pax_global_header00006660000000000000000000000064145726631360014527gustar00rootroot0000000000000052 comment=619e93fe896f23ac04fd424ac5223c2443fec296 xrootd-5.6.9/000077500000000000000000000000001457266313600130675ustar00rootroot00000000000000xrootd-5.6.9/.ci/000077500000000000000000000000001457266313600135405ustar00rootroot00000000000000xrootd-5.6.9/.ci/config.cmake000066400000000000000000000012061457266313600160060ustar00rootroot00000000000000set(FORCE_ENABLED 0 CACHE BOOL "") set(ENABLE_ASAN 0 CACHE BOOL "") set(ENABLE_TSAN 0 CACHE BOOL "") set(ENABLE_FUSE 1 CACHE BOOL "") set(ENABLE_HTTP 1 CACHE BOOL "") set(ENABLE_KRB5 1 CACHE BOOL "") set(ENABLE_MACAROONS 1 CACHE BOOL "") set(ENABLE_PYTHON 1 CACHE BOOL "") set(ENABLE_READLINE 1 CACHE BOOL "") set(ENABLE_SCITOKENS 1 CACHE BOOL "") set(ENABLE_TESTS 1 CACHE BOOL "") set(ENABLE_VOMS 1 CACHE BOOL "") set(ENABLE_XRDCL 1 CACHE BOOL "") set(ENABLE_XRDCLHTTP 1 CACHE BOOL "") set(ENABLE_XRDEC 1 CACHE BOOL "") set(XRDCL_LIB_ONLY 0 CACHE BOOL "") set(XRDCL_ONLY 0 CACHE BOOL "") xrootd-5.6.9/.gitattributes000066400000000000000000000000251457266313600157570ustar00rootroot00000000000000VERSION export-subst xrootd-5.6.9/.github/000077500000000000000000000000001457266313600144275ustar00rootroot00000000000000xrootd-5.6.9/.github/workflows/000077500000000000000000000000001457266313600164645ustar00rootroot00000000000000xrootd-5.6.9/.github/workflows/CI.yml000066400000000000000000000176371457266313600175200ustar00rootroot00000000000000name: CI on: push: branches-ignore: - master paths-ignore: - .gitignore - .gitlab-ci.yml - .mailmap - '**.md' - 'docs/**' - 'docker/**' tags-ignore: pull_request: workflow_dispatch: concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true defaults: run: shell: bash env: CDASH: ${{ vars.CDASH }} CMAKE_VERBOSE_MAKEFILE: true CTEST_OUTPUT_ON_FAILURE: true jobs: alpine: name: Alpine runs-on: ubuntu-latest container: alpine env: CMAKE_ARGS: -DCMAKE_INSTALL_PREFIX=/usr steps: - name: Install dependencies shell: sh run: | apk add \ bash \ cmake \ cppunit-dev \ ceph-dev \ curl-dev \ fuse-dev \ fuse3-dev \ g++ \ git \ gtest-dev \ isa-l-dev \ json-c-dev \ krb5-dev \ libxml2-dev \ linux-headers \ make \ openssl \ openssl-dev \ py3-pip \ py3-setuptools \ py3-wheel \ python3-dev \ readline-dev \ sudo \ tinyxml-dev \ util-linux-dev \ uuidgen \ zlib-dev - name: Clone repository uses: actions/checkout@v3 with: fetch-depth: 0 - name: Setup GitHub runner user within container run: adduser -D --uid 1001 runner && chown -R runner:runner ${GITHUB_WORKSPACE} - name: Build and Test with CTest run: sudo -E -u runner ctest -VV -S test.cmake - name: Install with CMake run: cmake --install build - name: Run post-install tests run: | tests/post-install.sh tests/check-headers.sh centos7: name: CentOS 7 runs-on: ubuntu-latest container: centos:7 env: CMAKE_ARGS: "-DCMAKE_INSTALL_PREFIX=/usr;-DCMAKE_INSTALL_RPATH='$ORIGIN/../$LIB'" steps: - name: Install dependencies run: | yum install -y centos-release-scl epel-release git yum install -y epel-rpm-macros rpmdevtools sudo yum-utils - name: Clone repository uses: actions/checkout@v3 with: fetch-depth: 0 - name: Setup GitHub runner user within container run: adduser --uid 1001 runner && chown -R runner:runner . - name: Install XRootD build dependencies run: yum-builddep -y xrootd.spec - name: Build and Test with CTest run: | source /opt/rh/devtoolset-7/enable su -p runner -c 'ctest3 -VV -S test.cmake' alma8: name: Alma 8 runs-on: ubuntu-latest container: almalinux:8 env: CMAKE_ARGS: "-DCMAKE_INSTALL_PREFIX=/usr;-DCMAKE_INSTALL_RPATH='$ORIGIN/../$LIB'" steps: - name: Install dependencies run: | dnf install -y dnf-plugins-core epel-release git rpmdevtools sudo dnf config-manager --set-enabled powertools - name: Clone repository uses: actions/checkout@v3 with: fetch-depth: 0 - name: Setup GitHub runner user within container run: adduser --uid 1001 runner && chown -R runner:runner ${GITHUB_WORKSPACE} - name: Install XRootD build dependencies run: dnf builddep -y xrootd.spec - name: Build and Test with CTest run: sudo -E -u runner ctest -VV -S test.cmake - name: Install with CMake run: cmake --install build - name: Run post-install tests run: | tests/post-install.sh tests/check-headers.sh alma9: name: Alma 9 runs-on: ubuntu-latest container: almalinux:9 env: CMAKE_ARGS: "-DCMAKE_INSTALL_PREFIX=/usr;-DCMAKE_INSTALL_RPATH='$ORIGIN/../$LIB'" steps: - name: Install dependencies run: | dnf install -y dnf-plugins-core epel-release git rpmdevtools sudo dnf config-manager --set-enabled crb - name: Clone repository uses: actions/checkout@v3 with: fetch-depth: 0 - name: Setup GitHub runner user within container run: adduser --uid 1001 runner && chown -R runner:runner ${GITHUB_WORKSPACE} - name: Install XRootD build dependencies run: dnf builddep -y xrootd.spec - name: Build and Test with CTest run: sudo -E -u runner ctest -VV -S test.cmake - name: Install with CMake run: cmake --install build - name: Run post-install tests run: | tests/post-install.sh tests/check-headers.sh fedora: name: Fedora runs-on: ubuntu-latest container: fedora env: CMAKE_GENERATOR: Ninja CMAKE_ARGS: "-DCMAKE_INSTALL_PREFIX=/usr;-DCMAKE_INSTALL_RPATH='$ORIGIN/../$LIB'" steps: - name: Install dependencies run: dnf install -y dnf-plugins-core git ninja-build rpmdevtools - name: Clone repository uses: actions/checkout@v3 with: fetch-depth: 0 - name: Setup GitHub runner user within container run: adduser --uid 1001 runner && chown -R runner:runner ${GITHUB_WORKSPACE} - name: Install XRootD build dependencies run: dnf builddep -y --define 'with_ceph 1' xrootd.spec - name: Build and Test with CTest run: sudo -E -u runner ctest -VV -S test.cmake - name: Install with CMake run: cmake --install build - name: Run post-install tests run: | tests/post-install.sh tests/check-headers.sh ubuntu: name: Ubuntu runs-on: ubuntu-latest strategy: matrix: compiler: [ gcc, clang ] env: CC: ${{ matrix.compiler }} DEBIAN_FRONTEND: noninteractive CMAKE_ARGS: '-DINSTALL_PYTHON_BINDINGS=0;-DUSE_SYSTEM_ISAL=1;-DCMAKE_INSTALL_PREFIX=/usr' steps: - name: Install dependencies run: | sudo apt update -q sudo apt install -y \ cmake \ clang \ davix-dev \ g++ \ libcppunit-dev \ libcurl4-openssl-dev \ libfuse-dev \ libgtest-dev \ libisal-dev \ libjson-c-dev \ libkrb5-dev \ libmacaroons-dev \ libreadline-dev \ libscitokens-dev \ libssl-dev \ libsystemd-dev \ libtinyxml-dev \ libxml2-dev \ make \ pkg-config \ python3-dev \ python3-pip \ python3-setuptools \ python3-wheel \ uuid-dev \ voms-dev \ zlib1g-dev - name: Clone repository uses: actions/checkout@v3 with: fetch-depth: 0 - name: Build and Test with CTest run: env CC=${CC} CXX=${CC/g*/g++} ctest -VV -S test.cmake - name: Install with CMake run: sudo cmake --install build - name: Install Python bindings run: | sudo python3 -m pip install \ --target /usr/lib/python3/dist-packages \ --use-pep517 --verbose build/bindings/python - name: Run post-install tests run: | tests/post-install.sh tests/check-headers.sh macos: name: macOS runs-on: macos-latest env: CC: clang CXX: clang++ CMAKE_ARGS: "-DPython_FIND_UNVERSIONED_NAMES=FIRST" CMAKE_PREFIX_PATH: /usr/local/opt/openssl@3 steps: - name: Workaround for issue 1772 run: sudo sed -i -e "s/localhost/localhost $(hostname)/g" /etc/hosts - name: Install dependencies with Homebrew run: brew install cppunit davix googletest isa-l - name: Install Python dependencies with pip run: python3 -m pip install --upgrade pip setuptools wheel - name: Clone repository uses: actions/checkout@v3 with: fetch-depth: 0 - name: Build and Test with CTest run: ctest -VV --repeat until-pass:3 -S test.cmake - name: Install with CMake run: cmake --install build - name: Run post-install tests run: | export PYVERSION=$(python3 --version | grep -o 3...) export PYTHONPATH=/usr/local/lib/python${PYVERSION}/site-packages tests/post-install.sh xrootd-5.6.9/.github/workflows/DEB.yml000066400000000000000000000055631457266313600176120ustar00rootroot00000000000000name: DEB on: push: branches: - master paths-ignore: - .gitignore - .gitlab-ci.yml - .mailmap - '**.md' - 'docs/**' - 'docker/**' tags-ignore: workflow_dispatch: concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true defaults: run: shell: bash env: DEBIAN_FRONTEND: noninteractive jobs: debian: name: Debian strategy: matrix: version: [ 11, 12 ] runs-on: ubuntu-latest container: debian:${{ matrix.version }} steps: - name: Install development tools run: | apt update -qq apt install -y build-essential devscripts git - name: Clone repository uses: actions/checkout@v3 with: fetch-depth: 0 - name: Install XRootD build dependencies run: mk-build-deps --install --remove debian/control <<< yes - name: Build DEBs run: | git config --global --add safe.directory "$GITHUB_WORKSPACE" VERSION=$(git describe --match 'v*' | sed -e 's/v//; s/-rc/~rc/; s/-g/+git/; s/-/.post/; s/-/./') dch --create --package xrootd -v ${VERSION} -M 'XRootD automated build.' debuild --no-tgz-check --no-sign -b - name: Install DEBs run: apt install -y ../*.deb - name: Run post-install tests run: tests/post-install.sh - name: Move DEBs to Artifact Directory run: | source /etc/os-release mkdir -p DEB/${ID}/${VERSION_CODENAME} mv ../*.* DEB/${ID}/${VERSION_CODENAME} - name: Upload Artifacts uses: actions/upload-artifact@v3 with: name: DEB path: DEB retention-days: 1 ubuntu: name: Ubuntu (22.04) runs-on: ubuntu-22.04 steps: - name: Install development tools run: | sudo apt update -qq sudo apt install -y build-essential devscripts equivs git - name: Clone repository uses: actions/checkout@v3 with: fetch-depth: 0 - name: Install XRootD build dependencies run: mk-build-deps --install --remove -s sudo debian/control <<< yes - name: Build DEBs run: | git config --global --add safe.directory "$GITHUB_WORKSPACE" VERSION=$(git describe --match 'v*' | sed -e 's/v//; s/-rc/~rc/; s/-g/+git/; s/-/.post/; s/-/./') dch --create --package xrootd -v ${VERSION} -M 'XRootD automated build.' debuild --no-tgz-check --no-sign -b - name: Install DEBs run: sudo apt install -y ../*.deb - name: Run post-install tests run: tests/post-install.sh - name: Move DEBs to Artifact Directory run: | source /etc/os-release mkdir -p DEB/${ID}/${VERSION_CODENAME} mv ../*.* DEB/${ID}/${VERSION_CODENAME} - name: Upload Artifacts uses: actions/upload-artifact@v3 with: name: DEB path: DEB retention-days: 1 xrootd-5.6.9/.github/workflows/QEMU.yml000066400000000000000000000022431457266313600177570ustar00rootroot00000000000000name: QEMU on: workflow_dispatch: inputs: os: description: 'OS' required: true default: 'fedora' type: choice options: - alma8 - alma9 - centos7 - debian - fedora - ubuntu arch: description: 'Architecture' required: true default: 's390x' type: choice options: - 386 - amd64 - arm - arm64 - ppc64le - s390x concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ inputs.os }}-${{ inputs.arch }} cancel-in-progress: true defaults: run: shell: bash env: DOCKER: podman jobs: buildx: name: QEMU (${{ inputs.os }}-${{ inputs.arch }}) runs-on: ubuntu-latest steps: - name: Clone repository uses: actions/checkout@v3 with: fetch-depth: 0 - name: Setup QEMU for cross-building images run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - name: Cross-build container with docker/podman buildx run: cd docker && ./xrd-docker buildx ${{ inputs.os }} ${{ inputs.arch }} xrootd-5.6.9/.github/workflows/RPM.yml000066400000000000000000000116301457266313600176460ustar00rootroot00000000000000name: RPM on: push: branches: - master paths-ignore: - .gitignore - .gitlab-ci.yml - .mailmap - '**.md' - 'docs/**' - 'docker/**' tags-ignore: workflow_dispatch: concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: centos7: name: CentOS 7 runs-on: ubuntu-latest container: centos:7 steps: - name: Install git run: yum install -y git - name: Clone repository uses: actions/checkout@v1 - name: Install RPM development tools run: | yum install -y centos-release-scl epel-release yum install -y epel-rpm-macros rpmdevtools yum-utils - name: Install XRootD build dependencies run: yum-builddep -y xrootd.spec - name: Build RPMs run: | rpmdev-setuptree git config --global --add safe.directory "$GITHUB_WORKSPACE" git archive --prefix xrootd/ -o $(rpm -E '%{_sourcedir}')/xrootd.tar.gz HEAD rpmbuild -bb --with git xrootd.spec - name: Install RPMs run: yum install -y $(rpm -E '%{_rpmdir}')/*/*.rpm - name: Run post-install tests run: tests/post-install.sh - name: Move RPMs to Artifact Directory run: mkdir RPMS && mv $(rpm -E '%{_rpmdir}')/ RPMS$(rpm -E '%{dist}' | tr . /) - name: Upload Artifacts uses: actions/upload-artifact@v3 with: name: RPM path: RPMS retention-days: 1 alma8: name: Alma Linux 8 runs-on: ubuntu-latest container: almalinux:8 steps: - name: Install git run: yum install -y git - name: Clone repository uses: actions/checkout@v3 with: fetch-depth: 0 - name: Install RPM development tools run: | dnf install -y epel-release rpmdevtools dnf-plugins-core dnf config-manager --set-enabled powertools - name: Install XRootD build dependencies run: dnf builddep -y xrootd.spec - name: Build RPMs run: | rpmdev-setuptree git config --global --add safe.directory "$GITHUB_WORKSPACE" git archive --prefix xrootd/ -o $(rpm -E '%{_sourcedir}')/xrootd.tar.gz HEAD rpmbuild -bb --with git xrootd.spec - name: Install RPMs run: dnf install -y $(rpm -E '%{_rpmdir}')/*/*.rpm - name: Run post-install tests run: tests/post-install.sh - name: Move RPMs to Artifact Directory run: mkdir RPMS && mv $(rpm -E '%{_rpmdir}')/ RPMS$(rpm -E '%{dist}' | tr . /) - name: Upload Artifacts uses: actions/upload-artifact@v3 with: name: RPM path: RPMS retention-days: 1 alma9: name: Alma Linux 9 runs-on: ubuntu-latest container: almalinux:9 steps: - name: Install git run: yum install -y git - name: Clone repository uses: actions/checkout@v3 with: fetch-depth: 0 - name: Install RPM development tools run: | dnf install -y epel-release rpmdevtools dnf-plugins-core dnf config-manager --set-enabled crb - name: Install XRootD build dependencies run: dnf builddep -y xrootd.spec - name: Build RPMs run: | rpmdev-setuptree git config --global --add safe.directory "$GITHUB_WORKSPACE" git archive --prefix xrootd/ -o $(rpm -E '%{_sourcedir}')/xrootd.tar.gz HEAD rpmbuild -bb --with git xrootd.spec - name: Install RPMs run: dnf install -y $(rpm -E '%{_rpmdir}')/*/*.rpm - name: Run post-install tests run: tests/post-install.sh - name: Move RPMs to Artifact Directory run: mkdir RPMS && mv $(rpm -E '%{_rpmdir}')/ RPMS$(rpm -E '%{dist}' | tr . /) - name: Upload Artifacts uses: actions/upload-artifact@v3 with: name: RPM path: RPMS retention-days: 1 fedora: name: Fedora 39 runs-on: ubuntu-latest container: fedora:39 steps: - name: Install git run: yum install -y git - name: Clone repository uses: actions/checkout@v3 with: fetch-depth: 0 - name: Install RPM development tools run: | dnf install -y rpmdevtools dnf-plugins-core - name: Install XRootD build dependencies run: dnf builddep -y --define 'with_ceph 1' xrootd.spec - name: Build RPMs run: | rpmdev-setuptree git config --global --add safe.directory "$GITHUB_WORKSPACE" git archive --prefix xrootd/ -o $(rpm -E '%{_sourcedir}')/xrootd.tar.gz HEAD rpmbuild -bb --with git --with ceph xrootd.spec - name: Install RPMs run: dnf install -y $(rpm -E '%{_rpmdir}')/*/*.rpm - name: Run post-install tests run: tests/post-install.sh - name: Move RPMs to Artifact Directory run: mkdir RPMS && mv $(rpm -E '%{_rpmdir}')/ RPMS$(rpm -E '%{dist}' | tr . /) - name: Upload Artifacts uses: actions/upload-artifact@v3 with: name: RPM path: RPMS retention-days: 1 xrootd-5.6.9/.github/workflows/python.yml000066400000000000000000000034701457266313600205340ustar00rootroot00000000000000name: Python on: push: branches: - devel - master paths: - setup.py - pyproject.toml - bindings/python tags-ignore: pull_request: paths: - setup.py - pyproject.toml - bindings/python workflow_dispatch: concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: manylinux: name: Python runs-on: ubuntu-latest container: quay.io/pypa/manylinux_2_28_x86_64 strategy: matrix: version: [ "3.6", "3.8", "3.10", "3.11", "3.12" ] env: PYTHON: python${{ matrix.version }} steps: - name: Install dependencies run: dnf install -y git krb5-devel libuuid-devel openssl-devel - name: Clone repository uses: actions/checkout@v3 with: fetch-depth: 0 - name: Build source distribution tarball run: | git config --global --add safe.directory "$GITHUB_WORKSPACE" ./genversion.sh >| VERSION ${PYTHON} -m build --sdist - name: Build binary wheel run: ${PYTHON} -m pip wheel --use-pep517 --verbose dist/*.tar.gz - name: Install binary wheel run: ${PYTHON} -m pip install xrootd*.whl - name: Run post-installation tests run: | command -v ${PYTHON} && ${PYTHON} --version ${PYTHON} -m pip show xrootd ${PYTHON} -c 'import XRootD; print(XRootD)' ${PYTHON} -c 'import pyxrootd; print(pyxrootd)' ${PYTHON} -c 'from XRootD import client; help(client)' ${PYTHON} -c 'from XRootD import client; print(client.FileSystem("root://localhost"))' - name: Move binary wheels to artifact directory run: mv *.whl dist/ - name: Upload Artifacts uses: actions/upload-artifact@v3 with: name: Python path: dist retention-days: 1 xrootd-5.6.9/.gitignore000066400000000000000000000001561457266313600150610ustar00rootroot00000000000000# CMake build directory build/ # docker builds docker/xrootd.tar.gz # Python build artifacts dist/ *.egg-info xrootd-5.6.9/.gitlab-ci.yml000066400000000000000000000046071457266313600155320ustar00rootroot00000000000000stages: - build default: tags: - docker_node .deb_build: &deb_build stage: build variables: DEBIAN_FRONTEND: noninteractive script: - source /etc/os-release - apt update -qq - apt install -y build-essential devscripts equivs git - mk-build-deps --install --remove debian/control <<< y - VERSION=$(git describe --match 'v*' | sed -e 's/v//; s/-rc/~rc/; s/-g/+git/; s/-/.post/; s/-/./') - dch --create --package xrootd -v ${VERSION} -M 'XRootD automated build.' - debuild --no-tgz-check --no-sign -b - apt install -y ../*.d*eb - mkdir -p DEB/${ID}/${VERSION_CODENAME} - mv ../*.* DEB/${ID}/${VERSION_CODENAME} - tests/post-install.sh artifacts: paths: [ DEB ] expire_in: 1d .rpm_build_yum: &rpm_build_yum stage: build script: - yum install -y centos-release-scl epel-release git - yum install -y epel-rpm-macros rpmdevtools yum-utils - yum-builddep -y xrootd.spec - rpmdev-setuptree - git archive --prefix xrootd/ -o $(rpm -E '%{_sourcedir}')/xrootd.tar.gz HEAD - rpmbuild -bb --with git xrootd.spec - yum install -y $(rpm -E '%{_rpmdir}')/*/*.rpm - tests/post-install.sh - mkdir -p RPMS - mv $(rpm -E '%{_rpmdir}')/ RPMS$(rpm -E '%{dist}' | tr . /) artifacts: paths: [ RPMS ] expire_in: 1d .rpm_build_dnf: &rpm_build_dnf stage: build script: - dnf install -y dnf-plugins-core git rpmdevtools - rpmdev-setuptree - dnf builddep -y xrootd.spec - git archive --prefix xrootd/ -o $(rpm -E '%{_sourcedir}')/xrootd.tar.gz HEAD - rpmbuild -bb --with git xrootd.spec - dnf install -y $(rpm -E '%{_rpmdir}')/*/*.rpm - tests/post-install.sh - mkdir -p RPMS - mv $(rpm -E '%{_rpmdir}')/ RPMS$(rpm -E '%{dist}' | tr . /) artifacts: paths: [ RPMS ] expire_in: 1d Debian 11: image: debian:11 <<: *deb_build Debian 12: image: debian:12 <<: *deb_build Ubuntu 22.04: image: ubuntu:22.04 <<: *deb_build CentOS 7: image: centos:7 <<: *rpm_build_yum AlmaLinux 8: image: almalinux:8 before_script: - dnf install -y epel-release - dnf config-manager --set-enabled powertools <<: *rpm_build_dnf AlmaLinux 9: image: almalinux:9 before_script: - dnf install -y epel-release - dnf config-manager --set-enabled crb <<: *rpm_build_dnf Fedora 38: image: fedora:38 <<: *rpm_build_dnf Fedora 39: image: fedora:39 <<: *rpm_build_dnf xrootd-5.6.9/.mailmap000066400000000000000000000127011457266313600145110ustar00rootroot00000000000000Alja Mrak-Tadel Alja MRak-Tadel Alja Mrak-Tadel alja Alja Mrak-Tadel alja Alja Mrak-Tadel alja Andreas Joachim Peters Andreas Peters Andreas Joachim Peters Andreas Peters Andreas Joachim Peters Andreas-Joachim Peters Andrew Hanushevsky Artem Harutyunyan Brian Bockelman Brian P Bockelman Brian Bockelman Brian P Bockelman Cedric Caffy ccaffy <85744538+ccaffy@users.noreply.github.com> Chris Burr Chris Burr Chris Green David Smith David Smith David Smith Ed J Edgar Fajardo efajardo Edgar Fajardo efajardo Elvin Sindrilaru Fabrizio Furano Fabrizio Furano Fabrizio Furano Fabrizio Furano Fabrizio Furano Fabrizio Furano Fabrizio Furano Fabrizio Furano Fabrizio Furano ffurano Fabrizio Furano furano Frank Winklmeier fwinkl <> Fritz Mueller Fritz Mueller Gerardo Ganis Gerardo GANIS Gerardo Ganis Gerri Ganis Gerardo Ganis Gerri Ganis Gerardo Ganis Gerri Ganis Gerardo Ganis ganis Gerardo Ganis ganis Gerardo Ganis gganis Jacek Becla James Walder snafus Jan Iven Jozsef Makai Jozsef Makai Jozsef Makai Jyothish Thomas Jo-stfc <71326101+Jo-stfc@users.noreply.github.com> Jyothish Thomas root Kian-Tat Lim ktlim Lukasz Janyst Lukasz Janyst Lukasz Janyst Lukasz Janyst Lukasz Janyst Lukasz Janyst Lukasz Janyst Lukasz Janyst Matevž Tadel Matevz Tadel Matevž Tadel Matevž Tadel Michal Simon Michal Simon Michal Simon Michal Simon Michal Simon simonmichal Nikola Hardi Nikola Hardi Paul-Niklas Kramp Paul Kramp Paul-Niklas Kramp niklas Paul-Niklas Kramp pkramp Remigius Mommsen mommsen <> Sebastien Ponce Thorsten Kollegger TKollegger Wei Yang Wei Yang Wei Yang Wei Yang Wei Yang Wei Yang Wei Yang Wei Yang Wei Yang Wei Yang Wei Yang Wei Yang Wei Yang Wei Yang Wei Yang Wei Yang Wei Yang Wei Yang Wei Yang Wei Yang Unknown bbrqa <> Unknown otron Unknown root Unknown root Unknown root Unknown root Unknown xrootd Unknown root xrootd-5.6.9/CMakeLists.txt000066400000000000000000000054161457266313600156350ustar00rootroot00000000000000#------------------------------------------------------------------------------- # Project description #------------------------------------------------------------------------------- cmake_minimum_required(VERSION 3.16...3.25) project( XRootD ) set( CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/src ${PROJECT_SOURCE_DIR}/cmake ) include(XRootDVersion) #------------------------------------------------------------------------------- # A 'plugins' phony target to simplify building build-tree binaries. # Plugins are responsible for adding themselves to this target, where # appropriate. #------------------------------------------------------------------------------- ADD_CUSTOM_TARGET(plugins) include( XRootDUtils ) CheckBuildDirectory() include( XRootDOSDefs ) include( XRootDDefaults ) include( XRootDSystemCheck ) include( XRootDFindLibs ) add_definitions( -DXRDPLUGIN_SOVERSION="${PLUGIN_VERSION}" ) #------------------------------------------------------------------------------- # Generate the version header #------------------------------------------------------------------------------- configure_file(src/XrdVersion.hh.in src/XrdVersion.hh) #------------------------------------------------------------------------------- # Build in subdirectories #------------------------------------------------------------------------------- include_directories(BEFORE ${CMAKE_SOURCE_DIR}/src ${CMAKE_BINARY_DIR}/src) include(CTest) add_subdirectory(src) add_subdirectory(bindings) add_subdirectory(tests) add_subdirectory(docs) add_subdirectory(utils) include( XRootDSummary ) #------------------------------------------------------------------------------- # Install XRootDConfig.cmake module #------------------------------------------------------------------------------- include(CMakePackageConfigHelpers) write_basic_package_version_file(cmake/${PROJECT_NAME}ConfigVersion.cmake VERSION ${XRootD_VERSION} COMPATIBILITY SameMajorVersion) configure_package_config_file(cmake/${PROJECT_NAME}Config.cmake.in cmake/${PROJECT_NAME}Config.cmake INSTALL_PREFIX ${CMAKE_INSTALL_PREFIX} INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME} PATH_VARS CMAKE_INSTALL_INCLUDEDIR CMAKE_INSTALL_LIBDIR CMAKE_INSTALL_DATADIR ) install(DIRECTORY ${PROJECT_BINARY_DIR}/cmake/ DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}) #------------------------------------------------------------------------------- # Configure an 'uninstall' target #------------------------------------------------------------------------------- CONFIGURE_FILE( "${CMAKE_CURRENT_SOURCE_DIR}/cmake_uninstall.cmake.in" "${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake" IMMEDIATE @ONLY) ADD_CUSTOM_TARGET(uninstall "${CMAKE_COMMAND}" -P "${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake") xrootd-5.6.9/COPYING000066400000000000000000001045131457266313600141260ustar00rootroot00000000000000 GNU GENERAL PUBLIC LICENSE Version 3, 29 June 2007 Copyright (C) 2007 Free Software Foundation, Inc. Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. Preamble The GNU General Public License is a free, copyleft license for software and other kinds of works. The licenses for most software and other practical works are designed to take away your freedom to share and change the works. By contrast, the GNU General Public License is intended to guarantee your freedom to share and change all versions of a program--to make sure it remains free software for all its users. We, the Free Software Foundation, use the GNU General Public License for most of our software; it applies also to any other work released this way by its authors. You can apply it to your programs, too. When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for them if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs, and that you know you can do these things. To protect your rights, we need to prevent others from denying you these rights or asking you to surrender the rights. Therefore, you have certain responsibilities if you distribute copies of the software, or if you modify it: responsibilities to respect the freedom of others. For example, if you distribute copies of such a program, whether gratis or for a fee, you must pass on to the recipients the same freedoms that you received. You must make sure that they, too, receive or can get the source code. And you must show them these terms so they know their rights. Developers that use the GNU GPL protect your rights with two steps: (1) assert copyright on the software, and (2) offer you this License giving you legal permission to copy, distribute and/or modify it. For the developers' and authors' protection, the GPL clearly explains that there is no warranty for this free software. For both users' and authors' sake, the GPL requires that modified versions be marked as changed, so that their problems will not be attributed erroneously to authors of previous versions. Some devices are designed to deny users access to install or run modified versions of the software inside them, although the manufacturer can do so. This is fundamentally incompatible with the aim of protecting users' freedom to change the software. The systematic pattern of such abuse occurs in the area of products for individuals to use, which is precisely where it is most unacceptable. Therefore, we have designed this version of the GPL to prohibit the practice for those products. If such problems arise substantially in other domains, we stand ready to extend this provision to those domains in future versions of the GPL, as needed to protect the freedom of users. Finally, every program is threatened constantly by software patents. States should not allow patents to restrict development and use of software on general-purpose computers, but in those that do, we wish to avoid the special danger that patents applied to a free program could make it effectively proprietary. To prevent this, the GPL assures that patents cannot be used to render the program non-free. The precise terms and conditions for copying, distribution and modification follow. TERMS AND CONDITIONS 0. Definitions. "This License" refers to version 3 of the GNU General Public License. "Copyright" also means copyright-like laws that apply to other kinds of works, such as semiconductor masks. "The Program" refers to any copyrightable work licensed under this License. Each licensee is addressed as "you". "Licensees" and "recipients" may be individuals or organizations. To "modify" a work means to copy from or adapt all or part of the work in a fashion requiring copyright permission, other than the making of an exact copy. The resulting work is called a "modified version" of the earlier work or a work "based on" the earlier work. A "covered work" means either the unmodified Program or a work based on the Program. To "propagate" a work means to do anything with it that, without permission, would make you directly or secondarily liable for infringement under applicable copyright law, except executing it on a computer or modifying a private copy. Propagation includes copying, distribution (with or without modification), making available to the public, and in some countries other activities as well. To "convey" a work means any kind of propagation that enables other parties to make or receive copies. Mere interaction with a user through a computer network, with no transfer of a copy, is not conveying. An interactive user interface displays "Appropriate Legal Notices" to the extent that it includes a convenient and prominently visible feature that (1) displays an appropriate copyright notice, and (2) tells the user that there is no warranty for the work (except to the extent that warranties are provided), that licensees may convey the work under this License, and how to view a copy of this License. If the interface presents a list of user commands or options, such as a menu, a prominent item in the list meets this criterion. 1. Source Code. The "source code" for a work means the preferred form of the work for making modifications to it. "Object code" means any non-source form of a work. A "Standard Interface" means an interface that either is an official standard defined by a recognized standards body, or, in the case of interfaces specified for a particular programming language, one that is widely used among developers working in that language. The "System Libraries" of an executable work include anything, other than the work as a whole, that (a) is included in the normal form of packaging a Major Component, but which is not part of that Major Component, and (b) serves only to enable use of the work with that Major Component, or to implement a Standard Interface for which an implementation is available to the public in source code form. A "Major Component", in this context, means a major essential component (kernel, window system, and so on) of the specific operating system (if any) on which the executable work runs, or a compiler used to produce the work, or an object code interpreter used to run it. The "Corresponding Source" for a work in object code form means all the source code needed to generate, install, and (for an executable work) run the object code and to modify the work, including scripts to control those activities. However, it does not include the work's System Libraries, or general-purpose tools or generally available free programs which are used unmodified in performing those activities but which are not part of the work. For example, Corresponding Source includes interface definition files associated with source files for the work, and the source code for shared libraries and dynamically linked subprograms that the work is specifically designed to require, such as by intimate data communication or control flow between those subprograms and other parts of the work. The Corresponding Source need not include anything that users can regenerate automatically from other parts of the Corresponding Source. The Corresponding Source for a work in source code form is that same work. 2. Basic Permissions. All rights granted under this License are granted for the term of copyright on the Program, and are irrevocable provided the stated conditions are met. This License explicitly affirms your unlimited permission to run the unmodified Program. The output from running a covered work is covered by this License only if the output, given its content, constitutes a covered work. This License acknowledges your rights of fair use or other equivalent, as provided by copyright law. You may make, run and propagate covered works that you do not convey, without conditions so long as your license otherwise remains in force. You may convey covered works to others for the sole purpose of having them make modifications exclusively for you, or provide you with facilities for running those works, provided that you comply with the terms of this License in conveying all material for which you do not control copyright. Those thus making or running the covered works for you must do so exclusively on your behalf, under your direction and control, on terms that prohibit them from making any copies of your copyrighted material outside their relationship with you. Conveying under any other circumstances is permitted solely under the conditions stated below. Sublicensing is not allowed; section 10 makes it unnecessary. 3. Protecting Users' Legal Rights From Anti-Circumvention Law. No covered work shall be deemed part of an effective technological measure under any applicable law fulfilling obligations under article 11 of the WIPO copyright treaty adopted on 20 December 1996, or similar laws prohibiting or restricting circumvention of such measures. When you convey a covered work, you waive any legal power to forbid circumvention of technological measures to the extent such circumvention is effected by exercising rights under this License with respect to the covered work, and you disclaim any intention to limit operation or modification of the work as a means of enforcing, against the work's users, your or third parties' legal rights to forbid circumvention of technological measures. 4. Conveying Verbatim Copies. You may convey verbatim copies of the Program's source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice; keep intact all notices stating that this License and any non-permissive terms added in accord with section 7 apply to the code; keep intact all notices of the absence of any warranty; and give all recipients a copy of this License along with the Program. You may charge any price or no price for each copy that you convey, and you may offer support or warranty protection for a fee. 5. Conveying Modified Source Versions. You may convey a work based on the Program, or the modifications to produce it from the Program, in the form of source code under the terms of section 4, provided that you also meet all of these conditions: a) The work must carry prominent notices stating that you modified it, and giving a relevant date. b) The work must carry prominent notices stating that it is released under this License and any conditions added under section 7. This requirement modifies the requirement in section 4 to "keep intact all notices". c) You must license the entire work, as a whole, under this License to anyone who comes into possession of a copy. This License will therefore apply, along with any applicable section 7 additional terms, to the whole of the work, and all its parts, regardless of how they are packaged. This License gives no permission to license the work in any other way, but it does not invalidate such permission if you have separately received it. d) If the work has interactive user interfaces, each must display Appropriate Legal Notices; however, if the Program has interactive interfaces that do not display Appropriate Legal Notices, your work need not make them do so. A compilation of a covered work with other separate and independent works, which are not by their nature extensions of the covered work, and which are not combined with it such as to form a larger program, in or on a volume of a storage or distribution medium, is called an "aggregate" if the compilation and its resulting copyright are not used to limit the access or legal rights of the compilation's users beyond what the individual works permit. Inclusion of a covered work in an aggregate does not cause this License to apply to the other parts of the aggregate. 6. Conveying Non-Source Forms. You may convey a covered work in object code form under the terms of sections 4 and 5, provided that you also convey the machine-readable Corresponding Source under the terms of this License, in one of these ways: a) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by the Corresponding Source fixed on a durable physical medium customarily used for software interchange. b) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by a written offer, valid for at least three years and valid for as long as you offer spare parts or customer support for that product model, to give anyone who possesses the object code either (1) a copy of the Corresponding Source for all the software in the product that is covered by this License, on a durable physical medium customarily used for software interchange, for a price no more than your reasonable cost of physically performing this conveying of source, or (2) access to copy the Corresponding Source from a network server at no charge. c) Convey individual copies of the object code with a copy of the written offer to provide the Corresponding Source. This alternative is allowed only occasionally and noncommercially, and only if you received the object code with such an offer, in accord with subsection 6b. d) Convey the object code by offering access from a designated place (gratis or for a charge), and offer equivalent access to the Corresponding Source in the same way through the same place at no further charge. You need not require recipients to copy the Corresponding Source along with the object code. If the place to copy the object code is a network server, the Corresponding Source may be on a different server (operated by you or a third party) that supports equivalent copying facilities, provided you maintain clear directions next to the object code saying where to find the Corresponding Source. Regardless of what server hosts the Corresponding Source, you remain obligated to ensure that it is available for as long as needed to satisfy these requirements. e) Convey the object code using peer-to-peer transmission, provided you inform other peers where the object code and Corresponding Source of the work are being offered to the general public at no charge under subsection 6d. A separable portion of the object code, whose source code is excluded from the Corresponding Source as a System Library, need not be included in conveying the object code work. A "User Product" is either (1) a "consumer product", which means any tangible personal property which is normally used for personal, family, or household purposes, or (2) anything designed or sold for incorporation into a dwelling. In determining whether a product is a consumer product, doubtful cases shall be resolved in favor of coverage. For a particular product received by a particular user, "normally used" refers to a typical or common use of that class of product, regardless of the status of the particular user or of the way in which the particular user actually uses, or expects or is expected to use, the product. A product is a consumer product regardless of whether the product has substantial commercial, industrial or non-consumer uses, unless such uses represent the only significant mode of use of the product. "Installation Information" for a User Product means any methods, procedures, authorization keys, or other information required to install and execute modified versions of a covered work in that User Product from a modified version of its Corresponding Source. The information must suffice to ensure that the continued functioning of the modified object code is in no case prevented or interfered with solely because modification has been made. If you convey an object code work under this section in, or with, or specifically for use in, a User Product, and the conveying occurs as part of a transaction in which the right of possession and use of the User Product is transferred to the recipient in perpetuity or for a fixed term (regardless of how the transaction is characterized), the Corresponding Source conveyed under this section must be accompanied by the Installation Information. But this requirement does not apply if neither you nor any third party retains the ability to install modified object code on the User Product (for example, the work has been installed in ROM). The requirement to provide Installation Information does not include a requirement to continue to provide support service, warranty, or updates for a work that has been modified or installed by the recipient, or for the User Product in which it has been modified or installed. Access to a network may be denied when the modification itself materially and adversely affects the operation of the network or violates the rules and protocols for communication across the network. Corresponding Source conveyed, and Installation Information provided, in accord with this section must be in a format that is publicly documented (and with an implementation available to the public in source code form), and must require no special password or key for unpacking, reading or copying. 7. Additional Terms. "Additional permissions" are terms that supplement the terms of this License by making exceptions from one or more of its conditions. Additional permissions that are applicable to the entire Program shall be treated as though they were included in this License, to the extent that they are valid under applicable law. If additional permissions apply only to part of the Program, that part may be used separately under those permissions, but the entire Program remains governed by this License without regard to the additional permissions. When you convey a copy of a covered work, you may at your option remove any additional permissions from that copy, or from any part of it. (Additional permissions may be written to require their own removal in certain cases when you modify the work.) You may place additional permissions on material, added by you to a covered work, for which you have or can give appropriate copyright permission. Notwithstanding any other provision of this License, for material you add to a covered work, you may (if authorized by the copyright holders of that material) supplement the terms of this License with terms: a) Disclaiming warranty or limiting liability differently from the terms of sections 15 and 16 of this License; or b) Requiring preservation of specified reasonable legal notices or author attributions in that material or in the Appropriate Legal Notices displayed by works containing it; or c) Prohibiting misrepresentation of the origin of that material, or requiring that modified versions of such material be marked in reasonable ways as different from the original version; or d) Limiting the use for publicity purposes of names of licensors or authors of the material; or e) Declining to grant rights under trademark law for use of some trade names, trademarks, or service marks; or f) Requiring indemnification of licensors and authors of that material by anyone who conveys the material (or modified versions of it) with contractual assumptions of liability to the recipient, for any liability that these contractual assumptions directly impose on those licensors and authors. All other non-permissive additional terms are considered "further restrictions" within the meaning of section 10. If the Program as you received it, or any part of it, contains a notice stating that it is governed by this License along with a term that is a further restriction, you may remove that term. If a license document contains a further restriction but permits relicensing or conveying under this License, you may add to a covered work material governed by the terms of that license document, provided that the further restriction does not survive such relicensing or conveying. If you add terms to a covered work in accord with this section, you must place, in the relevant source files, a statement of the additional terms that apply to those files, or a notice indicating where to find the applicable terms. Additional terms, permissive or non-permissive, may be stated in the form of a separately written license, or stated as exceptions; the above requirements apply either way. 8. Termination. You may not propagate or modify a covered work except as expressly provided under this License. Any attempt otherwise to propagate or modify it is void, and will automatically terminate your rights under this License (including any patent licenses granted under the third paragraph of section 11). However, if you cease all violation of this License, then your license from a particular copyright holder is reinstated (a) provisionally, unless and until the copyright holder explicitly and finally terminates your license, and (b) permanently, if the copyright holder fails to notify you of the violation by some reasonable means prior to 60 days after the cessation. Moreover, your license from a particular copyright holder is reinstated permanently if the copyright holder notifies you of the violation by some reasonable means, this is the first time you have received notice of violation of this License (for any work) from that copyright holder, and you cure the violation prior to 30 days after your receipt of the notice. Termination of your rights under this section does not terminate the licenses of parties who have received copies or rights from you under this License. If your rights have been terminated and not permanently reinstated, you do not qualify to receive new licenses for the same material under section 10. 9. Acceptance Not Required for Having Copies. You are not required to accept this License in order to receive or run a copy of the Program. Ancillary propagation of a covered work occurring solely as a consequence of using peer-to-peer transmission to receive a copy likewise does not require acceptance. However, nothing other than this License grants you permission to propagate or modify any covered work. These actions infringe copyright if you do not accept this License. Therefore, by modifying or propagating a covered work, you indicate your acceptance of this License to do so. 10. Automatic Licensing of Downstream Recipients. Each time you convey a covered work, the recipient automatically receives a license from the original licensors, to run, modify and propagate that work, subject to this License. You are not responsible for enforcing compliance by third parties with this License. An "entity transaction" is a transaction transferring control of an organization, or substantially all assets of one, or subdividing an organization, or merging organizations. If propagation of a covered work results from an entity transaction, each party to that transaction who receives a copy of the work also receives whatever licenses to the work the party's predecessor in interest had or could give under the previous paragraph, plus a right to possession of the Corresponding Source of the work from the predecessor in interest, if the predecessor has it or can get it with reasonable efforts. You may not impose any further restrictions on the exercise of the rights granted or affirmed under this License. For example, you may not impose a license fee, royalty, or other charge for exercise of rights granted under this License, and you may not initiate litigation (including a cross-claim or counterclaim in a lawsuit) alleging that any patent claim is infringed by making, using, selling, offering for sale, or importing the Program or any portion of it. 11. Patents. A "contributor" is a copyright holder who authorizes use under this License of the Program or a work on which the Program is based. The work thus licensed is called the contributor's "contributor version". A contributor's "essential patent claims" are all patent claims owned or controlled by the contributor, whether already acquired or hereafter acquired, that would be infringed by some manner, permitted by this License, of making, using, or selling its contributor version, but do not include claims that would be infringed only as a consequence of further modification of the contributor version. For purposes of this definition, "control" includes the right to grant patent sublicenses in a manner consistent with the requirements of this License. Each contributor grants you a non-exclusive, worldwide, royalty-free patent license under the contributor's essential patent claims, to make, use, sell, offer for sale, import and otherwise run, modify and propagate the contents of its contributor version. In the following three paragraphs, a "patent license" is any express agreement or commitment, however denominated, not to enforce a patent (such as an express permission to practice a patent or covenant not to sue for patent infringement). To "grant" such a patent license to a party means to make such an agreement or commitment not to enforce a patent against the party. If you convey a covered work, knowingly relying on a patent license, and the Corresponding Source of the work is not available for anyone to copy, free of charge and under the terms of this License, through a publicly available network server or other readily accessible means, then you must either (1) cause the Corresponding Source to be so available, or (2) arrange to deprive yourself of the benefit of the patent license for this particular work, or (3) arrange, in a manner consistent with the requirements of this License, to extend the patent license to downstream recipients. "Knowingly relying" means you have actual knowledge that, but for the patent license, your conveying the covered work in a country, or your recipient's use of the covered work in a country, would infringe one or more identifiable patents in that country that you have reason to believe are valid. If, pursuant to or in connection with a single transaction or arrangement, you convey, or propagate by procuring conveyance of, a covered work, and grant a patent license to some of the parties receiving the covered work authorizing them to use, propagate, modify or convey a specific copy of the covered work, then the patent license you grant is automatically extended to all recipients of the covered work and works based on it. A patent license is "discriminatory" if it does not include within the scope of its coverage, prohibits the exercise of, or is conditioned on the non-exercise of one or more of the rights that are specifically granted under this License. You may not convey a covered work if you are a party to an arrangement with a third party that is in the business of distributing software, under which you make payment to the third party based on the extent of your activity of conveying the work, and under which the third party grants, to any of the parties who would receive the covered work from you, a discriminatory patent license (a) in connection with copies of the covered work conveyed by you (or copies made from those copies), or (b) primarily for and in connection with specific products or compilations that contain the covered work, unless you entered into that arrangement, or that patent license was granted, prior to 28 March 2007. Nothing in this License shall be construed as excluding or limiting any implied license or other defenses to infringement that may otherwise be available to you under applicable patent law. 12. No Surrender of Others' Freedom. If conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot convey a covered work so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may not convey it at all. For example, if you agree to terms that obligate you to collect a royalty for further conveying from those to whom you convey the Program, the only way you could satisfy both those terms and this License would be to refrain entirely from conveying the Program. 13. Use with the GNU Affero General Public License. Notwithstanding any other provision of this License, you have permission to link or combine any covered work with a work licensed under version 3 of the GNU Affero General Public License into a single combined work, and to convey the resulting work. The terms of this License will continue to apply to the part which is the covered work, but the special requirements of the GNU Affero General Public License, section 13, concerning interaction through a network will apply to the combination as such. 14. Revised Versions of this License. The Free Software Foundation may publish revised and/or new versions of the GNU General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. Each version is given a distinguishing version number. If the Program specifies that a certain numbered version of the GNU General Public License "or any later version" applies to it, you have the option of following the terms and conditions either of that numbered version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of the GNU General Public License, you may choose any version ever published by the Free Software Foundation. If the Program specifies that a proxy can decide which future versions of the GNU General Public License can be used, that proxy's public statement of acceptance of a version permanently authorizes you to choose that version for the Program. Later license versions may give you additional or different permissions. However, no additional obligations are imposed on any author or copyright holder as a result of your choosing to follow a later version. 15. Disclaimer of Warranty. THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 16. Limitation of Liability. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. 17. Interpretation of Sections 15 and 16. If the disclaimer of warranty and limitation of liability provided above cannot be given local legal effect according to their terms, reviewing courts shall apply local law that most closely approximates an absolute waiver of all civil liability in connection with the Program, unless a warranty or assumption of liability accompanies a copy of the Program in return for a fee. END OF TERMS AND CONDITIONS How to Apply These Terms to Your New Programs If you develop a new program, and you want it to be of the greatest possible use to the public, the best way to achieve this is to make it free software which everyone can redistribute and change under these terms. To do so, attach the following notices to the program. It is safest to attach them to the start of each source file to most effectively state the exclusion of warranty; and each file should have at least the "copyright" line and a pointer to where the full notice is found. Copyright (C) This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see . Also add information on how to contact you by electronic and paper mail. If the program does terminal interaction, make it output a short notice like this when it starts in an interactive mode: Copyright (C) This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. This is free software, and you are welcome to redistribute it under certain conditions; type `show c' for details. The hypothetical commands `show w' and `show c' should show the appropriate parts of the General Public License. Of course, your program's commands might be different; for a GUI interface, you would use an "about box". You should also get your employer (if you work as a programmer) or school, if any, to sign a "copyright disclaimer" for the program, if necessary. For more information on this, and how to apply and follow the GNU GPL, see . The GNU General Public License does not permit incorporating your program into proprietary programs. If your program is a subroutine library, you may consider it more useful to permit linking proprietary applications with the library. If this is what you want to do, use the GNU Lesser General Public License instead of this License. But first, please read . xrootd-5.6.9/COPYING.BSD000066400000000000000000000053731457266313600145410ustar00rootroot00000000000000******************************************************************************** *Prior to September 2nd, 2012 the XRootD software suite was licensed under a * *modified BSD license shown below. This applies to all code that was in the * *XRootD git repository prior to that date. All code is now licensed under LGPL.* * See files LICENSE, COPYING.LGPL, and COPYING for license details. * ******************************************************************************** Copyright (c) 2005-2012, Board of Trustees of the Leland Stanford, Jr. University. Produced under contract DE-AC02-76-SF00515 with the US Department of Energy. All rights reserved. Conditions of Use Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: a. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. b. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. c. Neither the name of the Leland Stanford, Jr. University nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. d. Products derived from this software that do not adhere to the xrootd or cmsd protocol specifications may not use the acronyms 'cmsd', 'Scalla', 'xroot', and 'xrootd', regardless of capitalization, to describe such derivative works. DISCLAIMER THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 'AS IS' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. xrootd-5.6.9/COPYING.LGPL000066400000000000000000000167431457266313600146720ustar00rootroot00000000000000 GNU LESSER GENERAL PUBLIC LICENSE Version 3, 29 June 2007 Copyright (C) 2007 Free Software Foundation, Inc. Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. This version of the GNU Lesser General Public License incorporates the terms and conditions of version 3 of the GNU General Public License, supplemented by the additional permissions listed below. 0. Additional Definitions. As used herein, "this License" refers to version 3 of the GNU Lesser General Public License, and the "GNU GPL" refers to version 3 of the GNU General Public License. "The Library" refers to a covered work governed by this License, other than an Application or a Combined Work as defined below. An "Application" is any work that makes use of an interface provided by the Library, but which is not otherwise based on the Library. Defining a subclass of a class defined by the Library is deemed a mode of using an interface provided by the Library. A "Combined Work" is a work produced by combining or linking an Application with the Library. The particular version of the Library with which the Combined Work was made is also called the "Linked Version". The "Minimal Corresponding Source" for a Combined Work means the Corresponding Source for the Combined Work, excluding any source code for portions of the Combined Work that, considered in isolation, are based on the Application, and not on the Linked Version. The "Corresponding Application Code" for a Combined Work means the object code and/or source code for the Application, including any data and utility programs needed for reproducing the Combined Work from the Application, but excluding the System Libraries of the Combined Work. 1. Exception to Section 3 of the GNU GPL. You may convey a covered work under sections 3 and 4 of this License without being bound by section 3 of the GNU GPL. 2. Conveying Modified Versions. If you modify a copy of the Library, and, in your modifications, a facility refers to a function or data to be supplied by an Application that uses the facility (other than as an argument passed when the facility is invoked), then you may convey a copy of the modified version: a) under this License, provided that you make a good faith effort to ensure that, in the event an Application does not supply the function or data, the facility still operates, and performs whatever part of its purpose remains meaningful, or b) under the GNU GPL, with none of the additional permissions of this License applicable to that copy. 3. Object Code Incorporating Material from Library Header Files. The object code form of an Application may incorporate material from a header file that is part of the Library. You may convey such object code under terms of your choice, provided that, if the incorporated material is not limited to numerical parameters, data structure layouts and accessors, or small macros, inline functions and templates (ten or fewer lines in length), you do both of the following: a) Give prominent notice with each copy of the object code that the Library is used in it and that the Library and its use are covered by this License. b) Accompany the object code with a copy of the GNU GPL and this license document. 4. Combined Works. You may convey a Combined Work under terms of your choice that, taken together, effectively do not restrict modification of the portions of the Library contained in the Combined Work and reverse engineering for debugging such modifications, if you also do each of the following: a) Give prominent notice with each copy of the Combined Work that the Library is used in it and that the Library and its use are covered by this License. b) Accompany the Combined Work with a copy of the GNU GPL and this license document. c) For a Combined Work that displays copyright notices during execution, include the copyright notice for the Library among these notices, as well as a reference directing the user to the copies of the GNU GPL and this license document. d) Do one of the following: 0) Convey the Minimal Corresponding Source under the terms of this License, and the Corresponding Application Code in a form suitable for, and under terms that permit, the user to recombine or relink the Application with a modified version of the Linked Version to produce a modified Combined Work, in the manner specified by section 6 of the GNU GPL for conveying Corresponding Source. 1) Use a suitable shared library mechanism for linking with the Library. A suitable mechanism is one that (a) uses at run time a copy of the Library already present on the user's computer system, and (b) will operate properly with a modified version of the Library that is interface-compatible with the Linked Version. e) Provide Installation Information, but only if you would otherwise be required to provide such information under section 6 of the GNU GPL, and only to the extent that such information is necessary to install and execute a modified version of the Combined Work produced by recombining or relinking the Application with a modified version of the Linked Version. (If you use option 4d0, the Installation Information must accompany the Minimal Corresponding Source and Corresponding Application Code. If you use option 4d1, you must provide the Installation Information in the manner specified by section 6 of the GNU GPL for conveying Corresponding Source.) 5. Combined Libraries. You may place library facilities that are a work based on the Library side by side in a single library together with other library facilities that are not Applications and are not covered by this License, and convey such a combined library under terms of your choice, if you do both of the following: a) Accompany the combined library with a copy of the same work based on the Library, uncombined with any other library facilities, conveyed under the terms of this License. b) Give prominent notice with the combined library that part of it is a work based on the Library, and explaining where to find the accompanying uncombined form of the same work. 6. Revised Versions of the GNU Lesser General Public License. The Free Software Foundation may publish revised and/or new versions of the GNU Lesser General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. Each version is given a distinguishing version number. If the Library as you received it specifies that a certain numbered version of the GNU Lesser General Public License "or any later version" applies to it, you have the option of following the terms and conditions either of that published version or of any later version published by the Free Software Foundation. If the Library as you received it does not specify a version number of the GNU Lesser General Public License, you may choose any version of the GNU Lesser General Public License ever published by the Free Software Foundation. If the Library as you received it specifies that a proxy can decide whether future versions of the GNU Lesser General Public License shall apply, that proxy's public statement of acceptance of any version is permanent authorization for you to choose that version for the Library. xrootd-5.6.9/CTestConfig.cmake000066400000000000000000000002631457266313600162420ustar00rootroot00000000000000set(CTEST_PROJECT_NAME "XRootD") set(CTEST_NIGHTLY_START_TIME "00:00:00 UTC") set(CTEST_DROP_SITE_CDASH TRUE) set(CTEST_SUBMIT_URL https://my.cdash.org/submit.php?project=XRootD) xrootd-5.6.9/Doxyfile000066400000000000000000000175401457266313600146040ustar00rootroot00000000000000# Doxyfile 1.3.7 #--------------------------------------------------------------------------- # Project related configuration options #--------------------------------------------------------------------------- PROJECT_NAME = xrootd PROJECT_NUMBER = OUTPUT_DIRECTORY = doxydoc CREATE_SUBDIRS = NO OUTPUT_LANGUAGE = English USE_WINDOWS_ENCODING = NO BRIEF_MEMBER_DESC = YES REPEAT_BRIEF = YES ABBREVIATE_BRIEF = ALWAYS_DETAILED_SEC = NO INLINE_INHERITED_MEMB = NO FULL_PATH_NAMES = NO STRIP_FROM_PATH = STRIP_FROM_INC_PATH = SHORT_NAMES = NO JAVADOC_AUTOBRIEF = NO MULTILINE_CPP_IS_BRIEF = NO DETAILS_AT_TOP = NO INHERIT_DOCS = YES DISTRIBUTE_GROUP_DOC = NO TAB_SIZE = 8 ALIASES = OPTIMIZE_OUTPUT_FOR_C = NO OPTIMIZE_OUTPUT_JAVA = NO SUBGROUPING = YES #--------------------------------------------------------------------------- # Build related configuration options #--------------------------------------------------------------------------- EXTRACT_ALL = YES EXTRACT_PRIVATE = YES EXTRACT_STATIC = YES EXTRACT_LOCAL_CLASSES = YES EXTRACT_LOCAL_METHODS = NO HIDE_UNDOC_MEMBERS = NO HIDE_UNDOC_CLASSES = NO HIDE_FRIEND_COMPOUNDS = NO HIDE_IN_BODY_DOCS = NO INTERNAL_DOCS = NO CASE_SENSE_NAMES = YES HIDE_SCOPE_NAMES = NO SHOW_INCLUDE_FILES = YES INLINE_INFO = YES SORT_MEMBER_DOCS = YES SORT_BRIEF_DOCS = NO SORT_BY_SCOPE_NAME = NO GENERATE_TODOLIST = YES GENERATE_TESTLIST = YES GENERATE_BUGLIST = YES GENERATE_DEPRECATEDLIST= YES ENABLED_SECTIONS = MAX_INITIALIZER_LINES = 30 SHOW_USED_FILES = YES #--------------------------------------------------------------------------- # configuration options related to warning and progress messages #--------------------------------------------------------------------------- QUIET = NO WARNINGS = YES WARN_IF_UNDOCUMENTED = YES WARN_IF_DOC_ERROR = YES WARN_FORMAT = "$file:$line: $text" WARN_LOGFILE = #--------------------------------------------------------------------------- # configuration options related to the input files #--------------------------------------------------------------------------- INPUT = src/ FILE_PATTERNS = *.hh RECURSIVE = YES EXCLUDE = EXCLUDE_SYMLINKS = NO EXCLUDE_PATTERNS = EXAMPLE_PATH = EXAMPLE_PATTERNS = EXAMPLE_RECURSIVE = NO IMAGE_PATH = INPUT_FILTER = FILTER_SOURCE_FILES = NO #--------------------------------------------------------------------------- # configuration options related to source browsing #--------------------------------------------------------------------------- SOURCE_BROWSER = NO INLINE_SOURCES = NO STRIP_CODE_COMMENTS = YES REFERENCED_BY_RELATION = YES REFERENCES_RELATION = YES VERBATIM_HEADERS = YES #--------------------------------------------------------------------------- # configuration options related to the alphabetical class index #--------------------------------------------------------------------------- ALPHABETICAL_INDEX = NO COLS_IN_ALPHA_INDEX = 5 IGNORE_PREFIX = #--------------------------------------------------------------------------- # configuration options related to the HTML output #--------------------------------------------------------------------------- GENERATE_HTML = YES HTML_OUTPUT = html HTML_FILE_EXTENSION = .html HTML_HEADER = HTML_FOOTER = HTML_STYLESHEET = HTML_ALIGN_MEMBERS = YES GENERATE_HTMLHELP = NO CHM_FILE = HHC_LOCATION = GENERATE_CHI = NO BINARY_TOC = NO TOC_EXPAND = NO DISABLE_INDEX = NO ENUM_VALUES_PER_LINE = 4 GENERATE_TREEVIEW = NO TREEVIEW_WIDTH = 250 #--------------------------------------------------------------------------- # configuration options related to the LaTeX output #--------------------------------------------------------------------------- GENERATE_LATEX = NO LATEX_OUTPUT = latex LATEX_CMD_NAME = latex MAKEINDEX_CMD_NAME = makeindex COMPACT_LATEX = NO PAPER_TYPE = a4wide EXTRA_PACKAGES = LATEX_HEADER = PDF_HYPERLINKS = NO USE_PDFLATEX = NO LATEX_BATCHMODE = NO LATEX_HIDE_INDICES = NO #--------------------------------------------------------------------------- # configuration options related to the RTF output #--------------------------------------------------------------------------- GENERATE_RTF = NO RTF_OUTPUT = rtf COMPACT_RTF = NO RTF_HYPERLINKS = NO RTF_STYLESHEET_FILE = RTF_EXTENSIONS_FILE = #--------------------------------------------------------------------------- # configuration options related to the man page output #--------------------------------------------------------------------------- GENERATE_MAN = NO MAN_OUTPUT = man MAN_EXTENSION = .3 MAN_LINKS = NO #--------------------------------------------------------------------------- # configuration options related to the XML output #--------------------------------------------------------------------------- GENERATE_XML = NO XML_OUTPUT = xml XML_SCHEMA = XML_DTD = XML_PROGRAMLISTING = YES #--------------------------------------------------------------------------- # configuration options for the AutoGen Definitions output #--------------------------------------------------------------------------- GENERATE_AUTOGEN_DEF = NO #--------------------------------------------------------------------------- # configuration options related to the Perl module output #--------------------------------------------------------------------------- GENERATE_PERLMOD = NO PERLMOD_LATEX = NO PERLMOD_PRETTY = YES PERLMOD_MAKEVAR_PREFIX = #--------------------------------------------------------------------------- # Configuration options related to the preprocessor #--------------------------------------------------------------------------- ENABLE_PREPROCESSING = YES MACRO_EXPANSION = NO EXPAND_ONLY_PREDEF = NO SEARCH_INCLUDES = YES INCLUDE_PATH = INCLUDE_FILE_PATTERNS = PREDEFINED = EXPAND_AS_DEFINED = SKIP_FUNCTION_MACROS = YES #--------------------------------------------------------------------------- # Configuration::additions related to external references #--------------------------------------------------------------------------- TAGFILES = GENERATE_TAGFILE = ALLEXTERNALS = NO EXTERNAL_GROUPS = YES PERL_PATH = /usr/bin/perl #--------------------------------------------------------------------------- # Configuration options related to the dot tool #--------------------------------------------------------------------------- CLASS_DIAGRAMS = YES HIDE_UNDOC_RELATIONS = YES HAVE_DOT = YES CLASS_GRAPH = YES COLLABORATION_GRAPH = YES UML_LOOK = NO TEMPLATE_RELATIONS = NO INCLUDE_GRAPH = YES INCLUDED_BY_GRAPH = YES CALL_GRAPH = NO GRAPHICAL_HIERARCHY = YES DOT_IMAGE_FORMAT = png DOT_PATH = DOTFILE_DIRS = MAX_DOT_GRAPH_WIDTH = 1024 MAX_DOT_GRAPH_HEIGHT = 1024 MAX_DOT_GRAPH_DEPTH = 0 GENERATE_LEGEND = YES DOT_CLEANUP = YES #--------------------------------------------------------------------------- # Configuration::additions related to the search engine #--------------------------------------------------------------------------- SEARCHENGINE = YES xrootd-5.6.9/LICENSE000066400000000000000000000026301457266313600140750ustar00rootroot00000000000000"Copyright (c) 2005-2012, Board of Trustees of the Leland Stanford, Jr. University.\n" "Produced under contract DE-AC02-76-SF00515 with the US Department of Energy. \n" "All rights reserved. The copyright holder's institutional names may not be used to\n" "endorse or promote products derived from this software without specific prior \n" "written permission.\n\n" "This file is part of the XRootD software suite. \n\n" "XRootD is free software: you can redistribute it and/or modify it under the terms \n" "of the GNU Lesser General Public License as published by the Free Software \n" "Foundation, either version 3 of the License, or (at your option) any later version.\n\n" "XRootD is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;\n" "without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR \n" "PURPOSE. See the GNU Lesser General Public License for more details. \nn" "You should have received a copy of the GNU Lesser General Public License along \n" "with XRootD in a file called COPYING.LESSER (LGPL license) and file COPYING (GPL \n" "license). If not, see .\n\n" "Prior to September 2nd, 2012 the XRootD software suite was licensed under a\n" "modified BSD license (see file COPYING.BSD). This applies to all code that\n" "was in the XRootD git repository prior to that date.\n" xrootd-5.6.9/MANIFEST.in000066400000000000000000000004641457266313600146310ustar00rootroot00000000000000include *.sh *.py *.in include CMakeLists.txt include COPYING* LICENSE include VERSION README.md recursive-include bindings * recursive-include cmake * recursive-include docs * recursive-include packaging * recursive-include src * recursive-include tests * recursive-include ups * recursive-include utils * xrootd-5.6.9/README.md000066400000000000000000000105771457266313600143600ustar00rootroot00000000000000

## XRootD: eXtended ROOT Daemon The [XRootD](http://xrootd.org) project provides a high-performance, fault-tolerant, and secure solution for handling massive amounts of data distributed across multiple storage resources, such as disk servers, tape libraries, and remote sites. It enables efficient data access and movement in a transparent and uniform manner, regardless of the underlying storage technology or location. It was initially developed by the High Energy Physics (HEP) community to meet the data storage and access requirements of the BaBar experiment at SLAC and later extended to meet the needs of experiments at the Large Hadron Collider (LHC) at CERN. XRootD is the core technology powering the [EOS](https://eos-web.web.cern.ch/) distributed filesystem, which is the storage solution used by LHC experiments and the storage backend for [CERNBox](https://cernbox.web.cern.ch/). XRootD is also used as the core technology for global CDN deployments across multiple science domains. XRootD is based on a scalable architecture that supports multi-protocol communications. XRootD provides a set of plugins and tools that allows the user to configure it freely to deploy data access clusters of any size, and which can include sophisticated features such as erasure coded files, various methods of authentication and authorization, as well as integration with other storage systems like [ceph](https://ceph.io). ## Documentation General documentation such as configuration reference guides, and user manuals can be found on the XRootD website at http://xrootd.org/docs.html. ## Supported Operating Systems XRootD is officially supported on the following platforms: * RedHat Enterprise Linux 7 or later and their derivatives * Debian 11 and Ubuntu 22.04 or later * macOS 11 (Big Sur) or later Support for other operating systems is provided on a best-effort basis and by contributions from the community. ## Installation Instructions XRootD is available via official channels in most operating systems. Installation via your system's package manager should be preferred. In RPM-based distributions, like CentOS, Alma, Rocky, Fedora, etc, one can search and install XRootD packages with ```sh $ sudo yum install xrootd ``` or ```sh $ sudo dnf install xrootd ``` In RHEL-based distributions, it will be necessary to first install the EPEL release repository with `yum install epel-release` or `dnf install epel-release`. If you would like to use our official repository for XRootD RPMs, you can enable it on RHEL-based distributions with ```sh $ sudo curl -L https://cern.ch/xrootd/xrootd.repo -o /etc/yum.repos.d/xrootd.repo ``` and on Fedora with ```sh $ sudo curl -L https://cern.ch/xrootd/xrootd-fedora.repo -o /etc/yum.repos.d/xrootd.repo ``` On Debian 11 or later, and Ubuntu 22.04 or later, XRootD can be installed via apt ```sh $ sudo apt install xrootd-client xrootd-server python3-xrootd ``` On macOS, XRootD is available via Homebrew ```sh $ brew install xrootd ``` XRootD can also be installed with conda, as it is also available in conda-forge: ```sh $ conda config --add channels conda-forge $ conda config --set channel_priority strict $ conda install xrootd ``` Finally, it is possible to install the XRootD python bindings from PyPI using pip: ```sh $ pip install xrootd ``` For detailed instructions on how to build and install XRootD from source code, please see [docs/INSTALL.md](https://github.com/xrootd/xrootd/blob/master/docs/INSTALL.md) in the main repository on GitHub. ## User Support and Bug Reports Bugs should be reported using [GitHub issues](https://github.com/xrootd/xrootd/issues). You can open a new ticket by clicking [here](https://github.com/xrootd/xrootd/issues/new). For general questions about XRootD, please send a message to our user mailing list at xrootd-l@slac.stanford.edu or open a new [discussion](https://github.com/xrootd/xrootd/discussions) on GitHub. Please check XRootD's contact page at http://xrootd.org/contact.html for further information. ## Contributing User contributions can be submitted via pull request on GitHub. We recommend that you create your own fork of XRootD on GitHub and use it to submit your patches. For more detailed instructions on how to contribute, please refer to the file [docs/CONTRIBUTING.md](https://github.com/xrootd/xrootd/blob/master/docs/CONTRIBUTING.md). xrootd-5.6.9/VERSION000066400000000000000000000000071457266313600141340ustar00rootroot00000000000000v5.6.9 xrootd-5.6.9/bindings/000077500000000000000000000000001457266313600146645ustar00rootroot00000000000000xrootd-5.6.9/bindings/CMakeLists.txt000066400000000000000000000000711457266313600174220ustar00rootroot00000000000000 if( BUILD_PYTHON ) add_subdirectory( python ) endif() xrootd-5.6.9/bindings/python/000077500000000000000000000000001457266313600162055ustar00rootroot00000000000000xrootd-5.6.9/bindings/python/.gitignore000077500000000000000000000000711457266313600201760ustar00rootroot00000000000000build .project .pydevproject .cproject *.py[co] MANIFEST xrootd-5.6.9/bindings/python/CMakeLists.txt000066400000000000000000000020361457266313600207460ustar00rootroot00000000000000cmake_minimum_required(VERSION 3.16...3.25) project(PyXRootD LANGUAGES CXX) if( CMAKE_VERSION VERSION_LESS 3.18 ) set(PYTHON_COMPONENTS Interpreter Development) else() set(PYTHON_COMPONENTS Interpreter Development.Module) endif() find_package(Python REQUIRED COMPONENTS ${PYTHON_COMPONENTS}) if(CMAKE_SOURCE_DIR STREQUAL PROJECT_SOURCE_DIR OR PYPI_BUILD) add_subdirectory(src) else() configure_file(setup.py setup.py) file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/VERSION "${XRootD_VERSION_STRING}") option(INSTALL_PYTHON_BINDINGS "Install Python bindings" TRUE) if(INSTALL_PYTHON_BINDINGS) set(PIP_OPTIONS "" CACHE STRING "Install options for pip") install(CODE " execute_process(COMMAND ${Python_EXECUTABLE} -m pip install ${PIP_OPTIONS} --prefix \$ENV{DESTDIR}${CMAKE_INSTALL_PREFIX} ${CMAKE_CURRENT_BINARY_DIR} RESULT_VARIABLE INSTALL_STATUS COMMAND_ECHO STDOUT) if(NOT INSTALL_STATUS EQUAL 0) message(FATAL_ERROR \"Failed to install Python bindings\") endif() ") endif() endif() xrootd-5.6.9/bindings/python/MANIFEST.in000066400000000000000000000002021457266313600177350ustar00rootroot00000000000000include CMakeLists.txt recursive-include tests * recursive-include examples *.py recursive-include docs * recursive-include src * xrootd-5.6.9/bindings/python/README.md000066400000000000000000000003141457266313600174620ustar00rootroot00000000000000## XRootD Python Bindings This is a set of simple but pythonic bindings for XRootD. It is designed to make it easy to interface with the XRootD client, by writing Python instead of having to write C++. xrootd-5.6.9/bindings/python/VERSION000077700000000000000000000000001457266313600207462../../VERSIONustar00rootroot00000000000000xrootd-5.6.9/bindings/python/docs/000077500000000000000000000000001457266313600171355ustar00rootroot00000000000000xrootd-5.6.9/bindings/python/docs/Makefile000066400000000000000000000127151457266313600206030ustar00rootroot00000000000000# Makefile for Sphinx documentation # # You can set these variables from the command line. SPHINXOPTS = SPHINXBUILD = sphinx-build PAPER = BUILDDIR = build # Internal variables. PAPEROPT_a4 = -D latex_paper_size=a4 PAPEROPT_letter = -D latex_paper_size=letter ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source # the i18n builder cannot share the environment and doctrees with the others I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext help: @echo "Please use \`make ' where is one of" @echo " html to make standalone HTML files" @echo " dirhtml to make HTML files named index.html in directories" @echo " singlehtml to make a single large HTML file" @echo " pickle to make pickle files" @echo " json to make JSON files" @echo " htmlhelp to make HTML files and a HTML help project" @echo " qthelp to make HTML files and a qthelp project" @echo " devhelp to make HTML files and a Devhelp project" @echo " epub to make an epub" @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" @echo " latexpdf to make LaTeX files and run them through pdflatex" @echo " text to make text files" @echo " man to make manual pages" @echo " texinfo to make Texinfo files" @echo " info to make Texinfo files and run them through makeinfo" @echo " gettext to make PO message catalogs" @echo " changes to make an overview of all changed/added/deprecated items" @echo " linkcheck to check all external links for integrity" @echo " doctest to run all doctests embedded in the documentation (if enabled)" clean: -rm -rf $(BUILDDIR)/* html: $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." dirhtml: $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." singlehtml: $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml @echo @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." pickle: $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle @echo @echo "Build finished; now you can process the pickle files." json: $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json @echo @echo "Build finished; now you can process the JSON files." htmlhelp: $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp @echo @echo "Build finished; now you can run HTML Help Workshop with the" \ ".hhp project file in $(BUILDDIR)/htmlhelp." qthelp: $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp @echo @echo "Build finished; now you can run "qcollectiongenerator" with the" \ ".qhcp project file in $(BUILDDIR)/qthelp, like this:" @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/PyXRootD.qhcp" @echo "To view the help file:" @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/PyXRootD.qhc" devhelp: $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp @echo @echo "Build finished." @echo "To view the help file:" @echo "# mkdir -p $$HOME/.local/share/devhelp/PyXRootD" @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/PyXRootD" @echo "# devhelp" epub: $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub @echo @echo "Build finished. The epub file is in $(BUILDDIR)/epub." latex: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." @echo "Run \`make' in that directory to run these through (pdf)latex" \ "(use \`make latexpdf' here to do that automatically)." latexpdf: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo "Running LaTeX files through pdflatex..." $(MAKE) -C $(BUILDDIR)/latex all-pdf @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." text: $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text @echo @echo "Build finished. The text files are in $(BUILDDIR)/text." man: $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man @echo @echo "Build finished. The manual pages are in $(BUILDDIR)/man." texinfo: $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo @echo @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." @echo "Run \`make' in that directory to run these through makeinfo" \ "(use \`make info' here to do that automatically)." info: $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo @echo "Running Texinfo files through makeinfo..." make -C $(BUILDDIR)/texinfo info @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." gettext: $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale @echo @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." changes: $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes @echo @echo "The overview file is in $(BUILDDIR)/changes." linkcheck: $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck @echo @echo "Link check complete; look for any errors in the above output " \ "or in $(BUILDDIR)/linkcheck/output.txt." doctest: $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest @echo "Testing of doctests in the sources finished, look at the " \ "results in $(BUILDDIR)/doctest/output.txt." xrootd-5.6.9/bindings/python/docs/ReleaseNotes.txt000066400000000000000000000023111457266313600222640ustar00rootroot00000000000000======== PyXRootD ======== Release Notes ============= ------------- Version 0.3.0 ------------- + **New features** * Improve integration with xrootd 4.x CopyProcess API by passing to python all of the result dictionaties. This involves one API change, where the CopyProcess.run method now returns a tuple (Status, [Results]) instead of just Status. + Major bug fixes * Fix memory leaks by doing proper reference counting of objects created within C++ code. * Consistently create the URL objects from urls represented as strings. ------------- Version 0.2.0 ------------- + **New features** * Move copy process to xrootd4 API * Move file and filesystem to xrootd4 API * Implement file.fcntl and file.visa + Major bug fixes * Release the GIL while running copy jobs to allow other Python threads to run + **Miscellaneous** * Cleanup of compilation scripts ------------- Version 0.1.3 ------------- + **Minor bug fixes** * Make the MkDirFlags.MAKEPATH flag work (issue #6) * Fix a segfault when listing invalid directory (issues #3 and #4) * Fix a SystemError exception occuring when trying to copy a file (issue #2) + **Miscellaneous** * Fix file permissions xrootd-5.6.9/bindings/python/docs/source/000077500000000000000000000000001457266313600204355ustar00rootroot00000000000000xrootd-5.6.9/bindings/python/docs/source/.static/000077500000000000000000000000001457266313600220025ustar00rootroot00000000000000xrootd-5.6.9/bindings/python/docs/source/.static/css/000077500000000000000000000000001457266313600225725ustar00rootroot00000000000000xrootd-5.6.9/bindings/python/docs/source/.static/css/custom.css000077500000000000000000000365411457266313600246320ustar00rootroot00000000000000/* * rtd.css * ~~~~~~~~~~~~~~~ * * Sphinx stylesheet -- sphinxdoc theme. Originally created by * Armin Ronacher for Werkzeug. * * Customized for ReadTheDocs by Eric Pierce & Eric Holscher * * :copyright: Copyright 2007-2010 by the Sphinx team, see AUTHORS. * :license: BSD, see LICENSE for details. * */ /* RTD colors * light blue: #e8ecef * medium blue: #8ca1af * dark blue: #465158 * dark grey: #444444 * * white hover: #d1d9df; * medium blue hover: #697983; * green highlight: #8ecc4c * light blue (project bar): #e8ecef */ @import url(http://fonts.googleapis.com/css?family=Alegreya:700); /* PAGE LAYOUT -------------------------------------------------------------- */ body { font: 100%/1.5 "ff-meta-web-pro-1","ff-meta-web-pro-2",Arial,"Helvetica Neue",sans-serif; text-align: center; color: black; background-color: #465158; padding: 0; margin: 0; } div.document { text-align: left; background-color: #e8ecef; } div.bodywrapper { background-color: #ffffff; border-left: 1px solid #ccc; border-bottom: 1px solid #ccc; margin: 0 0 0 16em; } div.body { margin: 0; padding: 0.5em 1.3em; min-width: 20em; } div.related { font-size: 1em; background-color: #465158; } div.documentwrapper { float: left; width: 100%; background-color: #e8ecef; } /* HEADINGS --------------------------------------------------------------- */ h1 { margin: 0; padding: 0.7em 0 0.3em 0; font-size: 1.5em; line-height: 1.15; color: #111; clear: both; } h1.main { font-size: 3.0em; font-family: 'Alegreya', serif; } h2 { margin: 2em 0 0.2em 0; font-size: 1.35em; padding: 0; color: #465158; } h3 { margin: 1em 0 -0.3em 0; font-size: 1.2em; color: #6c818f; } div.body h1 a, div.body h2 a, div.body h3 a, div.body h4 a, div.body h5 a, div.body h6 a { color: black; } h1 a.anchor, h2 a.anchor, h3 a.anchor, h4 a.anchor, h5 a.anchor, h6 a.anchor { display: none; margin: 0 0 0 0.3em; padding: 0 0.2em 0 0.2em; color: #aaa !important; } h1:hover a.anchor, h2:hover a.anchor, h3:hover a.anchor, h4:hover a.anchor, h5:hover a.anchor, h6:hover a.anchor { display: inline; } h1 a.anchor:hover, h2 a.anchor:hover, h3 a.anchor:hover, h4 a.anchor:hover, h5 a.anchor:hover, h6 a.anchor:hover { color: #777; background-color: #eee; } /* LINKS ------------------------------------------------------------------ */ /* Normal links get a pseudo-underline */ a { color: #444; text-decoration: none; border-bottom: 1px solid #ccc; } /* Links in sidebar, TOC, index trees and tables have no underline */ .sphinxsidebar a, .toctree-wrapper a, .indextable a, #indices-and-tables a { color: #444; text-decoration: none; border-bottom: none; } /* Most links get an underline-effect when hovered */ a:hover, div.toctree-wrapper a:hover, .indextable a:hover, #indices-and-tables a:hover { color: #111; text-decoration: none; border-bottom: 1px solid #111; } /* Footer links */ div.footer a { color: #86989B; text-decoration: none; border: none; } div.footer a:hover { color: #a6b8bb; text-decoration: underline; border: none; } /* Permalink anchor (subtle grey with a red hover) */ div.body a.headerlink { color: #ccc; font-size: 1em; margin-left: 6px; padding: 0 4px 0 4px; text-decoration: none; border: none; } div.body a.headerlink:hover { color: #c60f0f; border: none; } /* NAVIGATION BAR --------------------------------------------------------- */ div.related ul { height: 2.5em; } div.related ul li { margin: 0; padding: 0.65em 0; float: left; display: block; color: white; /* For the >> separators */ font-size: 0.8em; } div.related ul li.right { float: right; margin-right: 5px; color: transparent; /* Hide the | separators */ } /* "Breadcrumb" links in nav bar */ div.related ul li a { order: none; background-color: inherit; font-weight: bold; margin: 6px 0 6px 4px; line-height: 1.75em; color: #ffffff; padding: 0.4em 0.8em; border: none; border-radius: 3px; } /* previous / next / modules / index links look more like buttons */ div.related ul li.right a { margin: 0.375em 0; background-color: #697983; text-shadow: 0 1px rgba(0, 0, 0, 0.5); border-radius: 3px; -webkit-border-radius: 3px; -moz-border-radius: 3px; } /* All navbar links light up as buttons when hovered */ div.related ul li a:hover { background-color: #8ca1af; color: #ffffff; text-decoration: none; border-radius: 3px; -webkit-border-radius: 3px; -moz-border-radius: 3px; } /* Take extra precautions for tt within links */ a tt, div.related ul li a tt { background: inherit !important; color: inherit !important; } /* SIDEBAR ---------------------------------------------------------------- */ div.sphinxsidebarwrapper { padding: 0; } div.sphinxsidebar { margin: 0; margin-left: -100%; float: left; top: 3em; left: 0; padding: 0 1em; width: 14em; font-size: 1em; text-align: left; background-color: #e8ecef; } div.sphinxsidebar img { max-width: 12em; } div.sphinxsidebar h3, div.sphinxsidebar h4, div.sphinxsidebar p.logo { margin: 1.2em 0 0.3em 0; font-size: 1em; padding: 0; color: #222222; font-family: "ff-meta-web-pro-1", "ff-meta-web-pro-2", "Arial", "Helvetica Neue", sans-serif; } div.sphinxsidebar h3 a { color: #444444; } div.sphinxsidebar ul, div.sphinxsidebar p { margin-top: 0; padding-left: 0; line-height: 130%; background-color: #e8ecef; } /* No bullets for nested lists, but a little extra indentation */ div.sphinxsidebar ul ul { list-style-type: none; margin-left: 1.5em; padding: 0; } /* A little top/bottom padding to prevent adjacent links' borders * from overlapping each other */ div.sphinxsidebar ul li { padding: 1px 0; } /* A little left-padding to make these align with the ULs */ div.sphinxsidebar p.topless { padding-left: 0 0 0 1em; } /* Make these into hidden one-liners */ div.sphinxsidebar ul li, div.sphinxsidebar p.topless { white-space: nowrap; overflow: hidden; } /* ...which become visible when hovered */ div.sphinxsidebar ul li:hover, div.sphinxsidebar p.topless:hover { overflow: visible; } /* Search text box and "Go" button */ #searchbox { margin-top: 2em; margin-bottom: 1em; background: #ddd; padding: 0.5em; border-radius: 6px; -moz-border-radius: 6px; -webkit-border-radius: 6px; } #searchbox h3 { margin-top: 0; } /* Make search box and button abut and have a border */ input, div.sphinxsidebar input { border: 1px solid #999; float: left; } /* Search textbox */ input[type="text"] { margin: 0; padding: 0 3px; height: 20px; width: 144px; border-top-left-radius: 3px; border-bottom-left-radius: 3px; -moz-border-radius-topleft: 3px; -moz-border-radius-bottomleft: 3px; -webkit-border-top-left-radius: 3px; -webkit-border-bottom-left-radius: 3px; } /* Search button */ input[type="submit"] { margin: 0 0 0 -1px; /* -1px prevents a double-border with textbox */ height: 22px; color: #444; background-color: #e8ecef; padding: 1px 4px; font-weight: bold; border-top-right-radius: 3px; border-bottom-right-radius: 3px; -moz-border-radius-topright: 3px; -moz-border-radius-bottomright: 3px; -webkit-border-top-right-radius: 3px; -webkit-border-bottom-right-radius: 3px; } input[type="submit"]:hover { color: #ffffff; background-color: #8ecc4c; } div.sphinxsidebar p.searchtip { clear: both; padding: 0.5em 0 0 0; background: #ddd; color: #666; font-size: 0.9em; } /* Sidebar links are unusual */ div.sphinxsidebar li a, div.sphinxsidebar p a { background: #e8ecef; /* In case links overlap main content */ border-radius: 3px; -moz-border-radius: 3px; -webkit-border-radius: 3px; border: 1px solid transparent; /* To prevent things jumping around on hover */ padding: 0 5px 0 5px; } div.sphinxsidebar li a:hover, div.sphinxsidebar p a:hover { color: #111; text-decoration: none; border: 1px solid #888; } div.sphinxsidebar p.logo a { border: 0; } /* Tweak any link appearing in a heading */ div.sphinxsidebar h3 a { } /* OTHER STUFF ------------------------------------------------------------ */ cite, code, tt { font-family: 'Consolas', 'Deja Vu Sans Mono', 'Bitstream Vera Sans Mono', monospace; font-size: 0.95em; letter-spacing: 0.01em; } tt { background-color: #f2f2f2; color: #444; } tt.descname, tt.descclassname, tt.xref { border: 0; } hr { border: 1px solid #abc; margin: 2em; } pre, #_fontwidthtest { font-family: 'Consolas', 'Deja Vu Sans Mono', 'Bitstream Vera Sans Mono', monospace; margin: 1em 2em; font-size: 0.95em; letter-spacing: 0.015em; line-height: 120%; padding: 0.5em; border: 1px solid #ccc; background-color: #eee; border-radius: 6px; -moz-border-radius: 6px; -webkit-border-radius: 6px; } pre a { color: inherit; text-decoration: underline; } td.linenos pre { margin: 1em 0em; } td.code pre { margin: 1em 0em; } div.quotebar { background-color: #f8f8f8; max-width: 250px; float: right; padding: 2px 7px; border: 1px solid #ccc; } div.topic { background-color: #f8f8f8; } table { border-collapse: collapse; margin: 0 -0.5em 0 -0.5em; } table td, table th { padding: 0.2em 0.5em 0.2em 0.5em; } /* ADMONITIONS AND WARNINGS ------------------------------------------------- */ /* Shared by admonitions, warnings and sidebars */ div.admonition, div.warning, div.sidebar { font-size: 0.9em; margin: 2em; padding: 0; /* border-radius: 6px; -moz-border-radius: 6px; -webkit-border-radius: 6px; */ } div.admonition p, div.warning p, div.sidebar p { margin: 0.5em 1em 0.5em 1em; padding: 0; } div.admonition pre, div.warning pre, div.sidebar pre { margin: 0.4em 1em 0.4em 1em; } div.admonition p.admonition-title, div.warning p.admonition-title, div.sidebar p.sidebar-title { margin: 0; padding: 0.1em 0 0.1em 0.5em; color: white; font-weight: bold; font-size: 1.1em; text-shadow: 0 1px rgba(0, 0, 0, 0.5); } div.admonition ul, div.admonition ol, div.warning ul, div.warning ol, div.sidebar ul, div.sidebar ol { margin: 0.1em 0.5em 0.5em 3em; padding: 0; } /* Admonitions and sidebars only */ div.admonition, div.sidebar { border: 1px solid #609060; background-color: #e9ffe9; } div.admonition p.admonition-title, div.sidebar p.sidebar-title { background-color: #70A070; border-bottom: 1px solid #609060; } /* Warnings only */ div.warning { border: 1px solid #900000; background-color: #ffe9e9; } div.warning p.admonition-title { background-color: #b04040; border-bottom: 1px solid #900000; } /* Sidebars only */ div.sidebar { max-width: 30%; } div.versioninfo { margin: 1em 0 0 0; border: 1px solid #ccc; background-color: #DDEAF0; padding: 8px; line-height: 1.3em; font-size: 0.9em; } .viewcode-back { font-family: 'Lucida Grande', 'Lucida Sans Unicode', 'Geneva', 'Verdana', sans-serif; } div.viewcode-block:target { background-color: #f4debf; border-top: 1px solid #ac9; border-bottom: 1px solid #ac9; } dl { margin: 1em 0 2.5em 0; } /* Highlight target when you click an internal link */ dt:target { background: #ffe080; } /* Don't highlight whole divs */ div.highlight { background: transparent; } /* But do highlight spans (so search results can be highlighted) */ span.highlight { background: #ffe080; } div.footer { background-color: #465158; color: #eeeeee; padding: 0 2em 2em 2em; clear: both; font-size: 0.8em; text-align: center; } p { margin: 0.8em 0 0.5em 0; } .section p img.math { margin: 0; } .section p img { margin: 1em 2em; } /* MOBILE LAYOUT -------------------------------------------------------------- */ @media screen and (max-width: 600px) { h1, h2, h3, h4, h5 { position: relative; } ul { padding-left: 1.25em; } div.bodywrapper a.headerlink, #indices-and-tables h1 a { color: #e6e6e6; font-size: 80%; float: right; line-height: 1.8; position: absolute; right: -0.7em; visibility: inherit; } div.bodywrapper h1 a.headerlink, #indices-and-tables h1 a { line-height: 1.5; } pre { font-size: 0.7em; overflow: auto; word-wrap: break-word; white-space: pre-wrap; } div.related ul { height: 2.5em; padding: 0; text-align: left; } div.related ul li { clear: both; color: #465158; padding: 0.2em 0; } div.related ul li:last-child { border-bottom: 1px dotted #8ca1af; padding-bottom: 0.4em; margin-bottom: 1em; width: 100%; } div.related ul li a { color: #465158; padding-right: 0; } div.related ul li a:hover { background: inherit; color: inherit; } div.related ul li.right { clear: none; padding: 0.65em 0; margin-bottom: 0.5em; } div.related ul li.right a { color: #fff; padding-right: 0.8em; } div.related ul li.right a:hover { background-color: #8ca1af; } div.body { clear: both; min-width: 0; word-wrap: break-word; } div.bodywrapper { margin: 0 0 0 0; } div.sphinxsidebar { float: none; margin: 0; width: auto; } div.sphinxsidebar input[type="text"] { height: 2em; line-height: 2em; width: 70%; } div.sphinxsidebar input[type="submit"] { height: 2em; margin-left: 0.5em; width: 20%; } div.sphinxsidebar p.searchtip { background: inherit; margin-bottom: 1em; } div.sphinxsidebar ul li, div.sphinxsidebar p.topless { white-space: normal; } .bodywrapper img { display: block; margin-left: auto; margin-right: auto; max-width: 100%; } div.documentwrapper { float: none; } div.admonition, div.warning, pre, blockquote { margin-left: 0em; margin-right: 0em; } .body p img { margin: 0; } #searchbox { background: transparent; } .related:not(:first-child) li { display: none; } .related:not(:first-child) li.right { display: block; } div.footer { padding: 1em; } .rtd_doc_footer .rtd-badge { float: none; margin: 1em auto; position: static; } .rtd_doc_footer .rtd-badge.revsys-inline { margin-right: auto; margin-bottom: 2em; } table.indextable { display: block; width: auto; } .indextable tr { display: block; } .indextable td { display: block; padding: 0; width: auto !important; } .indextable td dt { margin: 1em 0; } ul.search { margin-left: 0.25em; } ul.search li div.context { font-size: 90%; line-height: 1.1; margin-bottom: 1; margin-left: 0; } }xrootd-5.6.9/bindings/python/docs/source/.templates/000077500000000000000000000000001457266313600225115ustar00rootroot00000000000000xrootd-5.6.9/bindings/python/docs/source/.templates/layout.html000077500000000000000000000001311457266313600247120ustar00rootroot00000000000000{% extends "!layout.html" %} {% set css_files = css_files + ['_static/css/custom.css']%} xrootd-5.6.9/bindings/python/docs/source/conf.py000066400000000000000000000175361457266313600217500ustar00rootroot00000000000000# -*- coding: utf-8 -*- # # PyXRootD documentation build configuration file, created by # sphinx-quickstart on Tue Mar 19 22:57:10 2013. # # This file is execfile()d with the current directory set to its containing dir. # # Note that not all possible configuration values are present in this # autogenerated file. # # All configuration values have a default; values that are commented out # serve to show the default. import sys, os # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. #sys.path.insert(0, os.path.abspath('.')) # -- General configuration ----------------------------------------------------- # If your documentation needs a minimal Sphinx version, state it here. #needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.intersphinx', 'sphinx.ext.todo', 'sphinx.ext.coverage', 'sphinx.ext.viewcode'] # Add any paths that contain templates here, relative to this directory. templates_path = ['.templates'] # The suffix of source filenames. source_suffix = '.rst' # The encoding of source files. #source_encoding = 'utf-8-sig' # The master toctree document. master_doc = 'index' # General information about the project. project = u'pyxrootd' copyright = u'2013, CERN' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. version = 'current' # The full version, including alpha/beta/rc tags. release = 'current' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. #language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: #today = '' # Else, today_fmt is used as the format for a strftime call. #today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. exclude_patterns = [] # The reST default role (used for this markup: `text`) to use for all documents. #default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). #add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. #show_authors = False # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' # A list of ignored prefixes for module index sorting. #modindex_common_prefix = [] # -- Options for HTML output --------------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. html_theme = 'basic' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. # html_theme_options = {} # Add any paths that contain custom themes here, relative to this directory. #html_theme_path = [] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". #html_title = None # A shorter title for the navigation bar. Default is the same as html_title. #html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. html_logo = 'xrootd-200x68.png' # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. html_favicon = 'favicon.ico' # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['.static'] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. #html_last_updated_fmt = '%b %d, %Y' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. #html_use_smartypants = True # Custom sidebar templates, maps document names to template names. #html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. #html_additional_pages = {} # If false, no module index is generated. #html_domain_indices = True # If false, no index is generated. #html_use_index = True # If true, the index is split into individual pages for each letter. #html_split_index = False # If true, links to the reST sources are added to the pages. #html_show_sourcelink = True # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. #html_show_sphinx = True # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. #html_show_copyright = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. #html_use_opensearch = '' # This is the file name suffix for HTML files (e.g. ".xhtml"). #html_file_suffix = None # Output file base name for HTML help builder. htmlhelp_basename = 'pyxrootddoc' # -- Options for LaTeX output -------------------------------------------------- latex_elements = { # The paper size ('letterpaper' or 'a4paper'). #'papersize': 'letterpaper', # The font size ('10pt', '11pt' or '12pt'). #'pointsize': '10pt', # Additional stuff for the LaTeX preamble. #'preamble': '', } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [ ('index', 'pyxrootd.tex', u'pyxrootd Documentation', u'CERN', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of # the title page. #latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. #latex_use_parts = False # If true, show page references after internal links. #latex_show_pagerefs = False # If true, show URL addresses after external links. #latex_show_urls = False # Documents to append as an appendix to all manuals. #latex_appendices = [] # If false, no module index is generated. #latex_domain_indices = True # -- Options for manual page output -------------------------------------------- # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ ('index', 'pyxrootd', u'pyxrootd Documentation', [u'CERN'], 1) ] # If true, show URL addresses after external links. #man_show_urls = False # -- Options for Texinfo output ------------------------------------------------ # Grouping the document tree into Texinfo files. List of tuples # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ ('index', 'pyxrootd', u'pyxrootd Documentation', u'CERN', 'pyxrootd', 'One line description of project.', 'Miscellaneous'), ] # Documents to append as an appendix to all manuals. #texinfo_appendices = [] # If false, no module index is generated. #texinfo_domain_indices = True # How to display URL addresses: 'footnote', 'no', or 'inline'. #texinfo_show_urls = 'footnote' # Example configuration for intersphinx: refer to the Python standard library. intersphinx_mapping = { 'python': ('https://docs.python.org/3/', None) } xrootd-5.6.9/bindings/python/docs/source/examples.rst000066400000000000000000000002061457266313600230030ustar00rootroot00000000000000============ **Examples** ============ .. toctree:: :maxdepth: 2 examples/filesystem examples/file examples/copyprocess xrootd-5.6.9/bindings/python/docs/source/examples/000077500000000000000000000000001457266313600222535ustar00rootroot00000000000000xrootd-5.6.9/bindings/python/docs/source/examples/copyprocess.rst000066400000000000000000000004631457266313600253610ustar00rootroot00000000000000======================== ``CopyProcess`` examples ======================== .. include:: ../../../examples/copyprocess.py :start-line: 1 :end-line: 3 .. literalinclude:: ../../../examples/copyprocess.py :lines: 29- .. include:: ../../../examples/copyprocess.py :start-line: 4 :end-line: 27 xrootd-5.6.9/bindings/python/docs/source/examples/file.rst000066400000000000000000000054321457266313600237300ustar00rootroot00000000000000================= ``File`` examples ================= This page includes some simple examples of how to use the ``pyxrootd`` `File` object to manipulate files on an ``xrootd`` server. We'll use the following `File` object as a basis for the rest of the examples:: from XRootD import client from XRootD.client.flags import OpenFlags with client.File() as f: f.open('root://someserver//tmp/eggs', OpenFlags.UPDATE) f.write('green\neggs\nand\nham\n') .. ----------------------------------------------------------------------------- .. read .. ----------------------------------------------------------------------------- .. include:: ../../../examples/read.py :start-line: 1 :end-line: 3 .. literalinclude:: ../../../examples/read.py :lines: 17- .. include:: ../../../examples/read.py :start-line: 4 :end-line: 8 .. ----------------------------------------------------------------------------- .. write .. ----------------------------------------------------------------------------- .. include:: ../../../examples/write.py :start-line: 1 :end-line: 3 .. literalinclude:: ../../../examples/write.py :lines: 16- .. include:: ../../../examples/write.py :start-line: 4 :end-line: 8 .. ----------------------------------------------------------------------------- .. iterate .. ----------------------------------------------------------------------------- .. include:: ../../../examples/iterate.py :start-line: 1 :end-line: 3 .. literalinclude:: ../../../examples/iterate.py :lines: 18- .. include:: ../../../examples/iterate.py :start-line: 4 :end-line: 10 .. ----------------------------------------------------------------------------- .. readlines .. ----------------------------------------------------------------------------- .. include:: ../../../examples/readlines.py :start-line: 1 :end-line: 3 .. literalinclude:: ../../../examples/readlines.py :lines: 18- .. include:: ../../../examples/readlines.py :start-line: 4 :end-line: 10 .. ----------------------------------------------------------------------------- .. readchunks .. ----------------------------------------------------------------------------- .. include:: ../../../examples/readchunks.py :start-line: 1 :end-line: 3 .. literalinclude:: ../../../examples/readchunks.py :lines: 16- .. include:: ../../../examples/readchunks.py :start-line: 4 :end-line: 8 .. ----------------------------------------------------------------------------- .. vector_read .. ----------------------------------------------------------------------------- .. include:: ../../../examples/vector_read.py :start-line: 1 :end-line: 3 .. literalinclude:: ../../../examples/vector_read.py :lines: 17- .. include:: ../../../examples/vector_read.py :start-line: 4 :end-line: 9 xrootd-5.6.9/bindings/python/docs/source/examples/filesystem.rst000066400000000000000000000062531457266313600251770ustar00rootroot00000000000000======================= ``FileSystem`` examples ======================= This page includes some simple examples of basic usage of the ``pyxrootd`` `FileSystem` object to interact with an ``xrootd`` server. We'll use the following imports and `FileSystem` object as the basis for the rest of the examples:: from XRootD import client from XRootD.client.flags import DirListFlags, OpenFlags, MkDirFlags, QueryCode myclient = client.FileSystem('root://someserver:1094') .. ----------------------------------------------------------------------------- .. copy .. ----------------------------------------------------------------------------- .. include:: ../../../examples/copy.py :start-line: 1 :end-line: 3 .. literalinclude:: ../../../examples/copy.py :lines: 11- .. include:: ../../../examples/copy.py :start-line: 4 :end-line: 6 .. ----------------------------------------------------------------------------- .. dirlist .. ----------------------------------------------------------------------------- .. include:: ../../../examples/dirlist.py :start-line: 1 :end-line: 3 .. literalinclude:: ../../../examples/dirlist.py :lines: 16- .. include:: ../../../examples/dirlist.py :start-line: 4 :end-line: 9 .. ----------------------------------------------------------------------------- .. mkdir .. ----------------------------------------------------------------------------- .. include:: ../../../examples/mkdir.py :start-line: 1 :end-line: 3 .. literalinclude:: ../../../examples/mkdir.py :lines: 9- .. ----------------------------------------------------------------------------- .. rmdir .. ----------------------------------------------------------------------------- .. include:: ../../../examples/rmdir.py :start-line: 1 :end-line: 3 .. literalinclude:: ../../../examples/rmdir.py :lines: 8- .. ----------------------------------------------------------------------------- .. mv .. ----------------------------------------------------------------------------- .. include:: ../../../examples/mv.py :start-line: 1 :end-line: 3 .. literalinclude:: ../../../examples/mv.py :lines: 8- .. ----------------------------------------------------------------------------- .. rm .. ----------------------------------------------------------------------------- .. include:: ../../../examples/rm.py :start-line: 1 :end-line: 3 .. literalinclude:: ../../../examples/rm.py :lines: 8- .. ----------------------------------------------------------------------------- .. locate .. ----------------------------------------------------------------------------- .. include:: ../../../examples/locate.py :start-line: 1 :end-line: 3 .. literalinclude:: ../../../examples/locate.py :lines: 13- .. include:: ../../../examples/locate.py :start-line: 4 :end-line: 7 .. ----------------------------------------------------------------------------- .. query .. ----------------------------------------------------------------------------- .. include:: ../../../examples/query.py :start-line: 1 :end-line: 3 .. literalinclude:: ../../../examples/query.py :lines: 17- .. include:: ../../../examples/query.py :start-line: 4 :end-line: 11 xrootd-5.6.9/bindings/python/docs/source/favicon.ico000066400000000000000000000102761457266313600225640ustar00rootroot00000000000000  ( @ ******C))*`(()=|KeP-(%}&'(2& (%$)))|***I*****)))).o31/70,hLv~**********:*))P t~񫙂}te'((L=3u誗 *$!)**%******)**]@.uSimhoLn422YB3{***************m&()**)))**)(&&&+))952{Yy()***)******))*='•t\')*S5s~UJCiQw|}))*******)**8*!Iߺ&')S6yqɴ|kMGB/*'x`Mpyc0*&)**.**)(((tv+,,vM2w)))SA4òóƴPIC*'&iRnzgQC9(()4)**7y753fF0kzi,+*óƸƶĴ<84>5.|asoN<85-(&ɼlEC>K9.}[#$%iQ?ŷǹƸŶóua'((zaMp{iG)((PɿhbX&((qJp\.+*ôŸǹƸŶó}@81MB:zXZ3oTBqlfŻ.ø"$T3320TC6ĵŷƸƷĵbO@OHEǾȿ!qi¶ź951|psu_7õĵõ³nk°LȯźȽwu~qOzgȽʍɿùb}sR}\F543tgJD@μȸĹǽb{pN_I...?50wh϶ʿöŻiP@{ZyjEoVE421s|_xXoL/,+2 Ǽƻ[UOM:.m]P+**d=rhkEQC:LE>sSpMnTOA80*'HA=%ÍǻöDB?fN<'((>4-kHzzsQlN853eJ8nH}\F940mkhC')*IÞX-..aK'((vP7tQoy~_pNiUF//.[@hHA72da^uA71+++A))*xb;$δȼ!qXôWQJ%()gCX.`9gQ122_TIrM]D555ʿğ=/')**e***ܞйϺ #rQn+&$OHClIuU@BA?Ƽş&%%***\*** )))ywuJffE#'X-rPvrPtrPoQ>RNLǞ!))******* ***777NMLupl.-+*+*ljg<2+oJ|T;HECʾޤ **)***QPP !"|lͷʿ---P***[***9lmnyĶiObĠ%"""***aKKKiWʪlY{zy#$%z`Qhc~ɦ˩̩ĴN+38***ôC~}h_ZyKA<Ʒ|{z,,,&()|ȣپ0*,-***334ͭA;7baa,,,(((&&&)))*********(((LKK[ZWǹaa`)))I)))=*** *********)))73{^^]r/*********O**** ;`o7@?o?xrootd-5.6.9/bindings/python/docs/source/gettingstarted.rst000066400000000000000000000071311457266313600242210ustar00rootroot00000000000000=================== **Getting Started** =================== ``File`` and ``FileSystem`` usage ================================= Synchronous and Asynchronous requests ------------------------------------- The new XRootD client is capable of making both synchronous and asynchronous requests. Therefore, ``pyxrootd`` must also be capable of this, although most people will probably only need synchronous functionality most of the time. Each method in the `File` and `FileSystem` classes can take an optional ``callback`` argument. If you don't pass in a callback, you're asking for a synchronous request. If you do, the request becomes asynchronous (assuming the callback is valid, of course), and your callback will be invoked when the response is received. ``pyxrootd`` comes with a callback helper class: :mod:`XRootD.client.utils.AsyncResponseHandler`. If you use an instance of this class as your callback, you can call the :func:`wait` function whenever you like after the request is made, and it will block until the response is received. Return types ------------ .. note:: The return signature of the `File` and `FileSystem` functions changes depending on whether you make a synchronous or asynchronous request, so be careful. Synchronous requests ******************** You always get a **2-tuple** in return when you make a synchronous request. The first item in the tuple is always an :mod:`XRootD.client.responses.XRootDStatus` instance. The ``XRootDStatus`` object tells you whether the request was successful or not, along with some other information. The second item in the tuple depends on which request you made. If it's a simple request without any response information, such as :func:`XRootD.client.FileSystem.ping`, the second item is ``None``. Otherwise, you get one of the objects in :mod:`XRootD.client.responses`. For example, if you call :func:`XRootD.client.FileSystem.dirlist`, you get an instance of :mod:`XRootD.client.responses.DirectoryList`. Asynchronous requests ********************* You get a single object, an :mod:`XRootD.client.responses.XRootDStatus` instance, when you fire off an asynchronous request. This can inform you about any immediate problems in making the request, e.g. the network is not reachable (or something). However, when that callback you gave us (remember him?) gets triggered - you get not 2, but a **3-tuple**. The first, again, is an ``XRootDStatus``. The second follows the synchronous pattern, i.e. you get your response object, or ``None``. The third item is an :mod:`XRootD.client.responses.HostList` instance. This contains a list of all the hosts that were implicated while carrying out that request you made. Timeouts -------- All of the functions in this class accept an optional ``timeout`` keyword argument. The default timeout is `0`, which means that the environment default will be used. You can change the timeout value on a per-request basis with the optional parameter, or you can set it system-wide with the ``XRD_REQUESTTIMEOUT`` environment variable. Also, the timeout resolution (time interval between timeout detection) can be set with the ``XRD_TIMEOUTRESOLUTION`` environment variable. Copying files ============= If you want to copy files simply and quickly with default options, you can just use :func:`XRootD.client.FileSystem.copy`. But if you want more configurable copy jobs, or you want to copy a large number of files at once, you can use :mod:`XRootD.client.CopyProcess`. You can even pass in a copy progress handler to :func:`CopyProcess.run()` and use it to build some kind of progress display (much like the ``xrdcopy`` command does). xrootd-5.6.9/bindings/python/docs/source/index.rst000066400000000000000000000015141457266313600222770ustar00rootroot00000000000000======================================== ``pyxrootd``: Python bindings for XRootD ======================================== ``pyxrootd`` is a set of simple but pythonic bindings for `XRootD `_. It is designed to make it easy to interface with the XRootD client, by writing Python instead of having to write C++. For bug reporting and issue tracking, please see `the pyxrootd github issue tracker `_ User Guide ========== .. toctree:: :numbered: :maxdepth: 2 install gettingstarted examples API Reference ============= .. toctree:: :numbered: :maxdepth: 1 modules/client/filesystem modules/client/file modules/client/copyprocess modules/client/responses modules/client/flags modules/client/url modules/client/utils xrootd-5.6.9/bindings/python/docs/source/install.rst000066400000000000000000000203061457266313600226360ustar00rootroot00000000000000================================= Installing XRootD Python Bindings ================================= For general instructions on how to use ``pip`` to install Python packages, please take a look at https://packaging.python.org/en/latest/tutorials/installing-packages/. The installation of XRootD and its Python bindings follows for the most part the same procedure. However, there are some important things that are specific to XRootD, which we discuss here. Since XRootD 5.6, it is possible to use ``pip`` to install only the Python bindings, building it against a pre-installed version of XRootD. In this case, we recommend using the same version of XRootD for both parts, even if the newer Python bindings should be usable with older versions of XRootD 5.x. Suppose that XRootD is installed already into ``/usr``. Then, one can build and install the Python bindings as shown below:: xrootd $ cd bindings/python python $ python3 -m pip install --target install/ . Processing xrootd/bindings/python Installing build dependencies ... done Getting requirements to build wheel ... done Installing backend dependencies ... done Preparing metadata (pyproject.toml) ... done Building wheels for collected packages: xrootd Building wheel for xrootd (pyproject.toml) ... done Created wheel for xrootd: filename=xrootd-5.6-cp311-cp311-linux_x86_64.whl size=203460 sha256=8bbd9168... Stored in directory: /tmp/pip-ephem-wheel-cache-rc_kb_nx/wheels/af/1b/42/bb953908... Successfully built xrootd Installing collected packages: xrootd The command above installs the Python bindings into the ``install/`` directory in the current working directory. The structure is as shown below:: install/ |-- XRootD | |-- __init__.py | |-- __pycache__ | | |-- __init__.cpython-311.pyc | |-- client | |-- __init__.py | |-- __pycache__ | | |-- __init__.cpython-311.pyc | | |-- _version.cpython-311.pyc | | |-- copyprocess.cpython-311.pyc | | |-- env.cpython-311.pyc | | |-- file.cpython-311.pyc | | |-- filesystem.cpython-311.pyc | | |-- finalize.cpython-311.pyc | | |-- flags.cpython-311.pyc | | |-- glob_funcs.cpython-311.pyc | | |-- responses.cpython-311.pyc | | |-- url.cpython-311.pyc | | |-- utils.cpython-311.pyc | |-- _version.py | |-- copyprocess.py | |-- env.py | |-- file.py | |-- filesystem.py | |-- finalize.py | |-- flags.py | |-- glob_funcs.py | |-- responses.py | |-- url.py | |-- utils.py |-- pyxrootd | |-- __init__.py | |-- __pycache__ | | |-- __init__.cpython-311.pyc | |-- client.cpython-311-x86_64-linux-gnu.so |-- xrootd-5.6.dist-info |-- INSTALLER |-- METADATA |-- RECORD |-- REQUESTED |-- WHEEL |-- direct_url.json |-- top_level.txt 8 directories, 36 files If you would like to install it for your own user, then use ``pip install --user`` instead of ``--target``. If XRootD is not already installed into the system, then you will want to install both the client libraries and the Python bindings together using ``pip``. This is possible by using the ``setup.py`` at the top level of the project, rather than the one in the ``bindings/python`` subdirectory:: xrootd $ python3 -m pip install --target install/ . Processing xrootd Installing build dependencies ... done Getting requirements to build wheel ... done Installing backend dependencies ... done Preparing metadata (pyproject.toml) ... done Building wheels for collected packages: xrootd Building wheel for xrootd (pyproject.toml) ... done Created wheel for xrootd: filename=xrootd-5.6-cp311-cp311-linux_x86_64.whl size=65315683 sha256=a2e7ff52... Stored in directory: /tmp/pip-ephem-wheel-cache-9g6ovy4q/wheels/47/93/fc/a23666d3... Successfully built xrootd Installing collected packages: xrootd Successfully installed xrootd-5.6 In this case, the structure is a bit different than before:: xrootd $ tree install/ install/ |-- XRootD | |-- __init__.py | |-- __pycache__ | | |-- __init__.cpython-311.pyc | |-- client | |-- __init__.py | |-- __pycache__ | | |-- __init__.cpython-311.pyc | | |-- _version.cpython-311.pyc | | |-- copyprocess.cpython-311.pyc | | |-- env.cpython-311.pyc | | |-- file.cpython-311.pyc | | |-- filesystem.cpython-311.pyc | | |-- finalize.cpython-311.pyc | | |-- flags.cpython-311.pyc | | |-- glob_funcs.cpython-311.pyc | | |-- responses.cpython-311.pyc | | |-- url.cpython-311.pyc | | |-- utils.cpython-311.pyc | |-- _version.py | |-- copyprocess.py | |-- env.py | |-- file.py | |-- filesystem.py | |-- finalize.py | |-- flags.py | |-- glob_funcs.py | |-- responses.py | |-- url.py | |-- utils.py |-- pyxrootd | |-- __init__.py | |-- __pycache__ | | |-- __init__.cpython-311.pyc | |-- client.cpython-311-x86_64-linux-gnu.so | |-- libXrdAppUtils.so | |-- libXrdAppUtils.so.2 | |-- libXrdAppUtils.so.2.0.0 | |-- libXrdCl.so | |-- libXrdCl.so.3 | |-- libXrdCl.so.3.0.0 | |-- libXrdClHttp-5.so | |-- libXrdClProxyPlugin-5.so | |-- libXrdClRecorder-5.so | |-- libXrdCrypto.so | |-- libXrdCrypto.so.2 | |-- libXrdCrypto.so.2.0.0 | |-- libXrdCryptoLite.so | |-- libXrdCryptoLite.so.2 | |-- libXrdCryptoLite.so.2.0.0 | |-- libXrdCryptossl-5.so | |-- libXrdPosix.so | |-- libXrdPosix.so.3 | |-- libXrdPosix.so.3.0.0 | |-- libXrdPosixPreload.so | |-- libXrdPosixPreload.so.2 | |-- libXrdPosixPreload.so.2.0.0 | |-- libXrdSec-5.so | |-- libXrdSecProt-5.so | |-- libXrdSecgsi-5.so | |-- libXrdSecgsiAUTHZVO-5.so | |-- libXrdSecgsiGMAPDN-5.so | |-- libXrdSeckrb5-5.so | |-- libXrdSecpwd-5.so | |-- libXrdSecsss-5.so | |-- libXrdSecunix-5.so | |-- libXrdSecztn-5.so | |-- libXrdUtils.so | |-- libXrdUtils.so.3 | |-- libXrdUtils.so.3.0.0 | |-- libXrdXml.so | |-- libXrdXml.so.3 | |-- libXrdXml.so.3.0.0 |-- xrootd-5.6.dist-info |-- COPYING |-- COPYING.BSD |-- COPYING.LGPL |-- INSTALLER |-- LICENSE |-- METADATA |-- RECORD |-- REQUESTED |-- WHEEL |-- direct_url.json |-- top_level.txt 8 directories, 78 files As can be seen above, now all client libraries have been installed alongside the C++ Python bindings library (``client.cpython-311-x86_64-linux-gnu.so``). When installing via ``pip`` by simply calling ``pip install xrootd``, the package that gets installed is in this mode which includes the libraries. However, command line tools are not included. Binary wheels are supported as well. They can be built using the ``wheel`` subcommand instead of ``install``:: xrootd $ python3.12 -m pip wheel . Processing xrootd Installing build dependencies ... done Getting requirements to build wheel ... done Installing backend dependencies ... done Preparing metadata (pyproject.toml) ... done Building wheels for collected packages: xrootd Building wheel for xrootd (pyproject.toml) ... done Created wheel for xrootd: filename=xrootd-5.6-cp312-cp312-linux_x86_64.whl size=65318541 sha256=6c4ed389... Stored in directory: /tmp/pip-ephem-wheel-cache-etujwyx1/wheels/cf/67/3c/514b21dd... Successfully built xrootd If you want to have everything installed, that is, server, client, command line tools, etc, then it is recommended to use CMake to build the project, and use the options ``-DENABLE_PYTHON=ON -DINSTALL_PYTHON_BINDINGS=ON`` so that CMake takes care of calling ``pip`` to install the Python bindings compiled together with the other components in the end. The option ``-DPIP_OPTIONS`` can be used to pass on options to pip, but it should never be used to change the installation prefix, as that is handled by CMake. Please see INSTALL.md_ for instructions on how to build XRootD from source using CMake. .. _INSTALL.md: https://github.com/xrootd/xrootd/blob/master/docs/INSTALL.md xrootd-5.6.9/bindings/python/docs/source/modules/000077500000000000000000000000001457266313600221055ustar00rootroot00000000000000xrootd-5.6.9/bindings/python/docs/source/modules/client/000077500000000000000000000000001457266313600233635ustar00rootroot00000000000000xrootd-5.6.9/bindings/python/docs/source/modules/client/copyprocess.rst000066400000000000000000000006471457266313600264750ustar00rootroot00000000000000=============================================== :mod:`XRootD.client.CopyProcess`: Copying files =============================================== Class Reference --------------- .. module:: XRootD.client :noindex: .. autoclass:: XRootD.client.CopyProcess Methods ******* .. automethod:: XRootD.client.CopyProcess.add_job .. automethod:: XRootD.client.CopyProcess.prepare .. automethod:: XRootD.client.CopyProcess.run xrootd-5.6.9/bindings/python/docs/source/modules/client/file.rst000066400000000000000000000025511457266313600250370ustar00rootroot00000000000000================================================ :mod:`XRootD.client.File`: File-based operations ================================================ .. module:: XRootD.client :noindex: .. autoclass:: XRootD.client.File Similarities with Python built-in `file` object ----------------------------------------------- To provide an interface like the python built-in file object, the __iter__(), next(), readline() and readlines() methods have been implemented. These look for newlines in files, which may not always be appropriate, especially for binary data. Additionally, these methods can't be called asynchronously, and they don't return an ``XRootDStatus`` object like the others. You only get the data that was read. Class Reference --------------- Methods ******* .. automethod:: XRootD.client.File.open .. automethod:: XRootD.client.File.close .. automethod:: XRootD.client.File.stat .. automethod:: XRootD.client.File.read .. automethod:: XRootD.client.File.readline .. automethod:: XRootD.client.File.readlines .. automethod:: XRootD.client.File.readchunks .. automethod:: XRootD.client.File.write .. automethod:: XRootD.client.File.sync .. automethod:: XRootD.client.File.truncate .. automethod:: XRootD.client.File.vector_read .. automethod:: XRootD.client.File.is_open .. automethod:: XRootD.client.File.set_property .. automethod:: XRootD.client.File.get_property xrootd-5.6.9/bindings/python/docs/source/modules/client/filesystem.rst000066400000000000000000000024151457266313600263030ustar00rootroot00000000000000============================================================ :mod:`XRootD.client.FileSystem`: Filesystem-based operations ============================================================ Class Reference --------------- .. module:: XRootD.client .. autoclass:: XRootD.client.FileSystem Attributes ********** .. autoattribute:: XRootD.client.FileSystem.url Methods ******* .. automethod:: XRootD.client.FileSystem.copy .. automethod:: XRootD.client.FileSystem.locate .. automethod:: XRootD.client.FileSystem.deeplocate .. automethod:: XRootD.client.FileSystem.mv .. automethod:: XRootD.client.FileSystem.query .. automethod:: XRootD.client.FileSystem.truncate .. automethod:: XRootD.client.FileSystem.rm .. automethod:: XRootD.client.FileSystem.mkdir .. automethod:: XRootD.client.FileSystem.rmdir .. automethod:: XRootD.client.FileSystem.chmod .. automethod:: XRootD.client.FileSystem.ping .. automethod:: XRootD.client.FileSystem.stat .. automethod:: XRootD.client.FileSystem.statvfs .. automethod:: XRootD.client.FileSystem.protocol .. automethod:: XRootD.client.FileSystem.dirlist .. automethod:: XRootD.client.FileSystem.sendinfo .. automethod:: XRootD.client.FileSystem.prepare .. automethod:: XRootD.client.FileSystem.set_property .. automethod:: XRootD.client.FileSystem.get_property xrootd-5.6.9/bindings/python/docs/source/modules/client/flags.rst000066400000000000000000000121661457266313600252170ustar00rootroot00000000000000=============================================== :mod:`XRootD.client.flags`: Flags and constants =============================================== .. module:: XRootD.client.flags .. attribute:: OpenFlags | :mod:`OpenFlags.NONE`: Nothing | :mod:`OpenFlags.DELETE`: Open a new file, deleting any existing file | :mod:`OpenFlags.FORCE`: Ignore file usage rules | :mod:`OpenFlags.NEW`: Open the file only if it does not already exist | :mod:`OpenFlags.READ`: Open only for reading | :mod:`OpenFlags.UPDATE`: Open for reading and writing | :mod:`OpenFlags.REFRESH`: Refresh the cached information on file location. Voids `NoWait`. | :mod:`OpenFlags.MAKEPATH`: Create directory path if it doesn't already exist | :mod:`OpenFlags.APPEND`: Open only for appending | :mod:`OpenFlags.REPLICA`: The file is being opened for replica creation | :mod:`OpenFlags.POSC`: Enable `Persist On Successful Close` processing | :mod:`OpenFlags.NOWAIT`: Open the file only if it does not cause a wait. For :func:`XRootD.client.FileSystem.locate` : provide a location as soon as one becomes known. This means that not all locations are necessarily returned. If the file does not exist a wait is still imposed. | :mod:`OpenFlags.SEQIO`: File will be read or written sequentially .. attribute:: MkDirFlags | :mod:`MkDirFlags.NONE`: Nothing special | :mod:`MkDirFlags.MAKEPATH`: Create the entire directory tree if it doesn't exist .. attribute:: DirListFlags | :mod:`DirListFlags.NONE`: Nothing special | :mod:`DirListFlags.STAT`: Stat each entry | :mod:`DirListFlags.LOCATE`: Locate all servers hosting the directory and send the dirlist request to all of them .. attribute:: PrepareFlags | :mod:`PrepareFlags.STAGE`: Stage the file to disk if it is not online | :mod:`PrepareFlags.WRITEMODE`: The file will be accessed for modification | :mod:`PrepareFlags.COLOCATE`: Co-locate staged files, if possible | :mod:`PrepareFlags.FRESH`: Refresh file access time even if the location is known .. attribute:: AccessMode | :mod:`AccessMode.NONE`: Default, no flags | :mod:`AccessMode.UR`: Owner readable | :mod:`AccessMode.UW`: Owner writable | :mod:`AccessMode.UX`: Owner executable/browsable | :mod:`AccessMode.GR`: Group readable | :mod:`AccessMode.GW`: Group writable | :mod:`AccessMode.GX`: Group executable/browsable | :mod:`AccessMode.OR`: World readable | :mod:`AccessMode.OW`: World writable | :mod:`AccessMode.OX`: World executable/browsable .. attribute:: QueryCode | :mod:`QueryCode.STATS`: Query server stats | :mod:`QueryCode.PREPARE`: Query prepare status | :mod:`QueryCode.CHECKSUM`: Query file checksum | :mod:`QueryCode.XATTR`: Query file extended attributes | :mod:`QueryCode.SPACE`: Query logical space stats | :mod:`QueryCode.CHECKSUMCANCEL`: Query file checksum cancellation | :mod:`QueryCode.CONFIG`: Query server configuration | :mod:`QueryCode.VISA`: Query file visa attributes | :mod:`QueryCode.OPAQUE`: Implementation dependent | :mod:`QueryCode.OPAQUEFILE`: Implementation dependent .. attribute:: HostTypes | :mod:`HostTypes.IS_MANAGER`: Manager | :mod:`HostTypes.IS_SERVER`: Data server | :mod:`HostTypes.ATTR_META`: Meta manager attribute | :mod:`HostTypes.ATTR_PROXY`: Proxy server attribute | :mod:`HostTypes.ATTR_SUPER`: Supervisor attribute .. attribute:: StatInfoFlags | :mod:`StatInfoFlags.X_BIT_SET`: Executable/searchable bit set | :mod:`StatInfoFlags.IS_DIR`: This is a directory | :mod:`StatInfoFlags.OTHER`: Neither a file nor a directory | :mod:`StatInfoFlags.OFFLINE`: File is not online (ie. on disk) | :mod:`StatInfoFlags.POSC_PENDING`: File opened with POSC flag, not yet successfully closed | :mod:`StatInfoFlags.IS_READABLE`: Read access is allowed | :mod:`StatInfoFlags.IS_WRITABLE`: Write access is allowed .. attribute:: LocationType Describes the node type and file status for a given location. Used with the ``type`` attribute of :mod:`XRootD.client.responses.LocationInfo`. | :mod:`LocationType.MANAGER_ONLINE`: manager node where the file is online | :mod:`LocationType.MANAGER_PENDING`: manager node where the file is pending to be online | :mod:`LocationType.SERVER_ONLINE`: server node where the file is online | :mod:`LocationType.SERVER_PENDING`: server node where the file is pending to be online .. attribute:: AccessType Describes the allowed access type for the file at given location Used with the ``accesstype`` attribute of :mod:`XRootD.client.responses.LocationInfo`. | :mod:`AccessType.READ`: Read access is allowed | :mod:`AccessType.READ_WRITE`: Write access is allowed xrootd-5.6.9/bindings/python/docs/source/modules/client/responses.rst000066400000000000000000000016601457266313600261410ustar00rootroot00000000000000======================================================= :mod:`XRootD.client.responses`: Server response objects ======================================================= This page documents the various response objects that are returned by making requests to an `XRootD` server. .. module:: XRootD.client.responses .. autoclass:: XRootD.client.responses.XRootDStatus() .. autoclass:: XRootD.client.responses.DirectoryList() .. autoclass:: XRootD.client.responses.ListEntry() .. autoclass:: XRootD.client.responses.StatInfo() .. autoclass:: XRootD.client.responses.StatInfoVFS() .. autoclass:: XRootD.client.responses.VectorReadInfo() .. autoclass:: XRootD.client.responses.ChunkInfo() .. autoclass:: XRootD.client.responses.LocationInfo() .. autoclass:: XRootD.client.responses.Location() .. autoclass:: XRootD.client.responses.HostList() .. autoclass:: XRootD.client.responses.HostInfo() .. autoclass:: XRootD.client.responses.ProtocolInfo() xrootd-5.6.9/bindings/python/docs/source/modules/client/url.rst000066400000000000000000000005261457266313600247220ustar00rootroot00000000000000=========================================== :mod:`XRootD.client.URL`: XRootD URL object =========================================== Class Reference --------------- .. module:: XRootD.client :noindex: .. autoclass:: XRootD.client.URL() Methods ******* .. automethod:: XRootD.client.URL.is_valid .. automethod:: XRootD.client.URL.clear xrootd-5.6.9/bindings/python/docs/source/modules/client/utils.rst000066400000000000000000000004601457266313600252550ustar00rootroot00000000000000=========================================== :mod:`XRootD.client.utils`: Utility classes =========================================== .. module:: XRootD.client.utils .. autoclass:: XRootD.client.utils.AsyncResponseHandler :members: .. autoclass:: XRootD.client.utils.CopyProgressHandler :members: xrootd-5.6.9/bindings/python/docs/source/xrootd-200x68.png000066400000000000000000000352031457266313600233320ustar00rootroot00000000000000PNG  IHDRJ MgAMA a EiCCPICC profilexڝSgTS=BKKoR RB&*! J!QEEȠQ, !{kּ> H3Q5 B.@ $pd!s#~<<+"x M0B\t8K@zB@F&S`cbP-`'{[! eDh;VEX0fK9-0IWfH  0Q){`##xFW<+*x<$9E[-qWW.(I+6aa@.y24x6_-"bbϫp@t~,/;m%h^ uf@Wp~<5j>{-]cK'Xto(hw?G%fIq^D$.Tʳ?D*A, `6B$BB dr`)B(Ͱ*`/@4Qhp.U=pa( Aa!ڈbX#!H$ ɈQ"K5H1RT UH=r9\F;2G1Q= C7F dt1r=6Ыhڏ>C03l0.B8, c˱" VcϱwE 6wB aAHXLXNH $4 7 Q'"K&b21XH,#/{C7$C2'ITFnR#,4H#dk9, +ȅ3![ b@qS(RjJ4e2AURݨT5ZBRQ4u9̓IKhhitݕNWGw Ljg(gwLӋT071oUX**| J&*/Tު UUT^S}FU3S ԖUPSSg;goT?~YYLOCQ_ cx,!k u5&|v*=9C3J3WRf?qtN (~))4L1e\kXHQG6EYAJ'\'GgSSݧ M=:.kDwn^Loy}/TmG X $ <5qo</QC]@Caaᄑ.ȽJtq]zۯ6iܟ4)Y3sCQ? 0k߬~OCOg#/c/Wװwa>>r><72Y_7ȷOo_C#dz%gA[z|!?:eAAA!h쐭!ΑiP~aa~ 'W?pX15wCsDDDޛg1O9-J5*>.j<74?.fYXXIlK9.*6nl {/]py.,:@LN8A*%w% yg"/6шC\*NH*Mz쑼5y$3,幄'L Lݛ:v m2=:1qB!Mggfvˬen/kY- BTZ(*geWf͉9+̳ې7ᒶKW-X潬j9(xoʿܔĹdff-[n ڴ VE/(ۻCɾUUMfeI?m]Nmq#׹=TR+Gw- 6 U#pDy  :v{vg/jBFS[b[O>zG499?rCd&ˮ/~јѡ򗓿m|x31^VwwO| (hSЧ5bKGD pHYsaa?itIME  .^q IDATxye?z}K&!;٪B ad"^AUyEEDEEA ",$dJ!+Yg3Uu~TLg2 kgz}$%)IIJR;8P 17&`vXqQU TqkUxupGq]۶&MTߥH\8:s]lwvveS:awtɏꄮlv߮lOCnh88m_\x/llvfs]nvkO.#G jGg6zte+ۭ]];:d{ꊴS醗_mHkd::֩;+Ҏn.WwhSSvttommѥnjGWvue3ٕ=޼Uwtn|I5lpg\Wg~ܰk>Wʾjol)ƚ`-˔& DDPBAEI|tsY*93  zWS Un57xkm+>r_gU CE^ZbW笓si` 1LE2ei^}fqGr3[ZF9p8w"r%)@]u3f:PN>3=,[E:e!h14 ?#G b@Fںr~~0IJ,x1>93xge˖P.jjk  Vqm۾8F$8 ҙ,t^c?A{y-׷3crISC1MQT PB {aҲ%?P8 $ ꫿̆0Mt:ij¤)q)#$4T5*ˬ#Ddcu|CDnpuߕ$y'R)ή4}***ٛ$9vI2)4ؐP>/, _55U|GwR*SH\+/D"@U>ůcT`t2CtƤqkGN9|ug&"JT$)j"SFk{0~-+ːH$yjf=>F֢  "  4cڕ,7 P/{O<9gJ88o0o+ Қ'۝+ǒ[垿,o6?d$\-"'QbwvRYsE:0 ;wXIWOWH$  0@5W";$ B%BkҌR5fɖ<5i (Ъ0|GpE^OjHZT.<:O!&y"­*զI'XcD^ @;/aJye-= z&SMejTGȤ#aYg̊fLD4IAHlmw{*.0 %dR4z[*:; CDqg?U5zQx) D>#Llm嘣 h'q CKcTWDZ,eH$L#f C0$!a"G@""qpC؎S&P=Hx8}٬Nh˴i>j4Ud} =0"©?s0A* +.zMOOvb P ނ&_1tP=Tֈ@a C"i?*a)Vo~iu"Th௥  <=#8p&ldXۨ01cinnmeͦvX.4oz,6c֝wrao]-_HW%i ByPQU׿CʱL#Rma`vLB*=!~LoOqx6_e;}w]4%JR,^mHy X^=q쒕[y7yjj-X̙͝;~V.xY>2DEy) #/T'ϛKpϾeYp}>8Cy20 |XN˪zQAv$mZPU*jW(+g.ܷZ1 |*5 |1|ȋeDiďx 3䂐˚'kCmAu{08ȩoQ뺟*DPcnΓg꩞=qTտ^i<[iݹӀS;!mldS<ڪIEIee5)Fo$FTR(/3)O ),eАbKqg-Lmln<1G]DU`ڶ5nkLƏ$F0o^#KB}~||07 5&NB+r}Eß@U:(qCOi]Qճm>dw)"Ep(87L7xbM5m4nR8HD\}1:~fJ&%0(/KP(/(Od&ɐ!a" JAYBx̙C>џ S asgsi2dHlZϯw_IcV%s%Z{+wy|ޏ @)<7aiM)uMVp~ŋJ?Ͷ]W Z|CGe@UR dEZ ?P +aBw:A(e &\:> UMyڢ ]4Ce,}a\45o.bi}v%kxk.?SO?O8:]{pU,C+5VDV*k|u@Yli۶wID$\C=UB$Tqq]w;sѮ.E }Xlk77}1R4H'#XE\M{6F+wʢgfŲ(T;'eLٷ bhCq#ʙaXyK'C0hB Xi[x'A6MȣHeͫ.a[ˣ4C5샀OgwN+>j``@yOz'赎lSă~YmN^UD#~mtMUøǤ= 靼KD݊A\(xR3We$-e"q]xAKc|?'qO[g1mmm#R7cHCT`Pm$5t`aXCEu語ӾLU`mZźo"߶|ާۨg2ٺu 7?6mD.'a?U9`9b<=t믿Gn RcԩSm.)v~_/*Vy ~w\.p!EdV Z}A}? דcАaw+  MC##8#ҝgWX͏ Häf exSYI2"`YB*iNM2)1$MWw=0Heh[/ƪlO@Ww9X}vͥ=]y w/o$]ʛ벽gqfP9As_{N*hxڶmض%Z4 ]םx=9[Ǒ~^1^6"?qƽ{+xa@a0 #%rqU\jJеi ]^V *LYyR`J@ !QbiWԧ*XNﻄObXL= b4[&ILmo-pr׬[1 2JBHSs1,gtWpŶmsczP^AVw%:,?3d`8,Wa-"r0h^":p?YqӁ+Ti]۶//7{]QR[+|j,"pq7TuhN^"rppݦc=k # 0 {kVFx6Ҟ җd7]ͅ; h&4IS E*$y C5oԓ.dS1Ly,J9==-ݴw,``UCyPCy<ؓf 4M@սjkm#c;>s=ϻq/R=Tհm`4t2"{EOPY8$w'.\כ1x,mcJaĊc YF."qiFvJ2H% ʒ& Wc#se_|>@&mQSi1~D%u24p,dۛ:uxYo-0̏LSOD<cpPͮ<Q#)w]W\9EU7ŶmD.S}aE̋bMYUӽ?Rթ(Ɲ B=\@ GLwܪgWi DThF/QC!J5@$ 1BDLDB[涀}+1ܿ9"޵@`ņN6\7N< TA:iv]Zzvr{ّ^D?b6JWXgzjɓ'hѢ?c,)HX3(Z>8Nml\Vٶy9_m4UywH#\vAP M ?C0ٴS5%"Tf i,OS^{D"E&1"ai DN%{tu ںﻘjΟc"MC%oazḛ: 0.JYQ$ȅmG^.hp˲tѢE,]t;!"iNShѢy n'En*۶q绪Z^9Ê'Xb۶ (^ˮ%>0DvE` BrEw%UT않D)znEPDٸbϣb(t 2wxu ںFY*jpK}] 3 YK0`ڡ'0zHj+TeT-l{m4x_L0mY>0_U%!"fӠu*1y VD>ϡs^\}̉"d=ABtM188j Y|./BUYDˆ4!10RY(i> ODO 'z&NaeTW$hn6\m1ә:~6eLAE:JLXeʺw#R8D݅e/VPU绮;.<ﷱPّm#ٶm"΢t`w;'".<8rעG+eLbsA͔n:*_jX'jEwrm{4pA@wǪ7y_1EJEk;"2hU}qVWv9"f0@:: 5\R z|X3XP_7U+[[?x*LJ}u;x" x"ۤODKd"$k ,ˠǨga÷cO $~h.}CDB>}}xx`QGض=]n ۶'E+%T G*6xS$Qt}JUU 4e`b!a4" a!'A>(y980ɛUl= ،K4>[>FFw C~f&jo XEq Y8Nj &;nҳ/:$_vd!ܿX;boY2$֧LoŚ:a)lQ5}P Uzqod2IVyy ~ oٶD3/#z-z Z~[ Chö?qK'aupEJWEQՃ](DT%;0 "[V♶m'v0q+kǑT>Q]~9їyҢEvI'N8jqt~v&L e6(a/+U4$KMEm:ALHVQ1h iY ÐtI 3Ftʈiѧ1P4šոoAekYO6ngTWf0[߱.H+ %*zIDATy+mۨť"r8mb/UW~/cD"۶oU%KPRbz\ qĒ%K1rKbg㋮c2wD1a~OcdҸm# Ae`ZhHو#H'IGcJiVxŮ>!@cF4{Zh/ɟCʦI2~nk7sÏ˟Wy݊*pmd`Bn<xp:ӊ(D(H\j,\ÑHMk)ҕ\Z?Hm#>Muۋ[1WԶdr69(Ov#Wgq}+a۶ZR~ ZEUϛ2eʞEU-{!ލp bձfH5:4VP4Wk}%+ژ1.hjMQXݣ=ee,4 @{ӏ﷞VjyP3&Mŋw6wgQ/\p_XD@/P LjU5(!hGQt`yտMNfZ ӫՋvR#zʚu[yMyh0 =` F J2aDkvt]<䣨BkG{rD'>>Zŝ_u'3~2Gg|^R&zegض{`Dl-FxSDrp\/n ADk۶}p&Q%HU{E^uWFLp` D`">uW۶ݛ+WUUզW^y¶"2ljkq-V`,t]5vEv[KN [~Z6JBT& ,`xڷѽ6yI+0̸ 1}>P=zsBek/> kL84jkJԁ%'(SBUEm]06\u^Kp`E=cO~Nĉ{,Yr@_Df;v 2)(OTdRQi(ۛ!T-Lyn4M+R9J۶BdjnCS *ڻuV*%43v$K_{m&?g-g-%I^+ <(>gqb!Ʉ`Q.ogkg]/kν^mIJ dnnoΪ4ڲB+Ju Qp))K{]c\ov:4MB ǿQfX!! 31*QhWvA Chnyi󃧳f>[%+U٢'Zah~u/o- 6ogD #~Dr~w긡oFѲtH!X{Fn!JAtf!1$$ zhnnebSP.2ɤ t ˈ[n˵)}7D4Nn+'4fr+Pq>ׁ2T_p}Q_HXio^5Tslw_ FSN]uL 0zPFt z35 ٿeZ)Ȧ>m>Uǩ"jAs#Ula U$]GVU1QP=tj+}7ERVԭ4\G: ĂJTyC$ WZ:[-p%p ;qnLC_3A2)6(E2$H$"ZjHO.#đMأx{<{{ '84H"Ziޮ}u%Z!5W|cs/~qj IY2ں-R H&LL*2uuT4`aҹnJE$MXc!c "h+-K׏˗j4t^uhhhܼW)%)i HU])An ?ᐡ ,Ļ͆qBٸv%yVLBCbÏgp]ˊS),E%l(ih" ܁jVE*AWik}0B *3wk苕dq5E9[)  /0 qӀ-6`!DbA^(B_nF we[Du4L:*sCeFn{Mj]V R8u _xwpo*o *s~i[YZWϽ,n즪wK[yjVgrOY[wOUNJm%:UJ4\y?~.=#^dX/W{A/%egÒQ)`%*=JZƒ$%)IIJR$%)G {gjNtIENDB`xrootd-5.6.9/bindings/python/examples/000077500000000000000000000000001457266313600200235ustar00rootroot00000000000000xrootd-5.6.9/bindings/python/examples/copy.py000066400000000000000000000004251457266313600213500ustar00rootroot00000000000000""" Copy a file ----------- See :mod:`XRootD.client.CopyProcess` if you need multiple/more configurable copy jobs. """ from XRootD import client myclient = client.FileSystem('root://localhost') status = myclient.copy('/tmp/spam', '/tmp/eggs', force=True) assert status[0].ok xrootd-5.6.9/bindings/python/examples/copyprocess.py000066400000000000000000000031671457266313600227550ustar00rootroot00000000000000""" Add a number of copy jobs and run them in parallel with a progress handler -------------------------------------------------------------------------- Produces output similar to the following:: id: 1, total: 4 source: /tmp/spam target: /tmp/spam1 processed: 20, total: 20 end status: [SUCCESS] id: 2, total: 4 source: /tmp/spam target: root://localhost//tmp/spam2 processed: 20, total: 20 end status: [SUCCESS] id: 3, total: 4 source: root://localhost//tmp/spam target: /tmp/spam3 processed: 20, total: 20 end status: [SUCCESS] id: 4, total: 4 source: root://localhost//tmp/spam target: root://localhost//tmp/spam4 processed: 20, total: 20 end status: [SUCCESS] """ from XRootD import client class MyCopyProgressHandler(client.utils.CopyProgressHandler): def begin(self, jobId, total, source, target): print 'id: %d, total: %d' % (jobId, total) print 'source: %s' % source print 'target: %s' % target def end(self, jobId, result): print 'end status:', jobId, result def update(self, jobId, processed, total): print 'jobId: %d, processed: %d, total: %d' % (jobId, processed, total) def should_cancel( jobId ): return False process = client.CopyProcess() # From local to local process.add_job( '/tmp/spam', '/tmp/spam1' ) # From local to remote process.add_job( '/tmp/spam', 'root://localhost//tmp/spam2' ) # From remote to local process.add_job( 'root://localhost//tmp/spam', '/tmp/spam3' ) # From remote to remote process.add_job( 'root://localhost//tmp/spam', 'root://localhost//tmp/spam4' ) handler = MyCopyProgressHandler() process.prepare() process.run(handler) xrootd-5.6.9/bindings/python/examples/dirlist.py000066400000000000000000000010461457266313600220500ustar00rootroot00000000000000""" Ask a for a directory listing ----------------------------- Produces output similar to the following:: 2013-04-12 09:46:51 20 spam 2013-04-05 08:23:00 4096 .xrootd 2013-04-12 09:33:25 20 eggs """ from XRootD import client from XRootD.client.flags import DirListFlags myclient = client.FileSystem('root://localhost') status, listing = myclient.dirlist('/tmp', DirListFlags.STAT) print listing.parent for entry in listing: print "{0} {1:>10} {2}".format(entry.statinfo.modtimestr, entry.statinfo.size, entry.name)xrootd-5.6.9/bindings/python/examples/fcntl.py000066400000000000000000000003531457266313600215040ustar00rootroot00000000000000 from XRootD import client from XRootD.client.flags import OpenFlags with client.File() as f: status, response = f.open('root://localhost//tmp/eggs', OpenFlags.DELETE) status, response = f.fcntl( 'asdf' ) print status, response xrootd-5.6.9/bindings/python/examples/fcntl_async.py000066400000000000000000000005331457266313600227010ustar00rootroot00000000000000 from XRootD import client from XRootD.client.flags import OpenFlags from time import sleep def callback( status, response, hostlist ): print "Called:", status, response, hostlist with client.File() as f: status, response = f.open('root://localhost//tmp/eggs', OpenFlags.DELETE) status = f.fcntl( 'asdf', callback = callback ) sleep(20) xrootd-5.6.9/bindings/python/examples/fileproperties.py000066400000000000000000000017241457266313600234350ustar00rootroot00000000000000""" Write a chunk of data to a file and test it's property system ------------------------------------------------------------- Produces the following output:: spam\n true\n true\n true\n localhost:1094\n root://localhost:1094//tmp/eggs\n True\n True\n True\n false\n false\n false\n """ from XRootD import client from XRootD.client.flags import OpenFlags with client.File() as f: f.open('root://localhost//tmp/eggs', OpenFlags.DELETE) data = 'spam\n' f.write(data) print f.read()[1] print f.get_property( "ReadRecovery" ) print f.get_property( "WriteRecovery" ) print f.get_property( "FollowRedirects" ) print f.get_property( "DataServer" ) print f.get_property( "LastURL" ) print f.set_property( "ReadRecovery", "false" ) print f.set_property( "WriteRecovery", "false" ) print f.set_property( "FollowRedirects", "false" ) print f.get_property( "ReadRecovery" ) print f.get_property( "WriteRecovery" ) print f.get_property( "FollowRedirects" ) xrootd-5.6.9/bindings/python/examples/filesystemproperties.py000066400000000000000000000010141457266313600246720ustar00rootroot00000000000000""" Make a directory, remove it and test the filesystem properties -------------------------------------------------------------- """ from XRootD import client from XRootD.client.flags import MkDirFlags myclient = client.FileSystem("root://localhost") myclient.mkdir("/tmp/some/dir", MkDirFlags.MAKEPATH) myclient.rmdir("/tmp/some/dir") myclient.rmdir("/tmp/some") print myclient.get_property( "FollowRedirects" ) print myclient.set_property( "FollowRedirects", "false" ) print myclient.get_property( "FollowRedirects" ) xrootd-5.6.9/bindings/python/examples/iterate.py000066400000000000000000000005031457266313600220300ustar00rootroot00000000000000""" Iterate over a file, delimited by newline characters ---------------------------------------------------- Produces the following output:: 'green\n' 'eggs\n' 'and\n' 'spam\n' """ from XRootD import client with client.File() as f: f.open('root://localhost//tmp/eggs') for line in f: print '%r' % linexrootd-5.6.9/bindings/python/examples/locate.py000066400000000000000000000006041457266313600216440ustar00rootroot00000000000000""" Locate a file ------------- Produces output similar to the following:: ]> """ from XRootD import client from XRootD.client.flags import OpenFlags myclient = client.FileSystem("root://localhost") status, locations = myclient.locate("/tmp", OpenFlags.REFRESH) print locations xrootd-5.6.9/bindings/python/examples/mkdir.py000066400000000000000000000003261457266313600215040ustar00rootroot00000000000000""" Make a directory ---------------- """ from XRootD import client from XRootD.client.flags import MkDirFlags myclient = client.FileSystem("root://localhost") myclient.mkdir("/tmp/some/dir", MkDirFlags.MAKEPATH) xrootd-5.6.9/bindings/python/examples/mv.py000066400000000000000000000002461457266313600210210ustar00rootroot00000000000000""" Move/rename a file ------------------ """ from XRootD import client myclient = client.FileSystem("root://localhost") print myclient.mv("/tmp/spam", "/tmp/eggs") xrootd-5.6.9/bindings/python/examples/query.py000066400000000000000000000011551457266313600215440ustar00rootroot00000000000000""" Ask the server for some information ----------------------------------- Produces output similar to the following:: oss.cgroup=public&oss.space=52844687360&oss.free=27084992512&oss.maxf=27084992512&oss.used=25759694848&oss.quota=-1 For more information about XRootD query codes and arguments, see `the relevant section in the protocol reference `_. """ from XRootD import client from XRootD.client.flags import QueryCode myclient = client.FileSystem("root://localhost") status, response = myclient.query(QueryCode.SPACE, '/tmp') print responsexrootd-5.6.9/bindings/python/examples/read.py000066400000000000000000000013551457266313600213140ustar00rootroot00000000000000""" Read a certain amount of data from a certain offset in a file ------------------------------------------------------------- Produces the following output:: 'green\neggs\nand\nham\n' 'eggs' """ from XRootD import client from XRootD.client.flags import OpenFlags with client.File() as f: status, response = f.open('root://localhost//tmp/eggs', OpenFlags.DELETE) f.write('green\neggs\nand\nham\n') status, data = f.read() # Reads the whole file print '%r' % data print f.get_property('DataServer') print f.get_property('LastURL') print f.get_property('ReadRecovery') f.set_property('ReadRecovery', 'false') print f.get_property('ReadRecovery') status, data = f.read(offset=6, size=4) # Reads "eggs" print '%r' % dataxrootd-5.6.9/bindings/python/examples/readchunks.py000066400000000000000000000005351457266313600225270ustar00rootroot00000000000000""" Iterate over a file in chunks of the specified size --------------------------------------------------- Produces the following output:: 'green\neggs' '\nand\nspam\n' """ from XRootD import client with client.File() as f: f.open('root://localhost//tmp/eggs') for chunk in f.readchunks(offset=0, chunksize=10): print '%r' % chunk xrootd-5.6.9/bindings/python/examples/readlines.py000066400000000000000000000006761457266313600223540ustar00rootroot00000000000000""" Read all lines from a file into a list -------------------------------------- Produces the following output (Note how the first line is not returned with the call to :func:`readlines()` because we ate it with the first call to :func:`readline()`):: 'green\n' ['eggs\n', 'and\n', 'spam\n'] """ from XRootD import client with client.File() as f: f.open('root://localhost//tmp/eggs') print '%r' % f.readline() print f.readlines() xrootd-5.6.9/bindings/python/examples/rm.py000066400000000000000000000002171457266313600210130ustar00rootroot00000000000000""" Delete a file ------------- """ from XRootD import client myclient = client.FileSystem("root://localhost") print myclient.rm("/tmp/eggs") xrootd-5.6.9/bindings/python/examples/rmdir.py000066400000000000000000000002401457266313600215060ustar00rootroot00000000000000""" Delete a directory ------------------ """ from XRootD import client myclient = client.FileSystem("root://localhost") print myclient.rmdir("/tmp/some/dir") xrootd-5.6.9/bindings/python/examples/vector_read.py000066400000000000000000000015221457266313600226720ustar00rootroot00000000000000""" Read scattered data chunks in one operation ------------------------------------------- Produces the following output:: """ from XRootD import client from XRootD.client.flags import OpenFlags with client.File() as f: print f.open('root://localhost//tmp/eggs', OpenFlags.UPDATE) f.write(r'The XROOTD project aims at giving high performance, scalable ' +' fault tolerant access to data repositories of many kinds') size = f.stat()[1].size v = [(0, 40), (40, 40), (80, size - 80)] status, response = f.vector_read(chunks=v) print status for chunk in response.chunks: print chunk xrootd-5.6.9/bindings/python/examples/visa.py000066400000000000000000000003421457266313600213360ustar00rootroot00000000000000 from XRootD import client from XRootD.client.flags import OpenFlags with client.File() as f: status, response = f.open('root://localhost//tmp/eggs', OpenFlags.DELETE) status, response = f.visa() print status, response xrootd-5.6.9/bindings/python/examples/visa_async.py000066400000000000000000000005221457266313600225330ustar00rootroot00000000000000 from XRootD import client from XRootD.client.flags import OpenFlags from time import sleep def callback( status, response, hostlist ): print "Called:", status, response, hostlist with client.File() as f: status, response = f.open('root://localhost//tmp/eggs', OpenFlags.DELETE) status = f.visa( callback = callback ) sleep(20) xrootd-5.6.9/bindings/python/examples/write.py000066400000000000000000000005441457266313600215320ustar00rootroot00000000000000""" Write a chunk of data to a file ------------------------------- Produces the following output:: 'green\neggs\nand\nspam\n' """ from XRootD import client from XRootD.client.flags import OpenFlags with client.File() as f: print f.open('root://localhost//tmp/eggs', OpenFlags.UPDATE) data = 'spam\n' f.write(data, offset=15) print f.read() xrootd-5.6.9/bindings/python/libs/000077500000000000000000000000001457266313600171365ustar00rootroot00000000000000xrootd-5.6.9/bindings/python/libs/__init__.py000066400000000000000000000000001457266313600212350ustar00rootroot00000000000000xrootd-5.6.9/bindings/python/libs/client/000077500000000000000000000000001457266313600204145ustar00rootroot00000000000000xrootd-5.6.9/bindings/python/libs/client/__init__.py000066400000000000000000000010621457266313600225240ustar00rootroot00000000000000from __future__ import absolute_import, division, print_function from .glob_funcs import glob, iglob from .filesystem import FileSystem as FileSystem from .file import File as File from .url import URL as URL from .copyprocess import CopyProcess as CopyProcess from .env import EnvPutString as EnvPutString from .env import EnvGetString as EnvGetString from .env import EnvPutInt as EnvPutInt from .env import EnvGetInt as EnvGetInt from ._version import __version__ as __version__ from .env import EnvGetDefault as EnvGetDefault import XRootD.client.finalize xrootd-5.6.9/bindings/python/libs/client/_version.py000066400000000000000000000024411457266313600226130ustar00rootroot00000000000000#------------------------------------------------------------------------------- # Copyright (c) 2012-2014 by European Organization for Nuclear Research (CERN) # Author: Michal Simon #------------------------------------------------------------------------------- # This file is part of the XRootD software suite. # # XRootD is free software: you can redistribute it and/or modify # it under the terms of the GNU Lesser General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # XRootD is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU Lesser General Public License # along with XRootD. If not, see . # # In applying this licence, CERN does not waive the privileges and immunities # granted to it by virtue of its status as an Intergovernmental Organization # or submit itself to any jurisdiction. #------------------------------------------------------------------------------ from pyxrootd import client __version__ = client.XrdVersion_cpp()xrootd-5.6.9/bindings/python/libs/client/copyprocess.py000066400000000000000000000156161457266313600233500ustar00rootroot00000000000000#------------------------------------------------------------------------------- # Copyright (c) 2012-2014 by European Organization for Nuclear Research (CERN) # Author: Justin Salmon #------------------------------------------------------------------------------- # This file is part of the XRootD software suite. # # XRootD is free software: you can redistribute it and/or modify # it under the terms of the GNU Lesser General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # XRootD is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU Lesser General Public License # along with XRootD. If not, see . # # In applying this licence, CERN does not waive the privileges and immunities # granted to it by virtue of its status as an Intergovernmental Organization # or submit itself to any jurisdiction. #------------------------------------------------------------------------------- from __future__ import absolute_import, division, print_function from pyxrootd import client from XRootD.client.url import URL from XRootD.client.responses import XRootDStatus class ProgressHandlerWrapper(object): """Internal progress handler wrapper to convert parameters to friendly types""" def __init__(self, handler): self.handler = handler def begin(self, jobId, total, source, target): if self.handler: self.handler.begin(jobId, total, URL(source), URL(target)) def end(self, jobId, results): if 'status' in results: results['status'] = XRootDStatus(results['status']) if self.handler: self.handler.end(jobId, results) def update(self, jobId, processed, total): if self.handler: self.handler.update(jobId, processed, total) def should_cancel(self, jobId): if self.handler: return self.handler.should_cancel(jobId) else: return False class CopyProcess(object): """Add multiple individually-configurable copy jobs to a "copy process" and run them in parallel (yes, in parallel, because ``xrootd`` isn't limited by the `GIL`.""" def __init__(self): self.__process = client.CopyProcess() def parallel(self, parallel): """ Add a config job to the copy process in order to set the number of parallel copy jobs. :param parallel: number of parallel copy jobs :type parallel: integer """ self.__process.parallel(parallel) def add_job(self, source, target, sourcelimit = 1, force = False, posc = False, coerce = False, mkdir = False, thirdparty = 'none', checksummode = 'none', checksumtype = '', checksumpreset = '', dynamicsource = False, chunksize = 8388608, parallelchunks = 4, inittimeout = 600, tpctimeout = 1800, rmBadCksum = False, cptimeout = 0, xrateThreshold = 0, xrate = 0, retry = 0, cont = False, rtrplc = "force" ): """Add a job to the copy process. :param source: original source URL :type source: string :param target: target directory or file :type target: string :param sourcelimit: max number of download sources :type sourcelimit: integer :param force: overwrite target if it exists :type force: boolean :param posc: persist on successful close :type posc: boolean :param coerce: ignore file usage rules, i.e. apply `FORCE` flag to ``open()`` :type coerce: boolean :param mkdir: create the parent directories when creating a file :type mkdir: boolean :param thirdparty: third party copy mode :type thirdparty: string :param checksummode: checksum mode to be used :type checksummode: string :param checksumtype: type of the checksum to be computed :type checksumtype: string :param checksumpreset: pre-set checksum instead of computing it :type checksumpreset: string :param dynamicsource: read as much data from source as is available without checking the size :type dynamicsource: boolean :param chunksize: chunk size for remote transfers :type chunksize: integer :param parallelchunks: number of chunks that should be requested in parallel :type parallelchunks: integer :param inittimeout: copy initialization timeout :type inittimeout: integer :param tpctimeout: timeout for a third-party copy to finish :type tpctimeout: integer :param rmBadCksum: remove target file on bad checksum :type rmBadCksum: boolean :param cptimeout: timeout for classic cp operation :type cptimeout: integer :param xrateThreshold: data transfer rate threshold :type xrateThreshold: integer :param xrate: data transfer rate limit :type xrate: integer :param retry: number of retries :type retry: integer :param cont: continue copying a file from the point where the previous copy was interrupted :type cont: boolean :param rtrplc: the retry polic (force or continue) :type rtrplc: string """ self.__process.add_job(source, target, sourcelimit, force, posc, coerce, mkdir, thirdparty, checksummode, checksumtype, checksumpreset, dynamicsource, chunksize, parallelchunks, inittimeout, tpctimeout, rmBadCksum, cptimeout, retry, xrateThreshold, xrate, cont, rtrplc ) def prepare(self): """Prepare the copy jobs. **Must be called before** ``run()``.""" status = self.__process.prepare() return XRootDStatus(status) def run(self, handler=None): """Run the copy jobs with an optional progress handler. :param handler: a copy progress handler. You can subclass :mod:`XRootD.client.utils.CopyProgressHandler` and implement the three methods (``begin()``, ``progress()`` and ``end()`` ) to get regular progress updates for your copy jobs. """ status, results = self.__process.run(ProgressHandlerWrapper(handler)) for x in results: if 'status' in x: x['status'] = XRootDStatus(x['status']) return XRootDStatus(status), results xrootd-5.6.9/bindings/python/libs/client/env.py000066400000000000000000000045721457266313600215660ustar00rootroot00000000000000#------------------------------------------------------------------------------- # Copyright (c) 2012-2014 by European Organization for Nuclear Research (CERN) # Author: Michal Simon #------------------------------------------------------------------------------- # This file is part of the XRootD software suite. # # XRootD is free software: you can redistribute it and/or modify # it under the terms of the GNU Lesser General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # XRootD is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU Lesser General Public License # along with XRootD. If not, see . # # In applying this licence, CERN does not waive the privileges and immunities # granted to it by virtue of its status as an Intergovernmental Organization # or submit itself to any jurisdiction. #------------------------------------------------------------------------------ from pyxrootd import client def EnvPutString( key, value ): """Sets the given key in the xrootd client environment to the given value. Returns false if there is already a shell-imported setting for this key, true otherwise """ return client.EnvPutString_cpp( key, value ) def EnvGetString( key ): """Gets the given key from the xrootd client environment. If key does not exist in the environment returns None. """ return client.EnvGetString_cpp( key ) def EnvPutInt( key, value ): """Sets the given key in the xrootd client environment to the given value. Returns false if there is already a shell-imported setting for this key, true otherwise """ return client.EnvPutInt_cpp( key, value ) def EnvGetInt( key ): """Gets the given key from the xrootd client environment. If key does not exist in the environment returns None. """ return client.EnvGetInt_cpp( key ) def EnvGetDefault( key ): """ Get the default value for the given key. If key does not exist in the environment returns None. """ return client.EnvGetDefault_cpp( key )xrootd-5.6.9/bindings/python/libs/client/file.py000066400000000000000000000321311457266313600217050ustar00rootroot00000000000000#------------------------------------------------------------------------------- # Copyright (c) 2012-2014 by European Organization for Nuclear Research (CERN) # Author: Justin Salmon #------------------------------------------------------------------------------- # This file is part of the XRootD software suite. # # XRootD is free software: you can redistribute it and/or modify # it under the terms of the GNU Lesser General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # XRootD is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU Lesser General Public License # along with XRootD. If not, see . # # In applying this licence, CERN does not waive the privileges and immunities # granted to it by virtue of its status as an Intergovernmental Organization # or submit itself to any jurisdiction. #------------------------------------------------------------------------------- from __future__ import absolute_import, division, print_function from pyxrootd import client from XRootD.client.responses import XRootDStatus, StatInfo, VectorReadInfo from XRootD.client.utils import CallbackWrapper class File(object): """Interact with an ``xrootd`` server to perform file-based operations such as reading, writing, vector reading, etc.""" def __init__(self): self.__file = client.File() def __enter__(self): return self def __exit__(self, type, value, traceback): self.__file.__exit__() def __iter__(self): return self def __next__(self): return self.__file.__next__() # Python 2 compatibility def next(self): return self.__file.next() def open(self, url, flags=0, mode=0, timeout=0, callback=None): """Open the file pointed to by the given URL. :param url: url of the file to be opened :type url: string :param flags: An `ORed` combination of :mod:`XRootD.client.flags.OpenFlags` where the default is `OpenFlags.NONE` :param mode: access mode for new files, an `ORed` combination of :mod:`XRootD.client.flags.AccessMode` where the default is `AccessMode.NONE` :returns: tuple containing :mod:`XRootD.client.responses.XRootDStatus` object and None """ if callback: callback = CallbackWrapper(callback, None) return XRootDStatus(self.__file.open(url, flags, mode, timeout, callback)) status, response = self.__file.open(url, flags, mode, timeout) return XRootDStatus(status), None def close(self, timeout=0, callback=None): """Close the file. :returns: tuple containing :mod:`XRootD.client.responses.XRootDStatus` object and None As of Python 2.5, you can avoid having to call this method explicitly if you use the `with` statement. For example, the following code will automatically close *f* when the `with` block is exited:: from __future__ import with_statement # This isn't required in Python 2.6 with client.File() as f: f.open("root://someserver//somefile") for line in f: print line, """ if callback: callback = CallbackWrapper(callback, None) return XRootDStatus(self.__file.close(timeout, callback)) status, response = self.__file.close(timeout) return XRootDStatus(status), None def stat(self, force=False, timeout=0, callback=None): """Obtain status information for this file. :param force: do not use the cached information, force re-stating :type force: boolean :returns: tuple containing :mod:`XRootD.client.responses.XRootDStatus` object and :mod:`XRootD.client.responses.StatInfo` object """ if callback: callback = CallbackWrapper(callback, StatInfo) return XRootDStatus(self.__file.stat(force, timeout, callback)) status, response = self.__file.stat(force, timeout) if response: response = StatInfo(response) return XRootDStatus(status), response def read(self, offset=0, size=0, timeout=0, callback=None): """Read a data chunk from a given offset. :param offset: offset from the beginning of the file :type offset: integer :param size: number of bytes to be read :type size: integer :returns: tuple containing :mod:`XRootD.client.responses.XRootDStatus` object and the data that was read """ if callback: callback = CallbackWrapper(callback, None) return XRootDStatus(self.__file.read(offset, size, timeout, callback)) status, response = self.__file.read(offset, size, timeout) return XRootDStatus(status), response def readline(self, offset=0, size=0, chunksize=0): """Read a data chunk from a given offset, until the first newline or EOF encountered. :param offset: offset from the beginning of the file :type offset: integer :param size: maximum number of bytes to be read :type size: integer :param chunksize: size of chunk used for reading, in bytes :type chunksize: integer :returns: data that was read, including the trailing newline :rtype: string """ return self.__file.readline(offset, size, chunksize) def readlines(self, offset=0, size=0, chunksize=0): """Read lines from a given offset until EOF encountered. Return list of lines read. :param offset: offset from the beginning of the file :type offset: integer :param size: maximum number of bytes to be read :type size: integer :param chunksize: size of chunk used for reading, in bytes :type chunksize: integer :returns: data that was read, including trailing newlines :rtype: list of strings .. warning:: This method will read the whole file into memory if you don't specify an offset. Think twice about using it if your files are big. """ return self.__file.readlines(offset, size, chunksize) def readchunks(self, offset=0, chunksize=1024 * 1024 * 2): """Return an iterator object which will read data chunks from a given offset of the given chunksize until EOF. :param offset: offset from the beginning of the file :type offset: integer :param chunksize: size of chunk to read, in bytes :type chunksize: integer :returns: iterator object """ return self.__file.readchunks(offset, chunksize) def write(self, buffer, offset=0, size=0, timeout=0, callback=None): """Write a data chunk at a given offset. :param buffer: data to be written :param offset: offset from the beginning of the file :type offset: integer :param size: number of bytes to be written :type size: integer :returns: tuple containing :mod:`XRootD.client.responses.XRootDStatus` object and None """ if callback: callback = CallbackWrapper(callback, None) return XRootDStatus(self.__file.write(buffer, offset, size, timeout, callback)) status, response = self.__file.write(buffer, offset, size, timeout) return XRootDStatus(status), None def sync(self, timeout=0, callback=None): """Commit all pending disk writes. :returns: tuple containing :mod:`XRootD.client.responses.XRootDStatus` object and None """ if callback: callback = CallbackWrapper(callback, None) return XRootDStatus(self.__file.sync(timeout, callback)) status, response = self.__file.sync(timeout) return XRootDStatus(status), None def truncate(self, size, timeout=0, callback=None): """Truncate the file to a particular size. :param size: desired size of the file :type size: integer :returns: tuple containing :mod:`XRootD.client.responses.XRootDStatus` object and None """ if callback: callback = CallbackWrapper(callback, None) return XRootDStatus(self.__file.truncate(size, timeout, callback)) status, response = self.__file.truncate(size, timeout) return XRootDStatus(status), None def vector_read(self, chunks, timeout=0, callback=None): """Read scattered data chunks in one operation. :param chunks: list of the chunks to be read. The default maximum chunk size is 2097136 bytes and the default maximum number of chunks per request is 1024. The server may be queried using :func:`XRootD.client.FileSystem.query` for the actual settings. :type chunks: list of 2-tuples of the form (offset, size) :returns: tuple containing :mod:`XRootD.client.responses.XRootDStatus` object and :mod:`XRootD.client.responses.VectorReadInfo` object """ if callback: callback = CallbackWrapper(callback, VectorReadInfo) return XRootDStatus(self.__file.vector_read(chunks, timeout, callback)) status, response = self.__file.vector_read(chunks, timeout) if response: response = VectorReadInfo(response) return XRootDStatus(status), response def fcntl(self, arg, timeout=0, callback=None): """Perform a custom operation on an open file. :param arg: argument :type arg: string :returns: tuple containing :mod:`XRootD.client.responses.XRootDStatus` object and a string """ if callback: callback = CallbackWrapper(callback, None) return XRootDStatus(self.__file.fcntl( arg, timeout, callback)) status, response = self.__file.fcntl( arg, timeout ) return XRootDStatus(status), response def visa(self, timeout=0, callback=None): """Get access token to a file. :returns: tuple containing :mod:`XRootD.client.responses.XRootDStatus` object and a string """ if callback: callback = CallbackWrapper(callback, None) return XRootDStatus(self.__file.visa(timeout, callback)) status, response = self.__file.visa(timeout) return XRootDStatus(status), response def is_open(self): """Check if the file is open. :rtype: boolean """ return self.__file.is_open() def set_property(self, name, value): """Set file property. :param name: name of the property to set :type name: string :returns: boolean denoting if property setting was successful :rtype: boolean """ return self.__file.set_property(name, value) def get_property(self, name): """Get file property. :param name: name of the property :type name: string """ return self.__file.get_property(name) def set_xattr(self, attrs, timeout=0, callback=None): """Set extended file attributes. :param attrs: extended attributes to be set on the file :type attrs: list of tuples of name/value pairs :returns: tuple containing :mod:`XRootD.client.responses.XRootDStatus` object and :mod:`list of touples (string, XRootD.client.responses.XRootDStatus)` object """ if callback: callback = CallbackWrapper(callback, list) return XRootDStatus(self.__file.set_xattr(attrs, timeout, callback)) status, response = self.__file.set_xattr(attrs, timeout) return XRootDStatus(status), response def get_xattr(self, attrs, timeout=0, callback=None): """Get extended file attributes. :param attrs: list of extended attribute names to be retrived :type attrs: list of strings :returns: tuple containing :mod:`XRootD.client.responses.XRootDStatus` object and :mod:`list of touples (string, string, XRootD.client.responses.XRootDStatus)` object """ if callback: callback = CallbackWrapper(callback, list) return XRootDStatus(self.__file.get_xattr(attrs, timeout, callback)) status, response = self.__file.get_xattr(attrs, timeout) return XRootDStatus(status), response def del_xattr(self, attrs, timeout=0, callback=None): """Delete extended file attributes. :param attrs: list of extended attribute names to be deleted :type attrs: list of strings :returns: tuple containing :mod:`XRootD.client.responses.XRootDStatus` object and :mod:`list of touples (string, XRootD.client.responses.XRootDStatus)` object """ if callback: callback = CallbackWrapper(callback, list) return XRootDStatus(self.__file.del_xattr(attrs, timeout, callback)) status, response = self.__file.del_xattr(attrs, timeout) return XRootDStatus(status), response def list_xattr(self, timeout=0, callback=None): """List all extended file attributes. :returns: tuple containing :mod:`XRootD.client.responses.XRootDStatus` object and :mod:`list of touples (string, string, XRootD.client.responses.XRootDStatus)` object """ if callback: callback = CallbackWrapper(callback, list) return XRootDStatus(self.__file.list_xattr(timeout, callback)) status, response = self.__file.list_xattr(timeout) return XRootDStatus(status), response xrootd-5.6.9/bindings/python/libs/client/filesystem.py000066400000000000000000000417661457266313600231700ustar00rootroot00000000000000#------------------------------------------------------------------------------- # Copyright (c) 2012-2014 by European Organization for Nuclear Research (CERN) # Author: Justin Salmon #------------------------------------------------------------------------------- # This file is part of the XRootD software suite. # # XRootD is free software: you can redistribute it and/or modify # it under the terms of the GNU Lesser General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # XRootD is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU Lesser General Public License # along with XRootD. If not, see . # # In applying this licence, CERN does not waive the privileges and immunities # granted to it by virtue of its status as an Intergovernmental Organization # or submit itself to any jurisdiction. #------------------------------------------------------------------------------- from __future__ import absolute_import, division, print_function from pyxrootd import client from XRootD.client.responses import XRootDStatus, StatInfo, StatInfoVFS from XRootD.client.responses import LocationInfo, DirectoryList, ProtocolInfo from XRootD.client.utils import CallbackWrapper from XRootD.client.flags import AccessMode class FileSystem(object): """Interact with an ``xrootd`` server to perform filesystem-based operations such as copying files, creating directories, changing file permissions, listing directories, etc. :param url: The URL of the server to connect with :type url: string """ def __init__(self, url): self.__fs = client.FileSystem(url) @property def url(self): """The server URL object, instance of :mod:`XRootD.client.URL`""" return self.__fs.url def copy(self, source, target, force=False): """Copy a file. .. note:: This method is less configurable than using :mod:`XRootD.client.CopyProcess` - it is designed to be as simple as possible by using sensible defaults for the underlying copy job. If you need more configurability, or want to make multiple copy jobs run at once in parallel, use :mod:`XRootD.client.CopyProcess`. :param source: Source file path :type source: string :param target: Destination file path :type target: string :param force: overwrite target if it exists :type force: boolean :returns: tuple containing :mod:`XRootD.client.responses.XRootDStatus` object and None """ result = self.__fs.copy(source=source, target=target, force=force)[0] return XRootDStatus(result), None def locate(self, path, flags, timeout=0, callback=None): """Locate a file. :param path: path to the file to be located :type path: string :param flags: An `ORed` combination of :mod:`XRootD.client.flags.OpenFlags` :returns: tuple containing :mod:`XRootD.client.responses.XRootDStatus` object and :mod:`XRootD.client.responses.LocationInfo` object """ if callback: callback = CallbackWrapper(callback, LocationInfo) return XRootDStatus(self.__fs.locate(path, flags, timeout, callback)) status, response = self.__fs.locate(path, flags, timeout) if response: response = LocationInfo(response) return XRootDStatus(status), response def deeplocate(self, path, flags, timeout=0, callback=None): """Locate a file, recursively locate all disk servers. :param path: path to the file to be located :type path: string :param flags: An `ORed` combination of :mod:`XRootD.client.flags.OpenFlags` :returns: tuple containing :mod:`XRootD.client.responses.XRootDStatus` object and :mod:`XRootD.client.responses.LocationInfo` object """ if callback: callback = CallbackWrapper(callback, LocationInfo) return XRootDStatus(self.__fs.deeplocate(path, flags, timeout, callback)) status, response = self.__fs.deeplocate(path, flags, timeout) if response: response = LocationInfo(response) return XRootDStatus(status), response def mv(self, source, dest, timeout=0, callback=None): """Move a directory or a file. :param source: the file or directory to be moved :type source: string :param dest: the new name :type dest: string :returns: tuple containing :mod:`XRootD.client.responses.XRootDStatus` object and None """ if callback: callback = CallbackWrapper(callback, None) return XRootDStatus(self.__fs.mv(source, dest, timeout, callback)) status, response = self.__fs.mv(source, dest, timeout) return XRootDStatus(status), None def query(self, querycode, arg, timeout=0, callback=None): """Obtain server information. :param querycode: the query code as specified in :mod:`XRootD.client.flags.QueryCode` :param arg: query argument :type arg: string :returns: the query response or None if there was an error :rtype: string .. note:: For more information about XRootD query codes and arguments, see `the relevant section in the protocol reference `_. """ if callback: callback = CallbackWrapper(callback, None) return XRootDStatus(self.__fs.query(querycode, arg, timeout, callback)) status, response = self.__fs.query(querycode, arg, timeout) return XRootDStatus(status), response def truncate(self, path, size, timeout=0, callback=None): """Truncate a file. :param path: path to the file to be truncated :type path: string :param size: file size :type size: integer :returns: tuple containing :mod:`XRootD.client.responses.XRootDStatus` object and None """ if callback: callback = CallbackWrapper(callback, None) return XRootDStatus(self.__fs.truncate(path, size, timeout, callback)) status, response = self.__fs.truncate(path, size, timeout) return XRootDStatus(status), None def rm(self, path, timeout=0, callback=None): """Remove a file. :param path: path to the file to be removed :type path: string :returns: tuple containing :mod:`XRootD.client.responses.XRootDStatus` object and None """ if callback: callback = CallbackWrapper(callback, None) return XRootDStatus(self.__fs.rm(path, timeout, callback)) status, response = self.__fs.rm(path, timeout) return XRootDStatus(status), None def mkdir(self, path, flags=0, mode=0, timeout=0, callback=None): """Create a directory. :param path: path to the directory to create :type path: string :param flags: An `ORed` combination of :mod:`XRootD.client.flags.MkDirFlags` where the default is `MkDirFlags.NONE` :param mode: the initial file access mode, an `ORed` combination of :mod:`XRootD.client.flags.AccessMode` where the default is `rwxr-x---` :returns: tuple containing :mod:`XRootD.client.responses.XRootDStatus` object and None """ if mode == 0: mode = AccessMode.UR | AccessMode.UW | AccessMode.UX | \ AccessMode.GR | AccessMode.GX if callback: callback = CallbackWrapper(callback, None) return XRootDStatus(self.__fs.mkdir(path, flags, mode, timeout, callback)) status, response = self.__fs.mkdir(path, flags, mode, timeout) return XRootDStatus(status), None def rmdir(self, path, timeout=0, callback=None): """Remove a directory. :param path: path to the directory to remove :type path: string :returns: tuple containing :mod:`XRootD.client.responses.XRootDStatus` object and None """ if callback: callback = CallbackWrapper(callback, None) return XRootDStatus(self.__fs.rmdir(path, timeout, callback)) status, response = self.__fs.rmdir(path, timeout) return XRootDStatus(status), None def chmod(self, path, mode, timeout=0, callback=None): """Change access mode on a directory or a file. :param path: path to the file/directory to change access mode :type path: string :param mode: An `OR`ed` combination of :mod:`XRootD.client.flags.AccessMode` :returns: tuple containing :mod:`XRootD.client.responses.XRootDStatus` object and None """ if callback: callback = CallbackWrapper(callback, None) return XRootDStatus(self.__fs.chmod(path, mode, timeout, callback)) status, response = self.__fs.chmod(path, mode, timeout) return XRootDStatus(status), None def ping(self, timeout=0, callback=None): """Check if the server is alive. :returns: tuple containing :mod:`XRootD.client.responses.XRootDStatus` object and None """ if callback: callback = CallbackWrapper(callback, None) return XRootDStatus(self.__fs.ping(timeout, callback)) status, response = self.__fs.ping(timeout) return XRootDStatus(status), None def stat(self, path, timeout=0, callback=None): """Obtain status information for a path. :param path: path to the file/directory to stat :type path: string :returns: tuple containing :mod:`XRootD.client.responses.XRootDStatus` object and :mod:`XRootD.client.responses.StatInfo` object """ if callback: callback = CallbackWrapper(callback, StatInfo) return XRootDStatus(self.__fs.stat(path, timeout, callback)) status, response = self.__fs.stat(path, timeout) if response: response = StatInfo(response) return XRootDStatus(status), response def statvfs(self, path, timeout=0, callback=None): """Obtain status information for a Virtual File System. :param path: path to the file/directory to stat :type path: string :returns: tuple containing :mod:`XRootD.client.responses.XRootDStatus` object and :mod:`XRootD.client.responses.StatInfoVFS` object """ if callback: callback = CallbackWrapper(callback, StatInfoVFS) return XRootDStatus(self.__fs.statvfs(path, timeout, callback)) status, response = self.__fs.statvfs(path, timeout) if response: response = StatInfoVFS(response) return XRootDStatus(status), response def protocol(self, timeout=0, callback=None): """Obtain server protocol information. :returns: tuple containing :mod:`XRootD.client.responses.XRootDStatus` object and :mod:`XRootD.client.responses.ProtocolInfo` object """ if callback: callback = CallbackWrapper(callback, ProtocolInfo) return XRootDStatus(self.__fs.protocol(timeout, callback)) status, response = self.__fs.protocol(timeout) if response: response = ProtocolInfo(response) return XRootDStatus(status), response def dirlist(self, path, flags=0, timeout=0, callback=None): """List entries of a directory. :param path: path to the directory to list :type path: string :param flags: An `ORed` combination of :mod:`XRootD.client.flags.DirListFlags` where the default is `DirListFlags.NONE` :returns: tuple containing :mod:`XRootD.client.responses.XRootDStatus` object and :mod:`XRootD.client.responses.DirectoryList` object .. warning:: Currently, passing `DirListFlags.STAT` with an asynchronous call to :mod:`XRootD.client.FileSystem.dirlist()` does not work, due to an xrootd client limitation. So you'll get ``None`` instead of the ``StatInfo`` instance. See `the GitHub issue `_ for more details. """ if callback: callback = CallbackWrapper(callback, DirectoryList) return XRootDStatus(self.__fs.dirlist(path, flags, timeout, callback)) status, response = self.__fs.dirlist(path, flags, timeout) if response: response = DirectoryList(response) return XRootDStatus(status), response def sendinfo(self, info, timeout=0, callback=None): """Send info to the server (up to 1024 characters). :param info: the info string to be sent :type info: string :returns: tuple containing :mod:`XRootD.client.responses.XRootDStatus` object and None """ if callback: callback = CallbackWrapper(callback, None) return XRootDStatus(self.__fs.sendinfo(info, timeout, callback)) status, response = self.__fs.sendinfo(info, timeout) return XRootDStatus(status), response def prepare(self, files, flags, priority=0, timeout=0, callback=None): """Prepare one or more files for access. :param files: list of files to be prepared :type files: list :param flags: An `ORed` combination of :mod:`XRootD.client.flags.PrepareFlags` :param priority: priority of the request 0 (lowest) - 3 (highest) :type priority: integer :returns: tuple containing :mod:`XRootD.client.responses.XRootDStatus` object and None """ if callback: callback = CallbackWrapper(callback, None) return XRootDStatus(self.__fs.prepare(files, flags, priority, timeout, callback)) status, response = self.__fs.prepare(files, flags, priority, timeout) return XRootDStatus(status), response def set_property(self, name, value): """Set file system property. :param name: name of the property to set :type name: string :returns: boolean denoting if property setting was successful :rtype: boolean """ return self.__fs.set_property(name, value) def get_property(self, name): """Get file system property. :param name: name of the property :type name: string """ return self.__fs.get_property(name) def cat(self, path): """Cat the remote file. :param path: path to the remote file :type path: string """ source = self.__fs.url.hostid + '/' + path return self.__fs.cat(source) def set_xattr(self, path, attrs, timeout=0, callback=None): """Set extended file attributes. :param path: path to the file :type path: string :param attrs: extended attributes to be set on the file :type attrs: list of tuples of name/value pairs :returns: tuple containing :mod:`XRootD.client.responses.XRootDStatus` object and :mod:`list of touples (name, XRootD.client.responses.XRootDStatus)` object """ if callback: callback = CallbackWrapper(callback, list) return XRootDStatus(self.__fs.set_xattr(path, attrs, timeout, callback)) status, response = self.__fs.set_xattr(path, attrs, timeout) return XRootDStatus(status), response def get_xattr(self, path, attrs, timeout=0, callback=None): """Get extended file attributes. :param path: path to the file :type path: string :param attrs: extended attributes to be set on the file :type attrs: list of tuples of name/value pairs :returns: tuple containing :mod:`XRootD.client.responses.XRootDStatus` object and :mod:`list of touples (name, value, XRootD.client.responses.XRootDStatus)` object """ if callback: callback = CallbackWrapper(callback, list) return XRootDStatus(self.__fs.get_xattr(path, attrs, timeout, callback)) status, response = self.__fs.get_xattr(path, attrs, timeout) return XRootDStatus(status), response def del_xattr(self, path, attrs, timeout=0, callback=None): """Delete extended file attributes. :param path: path to the file :type path: string :param attrs: extended attributes to be set on the file :type attrs: list of tuples of name/value pairs :returns: tuple containing :mod:`XRootD.client.responses.XRootDStatus` object and :mod:`list of touples (name, XRootD.client.responses.XRootDStatus)` object """ if callback: callback = CallbackWrapper(callback, list) return XRootDStatus(self.__fs.del_xattr(path, attrs, timeout, callback)) status, response = self.__fs.del_xattr(path, attrs, timeout) return XRootDStatus(status), response def list_xattr(self, path, timeout=0, callback=None): """Delete extended file attributes. :param path: path to the file :type path: string :param attrs: extended attributes to be set on the file :type attrs: list of tuples of name/value pairs :returns: tuple containing :mod:`XRootD.client.responses.XRootDStatus` object and :mod:`list of touples (name, value, XRootD.client.responses.XRootDStatus)` object """ if callback: callback = CallbackWrapper(callback, list) return XRootDStatus(self.__fs.list_xattr(path, timeout, callback)) status, response = self.__fs.list_xattr(path, timeout) return XRootDStatus(status), response xrootd-5.6.9/bindings/python/libs/client/finalize.py000066400000000000000000000041011457266313600225630ustar00rootroot00000000000000#------------------------------------------------------------------------------- # Copyright (c) 2012-2014 by European Organization for Nuclear Research (CERN) # Author: Michal Simon #------------------------------------------------------------------------------- # This file is part of the XRootD software suite. # # XRootD is free software: you can redistribute it and/or modify # it under the terms of the GNU Lesser General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # XRootD is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU Lesser General Public License # along with XRootD. If not, see . # # In applying this licence, CERN does not waive the privileges and immunities # granted to it by virtue of its status as an Intergovernmental Organization # or submit itself to any jurisdiction. #------------------------------------------------------------------------------ from __future__ import absolute_import import gc import atexit from pyxrootd import client from .file import File @atexit.register def finalize(): """Python atexit handler, will stop all XRootD client threads (XrdCl JobManager, TaskManager and Poller) in order to ensure no Python APIs are called after the Python Interpreter gets finalized. """ # Ensure there are no files left open as calling their destructor will # cause "close" commands to be sent. # If this isn't done, there will be no running threads to process requests # after this function returns so the interpreter will deadlock. for obj in gc.get_objects(): try: if isinstance(obj, File) and obj.is_open(): obj.close() except ReferenceError: pass client.__XrdCl_Stop_Threads() xrootd-5.6.9/bindings/python/libs/client/flags.py000066400000000000000000000055141457266313600220670ustar00rootroot00000000000000#------------------------------------------------------------------------------- # Copyright (c) 2012-2013 by European Organization for Nuclear Research (CERN) # Author: Justin Salmon #------------------------------------------------------------------------------- # XRootD is free software: you can redistribute it and/or modify # it under the terms of the GNU Lesser General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # XRootD is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU Lesser General Public License # along with XRootD. If not, see . #------------------------------------------------------------------------------- from __future__ import absolute_import, division, print_function def enum(**enums): """Build the equivalent of a C++ enum""" reverse = dict((value, key) for key, value in enums.items()) enums['reverse_mapping'] = reverse return type('Enum', (), enums) QueryCode = enum( STATS = 1, PREPARE = 2, CHECKSUM = 3, XATTR = 4, SPACE = 5, CHECKSUMCANCEL = 6, CONFIG = 7, VISA = 8, OPAQUE = 16, OPAQUEFILE = 32 ) OpenFlags = enum( NONE = 0, # COMPRESS = 1, DELETE = 2, FORCE = 4, NEW = 8, READ = 16, UPDATE = 32, # ASYNC = 64, REFRESH = 128, MAKEPATH = 256, # APPEND = 512, # RETSTAT = 1024, REPLICA = 2048, POSC = 4096, NOWAIT = 8192, SEQIO = 16384, WRITE = 32768 ) AccessMode = enum( NONE = 0, UR = 0x100, UW = 0x080, UX = 0x040, GR = 0x020, GW = 0x010, GX = 0x008, OR = 0x004, OW = 0x002, OX = 0x001 ) MkDirFlags = enum( NONE = 0, MAKEPATH = 1 ) DirListFlags = enum( NONE = 0, STAT = 1, LOCATE = 2, RECURSIVE = 4, MERGE = 8, CHUNKED = 16, ZIP = 32 ) PrepareFlags = enum( # CANCEL = 1, # NOTIFY = 2, # NOERRS = 4, STAGE = 8, WRITEMODE = 16, COLOCATE = 32, FRESH = 64, EVICT = 1 << 8 ) HostTypes = enum( IS_MANAGER = 0x00000002, IS_SERVER = 0x00000001, ATTR_META = 0x00000100, ATTR_PROXY = 0x00000200, ATTR_SUPER = 0x00000400 ) StatInfoFlags = enum( X_BIT_SET = 1, IS_DIR = 2, OTHER = 4, OFFLINE = 8, IS_READABLE = 16, IS_WRITABLE = 32, POSC_PENDING = 64, BACKUP_EXISTS = 128 ) LocationType = enum( MANAGER_ONLINE = 0, MANAGER_PENDING = 1, SERVER_ONLINE = 2, SERVER_PENDING = 3 ) AccessType = enum( READ = 0, READ_WRITE = 1 ) xrootd-5.6.9/bindings/python/libs/client/glob_funcs.py000066400000000000000000000104051457266313600231070ustar00rootroot00000000000000#------------------------------------------------------------------------------- # Copyright (c) 2012-2018 by European Organization for Nuclear Research (CERN) # Author: Benjamin Krikler #------------------------------------------------------------------------------- # This file is part of the XRootD software suite. # # XRootD is free software: you can redistribute it and/or modify # it under the terms of the GNU Lesser General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # XRootD is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU Lesser General Public License # along with XRootD. If not, see . # # In applying this licence, CERN does not waive the privileges and immunities # granted to it by virtue of its status as an Intergovernmental Organization # or submit itself to any jurisdiction. #------------------------------------------------------------------------------- from __future__ import absolute_import, division, print_function from XRootD.client.filesystem import FileSystem import glob as gl import os import fnmatch import sys if sys.version_info[0] > 2: from urllib.parse import urlparse else: from urlparse import urlparse __all__ = ["glob", "iglob"] def split_url(url): parsed_uri = urlparse(url) domain = '{uri.scheme}://{uri.netloc}/'.format(uri=parsed_uri) path = parsed_uri.path return domain, path def iglob(pathname, raise_error=False): """ Generates paths based on a wild-carded path, potentially via xrootd. Multiple wild-cards can be present in the path. Args: pathname (str): The wild-carded path to be expanded. raise_error (bool): Whether or not to let xrootd raise an error if there's a problem. If False (default), and there's a problem for a particular directory or file, then that will simply be skipped, likely resulting in an empty list. Yields: (str): A single path that matches the wild-carded string """ # Let normal python glob try first generator = gl.iglob(pathname) path = next(generator, None) if path is not None: yield path for path in generator: yield path return # Else try xrootd instead for path in xrootd_iglob(pathname, raise_error=raise_error): yield path def xrootd_iglob(pathname, raise_error): """Handles the actual interaction with xrootd Provides a python generator over files that match the wild-card expression. """ # Split the pathname into a directory and basename dirs, basename = os.path.split(pathname) if gl.has_magic(dirs): dirs = list(xrootd_iglob(dirs, raise_error)) else: dirs = [dirs] for dirname in dirs: host, path = split_url(dirname) query = FileSystem(host) if not query: raise RuntimeError("Cannot prepare xrootd query") status, dirlist = query.dirlist(path) if status.error: if not raise_error: continue raise RuntimeError("'{!s}' for path '{}'".format(status, dirname)) for entry in dirlist.dirlist: filename = entry.name if filename in [".", ".."]: continue if not fnmatch.fnmatchcase(filename, basename): continue yield os.path.join(dirname, filename) def glob(pathname, raise_error=False): """ Creates a list of paths that match pathname. Multiple wild-cards can be present in the path. Args: pathname (str): The wild-carded path to be expanded. raise_error (bool): Whether or not to let xrootd raise an error if there's a problem. If False (default), and there's a problem for a particular directory or file, then that will simply be skipped, likely resulting in an empty list. Returns: (str): A single path that matches the wild-carded string """ return list(iglob(pathname, raise_error=raise_error)) xrootd-5.6.9/bindings/python/libs/client/responses.py000066400000000000000000000262711457266313600230170ustar00rootroot00000000000000#------------------------------------------------------------------------------- # Copyright (c) 2012-2013 by European Organization for Nuclear Research (CERN) # Author: Justin Salmon #------------------------------------------------------------------------------- # XRootD is free software: you can redistribute it and/or modify # it under the terms of the GNU Lesser General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # XRootD is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU Lesser General Public License # along with XRootD. If not, see . #------------------------------------------------------------------------------- from __future__ import absolute_import, division, print_function from XRootD.client.url import URL class Struct(object): """Convert a dict into an object by adding each dict entry to __dict__""" def __init__(self, entries): self.__dict__.update(**entries) def __repr__(self): return '<%s>' % str(', '.join('%s: %s' % (k, repr(v)) for (k, v) in self.__dict__.items())) class LocationInfo(Struct): """Path location information (a list of discovered file locations). :param locations: (List of :mod:`XRootD.client.responses.Location` objects) List of discovered locations This object is iterable:: >>> status, locations = filesystem.locate('/tmp', OpenFlags.REFRESH) >>> print locations >>> for location in locations: ... print location.address ... [::127.0.0.1]:1094 """ def __init__(self, locations): super(LocationInfo, self).__init__({'locations': [Location(l) for l in locations]}) def __iter__(self): return iter(self.locations) class Location(Struct): """Information about a single location. :var address: The address of this location :var type: The type of this location, one of :mod:`XRootD.client.flags.LocationType` :var accesstype: The allowed access type of this location, one of :mod:`XRootD.client.flags.AccessType` :var is_manager: Is the location a manager :var is_server: Is the location a server """ def __init__(self, location): super(Location, self).__init__(location) class XRootDStatus(Struct): """Status of a request. Returned with all requests. :var message: Message describing the status of this request :var ok: The request was successful :var error: Error making request :var fatal: Fatal error making request :var status: Status of the request :var code: Error type, or additional hints on what to do :var shellcode: Status code that may be returned to the shell :var errno: Errno, if any """ #---------------------------------------------------------------------------- # Additional info for the stOK status #---------------------------------------------------------------------------- suDone = 0 suContinue = 1 suRetry = 2 suPartial = 3 suAlreadyDone = 4 #---------------------------------------------------------------------------- # Generic errors #---------------------------------------------------------------------------- errNone = 0; # No error errRetry = 1; # Try again for whatever reason errUnknown = 2; # Unknown error errInvalidOp = 3; # The operation cannot be performed in the # given circumstances errFcntl = 4; # failed manipulate file descriptor errPoll = 5; # error while polling descriptors errConfig = 6; # System misconfigured errInternal = 7; # Internal error errUnknownCommand = 8; errInvalidArgs = 9; errInProgress = 10; errUninitialized = 11; errOSError = 12; errNotSupported = 13; errDataError = 14; # data is corrupted errNotImplemented = 15; # Operation is not implemented errNoMoreReplicas = 16; # No more replicas to try errPipelineError = 17; # Pipeline failed and operation couldn't be executed #---------------------------------------------------------------------------- # Socket related errors #---------------------------------------------------------------------------- errInvalidAddr = 101; errSocketError = 102; errSocketTimeout = 103; errSocketDisconnected = 104; errPollerError = 105; errSocketOptError = 106; errStreamDisconnect = 107; errConnectionError = 108; errInvalidSession = 109; #---------------------------------------------------------------------------- # Post Master related errors #---------------------------------------------------------------------------- errInvalidMessage = 201; errHandShakeFailed = 202; errLoginFailed = 203; errAuthFailed = 204; errQueryNotSupported = 205; errOperationExpired = 206; errOperationInterrupted = 207; #---------------------------------------------------------------------------- # XRootD related errors #---------------------------------------------------------------------------- errNoMoreFreeSIDs = 301; errInvalidRedirectURL = 302; errInvalidResponse = 303; errNotFound = 304; errCheckSumError = 305; errRedirectLimit = 306; errErrorResponse = 400; errRedirect = 401; errResponseNegative = 500; # Query response was negative def __init__(self, status): super(XRootDStatus, self).__init__(status) def __str__(self): return self.message class ProtocolInfo(Struct): """Protocol information for a server. :var version: The version of the protocol this server is speaking :var hostinfo: Informational flags for this host. An `ORed` combination of :mod:`XRootD.client.flags.HostTypes` """ def __init__(self, info): super(ProtocolInfo, self).__init__(info) class StatInfo(Struct): """Status information for files and directories. :var id: This file's unique identifier :var flags: Informational flags. An `ORed` combination of :mod:`XRootD.client.flags.StatInfoFlags` :var size: The file size (in bytes) :var modtime: Modification time (in seconds since epoch) :var modtimestr: Modification time (as readable string) """ def __init__(self, info): super(StatInfo, self).__init__(info) class StatInfoVFS(Struct): """Status information for Virtual File Systems. :var nodes_rw: Number of nodes that can provide read/write space :var free_rw: Size of the largest contiguous area of free r/w space (in MB) :var utilization_rw: Percentage of the partition utilization represented by ``free_rw`` :var nodes_staging: Number of nodes that can provide staging space :var free_staging: Size of the largest contiguous area of free staging space (in MB) :var utilization_staging: Percentage of the partition utilization represented by ``free_staging`` """ def __init__(self, info): super(StatInfoVFS, self).__init__(info) class DirectoryList(Struct): """Directory listing. This object is iterable:: >>> status, dirlist = filesystem.dirlist('/tmp', DirListFlags.STAT) >>> print dirlist >>> print 'Entries:', dirlist.size Entries: 2 >>> for item in dirlist: ... print item.name, item.statinfo.size ... spam 1024 eggs 2048 :var size: The size of this listing (number of entries) :var parent: The name of the parent directory of this directory :var dirlist: (List of :mod:`XRootD.client.responses.ListEntry` objects) - The list of directory entries """ def __init__(self, dirlist): dirlist.update({'dirlist': [ListEntry(e) for e in dirlist['dirlist']]}) super(DirectoryList, self).__init__(dirlist) def __iter__(self): return iter(self.dirlist) class ListEntry(Struct): """An entry in a directory listing. :var name: The name of the file/directory :var hostaddr: The address of the host on which this file/directory lives :var statinfo: (Instance of :mod:`XRootD.client.responses.StatInfo`) - Status information about this file/directory. You must pass `DirListFlags.STAT` with the call to :mod:`XRootD.client.FileSystem.dirlist()` to retrieve status information. """ def __init__(self, entry): if entry['statinfo']: entry.update({'statinfo': StatInfo(entry['statinfo'])}) super(ListEntry, self).__init__(entry) class ChunkInfo(Struct): """Describes a data chunk for a vector read. :var offset: The offset in the file from which this chunk came :var length: The length of this chunk :var buffer: The actual chunk data """ def __init__(self, info): super(ChunkInfo, self).__init__(info) class VectorReadInfo(Struct): """Vector read response object. Returned by :mod:`XRootD.client.File.vector_read()`. This object is iterable:: >>> f.open('root://localhost/tmp/spam') >>> status, chunks = file.vector_read([(0, 10), (10, 10)]) >>> print chunks >>> print chunks.size 20 >>> for chunk in chunks: ... print chunk.offset, chunk.length ... 0 10 10 10 :var size: Total size of all chunks :var chunks: (List of :mod:`XRootD.client.responses.ChunkInfo` objects) - The list of chunks that were read """ def __init__(self, info): info.update({'chunks': [ChunkInfo(c) for c in info['chunks']]}) super(VectorReadInfo, self).__init__(info) def __iter__(self): return iter(self.chunks) class HostList(Struct): """A list of hosts that were involved in the request. This object is iterable:: >>> print hostlist >>> for host in hostlist: ... print host.url ... root://localhost :var hosts: (List of :mod:`XRootD.client.responses.HostInfo` objects) - The list of hosts """ def __init__(self, hostlist): super(HostList, self).__init__({'hosts': [HostInfo(h) for h in hostlist]}) def __iter__(self): return iter(self.hosts) class HostInfo(Struct): """Information about a single host. :var url: URL of the host, instance of :mod:`XRootD.client.URL` :var protocol: Version of the protocol the host is speaking :var flags: Host type, an `ORed` combination of :mod:`XRootD.client.flags.HostTypes` :var load_balancer: Was the host used as a load balancer """ def __init__(self, info): super(HostInfo, self).__init__(info) xrootd-5.6.9/bindings/python/libs/client/url.py000066400000000000000000000054321457266313600215740ustar00rootroot00000000000000#------------------------------------------------------------------------------- # Copyright (c) 2012-2013 by European Organization for Nuclear Research (CERN) # Author: Justin Salmon #------------------------------------------------------------------------------- # XRootD is free software: you can redistribute it and/or modify # it under the terms of the GNU Lesser General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # XRootD is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU Lesser General Public License # along with XRootD. If not, see . #------------------------------------------------------------------------------- from __future__ import absolute_import, division, print_function from pyxrootd import client class URL(object): """Server URL object. This class has each portion of an `XRootD` URL split up as attributes. For example, given the URL:: >>> url = URL(root://user1:passwd1@host1:1234//path?param1=val1¶m2=val2) then ``url.hostid`` would return `user1:passwd1@host1:1234`. :var hostid: The host part of the URL, i.e. ``user1:passwd1@host1:1234`` :var protocol: The protocol part of the URL, i.e. ``root`` :var username: The username part of the URL, i.e. ``user1`` :var password: The password part of the URL, i.e. ``passwd1`` :var hostname: The name of the target host part of the URL, i.e. ``host1`` :var port: The target port part of the URL, i.e. ``1234`` :var path: The path part of the URL, i.e. ``path`` :var path_with_params: The path part of the URL with parameters, i.e. ``path?param1=val1¶m2=val2`` """ def __init__(self, url): self.__url = client.URL(url) def __str__(self): return str(self.__url) @property def hostid(self): return self.__url.hostid @property def protocol(self): return self.__url.protocol @property def username(self): return self.__url.username @property def password(self): return self.__url.password @property def hostname(self): return self.__url.hostname @property def port(self): return self.__url.port @property def path(self): return self.__url.path @property def path_with_params(self): return self.__url.path_with_params def is_valid(self): """Return the validity of the URL""" return self.__url.is_valid() def clear(self): """Clear the URL""" return self.__url.clear() xrootd-5.6.9/bindings/python/libs/client/utils.py000066400000000000000000000075261457266313600221400ustar00rootroot00000000000000#------------------------------------------------------------------------------- # Copyright (c) 2012-2013 by European Organization for Nuclear Research (CERN) # Author: Justin Salmon #------------------------------------------------------------------------------- # XRootD is free software: you can redistribute it and/or modify # it under the terms of the GNU Lesser General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # XRootD is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU Lesser General Public License # along with XRootD. If not, see . #------------------------------------------------------------------------------- from __future__ import absolute_import, division, print_function from threading import Lock from XRootD.client.responses import XRootDStatus, HostList class CallbackWrapper(object): def __init__(self, callback, responsetype): if not hasattr(callback, '__call__'): raise TypeError('callback must be callable function, class or lambda') self.callback = callback self.responsetype = responsetype def __call__(self, status, response, *argv): self.status = XRootDStatus(status) self.response = response if self.responsetype and self.response: self.response = self.responsetype(response) if argv: self.hostlist = HostList(argv[0]) else: self.hostlist = HostList([]) self.callback(self.status, self.response, self.hostlist) class AsyncResponseHandler(object): """Utility class to handle asynchronous method calls.""" def __init__(self): self.mutex = Lock() self.mutex.acquire() def __call__(self, status, response, hostlist): self.status = status self.response = response self.hostlist = hostlist self.mutex.release() def wait(self): """Block and wait for the async response""" self.mutex.acquire() self.mutex.release() return self.status, self.response, self.hostlist class CopyProgressHandler(object): """Utility class to handle progress updates from copy jobs .. note:: This class does nothing by itself. You have to subclass it and do something useful with the progress updates yourself. """ def begin(self, jobId, total, source, target): """Notify when a new job is about to start :param jobId: the job number of the copy job concerned :type jobId: integer :param total: total number of jobs being processed :type total: integer :param source: the source url of the current job :type source: :mod:`XRootD.client.URL` object :param target: the destination url of the current job :type target: :mod:`XRootD.client.URL` object """ pass def end(self, jobId, results): """Notify when the previous job has finished :param jobId: the job number of the copy job concerned :type jobId: integer :param status: status of the job :type status: :mod:`XRootD.client.responses.XRootDStatus` object """ pass def update(self, jobId, processed, total): """Notify about the progress of the current job :param jobId: the job number of the copy job concerned :type jobId: integer :param processed: bytes processed by the current job :type processed: integer :param total: total number of bytes to be processed by the current job :type total: integer """ pass def should_cancel( self, jobId ): """Check whether the current job should be canceled. :param jobId: the job number of the copy job concerned :type jobId: integer """ return False xrootd-5.6.9/bindings/python/pyproject.toml000077700000000000000000000000001457266313600244562../../pyproject.tomlustar00rootroot00000000000000xrootd-5.6.9/bindings/python/setup.py000066400000000000000000000123461457266313600177250ustar00rootroot00000000000000import os import platform import subprocess import sys from setuptools import setup, Extension from setuptools.command.build_ext import build_ext from subprocess import check_call, check_output try: from shutil import which except ImportError: from distutils.spawn import find_executable as which def get_cmake_args(): args = os.getenv('CMAKE_ARGS') if not args: return [] from shlex import split return split(args) srcdir = '${CMAKE_CURRENT_SOURCE_DIR}' cmdline_args = [] # Check for unexpanded srcdir to determine if this is part # of a regular CMake build or a Python build using setup.py. if not srcdir.startswith('$'): # When building the Python bindings as part of a standard CMake build, # propagate down which cmake command to use, and the build type, C++ # compiler, build flags, and how to link libXrdCl from the main build. cmake = '${CMAKE_COMMAND}' cmdline_args += [ '-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}', '-DCMAKE_CXX_STANDARD=${CMAKE_CXX_STANDARD}', '-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}', '-DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}', '-DXRootD_CLIENT_LIBRARY=${CMAKE_BINARY_DIR}/src/XrdCl/libXrdCl${CMAKE_SHARED_LIBRARY_SUFFIX}', '-DXRootD_INCLUDE_DIR=${CMAKE_SOURCE_DIR}/src;${CMAKE_BINARY_DIR}/src', ] else: srcdir = '.' cmake = which("cmake3") or which("cmake") cmdline_args += get_cmake_args() def get_version(): version = '${XRootD_VERSION_STRING}' if version.startswith('$'): try: version = open('VERSION').read().strip() if version.startswith('$'): output = check_output(['git', 'describe']) version = output.decode().strip() except: version = None pass if version is None: from datetime import date version = '5.7-rc' + date.today().strftime("%Y%m%d") if version.startswith('v'): version = version[1:] # Sanitize version to conform to PEP 440 # https://www.python.org/dev/peps/pep-0440 version = version.replace('-rc', 'rc') version = version.replace('-g', '+git.') version = version.replace('-', '.post', 1) version = version.replace('-', '.') return version class CMakeExtension(Extension): def __init__(self, name, src=srcdir, sources=[], **kwa): Extension.__init__(self, name, sources=sources, **kwa) self.src = os.path.abspath(src) class CMakeBuild(build_ext): def build_extensions(self): if cmake is None: raise RuntimeError('Cannot find CMake executable') for ext in self.extensions: extdir = os.path.abspath(os.path.dirname(self.get_ext_fullpath(ext.name))) # Use relative RPATHs to ensure the correct libraries are picked up. # The RPATH below covers most cases where a non-standard path is # used for installation. It allows to find libXrdCl with a relative # path from the site-packages directory. Build with install RPATH # because libraries are installed by Python/pip not CMake, so CMake # cannot fix the install RPATH later on. cmake_args = [ '-DPython_EXECUTABLE={}'.format(sys.executable), '-DCMAKE_BUILD_WITH_INSTALL_RPATH=TRUE', '-DCMAKE_ARCHIVE_OUTPUT_DIRECTORY={}/{}'.format(self.build_temp, ext.name), '-DCMAKE_LIBRARY_OUTPUT_DIRECTORY={}/{}'.format(extdir, ext.name), ] if sys.platform == 'darwin': cmake_args += [ '-DCMAKE_INSTALL_RPATH=@loader_path/../../..' ] else: cmake_args += [ '-DCMAKE_INSTALL_RPATH=$ORIGIN/../../../../$LIB' ] cmake_args += cmdline_args if not os.path.exists(self.build_temp): os.makedirs(self.build_temp) check_call([cmake, ext.src, '-B', self.build_temp] + cmake_args) check_call([cmake, '--build', self.build_temp, '--parallel']) version = get_version() setup(name='xrootd', version=version, description='XRootD Python bindings', author='XRootD Developers', author_email='xrootd-dev@slac.stanford.edu', url='http://xrootd.org', download_url='https://github.com/xrootd/xrootd/archive/v%s.tar.gz' % version, keywords=['XRootD', 'network filesystem'], license='LGPLv3+', long_description=open(srcdir + '/README.md').read(), long_description_content_type='text/plain', packages = ['XRootD', 'XRootD.client', 'pyxrootd'], package_dir = { 'pyxrootd' : srcdir + '/src', 'XRootD' : srcdir + '/libs', 'XRootD/client': srcdir + '/libs/client', }, ext_modules= [ CMakeExtension('pyxrootd') ], cmdclass={ 'build_ext': CMakeBuild }, zip_safe=False, classifiers=[ "Intended Audience :: Information Technology", "Intended Audience :: Science/Research", "License :: OSI Approved :: GNU Lesser General Public License v3 or later (LGPLv3+)", "Operating System :: MacOS", "Operating System :: POSIX :: Linux", "Operating System :: Unix", "Programming Language :: C++", "Programming Language :: Python", ] ) xrootd-5.6.9/bindings/python/src/000077500000000000000000000000001457266313600167745ustar00rootroot00000000000000xrootd-5.6.9/bindings/python/src/AsyncResponseHandler.hh000066400000000000000000000276071457266313600234230ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2012-2013 by European Organization for Nuclear Research (CERN) // Author: Justin Salmon //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #ifndef ASYNCRESPONSEHANDLER_HH_ #define ASYNCRESPONSEHANDLER_HH_ #include "PyXRootD.hh" #include "Conversions.hh" #include "Utils.hh" #include "XrdCl/XrdClXRootDResponses.hh" namespace PyXRootD { //---------------------------------------------------------------------------- //! Generic asynchronous response handler //---------------------------------------------------------------------------- template class AsyncResponseHandler: public XrdCl::ResponseHandler { public: //------------------------------------------------------------------------ //! Constructor //------------------------------------------------------------------------ AsyncResponseHandler( PyObject *callback ) : callback( callback ), state( PyGILState_UNLOCKED ) {} //------------------------------------------------------------------------ //! Destructor //------------------------------------------------------------------------ virtual ~AsyncResponseHandler() {}; //------------------------------------------------------------------------ //! Handle the asynchronous response call //------------------------------------------------------------------------ void HandleResponseWithHosts( XrdCl::XRootDStatus *status, XrdCl::AnyObject *response, XrdCl::HostList *hostList ) { // If we get called while the program's exit handlers are being called, // then calls to PyGILState_Ensure() deadlock. Py_IsInitialized() is // not thread-safe but we appear to be lacking in alternates. if (!Py_IsInitialized()) {return;} //---------------------------------------------------------------------- // Ensure we hold the Global Interpreter Lock //---------------------------------------------------------------------- state = PyGILState_Ensure(); if ( InitTypes() != 0) { delete status; delete response; delete hostList; return Exit(); } //---------------------------------------------------------------------- // Convert the XRootDStatus object //---------------------------------------------------------------------- PyObject *pystatus = ConvertType( status ); if ( !pystatus || PyErr_Occurred() ) { delete status; delete response; delete hostList; return Exit(); } //---------------------------------------------------------------------- // Convert the response object, if any //---------------------------------------------------------------------- PyObject *pyresponse = NULL; if ( response != NULL) { pyresponse = ParseResponse( response ); if ( pyresponse == NULL || PyErr_Occurred() ) { Py_XDECREF( pystatus ); delete status; delete response; delete hostList; return Exit(); } } //---------------------------------------------------------------------- // Convert the host list //---------------------------------------------------------------------- PyObject *pyhostlist = PyList_New( 0 ); if ( hostList != NULL ) { pyhostlist = ConvertType( hostList ); if ( pyhostlist == NULL || PyErr_Occurred() ) { Py_XDECREF( pystatus ); Py_XDECREF( pyresponse ); delete status; delete response; delete hostList; return Exit(); } } //---------------------------------------------------------------------- // Build the callback arguments //---------------------------------------------------------------------- if (pyresponse == NULL) pyresponse = Py_BuildValue( "" ); PyObject *args = Py_BuildValue( "(OOO)", pystatus, pyresponse, pyhostlist ); if ( !args || PyErr_Occurred() ) { Py_XDECREF( pystatus ); Py_XDECREF( pyresponse ); Py_XDECREF( pyhostlist ); delete status; delete response; delete hostList; return Exit(); } //---------------------------------------------------------------------- // Check if it is a final response or if it is just a chunk //---------------------------------------------------------------------- bool finalrsp = !( status->IsOK() && status->code == XrdCl::suContinue ); //---------------------------------------------------------------------- // Invoke the Python callback //---------------------------------------------------------------------- PyObject *callbackResult = PyObject_CallObject( this->callback, args ); Py_DECREF( args ); if ( !callbackResult || PyErr_Occurred() ) { Py_XDECREF( pystatus ); Py_XDECREF( pyresponse ); Py_XDECREF( pyhostlist ); delete status; delete response; delete hostList; return Exit(); } //---------------------------------------------------------------------- // Clean up //---------------------------------------------------------------------- Py_XDECREF( pystatus ); Py_XDECREF( pyresponse ); Py_XDECREF( pyhostlist ); Py_XDECREF( callbackResult ); if( finalrsp ) { Py_XDECREF( this->callback ); } PyGILState_Release( state ); delete status; delete response; delete hostList; if( finalrsp ) // Commit suicide... delete this; } //------------------------------------------------------------------------ //! Handle the asynchronous response call //------------------------------------------------------------------------ void HandleResponse( XrdCl::XRootDStatus *status, XrdCl::AnyObject *response ) { // If we get called while the program's exit handlers are being called, // then calls to PyGILState_Ensure() deadlock. Py_IsInitialized() is // not thread-safe but we appear to be lacking in alternates. if (!Py_IsInitialized()) {return;} //---------------------------------------------------------------------- // Ensure we hold the Global Interpreter Lock //---------------------------------------------------------------------- state = PyGILState_Ensure(); if ( InitTypes() != 0) { return Exit(); } //---------------------------------------------------------------------- // Convert the XRootDStatus object //---------------------------------------------------------------------- PyObject *pystatus = ConvertType( status ); if ( !pystatus || PyErr_Occurred() ) { return Exit(); } //---------------------------------------------------------------------- // Convert the response object, if any //---------------------------------------------------------------------- PyObject *pyresponse = NULL; if ( response != NULL) { pyresponse = ParseResponse( response ); if ( pyresponse == NULL || PyErr_Occurred() ) { Py_XDECREF( pystatus ); delete response; return Exit(); } } //---------------------------------------------------------------------- // Build the callback arguments //---------------------------------------------------------------------- if (pyresponse == NULL) pyresponse = Py_BuildValue( "" ); PyObject *args = Py_BuildValue( "(OO)", pystatus, pyresponse ); if ( !args || PyErr_Occurred() ) { Py_XDECREF( pystatus ); Py_XDECREF( pyresponse ); delete response; return Exit(); } //---------------------------------------------------------------------- // Check if it is a final response or if it is just a chunk //---------------------------------------------------------------------- bool finalrsp = !( status->IsOK() && status->code == XrdCl::suContinue ); //---------------------------------------------------------------------- // Invoke the Python callback //---------------------------------------------------------------------- PyObject *callbackResult = PyObject_CallObject( this->callback, args ); Py_DECREF( args ); if ( !callbackResult || PyErr_Occurred() ) { Py_XDECREF( pystatus ); Py_XDECREF( pyresponse ); delete response; return Exit(); } //---------------------------------------------------------------------- // Clean up //---------------------------------------------------------------------- Py_XDECREF( pystatus ); Py_XDECREF( pyresponse ); Py_XDECREF( callbackResult ); if( finalrsp ) { Py_XDECREF( this->callback ); } PyGILState_Release( state ); delete status; delete response; if( finalrsp ) // Commit suicide... delete this; } //------------------------------------------------------------------------ //! Parse out and convert the AnyObject response to a mapping type //------------------------------------------------------------------------ PyObject* ParseResponse( XrdCl::AnyObject *response ) { PyObject *pyresponse = 0; Type *type; response->Get( type ); pyresponse = ConvertType( type ); return ( !pyresponse || PyErr_Occurred() ) ? NULL : pyresponse; } //------------------------------------------------------------------------ //! Something went wrong, print error and release the GIL before returning //------------------------------------------------------------------------ void Exit() { PyErr_Print(); PyGILState_Release( state ); delete this; } private: PyObject *callback; PyGILState_STATE state; }; //---------------------------------------------------------------------------- //! Get an async response handler of the correct type //---------------------------------------------------------------------------- template XrdCl::ResponseHandler* GetHandler( PyObject *callback ) { if (!IsCallable(callback)) { return NULL; } return new AsyncResponseHandler( callback ); } } #endif /* ASYNCRESPONSEHANDLER_HH_ */ xrootd-5.6.9/bindings/python/src/CMakeLists.txt000066400000000000000000000033741457266313600215430ustar00rootroot00000000000000# WITH_SOABI was introduced in CMake 3.17 # https://cmake.org/cmake/help/latest/module/FindPython.html#commands if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.17) set(SOABI WITH_SOABI) endif() Python_add_library(client MODULE ${SOABI} # headers AsyncResponseHandler.hh ChunkIterator.hh Conversions.hh PyXRootD.hh PyXRootDCopyProcess.hh PyXRootDCopyProgressHandler.hh PyXRootDEnv.hh PyXRootDFile.hh PyXRootDFileSystem.hh PyXRootDFinalize.hh PyXRootDURL.hh Utils.hh # sources PyXRootDCopyProcess.cc PyXRootDCopyProgressHandler.cc PyXRootDFile.cc PyXRootDFileSystem.cc PyXRootDModule.cc PyXRootDURL.cc Utils.cc ) target_compile_options(client PRIVATE -w) # TODO: fix build warnings if(APPLE) set(CMAKE_MACOSX_RPATH TRUE) set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) set_target_properties(client PROPERTIES INSTALL_NAME_DIR "@rpath") endif() # Avoid a call to find_package(XRootD) in order to be able to override # variables when building the module as part of a standard CMake build. if(TARGET XrdCl) target_link_libraries(client PRIVATE XrdCl) else() find_library(XRootD_CLIENT_LIBRARY NAMES XrdCl) if(NOT XRootD_CLIENT_LIBRARY) message(FATAL_ERROR "Could not find XRootD client library") endif() find_path(XRootD_INCLUDE_DIR XrdVersion.hh PATH_SUFFIXES include/xrootd) if(NOT XRootD_INCLUDE_DIR) message(FATAL_ERROR "Could not find XRootD client include directory") endif() # The client library makes use of private XRootD headers, so add the # extra include for it to allow building the Python bindings against # a pre-installed XRootD. target_link_libraries(client PRIVATE ${XRootD_CLIENT_LIBRARY}) target_include_directories(client PRIVATE ${XRootD_INCLUDE_DIR} ${XRootD_INCLUDE_DIR}/private) endif() xrootd-5.6.9/bindings/python/src/ChunkIterator.hh000066400000000000000000000151131457266313600221000ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2012-2013 by European Organization for Nuclear Research (CERN) // Author: Justin Salmon //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #ifndef CHUNKITERATOR_HH_ #define CHUNKITERATOR_HH_ #include "PyXRootD.hh" #include "PyXRootDFile.hh" namespace PyXRootD { //---------------------------------------------------------------------------- //! Iterator class for looping over a file and yielding chunks from it //---------------------------------------------------------------------------- class ChunkIterator { public: PyObject_HEAD File *file; uint32_t chunksize; uint64_t startOffset; uint64_t currentOffset; }; //---------------------------------------------------------------------------- //! __init__ //---------------------------------------------------------------------------- static int ChunkIterator_init(ChunkIterator *self, PyObject *args) { PyObject *py_offset = NULL, *py_chunksize = NULL; if ( !PyArg_ParseTuple( args, "OOO", &self->file,&py_offset, &py_chunksize ) ) return -1; unsigned long long tmp_offset = 0; unsigned int tmp_chunksize = 2 * 1024 * 1024; // 2 MB if ( py_offset && PyObjToUllong( py_offset, &tmp_offset, "offset" ) ) return -1; if ( py_chunksize && PyObjToUint( py_chunksize, &tmp_chunksize, "chunksize" ) ) return -1; self->startOffset = (uint64_t)tmp_offset; self->chunksize = (uint32_t)tmp_chunksize; self->currentOffset = self->startOffset; return 0; } //---------------------------------------------------------------------------- //! __iter__ //---------------------------------------------------------------------------- static PyObject* ChunkIterator_iter(ChunkIterator *self) { Py_INCREF(self); return (PyObject*) self; } //---------------------------------------------------------------------------- //! __iternext__ //! //! Dumb implementation, should use prefetching/readv //---------------------------------------------------------------------------- static PyObject* ChunkIterator_iternext(ChunkIterator *self) { XrdCl::Buffer *chunk = self->file->ReadChunk( self->file, self->currentOffset, self->chunksize); PyObject *pychunk = NULL; if ( chunk->GetSize() == 0 ) { //------------------------------------------------------------------------ // Raise StopIteration exception when we are done //------------------------------------------------------------------------ PyErr_SetNone( PyExc_StopIteration ); } else { self->currentOffset += self->chunksize; pychunk = PyBytes_FromStringAndSize( (const char*) chunk->GetBuffer(), chunk->GetSize() ); } delete chunk; return pychunk; } //---------------------------------------------------------------------------- //! ChunkIterator type structure //---------------------------------------------------------------------------- static PyTypeObject ChunkIteratorType = { PyVarObject_HEAD_INIT(NULL, 0) "client.File.ChunkIterator", /* tp_name */ sizeof(ChunkIterator), /* tp_basicsize */ 0, /* tp_itemsize */ 0, /* tp_dealloc */ 0, /* tp_print */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_compare */ 0, /* tp_repr */ 0, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ 0, /* tp_hash */ 0, /* tp_call */ 0, /* tp_str */ 0, /* tp_getattro */ 0, /* tp_setattro */ 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_ITER, /* tp_flags */ "Internal chunk iterator object", /* tp_doc */ 0, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ (getiterfunc) ChunkIterator_iter, /* tp_iter */ (iternextfunc) ChunkIterator_iternext, /* tp_iternext */ 0, /* tp_methods */ 0, /* tp_members */ 0, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ 0, /* tp_descr_set */ 0, /* tp_dictoffset */ (initproc) ChunkIterator_init, /* tp_init */ }; } #endif /* CHUNKITERATOR_HH_ */ xrootd-5.6.9/bindings/python/src/Conversions.hh000066400000000000000000000325771457266313600216430ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2012-2015 by European Organization for Nuclear Research (CERN) // Author: Justin Salmon // Author: Lukasz Janyst //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #ifndef CONVERSIONS_HH_ #define CONVERSIONS_HH_ #include "PyXRootDURL.hh" #include "Utils.hh" #include #include "XrdCl/XrdClXRootDResponses.hh" #include "XrdCl/XrdClPropertyList.hh" namespace PyXRootD { //---------------------------------------------------------------------------- //! Convert an object of type T into a Python dictionary type. //---------------------------------------------------------------------------- template struct PyDict; template inline PyObject* ConvertType( T *response ) { if ( response != NULL ) { return PyDict::Convert( response ); } else { Py_RETURN_NONE; } } template<> inline PyObject* ConvertType >(const std::deque *list ) { if(list == NULL) Py_RETURN_NONE; PyObject *pylist = NULL; if(list) { pylist = PyList_New(list->size()); std::deque::const_iterator it = list->begin(); for(unsigned int i = 0; i < list->size(); ++i) { const XrdCl::PropertyList &result = *it++; PyObject *pyresult = ConvertType(&result); PyList_SetItem(pylist, i, pyresult); } } return pylist; } template<> inline PyObject* ConvertType >(std::deque *list ) { return ConvertType((const std::deque*) list); } template<> struct PyDict { static PyObject* Convert( XrdCl::AnyObject *object ) { Py_RETURN_NONE; } }; template<> struct PyDict { static PyObject* Convert( XrdCl::XRootDStatus *status ) { PyObject *error = PyBool_FromLong(status->IsError()); PyObject *fatal = PyBool_FromLong(status->IsFatal()); PyObject *ok = PyBool_FromLong(status->IsOK()); PyObject *obj = Py_BuildValue( "{sHsHsIsssisOsOsO}", "status", status->status, "code", status->code, "errno", status->errNo, "message", status->ToStr().c_str(), "shellcode", status->GetShellCode(), "error", error, "fatal", fatal, "ok", ok); Py_DECREF(error); Py_DECREF(fatal); Py_DECREF(ok); return obj; } }; template<> struct PyDict { static PyObject* Convert( XrdCl::ProtocolInfo *info ) { return Py_BuildValue( "{sIsI}", "version", info->GetVersion(), "hostinfo", info->GetHostInfo() ); } }; template<> struct PyDict { static PyObject* Convert( XrdCl::StatInfo *info ) { return Py_BuildValue("{sOsOsOsOsO}", "id", Py_BuildValue("s", info->GetId().c_str()), "size", Py_BuildValue("k", info->GetSize()), "flags", Py_BuildValue("I", info->GetFlags()), "modtime", Py_BuildValue("k", info->GetModTime()), "modtimestr", Py_BuildValue("s", info->GetModTimeAsString().c_str())); } }; template<> struct PyDict { static PyObject* Convert( XrdCl::StatInfoVFS *info ) { return Py_BuildValue( "{sksksksksbsb}", "nodes_rw", info->GetNodesRW(), "nodes_staging", info->GetNodesStaging(), "free_rw", info->GetFreeRW(), "free_staging", info->GetFreeStaging(), "utilization_rw", info->GetUtilizationRW(), "utilization_staging", info->GetUtilizationStaging() ); } }; template<> struct PyDict { static PyObject* Convert( XrdCl::DirectoryList *list ) { PyObject *directoryList = PyList_New( list->GetSize() ); PyObject *statInfo; int i = 0; for ( XrdCl::DirectoryList::Iterator it = list->Begin(); it < list->End(); ++it ) { statInfo = ConvertType( (*it)->GetStatInfo() ); PyList_SET_ITEM( directoryList, i, Py_BuildValue( "{sssssO}", "hostaddr", (*it)->GetHostAddress().c_str(), "name", (*it)->GetName().c_str(), "statinfo", statInfo ) ); Py_DECREF( statInfo ); i++; } PyObject *o = Py_BuildValue( "{sisssO}", "size", list->GetSize(), "parent", list->GetParentName().c_str(), "dirlist", directoryList ); Py_DECREF( directoryList ); return o; } }; template<> struct PyDict { static PyObject* Convert( XrdCl::HostList *list ) { URLType.tp_new = PyType_GenericNew; if ( PyType_Ready( &URLType ) < 0 ) return NULL; Py_INCREF( &URLType ); PyObject *pyhostlist = NULL; if ( list ) { pyhostlist = PyList_New( list->size() ); for ( unsigned int i = 0; i < list->size(); ++i ) { XrdCl::HostInfo *info = &list->at( i ); PyObject *url = PyObject_CallObject( (PyObject*) &URLType, Py_BuildValue( "(s)", info->url.GetURL().c_str() ) ); PyObject *pyhostinfo = Py_BuildValue( "{sIsIsOsO}", "flags", info->flags, "protocol", info->protocol, "load_balancer", PyBool_FromLong(info->loadBalancer), "url", url ); Py_DECREF( url ); PyList_SET_ITEM( pyhostlist, i, pyhostinfo ); } } return pyhostlist; } }; template<> struct PyDict { static PyObject* Convert( XrdCl::LocationInfo *info ) { PyObject *locationList = PyList_New( info->GetSize() ); int i = 0; for ( XrdCl::LocationInfo::Iterator it = info->Begin(); it < info->End(); ++it ) { PyList_SET_ITEM( locationList, i, Py_BuildValue( "{sssIsIsOsO}", "address", it->GetAddress().c_str(), "type", it->GetType(), "accesstype", it->GetAccessType(), "is_server", PyBool_FromLong( it->IsServer() ), "is_manager", PyBool_FromLong( it->IsManager() ) ) ); i++; } PyObject *o = Py_BuildValue( "O", locationList ); Py_DECREF( locationList ); return o; } }; template<> struct PyDict { static PyObject* Convert( XrdCl::Buffer *buffer ) { return PyBytes_FromStringAndSize( buffer->GetBuffer(), buffer->GetSize() ); } }; template<> struct PyDict { static PyObject* Convert( XrdCl::ChunkInfo *chunk ) { PyObject *o = PyBytes_FromStringAndSize( (const char*)chunk->buffer, chunk->length ); delete[] (char*) chunk->buffer; return o; } }; template<> struct PyDict { static PyObject* Convert( XrdCl::VectorReadInfo *info ) { if ( !info ) return Py_BuildValue( "" ); XrdCl::ChunkList chunks = info->GetChunks(); PyObject *pychunks = PyList_New( chunks.size() ); for ( uint32_t i = 0; i < chunks.size(); ++i ) { XrdCl::ChunkInfo chunk = chunks.at( i ); PyObject *buffer = PyBytes_FromStringAndSize( (const char *) chunk.buffer, chunk.length ); delete[] (char*) chunk.buffer; PyList_SET_ITEM( pychunks, i, Py_BuildValue( "{sOsOsO}", "offset", Py_BuildValue( "k", chunk.offset ), "length", Py_BuildValue( "I", chunk.length ), "buffer", buffer ) ); Py_DECREF( buffer ); } PyObject *o = Py_BuildValue( "{sIsO}", "size", info->GetSize(), "chunks", pychunks ); Py_DECREF( pychunks ); return o; } }; template<> struct PyDict { static PyObject* Convert(const XrdCl::PropertyList *result) { PyObject *pyresult = PyDict_New(); PyObject *kO = 0; PyObject *vO = 0; const char *key = "sourceCheckSum"; if(result->HasProperty(key)) { std::string s; result->Get(key, s); kO = Py_BuildValue("s", key); vO = Py_BuildValue("s", s.c_str()); PyDict_SetItem(pyresult, kO, vO); Py_DECREF(kO); Py_DECREF(vO); } key = "targetCheckSum"; if(result->HasProperty(key)) { std::string s; result->Get(key, s); kO = Py_BuildValue("s", key); vO = Py_BuildValue("s", s.c_str()); PyDict_SetItem(pyresult, kO, vO); Py_DECREF(kO); Py_DECREF(vO); } key = "size"; if(result->HasProperty(key)) { uint64_t s; result->Get(key, s); kO = Py_BuildValue("s", key); vO = Py_BuildValue("K", s); PyDict_SetItem(pyresult, kO, vO); Py_DECREF(kO); Py_DECREF(vO); } key = "status"; if(result->HasProperty(key)) { XrdCl::XRootDStatus s; result->Get(key, s); kO = Py_BuildValue("s", key); vO = ConvertType(&s); PyDict_SetItem(pyresult, kO, vO); Py_DECREF(kO); Py_DECREF(vO); } key = "sources"; if(result->HasProperty(key)) { std::vector s; result->Get(key, s); kO = Py_BuildValue("s", key); vO = ConvertType(&s); PyDict_SetItem(pyresult, kO, vO); Py_DECREF(kO); Py_DECREF(vO); } key = "realTarget"; if(result->HasProperty(key)) { std::string s; result->Get(key, s); kO = Py_BuildValue("s", key); vO = Py_BuildValue("s", s.c_str()); PyDict_SetItem(pyresult, kO, vO); Py_DECREF(kO); Py_DECREF(vO); } return pyresult; } }; template<> struct PyDict > { static PyObject* Convert( std::vector *list ) { PyObject *pylist = NULL; if(list) { pylist = PyList_New(list->size()); for(unsigned int i = 0; i < list->size(); ++i) { std::string &str = list->at(i); PyList_SetItem(pylist, i, Py_BuildValue("s", str.c_str())); } } return pylist; } }; template<> struct PyDict> { static PyObject* Convert( std::vector *resp ) { PyObject *pylist = NULL; if( resp ) { pylist = PyList_New( resp->size() ); for( size_t i = 0; i < resp->size(); ++i ) { XrdCl::XAttrStatus &xst = (*resp)[i]; PyObject *pystatus = ConvertType( &xst.status ); PyList_SetItem( pylist, i, Py_BuildValue( "(sO)", xst.name.c_str(), pystatus ) ); Py_DECREF( pystatus ); } } return pylist; } }; template<> struct PyDict> { static PyObject* Convert( std::vector *resp ) { PyObject *pylist = NULL; if( resp ) { pylist = PyList_New( resp->size() ); for( size_t i = 0; i < resp->size(); ++i ) { XrdCl::XAttr &xattr = (*resp)[i]; PyObject *pystatus = ConvertType( &xattr.status ); PyList_SetItem( pylist, i, Py_BuildValue( "(ssO)", xattr.name.c_str(), xattr.value.c_str(), pystatus ) ); Py_DECREF( pystatus ); } } return pylist; } }; } #endif /* CONVERSIONS_HH_ */ xrootd-5.6.9/bindings/python/src/PyXRootD.hh000066400000000000000000000035521457266313600210120ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2012-2013 by European Organization for Nuclear Research (CERN) // Author: Justin Salmon //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #ifndef PYXROOTD_HH_ #define PYXROOTD_HH_ #define PY_SSIZE_T_CLEAN #include #include #include "structmember.h" #if PY_MAJOR_VERSION >= 3 #define IS_PY3K #define Py_TPFLAGS_HAVE_ITER 0 #else #define PyUnicode_AsUTF8 PyString_AsString #define PyUnicode_Check PyString_Check #define PyUnicode_FromString PyString_FromString #define PyUnicode_FromStringAndSize PyString_FromStringAndSize #define PyUnicode_GET_LENGTH PyString_Size #endif #define async( func ) \ Py_BEGIN_ALLOW_THREADS \ func; \ Py_END_ALLOW_THREADS \ #endif /* PYXROOTD_HH_ */ xrootd-5.6.9/bindings/python/src/PyXRootDCopyProcess.cc000066400000000000000000000206571457266313600231770ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2012-2014 by European Organization for Nuclear Research (CERN) // Author: Justin Salmon // Author: Lukasz Janyst //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #include "PyXRootDCopyProcess.hh" #include "PyXRootDCopyProgressHandler.hh" #include "XrdCl/XrdClConstants.hh" #include "XrdCl/XrdClDefaultEnv.hh" #include "Conversions.hh" #include #include namespace PyXRootD { //---------------------------------------------------------------------------- // Set the number of parallel jobs //---------------------------------------------------------------------------- PyObject* CopyProcess::Parallel( CopyProcess *self, PyObject *args, PyObject *kwds ) { static const char *kwlist[] = { "parallel", NULL }; // we cannot submit a config job now because it needs to be the last one, // otherwise it will segv if ( !PyArg_ParseTupleAndKeywords( args, kwds, "I:parallel", (char**) kwlist, &self->parallel ) ) return NULL; XrdCl::XRootDStatus st; return ConvertType( &st ); } //---------------------------------------------------------------------------- // Add a job to the copy process //---------------------------------------------------------------------------- PyObject* CopyProcess::AddJob( CopyProcess *self, PyObject *args, PyObject *kwds ) { //-------------------------------------------------------------------------- // Initialize default parameters //-------------------------------------------------------------------------- XrdCl::Env *env = XrdCl::DefaultEnv::GetEnv(); static const char *kwlist[] = { "source", "target", "sourcelimit", "force", "posc", "coerce", "mkdir", "thirdparty", "checksummode", "checksumtype", "checksumpreset", "dynamicsource", "chunksize", "parallelchunks", "inittimeout", "tpctimeout", "rmBadCksum", "cptimeout", "xratethreshold", "xrate", "retry", "cont", "rtrplc", NULL }; const char *source; const char *target; uint16_t sourceLimit = 1; bool force = false; bool posc = false; bool coerce = false; bool mkdir = false; const char *thirdParty = "none"; const char *checkSumMode = "none"; const char *checkSumType = ""; const char *checkSumPreset = ""; bool dynamicSource = false; bool rmBadCksum = false; long long xRateThreshold = 0; long long xRate = 0; long long retry = 0; bool cont = false; const char *rtrplc = "force"; int val = XrdCl::DefaultCPChunkSize; env->GetInt( "CPChunkSize", val ); uint32_t chunkSize = val; val = XrdCl::DefaultCPParallelChunks; env->GetInt( "CPParallelChunks", val ); uint16_t parallelChunks = val; val = XrdCl::DefaultCPInitTimeout; env->GetInt( "CPInitTimeout", val ); uint16_t initTimeout = val; val = XrdCl::DefaultCPTPCTimeout; env->GetInt( "CPTPCTimeout", val ); uint16_t tpcTimeout = val; val = XrdCl::DefaultCPTimeout; env->GetInt( "CPTimeout", val ); uint16_t cpTimeout = val; if ( !PyArg_ParseTupleAndKeywords( args, kwds, "ss|HbbbbssssbIHHHbHLLLbs:add_job", (char**) kwlist, &source, &target, &sourceLimit, &force, &posc, &coerce, &mkdir, &thirdParty, &checkSumMode, &checkSumType, &checkSumPreset, &dynamicSource, &chunkSize, ¶llelChunks, &initTimeout, &tpcTimeout, &rmBadCksum, &cpTimeout, &xRateThreshold, &xRate, &retry, &cont, &rtrplc ) ) return NULL; XrdCl::PropertyList properties; self->results->push_back(XrdCl::PropertyList()); properties.Set( "source", source ); properties.Set( "target", target ); properties.Set( "force", force ); properties.Set( "posc", posc ); properties.Set( "coerce", coerce ); properties.Set( "makeDir", mkdir ); properties.Set( "dynamicSource", dynamicSource ); properties.Set( "thirdParty", thirdParty ); properties.Set( "checkSumMode", checkSumMode ); properties.Set( "checkSumType", checkSumType ); properties.Set( "checkSumPreset", checkSumPreset ); properties.Set( "chunkSize", chunkSize ); properties.Set( "parallelChunks", parallelChunks ); properties.Set( "initTimeout", initTimeout ); properties.Set( "tpcTimeout", tpcTimeout ); properties.Set( "rmOnBadCksum", rmBadCksum ); properties.Set( "cpTimeout", cpTimeout ); properties.Set( "xrateThreshold", xRateThreshold ); properties.Set( "xrate", xRate ); properties.Set( "continue", cont ); env->PutInt( "CpRetry", retry ); env->PutString( "CpRetryPolicy", rtrplc ); if( sourceLimit > 1 ) { int blockSize = XrdCl::DefaultXCpBlockSize; env->GetInt( "XCpBlockSize", blockSize ); properties.Set( "xcp", true ); properties.Set( "xcpBlockSize", blockSize ); properties.Set( "nbXcpSources", sourceLimit ); } XrdCl::XRootDStatus status = self->process->AddJob(properties, &self->results->back()); return ConvertType( &status ); } //---------------------------------------------------------------------------- // Prepare the copy jobs //---------------------------------------------------------------------------- PyObject* CopyProcess::Prepare( CopyProcess *self, PyObject *args, PyObject *kwds ) { // add a config job that sets the number of parallel copy jobs XrdCl::PropertyList processConfig; processConfig.Set( "jobType", "configuration" ); processConfig.Set( "parallel", self->parallel ); XrdCl::XRootDStatus status = self->process->AddJob(processConfig, 0); if( !status.IsOK() ) return ConvertType( &status ); status = self->process->Prepare(); return ConvertType( &status ); } //---------------------------------------------------------------------------- // Run the copy jobs //---------------------------------------------------------------------------- PyObject* CopyProcess::Run( CopyProcess *self, PyObject *args, PyObject *kwds ) { (void) CopyProcessType; // Suppress unused variable warning static const char *kwlist[] = { "handler", NULL }; PyObject *pyhandler = 0; XrdCl::CopyProgressHandler *handler = 0; if( !PyArg_ParseTupleAndKeywords( args, kwds, "|O", (char**) kwlist, &pyhandler ) ) return NULL; handler = new CopyProgressHandler( pyhandler ); XrdCl::XRootDStatus status; //-------------------------------------------------------------------------- //! Allow other threads to acquire the GIL while the copy jobs are running //-------------------------------------------------------------------------- Py_BEGIN_ALLOW_THREADS status = self->process->Run( handler ); Py_END_ALLOW_THREADS PyObject *tuple = PyTuple_New(2); PyTuple_SetItem(tuple, 0, ConvertType(&status)); PyTuple_SetItem(tuple, 1, ConvertType(self->results)); return tuple; } } xrootd-5.6.9/bindings/python/src/PyXRootDCopyProcess.hh000066400000000000000000000150071457266313600232020ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2012-2014 by European Organization for Nuclear Research (CERN) // Author: Justin Salmon //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #ifndef PYXROOTD_COPY_PROCESS_HH_ #define PYXROOTD_COPY_PROCESS_HH_ #include "PyXRootD.hh" #include "XrdCl/XrdClCopyProcess.hh" #include "XrdCl/XrdClXRootDResponses.hh" #include namespace PyXRootD { //---------------------------------------------------------------------------- //! XrdCl::CopyProcess binding class //---------------------------------------------------------------------------- class CopyProcess { public: static PyObject* Parallel( CopyProcess *self, PyObject *args, PyObject *kwds ); static PyObject* AddJob(CopyProcess *self, PyObject *args, PyObject *kwds); static PyObject* Prepare(CopyProcess *self, PyObject *args, PyObject *kwds); static PyObject* Run(CopyProcess *self, PyObject *args, PyObject *kwds); public: PyObject_HEAD XrdCl::CopyProcess *process; std::deque *results; int parallel; }; PyDoc_STRVAR(copyprocess_type_doc, "CopyProcess object (internal)"); //---------------------------------------------------------------------------- //! __init__() //---------------------------------------------------------------------------- static int CopyProcess_init( CopyProcess *self, PyObject *args ) { self->process = new XrdCl::CopyProcess(); self->results = new std::deque(); self->parallel = 1; return 0; } //---------------------------------------------------------------------------- //! Deallocation function, called when object is deleted //---------------------------------------------------------------------------- static void CopyProcess_dealloc( CopyProcess *self ) { delete self->process; delete self->results; Py_TYPE(self)->tp_free( (PyObject*) self ); } //---------------------------------------------------------------------------- //! Visible method definitions //---------------------------------------------------------------------------- static PyMethodDef CopyProcessMethods[] = { { "parallel", (PyCFunction) PyXRootD::CopyProcess::Parallel, METH_VARARGS | METH_KEYWORDS, NULL }, { "add_job", (PyCFunction) PyXRootD::CopyProcess::AddJob, METH_VARARGS | METH_KEYWORDS, NULL }, { "prepare", (PyCFunction) PyXRootD::CopyProcess::Prepare, METH_VARARGS | METH_KEYWORDS, NULL }, { "run", (PyCFunction) PyXRootD::CopyProcess::Run, METH_VARARGS | METH_KEYWORDS, NULL }, { NULL } /* Sentinel */ }; //---------------------------------------------------------------------------- //! Visible member definitions //---------------------------------------------------------------------------- static PyMemberDef CopyProcessMembers[] = { { NULL } /* Sentinel */ }; //---------------------------------------------------------------------------- //! CopyProcess binding type object //---------------------------------------------------------------------------- static PyTypeObject CopyProcessType = { PyVarObject_HEAD_INIT(NULL, 0) "pyxrootd.CopyProcess", /* tp_name */ sizeof(CopyProcess), /* tp_basicsize */ 0, /* tp_itemsize */ (destructor) CopyProcess_dealloc, /* tp_dealloc */ 0, /* tp_print */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_compare */ 0, /* tp_repr */ 0, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ 0, /* tp_hash */ 0, /* tp_call */ 0, /* tp_str */ 0, /* tp_getattro */ 0, /* tp_setattro */ 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ copyprocess_type_doc, /* tp_doc */ 0, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ 0, /* tp_iter */ 0, /* tp_iternext */ CopyProcessMethods, /* tp_methods */ CopyProcessMembers, /* tp_members */ 0, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ 0, /* tp_descr_set */ 0, /* tp_dictoffset */ (initproc) CopyProcess_init, /* tp_init */ }; } #endif /* PYXROOTD_COPY_PROCESS_HH_ */ xrootd-5.6.9/bindings/python/src/PyXRootDCopyProgressHandler.cc000066400000000000000000000104771457266313600246620ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2012-2014 by European Organization for Nuclear Research (CERN) // Author: Justin Salmon // Author: Lukasz Janyst //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #include "PyXRootDCopyProgressHandler.hh" #include "Conversions.hh" #include "XrdCl/XrdClXRootDResponses.hh" namespace PyXRootD { //---------------------------------------------------------------------------- // Notify when a new job is about to start //---------------------------------------------------------------------------- void CopyProgressHandler::BeginJob( uint16_t jobNum, uint16_t jobTotal, const XrdCl::URL *source, const XrdCl::URL *target ) { PyGILState_STATE state = PyGILState_Ensure(); PyObject *ret = 0; if (handler != NULL) { ret = PyObject_CallMethod( handler, (char*)"begin", (char*)"(HHss)", jobNum, jobTotal, source->GetURL().c_str(), target->GetURL().c_str() ); Py_XDECREF(ret); } PyGILState_Release(state); } //---------------------------------------------------------------------------- // Notify when the previous job has finished //---------------------------------------------------------------------------- void CopyProgressHandler::EndJob( uint16_t jobNum, const XrdCl::PropertyList *result ) { PyGILState_STATE state = PyGILState_Ensure(); PyObject *pyresult = ConvertType(result); PyObject *ret = 0; if (handler) { ret = PyObject_CallMethod( handler, (char*)"end", (char*)"HO", jobNum, pyresult ); Py_XDECREF(ret); } PyGILState_Release(state); } //---------------------------------------------------------------------------- // Notify about the progress of the current job //---------------------------------------------------------------------------- void CopyProgressHandler::JobProgress( uint16_t jobNum, uint64_t bytesProcessed, uint64_t bytesTotal ) { PyGILState_STATE state = PyGILState_Ensure(); PyObject *ret = 0; if(handler) { ret = PyObject_CallMethod( handler, (char*)"update", (char*)"HKK", jobNum, bytesProcessed, bytesTotal ); Py_XDECREF(ret); } PyGILState_Release(state); } //---------------------------------------------------------------------------- // Check if the job should be canceled //---------------------------------------------------------------------------- bool CopyProgressHandler::ShouldCancel( uint16_t jobNum ) { PyGILState_STATE state = PyGILState_Ensure(); bool ret = false; if(handler) { PyObject *val = PyObject_CallMethod( handler, (char*)"should_cancel", (char*)"H", jobNum ); if(val) { if(PyBool_Check(val)) ret = (val == Py_True); Py_DECREF(val); } } PyGILState_Release(state); return ret; } } xrootd-5.6.9/bindings/python/src/PyXRootDCopyProgressHandler.hh000066400000000000000000000067521457266313600246750ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2012-2014 by European Organization for Nuclear Research (CERN) // Author: Justin Salmon //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #ifndef PYXROOTD_COPY_PROGRESS_HANDLER_HH_ #define PYXROOTD_COPY_PROGRESS_HANDLER_HH_ #include "PyXRootD.hh" #include "XrdCl/XrdClCopyProcess.hh" #include "XrdCl/XrdClPropertyList.hh" #include "XrdCl/XrdClURL.hh" namespace PyXRootD { //---------------------------------------------------------------------------- //! Interface for copy progress notification //---------------------------------------------------------------------------- class CopyProgressHandler : public XrdCl::CopyProgressHandler { public: //------------------------------------------------------------------------ //! Constructor //------------------------------------------------------------------------ CopyProgressHandler( PyObject *handler ) : handler( handler ) {} //------------------------------------------------------------------------ //! Notify when a new job is about to start //------------------------------------------------------------------------ virtual void BeginJob( uint16_t jobNum, uint16_t jobTotal, const XrdCl::URL *source, const XrdCl::URL *target ); //------------------------------------------------------------------------ //! Notify when the previous job has finished //------------------------------------------------------------------------ virtual void EndJob( uint16_t jobNum, const XrdCl::PropertyList *result ); //------------------------------------------------------------------------ //! Notify about the progress of the current job //------------------------------------------------------------------------ virtual void JobProgress( uint16_t jobNum, uint64_t bytesProcessed, uint64_t bytesTotal ); //------------------------------------------------------------------------ //! Determine whether the job should be canceled //------------------------------------------------------------------------ virtual bool ShouldCancel(uint16_t jobNum); public: PyObject *handler; }; } #endif /* PYXROOTD_COPY_PROGRESS_HANDLER_HH_ */ xrootd-5.6.9/bindings/python/src/PyXRootDEnv.hh000066400000000000000000000117771457266313600214730ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2012-2013 by European Organization for Nuclear Research (CERN) // Author: Michal Simon //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #ifndef PYXROOTDENV_HH_ #define PYXROOTDENV_HH_ #include "PyXRootD.hh" #include "XrdCl/XrdClDefaultEnv.hh" #include "XrdVersion.hh" namespace PyXRootD { //---------------------------------------------------------------------------- //! Sets the given key in the xrootd client environment to the given value. //! @return : false if there is already a shell-imported setting for this key, // true otherwise //---------------------------------------------------------------------------- PyObject* EnvPutString_cpp( PyObject *self, PyObject *args ) { char *key = 0, *value = 0; // parse arguments if( !PyArg_ParseTuple( args, "ss", &key, &value ) ) { return NULL; } return PyBool_FromLong( XrdCl::DefaultEnv::GetEnv()->PutString( key, value ) ); } //---------------------------------------------------------------------------- //! Gets the given key from the xrootd client environment. If key does not //! exist in the environment returns None. //---------------------------------------------------------------------------- PyObject* EnvGetString_cpp( PyObject *self, PyObject *args ) { char *key = 0; // parse arguments if( !PyArg_ParseTuple( args, "s", &key) ) { return NULL; } std::string value; if( !XrdCl::DefaultEnv::GetEnv()->GetString( key, value ) ) Py_RETURN_NONE; return Py_BuildValue( "s", value.c_str() ); } //---------------------------------------------------------------------------- //! Sets the given key in the xrootd client environment to the given value. //! @return : false if there is already a shell-imported setting for this key, // true otherwise //---------------------------------------------------------------------------- PyObject* EnvPutInt_cpp( PyObject *self, PyObject *args ) { char *key = 0; int value = 0; // parse arguments if( !PyArg_ParseTuple( args, "si", &key, &value ) ) { return NULL; } return PyBool_FromLong( XrdCl::DefaultEnv::GetEnv()->PutInt( key, value ) ); } //---------------------------------------------------------------------------- //! Gets the given key from the xrootd client environment. If key does not //! exist in the environment returns None. //---------------------------------------------------------------------------- PyObject* EnvGetInt_cpp( PyObject *self, PyObject *args ) { char *key = 0; // parse arguments if( !PyArg_ParseTuple( args, "s", &key) ) { return NULL; } int value = 0; if( !XrdCl::DefaultEnv::GetEnv()->GetInt( key, value ) ) Py_RETURN_NONE; return Py_BuildValue( "i", value ); } //---------------------------------------------------------------------------- //! Gets default parameter value for given key //---------------------------------------------------------------------------- PyObject* EnvGetDefault_cpp( PyObject *self, PyObject *args ) { char *key = 0; // parse arguments if( !PyArg_ParseTuple( args, "s", &key) ) { return NULL; } std::string value; if( XrdCl::DefaultEnv::GetEnv()->GetDefaultStringValue( key, value ) ) return Py_BuildValue( "s", value.c_str() ); int val; if( XrdCl::DefaultEnv::GetEnv()->GetDefaultIntValue( key, val ) ) return Py_BuildValue( "s", std::to_string( val ).c_str() ); Py_RETURN_NONE; } //---------------------------------------------------------------------------- //! @return : client version, e.g.: v4.10.0 //---------------------------------------------------------------------------- PyObject* XrdVersion_cpp( PyObject *self, PyObject *args ) { static std::string verstr( XrdVERSION[0] == 'v' ? &XrdVERSION[1] : XrdVERSION ); return Py_BuildValue( "s", verstr.c_str() ); } } #endif /* PYENV_HH_ */ xrootd-5.6.9/bindings/python/src/PyXRootDFile.cc000066400000000000000000001056371457266313600216070ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2012-2014 by European Organization for Nuclear Research (CERN) // Author: Justin Salmon //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #include "PyXRootD.hh" #include "PyXRootDFile.hh" #include "AsyncResponseHandler.hh" #include "ChunkIterator.hh" #include "Utils.hh" #include "XrdCl/XrdClFile.hh" #include "XrdCl/XrdClFileSystem.hh" namespace PyXRootD { //---------------------------------------------------------------------------- //! Open the file pointed to by the given URL //---------------------------------------------------------------------------- PyObject* File::Open( File *self, PyObject *args, PyObject *kwds ) { static const char *kwlist[] = { "url", "flags", "mode", "timeout", "callback", NULL }; const char *url; XrdCl::OpenFlags::Flags flags = XrdCl::OpenFlags::None; XrdCl::Access::Mode mode = XrdCl::Access::None; uint16_t timeout = 0; PyObject *callback = NULL, *pystatus = NULL; XrdCl::XRootDStatus status; if ( !PyArg_ParseTupleAndKeywords( args, kwds, "s|HHHO:open", (char**) kwlist, &url, &flags, &mode, &timeout, &callback ) ) return NULL; if ( callback && callback != Py_None ) { XrdCl::ResponseHandler *handler = GetHandler( callback ); if ( !handler ) return NULL; async( status = self->file->Open( url, flags, mode, handler, timeout ) ); } else { async( status = self->file->Open( url, flags, mode, timeout ) ); } pystatus = ConvertType( &status ); PyObject *o = ( callback && callback != Py_None ) ? Py_BuildValue( "O", pystatus ) : Py_BuildValue( "OO", pystatus, Py_BuildValue( "" ) ); Py_DECREF( pystatus ); return o; } //---------------------------------------------------------------------------- //! Close the file //---------------------------------------------------------------------------- PyObject* File::Close( File *self, PyObject *args, PyObject *kwds ) { static const char *kwlist[] = { "timeout", "callback", NULL }; uint16_t timeout = 0; PyObject *callback = NULL, *pystatus = NULL; XrdCl::XRootDStatus status; if ( !PyArg_ParseTupleAndKeywords( args, kwds, "|HO:close", (char**) kwlist, &timeout, &callback ) ) return NULL; if ( callback && callback != Py_None ) { XrdCl::ResponseHandler *handler = GetHandler( callback ); if ( !handler ) return NULL; async( status = self->file->Close( handler, timeout ) ); } else { async( status = self->file->Close( timeout ) ) } pystatus = ConvertType( &status ); PyObject *o = ( callback && callback != Py_None ) ? Py_BuildValue( "O", pystatus ) : Py_BuildValue( "OO", pystatus, Py_BuildValue( "" ) ); Py_DECREF( pystatus ); return o; } //---------------------------------------------------------------------------- //! Obtain status information for this file //---------------------------------------------------------------------------- PyObject* File::Stat( File *self, PyObject *args, PyObject *kwds ) { static const char *kwlist[] = { "force", "timeout", "callback", NULL }; int force = 0; uint16_t timeout = 0; PyObject *callback = NULL, *pyresponse = NULL, *pystatus = NULL; XrdCl::XRootDStatus status; if ( !self->file->IsOpen() ) return FileClosedError(); if ( !PyArg_ParseTupleAndKeywords( args, kwds, "|iHO:stat", (char**) kwlist, &force, &timeout, &callback ) ) return NULL; if ( callback && callback != Py_None ) { XrdCl::ResponseHandler *handler = GetHandler( callback ); async( status = self->file->Stat( force, handler, timeout ) ); } else { XrdCl::StatInfo *response = 0; async( status = self->file->Stat( force, response, timeout ) ); pyresponse = ConvertType( response ); delete response; } pystatus = ConvertType( &status ); PyObject *o = ( callback && callback != Py_None ) ? Py_BuildValue( "O", pystatus ) : Py_BuildValue( "OO", pystatus, pyresponse ); Py_DECREF( pystatus ); Py_XDECREF( pyresponse ); return o; } //---------------------------------------------------------------------------- //! Read a data chunk at a given offset //---------------------------------------------------------------------------- PyObject* File::Read( File *self, PyObject *args, PyObject *kwds ) { static const char *kwlist[] = { "offset", "size", "timeout", "callback", NULL }; uint64_t offset = 0; uint32_t size = 0; uint16_t timeout = 0; PyObject *callback = NULL, *pystatus = NULL, *pyresponse = NULL; PyObject *py_offset = NULL, *py_size = NULL, *py_timeout = NULL; char *buffer = 0; XrdCl::XRootDStatus status; if ( !self->file->IsOpen() ) return FileClosedError(); if ( !PyArg_ParseTupleAndKeywords( args, kwds, "|OOOO:read", (char**) kwlist, &py_offset, &py_size, &py_timeout, &callback ) ) return NULL; unsigned long long tmp_offset = 0; unsigned int tmp_size = 0; unsigned short int tmp_timeout = 0; if ( py_offset && PyObjToUllong( py_offset, &tmp_offset, "offset" ) ) return NULL; if ( py_size && PyObjToUint(py_size, &tmp_size, "size" ) ) return NULL; if ( py_timeout && PyObjToUshrt(py_timeout, &tmp_timeout, "timeout" ) ) return NULL; offset = (uint64_t)tmp_offset; size = (uint32_t)tmp_size; timeout = (uint16_t)tmp_timeout; if (!size) { XrdCl::StatInfo *info = 0; async( XrdCl::XRootDStatus status = self->file->Stat(true, info, timeout) ); size = info->GetSize(); if (info) delete info; } buffer = new char[size]; if ( callback && callback != Py_None ) { XrdCl::ResponseHandler *handler = GetHandler( callback ); if ( !handler ) { delete[] buffer; return NULL; } async( status = self->file->Read( offset, size, buffer, handler, timeout ) ); } else { uint32_t bytesRead = 0; async( status = self->file->Read( offset, size, buffer, bytesRead, timeout ) ); pyresponse = PyBytes_FromStringAndSize( buffer, bytesRead ); delete[] buffer; } pystatus = ConvertType( &status ); PyObject *o = ( callback && callback != Py_None ) ? Py_BuildValue( "O", pystatus ) : Py_BuildValue( "OO", pystatus, pyresponse ); Py_DECREF( pystatus ); Py_XDECREF( pyresponse ); return o; } //---------------------------------------------------------------------------- // Read a data chunk at a given offset, until the first newline encountered // or size data read. //---------------------------------------------------------------------------- PyObject* File::ReadLine( File *self, PyObject *args, PyObject *kwds ) { static const char *kwlist[] = { "offset", "size", "chunksize", NULL }; uint64_t offset = 0; uint32_t size = 0; uint32_t chunksize = 0; PyObject *pyline = NULL; PyObject *py_offset = NULL, *py_size = NULL, *py_chunksize = NULL; if ( !self->file->IsOpen() ) return FileClosedError(); if ( !PyArg_ParseTupleAndKeywords( args, kwds, "|OOO:readline", (char**) kwlist, &py_offset, &py_size, &py_chunksize ) ) return NULL; unsigned long long tmp_offset = 0; unsigned int tmp_size = 0, tmp_chunksize = 0; if ( py_offset && PyObjToUllong(py_offset, &tmp_offset, "offset" ) ) return NULL; if ( py_size && PyObjToUint(py_size, &tmp_size, "size" ) ) return NULL; if ( py_chunksize && PyObjToUint(py_chunksize, &tmp_chunksize, "chunksize" ) ) return NULL; offset = (uint64_t)tmp_offset; size = (uint32_t)tmp_size; chunksize = (uint32_t)tmp_chunksize; uint64_t off_init = offset; if (offset == 0) offset = self->currentOffset; else self->currentOffset = offset; // Default chunk size is 2MB or equal to size if size less then 2MB if ( !chunksize ) chunksize = 1024 * 1024 * 2; if ( !size ) size = UINT_MAX; if ( size < chunksize ) chunksize = size; uint64_t off_end = offset + size; XrdCl::Buffer* chunk = new XrdCl::Buffer(); XrdCl::Buffer* line = new XrdCl::Buffer(); while ( offset < off_end ) { chunk = self->ReadChunk( self, offset, chunksize ); offset += chunk->GetSize(); // Reached end of file if ( !chunk->GetSize() ) break; // Check if we read a new line bool found_newline = false; for( uint32_t i = 0; i < chunk->GetSize(); ++i ) { chunk->SetCursor( i ); // Stop if found newline or read required amount of data if( ( *chunk->GetBufferAtCursor() == '\n') || ( line->GetSize() + i >= size)) { found_newline = true; line->Append( chunk->GetBuffer(), i + 1 ); break; } } if ( !found_newline ) line->Append( chunk->GetBuffer(), chunk->GetSize() ); else break; } if ( line->GetSize() != 0 ) { // Update file offset if default readline call if ( off_init == 0 ) self->currentOffset += line->GetSize(); pyline = PyUnicode_FromStringAndSize( line->GetBuffer(), line->GetSize() ); } else pyline = PyUnicode_FromString( "" ); delete line; delete chunk; return pyline; } //---------------------------------------------------------------------------- //! Read data chunks from a given offset, separated by newlines, until EOF //! encountered. Return list of lines read. A max read size can be specified, //! but it should be noted that using this method is probably a bad idea. //---------------------------------------------------------------------------- PyObject* File::ReadLines( File *self, PyObject *args, PyObject *kwds ) { static const char *kwlist[] = { "offset", "size", "chunksize", NULL }; uint64_t offset = 0; uint32_t size = 0; uint32_t chunksize = 0; PyObject *py_offset = NULL, *py_size = NULL, *py_chunksize = NULL; if ( !self->file->IsOpen() ) return FileClosedError(); if ( !PyArg_ParseTupleAndKeywords( args, kwds, "|kII:readlines", (char**) kwlist, &offset, &size, &chunksize ) ) return NULL; unsigned long long tmp_offset = 0; unsigned int tmp_size = 0, tmp_chunksize = 0; if ( py_offset && PyObjToUllong( py_offset, &tmp_offset, "offset" ) ) return NULL; if ( py_size && PyObjToUint( py_size, &tmp_size, "size" ) ) return NULL; if ( py_chunksize && PyObjToUint( py_chunksize, &tmp_chunksize, "chunksize" ) ) return NULL; offset = (uint64_t)tmp_offset; size = (uint32_t)tmp_size; chunksize = (uint16_t)tmp_chunksize; PyObject *lines = PyList_New( 0 ); PyObject *line = NULL; for (;;) { line = self->ReadLine( self, args, kwds ); if ( !line || PyUnicode_GET_LENGTH( line ) == 0 ) break; PyList_Append( lines, line ); } return lines; } //---------------------------------------------------------------------------- //! Read a chunk of the given size from the given offset as a string //---------------------------------------------------------------------------- XrdCl::Buffer* File::ReadChunk( File *self, uint64_t offset, uint32_t size ) { XrdCl::XRootDStatus status; XrdCl::Buffer *buffer; XrdCl::Buffer *temp; uint32_t bytesRead = 0; temp = new XrdCl::Buffer( size ); status = self->file->Read( offset, size, temp->GetBuffer(), bytesRead ); buffer = new XrdCl::Buffer( bytesRead ); buffer->Append( temp->GetBuffer(), bytesRead ); delete temp; return buffer; } //---------------------------------------------------------------------------- //! Read data chunks from a given offset of the given size, until EOF //! encountered. Return chunk iterator. //---------------------------------------------------------------------------- PyObject* File::ReadChunks( File *self, PyObject *args, PyObject *kwds ) { static const char *kwlist[] = { "offset", "chunksize", NULL }; uint64_t offset = 0; uint32_t chunksize = 0; ChunkIterator *iterator; PyObject *py_offset = NULL, *py_chunksize = NULL; if ( !self->file->IsOpen() ) return FileClosedError(); if ( !PyArg_ParseTupleAndKeywords( args, kwds, "|OO:readchunks", (char**) kwlist, &py_offset, &py_chunksize ) ) return NULL; unsigned long long tmp_offset = 0; unsigned int tmp_chunksize = 1024 * 1024 *2; // 2 MB if ( py_offset && PyObjToUllong( py_offset, &tmp_offset, "offset" ) ) return NULL; if ( py_chunksize && PyObjToUint( py_chunksize, &tmp_chunksize, "chunksize" ) ) return NULL; offset = (uint64_t)tmp_offset; chunksize = (uint32_t)tmp_chunksize; ChunkIteratorType.tp_new = PyType_GenericNew; if ( PyType_Ready( &ChunkIteratorType ) < 0 ) return NULL; args = Py_BuildValue( "OOO", self, Py_BuildValue("k", offset), Py_BuildValue("I", chunksize) ); iterator = (ChunkIterator*) PyObject_CallObject( (PyObject *) &ChunkIteratorType, args ); Py_DECREF( args ); if ( !iterator ) return NULL; return (PyObject *) iterator; } //---------------------------------------------------------------------------- //! Write a data chunk at a given offset //---------------------------------------------------------------------------- PyObject* File::Write( File *self, PyObject *args, PyObject *kwds ) { static const char *kwlist[] = { "buffer", "offset", "size", "timeout", "callback", NULL }; const char *buffer; Py_ssize_t buffsize; uint64_t offset = 0; uint32_t size = 0; uint16_t timeout = 0; PyObject *callback = NULL, *pystatus = NULL; PyObject *py_offset = NULL, *py_size = NULL, *py_timeout = NULL; XrdCl::XRootDStatus status; if ( !self->file->IsOpen() ) return FileClosedError(); if ( !PyArg_ParseTupleAndKeywords( args, kwds, "s#|OOOO:write", (char**) kwlist, &buffer, &buffsize, &py_offset, &py_size, &py_timeout, &callback ) ) return NULL; unsigned long long tmp_offset = 0; unsigned int tmp_size = 0; unsigned short int tmp_timeout = 0; if (py_offset && PyObjToUllong(py_offset, &tmp_offset, "offset")) return NULL; if (py_size && PyObjToUint(py_size, &tmp_size, "size")) return NULL; if (py_timeout && PyObjToUshrt(py_timeout, &tmp_timeout, "timeout")) return NULL; offset = (uint64_t)tmp_offset; size = (uint32_t)tmp_size; timeout = (uint16_t)tmp_timeout; if (!size) { size = buffsize; } if ( callback && callback != Py_None ) { XrdCl::ResponseHandler *handler = GetHandler( callback ); if ( !handler ) return NULL; async( status = self->file->Write( offset, size, buffer, handler, timeout ) ); } else { async( status = self->file->Write( offset, size, buffer, timeout ) ); } pystatus = ConvertType( &status ); PyObject *o = ( callback && callback != Py_None ) ? Py_BuildValue( "O", pystatus ) : Py_BuildValue( "OO", pystatus, Py_BuildValue( "" ) ); Py_DECREF( pystatus ); return o; } //---------------------------------------------------------------------------- //! Commit all pending disk writes //---------------------------------------------------------------------------- PyObject* File::Sync( File *self, PyObject *args, PyObject *kwds ) { static const char *kwlist[] = { "timeout", "callback", NULL }; uint16_t timeout = 0; PyObject *callback = NULL, *pystatus = NULL; XrdCl::XRootDStatus status; if ( !self->file->IsOpen() ) return FileClosedError(); if ( !PyArg_ParseTupleAndKeywords( args, kwds, "|HO:sync", (char**) kwlist, &timeout, &callback ) ) return NULL; if ( callback && callback != Py_None ) { XrdCl::ResponseHandler *handler = GetHandler( callback ); if ( !handler ) return NULL; async( status = self->file->Sync( handler, timeout ) ); } else { async( status = self->file->Sync( timeout ) ); } pystatus = ConvertType( &status ); PyObject *o = ( callback && callback != Py_None ) ? Py_BuildValue( "O", pystatus ) : Py_BuildValue( "OO", pystatus, Py_BuildValue( "" ) ); Py_DECREF( pystatus ); return o; } //---------------------------------------------------------------------------- //! Truncate the file to a particular size //---------------------------------------------------------------------------- PyObject* File::Truncate( File *self, PyObject *args, PyObject *kwds ) { static const char *kwlist[] = { "size", "timeout", "callback", NULL }; uint64_t size; uint16_t timeout = 0; PyObject *callback = NULL, *pystatus = NULL; PyObject *py_size = NULL, *py_timeout = NULL; XrdCl::XRootDStatus status; if ( !self->file->IsOpen() ) return FileClosedError(); if ( !PyArg_ParseTupleAndKeywords( args, kwds, "O|OO:truncate", (char**) kwlist, &py_size, &py_timeout, &callback ) ) return NULL; unsigned long long tmp_size = 0; unsigned short int tmp_timeout = 0; if ( py_size && PyObjToUllong( py_size, &tmp_size, "size" ) ) return NULL; if ( py_timeout && PyObjToUshrt( py_timeout, &tmp_timeout, "timeout" ) ) return NULL; size = (uint64_t)tmp_size; timeout = (uint16_t)tmp_timeout; if ( callback && callback != Py_None ) { XrdCl::ResponseHandler *handler = GetHandler( callback ); if ( !handler ) return NULL; async( status = self->file->Truncate( size, handler, timeout ) ); } else { async( status = self->file->Truncate( size, timeout ) ); } pystatus = ConvertType( &status ); PyObject *o = ( callback && callback != Py_None ) ? Py_BuildValue( "O", pystatus ) : Py_BuildValue( "OO", pystatus, Py_BuildValue( "" ) ); Py_DECREF( pystatus ); return o; } //---------------------------------------------------------------------------- //! Read scattered data chunks in one operation //---------------------------------------------------------------------------- PyObject* File::VectorRead( File *self, PyObject *args, PyObject *kwds ) { static const char *kwlist[] = { "chunks", "timeout", "callback", NULL }; uint16_t timeout = 0; uint64_t offset = 0; uint32_t length = 0; PyObject *pychunks = NULL, *callback = NULL; PyObject *pyresponse = NULL, *pystatus = NULL, *py_timeout = NULL; XrdCl::XRootDStatus status; XrdCl::ChunkList chunks; if ( !self->file->IsOpen() ) return FileClosedError(); if ( !PyArg_ParseTupleAndKeywords( args, kwds, "O|OO:vector_read", (char**) kwlist, &pychunks, &py_timeout, &callback ) ) return NULL; unsigned short int tmp_timeout = 0; if ( py_timeout && PyObjToUshrt( py_timeout, &tmp_timeout, "timeout" ) ) return NULL; timeout = (uint16_t)tmp_timeout; if ( !PyList_Check( pychunks ) ) { PyErr_SetString( PyExc_TypeError, "chunks parameter must be a list" ); return NULL; } for ( int i = 0; i < PyList_Size( pychunks ); ++i ) { PyObject *chunk = PyList_GetItem( pychunks, i ); if ( !PyTuple_Check( chunk ) || ( PyTuple_Size( chunk ) != 2 ) ) { PyErr_SetString( PyExc_TypeError, "vector_read() expects list of tuples" " of length 2" ); return NULL; } // Check that offset and length values are valid unsigned long long tmp_offset = 0; unsigned int tmp_length = 0; if ( PyObjToUllong( PyTuple_GetItem( chunk, 0 ), &tmp_offset, "offset" ) ) return NULL; if ( PyObjToUint( PyTuple_GetItem( chunk, 1 ), &tmp_length, "length" ) ) return NULL; offset = (uint64_t)tmp_offset; length = (uint32_t)tmp_length; char *buffer = new char[length]; chunks.push_back( XrdCl::ChunkInfo( offset, length, buffer ) ); } if ( callback && callback != Py_None ) { XrdCl::ResponseHandler *handler = GetHandler( callback ); if ( !handler ) return NULL; async( status = self->file->VectorRead( chunks, 0, handler, timeout ) ); } else { XrdCl::VectorReadInfo *info = 0; async( status = self->file->VectorRead( chunks, 0, info, timeout ) ); pyresponse = ConvertType( info ); delete info; } pystatus = ConvertType( &status ); PyObject *o = ( callback && callback != Py_None ) ? Py_BuildValue( "O", pystatus ) : Py_BuildValue( "OO", pystatus, pyresponse ); Py_DECREF( pystatus ); Py_XDECREF( pyresponse ); return o; } //---------------------------------------------------------------------------- // Perform a custom operation on an open file //---------------------------------------------------------------------------- PyObject* File::Fcntl( File *self, PyObject *args, PyObject *kwds ) { static const char *kwlist[] = { "arg", "timeout", "callback", NULL }; const char *buffer = 0; Py_ssize_t buffSize = 0; uint16_t timeout = 0; PyObject *callback = NULL, *pystatus = NULL, *pyresponse = NULL; XrdCl::XRootDStatus status; if ( !self->file->IsOpen() ) return FileClosedError(); if ( !PyArg_ParseTupleAndKeywords( args, kwds, "s#|HO:fcntl", (char**) kwlist, &buffer, &buffSize, &timeout, &callback ) ) return NULL; XrdCl::Buffer arg; arg.Append( buffer, buffSize ); if ( callback && callback != Py_None ) { XrdCl::ResponseHandler *handler = GetHandler( callback ); if( !handler ) return NULL; async( status = self->file->Fcntl( arg, handler, timeout ) ); } else { XrdCl::Buffer *response = 0; async( status = self->file->Fcntl( arg, response, timeout ) ); pyresponse = ConvertType( response ); delete response; } pystatus = ConvertType( &status ); PyObject *o = ( callback && callback != Py_None ) ? Py_BuildValue( "O", pystatus ) : Py_BuildValue( "OO", pystatus, pyresponse ); Py_DECREF( pystatus ); Py_XDECREF( pyresponse ); return o; } //---------------------------------------------------------------------------- // Perform a custom operation on an open file //---------------------------------------------------------------------------- PyObject* File::Visa( File *self, PyObject *args, PyObject *kwds ) { static const char *kwlist[] = { "timeout", "callback", NULL }; uint16_t timeout = 0; PyObject *callback = NULL, *pystatus = NULL, *pyresponse = NULL; XrdCl::XRootDStatus status; if ( !self->file->IsOpen() ) return FileClosedError(); if ( !PyArg_ParseTupleAndKeywords( args, kwds, "|HO:visa", (char**) kwlist, &timeout, &callback ) ) return NULL; if ( callback && callback != Py_None ) { XrdCl::ResponseHandler *handler = GetHandler( callback ); if( !handler ) return NULL; async( status = self->file->Visa( handler, timeout ) ); } else { XrdCl::Buffer *response = 0; async( status = self->file->Visa( response, timeout ) ); pyresponse = ConvertType( response ); delete response; } pystatus = ConvertType( &status ); PyObject *o = ( callback && callback != Py_None ) ? Py_BuildValue( "O", pystatus ) : Py_BuildValue( "OO", pystatus, pyresponse ); Py_DECREF( pystatus ); Py_XDECREF( pyresponse ); return o; } //---------------------------------------------------------------------------- //! Check if the file is open //---------------------------------------------------------------------------- PyObject* File::IsOpen( File *self, PyObject *args, PyObject *kwds ) { if ( !PyArg_ParseTuple( args, ":is_open" ) ) return NULL; // Allow no args return PyBool_FromLong(self->file->IsOpen()); } //---------------------------------------------------------------------------- //! Get property //---------------------------------------------------------------------------- PyObject* File::GetProperty( File *self, PyObject *args, PyObject *kwds ) { static const char *kwlist[] = { "name", NULL }; char *name = 0; std::string value; if ( !PyArg_ParseTupleAndKeywords( args, kwds, "s:get_property", (char**) kwlist, &name ) ) return NULL; bool status = self->file->GetProperty( name, value ); return status ? Py_BuildValue( "s", value.c_str() ) : Py_None; } //---------------------------------------------------------------------------- //! Set property //---------------------------------------------------------------------------- PyObject* File::SetProperty( File *self, PyObject *args, PyObject *kwds ) { (void) FileType; // Suppress unused variable warning static const char *kwlist[] = { "name", "value", NULL }; char *name = 0; char *value = 0; if ( !PyArg_ParseTupleAndKeywords( args, kwds, "ss:set_property", (char**) kwlist, &name, &value ) ) return NULL; bool status = self->file->SetProperty( name, value ); return status ? Py_True : Py_False; } //---------------------------------------------------------------------------- //! Set Extended File Attributes //---------------------------------------------------------------------------- PyObject* File::SetXAttr( File *self, PyObject *args, PyObject *kwds ) { static const char *kwlist[] = { "attrs", "timeout", "callback", NULL }; std::vector attrs; uint16_t timeout = 0; PyObject *callback = NULL, *pystatus = NULL; PyObject *pyattrs = NULL, *pyresponse = NULL; XrdCl::XRootDStatus status; if ( !self->file->IsOpen() ) return FileClosedError(); if ( !PyArg_ParseTupleAndKeywords( args, kwds, "O|HO:set_xattr", (char**) kwlist, &pyattrs, &timeout, &callback ) ) return NULL; // it should be a list if( !PyList_Check( pyattrs ) ) return NULL; // now parse the input Py_ssize_t size = PyList_Size( pyattrs ); attrs.reserve( size ); for( ssize_t i = 0; i < size; ++i ) { // get the item at respective index PyObject *item = PyList_GetItem( pyattrs, i ); // make sure the item is a tuple if( !item || !PyTuple_Check( item ) ) return NULL; // make sure the tuple size equals to 2 if( PyTuple_Size( item ) != 2 ) return NULL; // extract the attribute name from the tuple PyObject *py_name = PyTuple_GetItem( item, 0 ); if( !PyUnicode_Check( py_name ) ) return NULL; std::string name = PyUnicode_AsUTF8( py_name ); // extract the attribute value from the tuple PyObject *py_value = PyTuple_GetItem( item, 1 ); if( !PyUnicode_Check( py_value ) ) return NULL; std::string value = PyUnicode_AsUTF8( py_value ); // update the C++ list of xattrs attrs.push_back( XrdCl::xattr_t( name, value ) ); } if ( callback && callback != Py_None ) { XrdCl::ResponseHandler *handler = GetHandler>( callback ); if ( !handler ) return NULL; async( status = self->file->SetXAttr( attrs, handler, timeout ) ); } else { std::vector result; async( status = self->file->SetXAttr( attrs, result, timeout ) ); pyresponse = ConvertType( &result ); } pystatus = ConvertType( &status ); PyObject *o = ( callback && callback != Py_None ) ? Py_BuildValue( "O", pystatus ) : Py_BuildValue( "OO", pystatus, pyresponse ); Py_DECREF( pystatus ); Py_XDECREF( pyresponse ); return o; } //---------------------------------------------------------------------------- //! Get Extended File Attributes //---------------------------------------------------------------------------- PyObject* File::GetXAttr( File *self, PyObject *args, PyObject *kwds ) { static const char *kwlist[] = { "attrs", "timeout", "callback", NULL }; std::vector attrs; uint16_t timeout = 0; PyObject *callback = NULL, *pystatus = NULL; PyObject *pyattrs = NULL, *pyresponse = NULL; XrdCl::XRootDStatus status; if ( !self->file->IsOpen() ) return FileClosedError(); if ( !PyArg_ParseTupleAndKeywords( args, kwds, "O|HO:set_xattr", (char**) kwlist, &pyattrs, &timeout, &callback ) ) return NULL; // it should be a list if( !PyList_Check( pyattrs ) ) return NULL; // now parse the input Py_ssize_t size = PyList_Size( pyattrs ); attrs.reserve( size ); for( ssize_t i = 0; i < size; ++i ) { // get the item at respective index PyObject *item = PyList_GetItem( pyattrs, i ); // make sure the item is a string if( !item || !PyUnicode_Check( item ) ) return NULL; std::string name = PyUnicode_AsUTF8( item ); // update the C++ list of xattrs attrs.push_back( name ); } if ( callback && callback != Py_None ) { XrdCl::ResponseHandler *handler = GetHandler>( callback ); if ( !handler ) return NULL; async( status = self->file->GetXAttr( attrs, handler, timeout ) ); } else { std::vector result; async( status = self->file->GetXAttr( attrs, result, timeout ) ); pyresponse = ConvertType( &result ); } pystatus = ConvertType( &status ); PyObject *o = ( callback && callback != Py_None ) ? Py_BuildValue( "O", pystatus ) : Py_BuildValue( "OO", pystatus, pyresponse ); Py_DECREF( pystatus ); Py_XDECREF( pyresponse ); return o; } //---------------------------------------------------------------------------- //! Delete Extended File Attributes //---------------------------------------------------------------------------- PyObject* File::DelXAttr( File *self, PyObject *args, PyObject *kwds ) { static const char *kwlist[] = { "attrs", "timeout", "callback", NULL }; std::vector attrs; uint16_t timeout = 0; PyObject *callback = NULL, *pystatus = NULL; PyObject *pyattrs = NULL, *pyresponse = NULL; XrdCl::XRootDStatus status; if ( !self->file->IsOpen() ) return FileClosedError(); if ( !PyArg_ParseTupleAndKeywords( args, kwds, "O|HO:set_xattr", (char**) kwlist, &pyattrs, &timeout, &callback ) ) return NULL; // it should be a list if( !PyList_Check( pyattrs ) ) return NULL; // now parse the input Py_ssize_t size = PyList_Size( pyattrs ); attrs.reserve( size ); for( ssize_t i = 0; i < size; ++i ) { // get the item at respective index PyObject *item = PyList_GetItem( pyattrs, i ); // make sure the item is a string if( !item || !PyUnicode_Check( item ) ) return NULL; std::string name = PyUnicode_AsUTF8( item ); // update the C++ list of xattrs attrs.push_back( name ); } if ( callback && callback != Py_None ) { XrdCl::ResponseHandler *handler = GetHandler>( callback ); if ( !handler ) return NULL; async( status = self->file->DelXAttr( attrs, handler, timeout ) ); } else { std::vector result; async( status = self->file->DelXAttr( attrs, result, timeout ) ); pyresponse = ConvertType( &result ); } pystatus = ConvertType( &status ); PyObject *o = ( callback && callback != Py_None ) ? Py_BuildValue( "O", pystatus ) : Py_BuildValue( "OO", pystatus, pyresponse ); Py_DECREF( pystatus ); Py_XDECREF( pyresponse ); return o; } //---------------------------------------------------------------------------- //! List Extended File Attributes //---------------------------------------------------------------------------- PyObject* File::ListXAttr( File *self, PyObject *args, PyObject *kwds ) { static const char *kwlist[] = { "timeout", "callback", NULL }; uint16_t timeout = 0; PyObject *callback = NULL, *pystatus = NULL; PyObject *pyresponse = NULL; XrdCl::XRootDStatus status; if ( !self->file->IsOpen() ) return FileClosedError(); if ( !PyArg_ParseTupleAndKeywords( args, kwds, "|HO:set_xattr", (char**) kwlist, &timeout, &callback ) ) return NULL; if ( callback && callback != Py_None ) { XrdCl::ResponseHandler *handler = GetHandler>( callback ); if ( !handler ) return NULL; async( status = self->file->ListXAttr( handler, timeout ) ); } else { std::vector result; async( status = self->file->ListXAttr( result, timeout ) ); pyresponse = ConvertType( &result ); } pystatus = ConvertType( &status ); PyObject *o = ( callback && callback != Py_None ) ? Py_BuildValue( "O", pystatus ) : Py_BuildValue( "OO", pystatus, pyresponse ); Py_DECREF( pystatus ); Py_XDECREF( pyresponse ); return o; } } xrootd-5.6.9/bindings/python/src/PyXRootDFile.hh000066400000000000000000000277171457266313600216230ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2012-2014 by European Organization for Nuclear Research (CERN) // Author: Justin Salmon //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #ifndef PYXROOTDFILE_HH_ #define PYXROOTDFILE_HH_ #include "PyXRootD.hh" #include "Utils.hh" #include "XrdCl/XrdClFile.hh" #include namespace PyXRootD { //---------------------------------------------------------------------------- //! XrdCl::File binding class //---------------------------------------------------------------------------- class File { public: static PyObject* Open( File *self, PyObject *args, PyObject *kwds ); static PyObject* Close( File *self, PyObject *args, PyObject *kwds ); static PyObject* Stat( File *self, PyObject *args, PyObject *kwds ); static PyObject* Read( File *self, PyObject *args, PyObject *kwds ); static PyObject* ReadLine( File *self, PyObject *args, PyObject *kwds ); static PyObject* ReadLines( File *self, PyObject *args, PyObject *kwds ); static XrdCl::Buffer* ReadChunk( File *self, uint64_t offset, uint32_t size ); static PyObject* ReadChunks( File *self, PyObject *args, PyObject *kwds ); static PyObject* Write( File *self, PyObject *args, PyObject *kwds ); static PyObject* Sync( File *self, PyObject *args, PyObject *kwds ); static PyObject* Truncate( File *self, PyObject *args, PyObject *kwds ); static PyObject* VectorRead( File *self, PyObject *args, PyObject *kwds ); static PyObject* Fcntl( File *self, PyObject *args, PyObject *kwds ); static PyObject* Visa( File *self, PyObject *args, PyObject *kwds ); static PyObject* IsOpen( File *self, PyObject *args, PyObject *kwds ); static PyObject* GetProperty( File *self, PyObject *args, PyObject *kwds ); static PyObject* SetProperty( File *self, PyObject *args, PyObject *kwds ); static PyObject* SetXAttr( File *self, PyObject *args, PyObject *kwds ); static PyObject* GetXAttr( File *self, PyObject *args, PyObject *kwds ); static PyObject* DelXAttr( File *self, PyObject *args, PyObject *kwds ); static PyObject* ListXAttr( File *self, PyObject *args, PyObject *kwds ); public: PyObject_HEAD XrdCl::File *file; uint64_t currentOffset; }; PyDoc_STRVAR(file_type_doc, "File object (internal)"); //---------------------------------------------------------------------------- //! Set exception and return null if I/O op on closed file is attempted //---------------------------------------------------------------------------- static PyObject* FileClosedError() { PyErr_SetString( PyExc_ValueError, "I/O operation on closed file" ); return NULL; } //---------------------------------------------------------------------------- //! __init__() equivalent //---------------------------------------------------------------------------- static int File_init( File *self, PyObject *args ) { self->file = new XrdCl::File(); self->currentOffset = 0; return 0; } //---------------------------------------------------------------------------- //! Deallocation function, called when object is deleted //---------------------------------------------------------------------------- static void File_dealloc( File *self ) { delete self->file; Py_TYPE(self)->tp_free( (PyObject*) self ); } //---------------------------------------------------------------------------- //! __iter__ //---------------------------------------------------------------------------- static PyObject* File_iter( File *self ) { if ( !self->file->IsOpen() ) return FileClosedError(); //-------------------------------------------------------------------------- // Return ourselves for iteration //-------------------------------------------------------------------------- Py_INCREF( self ); return (PyObject*) self; } //---------------------------------------------------------------------------- //! __iternext__ //---------------------------------------------------------------------------- static PyObject* File_iternext( File *self ) { if ( !self->file->IsOpen() ) return FileClosedError(); PyObject *line = PyObject_CallMethod( (PyObject*) self, const_cast("readline"), NULL ); if( !line ) return NULL; //-------------------------------------------------------------------------- // Raise StopIteration if the line we just read is empty //-------------------------------------------------------------------------- if ( PyUnicode_GET_LENGTH( line ) == 0 ) { PyErr_SetNone( PyExc_StopIteration ); return NULL; } return line; } //---------------------------------------------------------------------------- //! __enter__ //---------------------------------------------------------------------------- static PyObject* File_enter( File *self ) { Py_INCREF( self ); return (PyObject*) self; } //---------------------------------------------------------------------------- //! __exit__ //---------------------------------------------------------------------------- static PyObject* File_exit( File *self ) { PyObject *ret = PyObject_CallMethod( (PyObject*) self, const_cast("close"), NULL ); if ( !ret ) return NULL; Py_DECREF( ret ); Py_RETURN_NONE ; } //---------------------------------------------------------------------------- //! Visible method definitions //---------------------------------------------------------------------------- static PyMethodDef FileMethods[] = { { "open", (PyCFunction) PyXRootD::File::Open, METH_VARARGS | METH_KEYWORDS, NULL }, { "close", (PyCFunction) PyXRootD::File::Close, METH_VARARGS | METH_KEYWORDS, NULL }, { "stat", (PyCFunction) PyXRootD::File::Stat, METH_VARARGS | METH_KEYWORDS, NULL }, { "read", (PyCFunction) PyXRootD::File::Read, METH_VARARGS | METH_KEYWORDS, NULL }, { "readline", (PyCFunction) PyXRootD::File::ReadLine, METH_VARARGS | METH_KEYWORDS, NULL }, { "readlines", (PyCFunction) PyXRootD::File::ReadLines, METH_VARARGS | METH_KEYWORDS, NULL }, { "readchunks", (PyCFunction) PyXRootD::File::ReadChunks, METH_VARARGS | METH_KEYWORDS, NULL }, { "write", (PyCFunction) PyXRootD::File::Write, METH_VARARGS | METH_KEYWORDS, NULL }, { "sync", (PyCFunction) PyXRootD::File::Sync, METH_VARARGS | METH_KEYWORDS, NULL }, { "truncate", (PyCFunction) PyXRootD::File::Truncate, METH_VARARGS | METH_KEYWORDS, NULL }, { "vector_read", (PyCFunction) PyXRootD::File::VectorRead, METH_VARARGS | METH_KEYWORDS, NULL }, { "fcntl", (PyCFunction) PyXRootD::File::Fcntl, METH_VARARGS | METH_KEYWORDS, NULL }, { "visa", (PyCFunction) PyXRootD::File::Visa, METH_VARARGS | METH_KEYWORDS, NULL }, { "is_open", (PyCFunction) PyXRootD::File::IsOpen, METH_VARARGS | METH_KEYWORDS, NULL }, { "get_property", (PyCFunction) PyXRootD::File::GetProperty, METH_VARARGS | METH_KEYWORDS, NULL }, { "set_property", (PyCFunction) PyXRootD::File::SetProperty, METH_VARARGS | METH_KEYWORDS, NULL }, { "set_xattr", (PyCFunction) PyXRootD::File::SetXAttr, METH_VARARGS | METH_KEYWORDS, NULL }, { "get_xattr", (PyCFunction) PyXRootD::File::GetXAttr, METH_VARARGS | METH_KEYWORDS, NULL }, { "del_xattr", (PyCFunction) PyXRootD::File::DelXAttr, METH_VARARGS | METH_KEYWORDS, NULL }, { "list_xattr", (PyCFunction) PyXRootD::File::ListXAttr, METH_VARARGS | METH_KEYWORDS, NULL }, {"__enter__", (PyCFunction) File_enter, METH_NOARGS, NULL}, {"__exit__", (PyCFunction) File_exit, METH_VARARGS, NULL}, { NULL } /* Sentinel */ }; //---------------------------------------------------------------------------- //! Visible member definitions //---------------------------------------------------------------------------- static PyMemberDef FileMembers[] = { { NULL } /* Sentinel */ }; //---------------------------------------------------------------------------- //! File binding type object //---------------------------------------------------------------------------- static PyTypeObject FileType = { PyVarObject_HEAD_INIT(NULL, 0) "pyxrootd.File", /* tp_name */ sizeof(File), /* tp_basicsize */ 0, /* tp_itemsize */ (destructor) File_dealloc, /* tp_dealloc */ 0, /* tp_print */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_compare */ 0, /* tp_repr */ 0, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ 0, /* tp_hash */ 0, /* tp_call */ 0, /* tp_str */ 0, /* tp_getattro */ 0, /* tp_setattro */ 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_ITER, /* tp_flags */ file_type_doc, /* tp_doc */ 0, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ (getiterfunc) File_iter, /* tp_iter */ (iternextfunc) File_iternext, /* tp_iternext */ FileMethods, /* tp_methods */ FileMembers, /* tp_members */ 0, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ 0, /* tp_descr_set */ 0, /* tp_dictoffset */ (initproc) File_init, /* tp_init */ }; } #endif /* PYXROOTDFILE_HH_ */ xrootd-5.6.9/bindings/python/src/PyXRootDFileSystem.cc000066400000000000000000001110351457266313600230010ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2012-2014 by European Organization for Nuclear Research (CERN) // Author: Justin Salmon //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #include "PyXRootD.hh" #include "PyXRootDFileSystem.hh" #include "PyXRootDCopyProcess.hh" #include "AsyncResponseHandler.hh" #include "Utils.hh" #include "XrdCl/XrdClFileSystem.hh" #include "XrdCl/XrdClCopyProcess.hh" namespace PyXRootD { //---------------------------------------------------------------------------- //! Copy a file //---------------------------------------------------------------------------- PyObject* FileSystem::Copy( FileSystem *self, PyObject *args, PyObject *kwds ) { static const char *kwlist[] = { "source", "target", "force", NULL }; const char *source, *target; bool force = false; PyObject *pystatus = NULL; CopyProcess *copyprocess = NULL; if ( !PyArg_ParseTupleAndKeywords( args, kwds, "ss|i:copy", (char**) kwlist, &source, &target, &force ) ) return NULL; CopyProcessType.tp_new = PyType_GenericNew; if ( PyType_Ready( &CopyProcessType ) < 0 ) return NULL; copyprocess = (CopyProcess*) PyObject_CallObject( (PyObject *) &CopyProcessType, NULL ); if ( !copyprocess ) return NULL; copyprocess->AddJob( copyprocess, args, kwds ); pystatus = copyprocess->Prepare( copyprocess, NULL, NULL ); if ( !pystatus ) return NULL; if ( PyDict_GetItemString( pystatus, "ok" ) == Py_False ) { PyObject *tuple = PyTuple_New(2); PyTuple_SetItem(tuple, 0, pystatus); Py_INCREF(Py_None); PyTuple_SetItem(tuple, 1, Py_None); return tuple; } pystatus = copyprocess->Run( copyprocess, PyTuple_New(0), PyDict_New() ); if ( !pystatus ) return NULL; Py_DECREF( copyprocess ); return pystatus; } //---------------------------------------------------------------------------- //! Locate a file //---------------------------------------------------------------------------- PyObject* FileSystem::Locate( FileSystem *self, PyObject *args, PyObject *kwds ) { static const char *kwlist[] = { "path", "flags", "timeout", "callback", NULL }; const char *path; XrdCl::OpenFlags::Flags flags = XrdCl::OpenFlags::None; uint16_t timeout = 0; PyObject *callback = NULL, *pyresponse = NULL, *pystatus = NULL; XrdCl::XRootDStatus status; if ( !PyArg_ParseTupleAndKeywords( args, kwds, "sH|HO:locate", (char**) kwlist, &path, &flags, &timeout, &callback ) ) return NULL; if ( callback && callback != Py_None ) { XrdCl::ResponseHandler *handler = GetHandler( callback ); if ( !handler ) return NULL; async( status = self->filesystem->Locate( path, flags, handler, timeout ) ); } else { XrdCl::LocationInfo *response = 0; async( status = self->filesystem->Locate( path, flags, response, timeout ) ); pyresponse = ConvertType( response ); delete response; } pystatus = ConvertType( &status ); PyObject *o = ( callback && callback != Py_None ) ? Py_BuildValue( "O", pystatus ) : Py_BuildValue( "OO", pystatus, pyresponse ); Py_DECREF( pystatus ); Py_XDECREF( pyresponse ); return o; } //---------------------------------------------------------------------------- //! Locate a file, recursively locate all disk servers //---------------------------------------------------------------------------- PyObject* FileSystem::DeepLocate( FileSystem *self, PyObject *args, PyObject *kwds ) { static const char *kwlist[] = { "path", "flags", "timeout", "callback", NULL }; const char *path; XrdCl::OpenFlags::Flags flags = XrdCl::OpenFlags::None; uint16_t timeout = 0; PyObject *callback = NULL, *pyresponse = NULL, *pystatus = NULL; XrdCl::XRootDStatus status; if ( !PyArg_ParseTupleAndKeywords( args, kwds, "sH|HO:deeplocate", (char**) kwlist, &path, &flags, &timeout, &callback ) ) return NULL; if ( callback && callback != Py_None ) { XrdCl::ResponseHandler *handler = GetHandler( callback ); if ( !handler ) return NULL; async( status = self->filesystem->DeepLocate( path, flags, handler, timeout ) ); } else { XrdCl::LocationInfo *response = 0; async( status = self->filesystem->DeepLocate( path, flags, response, timeout ) ); pyresponse = ConvertType( response ); delete response; } pystatus = ConvertType( &status ); PyObject *o = ( callback && callback != Py_None ) ? Py_BuildValue( "O", pystatus ) : Py_BuildValue( "OO", pystatus, pyresponse ); Py_DECREF( pystatus ); Py_XDECREF( pyresponse ); return o; } //---------------------------------------------------------------------------- //! Move a directory or a file //---------------------------------------------------------------------------- PyObject* FileSystem::Mv( FileSystem *self, PyObject *args, PyObject *kwds ) { static const char *kwlist[] = { "source", "dest", "timeout", "callback", NULL }; const char *source; const char *dest; uint16_t timeout = 0; PyObject *callback = NULL, *pystatus = NULL; XrdCl::XRootDStatus status; if ( !PyArg_ParseTupleAndKeywords( args, kwds, "ss|HO:mv", (char**) kwlist, &source, &dest, &timeout, &callback ) ) return NULL; if ( callback && callback != Py_None ) { XrdCl::ResponseHandler *handler = GetHandler( callback ); if ( !handler ) return NULL; async( status = self->filesystem->Mv( source, dest, handler, timeout ) ); } else { async( status = self->filesystem->Mv( source, dest, timeout ) ); } pystatus = ConvertType( &status ); PyObject *o = ( callback && callback != Py_None ) ? Py_BuildValue( "O", pystatus ) : Py_BuildValue( "OO", pystatus, Py_BuildValue( "" ) ); Py_DECREF( pystatus ); return o; } //---------------------------------------------------------------------------- //! Obtain server information //---------------------------------------------------------------------------- PyObject* FileSystem::Query( FileSystem *self, PyObject *args, PyObject *kwds ) { static const char *kwlist[] = { "querycode", "arg", "timeout", "callback", NULL }; const char *arg; uint16_t timeout = 0; PyObject *callback = NULL, *pyresponse = NULL, *pystatus = NULL; XrdCl::QueryCode::Code queryCode; XrdCl::XRootDStatus status; XrdCl::Buffer argbuffer; if ( !PyArg_ParseTupleAndKeywords( args, kwds, "is|HO:query", (char**) kwlist, &queryCode, &arg, &timeout, &callback ) ) return NULL; argbuffer.FromString(arg); if ( callback && callback != Py_None ) { XrdCl::ResponseHandler *handler = GetHandler( callback ); if ( !handler ) return NULL; async( status = self->filesystem->Query( queryCode, argbuffer, handler, timeout ) ); } else { XrdCl::Buffer *response = 0; async( status = self->filesystem->Query( queryCode, argbuffer, response, timeout ) ); pyresponse = ConvertType( response ); delete response; } pystatus = ConvertType( &status ); PyObject *o = ( callback && callback != Py_None ) ? Py_BuildValue( "O", pystatus ) : Py_BuildValue( "OO", pystatus, pyresponse ); Py_DECREF( pystatus ); Py_XDECREF( pyresponse ); return o; } //---------------------------------------------------------------------------- //! Truncate a file //---------------------------------------------------------------------------- PyObject* FileSystem::Truncate( FileSystem *self, PyObject *args, PyObject *kwds ) { static const char *kwlist[] = { "path", "size", "timeout", "callback", NULL }; const char *path; uint64_t size = 0; uint16_t timeout = 0; PyObject *callback = NULL, *pystatus = NULL; XrdCl::XRootDStatus status; if ( !PyArg_ParseTupleAndKeywords( args, kwds, "sK|HO:truncate", (char**) kwlist, &path, &size, &timeout, &callback ) ) return NULL; if ( callback && callback != Py_None ) { XrdCl::ResponseHandler *handler = GetHandler( callback ); if ( !handler ) return NULL; async( status = self->filesystem->Truncate( path, size, handler, timeout ) ); } else { async( status = self->filesystem->Truncate( path, size, timeout ) ); } pystatus = ConvertType( &status ); PyObject *o = ( callback && callback != Py_None ) ? Py_BuildValue( "O", pystatus ) : Py_BuildValue( "OO", pystatus, Py_BuildValue( "" ) ); Py_DECREF( pystatus ); return o; } //---------------------------------------------------------------------------- //! Remove a file //---------------------------------------------------------------------------- PyObject* FileSystem::Rm( FileSystem *self, PyObject *args, PyObject *kwds ) { static const char *kwlist[] = { "path", "timeout", "callback", NULL }; const char *path; uint16_t timeout = 0; PyObject *callback = NULL, *pystatus = NULL; XrdCl::XRootDStatus status; if ( !PyArg_ParseTupleAndKeywords( args, kwds, "s|HO:rm", (char**) kwlist, &path, &timeout, &callback ) ) return NULL; if ( callback && callback != Py_None ) { XrdCl::ResponseHandler *handler = GetHandler( callback ); if ( !handler ) return NULL; async( status = self->filesystem->Rm( path, handler, timeout ) ); } else { async( status = self->filesystem->Rm( path, timeout ) ); } pystatus = ConvertType( &status ); PyObject *o = ( callback && callback != Py_None ) ? Py_BuildValue( "O", pystatus ) : Py_BuildValue( "OO", pystatus, Py_BuildValue( "" ) ); Py_DECREF( pystatus ); return o; } //---------------------------------------------------------------------------- //! Create a directory //---------------------------------------------------------------------------- PyObject* FileSystem::MkDir( FileSystem *self, PyObject *args, PyObject *kwds ) { static const char *kwlist[] = { "path", "flags", "mode", "timeout", "callback", NULL }; const char *path; XrdCl::MkDirFlags::Flags flags = XrdCl::MkDirFlags::None; XrdCl::Access::Mode mode = XrdCl::Access::None; uint16_t timeout = 0; PyObject *callback = NULL, *pystatus = NULL; XrdCl::XRootDStatus status; if ( !PyArg_ParseTupleAndKeywords( args, kwds, "s|HHHO:mkdir", (char**) kwlist, &path, &flags, &mode, &timeout, &callback ) ) return NULL; if ( callback && callback != Py_None ) { XrdCl::ResponseHandler *handler = GetHandler( callback ); if ( !handler ) return NULL; async( status = self->filesystem->MkDir( path, flags, mode, handler, timeout ) ); } else { async( status = self->filesystem->MkDir( path, flags, mode, timeout ) ); } pystatus = ConvertType( &status ); PyObject *o = ( callback && callback != Py_None ) ? Py_BuildValue( "O", pystatus ) : Py_BuildValue( "OO", pystatus, Py_BuildValue( "" ) ); Py_DECREF( pystatus ); return o; } //---------------------------------------------------------------------------- //! Remove a directory //---------------------------------------------------------------------------- PyObject* FileSystem::RmDir( FileSystem *self, PyObject *args, PyObject *kwds ) { static const char *kwlist[] = { "path", "timeout", "callback", NULL }; const char *path; uint16_t timeout = 0; PyObject *callback = NULL, *pystatus = NULL; XrdCl::XRootDStatus status; if ( !PyArg_ParseTupleAndKeywords( args, kwds, "s|HO:rmdir", (char**) kwlist, &path, &timeout, &callback ) ) return NULL; if ( callback && callback != Py_None ) { XrdCl::ResponseHandler *handler = GetHandler( callback ); if ( !handler ) return NULL; async( status = self->filesystem->RmDir( path, handler, timeout ) ); } else { async( status = self->filesystem->RmDir( path, timeout ) ); } pystatus = ConvertType( &status ); PyObject *o = ( callback && callback != Py_None ) ? Py_BuildValue( "O", pystatus ) : Py_BuildValue( "OO", pystatus, Py_BuildValue( "" ) ); Py_DECREF( pystatus ); return o; } //---------------------------------------------------------------------------- //! Change access mode on a directory or a file //---------------------------------------------------------------------------- PyObject* FileSystem::ChMod( FileSystem *self, PyObject *args, PyObject *kwds ) { static const char *kwlist[] = { "path", "mode", "timeout", "callback", NULL }; const char *path; XrdCl::Access::Mode mode = XrdCl::Access::None; uint16_t timeout = 0; PyObject *callback = NULL, *pystatus = NULL; XrdCl::XRootDStatus status; if ( !PyArg_ParseTupleAndKeywords( args, kwds, "sH|HO:chmod", (char**) kwlist, &path, &mode, &timeout, &callback ) ) return NULL; if ( callback && callback != Py_None ) { XrdCl::ResponseHandler *handler = GetHandler( callback ); if ( !handler ) return NULL; async( status = self->filesystem->ChMod( path, mode, handler, timeout ) ); } else { async( status = self->filesystem->ChMod( path, mode, timeout ) ); } pystatus = ConvertType( &status ); PyObject *o = ( callback && callback != Py_None ) ? Py_BuildValue( "O", pystatus ) : Py_BuildValue( "OO", pystatus, Py_BuildValue( "" ) ); Py_DECREF( pystatus ); return o; } //---------------------------------------------------------------------------- // Check if the server is alive //---------------------------------------------------------------------------- PyObject* FileSystem::Ping( FileSystem *self, PyObject *args, PyObject *kwds ) { static const char *kwlist[] = { "timeout", "callback", NULL }; uint16_t timeout = 0; PyObject *callback = NULL, *pystatus = NULL; XrdCl::XRootDStatus status; if ( !PyArg_ParseTupleAndKeywords( args, kwds, "|HO:ping", (char**) kwlist, &timeout, &callback ) ) return NULL; if ( callback && callback != Py_None ) { XrdCl::ResponseHandler *handler = GetHandler( callback ); if ( !handler ) return NULL; async( status = self->filesystem->Ping( handler, timeout ) ); } else { async( status = self->filesystem->Ping( timeout ) ); } pystatus = ConvertType( &status ); PyObject *o = ( callback && callback != Py_None ) ? Py_BuildValue( "O", pystatus ) : Py_BuildValue( "OO", pystatus, Py_BuildValue( "" ) ); Py_DECREF( pystatus ); return o; } //---------------------------------------------------------------------------- //! Obtain status information for a path //---------------------------------------------------------------------------- PyObject* FileSystem::Stat( FileSystem *self, PyObject *args, PyObject *kwds ) { static const char *kwlist[] = { "path", "timeout", "callback", NULL }; const char *path; uint16_t timeout = 0; PyObject *callback = NULL, *pyresponse = NULL, *pystatus = NULL; XrdCl::XRootDStatus status; if ( !PyArg_ParseTupleAndKeywords( args, kwds, "s|HO:stat", (char**) kwlist, &path, &timeout, &callback ) ) return NULL; if ( callback && callback != Py_None ) { XrdCl::ResponseHandler *handler = GetHandler( callback ); if ( !handler ) return NULL; async( status = self->filesystem->Stat( path, handler, timeout ) ); } else { XrdCl::StatInfo *response = 0; async( status = self->filesystem->Stat( path, response, timeout ) ); pyresponse = ConvertType( response ); delete response; } pystatus = ConvertType( &status ); PyObject *o = ( callback && callback != Py_None ) ? Py_BuildValue( "O", pystatus ) : Py_BuildValue( "OO", pystatus, pyresponse ); Py_DECREF( pystatus ); Py_XDECREF( pyresponse ); return o; } //---------------------------------------------------------------------------- //! Obtain status information for a Virtual File System //---------------------------------------------------------------------------- PyObject* FileSystem::StatVFS( FileSystem *self, PyObject *args, PyObject *kwds ) { static const char *kwlist[] = { "path", "timeout", "callback", NULL }; const char *path; uint16_t timeout = 0; PyObject *callback = NULL, *pyresponse = NULL, *pystatus = NULL; XrdCl::XRootDStatus status; if ( !PyArg_ParseTupleAndKeywords( args, kwds, "s|HO:statvfs", (char**) kwlist, &path, &timeout, &callback ) ) return NULL; if ( callback && callback != Py_None ) { XrdCl::ResponseHandler *handler = GetHandler( callback ); if ( !handler ) return NULL; async( status = self->filesystem->StatVFS( path, handler, timeout ) ); } else { XrdCl::StatInfoVFS *response = 0; async( status = self->filesystem->StatVFS( path, response, timeout ) ); pyresponse = ConvertType( response ); delete response; } pystatus = ConvertType( &status ); PyObject *o = ( callback && callback != Py_None ) ? Py_BuildValue( "O", pystatus ) : Py_BuildValue( "OO", pystatus, pyresponse ); Py_DECREF( pystatus ); Py_XDECREF( pyresponse ); return o; } //---------------------------------------------------------------------------- //! Obtain server protocol information //---------------------------------------------------------------------------- PyObject* FileSystem::Protocol( FileSystem *self, PyObject *args, PyObject *kwds ) { static const char *kwlist[] = { "timeout", "callback", NULL }; uint16_t timeout = 0; PyObject *callback = NULL, *pyresponse = NULL, *pystatus = NULL; XrdCl::XRootDStatus status; if ( !PyArg_ParseTupleAndKeywords( args, kwds, "|HO:protocol", (char**) kwlist, &timeout, &callback ) ) return NULL; if ( callback && callback != Py_None ) { XrdCl::ResponseHandler *handler = GetHandler( callback ); if ( !handler ) return NULL; async( status = self->filesystem->Protocol( handler, timeout ) ); } else { XrdCl::ProtocolInfo *response = 0; async( status = self->filesystem->Protocol( response, timeout ) ); pyresponse = ConvertType( response ); delete response; } pystatus = ConvertType( &status ); PyObject *o = ( callback && callback != Py_None ) ? Py_BuildValue( "O", pystatus ) : Py_BuildValue( "OO", pystatus, pyresponse ); Py_DECREF( pystatus ); Py_XDECREF( pyresponse ); return o; } //---------------------------------------------------------------------------- //! List entries of a directory //---------------------------------------------------------------------------- PyObject* FileSystem::DirList( FileSystem *self, PyObject *args, PyObject *kwds ) { static const char *kwlist[] = { "path", "flags", "timeout", "callback", NULL }; const char *path; XrdCl::DirListFlags::Flags flags = XrdCl::DirListFlags::None; uint16_t timeout = 0; PyObject *callback = NULL, *pyresponse = NULL, *pystatus = NULL; XrdCl::XRootDStatus status; if ( !PyArg_ParseTupleAndKeywords( args, kwds, "s|bHO:dirlist", (char**) kwlist, &path, &flags, &timeout, &callback ) ) return NULL; if ( callback && callback != Py_None ) { XrdCl::ResponseHandler *handler = GetHandler( callback ); if ( !handler ) return NULL; async( status = self->filesystem->DirList( path, flags, handler, timeout ) ); } else { XrdCl::DirectoryList *list = 0; async( status = self->filesystem->DirList( path, flags, list, timeout ) ); pyresponse = ConvertType( list ); delete list; } pystatus = ConvertType( &status ); PyObject *o = ( callback && callback != Py_None ) ? Py_BuildValue( "O", pystatus ) : Py_BuildValue( "OO", pystatus, pyresponse ); Py_DECREF( pystatus ); Py_XDECREF( pyresponse ); return o; } //---------------------------------------------------------------------------- //! Send info to the server (up to 1024 characters) //---------------------------------------------------------------------------- PyObject* FileSystem::SendInfo( FileSystem *self, PyObject *args, PyObject *kwds ) { static const char *kwlist[] = { "info", "timeout", "callback", NULL }; const char *info; uint16_t timeout = 0; PyObject *callback = NULL, *pyresponse = NULL, *pystatus = NULL; XrdCl::XRootDStatus status; if ( !PyArg_ParseTupleAndKeywords( args, kwds, "s|HO:sendinfo", (char**) kwlist, &info, &timeout, &callback ) ) return NULL; if ( callback && callback != Py_None ) { XrdCl::ResponseHandler *handler = GetHandler( callback ); if ( !handler ) return NULL; async( status = self->filesystem->SendInfo( info, handler, timeout ) ); } else { XrdCl::Buffer *response = 0; async( status = self->filesystem->SendInfo( info, response, timeout ) ); pyresponse = ConvertType( response ); delete response; } pystatus = ConvertType( &status ); PyObject *o = ( callback && callback != Py_None ) ? Py_BuildValue( "O", pystatus ) : Py_BuildValue( "OO", pystatus, pyresponse ); Py_DECREF( pystatus ); Py_XDECREF( pyresponse ); return o; } //---------------------------------------------------------------------------- //! Prepare one or more files for access //---------------------------------------------------------------------------- PyObject* FileSystem::Prepare( FileSystem *self, PyObject *args, PyObject *kwds ) { static const char *kwlist[] = { "files", "flags", "priority", "timeout", "callback", NULL }; uint16_t flagval = 0; uint8_t priority = 0; uint16_t timeout = 0; PyObject *pyfiles = NULL, *callback = NULL; PyObject *pyresponse = NULL, *pystatus = NULL; XrdCl::XRootDStatus status; if ( !PyArg_ParseTupleAndKeywords( args, kwds, "OH|bHO:prepare", (char**) kwlist, &pyfiles, &flagval, &priority, &timeout, &callback ) ) return NULL; if ( !PyList_Check( pyfiles ) ) { PyErr_SetString( PyExc_TypeError, "files parameter must be a list" ); return NULL; } std::vector files; for (int i = 0; i < PyList_Size(pyfiles); ++i) { PyObject *item = PyList_GetItem(pyfiles, i); if (!PyUnicode_Check(item)) { PyErr_SetString(PyExc_TypeError, "files parameter must be a list of strings"); return NULL; } files.emplace_back(PyUnicode_AsUTF8(item)); } XrdCl::PrepareFlags::Flags flags; flags = static_cast(flagval); if ( callback && callback != Py_None ) { XrdCl::ResponseHandler *handler = GetHandler( callback ); if ( !handler ) return NULL; async( status = self->filesystem->Prepare( files, flags, priority, handler, timeout ) ); } else { XrdCl::Buffer *response = 0; async( status = self->filesystem->Prepare( files, flags, priority, response, timeout ) ); pyresponse = ConvertType( response ); delete response; } (void) FileSystemType; // Suppress unused variable warning pystatus = ConvertType( &status ); PyObject *o = ( callback && callback != Py_None ) ? Py_BuildValue( "O", pystatus ) : Py_BuildValue( "OO", pystatus, pyresponse ); Py_DECREF( pystatus ); Py_XDECREF( pyresponse ); return o; } //---------------------------------------------------------------------------- //! Get property //---------------------------------------------------------------------------- PyObject* FileSystem::GetProperty( FileSystem *self, PyObject *args, PyObject *kwds ) { static const char *kwlist[] = { "name", NULL }; char *name = 0; std::string value; if ( !PyArg_ParseTupleAndKeywords( args, kwds, "s:get_property", (char**) kwlist, &name ) ) return NULL; bool status = self->filesystem->GetProperty( name, value ); return status ? Py_BuildValue( "s", value.c_str() ) : Py_None; } //---------------------------------------------------------------------------- //! Set property //---------------------------------------------------------------------------- PyObject* FileSystem::SetProperty( FileSystem *self, PyObject *args, PyObject *kwds ) { static const char *kwlist[] = { "name", "value", NULL }; char *name = 0; char *value = 0; if ( !PyArg_ParseTupleAndKeywords( args, kwds, "ss:set_property", (char**) kwlist, &name, &value ) ) return NULL; bool status = self->filesystem->SetProperty( name, value ); return status ? Py_True : Py_False; } //---------------------------------------------------------------------------- //! Do a remote cat //---------------------------------------------------------------------------- PyObject* FileSystem::Cat( FileSystem *self, PyObject *args, PyObject *kwds ) { static const char *kwlist[] = { "source" , NULL }; const char *source = 0; if ( !PyArg_ParseTupleAndKeywords( args, kwds, "s", (char**) kwlist, &source ) ) Py_RETURN_NONE; XrdCl::CopyProcess process; XrdCl::PropertyList props, results; props.Set( "source", source ); props.Set( "target", "stdio://-" ); props.Set( "dynamicSource", true ); XrdCl::XRootDStatus st = process.AddJob( props, &results ); if( !st.IsOK() ) return ConvertType( &st ); st = process.Prepare(); if( !st.IsOK() ) return ConvertType( &st ); st = process.Run( 0 ); return ConvertType( &st ); } //---------------------------------------------------------------------------- //! Set Extended File Attributes //---------------------------------------------------------------------------- PyObject* FileSystem::SetXAttr( FileSystem *self, PyObject *args, PyObject *kwds ) { static const char *kwlist[] = { "path", "attrs", "timeout", "callback", NULL }; char *path = 0; std::vector attrs; uint16_t timeout = 0; PyObject *callback = NULL, *pystatus = NULL; PyObject *pyattrs = NULL, *pyresponse = NULL; XrdCl::XRootDStatus status; if ( !PyArg_ParseTupleAndKeywords( args, kwds, "sO|HO:set_xattr", (char**) kwlist, &path, &pyattrs, &timeout, &callback ) ) return NULL; // it should be a list if( !PyList_Check( pyattrs ) ) return NULL; // now parse the input Py_ssize_t size = PyList_Size( pyattrs ); attrs.reserve( size ); for( ssize_t i = 0; i < size; ++i ) { // get the item at respective index PyObject *item = PyList_GetItem( pyattrs, i ); // make sure the item is a tuple if( !item || !PyTuple_Check( item ) ) return NULL; // make sure the tuple size equals to 2 if( PyTuple_Size( item ) != 2 ) return NULL; // extract the attribute name from the tuple PyObject *py_name = PyTuple_GetItem( item, 0 ); if( !PyUnicode_Check( py_name ) ) return NULL; std::string name = PyUnicode_AsUTF8( py_name ); // extract the attribute value from the tuple PyObject *py_value = PyTuple_GetItem( item, 1 ); if( !PyUnicode_Check( py_value ) ) return NULL; std::string value = PyUnicode_AsUTF8( py_value ); // update the C++ list of xattrs attrs.push_back( XrdCl::xattr_t( name, value ) ); } if ( callback && callback != Py_None ) { XrdCl::ResponseHandler *handler = GetHandler>( callback ); if ( !handler ) return NULL; async( status = self->filesystem->SetXAttr( path, attrs, handler, timeout ) ); } else { std::vector result; async( status = self->filesystem->SetXAttr( path, attrs, result, timeout ) ); pyresponse = ConvertType( &result ); } pystatus = ConvertType( &status ); PyObject *o = ( callback && callback != Py_None ) ? Py_BuildValue( "O", pystatus ) : Py_BuildValue( "OO", pystatus, pyresponse ); Py_DECREF( pystatus ); Py_XDECREF( pyresponse ); return o; } //---------------------------------------------------------------------------- //! Get Extended File Attributes //---------------------------------------------------------------------------- PyObject* FileSystem::GetXAttr( FileSystem *self, PyObject *args, PyObject *kwds ) { static const char *kwlist[] = { "path", "attrs", "timeout", "callback", NULL }; char *path = 0; std::vector attrs; uint16_t timeout = 0; PyObject *callback = NULL, *pystatus = NULL; PyObject *pyattrs = NULL, *pyresponse = NULL; XrdCl::XRootDStatus status; if ( !PyArg_ParseTupleAndKeywords( args, kwds, "sO|HO:set_xattr", (char**) kwlist, &path, &pyattrs, &timeout, &callback ) ) return NULL; // it should be a list if( !PyList_Check( pyattrs ) ) return NULL; // now parse the input Py_ssize_t size = PyList_Size( pyattrs ); attrs.reserve( size ); for( ssize_t i = 0; i < size; ++i ) { // get the item at respective index PyObject *item = PyList_GetItem( pyattrs, i ); // make sure the item is a string if( !item || !PyUnicode_Check( item ) ) return NULL; std::string name = PyUnicode_AsUTF8( item ); // update the C++ list of xattrs attrs.push_back( name ); } if ( callback && callback != Py_None ) { XrdCl::ResponseHandler *handler = GetHandler>( callback ); if ( !handler ) return NULL; async( status = self->filesystem->GetXAttr( path, attrs, handler, timeout ) ); } else { std::vector result; async( status = self->filesystem->GetXAttr( path, attrs, result, timeout ) ); pyresponse = ConvertType( &result ); } pystatus = ConvertType( &status ); PyObject *o = ( callback && callback != Py_None ) ? Py_BuildValue( "O", pystatus ) : Py_BuildValue( "OO", pystatus, pyresponse ); Py_DECREF( pystatus ); Py_XDECREF( pyresponse ); return o; } //---------------------------------------------------------------------------- //! Delete Extended File Attributes //---------------------------------------------------------------------------- PyObject* FileSystem::DelXAttr( FileSystem *self, PyObject *args, PyObject *kwds ) { static const char *kwlist[] = { "path", "attrs", "timeout", "callback", NULL }; char *path = 0; std::vector attrs; uint16_t timeout = 0; PyObject *callback = NULL, *pystatus = NULL; PyObject *pyattrs = NULL, *pyresponse = NULL; XrdCl::XRootDStatus status; if ( !PyArg_ParseTupleAndKeywords( args, kwds, "sO|HO:set_xattr", (char**) kwlist, &path, &pyattrs, &timeout, &callback ) ) return NULL; // it should be a list if( !PyList_Check( pyattrs ) ) return NULL; // now parse the input Py_ssize_t size = PyList_Size( pyattrs ); attrs.reserve( size ); for( ssize_t i = 0; i < size; ++i ) { // get the item at respective index PyObject *item = PyList_GetItem( pyattrs, i ); // make sure the item is a string if( !item || !PyUnicode_Check( item ) ) return NULL; std::string name = PyUnicode_AsUTF8( item ); // update the C++ list of xattrs attrs.push_back( name ); } if ( callback && callback != Py_None ) { XrdCl::ResponseHandler *handler = GetHandler>( callback ); if ( !handler ) return NULL; async( status = self->filesystem->DelXAttr( path, attrs, handler, timeout ) ); } else { std::vector result; async( status = self->filesystem->DelXAttr( path, attrs, result, timeout ) ); pyresponse = ConvertType( &result ); } pystatus = ConvertType( &status ); PyObject *o = ( callback && callback != Py_None ) ? Py_BuildValue( "O", pystatus ) : Py_BuildValue( "OO", pystatus, pyresponse ); Py_DECREF( pystatus ); Py_XDECREF( pyresponse ); return o; } //---------------------------------------------------------------------------- //! List Extended File Attributes //---------------------------------------------------------------------------- PyObject* FileSystem::ListXAttr( FileSystem *self, PyObject *args, PyObject *kwds ) { static const char *kwlist[] = { "path", "timeout", "callback", NULL }; char *path = 0; uint16_t timeout = 0; PyObject *callback = NULL, *pystatus = NULL, *pyresponse = NULL; XrdCl::XRootDStatus status; if ( !PyArg_ParseTupleAndKeywords( args, kwds, "s|HO:set_xattr", (char**) kwlist, &path, &timeout, &callback ) ) return NULL; if ( callback && callback != Py_None ) { XrdCl::ResponseHandler *handler = GetHandler>( callback ); if ( !handler ) return NULL; async( status = self->filesystem->ListXAttr( path, handler, timeout ) ); } else { std::vector result; async( status = self->filesystem->ListXAttr( path, result, timeout ) ); pyresponse = ConvertType( &result ); } pystatus = ConvertType( &status ); PyObject *o = ( callback && callback != Py_None ) ? Py_BuildValue( "O", pystatus ) : Py_BuildValue( "OO", pystatus, pyresponse ); Py_DECREF( pystatus ); Py_XDECREF( pyresponse ); return o; } } xrootd-5.6.9/bindings/python/src/PyXRootDFileSystem.hh000066400000000000000000000247641457266313600230270ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2012-2014 by European Organization for Nuclear Research (CERN) // Author: Justin Salmon //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #ifndef PYXROOTD_FILESYSTEM_HH_ #define PYXROOTD_FILESYSTEM_HH_ #include "PyXRootD.hh" #include "PyXRootDURL.hh" #include "Conversions.hh" #include "XrdCl/XrdClFileSystem.hh" namespace PyXRootD { //---------------------------------------------------------------------------- //! XrdCl::FileSystem binding class //---------------------------------------------------------------------------- class FileSystem { public: static PyObject* Copy( FileSystem *self, PyObject *args, PyObject *kwds ); static PyObject* Locate( FileSystem *self, PyObject *args, PyObject *kwds ); static PyObject* DeepLocate( FileSystem *self, PyObject *args, PyObject *kwds ); static PyObject* Mv( FileSystem *self, PyObject *args, PyObject *kwds ); static PyObject* Query( FileSystem *self, PyObject *args, PyObject *kwds ); static PyObject* Truncate( FileSystem *self, PyObject *args, PyObject *kwds ); static PyObject* Rm( FileSystem *self, PyObject *args, PyObject *kwds ); static PyObject* MkDir( FileSystem *self, PyObject *args, PyObject *kwds ); static PyObject* RmDir( FileSystem *self, PyObject *args, PyObject *kwds ); static PyObject* ChMod( FileSystem *self, PyObject *args, PyObject *kwds ); static PyObject* Ping( FileSystem *self, PyObject *args, PyObject *kwds ); static PyObject* Stat( FileSystem *self, PyObject *args, PyObject *kwds ); static PyObject* StatVFS( FileSystem *self, PyObject *args, PyObject *kwds ); static PyObject* Protocol( FileSystem *self, PyObject *args, PyObject *kwds ); static PyObject* DirList( FileSystem *self, PyObject *args, PyObject *kwds ); static PyObject* SendInfo( FileSystem *self, PyObject *args, PyObject *kwds ); static PyObject* Prepare( FileSystem *self, PyObject *args, PyObject *kwds ); static PyObject* GetProperty( FileSystem *self, PyObject *args, PyObject *kwds ); static PyObject* SetProperty( FileSystem *self, PyObject *args, PyObject *kwds ); static PyObject* Cat( FileSystem *self, PyObject *args, PyObject *kwds ); static PyObject* SetXAttr( FileSystem *self, PyObject *args, PyObject *kwds ); static PyObject* GetXAttr( FileSystem *self, PyObject *args, PyObject *kwds ); static PyObject* DelXAttr( FileSystem *self, PyObject *args, PyObject *kwds ); static PyObject* ListXAttr( FileSystem *self, PyObject *args, PyObject *kwds ); public: PyObject_HEAD URL *url; XrdCl::FileSystem *filesystem; }; PyDoc_STRVAR(filesystem_type_doc, "FileSystem object (internal)"); //---------------------------------------------------------------------------- //! Visible method definitions //---------------------------------------------------------------------------- static PyMethodDef FileSystemMethods[] = { { "copy", (PyCFunction) PyXRootD::FileSystem::Copy, METH_VARARGS | METH_KEYWORDS, NULL }, { "locate", (PyCFunction) PyXRootD::FileSystem::Locate, METH_VARARGS | METH_KEYWORDS, NULL }, { "deeplocate", (PyCFunction) PyXRootD::FileSystem::DeepLocate, METH_VARARGS | METH_KEYWORDS, NULL }, { "mv", (PyCFunction) PyXRootD::FileSystem::Mv, METH_VARARGS | METH_KEYWORDS, NULL }, { "query", (PyCFunction) PyXRootD::FileSystem::Query, METH_VARARGS | METH_KEYWORDS, NULL }, { "truncate", (PyCFunction) PyXRootD::FileSystem::Truncate, METH_VARARGS | METH_KEYWORDS, NULL }, { "rm", (PyCFunction) PyXRootD::FileSystem::Rm, METH_VARARGS | METH_KEYWORDS, NULL }, { "mkdir", (PyCFunction) PyXRootD::FileSystem::MkDir, METH_VARARGS | METH_KEYWORDS, NULL }, { "rmdir", (PyCFunction) PyXRootD::FileSystem::RmDir, METH_VARARGS | METH_KEYWORDS, NULL }, { "chmod", (PyCFunction) PyXRootD::FileSystem::ChMod, METH_VARARGS | METH_KEYWORDS, NULL }, { "ping", (PyCFunction) PyXRootD::FileSystem::Ping, METH_VARARGS | METH_KEYWORDS, NULL }, { "stat", (PyCFunction) PyXRootD::FileSystem::Stat, METH_VARARGS | METH_KEYWORDS, NULL }, { "statvfs", (PyCFunction) PyXRootD::FileSystem::StatVFS, METH_VARARGS | METH_KEYWORDS, NULL }, { "protocol", (PyCFunction) PyXRootD::FileSystem::Protocol, METH_VARARGS | METH_KEYWORDS, NULL }, { "dirlist", (PyCFunction) PyXRootD::FileSystem::DirList, METH_VARARGS | METH_KEYWORDS, NULL }, { "sendinfo", (PyCFunction) PyXRootD::FileSystem::SendInfo, METH_VARARGS | METH_KEYWORDS, NULL }, { "prepare", (PyCFunction) PyXRootD::FileSystem::Prepare, METH_VARARGS | METH_KEYWORDS, NULL }, { "get_property", (PyCFunction) PyXRootD::FileSystem::GetProperty, METH_VARARGS | METH_KEYWORDS, NULL }, { "set_property", (PyCFunction) PyXRootD::FileSystem::SetProperty, METH_VARARGS | METH_KEYWORDS, NULL }, { "cat", (PyCFunction) PyXRootD::FileSystem::Cat, METH_VARARGS | METH_KEYWORDS, NULL }, { "set_xattr", (PyCFunction) PyXRootD::FileSystem::SetXAttr, METH_VARARGS | METH_KEYWORDS, NULL }, { "get_xattr", (PyCFunction) PyXRootD::FileSystem::GetXAttr, METH_VARARGS | METH_KEYWORDS, NULL }, { "del_xattr", (PyCFunction) PyXRootD::FileSystem::DelXAttr, METH_VARARGS | METH_KEYWORDS, NULL }, { "list_xattr", (PyCFunction) PyXRootD::FileSystem::ListXAttr, METH_VARARGS | METH_KEYWORDS, NULL }, { NULL } /* Sentinel */ }; //---------------------------------------------------------------------------- //! __init__() equivalent //---------------------------------------------------------------------------- static int FileSystem_init( FileSystem *self, PyObject *args ) { self->url = (URL *) PyObject_CallObject( (PyObject*) &URLType, args ); if ( !self->url ) return -1; self->filesystem = new XrdCl::FileSystem( *self->url->url ); return 0; } //---------------------------------------------------------------------------- //! Deallocation function, called when object is deleted //---------------------------------------------------------------------------- static void FileSystem_dealloc( FileSystem *self ) { delete self->filesystem; Py_XDECREF( self->url ); Py_TYPE(self)->tp_free( (PyObject*) self ); } //---------------------------------------------------------------------------- //! Visible member definitions //---------------------------------------------------------------------------- static PyMemberDef FileSystemMembers[] = { { const_cast("url"), T_OBJECT_EX, offsetof(FileSystem, url), 0, const_cast("Server URL") }, { NULL } /* Sentinel */ }; //---------------------------------------------------------------------------- //! FileSystem binding type object //---------------------------------------------------------------------------- static PyTypeObject FileSystemType = { PyVarObject_HEAD_INIT(NULL, 0) "pyxrootd.FileSystem", /* tp_name */ sizeof(FileSystem), /* tp_basicsize */ 0, /* tp_itemsize */ (destructor) FileSystem_dealloc, /* tp_dealloc */ 0, /* tp_print */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_compare */ 0, /* tp_repr */ 0, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ 0, /* tp_hash */ 0, /* tp_call */ 0, /* tp_str */ 0, /* tp_getattro */ 0, /* tp_setattro */ 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ filesystem_type_doc, /* tp_doc */ 0, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ 0, /* tp_iter */ 0, /* tp_iternext */ FileSystemMethods, /* tp_methods */ FileSystemMembers, /* tp_members */ 0, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ 0, /* tp_descr_set */ 0, /* tp_dictoffset */ (initproc) FileSystem_init, /* tp_init */ }; } #endif /* PYXROOTD_FILESYSTEM_HH_ */ xrootd-5.6.9/bindings/python/src/PyXRootDFinalize.hh000066400000000000000000000041271457266313600224730ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2012-2013 by European Organization for Nuclear Research (CERN) // Author: Michal Simon //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #ifndef BINDINGS_PYTHON_SRC_PYXROOTDFINALIZE_HH_ #define BINDINGS_PYTHON_SRC_PYXROOTDFINALIZE_HH_ #include "PyXRootD.hh" #include "XrdCl/XrdClDefaultEnv.hh" #include "XrdCl/XrdClPostMaster.hh" namespace PyXRootD { //---------------------------------------------------------------------------- //! Waits until XrdCl JobManager, TaskManager and Poller will stop gracefully. //! //! To be used in Python native atexit handler in order to make sure there are //! no outstanding threads after the Python interpreter got finalized. //---------------------------------------------------------------------------- PyObject* __XrdCl_Stop_Threads( PyObject *self, PyObject* ) { Py_BEGIN_ALLOW_THREADS XrdCl::DefaultEnv::GetPostMaster()->Stop(); Py_END_ALLOW_THREADS Py_RETURN_NONE; } } #endif /* BINDINGS_PYTHON_SRC_PYXROOTDFINALIZE_HH_ */ xrootd-5.6.9/bindings/python/src/PyXRootDModule.cc000066400000000000000000000121171457266313600221430ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2012-2013 by European Organization for Nuclear Research (CERN) // Author: Justin Salmon //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #include "PyXRootD.hh" #include "PyXRootDFileSystem.hh" #include "PyXRootDFile.hh" #include "PyXRootDCopyProcess.hh" #include "PyXRootDURL.hh" #include "PyXRootDFinalize.hh" #include "PyXRootDEnv.hh" namespace PyXRootD { // Global module object PyObject* ClientModule; PyDoc_STRVAR(client_module_doc, "XRootD Client extension module"); //---------------------------------------------------------------------------- //! Visible module-level method declarations //---------------------------------------------------------------------------- static PyMethodDef module_methods[] = { // The finalization routine used in atexit handler. { "__XrdCl_Stop_Threads", __XrdCl_Stop_Threads, METH_NOARGS, "Stop XrdCl threads." }, // Ths XRootD Env { "EnvPutString_cpp", EnvPutString_cpp, METH_VARARGS, "Puts a string into XrdCl environment." }, { "EnvGetString_cpp", EnvGetString_cpp, METH_VARARGS, "Gets a string from XrdCl environment." }, { "EnvPutInt_cpp", EnvPutInt_cpp, METH_VARARGS, "Puts an int into XrdCl environment." }, { "EnvGetInt_cpp", EnvGetInt_cpp, METH_VARARGS, "Gets an int from XrdCl environment." }, { "XrdVersion_cpp", XrdVersion_cpp, METH_VARARGS, "Get the XRootD client version." }, { "EnvGetDefault_cpp", EnvGetDefault_cpp, METH_VARARGS, "Get default values from XrdCl environment" }, { NULL, NULL, 0, NULL } }; #if PY_MAJOR_VERSION >= 3 //---------------------------------------------------------------------------- //! Module properties //---------------------------------------------------------------------------- static struct PyModuleDef moduledef = { PyModuleDef_HEAD_INIT, "client", /* m_name */ client_module_doc, /* m_doc */ -1, /* m_size */ module_methods, /* m_methods */ NULL, /* m_reload */ NULL, /* m_traverse */ NULL, /* m_clear */ NULL, /* m_free */ }; #endif //---------------------------------------------------------------------------- //! Module initialization function //---------------------------------------------------------------------------- #ifdef IS_PY3K PyMODINIT_FUNC PyInit_client( void ) #else PyMODINIT_FUNC initclient( void ) #endif { #if PY_VERSION_HEX < 0x03070000 // Ensure GIL state is initialized if ( !PyEval_ThreadsInitialized() ) { PyEval_InitThreads(); } #endif FileSystemType.tp_new = PyType_GenericNew; if ( PyType_Ready( &FileSystemType ) < 0 ) { #ifdef IS_PY3K return NULL; #else return; #endif } Py_INCREF( &FileSystemType ); FileType.tp_new = PyType_GenericNew; if ( PyType_Ready( &FileType ) < 0 ) { #ifdef IS_PY3K return NULL; #else return; #endif } Py_INCREF( &FileType ); URLType.tp_new = PyType_GenericNew; if ( PyType_Ready( &URLType ) < 0 ) { #ifdef IS_PY3K return NULL; #else return; #endif } Py_INCREF( &URLType ); CopyProcessType.tp_new = PyType_GenericNew; if ( PyType_Ready( &CopyProcessType ) < 0 ) { #ifdef IS_PY3K return NULL; #else return; #endif } Py_INCREF( &CopyProcessType ); #ifdef IS_PY3K ClientModule = PyModule_Create(&moduledef); #else ClientModule = Py_InitModule3("client", module_methods, client_module_doc); #endif if (ClientModule == NULL) { #ifdef IS_PY3K return NULL; #else return; #endif } PyModule_AddObject( ClientModule, "FileSystem", (PyObject *) &FileSystemType ); PyModule_AddObject( ClientModule, "File", (PyObject *) &FileType ); PyModule_AddObject( ClientModule, "URL", (PyObject *) &URLType ); PyModule_AddObject( ClientModule, "CopyProcess", (PyObject *) &CopyProcessType ); #ifdef IS_PY3K return ClientModule; #endif } } xrootd-5.6.9/bindings/python/src/PyXRootDURL.cc000066400000000000000000000167061457266313600213700ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2012-2013 by European Organization for Nuclear Research (CERN) // Author: Justin Salmon //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #include "PyXRootD.hh" #include "PyXRootDURL.hh" namespace PyXRootD { //---------------------------------------------------------------------------- //! Is the url valid //---------------------------------------------------------------------------- PyObject* URL::IsValid( URL *self ) { return PyBool_FromLong( self->url->IsValid() ); } //---------------------------------------------------------------------------- //! Get the host part of the URL (user:password\@host:port) //---------------------------------------------------------------------------- PyObject* URL::GetHostId( URL *self, void *closure ) { return PyUnicode_FromString( self->url->GetHostId().c_str() ); } //---------------------------------------------------------------------------- //! Get the protocol //---------------------------------------------------------------------------- PyObject* URL::GetProtocol( URL *self, void *closure ) { return PyUnicode_FromString( self->url->GetProtocol().c_str() ); } //---------------------------------------------------------------------------- //! Set protocol //---------------------------------------------------------------------------- int URL::SetProtocol( URL *self, PyObject *protocol, void *closure ) { if ( !PyUnicode_Check( protocol ) ) { PyErr_SetString( PyExc_TypeError, "protocol must be string" ); return -1; } self->url->SetProtocol( std::string ( PyUnicode_AsUTF8( protocol ) ) ); return 0; } //---------------------------------------------------------------------------- //! Get the username //---------------------------------------------------------------------------- PyObject* URL::GetUserName( URL *self, void *closure ) { return PyUnicode_FromString( self->url->GetUserName().c_str() ); } //---------------------------------------------------------------------------- //! Set the username //---------------------------------------------------------------------------- int URL::SetUserName( URL *self, PyObject *username, void *closure ) { if ( !PyUnicode_Check( username ) ) { PyErr_SetString( PyExc_TypeError, "username must be string" ); return -1; } self->url->SetUserName( std::string( PyUnicode_AsUTF8( username ) ) ); return 0; } //---------------------------------------------------------------------------- //! Get the password //---------------------------------------------------------------------------- PyObject* URL::GetPassword( URL *self, void *closure ) { return PyUnicode_FromString( self->url->GetPassword().c_str() ); } //---------------------------------------------------------------------------- //! Set the password //---------------------------------------------------------------------------- int URL::SetPassword( URL *self, PyObject *password, void *closure ) { if ( !PyUnicode_Check( password ) ) { PyErr_SetString( PyExc_TypeError, "password must be string" ); return -1; } self->url->SetPassword( std::string( PyUnicode_AsUTF8( password ) ) ); return 0; } //---------------------------------------------------------------------------- //! Get the name of the target host //---------------------------------------------------------------------------- PyObject* URL::GetHostName( URL *self, void *closure ) { return PyUnicode_FromString( self->url->GetHostName().c_str() ); } //---------------------------------------------------------------------------- //! Set the host name //---------------------------------------------------------------------------- int URL::SetHostName( URL *self, PyObject *hostname, void *closure ) { if ( !PyUnicode_Check( hostname ) ) { PyErr_SetString( PyExc_TypeError, "hostname must be string" ); return -1; } self->url->SetHostName( std::string( PyUnicode_AsUTF8( hostname ) ) ); return 0; } //---------------------------------------------------------------------------- //! Get the target port //---------------------------------------------------------------------------- PyObject* URL::GetPort( URL *self, void *closure ) { #ifdef IS_PY3K return PyLong_FromLong( self->url->GetPort() ); #else return PyInt_FromLong( self->url->GetPort() ); #endif } //---------------------------------------------------------------------------- //! Is the url valid //---------------------------------------------------------------------------- int URL::SetPort( URL *self, PyObject *port, void *closure ) { #ifdef IS_PY3K if ( !PyLong_Check( port ) ) { #else if ( !PyInt_Check( port ) ) { #endif PyErr_SetString( PyExc_TypeError, "port must be int" ); return -1; } #ifdef IS_PY3K self->url->SetPort( PyLong_AsLong( port ) ); #else self->url->SetPort( PyInt_AsLong( port ) ); #endif return 0; } //---------------------------------------------------------------------------- //! Get the path //---------------------------------------------------------------------------- PyObject* URL::GetPath( URL *self, void *closure ) { return PyUnicode_FromString( self->url->GetPath().c_str() ); } //---------------------------------------------------------------------------- //! Set the path //---------------------------------------------------------------------------- int URL::SetPath( URL *self, PyObject *path, void *closure ) { if ( !PyUnicode_Check( path ) ) { PyErr_SetString( PyExc_TypeError, "path must be string" ); return -1; } self->url->SetPath( std::string( PyUnicode_AsUTF8( path ) ) ); return 0; } //---------------------------------------------------------------------------- //! Get the path with params //---------------------------------------------------------------------------- PyObject* URL::GetPathWithParams( URL *self, void *closure ) { return PyUnicode_FromString( self->url->GetPathWithParams().c_str() ); } //---------------------------------------------------------------------------- //! Clear the url //---------------------------------------------------------------------------- PyObject* URL::Clear( URL *self ) { (void) URLType; // Suppress unused variable warning self->url->Clear(); Py_RETURN_NONE ; } } xrootd-5.6.9/bindings/python/src/PyXRootDURL.hh000066400000000000000000000175171457266313600214030ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2012-2013 by European Organization for Nuclear Research (CERN) // Author: Justin Salmon //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #ifndef PYXROOTD_URL_HH_ #define PYXROOTD_URL_HH_ #include "PyXRootD.hh" #include "XrdCl/XrdClURL.hh" namespace PyXRootD { //---------------------------------------------------------------------------- //! XrdCl::URL binding class //---------------------------------------------------------------------------- class URL { public: static PyObject* IsValid( URL *self ); static PyObject* GetHostId( URL *self, void *closure ); static PyObject* GetProtocol( URL *self, void *closure ); static int SetProtocol( URL *self, PyObject *protocol, void *closure ); static PyObject* GetUserName( URL *self, void *closure ); static int SetUserName( URL *self, PyObject *username, void *closure ); static PyObject* GetPassword( URL *self, void *closure ); static int SetPassword( URL *self, PyObject *password, void *closure ); static PyObject* GetHostName( URL *self, void *closure ); static int SetHostName( URL *self, PyObject *hostname, void *closure ); static PyObject* GetPort( URL *self, void *closure ); static int SetPort( URL *self, PyObject *port, void *closure ); static PyObject* GetPath( URL *self, void *closure ); static int SetPath( URL *self, PyObject *path, void *closure ); static PyObject* GetPathWithParams( URL *self, void *closure ); static PyObject* Clear( URL *self ); public: PyObject_HEAD XrdCl::URL *url; }; PyDoc_STRVAR(url_type_doc, "URL object (internal)"); //---------------------------------------------------------------------------- //! __init__() equivalent //---------------------------------------------------------------------------- static int URL_init( URL *self, PyObject *args ) { const char *url; if ( !PyArg_ParseTuple( args, "s", &url ) ) return -1; self->url = new XrdCl::URL( url ); return 0; } //---------------------------------------------------------------------------- //! Deallocation function, called when object is deleted //---------------------------------------------------------------------------- static void URL_dealloc( URL *self ) { delete self->url; Py_TYPE(self)->tp_free( (PyObject*) self ); } //---------------------------------------------------------------------------- //! __str__() equivalent //---------------------------------------------------------------------------- static PyObject* URL_str( URL *self ) { return PyUnicode_FromString( self->url->GetURL().c_str() ); } //---------------------------------------------------------------------------- //! Visible member definitions //---------------------------------------------------------------------------- static PyMemberDef URLMembers[] = { { NULL } /* Sentinel */ }; static PyGetSetDef URLGetSet[] = { { const_cast("hostid"), (getter) URL::GetHostId, NULL, NULL, NULL }, { const_cast("protocol"), (getter) URL::GetProtocol, (setter) URL::SetProtocol, NULL, NULL }, { const_cast("username"), (getter) URL::GetUserName, (setter) URL::SetUserName, NULL, NULL }, { const_cast("password"), (getter) URL::GetPassword, (setter) URL::SetPassword, NULL, NULL }, { const_cast("hostname"), (getter) URL::GetHostName, (setter) URL::SetHostName, NULL, NULL }, { const_cast("port"), (getter) URL::GetPort, (setter) URL::SetPort, NULL, NULL }, { const_cast("path"), (getter) URL::GetPath, (setter) URL::SetPath, NULL, NULL }, { const_cast("path_with_params"), (getter) URL::GetPathWithParams, NULL, NULL, NULL }, { NULL } /* Sentinel */ }; //---------------------------------------------------------------------------- //! Visible method definitions //---------------------------------------------------------------------------- static PyMethodDef URLMethods[] = { { "is_valid", (PyCFunction) URL::IsValid, METH_NOARGS, NULL }, { "clear", (PyCFunction) URL::Clear, METH_NOARGS, NULL }, { NULL } /* Sentinel */ }; //---------------------------------------------------------------------------- //! URL binding type object //---------------------------------------------------------------------------- static PyTypeObject URLType = { PyVarObject_HEAD_INIT(NULL, 0) "pyxrootd.URL", /* tp_name */ sizeof(URL), /* tp_basicsize */ 0, /* tp_itemsize */ (destructor) URL_dealloc, /* tp_dealloc */ 0, /* tp_print */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_compare */ 0, /* tp_repr */ 0, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ 0, /* tp_hash */ 0, /* tp_call */ (reprfunc) URL_str, /* tp_str */ 0, /* tp_getattro */ 0, /* tp_setattro */ 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ url_type_doc, /* tp_doc */ 0, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ 0, /* tp_iter */ 0, /* tp_iternext */ URLMethods, /* tp_methods */ URLMembers, /* tp_members */ URLGetSet, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ 0, /* tp_descr_set */ 0, /* tp_dictoffset */ (initproc) URL_init, /* tp_init */ }; } #endif /* PYXROOTD_URL_HH_ */ xrootd-5.6.9/bindings/python/src/Utils.cc000066400000000000000000000140001457266313600203760ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2012-2013 by European Organization for Nuclear Research (CERN) // Author: Justin Salmon //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #include "Utils.hh" #include "PyXRootDURL.hh" namespace PyXRootD { //---------------------------------------------------------------------------- // Check that the given callback is actually callable. //---------------------------------------------------------------------------- bool IsCallable( PyObject *callable ) { if ( !PyCallable_Check( callable ) ) { PyErr_SetString( PyExc_TypeError, "callback must be callable function, class or lambda" ); return false; } // We need to keep this callback Py_INCREF( callable ); return true; } //---------------------------------------------------------------------------- // Initialize the Python types for the extension. //---------------------------------------------------------------------------- int InitTypes() { URLType.tp_new = PyType_GenericNew; if ( PyType_Ready( &URLType ) < 0 ) return -1; Py_INCREF( &URLType ); return 0; } //---------------------------------------------------------------------------- // Convert PyInt to unsigned long (uint64_t) //---------------------------------------------------------------------------- int PyIntToUlong(PyObject *py_val, unsigned long *val, const char *name) { #ifdef IS_PY3K const long tmp = PyLong_AsLong(py_val); #else const long tmp = PyInt_AsLong(py_val); #endif if (tmp == -1 && PyErr_Occurred()) { if (PyErr_ExceptionMatches(PyExc_OverflowError)) PyErr_Format(PyExc_OverflowError, "%s too big for unsigned long", name); return -1; } if (tmp < 0) { PyErr_Format(PyExc_OverflowError, "negative %s cannot be converted to unsigned long", name); return -1; } *val = tmp; return 0; } //---------------------------------------------------------------------------- // Convert Python object to unsigned long (uint64_t) //---------------------------------------------------------------------------- int PyObjToUlong(PyObject *py_val, unsigned long *val, const char *name) { #ifdef IS_PY3K if (PyLong_Check(py_val)) #else if (PyInt_Check(py_val)) #endif return PyIntToUlong(py_val, val, name); if (!PyLong_Check(py_val)) { PyErr_Format(PyExc_TypeError, "expected integer %s", name); return -1; } const unsigned long tmp = PyLong_AsUnsignedLong(py_val); if (PyErr_Occurred()) { if (PyErr_ExceptionMatches(PyExc_OverflowError)) PyErr_Format(PyExc_OverflowError, "%s too big for unsigned long", name); return -1; } *val = tmp; return 0; } //---------------------------------------------------------------------------- // Convert Python object to unsigned int (uint32_t) //---------------------------------------------------------------------------- int PyObjToUint(PyObject *py_val, unsigned int *val, const char *name) { unsigned long tmp; if (PyObjToUlong(py_val, &tmp, name)) return -1; if (tmp > UINT_MAX) { PyErr_Format(PyExc_OverflowError, "%s too big for unsigned int (uint32_t)", name); return -1; } *val = tmp; return 0; } //---------------------------------------------------------------------------- // Convert Python object to unsigned short int (uint16_t) //---------------------------------------------------------------------------- int PyObjToUshrt(PyObject *py_val, unsigned short int *val, const char *name) { unsigned int tmp; if (PyObjToUint(py_val, &tmp, name)) return -1; if (tmp > USHRT_MAX) { PyErr_Format(PyExc_OverflowError, "%s too big for unsigned short int " "(uint16_t)", name); return -1; } *val = tmp; return 0; } //---------------------------------------------------------------------------- // Convert Python object to unsigned long long (uint64_t) //---------------------------------------------------------------------------- int PyObjToUllong(PyObject *py_val, unsigned PY_LONG_LONG *val, const char *name) { #ifdef IS_PY3K if (PyLong_Check(py_val)) #else if (PyInt_Check(py_val)) #endif { unsigned long tmp; if (!PyIntToUlong(py_val, &tmp, name)) { *val = tmp; return 0; } return -1; } if (!PyLong_Check(py_val)) { PyErr_Format(PyExc_TypeError, "integer argument expected for %s", name); return -1; } const unsigned PY_LONG_LONG tmp = PyLong_AsUnsignedLongLong(py_val); if ((tmp == (unsigned long long) -1) && PyErr_Occurred()) { if (PyErr_ExceptionMatches(PyExc_OverflowError)) PyErr_Format(PyExc_OverflowError, "%s too big for unsigned long long", name); return -1; } *val = tmp; return 0; } } xrootd-5.6.9/bindings/python/src/Utils.hh000066400000000000000000000103051457266313600204140ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2012-2013 by European Organization for Nuclear Research (CERN) // Author: Justin Salmon //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #ifndef UTILS_HH_ #define UTILS_HH_ #include "PyXRootD.hh" #include "XrdCl/XrdClXRootDResponses.hh" namespace PyXRootD { //---------------------------------------------------------------------------- //! Check that the given callback is actually callable. //---------------------------------------------------------------------------- bool IsCallable( PyObject *callable ); //---------------------------------------------------------------------------- //! Initialize the Python types for the extension. //---------------------------------------------------------------------------- int InitTypes(); //---------------------------------------------------------------------------- //! Convert PyInt to unsigned long (uint64_t) //! //! @param py_val python object to be converted //! @param val converted value //! @param name name of the object //! //! return 0 if successful, otherwise failed //---------------------------------------------------------------------------- int PyIntToUlong(PyObject *py_val, unsigned long *val, const char *name); //---------------------------------------------------------------------------- //! Convert Python object to unsigned long (uint64_t) //! //! @param py_val python object to be converted //! @param val converted value //! @param name name of the object //! //! return 0 if successful, otherwise failed //---------------------------------------------------------------------------- int PyObjToUlong(PyObject *py_val, unsigned long *val, const char *name); //---------------------------------------------------------------------------- //! Convert Python object to unsigned int (uint32_t) //! //! @param py_val python object to be converted //! @param val converted value //! @param name name of the object //! //! return 0 if successful, otherwise failed //---------------------------------------------------------------------------- int PyObjToUint(PyObject *py_val, unsigned int *val, const char *name); //---------------------------------------------------------------------------- //! Convert Python object to unsigned short int (uint16_t) //! //! @param py_val python object to be converted //! @param val converted value //! @param name name of the object //! //! return 0 if successful, otherwise failed //---------------------------------------------------------------------------- int PyObjToUshrt(PyObject *py_val, unsigned short int *val, const char *name); //---------------------------------------------------------------------------- //! Convert Python object to unsigned long long (uint64_t) //! //! @param py_val python object to be converted //! @param val converted value //! @param name name of the object //! //! return 0 if successful, otherwise failed //---------------------------------------------------------------------------- int PyObjToUllong(PyObject *py_val, unsigned PY_LONG_LONG *val, const char *name); } #endif /* UTILS_HH_ */ xrootd-5.6.9/bindings/python/src/__init__.py000066400000000000000000000000001457266313600210730ustar00rootroot00000000000000xrootd-5.6.9/bindings/python/tests/000077500000000000000000000000001457266313600173475ustar00rootroot00000000000000xrootd-5.6.9/bindings/python/tests/env.py000066400000000000000000000003551457266313600205140ustar00rootroot00000000000000SERVER_URL = 'root://localhost/' smallfile = SERVER_URL + '/tmp/spam' smallcopy = SERVER_URL + '/tmp/eggs' smallbuffer = 'gre\0en\neggs\nand\nham\n' bigfile = SERVER_URL + '/tmp/bigfile' bigcopy = SERVER_URL + '/tmp/bigcopy'xrootd-5.6.9/bindings/python/tests/test_copy.py000066400000000000000000000034571457266313600217430ustar00rootroot00000000000000from XRootD import client from XRootD.client.flags import OpenFlags from env import * def test_copy_smallfile(): f = client.File() s, r = f.open(smallfile, OpenFlags.DELETE ) assert s.ok f.write(smallbuffer) size1 = f.stat(force=True)[1].size f.close() c = client.CopyProcess() c.add_job( source=smallfile, target=smallcopy, force=True ) s = c.prepare() assert s.ok s, __ = c.run() assert s.ok f = client.File() s, r = f.open(smallcopy, OpenFlags.READ) size2 = f.stat()[1].size assert size1 == size2 f.close() def test_copy_bigfile(): f = client.File() s, r = f.open(bigfile) assert s.ok size1 = f.stat(force=True)[1].size f.close() c = client.CopyProcess() c.add_job( source=bigfile, target=bigcopy, force=True ) s = c.prepare() assert s.ok s, __ = c.run() assert s.ok f = client.File() s, r = f.open(bigcopy, OpenFlags.READ) size2 = f.stat()[1].size assert size1 == size2 f.close() def test_copy_nojobs(): c = client.CopyProcess() s = c.prepare() assert s.ok s, __ = c.run() assert s.ok def test_copy_noprep(): c = client.CopyProcess() c.add_job( source=bigfile, target=bigcopy, force=True ) s, __ = c.run() assert s.ok class TestProgressHandler(object): def begin(self, id, total, source, target): print '+++ begin(): %d, total: %d' % (id, total) print '+++ source: %s' % source print '+++ target: %s' % target def end(self, jobId, status): print '+++ end(): jobId: %s, status: %s' % (jobId, status) def update(self, jobId, processed, total): print '+++ update(): jobid: %s, processed: %d, total: %d' % (jobId, processed, total) def test_copy_progress_handler(): c = client.CopyProcess() c.add_job( source=bigfile, target=bigcopy, force=True ) c.prepare() h = TestProgressHandler() c.run(handler=h) xrootd-5.6.9/bindings/python/tests/test_file.py000066400000000000000000000270251457266313600217050ustar00rootroot00000000000000from XRootD import client from XRootD.client.utils import AsyncResponseHandler from XRootD.client.flags import OpenFlags, AccessMode from env import * import pytest import sys import os # Global open mode open_mode = (AccessMode.UR | AccessMode.UW | AccessMode.GR | AccessMode.GW | AccessMode.OR | AccessMode.OW) def test_write_sync(): f = client.File() pytest.raises(ValueError, "f.write(smallbuffer)") status, __ = f.open(smallfile, OpenFlags.DELETE, open_mode ) assert status.ok # Write status, __ = f.write(smallbuffer) assert status.ok # Read status, response = f.read() assert status.ok assert len(response) == len(smallbuffer) buffer = 'eggs and ham\n' status, __ = f.write(buffer, offset=13, size=len(buffer) - 2) assert status.ok status, response = f.read() assert status.ok assert len(response) == len(buffer * 2) - 2 f.close() def test_write_async(): f = client.File() status, __ = f.open(smallfile, OpenFlags.DELETE, open_mode) assert status.ok # Write async handler = AsyncResponseHandler() status = f.write(smallbuffer, callback=handler) status, __, __ = handler.wait() assert status.ok # Read sync status, response = f.read() assert status.ok assert len(response) == len(smallbuffer) f.close() def test_open_close_sync(): f = client.File() pytest.raises(ValueError, "f.stat()") status, __ = f.open(smallfile, OpenFlags.READ) assert status.ok assert f.is_open() # Close status, __ = f.close() assert status.ok assert f.is_open() == False pytest.raises(ValueError, "f.stat()") f.close() f.close() def test_open_close_async(): f = client.File() handler = AsyncResponseHandler() status = f.open(smallfile, OpenFlags.READ, callback=handler) assert status.ok status, __, __ = handler.wait() assert status.ok assert f.is_open() # Close async handler = AsyncResponseHandler() status = f.close(callback=handler) assert status.ok status, __, __ = handler.wait() assert status.ok assert f.is_open() == False def test_io_limits(): f = client.File() pytest.raises(ValueError, 'f.read()') status, __ = f.open(smallfile, OpenFlags.UPDATE) assert status.ok status, __ = f.stat() assert status.ok # Test read limits pytest.raises(TypeError, 'f.read(0, [1, 2])') pytest.raises(TypeError, 'f.read([1, 2], 0)') pytest.raises(TypeError, 'f.read(0, 10, [0, 1, 2])') pytest.raises(OverflowError, 'f.read(0, -10)') pytest.raises(OverflowError, 'f.read(-1, 1)') pytest.raises(OverflowError, 'f.read(0, 1, -1)') pytest.raises(OverflowError, 'f.read(0, 10**11)') pytest.raises(OverflowError, 'f.read(0, 10, 10**6)') # Test readline limits pytest.raises(TypeError, 'f.readline([0, 1], 1)') pytest.raises(TypeError, 'f.readline(0, [0, 1])') pytest.raises(TypeError, 'f.readline(0, 10, [0, 1])') pytest.raises(OverflowError, 'f.readline(-1, 1)') pytest.raises(OverflowError, 'f.readline(0, -1)') pytest.raises(OverflowError, 'f.readline(0, 1, -1)') pytest.raises(OverflowError, 'f.readline(0, 10**11)') pytest.raises(OverflowError, 'f.readline(0, 10, 10**11)') # Test write limits data = "data that will never get written" pytest.raises(TypeError, 'f.write(data, 0, [1, 2])') pytest.raises(TypeError, 'f.write(data, [1, 2], 0)') pytest.raises(TypeError, 'f.write(data, 0, 10, [0, 1, 2])') pytest.raises(OverflowError, 'f.write(data, 0, -10)') pytest.raises(OverflowError, 'f.write(data, -1, 1)') pytest.raises(OverflowError, 'f.write(data, 0, 1, -1)') pytest.raises(OverflowError, 'f.write(data, 0, 10**11)') pytest.raises(OverflowError, 'f.write(data, 0, 10, 10**6)') # Test vector_read limits pytest.raises(TypeError, 'f.vector_read(chunks=100)') pytest.raises(TypeError, 'f.vector_read(chunks=[1,2,3])') pytest.raises(TypeError, 'f.vector_read(chunks=[("lol", "cakes")])') pytest.raises(TypeError, 'f.vector_read(chunks=[(1), (2)])') pytest.raises(TypeError, 'f.vector_read(chunks=[(1, 2), (3)])') pytest.raises(OverflowError, 'f.vector_read(chunks=[(-1, -100), (-100, -100)])') pytest.raises(OverflowError, 'f.vector_read(chunks=[(0, 10**10*10)])') # Test truncate limits pytest.raises(TypeError, 'f.truncate(0, [1, 2])') pytest.raises(TypeError, 'f.truncate([1, 2], 0)') pytest.raises(OverflowError, 'f.truncate(-1)') pytest.raises(OverflowError, 'f.truncate(100, -10)') pytest.raises(OverflowError, 'f.truncate(0, 10**6)') status, __ = f.close() assert status.ok def test_write_big_async(): f = client.File() pytest.raises(ValueError, 'f.read()') status, __ = f.open(bigfile, OpenFlags.DELETE, open_mode) assert status.ok rand_data = os.urandom(64 * 1024) max_size = 512 * 1024 # 512 K offset = 0 lst_handlers = [] while offset <= max_size: status, __ = f.write(smallbuffer) assert status.ok handler = AsyncResponseHandler() lst_handlers.append(handler) status = f.write(rand_data, offset, callback=handler) assert status.ok offset = offset + len(smallbuffer) + len(rand_data) # Wait for async write responses for handler in lst_handlers: status, __, __ = handler.wait() assert status.ok f.close() def test_read_sync(): f = client.File() pytest.raises(ValueError, 'f.read()') status, response = f.open(bigfile, OpenFlags.READ) assert status.ok status, response = f.stat() size = response.size status, response = f.read() assert status.ok assert len(response) == size f.close() def test_read_async(): f = client.File() status, response = f.open(bigfile, OpenFlags.READ) assert status.ok status, response = f.stat() size = response.size handler = AsyncResponseHandler() status = f.read(callback=handler) assert status.ok status, response, hostlist = handler.wait() assert status.ok assert len(response) == size f.close() def test_iter_small(): f = client.File() status, __ = f.open(smallfile, OpenFlags.DELETE) assert status.ok status, __ = f.write(smallbuffer) assert status.ok size = f.stat(force=True)[1].size pylines = open('/tmp/spam').readlines() total = 0 for i, line in enumerate(f): total += len(line) if pylines[i].endswith('\n'): assert line.endswith('\n') assert total == size f.close() def test_iter_big(): f = client.File() status, __ = f.open(bigfile, OpenFlags.READ) assert status.ok size = f.stat()[1].size pylines = open('/tmp/bigfile').readlines() total = 0 for i, line in enumerate(f): total += len(line) if pylines[i].endswith('\n'): assert line.endswith('\n') assert total == size f.close() def test_readline(): f = client.File() f.open(smallfile, OpenFlags.DELETE, open_mode) f.write(smallbuffer) response = f.readline(offset=0, size=100) assert response == 'gre\0en\n' response = f.readline() assert response == 'eggs\n' response = f.readline() assert response == 'and\n' response = f.readline() assert response == 'ham\n' f.close() f = client.File() f.open(smallfile, OpenFlags.DELETE, open_mode) f.write(smallbuffer[:-1]) f.readline() f.readline() f.readline() response = f.readline() assert response == 'ham' f.close() def test_readlines_small(): f = client.File() f.open(smallfile, OpenFlags.DELETE, open_mode) f.write(smallbuffer) f.close() pylines = open('/tmp/spam').readlines() for i in range(1, 100): f = client.File() f.open(smallfile) response = f.readlines(offset=0, chunksize=i) assert len(response) == 4 for j, line in enumerate(response): if pylines[j].endswith('\n'): assert line.endswith('\n') f.close() def test_readlines_big(): f = client.File() f.open(bigfile, OpenFlags.READ) size = f.stat()[1].size lines = f.readlines() pylines = open('/tmp/bigfile').readlines() assert len(lines) == len(pylines) nlines = len(pylines) total = 0 for i, l in enumerate(lines): total += len(l) if l != pylines[i]: print '!!!!!', total, i print '+++++ py: %r' % pylines[i] print '+++++ me: %r' % l break if pylines[i].endswith('\n'): assert l.endswith('\n') assert total == size f.close() def test_readchunks_small(): f = client.File() f.open(smallfile, OpenFlags.READ) size = f.stat()[1].size total = 0 chunks = ['gre', '\0en', '\neg', 'gs\n', 'and', '\nha', 'm\n'] for i, chunk in enumerate(f.readchunks(chunksize=3)): assert chunk == chunks[i] total += len(chunk) assert total == size f.close() def test_readchunks_big(): f = client.File() f.open(bigfile, OpenFlags.READ) size = f.stat()[1].size total = 0 for chunk in f.readchunks(chunksize=1024 * 1024 * 2): total += len(chunk) assert total == size f.close() def test_vector_read_sync(): v = [(0, 100), (101, 200), (201, 200)] vlen = sum([vec[1] for vec in v]) f = client.File() status, __ = f.open(bigfile, OpenFlags.READ) assert status.ok status, stat_info = f.stat() assert status.ok status, response = f.vector_read(chunks=v) # If big enough file everything shoud be ok if (stat_info.size > max([off + sz for (off, sz) in v])): assert status.ok assert response.size == vlen else: # If file not big enough this should fail status, __ = f.vector_read(chunks=v) assert not status.ok f.close() def test_vector_read_async(): v = [(0, 100), (101, 200), (201, 200)] vlen = sum([vec[1] for vec in v]) f = client.File() status, __ = f.open(bigfile, OpenFlags.READ) assert status.ok status, stat_info = f.stat() assert status.ok handler = AsyncResponseHandler() status = f.vector_read(chunks=v, callback=handler) assert status.ok # If big enough file everything shoud be ok if (stat_info.size > max([off + sz for (off, sz) in v])): status, response, hostlist = handler.wait() assert status.ok assert response.size == vlen else: status, __, __ = handler.wait() assert not status.ok f.close() def test_stat_sync(): f = client.File() pytest.raises(ValueError, 'f.stat()') status, __ = f.open(bigfile) assert status.ok status, __ = f.stat() assert status.ok f.close() def test_stat_async(): f = client.File() status, response = f.open(bigfile) assert status.ok handler = AsyncResponseHandler() status = f.stat(callback=handler) assert status.ok status, __, __ = handler.wait() assert status.ok f.close() def test_sync_sync(): f = client.File() pytest.raises(ValueError, 'f.sync()') status, __ = f.open(bigfile) assert status.ok status, __ = f.sync() assert status.ok f.close() def test_sync_async(): f = client.File() status, response = f.open(bigfile) assert status.ok handler = AsyncResponseHandler() status = f.sync(callback=handler) status, __, __ = handler.wait() assert status.ok f.close() def test_truncate_sync(): f = client.File() pytest.raises(ValueError, 'f.truncate(10000)') status, __ = f.open(smallfile, OpenFlags.DELETE) assert status.ok status, __ = f.truncate(size=10000) assert status.ok f.close() def test_truncate_async(): f = client.File() status, __ = f.open(smallfile, OpenFlags.DELETE) assert status.ok handler = AsyncResponseHandler() status = f.truncate(size=10000, callback=handler) assert status.ok status, __, __ = handler.wait() assert status.ok f.close() def test_misc(): f = client.File() assert not f.is_open() # Open status, response = f.open(smallfile, OpenFlags.READ) assert status.ok assert f.is_open() # Set/get file properties f.set_property("ReadRecovery", "true") f.set_property("WriteRecovery", "true") assert f.get_property("DataServer") # Testing context manager f.close() assert not f.is_open() xrootd-5.6.9/bindings/python/tests/test_filesystem.py000066400000000000000000000124131457266313600231450ustar00rootroot00000000000000from XRootD import client from XRootD.client.utils import AsyncResponseHandler from XRootD.client.flags import OpenFlags, QueryCode, MkDirFlags, AccessMode, \ DirListFlags, PrepareFlags from env import * import pytest import sys import os import inspect def test_filesystem(): c = client.FileSystem(SERVER_URL) funcspecs = [(c.locate, ('/tmp', OpenFlags.REFRESH), True), (c.deeplocate, ('/tmp', OpenFlags.REFRESH), True), (c.query, (QueryCode.SPACE, '/tmp'), True), (c.truncate, ('/tmp/spam', 1000), False), (c.mv, ('/tmp/spam', '/tmp/ham'), False), (c.chmod, ('/tmp/ham', AccessMode.UR | AccessMode.UW), False), (c.rm, ('/tmp/ham',), False), (c.mkdir, ('/tmp/somedir', MkDirFlags.MAKEPATH), False), (c.rmdir, ('/tmp/somedir',), False), (c.ping, (), False), (c.stat, ('/tmp',), True), (c.statvfs, ('/tmp',), True), (c.protocol, (), True), (c.dirlist, ('/tmp', DirListFlags.STAT), True), (c.sendinfo, ('important info',), False), (c.prepare, (['/tmp/foo'], PrepareFlags.STAGE), True), ] for func, args, hasReturnObject in funcspecs: sync (func, args, hasReturnObject) # Create new temp file f = client.File() status, response = f.open(smallfile, OpenFlags.NEW) for func, args, hasReturnObject in funcspecs: async(func, args, hasReturnObject) def sync(func, args, hasReturnObject): status, response = func(*args) print status assert status.ok if hasReturnObject: print response assert response def async(func, args, hasReturnObject): handler = AsyncResponseHandler() status = func(callback=handler, *args) print status assert status.ok status, response, hostlist = handler.wait() assert status.ok if response: assert response for host in hostlist: assert host.url print host.url if hasReturnObject: assert response def test_copy_sync(): c = client.FileSystem(SERVER_URL) f = client.File() status, response = f.open(smallfile, OpenFlags.DELETE) assert status.ok status, response = c.copy(smallfile, '/tmp/eggs', force=True) assert status.ok status, response = c.copy('/tmp/nonexistent', '/tmp/eggs') assert not status.ok try: os.remove('/tmp/eggs') except OSError, __: pass def test_locate_sync(): c = client.FileSystem(SERVER_URL) status, response = c.locate('/tmp', OpenFlags.REFRESH) assert status.ok for item in response: assert item def test_locate_async(): c = client.FileSystem(SERVER_URL) handler = AsyncResponseHandler() response = c.locate('/tmp', OpenFlags.REFRESH, callback=handler) status, response, hostlist = handler.wait() assert status.ok for item in response: assert item def test_deeplocate_sync(): c = client.FileSystem(SERVER_URL) status, response = c.deeplocate('/tmp', OpenFlags.REFRESH) assert status.ok for item in response: assert item def test_deeplocate_async(): c = client.FileSystem(SERVER_URL) handler = AsyncResponseHandler() response = c.deeplocate('/tmp', OpenFlags.REFRESH, callback=handler) status, response, hostlist = handler.wait() assert status.ok for item in response: assert item def test_dirlist_sync(): c = client.FileSystem(SERVER_URL) status, response = c.dirlist('/tmp', DirListFlags.STAT) assert status.ok for item in response: assert item.name print item.statinfo assert item.statinfo assert item.hostaddr status, response = c.dirlist('invalid', DirListFlags.STAT) assert not status.ok def test_dirlist_async(): c = client.FileSystem(SERVER_URL) handler = AsyncResponseHandler() status = c.dirlist('/tmp', DirListFlags.STAT, callback=handler) assert status.ok status, response, hostlist = handler.wait() assert status.ok for h in hostlist: print h.url for item in response: assert item.name print item.statinfo assert item.statinfo assert item.hostaddr assert hostlist def test_query_sync(): c = client.FileSystem(SERVER_URL) status, response = c.query(QueryCode.STATS, 'a') assert status.ok assert response print response def test_query_async(): c = client.FileSystem(SERVER_URL) handler = AsyncResponseHandler() status = c.query(QueryCode.STATS, 'a', callback=handler) assert status.ok status, response, hostlist = handler.wait() assert status.ok assert response print response def test_mkdir_flags(): c = client.FileSystem(SERVER_URL) status, response = c.mkdir('/tmp/dir1/dir2', MkDirFlags.MAKEPATH) assert status.ok c.rm('/tmp/dir1/dir2') c.rm('/tmp/dir1') def test_args(): c = client.FileSystem(url=SERVER_URL) assert c pytest.raises(TypeError, "c = client.FileSystem(foo='root://localhost')") pytest.raises(TypeError, "c = client.FileSystem(path='root://localhost', foo='bar')") def test_creation(): c = client.FileSystem(SERVER_URL) assert c.url is not None def test_deletion(): c = client.FileSystem(SERVER_URL) del c if sys.hexversion > 0x03000000: pytest.raises(UnboundLocalError, 'assert c') else: pytest.raises(NameError, 'assert c') xrootd-5.6.9/bindings/python/tests/test_glob.py000066400000000000000000000033001457266313600216770ustar00rootroot00000000000000import pytest import os import glob as norm_glob import XRootD.client.glob_funcs as glob from pathlib2 import Path @pytest.fixture def tmptree(tmpdir): subdir1 = tmpdir / "subdir1" subdir1.mkdir() subdir2 = tmpdir / "subdir2" subdir2.mkdir() for i in range(3): dummy = subdir1 / ("a_file_%d.txt" % i) dummy.write_text(u"This is file %d\n" % i, encoding="utf-8") return tmpdir def test_glob_local(tmptree): normal_glob_result = norm_glob.glob(str(tmptree / "not-there")) assert glob.glob(str(tmptree / "not-there")) == normal_glob_result assert len(glob.glob(str(tmptree / "not-there"))) == 0 assert len(glob.glob(str(tmptree / "not-there*"))) == 0 assert len(glob.glob(str(tmptree / "sub*"))) == 2 assert len(glob.glob(str(tmptree / "subdir1" / "*txt"))) == 3 assert len(glob.glob(str(tmptree / "subdir*" / "*txt"))) == 3 with pytest.raises(RuntimeError) as excinfo: glob.glob(str(tmptree / "not-there"), raise_error=True) assert "[ERROR]" in str(excinfo.value) assert str(tmptree) in str(excinfo.value) def test_glob_remote(tmptree): assert len(glob.glob("root://eospublic.cern.ch//eos/root-eos/cms_opendata_2012_nanoad/")) == 0 assert len(glob.glob("root://eospublic.cern.ch//eos/root-eos/cms_opendata_2012_nanoa*")) == 1 assert len(glob.glob("root://eospublic.cern.ch//eos/root-eos/cms_opendata_2012_nanoaod/*")) > 0 assert len(glob.glob("root://eospublic.cern.ch//eos/root-*/cms_opendata_2012_nanoaod/*")) > 0 with pytest.raises(RuntimeError) as excinfo: glob.glob("root://eospublic.cern.ch//eos/root-NOTREAL/cms_opendata_2012_nanoaod/*", raise_error=True) assert "[ERROR]" in str(excinfo.value) xrootd-5.6.9/bindings/python/tests/test_threads.py000066400000000000000000000014211457266313600224100ustar00rootroot00000000000000from XRootD import client from XRootD.client.flags import OpenFlags from threading import Thread from env import * class TestThread(Thread): def __init__(self, file, id): Thread.__init__(self) self.file = file self.id = id def run(self): self.file.open(smallfile, OpenFlags.DELETE) assert self.file.is_open() s, _ = self.file.write(smallbuffer) assert s.ok print '+++ thread %d says: %s' % (self.id, self.file.read()) for line in self.file: print '+++ thread %d says: %s' % (self.id, line) self.file.close() def test_threads(): f = client.File() # f.open(smallfile, OpenFlags.DELETE) # assert f.is_open() # f.write(smallbuffer) for i in xrange(3): tt = TestThread(f, i) tt.start() tt.join() # f.close() xrootd-5.6.9/bindings/python/tests/test_url.py000066400000000000000000000024741457266313600215710ustar00rootroot00000000000000from XRootD import client import pytest, sys from env import * def test_creation(): u = client.FileSystem(SERVER_URL).url assert u is not None def test_deletion(): u = client.FileSystem(SERVER_URL).url del u if sys.hexversion > 0x03000000: pytest.raises(UnboundLocalError, 'assert u') else: pytest.raises(NameError, 'assert u') def test_valid(): u = client.FileSystem(SERVER_URL).url assert u.is_valid() def test_invalid(): u = client.FileSystem('root://').url assert u.is_valid() == False def test_getters(): u = client.FileSystem("root://user1:passwd1@host1:123//path?param1=val1¶m2=val2").url assert u.is_valid() assert u.hostid == 'user1:passwd1@host1:123' assert u.protocol == 'root' assert u.username == 'user1' assert u.password == 'passwd1' assert u.hostname == 'host1' assert u.port == 123 assert u.path == '/path' assert u.path_with_params == '/path?param1=val1¶m2=val2' def test_setters(): u = client.FileSystem(SERVER_URL).url u.protocol = 'root' assert u.protocol == 'root' u.username = 'user1' assert u.username == 'user1' u.password = 'passwd1' assert u.password == 'passwd1' u.hostname = 'host1' assert u.hostname == 'host1' u.port = 123 assert u.port == 123 u.path = '/path' assert u.path == '/path' u.clear() assert str(u) == '' xrootd-5.6.9/cmake/000077500000000000000000000000001457266313600141475ustar00rootroot00000000000000xrootd-5.6.9/cmake/FindAutoConf.cmake000066400000000000000000000004001457266313600174620ustar00rootroot00000000000000FIND_PROGRAM( AUTOCONF_FOUND autoconf HINTS ${AUTOCONF_DIR} /usr PATH_SUFFIXES bin ) INCLUDE( FindPackageHandleStandardArgs ) FIND_PACKAGE_HANDLE_STANDARD_ARGS( AutoConf DEFAULT_MSG AUTOCONF_FOUND ) xrootd-5.6.9/cmake/FindAutoMake.cmake000066400000000000000000000004001457266313600174520ustar00rootroot00000000000000FIND_PROGRAM( AUTOMAKE_FOUND automake HINTS ${AUTOMAKE_DIR} /usr PATH_SUFFIXES bin ) INCLUDE( FindPackageHandleStandardArgs ) FIND_PACKAGE_HANDLE_STANDARD_ARGS( AutoMake DEFAULT_MSG AUTOMAKE_FOUND ) xrootd-5.6.9/cmake/FindCppUnit.cmake000066400000000000000000000012671457266313600173420ustar00rootroot00000000000000# Try to find CPPUnit # Once done, this will define # # CPPUNIT_FOUND - system has cppunit # CPPUNIT_INCLUDE_DIRS - the cppunit include directories # CPPUNIT_LIBRARIES - cppunit libraries directories find_path( CPPUNIT_INCLUDE_DIRS cppunit/ui/text/TestRunner.h HINTS ${CPPUNIT_DIR} $ENV{CPPUNIT_DIR} /usr /opt PATH_SUFFIXES include ) find_library( CPPUNIT_LIBRARIES cppunit HINTS ${CPPUNIT_DIR} $ENV{CPPUNIT_DIR} /usr /opt PATH_SUFFIXES lib ) set(CPPUNIT_INCLUDE_DIRS ${CPPUNIT_INCLUDE_DIR}) set(CPPUNIT_LIBRARIES ${CPPUNIT_LIBRARY}) include(FindPackageHandleStandardArgs) find_package_handle_standard_args(CppUnit DEFAULT_MSG CPPUNIT_INCLUDE_DIRS CPPUNIT_LIBRARIES) xrootd-5.6.9/cmake/FindDavix.cmake000066400000000000000000000034301457266313600170250ustar00rootroot00000000000000#.rst: # FindDavix # ------- # # Find Davix library for file management over HTTP-based protocols. # # Imported Targets # ^^^^^^^^^^^^^^^^ # # This module defines :prop_tgt:`IMPORTED` target: # # ``Davix::Davix`` # The libdavix library, if found. # # Result Variables # ^^^^^^^^^^^^^^^^ # # This module will set the following variables in your project: # # ``DAVIX_FOUND`` # True if Davix has been found. # ``DAVIX_INCLUDE_DIRS`` # Where to find davix.hpp, etc. # ``DAVIX_LIBRARIES`` # The libraries to link against to use Davix. # ``DAVIX_VERSION`` # The version of the Davix library found (e.g. 0.6.4) # # Obsolete variables # ^^^^^^^^^^^^^^^^^^ # # The following variables may also be set, for backwards compatibility: # # ``DAVIX_LIBRARY`` # where to find the DAVIX library. # ``DAVIX_INCLUDE_DIR`` # where to find the DAVIX headers (same as DAVIX_INCLUDE_DIRS) # foreach(var FOUND INCLUDE_DIR INCLUDE_DIRS LIBRARY LIBRARIES) unset(DAVIX_${var} CACHE) endforeach() find_package(PkgConfig) if(PKG_CONFIG_FOUND) if(${Davix_FIND_REQUIRED}) set(Davix_REQUIRED REQUIRED) endif() if(NOT DEFINED Davix_FIND_VERSION) pkg_check_modules(DAVIX ${Davix_REQUIRED} davix) else() pkg_check_modules(DAVIX ${Davix_REQUIRED} davix>=${Davix_FIND_VERSION}) endif() set(DAVIX_LIBRARIES ${DAVIX_LDFLAGS}) set(DAVIX_LIBRARY ${DAVIX_LIBRARIES}) set(DAVIX_INCLUDE_DIRS ${DAVIX_INCLUDE_DIRS}) set(DAVIX_INCLUDE_DIR ${DAVIX_INCLUDE_DIRS}) endif() if(DAVIX_FOUND AND NOT TARGET Davix::Davix) add_library(Davix::Davix INTERFACE IMPORTED) set_property(TARGET Davix::Davix PROPERTY INTERFACE_INCLUDE_DIRECTORIES "${DAVIX_INCLUDE_DIRS}") set_property(TARGET Davix::Davix PROPERTY INTERFACE_LINK_LIBRARIES "${DAVIX_LIBRARIES}") endif() mark_as_advanced(DAVIX_INCLUDE_DIR DAVIX_LIBRARY) xrootd-5.6.9/cmake/FindKerberos5.cmake000066400000000000000000000015071457266313600176160ustar00rootroot00000000000000include( FindPackageHandleStandardArgs ) if( KERBEROS5_INCLUDE_DIR AND KERBEROS5_LIBRARIES ) set( KERBEROS5_FOUND TRUE ) else() find_path( KERBEROS5_INCLUDE_DIR NAMES krb5.h HINTS ${KERBEROS5_ROOT_DIR} PATH_SUFFIXES include ) find_library( KERBEROS5_LIBRARY NAMES krb5 HINTS ${KERBEROS5_ROOT_DIR} PATH_SUFFIXES ${LIBRARY_PATH_PREFIX} ${LIB_SEARCH_OPTIONS}) find_library( COM_ERR_LIBRARY NAMES com_err HINTS ${KERBEROS5_ROOT_DIR} PATH_SUFFIXES ${LIBRARY_PATH_PREFIX} ${LIB_SEARCH_OPTIONS}) set( KERBEROS5_LIBRARIES ${KERBEROS5_LIBRARY} ${COM_ERR_LIBRARY} ) find_package_handle_standard_args( Kerberos5 DEFAULT_MSG KERBEROS5_LIBRARIES KERBEROS5_INCLUDE_DIR ) mark_as_advanced( KERBEROS5_INCLUDE_DIR KERBEROS5_LIBRARIES ) endif() xrootd-5.6.9/cmake/FindLibTool.cmake000066400000000000000000000003731457266313600173210ustar00rootroot00000000000000FIND_PROGRAM( LIBTOOL_FOUND libtool HINTS ${LIBTOOL_DIR} /usr PATH_SUFFIXES bin ) INCLUDE( FindPackageHandleStandardArgs ) FIND_PACKAGE_HANDLE_STANDARD_ARGS( LibTool DEFAULT_MSG LIBTOOL_FOUND ) xrootd-5.6.9/cmake/FindMacaroons.cmake000066400000000000000000000006111457266313600176720ustar00rootroot00000000000000 FIND_PATH(MACAROONS_INCLUDES macaroons.h HINTS ${MACAROONS_DIR} $ENV{MACAROONS_DIR} /usr PATH_SUFFIXES include ) FIND_LIBRARY(MACAROONS_LIB macaroons HINTS ${MACAROONS_DIR} $ENV{MACAROONS_DIR} /usr PATH_SUFFIXES lib PATH_SUFFIXES .libs ) INCLUDE(FindPackageHandleStandardArgs) FIND_PACKAGE_HANDLE_STANDARD_ARGS(Macaroons DEFAULT_MSG MACAROONS_INCLUDES MACAROONS_LIB) xrootd-5.6.9/cmake/FindReadline.cmake000066400000000000000000000032431457266313600174770ustar00rootroot00000000000000include( FindPackageHandleStandardArgs ) include( CheckCXXSourceCompiles ) if( READLINE_INCLUDE_DIR AND READLINE_LIBRARY ) set( READLINE_FOUND TRUE ) else( READLINE_INCLUDE_DIR AND READLINE_LIBRARY ) find_path( READLINE_INCLUDE_DIR readline/readline.h /usr/include/readline ) find_library( READLINE_LIB NAMES readline HINTS ${READLINE_ROOT_DIR} PATH_SUFFIXES ${LIBRARY_PATH_PREFIX} ${LIB_SEARCH_OPTIONS}) #----------------------------------------------------------------------------- # Check if we need ncurses - a hack required for SLC5 #----------------------------------------------------------------------------- if( READLINE_LIB ) set( CMAKE_REQUIRED_LIBRARIES ${READLINE_LIB} ) set( CMAKE_REQUIRED_INCLUDES ${READLINE_INCLUDE_DIR} ) check_cxx_source_compiles( " #include #include int main() { char shell_prompt[100]; readline(shell_prompt); return 0; } " READLINE_OK ) if( READLINE_OK ) set( READLINE_LIBRARY ${READLINE_LIB} ) else() find_library( NCURSES_LIBRARY NAMES ncurses HINTS ${READLINE_ROOT_DIR} PATH_SUFFIXES ${LIBRARY_PATH_PREFIX} ${LIB_SEARCH_OPTIONS}) if( NCURSES_LIBRARY ) set( READLINE_LIBRARY "${READLINE_LIB};${NCURSES_LIBRARY}" ) endif() endif() endif() find_package_handle_standard_args( Readline DEFAULT_MSG READLINE_LIBRARY READLINE_INCLUDE_DIR ) mark_as_advanced( READLINE_INCLUDE_DIR READLINE_LIBRARY ) endif( READLINE_INCLUDE_DIR AND READLINE_LIBRARY) xrootd-5.6.9/cmake/FindSciTokensCpp.cmake000066400000000000000000000014061457266313600203200ustar00rootroot00000000000000 include(CheckSymbolExists) FIND_PATH(SCITOKENS_CPP_INCLUDE_DIR scitokens/scitokens.h HINTS ${SCITOKENS_CPP_DIR} $ENV{SCITOKENS_CPP_DIR} /usr PATH_SUFFIXES include ) FIND_LIBRARY(SCITOKENS_CPP_LIBRARIES SciTokens HINTS ${SCITOKENS_CPP_DIR} $ENV{SCITOKENS_CPP_DIR} /usr PATH_SUFFIXES lib ) INCLUDE(FindPackageHandleStandardArgs) FIND_PACKAGE_HANDLE_STANDARD_ARGS(SciTokensCpp DEFAULT_MSG SCITOKENS_CPP_LIBRARIES SCITOKENS_CPP_INCLUDE_DIR) IF (SCITOKENS_CPP_INCLUDE_DIR) SET( CMAKE_REQUIRED_INCLUDES ${SCITOKENS_CPP_INCLUDE_DIR} ) SET( CMAKE_REQUIRED_LIBRARIES ${SCITOKENS_CPP_LIBRARIES} ) CHECK_SYMBOL_EXISTS(scitoken_config_set_str "scitokens/scitokens.h" HAVE_SCITOKEN_CONFIG_SET_STR) MARK_AS_ADVANCED(HAVE_SCITOKEN_CONFIG_SET_STR) ENDIF () xrootd-5.6.9/cmake/FindTinyXml.cmake000066400000000000000000000005551457266313600173630ustar00rootroot00000000000000FIND_PATH(TINYXML_INCLUDE_DIR tinyxml.h HINTS ${TINYXML_DIR} $ENV{TINYXML_DIR} /usr PATH_SUFFIXES include ) FIND_LIBRARY(TINYXML_LIBRARIES tinyxml HINTS ${TINYXML_DIR} $ENV{TINYXML_DIR} /usr PATH_SUFFIXES lib ) INCLUDE(FindPackageHandleStandardArgs) FIND_PACKAGE_HANDLE_STANDARD_ARGS(TinyXml DEFAULT_MSG TINYXML_LIBRARIES TINYXML_INCLUDE_DIR) xrootd-5.6.9/cmake/FindVOMS.cmake000066400000000000000000000012111457266313600165310ustar00rootroot00000000000000include( FindPackageHandleStandardArgs ) if( VOMS_INCLUDE_DIRS AND VOMS_LIBRARIES ) set( VOMS_FOUND TRUE ) else() find_path( VOMS_INCLUDE_DIR NAMES voms/voms_api.h HINTS ${VOMS_ROOT_DIR} PATH_SUFFIXES include ) set( VOMS_INCLUDE_DIRS ${VOMS_INCLUDE_DIR}) find_library( VOMS_LIBRARY NAMES vomsapi HINTS ${VOMS_ROOT_DIR} PATH_SUFFIXES lib64 ${LIBRARY_PATH_PREFIX} ${LIB_SEARCH_OPTIONS}) set( VOMS_LIBRARIES ${VOMS_LIBRARY} ) find_package_handle_standard_args( VOMS DEFAULT_MSG VOMS_LIBRARY VOMS_INCLUDE_DIR ) mark_as_advanced( VOMS_INCLUDE_DIR VOMS_LIBRARY ) endif() xrootd-5.6.9/cmake/FindYasm.cmake000066400000000000000000000003541457266313600166650ustar00rootroot00000000000000FIND_PROGRAM( YASM_FOUND yasm HINTS ${YASM_DIR} /usr PATH_SUFFIXES bin ) INCLUDE( FindPackageHandleStandardArgs ) FIND_PACKAGE_HANDLE_STANDARD_ARGS( Yasm DEFAULT_MSG YASM_FOUND ) xrootd-5.6.9/cmake/Findfuse.cmake000066400000000000000000000013441457266313600167160ustar00rootroot00000000000000# Try to find fuse (devel) # Once done, this will define # # FUSE_FOUND - system has fuse # FUSE_INCLUDE_DIRS - the fuse include directories # FUSE_LIBRARIES - fuse libraries directories if(FUSE_INCLUDE_DIRS AND FUSE_LIBRARIES) set(FUSE_FIND_QUIETLY TRUE) endif(FUSE_INCLUDE_DIRS AND FUSE_LIBRARIES) find_path(FUSE_INCLUDE_DIR fuse/fuse_lowlevel.h) find_library(FUSE_LIBRARY fuse) set(FUSE_INCLUDE_DIRS ${FUSE_INCLUDE_DIR}) set(FUSE_LIBRARIES ${FUSE_LIBRARY}) # handle the QUIETLY and REQUIRED arguments and set FUSE_FOUND to TRUE if # all listed variables are TRUE include(FindPackageHandleStandardArgs) find_package_handle_standard_args(fuse DEFAULT_MSG FUSE_INCLUDE_DIR FUSE_LIBRARY) mark_as_advanced(FUSE_INCLUDE_DIR FUSE_LIBRARY) xrootd-5.6.9/cmake/Findisal.cmake000066400000000000000000000032501457266313600167020ustar00rootroot00000000000000#.rst: # Findisal # --------- # # Find Intelligent Storage Acceleration Library. # # Result Variables # ^^^^^^^^^^^^^^^^ # # This module defines the following variables: # # :: # # ISAL_FOUND - True if isa-l is found. # ISAL_INCLUDE_DIRS - Where to find isa-l.h # ISAL_LIBRARIES - Where to find libisal.so # # :: # # ISAL_VERSION - The version of ISAL found (x.y.z) # ISAL_VERSION_MAJOR - The major version of isa-l # ISAL_VERSION_MINOR - The minor version of isa-l # ISAL_VERSION_PATCH - The patch version of isa-l foreach(var ISAL_FOUND ISAL_INCLUDE_DIR ISAL_ISAL_LIBRARY ISAL_LIBRARIES) unset(${var} CACHE) endforeach() find_path(ISAL_INCLUDE_DIR NAME isa-l.h PATH_SUFFIXES include) if(NOT ISAL_LIBRARY) find_library(ISAL_LIBRARY NAMES isal PATH_SUFFIXES lib) endif() mark_as_advanced(ISAL_INCLUDE_DIR) if(ISAL_INCLUDE_DIR AND EXISTS "${ISAL_INCLUDE_DIR}/isa-l.h") file(STRINGS "${ISAL_INCLUDE_DIR}/isa-l.h" ISAL_H REGEX "^#define ISAL_[A-Z_]+[ ]+[0-9]+.*$") string(REGEX REPLACE ".+ISAL_MAJOR_VERSION[ ]+([0-9]+).*$" "\\1" ISAL_VERSION_MAJOR "${ISAL_H}") string(REGEX REPLACE ".+ISAL_MINOR_VERSION[ ]+([0-9]+).*$" "\\1" ISAL_VERSION_MINOR "${ISAL_H}") string(REGEX REPLACE ".+ISAL_PATCH_VERSION[ ]+([0-9]+).*$" "\\1" ISAL_VERSION_PATCH "${ISAL_H}") set(ISAL_VERSION "${ISAL_VERSION_MAJOR}.${ISAL_VERSION_MINOR}.${ISAL_VERSION_PATCH}") endif() include(FindPackageHandleStandardArgs) find_package_handle_standard_args(isal REQUIRED_VARS ISAL_LIBRARY ISAL_INCLUDE_DIR VERSION_VAR ISAL_VERSION) if(ISAL_FOUND) set(ISAL_INCLUDE_DIRS "${ISAL_INCLUDE_DIR}") if(NOT ISAL_LIBRARIES) set(ISAL_LIBRARIES ${ISAL_LIBRARY}) endif() endif() xrootd-5.6.9/cmake/Findlibuuid.cmake000066400000000000000000000070221457266313600174100ustar00rootroot00000000000000#.rst: # Findlibuuid # ----------- # # Find libuuid, DCE compatible Universally Unique Identifier library. # # Imported Targets # ^^^^^^^^^^^^^^^^ # # This module defines :prop_tgt:`IMPORTED` target: # # ``uuid::uuid`` # The libuuid library, if found. # # Result Variables # ^^^^^^^^^^^^^^^^ # # This module will set the following variables in your project: # # ``LIBUUID_FOUND`` # True if libuuid has been found. # ``UUID_INCLUDE_DIRS`` # Where to find uuid/uuid.h. # ``UUID_LIBRARIES`` # The libraries to link against to use libuuid. # # Obsolete variables # ^^^^^^^^^^^^^^^^^^ # # The following variables may also be set, for backwards compatibility: # # ``UUID_LIBRARY`` # where to find the libuuid library (same as UUID_LIBRARIES). # ``UUID_INCLUDE_DIR`` # where to find the uuid/uuid.h header (same as UUID_INCLUDE_DIRS). include(CheckCXXSymbolExists) if(NOT UUID_INCLUDE_DIR) set(CMAKE_FIND_FRAMEWORK LAST) find_path(UUID_INCLUDE_DIR uuid/uuid.h) endif() if(IS_DIRECTORY "${UUID_INCLUDE_DIR}") set(CMAKE_REQUIRED_INCLUDES ${UUID_INCLUDE_DIR}) check_cxx_symbol_exists("uuid_generate_random" "uuid/uuid.h" _uuid_header_only) unset(CMAKE_REQUIRED_INCLUDES) endif() if(NOT UUID_LIBRARY AND NOT _uuid_header_only) find_library(UUID_LIBRARY NAMES uuid) if(UUID_LIBRARY) set(CMAKE_REQUIRED_INCLUDES ${UUID_INCLUDE_DIR}) set(CMAKE_REQUIRED_LIBRARIES ${UUID_LIBRARY}) check_cxx_symbol_exists("uuid_generate_random" "uuid/uuid.h" _have_libuuid) unset(CMAKE_REQUIRED_INCLUDES) unset(CMAKE_REQUIRED_LIBRARIES) endif() if(NOT _have_libuuid) find_package(PkgConfig QUIET) if(PKG_CONFIG_FOUND) # We need to clear cache variables set above, which pkg-config may set, # otherwise the call to pkg_check_modules will have no effect, as it does # not override cache variables. foreach(var FOUND INCLUDE_DIR INCLUDE_DIRS LIBRARY LIBRARIES) unset(UUID_${var} CACHE) endforeach() if(${libuuid_FIND_REQUIRED}) set(libuuid_REQUIRED REQUIRED) endif() pkg_check_modules(UUID ${libuuid_REQUIRED} uuid) # The include directory returned by pkg-config is /include/uuid, # while we expect just /include, so strip the last component to # allow #include to actually work. get_filename_component(UUID_INCLUDE_DIR ${UUID_INCLUDE_DIRS} DIRECTORY) set(UUID_INCLUDE_DIR ${UUID_INCLUDE_DIR} CACHE PATH "") set(UUID_LIBRARY ${UUID_LDFLAGS} CACHE STRING "") unset(UUID_INCLUDE_DIRS CACHE) unset(UUID_LIBRARIES CACHE) endif() endif() endif() if(_uuid_header_only) find_package_handle_standard_args(libuuid DEFAULT_MSG UUID_INCLUDE_DIR) else() find_package_handle_standard_args(libuuid DEFAULT_MSG UUID_INCLUDE_DIR UUID_LIBRARY) endif() if(LIBUUID_FOUND) set(UUID_INCLUDE_DIRS ${UUID_INCLUDE_DIR}) set(UUID_LIBRARIES ${UUID_LIBRARY}) if(NOT TARGET uuid::uuid) add_library(uuid::uuid INTERFACE IMPORTED) target_include_directories(uuid::uuid SYSTEM INTERFACE "${UUID_INCLUDE_DIRS}") target_link_libraries(uuid::uuid INTERFACE "${UUID_LIBRARIES}") endif() endif() mark_as_advanced(UUID_INCLUDE_DIR UUID_LIBRARY) if(NOT "${libuuid_FIND_QUIET}") message(DEBUG "UUID_FOUND = ${LIBUUID_FOUND}") message(DEBUG "UUID_HEADER_ONLY = ${_uuid_header_only}") message(DEBUG "UUID_INCLUDE_DIR = ${UUID_INCLUDE_DIR}") message(DEBUG "UUID_INCLUDE_DIRS = ${UUID_INCLUDE_DIRS}") message(DEBUG "UUID_LIBRARY = ${UUID_LIBRARY}") message(DEBUG "UUID_LIBRARIES = ${UUID_LIBRARIES}") endif() xrootd-5.6.9/cmake/Findsystemd.cmake000066400000000000000000000012531457266313600174430ustar00rootroot00000000000000# Try to find systemd # Once done, this will define # # SYSTEMD_FOUND - system has systemd # SYSTEMD_INCLUDE_DIRS - the systemd include directories # SYSTEMD_LIBRARIES - systemd libraries directories find_path( SYSTEMD_INCLUDE_DIR systemd/sd-daemon.h HINTS ${SYSTEMD_DIR} $ENV{SYSTEMD_DIR} /usr /opt PATH_SUFFIXES include ) find_library( SYSTEMD_LIBRARY systemd HINTS ${SYSTEMD_DIR} $ENV{SYSTEMD_DIR} /usr /opt PATH_SUFFIXES lib ) set(SYSTEMD_INCLUDE_DIRS ${SYSTEMD_INCLUDE_DIR}) set(SYSTEMD_LIBRARIES ${SYSTEMD_LIBRARY}) include(FindPackageHandleStandardArgs) find_package_handle_standard_args(systemd DEFAULT_MSG SYSTEMD_INCLUDE_DIRS SYSTEMD_LIBRARIES) xrootd-5.6.9/cmake/XRootDConfig.cmake.in000066400000000000000000000163561457266313600200760ustar00rootroot00000000000000################################################################################ # Module for locating XRootD. # # XROOTD_FOUND # Indicates whether the library has been found. # # XROOTD_INCLUDE_DIRS # Specifies XRootD include directory. # # XROOTD_LIBRARIES # Specifies XRootD libraries that should be passed to target_link_libararies. # # XROOTD__LIBRARIES # Specifies the libraries of a specific # # XROOTD__FOUND # Indicates whether the specified was found. # # List of components: CLIENT, UTILS, SERVER, POSIX, HTTP and SSI ################################################################################ @PACKAGE_INIT@ set(XRootD_FOUND TRUE) set(XRootD_VERSION @XRootD_VERSION@) set(XRootD_PLUGIN_VERSION @XRootD_VERSION_MAJOR@) set(XRootD_VERSION_MAJOR @XRootD_VERSION_MAJOR@) set(XRootD_VERSION_MINOR @XRootD_VERSION_MINOR@) set(XRootD_VERSION_PATCH @XRootD_VERSION_PATCH@) set(XRootD_VERSION_TWEAK @XRootD_VERSION_TWEAK@) set(XRootD_VERSION_NUMBER @XRootD_VERSION_NUMBER@) set(XRootD_VERSION_STRING @XRootD_VERSION_STRING@) ################################################################################ # Make sure all *_FOUND variables are intialized to FALSE ################################################################################ SET( XROOTD_FOUND FALSE ) SET( XROOTD_CLIENT_FOUND FALSE ) SET( XROOTD_UTILS_FOUND FALSE ) SET( XROOTD_SERVER_FOUND FALSE ) SET( XROOTD_POSIX_FOUND FALSE ) SET( XROOTD_HTTP_FOUND FALSE ) SET( XROOTD_SSI_FOUND FALSE ) ################################################################################ # Set XRootD include paths ################################################################################ set_and_check(XRootD_CMAKE_DIR "${CMAKE_CURRENT_LIST_DIR}") set_and_check(XRootD_DATA_DIR "@PACKAGE_CMAKE_INSTALL_DATADIR@") set_and_check(XRootD_INCLUDE_DIR "@PACKAGE_CMAKE_INSTALL_INCLUDEDIR@/xrootd") set_and_check(XRootD_LIB_DIR "@PACKAGE_CMAKE_INSTALL_LIBDIR@") set(XRootD_INCLUDE_DIRS "${XRootD_INCLUDE_DIR};${XRootD_INCLUDE_DIR}/private") set(XROOTD_INCLUDE_DIRS "${XRootD_INCLUDE_DIRS}") # backward compatibility ################################################################################ # XRootD client libs # - libXrdCl ################################################################################ FIND_LIBRARY( XROOTD_CLIENT_LIBRARIES XrdCl PATHS ${XRootD_LIB_DIR} NO_DEFAULT_PATH ) IF( NOT "${XROOTD_CLIENT_LIBRARIES}" STREQUAL "XROOTD_CLIENT_LIBRARIES-NOTFOUND" ) SET( XROOTD_CLIENT_FOUND TRUE ) LIST( APPEND XROOTD_LIBRARIES ${XROOTD_CLIENT_LIBRARIES} ) ENDIF() ################################################################################ # XRootD utils libs # - libXrdUtils ################################################################################ FIND_LIBRARY( XROOTD_UTILS_LIBRARIES XrdUtils PATHS ${XRootD_LIB_DIR} NO_DEFAULT_PATH ) IF( NOT "${XROOTD_UTILS_LIBRARIES}" STREQUAL "XROOTD_UTILS_LIBRARIES-NOTFOUND" ) SET( XROOTD_UTILS_FOUND TRUE ) LIST( APPEND XROOTD_LIBRARIES ${XROOTD_UTILS_LIBRARIES} ) ENDIF() ################################################################################ # XRootD server libs # - libXrdServer ################################################################################ FIND_LIBRARY( XROOTD_SERVER_LIBRARIES XrdServer PATHS ${XRootD_LIB_DIR} NO_DEFAULT_PATH ) IF( NOT "${XROOTD_SERVER_LIBRARIES}" STREQUAL "XROOTD_SERVER_LIBRARIES-NOTFOUND" ) SET( XROOTD_SERVER_FOUND TRUE ) LIST( APPEND XROOTD_LIBRARIES ${XROOTD_SERVER_LIBRARIES} ) ENDIF() ################################################################################ # XRootD posix libs # - libXrdPosix # - libXrdPosixPreload ################################################################################ FIND_LIBRARY( XROOTD_POSIX_LIBRARY XrdPosix PATHS ${XRootD_LIB_DIR} NO_DEFAULT_PATH ) FIND_LIBRARY( XROOTD_POSIX_PRELOAD_LIBRARY XrdPosixPreload PATHS ${XRootD_LIB_DIR} NO_DEFAULT_PATH ) IF( NOT "${XROOTD_POSIX_LIBRARY}" STREQUAL "XROOTD_POSIX_LIBRARY-NOTFOUND" ) IF( NOT "${XROOTD_POSIX_PRELOAD_LIBRARY}" STREQUAL "XROOTD_POSIX_PRELOAD_LIBRARY-NOTFOUND" ) SET( XROOTD_POSIX_LIBRARIES ${XROOTD_POSIX_LIBRARY} ${XROOTD_POSIX_PRELOAD_LIBRARY} ) SET( XROOTD_POSIX_FOUND TRUE ) LIST( APPEND XROOTD_LIBRARIES ${XROOTD_POSIX_LIBRARIES} ) ENDIF() ENDIF() ################################################################################ # XRootD HTTP (XrdHttp) libs # - libXrdHtppUtils ################################################################################ FIND_LIBRARY( XROOTD_HTTP_LIBRARIES XrdHttpUtils PATHS ${XRootD_LIB_DIR} NO_DEFAULT_PATH ) IF( NOT "${XROOTD_HTTP_LIBRARIES}" STREQUAL "XROOTD_HTTP_LIBRARIES-NOTFOUND" ) SET( XROOTD_HTTP_FOUND TRUE ) LIST( APPEND XROOTD_LIBRARIES ${XROOTD_HTTP_LIBRARIES} ) ENDIF() ################################################################################ # XRootD SSI libs # - XrdSsiLib # - XrdSsiShMap ################################################################################ FIND_LIBRARY( XROOTD_SSI_LIBRARY XrdSsiLib PATHS ${XRootD_LIB_DIR} NO_DEFAULT_PATH ) FIND_LIBRARY( XROOTD_SSI_SHMAP_LIBRARY XrdSsiShMap PATHS ${XRootD_LIB_DIR} NO_DEFAULT_PATH ) IF( NOT "${XROOTD_SSI_LIBRARY}" STREQUAL "XROOTD_SSI_LIBRARY-NOTFOUND" ) IF( NOT "${XROOTD_SSI_SHMAP_LIBRARY}" STREQUAL "XROOTD_SSI_SHMAP_LIBRARY-NOTFOUND" ) SET( XROOTD_SSI_LIBRARIES ${XROOTD_SSI_LIBRARY} ${XROOTD_SSI_SHMAP_LIBRARY} ) SET( XROOTD_SSI_FOUND TRUE ) LIST( APPEND XROOTD_LIBRARIES ${XROOTD_SSI_LIBRARIES} ) ENDIF() ENDIF() ################################################################################ # Set up the XRootD find module ################################################################################ foreach(COMPONENT UTILS CLIENT SERVER HTTP POSIX SSI) # Set uppercase names to keep backward compatibility set(XRootD_${COMPONENT}_FOUND ${XROOTD_${COMPONENT}_FOUND}) set(XRootD_${COMPONENT}_LIBRARIES ${XROOTD_${COMPONENT}_LIBRARIES}) endforeach() check_required_components(XRootD) set(XROOTD_FOUND ${XRootD_FOUND}) set(XRootD_LIBRARIES ${XROOTD_LIBRARIES}) message(DEBUG "XRootD_VERSION = '${XRootD_VERSION}'") message(DEBUG "XRootD_VERSION_MAJOR = '${XRootD_VERSION_MAJOR}'") message(DEBUG "XRootD_VERSION_MINOR = '${XRootD_VERSION_MINOR}'") message(DEBUG "XRootD_VERSION_PATCH = '${XRootD_VERSION_PATCH}'") message(TRACE "XRootD_VERSION_TWEAK = '${XRootD_VERSION_TWEAK}'") message(TRACE "XRootD_VERSION_NUMBER = '${XRootD_VERSION_NUMBER}'") message(TRACE "XRootD_FOUND = '${XRootD_FOUND}'") message(TRACE "XRootD_INSTALL_PREFIX = '@CMAKE_INSTALL_PREFIX@'") message(TRACE "XRootD_INCLUDE_DIR = '${XRootD_INCLUDE_DIR}'") message(TRACE "XRootD_LIBRARY_DIR = '${XRootD_LIBRARY_DIR}'") message(TRACE "XRootD_LIBRARIES = '${XRootD_LIBRARIES}'") foreach(COMPONENT UTILS CLIENT SERVER HTTP POSIX SSI) message(TRACE "XRootD_${COMPONENT}_FOUND\t\t= '${XRootD_${COMPONENT}_FOUND}'") endforeach() foreach(COMPONENT UTILS CLIENT SERVER HTTP POSIX SSI) message(TRACE "XRootD_${COMPONENT}_LIBRARIES\t= '${XRootD_${COMPONENT}_LIBRARIES}'") endforeach() include(FindPackageHandleStandardArgs) set(XRootD_CONFIG ${CMAKE_CURRENT_LIST_FILE}) find_package_handle_standard_args(XRootD CONFIG_MODE HANDLE_COMPONENTS) xrootd-5.6.9/cmake/XRootDDefaults.cmake000066400000000000000000000044731457266313600200300ustar00rootroot00000000000000#------------------------------------------------------------------------------- # Define the default build parameters #------------------------------------------------------------------------------- if( "${CMAKE_BUILD_TYPE}" STREQUAL "" ) if( Solaris AND NOT SUNCC_CAN_DO_OPTS ) set( CMAKE_BUILD_TYPE Debug ) else() set( CMAKE_BUILD_TYPE RelWithDebInfo ) endif() endif() include( CMakeDependentOption ) define_default( PLUGIN_VERSION 5 ) option( ENABLE_FUSE "Enable the fuse filesystem driver if possible." TRUE ) option( ENABLE_KRB5 "Enable the Kerberos 5 authentication if possible." TRUE ) option( ENABLE_READLINE "Enable the lib readline support in the commandline utilities." TRUE ) option( ENABLE_XRDCL "Enable XRootD client." TRUE ) option( ENABLE_TESTS "Enable unit tests." FALSE ) option( ENABLE_HTTP "Enable HTTP component." TRUE ) option( ENABLE_PYTHON "Enable python bindings." TRUE ) option( XRDCL_ONLY "Build only the client and necessary dependencies" FALSE ) option( XRDCL_LIB_ONLY "Build only the client libraries and necessary dependencies" FALSE ) option( PYPI_BUILD "The project is being built for PyPI release" FALSE ) option( ENABLE_VOMS "Enable VOMS plug-in if possible." TRUE ) option( ENABLE_XRDEC "Enable erasure coding component." FALSE ) option( ENABLE_ASAN "Enable adress sanitizer." FALSE ) option( ENABLE_TSAN "Enable thread sanitizer." FALSE ) option( ENABLE_XRDCLHTTP "Enable xrdcl-http plugin." TRUE ) cmake_dependent_option( ENABLE_SCITOKENS "Enable SciTokens plugin." TRUE "NOT XRDCL_ONLY" FALSE ) cmake_dependent_option( ENABLE_MACAROONS "Enable Macaroons plugin." TRUE "NOT XRDCL_ONLY" FALSE ) option( FORCE_ENABLED "Fail build if enabled components cannot be built." FALSE ) cmake_dependent_option( USE_SYSTEM_ISAL "Use isa-l installed in the system" FALSE "ENABLE_XRDEC" FALSE ) define_default( XRD_PYTHON_REQ_VERSION 3 ) xrootd-5.6.9/cmake/XRootDFindLibs.cmake000066400000000000000000000114471457266313600177520ustar00rootroot00000000000000#------------------------------------------------------------------------------- # Find the required libraries #------------------------------------------------------------------------------- find_package( ZLIB REQUIRED) find_package( libuuid REQUIRED ) if( ENABLE_READLINE ) if( FORCE_ENABLED ) find_package( Readline REQUIRED ) else() find_package( Readline ) endif() if( READLINE_FOUND ) add_definitions( -DHAVE_READLINE ) else() set( READLINE_LIBRARY "" ) set( NCURSES_LIBRARY "" ) endif() endif() if( ZLIB_FOUND ) add_definitions( -DHAVE_LIBZ ) endif() find_package( TinyXml ) find_package( LibXml2 ) if( LIBXML2_FOUND ) add_definitions( -DHAVE_XML2 ) endif() find_package( systemd ) if( SYSTEMD_FOUND ) add_definitions( -DHAVE_SYSTEMD ) endif() find_package( CURL ) find_package( OpenSSL 1.0.2 REQUIRED ) add_definitions( -DHAVE_DH_PADDED ) add_definitions( -DHAVE_XRDCRYPTO ) add_definitions( -DHAVE_SSL ) if( ENABLE_KRB5 ) if( FORCE_ENABLED ) find_package( Kerberos5 REQUIRED ) else() find_package( Kerberos5 ) endif() if( KERBEROS5_FOUND ) set( BUILD_KRB5 TRUE ) else() set( BUILD_KRB5 FALSE ) endif() endif() # mac fuse not supported if( ENABLE_FUSE AND (LINUX OR KFREEBSD) ) if( FORCE_ENABLED ) find_package( fuse REQUIRED ) else() find_package( fuse ) endif() if( FUSE_FOUND ) add_definitions( -DHAVE_FUSE ) set( BUILD_FUSE TRUE ) else() set( BUILD_FUSE FALSE ) endif() endif() if( ENABLE_TESTS ) if( FORCE_ENABLED ) find_package( CppUnit REQUIRED ) find_package( GTest REQUIRED ) else() find_package( CppUnit ) find_package( GTest ) endif() if( CPPUNIT_FOUND AND GTEST_FOUND ) set( BUILD_TESTS TRUE ) else() set( BUILD_TESTS FALSE ) endif() endif() if( ENABLE_HTTP ) set( BUILD_HTTP TRUE ) if( CURL_FOUND ) set( BUILD_TPC TRUE ) else() if( FORCE_ENABLED ) message( FATAL_ERROR "Cannot build HttpTpc: missing CURL." ) endif() set( BUILD_TPC FALSE ) endif() endif() if( BUILD_TPC ) set ( CMAKE_REQUIRED_LIBRARIES ${CURL_LIBRARIES} ) check_function_exists( curl_multi_wait HAVE_CURL_MULTI_WAIT ) compiler_define_if_found( HAVE_CURL_MULTI_WAIT HAVE_CURL_MULTI_WAIT ) endif() if( ENABLE_MACAROONS ) if( FORCE_ENABLED ) if( NOT BUILD_HTTP ) message(SEND_ERROR "Cannot enable XrdMacaroons without HTTP support") endif() find_package( Macaroons REQUIRED ) else() find_package( Macaroons ) endif() if( NOT MacOSX ) include(FindPkgConfig REQUIRED) if( FORCE_ENABLED ) pkg_check_modules(JSON REQUIRED json-c) else() pkg_check_modules(JSON json-c) endif() endif() if( MACAROONS_FOUND AND JSON_FOUND AND BUILD_HTTP ) set( BUILD_MACAROONS TRUE ) else() set( BUILD_MACAROONS FALSE ) endif() endif() if( ENABLE_SCITOKENS ) if( FORCE_ENABLED ) find_package( SciTokensCpp REQUIRED ) else() find_package( SciTokensCpp ) endif() if( SCITOKENSCPP_FOUND ) set( BUILD_SCITOKENS TRUE ) else() set( BUILD_SCITOKENS FALSE ) endif() endif() if( ENABLE_XRDEC ) if( USE_SYSTEM_ISAL ) if( FORCE_ENABLED ) find_package(isal REQUIRED) else() find_package(isal) endif() if( ISAL_FOUND ) set(BUILD_XRDEC TRUE) else() set(BUILD_XRDEC FALSE) endif() else() if( FORCE_ENABLED ) find_package( Yasm REQUIRED ) find_package( LibTool REQUIRED ) find_package( AutoMake REQUIRED ) find_package( AutoConf REQUIRED ) else() find_package( Yasm ) find_package( LibTool ) find_package( AutoMake ) find_package( AutoConf ) endif() if( YASM_FOUND AND LIBTOOL_FOUND AND AUTOMAKE_FOUND AND AUTOCONF_FOUND ) set( BUILD_XRDEC TRUE ) else() set( BUILD_XRDEC FALSE ) endif() endif() endif() if( ENABLE_PYTHON OR PYPI_BUILD ) if( CMAKE_VERSION VERSION_LESS 3.18 ) set(PYTHON_COMPONENTS Interpreter Development) else() set(PYTHON_COMPONENTS Interpreter Development.Module) endif() if( FORCE_ENABLED OR PYPI_BUILD ) find_package( Python ${XRD_PYTHON_REQ_VERSION} REQUIRED COMPONENTS ${PYTHON_COMPONENTS} ) else() find_package( Python ${XRD_PYTHON_REQ_VERSION} COMPONENTS ${PYTHON_COMPONENTS} ) endif() if( Python_FOUND ) set( BUILD_PYTHON TRUE ) else() set( BUILD_PYTHON FALSE ) endif() endif() if( ENABLE_VOMS AND (LINUX OR KFREEBSD OR Hurd) ) if( FORCE_ENABLED ) find_package( VOMS REQUIRED ) else() find_package( VOMS ) endif() if( VOMS_FOUND ) set( BUILD_VOMS TRUE ) else() set( BUILD_VOMS FALSE ) endif() endif() if( ENABLE_XRDCLHTTP ) if( FORCE_ENABLED ) find_package( Davix REQUIRED ) else() find_package( Davix ) endif() if( DAVIX_FOUND ) set( BUILD_XRDCLHTTP TRUE ) else() set( BUILD_XRDCLHTTP FALSE ) endif() endif() xrootd-5.6.9/cmake/XRootDOSDefs.cmake000066400000000000000000000160631457266313600174020ustar00rootroot00000000000000#------------------------------------------------------------------------------- # Define the OS variables #------------------------------------------------------------------------------- include( CheckCXXSourceRuns ) set( LINUX FALSE ) set( KFREEBSD FALSE ) set( Hurd FALSE ) set( MacOSX FALSE ) set( Solaris FALSE ) set( XrdClPipelines FALSE ) add_definitions( -D_LARGEFILE_SOURCE -D_LARGEFILE64_SOURCE -D_FILE_OFFSET_BITS=64 ) define_default( LIBRARY_PATH_PREFIX "lib" ) #------------------------------------------------------------------------------- # Enable c++14 #------------------------------------------------------------------------------- set(CMAKE_CXX_STANDARD 14) set(CMAKE_CXX_STANDARD_REQUIRED TRUE) if( ENABLE_ASAN ) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address") endif() if( ENABLE_TSAN ) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=thread") endif() #------------------------------------------------------------------------------- # Enable XrdCl::Pipelines for clang compiler # Note: once we move to c++14 globaly we can remove this #------------------------------------------------------------------------------- if( CMAKE_CXX_COMPILER_ID STREQUAL "Clang" ) set( XrdClPipelines TRUE ) endif() #------------------------------------------------------------------------------- # GCC #------------------------------------------------------------------------------- if( CMAKE_COMPILER_IS_GNUCXX ) set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra" ) #----------------------------------------------------------------------------- # Set -Werror only for Debug (or undefined) build type or if we have been # explicitly asked to do so #----------------------------------------------------------------------------- if( ( CMAKE_BUILD_TYPE STREQUAL "Debug" OR "${CMAKE_BUILD_TYPE}" STREQUAL "" OR FORCE_WERROR ) ) set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror" ) endif() set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-parameter" ) execute_process( COMMAND ${CMAKE_C_COMPILER} -dumpversion OUTPUT_VARIABLE GCC_VERSION ) if( GCC_VERSION VERSION_GREATER 4.8.0 ) set( XrdClPipelines TRUE ) endif() endif() #------------------------------------------------------------------------------- # Linux #------------------------------------------------------------------------------- if( ${CMAKE_SYSTEM_NAME} STREQUAL "Linux" ) set( LINUX TRUE ) include( GNUInstallDirs ) set( EXTRA_LIBS rt ) # Check for musl libc with the compiler, since it provides way to detect it execute_process(COMMAND ${CMAKE_CXX_COMPILER} -dumpmachine OUTPUT_VARIABLE TARGET_TRIPLE ERROR_VARIABLE TARGET_ERROR) if (NOT TARGET_ERROR) if ("${TARGET_TRIPLE}" MATCHES "musl") message(STATUS "Detected musl libc") add_definitions(-DMUSL=1) endif() else() message(WARNING "Could not detect system information!") endif() unset(TARGET_ERROR) endif() #------------------------------------------------------------------------------- # GNU/kFreeBSD #------------------------------------------------------------------------------- if( ${CMAKE_SYSTEM_NAME} STREQUAL "kFreeBSD" ) set( KFREEBSD TRUE ) include( GNUInstallDirs ) set( EXTRA_LIBS rt ) endif() #------------------------------------------------------------------------------- # GNU/Hurd #------------------------------------------------------------------------------- if( ${CMAKE_SYSTEM_NAME} STREQUAL "GNU" ) set( Hurd TRUE ) include( GNUInstallDirs ) set( EXTRA_LIBS rt ) endif() #------------------------------------------------------------------------------- # MacOSX #------------------------------------------------------------------------------- if( APPLE ) set( MacOSX TRUE ) set( XrdClPipelines TRUE ) set(CMAKE_MACOSX_RPATH TRUE) set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) set(CMAKE_INSTALL_RPATH "@loader_path/../lib") # this is here because of Apple deprecating openssl and krb5 set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-deprecated-declarations" ) add_definitions( -DLT_MODULE_EXT=".dylib" ) define_default( CMAKE_INSTALL_LIBDIR "lib" ) define_default( CMAKE_INSTALL_BINDIR "bin" ) define_default( CMAKE_INSTALL_MANDIR "share/man" ) define_default( CMAKE_INSTALL_INCLUDEDIR "include" ) define_default( CMAKE_INSTALL_DATADIR "share" ) endif() #------------------------------------------------------------------------------- # FreeBSD #------------------------------------------------------------------------------- if( ${CMAKE_SYSTEM_NAME} STREQUAL "FreeBSD" ) define_default( CMAKE_INSTALL_LIBDIR "lib" ) define_default( CMAKE_INSTALL_BINDIR "bin" ) define_default( CMAKE_INSTALL_MANDIR "man" ) define_default( CMAKE_INSTALL_INCLUDEDIR "include" ) define_default( CMAKE_INSTALL_DATADIR "share" ) endif() #------------------------------------------------------------------------------- # Solaris #------------------------------------------------------------------------------- if( ${CMAKE_SYSTEM_NAME} STREQUAL "SunOS" ) define_default( FORCE_32BITS FALSE ) define_default( CMAKE_INSTALL_LIBDIR "lib" ) define_default( CMAKE_INSTALL_BINDIR "bin" ) define_default( CMAKE_INSTALL_MANDIR "man" ) define_default( CMAKE_INSTALL_INCLUDEDIR "include" ) define_default( CMAKE_INSTALL_DATADIR "share" ) set( Solaris TRUE ) add_definitions( -D__solaris__=1 ) add_definitions( -DSUNCC -D_REENTRANT -D_POSIX_PTHREAD_SEMANTICS ) set( EXTRA_LIBS rt Crun Cstd ) set( CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -fast" ) set( CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -fast" ) define_solaris_flavor() #----------------------------------------------------------------------------- # Define solaris version #----------------------------------------------------------------------------- execute_process( COMMAND uname -r OUTPUT_VARIABLE SOLARIS_VER ) string( REPLACE "." ";" SOLARIS_VER_LIST ${SOLARIS_VER} ) list( GET SOLARIS_VER_LIST 1 SOLARIS_VERSION ) string( REPLACE "\n" "" SOLARIS_VERSION ${SOLARIS_VERSION} ) add_definitions( -DSOLARIS_VERSION=${SOLARIS_VERSION} ) #----------------------------------------------------------------------------- # AMD64 (opteron) #----------------------------------------------------------------------------- if( ${SOLARIS_VERSION} STREQUAL "10" AND SOLARIS_AMD64 AND NOT FORCE_32BITS ) set( CMAKE_CXX_FLAGS " -m64 -xtarget=opteron -xs ${CMAKE_CXX_FLAGS} " ) set( CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -G" ) define_default( CMAKE_LIBRARY_PATH "/lib/64;/usr/lib/64" ) add_definitions( -DSUNX86 ) set( LIB_SEARCH_OPTIONS NO_DEFAULT_PATH ) define_default( LIBRARY_PATH_PREFIX "lib/64" ) endif() #----------------------------------------------------------------------------- # Check if the SunCC compiler can do optimizations #----------------------------------------------------------------------------- check_cxx_source_runs( " int main() { #if __SUNPRO_CC > 0x5100 return 0; #else return 1; #endif } " SUNCC_CAN_DO_OPTS ) endif() xrootd-5.6.9/cmake/XRootDSummary.cmake000066400000000000000000000052031457266313600177060ustar00rootroot00000000000000#------------------------------------------------------------------------------- # Print the configuration summary #------------------------------------------------------------------------------- set( TRUE_VAR TRUE ) component_status( CEPH XRDCEPH_SUBMODULE TRUE_VAR) component_status( FUSE BUILD_FUSE FUSE_FOUND ) component_status( HTTP BUILD_HTTP OPENSSL_FOUND ) component_status( KRB5 BUILD_KRB5 KERBEROS5_FOUND ) component_status( MACAROONS BUILD_MACAROONS MACAROONS_FOUND AND JSON_FOUND AND BUILD_HTTP ) component_status( PYTHON BUILD_PYTHON Python_Interpreter_FOUND AND Python_Development_FOUND ) component_status( READLINE ENABLE_READLINE READLINE_FOUND ) component_status( SCITOKENS BUILD_SCITOKENS SCITOKENSCPP_FOUND ) component_status( TESTS BUILD_TESTS CPPUNIT_FOUND AND GTEST_FOUND ) component_status( TPC BUILD_TPC CURL_FOUND ) component_status( VOMSXRD BUILD_VOMS VOMS_FOUND ) component_status( XRDCL ENABLE_XRDCL TRUE_VAR ) component_status( XRDCLHTTP ENABLE_XRDCLHTTP DAVIX_FOUND ) component_status( XRDEC BUILD_XRDEC TRUE_VAR ) message( STATUS "----------------------------------------" ) message( STATUS "Installation path: " ${CMAKE_INSTALL_PREFIX} ) message( STATUS "C Compiler: " ${CMAKE_C_COMPILER} ) message( STATUS "C++ Compiler: " ${CMAKE_CXX_COMPILER} ) message( STATUS "Build type: " ${CMAKE_BUILD_TYPE} ) message( STATUS "Plug-in version: " ${PLUGIN_VERSION} ) message( STATUS "" ) message( STATUS "Ceph support: " ${STATUS_CEPH} ) message( STATUS "Readline support: " ${STATUS_READLINE} ) message( STATUS "FUSE support: " ${STATUS_FUSE} ) message( STATUS "Kerberos5 support: " ${STATUS_KRB5} ) message( STATUS "XrdCl: " ${STATUS_XRDCL} ) message( STATUS "XrdClHttp: " ${STATUS_XRDCLHTTP} ) message( STATUS "HTTP support: " ${STATUS_HTTP} ) message( STATUS "HTTP TPC support: " ${STATUS_TPC} ) message( STATUS "VOMS support: " ${STATUS_VOMSXRD} ) message( STATUS "Python support: " ${STATUS_PYTHON} ) message( STATUS "Erasure coding: " ${STATUS_XRDEC} ) message( STATUS "Macaroons: " ${STATUS_MACAROONS} ) message( STATUS "SciTokens: " ${STATUS_SCITOKENS} ) message( STATUS "Tests: " ${STATUS_TESTS} ) message( STATUS "----------------------------------------" ) if( FORCE_ENABLED ) foreach(FEATURE CEPH FUSE HTTP KRB5 MACAROONS PYTHON READLINE SCITOKENS TESTS VOMSXRD XRDCL XRDCLHTTP) if(ENABLE_${FEATURE} AND NOT STATUS_${FEATURE} STREQUAL "yes") message(SEND_ERROR "Could not enable feature: ${FEATURE}") endif() endforeach() endif() xrootd-5.6.9/cmake/XRootDSystemCheck.cmake000066400000000000000000000150741457266313600205020ustar00rootroot00000000000000#------------------------------------------------------------------------------- # Probe the system libraries #------------------------------------------------------------------------------- include( CheckFunctionExists ) include( CheckSymbolExists ) include( CheckLibraryExists ) include( CheckIncludeFile ) include( CheckCXXSourceCompiles ) include( CheckCXXSourceRuns ) include( XRootDUtils ) #------------------------------------------------------------------------------- # OS stuff #------------------------------------------------------------------------------- check_function_exists( setresuid HAVE_SETRESUID ) compiler_define_if_found( HAVE_SETRESUID HAVE_SETRESUID ) check_function_exists( strlcpy HAVE_STRLCPY ) compiler_define_if_found( HAVE_STRLCPY HAVE_STRLCPY ) check_function_exists( fstatat HAVE_FSTATAT ) compiler_define_if_found( HAVE_FSTATAT HAVE_FSTATAT ) check_function_exists( sigwaitinfo HAVE_SIGWTI ) compiler_define_if_found( HAVE_SIGWTI HAVE_SIGWTI ) if( NOT HAVE_SIGWTI ) check_library_exists( rt sigwaitinfo "" HAVE_SIGWTI_IN_RT ) compiler_define_if_found( HAVE_SIGWTI_IN_RT HAVE_SIGWTI ) endif() check_include_file( shadow.h HAVE_SHADOWPW ) compiler_define_if_found( HAVE_SHADOWPW HAVE_SHADOWPW ) #------------------------------------------------------------------------------- # Some socket related functions #------------------------------------------------------------------------------- check_function_exists( getifaddrs HAVE_GETIFADDRS ) compiler_define_if_found( HAVE_GETIFADDRS HAVE_GETIFADDRS ) check_function_exists( getnameinfo HAVE_NAMEINFO ) compiler_define_if_found( HAVE_NAMEINFO HAVE_NAMEINFO ) if( NOT HAVE_NAMEINFO ) check_library_exists( socket getnameinfo "" HAVE_NAMEINFO_IN_SOCKET ) compiler_define_if_found( HAVE_NAMEINFO_IN_SOCKET HAVE_NAMEINFO ) endif() check_function_exists( getprotobyname_r HAVE_PROTOR ) compiler_define_if_found( HAVE_PROTOR HAVE_PROTOR ) if( NOT HAVE_PROTOR ) check_library_exists( socket getprotobyname_r "" HAVE_PROTOR_IN_SOCKET ) compiler_define_if_found( HAVE_PROTOR_IN_SOCKET HAVE_PROTOR ) endif() check_function_exists( gethostbyaddr_r HAVE_GETHBYXR ) compiler_define_if_found( HAVE_GETHBYXR HAVE_GETHBYXR ) if( NOT HAVE_GETHBYXR ) check_library_exists( socket gethostbyaddr_r "" HAVE_GETHBYXR_IN_SOCKET ) compiler_define_if_found( HAVE_GETHBYXR_IN_SOCKET HAVE_GETHBYXR ) endif() if( HAVE_GETHBYXR_IN_SOCKET OR HAVE_PROTOR_IN_SOCKET OR HAVE_NAMEINFO_IN_SOCKET ) set( SOCKET_LIBRARY "socket" ) else() set( SOCKET_LIBRARY "" ) endif() #------------------------------------------------------------------------------- # Sendfile #------------------------------------------------------------------------------- if( NOT MacOSX ) check_function_exists( sendfile HAVE_SENDFILE ) compiler_define_if_found( HAVE_SENDFILE HAVE_SENDFILE ) set( SENDFILE_LIBRARY "" ) if( NOT HAVE_SENDFILE ) check_library_exists( sendfile sendfile "" HAVE_SENDFILE_IN_SENDFILE ) compiler_define_if_found( HAVE_SENDFILE_IN_SENDFILE HAVE_SENDFILE ) if( HAVE_SENDFILE_IN_SENDFILE ) set( SENDFILE_LIBRARY "sendfile" ) endif() endif() endif() #------------------------------------------------------------------------------- # Check for libcrypt #------------------------------------------------------------------------------- check_function_exists( crypt HAVE_CRYPT ) compiler_define_if_found( HAVE_CRYPT HAVE_CRYPT ) set( CRYPT_LIBRARY "" ) if( NOT HAVE_CRYPT ) check_library_exists( crypt crypt "" HAVE_CRYPT_IN_CRYPT ) compiler_define_if_found( HAVE_CRYPT_IN_CRYPT HAVE_CRYPT ) if( HAVE_CRYPT_IN_CRYPT ) set( CRYPT_LIBRARY "crypt" ) endif() endif() check_include_file( et/com_err.h HAVE_ET_COM_ERR_H ) compiler_define_if_found( HAVE_ET_COM_ERR_H HAVE_ET_COM_ERR_H ) #------------------------------------------------------------------------------- # Check for pthreads #------------------------------------------------------------------------------- set( THREADS_PREFER_PTHREAD_FLAG TRUE ) find_package( Threads ) #------------------------------------------------------------------------------- # Check for the atomics #------------------------------------------------------------------------------- if (CMAKE_CROSSCOMPILING) message(WARNING "Cannot detect atomics support when cross-compiling, assuming atomics are available") set(HAVE_ATOMICS ON) else() check_cxx_source_runs( " int main() { unsigned long long val = 111, *mem = &val; if (__sync_fetch_and_add(&val, 111) != 111 || val != 222) return 1; if (__sync_add_and_fetch(&val, 111) != 333) return 1; if (__sync_sub_and_fetch(&val, 111) != 222) return 1; if (__sync_fetch_and_sub(&val, 111) != 222 || val != 111) return 1; if (__sync_fetch_and_or (&val, 0) != 111 || val != 111) return 1; if (__sync_fetch_and_and(&val, 0) != 111 || val != 0 ) return 1; if (__sync_bool_compare_and_swap(mem, 0, 444) == 0 || val != 444) return 1; return 0; } " HAVE_ATOMICS ) endif() option(EnableAtomicsIfPresent "EnableAtomicsIfPresent" ON) if ( EnableAtomicsIfPresent ) compiler_define_if_found( HAVE_ATOMICS HAVE_ATOMICS ) endif () #------------------------------------------------------------------------------- # Check if we need libatomic to use atomic operations in the C++ code. # (This is required for using 64 bit atomics on some 32 bit architectures.) #------------------------------------------------------------------------------- function(check_working_cxx_atomics varname) set(OLD_CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS}) set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} -std=c++11") check_cxx_source_compiles(" #include #include int main() { std::atomic a1; std::atomic a2; std::atomic a3; std::atomic a4; return a1++ + a2++ + a3++ + a4++; } " ${varname}) set(CMAKE_REQUIRED_FLAGS ${OLD_CMAKE_REQUIRED_FLAGS}) endfunction(check_working_cxx_atomics varname) check_working_cxx_atomics(HAVE_CXX_ATOMICS_WITHOUT_LIB) set(ATOMIC_LIBRARY "") if(NOT HAVE_CXX_ATOMICS_WITHOUT_LIB) check_library_exists(atomic __atomic_fetch_add_4 "" HAVE_LIBATOMIC) if(HAVE_LIBATOMIC) set(OLD_CMAKE_REQUIRED_LIBRARIES ${CMAKE_REQUIRED_LIBRARIES}) list(APPEND CMAKE_REQUIRED_LIBRARIES "atomic") check_working_cxx_atomics(HAVE_CXX_ATOMICS_WITH_LIB) set(CMAKE_REQUIRED_LIBRARIES ${OLD_CMAKE_REQUIRED_LIBRARIES}) if(HAVE_CXX_ATOMICS_WITH_LIB) set(ATOMIC_LIBRARY "atomic") endif() endif() endif() if (NOT HAVE_CXX_ATOMICS_WITHOUT_LIB AND NOT HAVE_CXX_ATOMICS_WITH_LIB) message(FATAL_ERROR "Compiler must support std::atomic!") endif() xrootd-5.6.9/cmake/XRootDUtils.cmake000066400000000000000000000050051457266313600173510ustar00rootroot00000000000000 #------------------------------------------------------------------------------- # Add a compiler define flag if a variable is defined #------------------------------------------------------------------------------- function( compiler_define_if_found predicate name ) if( ${predicate} ) add_definitions( -D${name} ) endif() endfunction() macro( define_default variable value ) if( NOT DEFINED ${variable} ) set( ${variable} ${value} ) endif() endmacro() macro( component_status name flag found ) if( ${flag} AND ${found} ) set( STATUS_${name} "yes" ) elseif( ${flag} AND NOT ${found} ) set( STATUS_${name} "libs not found" ) else() set( STATUS_${name} "disabled" ) endif() endmacro() #------------------------------------------------------------------------------- # Detect in source builds #------------------------------------------------------------------------------- function( CheckBuildDirectory ) # Get Real Paths of the source and binary directories get_filename_component( srcdir "${CMAKE_SOURCE_DIR}" REALPATH ) get_filename_component( bindir "${CMAKE_BINARY_DIR}" REALPATH ) # Check for in-source builds if( ${srcdir} STREQUAL ${bindir} ) message( FATAL_ERROR "XRootD cannot be built in-source! " "Please run cmake outside the " "source directory and be sure to remove " "CMakeCache.txt or CMakeFiles if they " "exist in the source directory." ) endif() endfunction() #------------------------------------------------------------------------------- # Detect what kind of solaris machine we're running #------------------------------------------------------------------------------- macro( define_solaris_flavor ) execute_process( COMMAND isainfo OUTPUT_VARIABLE SOLARIS_ARCH ) string( REPLACE " " ";" SOLARIS_ARCH_LIST ${SOLARIS_ARCH} ) # amd64 (opteron) list( FIND SOLARIS_ARCH_LIST amd64 SOLARIS_AMD64 ) if( SOLARIS_AMD64 EQUAL -1 ) set( SOLARIS_AMD64 FALSE ) else() set( SOLARIS_AMD64 TRUE ) endif() endmacro() #------------------------------------------------------------------------------- # Install headers from a directory #------------------------------------------------------------------------------- function( install_headers destination files ) foreach( file ${files} ) string( REGEX MATCH "^(.+)/(.+)$" fileAr ${file} ) install( FILES ${file} DESTINATION ${destination}/${CMAKE_MATCH_1} ) endforeach() endfunction()xrootd-5.6.9/cmake/XRootDVersion.cmake000066400000000000000000000057561457266313600177130ustar00rootroot00000000000000#.rst: # # XRootDVersion # ------------- # # This module sets the version of XRootD. # # The version is determined in the following order: # * If a version is set with -DXRootD_VERSION_STRING=x.y.z during configuration, it is used. # * The version is read from the 'VERSION' file at the top directory of the repository. # * If the 'VERSION' file has not been expanded, a version is set using git describe. # * If none of the above worked, a fallback version is set using the current date. # if(NOT DEFINED XRootD_VERSION_STRING) file(READ "${PROJECT_SOURCE_DIR}/VERSION" XRootD_VERSION_STRING) string(STRIP ${XRootD_VERSION_STRING} XRootD_VERSION_STRING) endif() if(XRootD_VERSION_STRING MATCHES "Format:" AND IS_DIRECTORY ${PROJECT_SOURCE_DIR}/.git) find_package(Git QUIET) if(Git_FOUND) message(VERBOSE "Determining version with git") execute_process(COMMAND ${GIT_EXECUTABLE} --git-dir ${PROJECT_SOURCE_DIR}/.git describe OUTPUT_VARIABLE XRootD_VERSION_STRING ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE) set(XRootD_VERSION_STRING "${XRootD_VERSION_STRING}" CACHE INTERNAL "XRootD Version") endif() endif() if(XRootD_VERSION_STRING MATCHES "^v?([0-9]+)[.]*([0-9]*)[.]*([0-9]*)[.]*([0-9]*)") set(XRootD_VERSION_MAJOR ${CMAKE_MATCH_1}) if(${CMAKE_MATCH_COUNT} GREATER 1) set(XRootD_VERSION_MINOR ${CMAKE_MATCH_2}) else() set(XRootD_VERSION_MINOR 0) endif() if(${CMAKE_MATCH_COUNT} GREATER 2) set(XRootD_VERSION_PATCH ${CMAKE_MATCH_3}) else() set(XRootD_VERSION_PATCH 0) endif() if(${CMAKE_MATCH_COUNT} GREATER 3) set(XRootD_VERSION_TWEAK ${CMAKE_MATCH_4}) else() set(XRootD_VERSION_TWEAK 0) endif() math(EXPR XRootD_VERSION_NUMBER "10000 * ${XRootD_VERSION_MAJOR} + 100 * ${XRootD_VERSION_MINOR} + ${XRootD_VERSION_PATCH}" OUTPUT_FORMAT DECIMAL) else() message(WARNING "Failed to determine XRootD version, using a timestamp as fallback." "You can override this by setting -DXRootD_VERSION_STRING=x.y.z during configuration.") set(XRootD_VERSION_MAJOR 5) set(XRootD_VERSION_MINOR 7) set(XRootD_VERSION_PATCH 0) set(XRootD_VERSION_TWEAK 0) set(XRootD_VERSION_NUMBER 1000000) string(TIMESTAMP XRootD_VERSION_STRING "v${XRootD_VERSION_MAJOR}.${XRootD_VERSION_MINOR}-rc%Y%m%d" UTC) endif() if(XRootD_VERSION_STRING MATCHES "[_-](.*)$") set(XRootD_VERSION_SUFFIX ${CMAKE_MATCH_1}) endif() string(REGEX MATCH "[0-9]+[.]*[0-9]*[.]*[0-9]*[.]*[0-9]*(-rc)?[0-9].*" XRootD_VERSION ${XRootD_VERSION_STRING}) message(DEBUG "XRootD_VERSION_STRING = '${XRootD_VERSION_STRING}'") message(DEBUG "XRootD_VERSION_NUMBER = '${XRootD_VERSION_NUMBER}'") message(DEBUG "XRootD_VERSION_MAJOR = '${XRootD_VERSION_MAJOR}'") message(DEBUG "XRootD_VERSION_MINOR = '${XRootD_VERSION_MINOR}'") message(DEBUG "XRootD_VERSION_PATCH = '${XRootD_VERSION_PATCH}'") message(DEBUG "XRootD_VERSION_TWEAK = '${XRootD_VERSION_TWEAK}'") message(DEBUG "XRootD_VERSION_SUFFIX = '${XRootD_VERSION_SUFFIX}'") message(DEBUG "XRootD_VERSION = '${XRootD_VERSION}'") xrootd-5.6.9/cmake_uninstall.cmake.in000066400000000000000000000016551457266313600176560ustar00rootroot00000000000000IF(NOT EXISTS "@CMAKE_CURRENT_BINARY_DIR@/install_manifest.txt") MESSAGE(FATAL_ERROR "Cannot find install manifest: \"@CMAKE_CURRENT_BINARY_DIR@/install_manifest.txt\"") ENDIF(NOT EXISTS "@CMAKE_CURRENT_BINARY_DIR@/install_manifest.txt") FILE(READ "@CMAKE_CURRENT_BINARY_DIR@/install_manifest.txt" files) STRING(REGEX REPLACE "\n" ";" files "${files}") FOREACH(file ${files}) MESSAGE(STATUS "Uninstalling \"$ENV{DESTDIR}${file}\"") IF(EXISTS "$ENV{DESTDIR}${file}") EXEC_PROGRAM( "@CMAKE_COMMAND@" ARGS "-E remove \"$ENV{DESTDIR}${file}\"" OUTPUT_VARIABLE rm_out RETURN_VALUE rm_retval ) IF(NOT "${rm_retval}" STREQUAL 0) MESSAGE(FATAL_ERROR "Problem when removing \"$ENV{DESTDIR}${file}\"") ENDIF(NOT "${rm_retval}" STREQUAL 0) ELSE(EXISTS "$ENV{DESTDIR}${file}") MESSAGE(STATUS "File \"$ENV{DESTDIR}${file}\" does not exist.") ENDIF(EXISTS "$ENV{DESTDIR}${file}") ENDFOREACH(file) xrootd-5.6.9/debian/000077500000000000000000000000001457266313600143115ustar00rootroot00000000000000xrootd-5.6.9/debian/compat000066400000000000000000000000031457266313600155100ustar00rootroot0000000000000013 xrootd-5.6.9/debian/control000066400000000000000000000246321457266313600157230ustar00rootroot00000000000000Source: xrootd Section: net Priority: optional Standards-Version: 4.6.2 Build-Depends: debhelper (>= 13), dh-python, attr, cmake, libgtest-dev, libcppunit-dev, libisal-dev, pkg-config, libfuse-dev [linux-any kfreebsd-any], libkrb5-dev, libcurl4-openssl-dev, libtinyxml-dev, libxml2-dev, ncurses-dev, libssl-dev, libreadline-dev, zlib1g-dev, libsystemd-dev [linux-any], python3-dev, python3-pip, python3-wheel, libjson-c-dev, libmacaroons-dev, uuid-dev, uuid-runtime, voms-dev, libscitokens-dev, davix-dev, librados-dev [i386 amd64 armel armhf arm64 mips mipsel mips64el powerpc ppc64el riscv64 s390x], libradospp-dev [i386 amd64 armel armhf arm64 mips mipsel mips64el powerpc ppc64el riscv64 s390x], libradosstriper-dev [i386 amd64 armel armhf arm64 mips mipsel mips64el powerpc ppc64el riscv64 s390x] Build-Depends-Indep: dh-sequence-sphinxdoc, doxygen, graphviz, python3-sphinx Homepage: http://xrootd.org/ Maintainer: XRootD Developers Vcs-Browser: https://github.com/xrootd/xrootd Vcs-Git: https://github.com/xrootd/xrootd.git Package: xrootd-server Architecture: any Multi-Arch: foreign Section: net Depends: xrootd-plugins (= ${binary:Version}), xrootd-server-plugins (= ${binary:Version}), expect, logrotate, adduser, ${perl:Depends}, ${shlibs:Depends}, ${misc:Depends} Description: Extended ROOT file server The Extended root file server consists of a file server called xrootd and a cluster management server called cmsd. . The xrootd server was developed for the root analysis framework to serve root files. However, the server is agnostic to file types and provides POSIX-like access to any type of file. . The cmsd server is the next generation version of the olbd server, originally developed to cluster and load balance Objectivity/DB AMS database servers. It provides enhanced capability along with lower latency and increased throughput. Package: libxrdapputils2 Architecture: any Multi-Arch: same Section: libs Depends: ${shlibs:Depends}, ${misc:Depends} Description: Utilities library for xrootd applications This package contains the xrootd utilities library for applications. Package: libxrdcrypto2 Architecture: any Multi-Arch: same Section: libs Depends: ${shlibs:Depends}, ${misc:Depends} Description: Cryptograpic library for xrootd This package contains the xrootd cryptograpic library. Package: libxrdcryptolite2 Architecture: any Multi-Arch: same Section: libs Depends: ${shlibs:Depends}, ${misc:Depends} Description: Light version of cryptograpic library for xrootd This package contains the light version of the xrootd cryptograpic library. Package: libxrdutils3 Architecture: any Multi-Arch: same Section: libs Depends: ${shlibs:Depends}, ${misc:Depends} Description: Utilities library for xrootd This package contains the xrootd utilities library. Package: libxrdxml3 Architecture: any Multi-Arch: same Section: libs Depends: ${shlibs:Depends}, ${misc:Depends} Description: XML library for xrootd This package contains the xrootd XML library. Package: xrootd-plugins Architecture: any Multi-Arch: same Section: libs Depends: ${shlibs:Depends}, ${misc:Depends} Description: Plugins used by xrootd servers and clients This package contains plugins used by the xrootd servers and clients. Package: libxrootd-dev Architecture: any Multi-Arch: same Section: libdevel Depends: libxrdapputils2 (= ${binary:Version}), libxrdcrypto2 (= ${binary:Version}), libxrdcryptolite2 (= ${binary:Version}), libxrdutils3 (= ${binary:Version}), libxrdxml3 (= ${binary:Version}), ${misc:Depends} Description: Development files for xrootd This package contains header files and development libraries for xrootd development. Package: libxrdcl3 Architecture: any Multi-Arch: same Section: libs Depends: ${shlibs:Depends}, ${misc:Depends} Recommends: xrootd-plugins (= ${binary:Version}), xrootd-client-plugins(= ${binary:Version}) Description: Client library for xrootd This package contains the xrootd client library. Package: libxrdec1 Architecture: any Multi-Arch: same Section: libs Depends: ${shlibs:Depends}, ${misc:Depends} Recommends: xrootd-plugins (= ${binary:Version}), xrootd-client-plugins(= ${binary:Version}) Description: Client library for xrootd This package contains the xrootd client library for erasure coding. Package: libxrdffs3 Architecture: any Multi-Arch: same Section: libs Depends: ${shlibs:Depends}, ${misc:Depends} Description: File protocol library for xrootd This package contains the xrootd file protocol library. Package: libxrdposix3 Architecture: any Multi-Arch: same Section: libs Depends: ${shlibs:Depends}, ${misc:Depends} Description: Posix interface library for xrootd This package contains the xrootd Posix interface library. Package: libxrdssilib2 Architecture: any Multi-Arch: same Section: libs Depends: ${shlibs:Depends}, ${misc:Depends} Description: Server internals library for xrootd This package contains an xrootd server internals library. Package: libxrdssishmap2 Architecture: any Multi-Arch: same Section: libs Depends: ${shlibs:Depends}, ${misc:Depends} Description: Server internals library for xrootd This package contains an xrootd server internals library. Package: xrootd-client-plugins Architecture: any Multi-Arch: same Section: libs Depends: ${shlibs:Depends}, ${misc:Depends} Description: Plugins used by xrootd clients This package contains plugins used by xrootd clients. Package: xrootd-client-http-plugins Architecture: any Multi-Arch: same Section: libs Depends: ${shlibs:Depends}, ${misc:Depends} Description: HTTP client plugin for XRootD client This package contains an XRootD client plugin which allows XRootD to interact with HTTP repositories. Package: libxrootd-client-dev Architecture: any Multi-Arch: same Section: libdevel Depends: libxrdcl3 (= ${binary:Version}), libxrdffs3 (= ${binary:Version}), libxrdposix3 (= ${binary:Version}), libxrootd-dev (= ${binary:Version}), ${misc:Depends} Description: Development files for xrootd clients This package contains header files and development libraries for xrootd client development. Package: libxrdhttputils2 Architecture: any Multi-Arch: same Section: libs Depends: ${shlibs:Depends}, ${misc:Depends} Description: HTTP protocol utilities library for xrootd This package contains the xrootd HTTP protocol utilities library. Package: libxrdserver3 Architecture: any Multi-Arch: same Section: libs Depends: ${shlibs:Depends}, ${misc:Depends} Recommends: xrootd-plugins (= ${binary:Version}), xrootd-server-plugins(= ${binary:Version}) Description: Server library for xrootd This package contains the xrootd server library. Package: xrootd-server-plugins Architecture: any Multi-Arch: same Section: libs Depends: ${shlibs:Depends}, ${misc:Depends} Description: Plugins used by xrootd servers This package contains plugins used by xrootd servers. Package: libxrootd-server-dev Architecture: any Multi-Arch: same Section: libdevel Depends: libxrdhttputils2 (= ${binary:Version}), libxrdserver3 (= ${binary:Version}), libxrootd-dev (= ${binary:Version}), libxrootd-client-dev (= ${binary:Version}), ${misc:Depends} Description: Development files for xrootd servers This package contains header files and development libraries for xrootd server development. Package: libxrootd-private-dev Architecture: any Multi-Arch: same Section: libdevel Depends: libxrdssilib2 (= ${binary:Version}), libxrdssishmap2 (= ${binary:Version}), libxrootd-dev (= ${binary:Version}), libxrootd-client-dev (= ${binary:Version}), libxrootd-server-dev (= ${binary:Version}), ${misc:Depends} Description: Private xrootd headers This package contains some private xrootd headers. Backward and forward compatibility between versions is not guaranteed for these headers. Package: xrootd-client Architecture: any Multi-Arch: foreign Section: net Depends: xrootd-plugins (= ${binary:Version}), xrootd-client-plugins(= ${binary:Version}), ${shlibs:Depends}, ${misc:Depends} Description: Xrootd command line client tools This package contains the command line tools used to communicate with xrootd servers. Package: xrootd-fuse Architecture: linux-any kfreebsd-any Multi-Arch: foreign Section: net Depends: xrootd-plugins (= ${binary:Version}), xrootd-client-plugins (= ${binary:Version}), fuse, ${shlibs:Depends}, ${misc:Depends} Description: Xrootd FUSE tool This package contains the FUSE (file system in user space) xrootd mount tool. Package: xrootd-voms-plugins Architecture: any Multi-Arch: same Section: libs Depends: ${shlibs:Depends}, ${misc:Depends} Description: VOMS attribute extractor plugin for XRootD This package contains the xrootd VOMS attribute extractor plugin. Package: xrootd-scitokens-plugins Architecture: any Multi-Arch: same Section: libs Depends: ${shlibs:Depends}, ${misc:Depends} Description: SciTokens authorization support for XRootD This ACC (authorization) plugin for the XRootD framework utilizes the SciTokens library to validate and extract authorization claims from a SciToken passed during a transfer. Configured appropriately, this allows the XRootD server admin to delegate authorization decisions for a subset of the namespace to an external issuer. Package: libxrdcephposix0 Architecture: i386 amd64 armel armhf arm64 mips mipsel mips64el powerpc ppc64el riscv64 s390x Multi-Arch: same Section: libs Depends: ${shlibs:Depends}, ${misc:Depends} Description: Ceph posix library for xrootd This package contains an xrootd library used by the ceph plugins. Package: xrootd-ceph-plugins Architecture: i386 amd64 armel armhf arm64 mips mipsel mips64el powerpc ppc64el riscv64 s390x Multi-Arch: same Section: libs Depends: ${shlibs:Depends}, ${misc:Depends} Description: XRootD plugin for interfacing with the Ceph storage platform The xrootd-ceph is an OSS layer plugin for the XRootD server for interfacing with the Ceph storage platform. Package: python3-xrootd Architecture: any Multi-Arch: foreign Section: python Provides: ${python3:Provides} Depends: xrootd-plugins (= ${binary:Version}), xrootd-client-plugins (= ${binary:Version}), ${python3:Depends}, ${shlibs:Depends}, ${misc:Depends} Description: Python 3 bindings for xrootd This package contains Python 3 bindings for xrootd. Package: xrootd-doc Architecture: all Multi-Arch: foreign Section: doc Depends: ${sphinxdoc:Depends}, ${misc:Depends} Built-Using: ${sphinxdoc:Built-Using} Description: Developer documentation for the xrootd libraries This package contains the API documentation of the xrootd libraries. xrootd-5.6.9/debian/copyright000066400000000000000000000306321457266313600162500ustar00rootroot00000000000000Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ Upstream-Name: xrootd Upstream-Contact: https://xrootd.slac.stanford.edu/ Files: * Copyright: 2000-2023 Board of Trustees of the Leland Stanford, Jr. University. Produced under contract DE-AC02-76-SF00515 with the US Department of Energy. All rights reserved. License: LGPL-3 and BSD-3-clause-modified The copyright holder's institutional names may not be used to endorse or promote products derived from this software without specific prior written permission. . This file is part of the XRootD software suite. . XRootD is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. . XRootD is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. . You should have received a copy of the GNU Lesser General Public License along with XRootD in a file called COPYING.LESSER (LGPL license) and file COPYING (GPL license). If not, see . . On Debian systems the full text of the GNU lesser general public license version 3 can be found in /usr/share/common-licenses/LGPL-3. . Prior to September 2nd, 2012 the XRootD software suite was licensed under a modified BSD license shown below. This applies to all code that was in the XRootD git repository prior to that date. All code is now licensed under LGPL. . Conditions of Use . Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: . a. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. b. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. c. Neither the name of the Leland Stanford, Jr. University nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. d. Products derived from this software that do not adhere to the xrootd or cmsd protocol specifications may not use the acronyms 'cmsd', 'Scalla', 'xroot', and 'xrootd', regardless of capitalization, to describe such derivative works. . DISCLAIMER . THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 'AS IS' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. Files: bindings/python/* src/XrdApps/XrdClProxyPlugin/* src/XrdApps/XrdClRecordPlugin/* src/XrdCeph/* src/XrdCks/XrdCksCalczcrc32.cc src/XrdCl/* src/XrdEc/* src/XrdHttp/* src/XrdOssCsi/* src/XrdOuc/XrdOucCompiler.hh src/XrdOuc/XrdOucEnum.hh src/XrdSys/XrdSysKernelBuffer.hh src/XrdTls/XrdTlsContext.* src/XrdTls/XrdTlsSocket.* src/XrdZip/* tests/common/* tests/XrdCephTests/* tests/XrdClTests/* tests/XrdEcTests/* utils/xrootd-config Copyright: 2011-2023 European Organization for Nuclear Research (CERN) Comment: In applying this licence, CERN does not waive the privileges and immunities granted to it by virtue of its status as an Intergovernmental Organization or submit itself to any jurisdiction. License: LGPL-3 Files: src/XrdCks/XrdCksCalcadler32.hh src/XrdOuc/XrdOucCRC32C.* Copyright: 1995-1998, 2013, 2015 Mark Adler License: zlib/libpng Files: src/XrdCl/XrdClLocalFileHandler.* src/XrdCl/XrdClLocalFileTask.* src/XrdCms/XrdCmsRedirLocal.* tests/XrdClTests/LocalFileHandlerTest.cc Copyright: 2017, 2019 GSI Helmholtzzentrum für Schwerionenforschung GmbH License: LGPL-3 Files: src/XrdCl/XrdClMonitor.hh Copyright: 2012 Board of Trustees of the Leland Stanford, Jr., University 2012 European Organization for Nuclear Research (CERN) License: LGPL-3 Files: src/XrdOuc/XrdOucJson.hh Copyright: 2013-2019 Niels Lohmann License: Expat Files: src/XrdOuc/XrdOucSHA3.* Copyright: 2015 Markku-Juhani O. Saarinen License: Expat Files: src/XrdOuc/XrdOucUri.* Copyright: 2016 by David Farrell License: BSD-2-clause Files: src/XrdSciTokens/* Copyright: Brian Bockelman, Andrew Hanushevsky, Derek Weitzel License: Apache-2.0 On Debian systems the full text of the Apache license version 2 can be found in /usr/share/common-licenses/Apache-2.0. Files: src/XrdSciTokens/vendor/inih/* Copyright: 2009, Ben Hoyt. All rights reserved. License: BSD-3-clause Files: src/XrdSciTokens/vendor/picojson/* Copyright: 2009-2010 Cybozu Labs, Inc. 2011-2014 Kazuho Oku All rights reserved. License: BSD-2-clause Files: src/XrdSciTokens/vendor/picojson/picotest/* Copyright: 2014 DeNA Co., Ltd. License: Expat Files: src/XrdSecztn/XrdSecztn.cc Copyright: 2015 Erwin Jansen License: Expat Files: src/XrdSys/XrdSysFallocate.* Copyright: 2010 Mozilla Foundation. All Rights Reserved. 2015, 2016 R.J.V. Bertin for KDE project. Comment: Original license allows modification / redistribution under LGPL 2.1 or higher. Redistributed under LGPL-3 or higher. License: LGPL-3 Files: src/XrdTls/XrdTlsHostcheck.* Copyright: 1998-2012 Daniel Stenberg, , et al. License: curl This software is licensed as described in the file COPYING, which you should have received as part of this distribution. The terms are also available at http://curl.haxx.se/docs/copyright.html. . You may opt to use, copy, modify, merge, publish, distribute and/or sell copies of the Software, and permit persons to whom the Software is furnished to do so, under the terms of the COPYING file. . This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. Files: src/XrdTls/XrdTlsNotaryUtils.* Copyright: 2012 iSEC Partners. License: Expat Files: src/XrdXml/tinyxml/* Copyright: 2000-2006 Lee Thomason (www.grinninglizard.com) Comment: Not used in debian build, libtinyxml-dev used instead. License: zlib/libpng Files: debian/* Copyright: 2020-2023 Mattias Ellert License: LGPL-3 License: LGPL-3 This file is part of the XRootD software suite. . XRootD is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. . XRootD is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. . You should have received a copy of the GNU Lesser General Public License along with XRootD in a file called COPYING.LESSER (LGPL license) and file COPYING (GPL license). If not, see . . On Debian systems the full text of the GNU lesser general public license version 3 can be found in /usr/share/common-licenses/LGPL-3. License: zlib/libpng This software is provided 'as-is', without any express or implied warranty. In no event will the authors be held liable for any damages arising from the use of this software. . Permission is granted to anyone to use this software for any purpose, including commercial applications, and to alter it and redistribute it freely, subject to the following restrictions: . 1. The origin of this software must not be misrepresented; you must not claim that you wrote the original software. If you use this software in a product, an acknowledgment in the product documentation would be appreciated but is not required. . 2. Altered source versions must be plainly marked as such, and must not be misrepresented as being the original software. . 3. This notice may not be removed or altered from any source distribution. License: BSD-2-clause Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: . 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. . THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. License: BSD-3-clause Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: . * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of [insert "CERN" or "Ben Hoyt" as appropriate] nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. . THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. License: Expat Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: . The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. . THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. xrootd-5.6.9/debian/libxrdapputils1.install000066400000000000000000000000361457266313600210270ustar00rootroot00000000000000usr/lib/*/libXrdAppUtils.so.* xrootd-5.6.9/debian/libxrdapputils2.install000066400000000000000000000000371457266313600210310ustar00rootroot00000000000000/usr/lib/*/libXrdAppUtils.so.* xrootd-5.6.9/debian/libxrdcephposix0.install000066400000000000000000000000401457266313600211620ustar00rootroot00000000000000/usr/lib/*/libXrdCephPosix.so.* xrootd-5.6.9/debian/libxrdcl2.install000066400000000000000000000000301457266313600175570ustar00rootroot00000000000000usr/lib/*/libXrdCl.so.* xrootd-5.6.9/debian/libxrdcl3.install000066400000000000000000000000311457266313600175610ustar00rootroot00000000000000/usr/lib/*/libXrdCl.so.* xrootd-5.6.9/debian/libxrdcrypto1.install000066400000000000000000000000341457266313600205040ustar00rootroot00000000000000usr/lib/*/libXrdCrypto.so.* xrootd-5.6.9/debian/libxrdcrypto2.install000066400000000000000000000000351457266313600205060ustar00rootroot00000000000000/usr/lib/*/libXrdCrypto.so.* xrootd-5.6.9/debian/libxrdcryptolite1.install000066400000000000000000000000401457266313600213570ustar00rootroot00000000000000usr/lib/*/libXrdCryptoLite.so.* xrootd-5.6.9/debian/libxrdcryptolite2.install000066400000000000000000000000411457266313600213610ustar00rootroot00000000000000/usr/lib/*/libXrdCryptoLite.so.* xrootd-5.6.9/debian/libxrdec1.install000066400000000000000000000000311457266313600175500ustar00rootroot00000000000000/usr/lib/*/libXrdEc.so.* xrootd-5.6.9/debian/libxrdffs2.install000066400000000000000000000000311457266313600177400ustar00rootroot00000000000000usr/lib/*/libXrdFfs.so.* xrootd-5.6.9/debian/libxrdffs3.install000066400000000000000000000000321457266313600177420ustar00rootroot00000000000000/usr/lib/*/libXrdFfs.so.* xrootd-5.6.9/debian/libxrdhttputils1.install000066400000000000000000000000371457266313600212270ustar00rootroot00000000000000usr/lib/*/libXrdHttpUtils.so.* xrootd-5.6.9/debian/libxrdhttputils2.install000066400000000000000000000000401457266313600212220ustar00rootroot00000000000000/usr/lib/*/libXrdHttpUtils.so.* xrootd-5.6.9/debian/libxrdposix2.install000066400000000000000000000000331457266313600203260ustar00rootroot00000000000000usr/lib/*/libXrdPosix.so.* xrootd-5.6.9/debian/libxrdposix3.install000066400000000000000000000002531457266313600203330ustar00rootroot00000000000000/usr/lib/*/libXrdPosix.so.* /usr/lib/*/libXrdPosixPreload.so.* # This lib may be used for LD_PRELOAD so the .so link needs to be included /usr/lib/*/libXrdPosixPreload.so xrootd-5.6.9/debian/libxrdposix3.lintian-overrides000066400000000000000000000003461457266313600223260ustar00rootroot00000000000000# This lib may be used for LD_PRELOAD so the .so link needs to be included link-to-shared-library-in-wrong-package * [usr/lib/*/libXrdPosixPreload.so] lacks-unversioned-link-to-shared-library * [usr/lib/*/libXrdPosixPreload.so.*] xrootd-5.6.9/debian/libxrdserver2.install000066400000000000000000000000341457266313600204730ustar00rootroot00000000000000usr/lib/*/libXrdServer.so.* xrootd-5.6.9/debian/libxrdserver3.install000066400000000000000000000000351457266313600204750ustar00rootroot00000000000000/usr/lib/*/libXrdServer.so.* xrootd-5.6.9/debian/libxrdssilib1.install000066400000000000000000000000341457266313600204510ustar00rootroot00000000000000usr/lib/*/libXrdSsiLib.so.* xrootd-5.6.9/debian/libxrdssilib2.install000066400000000000000000000000351457266313600204530ustar00rootroot00000000000000/usr/lib/*/libXrdSsiLib.so.* xrootd-5.6.9/debian/libxrdssishmap1.install000066400000000000000000000000361457266313600210150ustar00rootroot00000000000000usr/lib/*/libXrdSsiShMap.so.* xrootd-5.6.9/debian/libxrdssishmap2.install000066400000000000000000000000371457266313600210170ustar00rootroot00000000000000/usr/lib/*/libXrdSsiShMap.so.* xrootd-5.6.9/debian/libxrdutils2.install000066400000000000000000000000331457266313600203240ustar00rootroot00000000000000usr/lib/*/libXrdUtils.so.* xrootd-5.6.9/debian/libxrdutils3.install000066400000000000000000000000341457266313600203260ustar00rootroot00000000000000/usr/lib/*/libXrdUtils.so.* xrootd-5.6.9/debian/libxrdxml2.install000066400000000000000000000000311457266313600177620ustar00rootroot00000000000000usr/lib/*/libXrdXml.so.* xrootd-5.6.9/debian/libxrdxml3.install000066400000000000000000000000321457266313600177640ustar00rootroot00000000000000/usr/lib/*/libXrdXml.so.* xrootd-5.6.9/debian/libxrootd-client-dev.install000066400000000000000000000002271457266313600217400ustar00rootroot00000000000000/usr/include/xrootd/XrdCl /usr/include/xrootd/XrdPosix /usr/lib/*/libXrdCl.so /usr/lib/*/libXrdEc.so /usr/lib/*/libXrdFfs.so /usr/lib/*/libXrdPosix.so xrootd-5.6.9/debian/libxrootd-dev.install000066400000000000000000000006621457266313600204670ustar00rootroot00000000000000/usr/bin/xrootd-config /usr/include/xrootd/XProtocol /usr/include/xrootd/Xrd /usr/include/xrootd/XrdCks /usr/include/xrootd/XrdNet /usr/include/xrootd/XrdOuc /usr/include/xrootd/XrdSec /usr/include/xrootd/XrdSys /usr/include/xrootd/XrdXml /usr/include/xrootd/XrdVersion.hh /usr/lib/*/libXrdAppUtils.so /usr/lib/*/libXrdCrypto.so /usr/lib/*/libXrdCryptoLite.so /usr/lib/*/libXrdUtils.so /usr/lib/*/libXrdXml.so /usr/lib/*/cmake/XRootD xrootd-5.6.9/debian/libxrootd-private-dev.install000066400000000000000000000001241457266313600221300ustar00rootroot00000000000000/usr/include/xrootd/private /usr/lib/*/libXrdSsiLib.so /usr/lib/*/libXrdSsiShMap.so xrootd-5.6.9/debian/libxrootd-server-dev.install000066400000000000000000000004251457266313600217700ustar00rootroot00000000000000/usr/include/xrootd/XrdAcc /usr/include/xrootd/XrdCms /usr/include/xrootd/XrdHttp /usr/include/xrootd/XrdOfs /usr/include/xrootd/XrdOss /usr/include/xrootd/XrdPfc /usr/include/xrootd/XrdSfs /usr/include/xrootd/XrdXrootd /usr/lib/*/libXrdHttpUtils.so /usr/lib/*/libXrdServer.so xrootd-5.6.9/debian/python3-xrootd.install000066400000000000000000000001751457266313600206250ustar00rootroot00000000000000/usr/lib/python3/dist-packages/xrootd-*.*-info /usr/lib/python3/dist-packages/pyxrootd /usr/lib/python3/dist-packages/XRootD xrootd-5.6.9/debian/rules000077500000000000000000000103411457266313600153700ustar00rootroot00000000000000#!/usr/bin/make -f export LC_ALL=C export DH_VERBOSE=1 export PYBUILD_NAME=xrootd export DEB_BUILD_MAINT_OPTIONS = hardening=+all optimize=-lto %: dh $@ --with python3 --buildsystem cmake override_dh_auto_configure: dh_auto_configure -- \ -DENABLE_FUSE:BOOL=1 \ -DENABLE_HTTP:BOOL=1 \ -DENABLE_KRB5:BOOL=1 \ -DENABLE_MACAROONS:BOOL=1 \ -DENABLE_PYTHON:BOOL=1 \ -DENABLE_READLINE:BOOL=1 \ -DENABLE_SCITOKENS:BOOL=1 \ -DENABLE_VOMS:BOOL=1 \ -DENABLE_XRDCLHTTP:BOOL=1 \ -DENABLE_XRDEC:BOOL=1 \ -DENABLE_TESTS:BOOL=1 \ -DFORCE_ENABLED:BOOL=1 \ -DINSTALL_PYTHON_BINDINGS:BOOL=0 \ -DUSE_SYSTEM_ISAL:BOOL=1 \ -DXRDCEPH_SUBMODULE:BOOL=1 override_dh_auto_build: dh_auto_build doxygen Doxyfile override_dh_auto_clean: dh_auto_clean rm -rf doxydoc rm -rf bindings/python/docs/build override_dh_auto_install: dh_auto_install python3 -m pip install --target debian/tmp/usr/lib/python3/dist-packages \ --no-deps --no-build-isolation --disable-pip-version-check --verbose \ --ignore-installed --use-pep517 obj-$(DEB_HOST_MULTIARCH)/bindings/python rm -f debian/tmp/usr/bin/xrdshmap rm -f debian/tmp/usr/bin/test-runner rm -f debian/tmp/usr/lib/*/libXrd*Test* rm -f debian/tmp/usr/lib/*/libXrdCephPosix.so rm -f debian/tmp/usr/lib/python3/dist-packages/xrootd-*.*-info/direct_url.json rm -f debian/tmp/usr/lib/python3/dist-packages/xrootd-*.*-info/RECORD [ -r debian/tmp/usr/lib/python3/dist-packages/xrootd-*.*-info/INSTALLER ] && \ sed -i -e s/pip/dpkg/ \ debian/tmp/usr/lib/python3/dist-packages/xrootd-*.*-info/INSTALLER LD_LIBRARY_PATH=$${LD_LIBRARY_PATH}:$(CURDIR)/debian/tmp/usr/lib/$(DEB_HOST_MULTIARCH) \ PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=$(CURDIR)/debian/tmp/usr/lib/python3/dist-packages \ make -C bindings/python/docs html && \ mv bindings/python/docs/build/html bindings/python/docs/build/python # Service unit files mkdir -p debian/tmp/lib/systemd/system install -m 644 packaging/common/xrootd@.service debian/tmp/lib/systemd/system install -m 644 packaging/common/xrootd@.socket debian/tmp/lib/systemd/system install -m 644 packaging/common/xrdhttp@.socket debian/tmp/lib/systemd/system install -m 644 packaging/common/cmsd@.service debian/tmp/lib/systemd/system install -m 644 packaging/common/frm_xfrd@.service debian/tmp/lib/systemd/system install -m 644 packaging/common/frm_purged@.service debian/tmp/lib/systemd/system mkdir -p debian/tmp/usr/lib/tmpfiles.d install -m 644 packaging/rhel/xrootd.tmpfiles debian/tmp/usr/lib/tmpfiles.d/xrootd.conf # Server config mkdir -p debian/tmp/etc/xrootd install -m 644 -p packaging/common/xrootd-clustered.cfg \ debian/tmp/etc/xrootd/xrootd-clustered.cfg install -m 644 -p packaging/common/xrootd-standalone.cfg \ debian/tmp/etc/xrootd/xrootd-standalone.cfg install -m 644 -p packaging/common/xrootd-filecache-clustered.cfg \ debian/tmp/etc/xrootd/xrootd-filecache-clustered.cfg install -m 644 -p packaging/common/xrootd-filecache-standalone.cfg \ debian/tmp/etc/xrootd/xrootd-filecache-standalone.cfg sed 's!/usr/lib64/!!' packaging/common/xrootd-http.cfg > \ debian/tmp/etc/xrootd/xrootd-http.cfg # Client config mkdir -p debian/tmp/etc/xrootd/client.plugins.d install -m 644 -p packaging/common/client.conf \ debian/tmp/etc/xrootd/client.conf sed 's!/usr/lib/!!' packaging/common/client-plugin.conf.example > \ debian/tmp/etc/xrootd/client.plugins.d/client-plugin.conf.example sed -e 's!/usr/lib64/!!' -e 's!-5!!' packaging/common/recorder.conf > \ debian/tmp/etc/xrootd/client.plugins.d/recorder.conf sed 's!/usr/lib64/!!' packaging/common/http.client.conf.example > \ debian/tmp/etc/xrootd/client.plugins.d/xrdcl-http-plugin.conf chmod 644 debian/tmp/usr/share/xrootd/utils/XrdCmsNotify.pm sed 's!/usr/bin/env perl!/usr/bin/perl!' -i \ debian/tmp/usr/share/xrootd/utils/netchk \ debian/tmp/usr/share/xrootd/utils/XrdCmsNotify.pm \ debian/tmp/usr/share/xrootd/utils/XrdOlbMonPerf sed 's!/usr/bin/env bash!/bin/bash!' -i \ debian/tmp/usr/bin/xrootd-config mkdir -p debian/tmp/etc/xrootd/config.d mkdir -p debian/tmp/var/log/xrootd mkdir -p debian/tmp/var/spool/xrootd mkdir -p debian/tmp/etc/logrotate.d install -m 644 -p packaging/common/xrootd.logrotate \ debian/tmp/etc/logrotate.d/xrootd xrootd-5.6.9/debian/source/000077500000000000000000000000001457266313600156115ustar00rootroot00000000000000xrootd-5.6.9/debian/source/format000066400000000000000000000000141457266313600170170ustar00rootroot000000000000003.0 (quilt) xrootd-5.6.9/debian/xrootd-ceph-plugins.install000066400000000000000000000000731457266313600216140ustar00rootroot00000000000000/usr/lib/*/libXrdCeph-5.so /usr/lib/*/libXrdCephXattr-5.so xrootd-5.6.9/debian/xrootd-ceph-plugins.lintian-overrides000066400000000000000000000001651457266313600236060ustar00rootroot00000000000000# These are plugins - no soname on purpose sharedobject-in-library-directory-missing-soname [usr/lib/*/libXrd*-5.so] xrootd-5.6.9/debian/xrootd-client-devel.install000066400000000000000000000000001457266313600215570ustar00rootroot00000000000000xrootd-5.6.9/debian/xrootd-client-http-plugins.install000066400000000000000000000001211457266313600231220ustar00rootroot00000000000000/usr/lib/*/libXrdClHttp-5.so /etc/xrootd/client.plugins.d/xrdcl-http-plugin.conf xrootd-5.6.9/debian/xrootd-client-http-plugins.lintian-overrides000066400000000000000000000001651457266313600251220ustar00rootroot00000000000000# These are plugins - no soname on purpose sharedobject-in-library-directory-missing-soname [usr/lib/*/libXrd*-5.so] xrootd-5.6.9/debian/xrootd-client-libs.install000066400000000000000000000000001457266313600214110ustar00rootroot00000000000000xrootd-5.6.9/debian/xrootd-client-plugins.install000066400000000000000000000003001457266313600221440ustar00rootroot00000000000000/usr/lib/*/libXrdClProxyPlugin-5.so /usr/lib/*/libXrdClRecorder-5.so /etc/xrootd/client.conf /etc/xrootd/client.plugins.d/client-plugin.conf.example /etc/xrootd/client.plugins.d/recorder.conf xrootd-5.6.9/debian/xrootd-client-plugins.lintian-overrides000066400000000000000000000001651457266313600241450ustar00rootroot00000000000000# These are plugins - no soname on purpose sharedobject-in-library-directory-missing-soname [usr/lib/*/libXrd*-5.so] xrootd-5.6.9/debian/xrootd-client.install000066400000000000000000000006351457266313600205000ustar00rootroot00000000000000/usr/bin/xrdadler32 /usr/bin/xrdcks /usr/bin/xrdcopy /usr/bin/xrdcp /usr/bin/xrdcrc32c /usr/bin/xrdfs /usr/bin/xrdgsiproxy /usr/bin/xrdgsitest /usr/bin/xrdmapc /usr/bin/xrdpinls /usr/bin/xrdreplay /usr/share/man/man1/xrdadler32.1 /usr/share/man/man1/xrdcopy.1 /usr/share/man/man1/xrdcp.1 /usr/share/man/man1/xrdfs.1 /usr/share/man/man1/xrdgsiproxy.1 /usr/share/man/man1/xrdgsitest.1 /usr/share/man/man1/xrdmapc.1 xrootd-5.6.9/debian/xrootd-clients.install000066400000000000000000000005721457266313600206630ustar00rootroot00000000000000usr/bin/xrdadler32 usr/bin/xrdcrc32c usr/bin/xrdcopy usr/bin/xrdcp usr/bin/xrdfs usr/bin/xrdgsiproxy usr/bin/xrdmapc usr/share/man/man1/xrdadler32.1* usr/share/man/man1/xrdcopy.1* usr/share/man/man1/xrdcp.1* usr/share/man/man1/xrdfs.1* usr/share/man/man1/xrdgsiproxy.1* usr/share/man/man1/xrdmapc.1* etc/xrootd/client.plugins.d/client-plugin.conf.example etc/xrootd/client.conf xrootd-5.6.9/debian/xrootd-devel.install000066400000000000000000000000001457266313600203030ustar00rootroot00000000000000xrootd-5.6.9/debian/xrootd-doc.install000066400000000000000000000001331457266313600177600ustar00rootroot00000000000000doxydoc/html /usr/share/doc/xrootd bindings/python/docs/build/python /usr/share/doc/xrootd xrootd-5.6.9/debian/xrootd-fuse.install000066400000000000000000000000611457266313600201550ustar00rootroot00000000000000/usr/bin/xrootdfs /usr/share/man/man1/xrootdfs.1 xrootd-5.6.9/debian/xrootd-libs.install000066400000000000000000000000001457266313600201350ustar00rootroot00000000000000xrootd-5.6.9/debian/xrootd-plugins.install000066400000000000000000000005631457266313600207030ustar00rootroot00000000000000/usr/lib/*/libXrdCksCalczcrc32-5.so /usr/lib/*/libXrdCryptossl-5.so /usr/lib/*/libXrdSec-5.so /usr/lib/*/libXrdSecProt-5.so /usr/lib/*/libXrdSecgsi-5.so /usr/lib/*/libXrdSecgsiAUTHZVO-5.so /usr/lib/*/libXrdSecgsiGMAPDN-5.so /usr/lib/*/libXrdSeckrb5-5.so /usr/lib/*/libXrdSecpwd-5.so /usr/lib/*/libXrdSecsss-5.so /usr/lib/*/libXrdSecunix-5.so /usr/lib/*/libXrdSecztn-5.so xrootd-5.6.9/debian/xrootd-plugins.lintian-overrides000066400000000000000000000003141457266313600226650ustar00rootroot00000000000000# These are plugins - no soname on purpose sharedobject-in-library-directory-missing-soname [usr/lib/*/libXrd*-5.so] # False positive library-not-linked-against-libc [usr/lib/*/libXrdCksCalczcrc32-5.so] xrootd-5.6.9/debian/xrootd-private-devel.install000066400000000000000000000000001457266313600217530ustar00rootroot00000000000000xrootd-5.6.9/debian/xrootd-scitokens-plugins.docs000066400000000000000000000000331457266313600221550ustar00rootroot00000000000000src/XrdSciTokens/README.md xrootd-5.6.9/debian/xrootd-scitokens-plugins.install000066400000000000000000000000431457266313600226740ustar00rootroot00000000000000/usr/lib/*/libXrdAccSciTokens-5.so xrootd-5.6.9/debian/xrootd-scitokens-plugins.lintian-overrides000066400000000000000000000001651457266313600246710ustar00rootroot00000000000000# These are plugins - no soname on purpose sharedobject-in-library-directory-missing-soname [usr/lib/*/libXrd*-5.so] xrootd-5.6.9/debian/xrootd-server-devel.install000066400000000000000000000000001457266313600216070ustar00rootroot00000000000000xrootd-5.6.9/debian/xrootd-server-libs.install000066400000000000000000000000001457266313600214410ustar00rootroot00000000000000xrootd-5.6.9/debian/xrootd-server-plugins.install000066400000000000000000000010051457266313600221770ustar00rootroot00000000000000/usr/lib/*/libXrdBlacklistDecision-5.so /usr/lib/*/libXrdBwm-5.so /usr/lib/*/libXrdCmsRedirectLocal-5.so /usr/lib/*/libXrdFileCache-5.so /usr/lib/*/libXrdHttp-5.so /usr/lib/*/libXrdHttpTPC-5.so /usr/lib/*/libXrdMacaroons-5.so /usr/lib/*/libXrdN2No2p-5.so /usr/lib/*/libXrdOfsPrepGPI-5.so /usr/lib/*/libXrdOssCsi-5.so /usr/lib/*/libXrdOssSIgpfsT-5.so /usr/lib/*/libXrdPfc-5.so /usr/lib/*/libXrdPss-5.so /usr/lib/*/libXrdSsi-5.so /usr/lib/*/libXrdSsiLog-5.so /usr/lib/*/libXrdThrottle-5.so /usr/lib/*/libXrdXrootd-5.so xrootd-5.6.9/debian/xrootd-server-plugins.lintian-overrides000066400000000000000000000001651457266313600241750ustar00rootroot00000000000000# These are plugins - no soname on purpose sharedobject-in-library-directory-missing-soname [usr/lib/*/libXrd*-5.so] xrootd-5.6.9/debian/xrootd-server.install000066400000000000000000000013441457266313600205260ustar00rootroot00000000000000/usr/bin/cconfig /usr/bin/cmsd /usr/bin/frm_admin /usr/bin/frm_purged /usr/bin/frm_xfragent /usr/bin/frm_xfrd /usr/bin/mpxstats /usr/bin/wait41 /usr/bin/xrdacctest /usr/bin/xrdpfc_print /usr/bin/xrdpwdadmin /usr/bin/xrdsssadmin /usr/bin/xrootd /usr/share/man/man8/cmsd.8 /usr/share/man/man8/frm_admin.8 /usr/share/man/man8/frm_purged.8 /usr/share/man/man8/frm_xfragent.8 /usr/share/man/man8/frm_xfrd.8 /usr/share/man/man8/mpxstats.8 /usr/share/man/man8/xrdpfc_print.8 /usr/share/man/man8/xrdpwdadmin.8 /usr/share/man/man8/xrdsssadmin.8 /usr/share/man/man8/xrootd.8 /usr/share/xrootd/utils /lib/systemd/system/* /usr/lib/tmpfiles.d/xrootd.conf /etc/logrotate.d/xrootd /etc/xrootd/config.d /etc/xrootd/*.cfg /var/log/xrootd /var/spool/xrootd xrootd-5.6.9/debian/xrootd-server.postinst000066400000000000000000000010251457266313600207370ustar00rootroot00000000000000#!/bin/sh set -e if test "$1" = "configure" -o "$1" = "reconfigure" ; then getent group xrootd > /dev/null || \ addgroup --quiet --system xrootd getent passwd xrootd > /dev/null || \ adduser --quiet --system --home /var/spool/xrootd --shell /bin/false \ --ingroup xrootd --disabled-password --disabled-login \ --gecos "XRootD runtime user" xrootd chown xrootd:xrootd /etc/xrootd/*.cfg chown xrootd:xrootd /var/log/xrootd chown xrootd:xrootd /var/spool/xrootd fi #DEBHELPER# xrootd-5.6.9/debian/xrootd-voms-plugins.install000066400000000000000000000002431457266313600216600ustar00rootroot00000000000000/usr/lib/*/libXrdVoms-5.so /usr/lib/*/libXrdHttpVOMS-5.so /usr/lib/*/libXrdSecgsiVOMS-5.so /usr/share/man/man1/libXrdVoms.1 /usr/share/man/man1/libXrdSecgsiVOMS.1 xrootd-5.6.9/debian/xrootd-voms-plugins.lintian-overrides000066400000000000000000000001651457266313600236530ustar00rootroot00000000000000# These are plugins - no soname on purpose sharedobject-in-library-directory-missing-soname [usr/lib/*/libXrd*-5.so] xrootd-5.6.9/docker/000077500000000000000000000000001457266313600143365ustar00rootroot00000000000000xrootd-5.6.9/docker/.dockerignore000066400000000000000000000000061457266313600170060ustar00rootroot00000000000000data/ xrootd-5.6.9/docker/build/000077500000000000000000000000001457266313600154355ustar00rootroot00000000000000xrootd-5.6.9/docker/build/Dockerfile.alma8000066400000000000000000000015241457266313600204320ustar00rootroot00000000000000FROM almalinux:8 # Install tools necessary for RPM development RUN dnf install -y dnf-plugins-core epel-release rpmdevtools sudo \ && dnf config-manager --set-enabled powertools # Create xrootd user to avoid running tests as root RUN groupadd xrootd && useradd -g xrootd -m xrootd USER xrootd WORKDIR /home/xrootd # Create directory tree for building RPMs RUN rpmdev-setuptree # XRootD source tarball must be created in the # current directory in order to build this image COPY xrootd.tar.gz rpmbuild/SOURCES # Extract spec file to build RPMs RUN tar xzf rpmbuild/SOURCES/xrootd.tar.gz --strip-components=1 xrootd/xrootd.spec USER root # Install build dependencies with dnf RUN dnf builddep -y xrootd.spec # Build RPMS for XRootD RUN sudo -u xrootd rpmbuild -bb --with git xrootd.spec # Install RPMS RUN yum install -y rpmbuild/RPMS/*/*.rpm xrootd-5.6.9/docker/build/Dockerfile.alma9000066400000000000000000000015151457266313600204330ustar00rootroot00000000000000FROM almalinux:9 # Install tools necessary for RPM development RUN dnf install -y dnf-plugins-core epel-release rpmdevtools sudo \ && dnf config-manager --set-enabled crb # Create xrootd user to avoid running tests as root RUN groupadd xrootd && useradd -g xrootd -m xrootd USER xrootd WORKDIR /home/xrootd # Create directory tree for building RPMs RUN rpmdev-setuptree # XRootD source tarball must be created in the # current directory in order to build this image COPY xrootd.tar.gz rpmbuild/SOURCES # Extract spec file to build RPMs RUN tar xzf rpmbuild/SOURCES/xrootd.tar.gz --strip-components=1 xrootd/xrootd.spec USER root # Install build dependencies with dnf RUN dnf builddep -y xrootd.spec # Build RPMS for XRootD RUN sudo -u xrootd rpmbuild -bb --with git xrootd.spec # Install RPMS RUN yum install -y rpmbuild/RPMS/*/*.rpm xrootd-5.6.9/docker/build/Dockerfile.centos7000066400000000000000000000015241457266313600210120ustar00rootroot00000000000000FROM centos:7 # Install tools necessary for RPM development RUN yum install -y centos-release-scl epel-release git \ && yum install -y epel-rpm-macros rpmdevtools sudo yum-utils # Create xrootd user to avoid running tests as root RUN groupadd xrootd && useradd -g xrootd -m xrootd USER xrootd WORKDIR /home/xrootd # Create directory tree for building RPMs RUN rpmdev-setuptree # XRootD source tarball must be created in the # current directory in order to build this image COPY xrootd.tar.gz rpmbuild/SOURCES # Extract spec file to build RPMs RUN tar xzf rpmbuild/SOURCES/xrootd.tar.gz --strip-components=1 xrootd/xrootd.spec USER root # Install build dependencies with yum RUN yum-builddep -y xrootd.spec # Build RPMS for XRootD RUN sudo -u xrootd rpmbuild -bb --with git xrootd.spec # Install RPMS RUN yum install -y rpmbuild/RPMS/*/*.rpm xrootd-5.6.9/docker/build/Dockerfile.debian000066400000000000000000000015321457266313600206510ustar00rootroot00000000000000ARG version=12 FROM debian:$version RUN apt update RUN apt install -y build-essential devscripts equivs sudo # Create xrootd user to avoid running tests as root RUN groupadd xrootd && useradd -g xrootd -m xrootd USER xrootd WORKDIR /home/xrootd # XRootD source tarball must be created in the # current directory in order to build this image COPY xrootd.tar.gz . # Extract tarball RUN tar xzf xrootd.tar.gz USER root WORKDIR /home/xrootd/xrootd # Install build dependencies with dnf RUN echo yes | mk-build-deps --install --remove debian/control # Build DEB packages for XRootD RUN export VERSION=$(sed -e 's/v//; s/-rc/~rc/; s/-g/+git/; s/-/.post/; s/-/./' < VERSION) \ && sudo -u xrootd dch --create --package xrootd -v ${VERSION} -M 'XRootD automated build.' \ && sudo -u xrootd debuild --no-tgz-check --no-sign -b RUN apt install -y ../*.d*eb xrootd-5.6.9/docker/build/Dockerfile.fedora000066400000000000000000000014551457266313600206730ustar00rootroot00000000000000ARG version=rawhide FROM fedora:$version # Install tools necessary for RPM development RUN dnf install -y dnf-plugins-core rpmdevtools sudo # Create xrootd user to avoid running tests as root RUN groupadd xrootd && useradd -g xrootd -m xrootd USER xrootd WORKDIR /home/xrootd # Create directory tree for building RPMs RUN rpmdev-setuptree # XRootD source tarball must be created in the # current directory in order to build this image COPY xrootd.tar.gz rpmbuild/SOURCES # Extract spec file to build RPMs RUN tar xzf rpmbuild/SOURCES/xrootd.tar.gz --strip-components=1 xrootd/xrootd.spec USER root # Install build dependencies with dnf RUN dnf builddep -y xrootd.spec # Build RPMS for XRootD RUN sudo -u xrootd rpmbuild -bb --with git xrootd.spec # Install RPMS RUN yum install -y rpmbuild/RPMS/*/*.rpm xrootd-5.6.9/docker/build/Dockerfile.ubuntu000066400000000000000000000016011457266313600207460ustar00rootroot00000000000000ARG version=22.04 FROM ubuntu:$version ENV DEBIAN_FRONTEND=noninteractive RUN apt update RUN apt install -y build-essential devscripts equivs sudo # Create xrootd user to avoid running tests as root RUN groupadd xrootd && useradd -g xrootd -m xrootd USER xrootd WORKDIR /home/xrootd # XRootD source tarball must be created in the # current directory in order to build this image COPY xrootd.tar.gz . # Extract tarball RUN tar xzf xrootd.tar.gz USER root WORKDIR /home/xrootd/xrootd # Install build dependencies with dnf RUN echo yes | mk-build-deps --install --remove debian/control # Build DEB packages for XRootD RUN export VERSION=$(sed -e 's/v//; s/-rc/~rc/; s/-g/+git/; s/-/.post/; s/-/./' < VERSION) \ && sudo -u xrootd dch --create --package xrootd -v ${VERSION} -M 'XRootD automated build.' \ && sudo -u xrootd debuild --no-tgz-check --no-sign -b RUN apt install -y ../*.d*eb xrootd-5.6.9/docker/xrd-docker000077500000000000000000000040261457266313600163300ustar00rootroot00000000000000#!/usr/bin/env bash trap 'exit 1' TERM KILL INT QUIT ABRT : ${DOCKER:=$(command -v docker || command -v podman)} build() { OS=${1:-centos7} [[ -f xrootd.tar.gz ]] || package [[ -f build/Dockerfile.${OS} ]] || die "unknwon OS: $OS" ${DOCKER} build -f build/Dockerfile.${OS} -t xrootd -t xrootd:${OS} . } buildx() { OS=${1:-fedora} ARCH=${2:-amd64} ARCH=${ARCH/linux\/} [[ -f xrootd.tar.gz ]] || package [[ -f build/Dockerfile.${OS} ]] || die "unknwon OS: $OS" ${DOCKER} buildx build --platform linux/${ARCH} -f build/Dockerfile.${OS} -t xrootd:${OS}-${ARCH/\/} . } qemu() { ${DOCKER} run --rm --privileged multiarch/qemu-user-static --reset -p yes } clean() { rm -f xrootd.tar.gz } die() { echo "$(basename $BASH_ARGV0): error: $@" exit 1 } package() { REPODIR=$(git rev-parse --show-toplevel) VERSION=$(git describe ${1:-HEAD}) echo Creating tarball for XRootD ${VERSION} pushd ${REPODIR} >/dev/null git archive --prefix=xrootd/ -o xrootd.tar.gz ${1:-HEAD} popd >/dev/null mv ${REPODIR}/xrootd.tar.gz . } usage() { echo $(basename $BASH_ARGV0) [COMMAND] [ARGS] echo echo COMMANDS: echo echo " clean -- remove tarball created by package command" echo " package [VERSION] -- create xrootd.tar.gz tarball (VERSION=HEAD by default)" echo " build [OS] -- build docker image based on OS: centos7 (default), alma8, alma9" echo " buildx [OS] [ARCH] -- cross-build docker image based on OS/ARCH pair. Supported architectures" echo " are amd64, aarch64, ppc64le, s390x (big-endian). Default OS is fedora." echo " You can see supported platforms with docker buildx inspect --bootstrap." echo " qemu -- setup QEMU to be able to run cross-builds with buildx command." echo echo " Note: The test suite runs automatically during the container builds" echo } [[ $# == 0 ]] && usage && exit 0 CMD=$1 shift [[ $(type -t ${CMD}) == "function" ]] || die "unknown command: ${CMD}" cd $(dirname "${BASH_SOURCE[0]}") || die "cannot change directory" $CMD $@ xrootd-5.6.9/docs/000077500000000000000000000000001457266313600140175ustar00rootroot00000000000000xrootd-5.6.9/docs/CMakeLists.txt000066400000000000000000000000301457266313600165500ustar00rootroot00000000000000 add_subdirectory(man) xrootd-5.6.9/docs/CONTRIBUTING.md000066400000000000000000000261441457266313600162570ustar00rootroot00000000000000## XRootD Development Model and How to Contribute ### Versioning XRootD software releases are organized into major, minor and patch versions, with the intent that installing minor version releases with the same major version do not take long to perform, cause significant downtime, or break dependent software. New major versions can be more disruptive, and may substantially change or remove software components. Releases are assigned version numbers, such as "5.6.0", where: * The first number designates a major version. Major versions may introduce binary incompatibility with previous major versions and may require code dependent on libraries in the new major version to be recompiled. Generally, such requirements are limited to code that enhances XRootD functionality (e.g. plug-ins). User application code that only uses public APIs should continue to work unchanged. Consequently, major versions are infrequent and are introduced approximately every 5 years. * The second number increments within the major version and designates a minor version. Minor versions introduce new features within a major version. They are binary compatible with all versions within the major version and occur as often as needed to address community needs. On average, there are a few minor versions per year. * The last digit increments whenever changes are applied to a minor version to fix problems. These occur at random frequency, as often as necessary to fix problems. Since patch versions represent the minimum change necessary to fix a problem they provide the forward path for problem resolution. * When the first number increments, the second and third numbers are reset to zero and when the second number increments the third number is reset to zero. * A fourth number may be added by EOS as indication that the version of XRootD used by EOS has been patched after the official release. Patches introduced in an intermediate release for EOS will be likely included into the following patch release, unless it is a temporary fix affecting only EOS. #### Library versions When a library evolves compatibly: existing interfaces are preserved, but new ones are added the library’s minor version number must be incremented. Since nothing has been done that would break applications constructed earlier, it is OK for older applications to be linked with the newer library at run-time. If the interfaces in a library shared object change incompatibly, then the major revision number associated with the library must be incremented. Doing so will cause run-time linking errors for the applictions constructed with the older versions of the library and thus will prevent them from running, as opposed to crashing in an uncontrollable way. More information on library versioning is available [here](https://www.usenix.org/legacy/publications/library/proceedings/als00/2000papers/papers/full_papers/browndavid/browndavid_html/) and [here](https://www.akkadia.org/drepper/dsohowto.pdf). The project policy is that a change to public interfaces (as defined in the installed headers) requires a major release - bumping the major version number. ### Releases and Release Procedure Feature releases with current developments will normally be built a few times per year. Each `master` release is preceded by one or more release candidates that get tested for bugs and deployment issues in a possibly wide range of environments. When the release candidates are deemed sufficiently stable, then the final release is built. In addition to the `master` or "feature" releases, "bug fix" releases may be built whenever needed. These are for bug fixes only, so they normally should not need release candidates (due to the reduced need for additional testing). RPM packages are built for each release, including release candidates. All the packages are pushed to the testing yum repository. Additionally, all the bug fix releases and all the final `master` releases are pushed to the stable repository. See the [download](https://xrootd.slac.stanford.edu/dload.html) page for details. ### Stable and Develoment Branches Beginning with XRootD 5.6.0, the development model is based on two long-term branches: `master`, and `devel`. The `master` branch is the stable branch. It contains released versions of XRootD and may also contain unreleased bug fixes which do not require a new minor release. Each patch release for a given major+minor series is created from the `master` branch by adding any required bug fixes from the `devel` branch to the `master` branch and tagging a new release, such that all XRootD releases may be found linearly in git history. The `devel` branch is the development branch where all new features and other developments happen. Each new feature release is created by rebasing, then (perharps partially) merging the `devel` branch into the `master` branch, then tagging the relase on `master`. The `devel` branch will be kept current with the `master` branch by rebasing it after each patch release, to ensure that all bug fixes are always included in both `master` and `devel`. ### Guidelines for Contributors This section provides guidelines for people who want to contribute code to the XRootD project. It is adapted from git's own guidelines for contributors, which can be found in their repository on GitHub at https://github.com/git/git/blob/master/Documentation/SubmittingPatches. #### Deciding what to base your work on In general, always base your work on the oldest branch that your change is relevant to. * A bug fix should be based on the latest release tag in general. If the bug is not present there, then base it on `master`. Otherwise, if it is only present on `devel`, or a feature branch, then base it on the tip of `devel` or the relevant feature branch. * A new feature should be based on `devel` in general. If the new feature depends on topics which are not yet merged, fork a branch from the tip of `devel`, merge these topics to the branch, and work on that branch. You can get an idea of how the branches relate to each other with `git log --first-parent master..` or with `git log --all --decorate --graph --oneline`. * Corrections and enhancements to a topic not yet merged into `devel` should be based on the tip of that topic. Before merging, we recommend cleaning up the history by squashing commits that are fixups for earlier commits in the same branch rather than committing a bad change and the fix for it in separate commits. This is important to preserve the ability to use git bisect to find which commit introduced a bug. #### Make separate commits for logically separate changes In your commits, you should give an explanation for the change(s) that is detailed enough so that a code reviewer can judge if it is a good thing to do or not without reading the actual patch text to determine how well the code actually does it. If your description is too long, that's probably a sign that the commit should be split into finer grained pieces. That being said, patches which plainly describe the things that help reviewers checking the patch and future maintainers understand the code are very welcome. If you are fixing a bug, it would be immensely useful to include a test demonstrating the problem being fixed, so that not only the problem is avoided in the future, but also reviewers can more easily verify that the proposed fix works as expected. Similarly, new features which come with accompanying tests are much more likely to be reviewed and merged in a timely fashion. When developing XRootD on your own fork, please make sure that the existing test suite is not broken by any of your changes by pushing to a branch in your own fork and checking the result of the GitHub Actions runs. #### Describe your changes well The log message that explains your changes is just as important as the changes themselves. The commit messages are the base for creating the release notes for each release. Hence, each commit message should clearly state whether it is a bug fix or a new feature whenever that is not immediately obvious from the nature of the change itself. Moreover, it is very important to explain not only _what_ your code does, but also _why_ it does it. The first line of the commit message should be a short description of up to about 50 characters (soft limit, hard limit at 80 characters), and should skip the full stop. It is encouraged, although not necessary, to include a tag which identifies the general area of code being modified, for example "[Server]", "[XrdHttp]", etc. If in doubt, please check the git log for similar files to see what are the current conventions. After the title sentence, you should include a blank line and then the body of the commit message, which should be a meaningful description that * explains the problem the change tries to solve, i.e. what is wrong with the current code without the change. * justifies the way the change solves the problem, i.e. why the result with the change is better. * alternate solutions considered but discarded, if any. You should use the imperative to describe your changes, for example: ``` Change default value of foo to 1024 ``` instead of ``` This commit changes the default value of foo to 1024 ``` or ``` Changed default default value of foo to 1024 ``` Examples of good commit messages: ``` Author: Andrew Hanushevsky Date: Thu Jun 8 18:06:01 2023 -0700 [Server] Allow generic prepare plug-in to handle large responses, fixes #2023 ``` ``` Author: Brian Bockelman Date: Sat Feb 18 13:15:49 2023 -0600 Map authentication failure to HTTP 401 The authentication failure error message was previously mapped to HTTP 500 (internal server error). 401 Unauthorized (despite its name) is what HTTP servers typically utilize for authentication problems. ``` #### References to other commits, issues, pull requests, etc Sometimes, it may be useful to refer to the pull request on GitHub, an open issue which a commit fixes/closes, or simply an older commit which may have introduced a regression fixed by the current change. When referring to older commits, try to use the same format as produced by ``` git show -s --pretty=reference ``` For issues, add a "Closes: #nnnn" or "Fixes: #nnnn" tag to the body of the commit message (or, even better, to the pull request description). When linking a change to a specific issue or pull request, please verify in the GitHub website that the association actually worked. Depending on how you phrase your message, this may not happen automatically. In that case, it is also possible to use the "Development" side panel on the right to manually create the connection between pull requests and issues. If you intend to have your changes be part of a particular release which is not the next release being planned, you may also mark your pull request for inclusion in the desired release by using the "Milestone" side panel on the right. This can be used as an alternative method of marking a change as "bug fix" or "feature", depending on if it will only be included in the next patch release or feature release. Any changes which require a major release must be marked with the appropriate milestone. xrootd-5.6.9/docs/INSTALL.md000066400000000000000000000172461457266313600154610ustar00rootroot00000000000000## Building and Installing XRootD from Source Code ### XRootD Required and Optional Build Dependencies The required build-time dependencies are: bash, cmake, make, a C++ compiler with support for C++14, kernel headers from the OS, and development packages for libuuid, OpenSSL, and zlib. The optional features are shown in the table below together with the dependencies required to enable them. | Feature | Dependencies | |:----------------------|:-----------------------------------------| | FUSE support | fuse-devel | | HTTP support (client) | davix-devel | | HTTP support (server) | libcurl-devel | | Erasure coding | isa-l / autoconf automake libtool yasm | | Kerberos5 | krb5-devel | | Macaroons | json-c-devel libmacaroons-devel | | Python bindings | python3-devel python3-setuptools | | readline | readline-devel | | SciTokens | scitokens-cpp-devel | | systemd support | systemd-devel | | VOMS | voms-devel | | Testing | cppunit-devel gtest-devel | Other optional dependencies: tinyxml-devel, libxml2-devel. These have bundled copies which are used if not found in the system. In the following sections, we show how to install all available dependencies in most of the supported operating systems. #### RPM-based distributions: RedHat, Fedora, CentOS, Alma, Rocky On RedHat Enterprise Linux and derivatives, all dependencies are available, except for Intel's [isa-l](https://github.com/intel/isa-l) library. You may build and install isa-l from source, but alternatively, you can simple install isa-l dependencies (i.e. `autoconf`, `automake`, `libtool`, and `yasm`) and let the bundled isa-l be built along XRootD. On Fedora, it's not necessary to install the `epel-release` package, but on most others it is required, as some dependencies are only available in [EPEL](https://docs.fedoraproject.org/en-US/epel/). It may also be necessary to enable other repositories manually if they are not already enabled by default, like the `PowerTools`, `crb`, or `scl` repositories. On CentOS 7, this can be done by installing `centos-release-scl` in addition to `epel-release`. The command to install all dependencies is shown below. You may, however, need to replace `dnf` with `yum` or prepend `sudo` to this command, depending on the distribution: ```sh dnf install \ cmake \ cppunit-devel \ curl-devel \ davix-devel \ diffutils \ file \ fuse-devel \ gcc-c++ \ git \ gtest-devel \ json-c-devel \ krb5-devel \ libmacaroons-devel \ libtool \ libuuid-devel \ libxml2-devel \ make \ openssl-devel \ python3-devel \ python3-setuptools \ readline-devel \ scitokens-cpp-devel \ systemd-devel \ tinyxml-devel \ voms-devel \ yasm \ zlib-devel ``` On CentOS 7, the default compiler is too old, so `devtoolset-11` should be used instead. It can be enabled afterwards with `source /opt/rh/devtoolset-11/enable`. Moreover, `cmake` installs CMake 2.x on CentOS 7, so `cmake3` needs to be installed instead and `cmake3`/`ctest3` used in the command line. #### DEB-based distrubutions: Debian 11, Ubuntu 22.04 On Debian 11 and Ubuntu 22.04, all necessary dependencies are available in the system. In earlier versions, some of XRootD's optional features may have to be disabled if their dependencies are not available to be installed via apt. ```sh apt install \ cmake \ davix-dev \ g++ \ libcppunit-dev \ libcurl4-openssl-dev \ libfuse-dev \ libgtest-dev \ libisal-dev \ libjson-c-dev \ libkrb5-dev \ libmacaroons-dev \ libreadline-dev \ libscitokens-dev \ libssl-dev \ libsystemd-dev \ libtinyxml-dev \ libxml2-dev \ make \ pkg-config \ python3-dev \ python3-setuptools \ uuid-dev \ voms-dev \ zlib1g-dev ``` ### Homebrew on macOS On macOS, XRootD is available to install via Homebrew. We recommend using it to install dependencies as well when building XRootD from source: ```sh brew install \ cmake \ cppunit \ curl \ davix \ gcc \ googletest \ isa-l \ krb5 \ libxml2 \ libxcrypt \ make \ openssl@1.1 \ pkg-config \ python@3.11 \ readline \ zlib \ ``` Homebrew is also available on Linux, where `utils-linux` is required as an extra dependency since uuid symbols are not provided by the kernel like on macOS. On Linux, `libfuse@2` may be installed to enable FUSE support. ## Building from Source Code with CMake XRootD uses [CMake](https://cmake.org) as its build generator. CMake is used during configuration to generate the actual build system that is used to build the project with a build tool like `make` or `ninja`. If you are new to CMake, we recommend reading the official [tutorial](https://cmake.org/cmake/help/latest/guide/tutorial/index.html) which provides a step-by-step guide on how to get started with CMake. For the anxious user, assuming the repository is cloned into the `xrootd` directory under the current working directory, the basic workflow is ```sh cmake -S xrootd -B xrootd_build cmake --build xrootd_build --parallel cmake --install xrootd_build ``` If you'd like to install somewhere other than the default of `/usr/local`, then you need to pass the option `-DCMAKE_INSTALL_PREFIX=` to the first command, with the location where you'd like to install as argument. The table below documents the main build options that can be used to customize the build: | CMake Option | Default | Description |:-------------------|:--------|:-------------------------------------------------------------- | `ENABLE_FUSE` | TRUE | Enable FUSE filesystem driver | `ENABLE_HTTP` | TRUE | Enable HTTP component (XrdHttp) | `ENABLE_KRB5` | TRUE | Enable Kerberos 5 authentication | `ENABLE_MACAROONS` | TRUE | Enable Macaroons plugin (server only) | `ENABLE_PYTHON` | TRUE | Enable building of the Python bindings | `ENABLE_READLINE` | TRUE | Enable readline support in the commandline utilities | `ENABLE_SCITOKENS` | TRUE | Enable SciTokens plugin (server only) | `ENABLE_VOMS` | TRUE | Enable VOMS plug-in | `ENABLE_XRDCLHTTP` | TRUE | Enable xrdcl-http plugin | `ENABLE_XRDCL` | TRUE | Enable XRootD client | `ENABLE_XRDEC` | FALSE | Enable support for erasure coding | `ENABLE_ASAN` | FALSE | Build with adress sanitizer enabled | `ENABLE_TSAN` | FALSE | Build with thread sanitizer enabled | `ENABLE_TESTS` | FALSE | Enable unit tests | `FORCE_ENABLED` | FALSE | Fail CMake configuration if enabled components cannot be built | `XRDCL_LIB_ONLY` | FALSE | Build only the client libraries and necessary dependencies | `XRDCL_ONLY` | FALSE | Build only the client and necessary dependencies | `USE_SYSTEM_ISAL` | FALSE | Use isa-l library installed in the system ### Running XRootD Tests After you've built the project, you should run the unit and integration tests with CTest to ensure that they all pass. This can be done simply by running `ctest` from the build directory. However, we also provide a CMake script to automate more advanced testing, including enabling a coverage report, memory checking with `valgrind`, and static analysis with `clang-tidy`. The script is named [test.cmake](../test.cmake) and can be found in the top directory of the repository. Its usage is documented in the file [TESTING.md](TESTING.md). xrootd-5.6.9/docs/README_IPV4_To_IPV6000066400000000000000000000104771457266313600167600ustar00rootroot00000000000000This major release of xrootd/cmsd provides full IPV6 support with IPV4 compatibility. As such, it is no longer ABI compatible in two major respects: 1) Classes dealing with sockets and the network interfaces have substantially changed, and 2) The security interface has changed so as to provide a consistent connection context. *** Socket and Network Interfaces *** The XRDSysDNS class is now deprecated and essentially obsolete. It is still provided for backward compatibly but is no longer supported. This class works only in IPV4 contexts. It has been replaced by four address-format agnostic classes: XrdNetAddrInfo, XrdNetAddr, XrdNetSockAddr, and XrdNetUtils. All uses of XrdSysDNS should convert to using one or more of the new classes. The XrdNetLink and XrdNetWork classes have been deleted from the distribution. These appeared to be never used and it was easier to remove them than to upgrade them. The XrdNetPeer class is also deprecated but exists for backward compatibility. However, this class embedded an IPV4 structure, sockaddr, and was changed to use XrdNetSockAddr which defines a larger structure suitable for IPV6. To prevent programs from unknowingly using the smaller structure, the variable name used for the sockaddr structure was changed to trigger a compilation error. You should make sure you are not taking the size of the smaller structure when copying network addresses. For instance, the variable InetAddr referred to the sockaddr structure. This has change so that now Intet.Addr refers to the same structure. In general, interfaces using the XrdNetPeer classes should switch to using equivalent interfaces based on the XrdNetAddr class. The XrdNetAddr class provides comprehensive address handling while XrdNetPeer did not. In a less important area, the class XrdProtocol_Config which also contained the socket address describing the servers IP address using the variable myAddr has been sized for IPV6 using a union. While the variable name has not changed, programs referring to this variable should use the urAddr variable instead. However, it was deemed unlikely that this would cause a problem. *** Security Interfaces *** The security interfaces have substantially changed. All interfaces that used to accept a sockaddr structure now only accept the XrdNetAddrInfo object. One can obtain the hostname using the XrdNetAddrInfo object however, that use is discouraged unless absolutely necessary. This is to prevent defeating the nodnr network directive option. Additional details on how to accomplish this follow later. The XrdSecEntity object has changed, as follows: 1) It now includes a pointer to the XrdNetAddrInfo object that describes the connection details of the end-point that is associated with the entity description as member addrInfo. 2) In order to better accommodate this additions, the layout has slightly changed. In order to assist authorization and other host address sensitive plug-ins, each supported security plug-in now sets the XrdNetAddrInfo member addrInfo to point to a copy of the XrdNetAddrInfo object passed when requesting a new instance of the protocol. The default authorization plug-in has been changed to capitalize on this new information. Similar changes should be made to all private plug-ins. Failure to set the addrInfo member will likely result in a SEGV. *** Summary Of Required Plug-In Changes *** The changes that you should make to your plug-ins are: 1) Substitute XrdNetAddr for any use of sockaddr. If that is not possible, at the very least, use XrdNetSockAddr instead. 2) After converting remove, if possible, redundant include files arpa/inet.h, netinet/in.h, sys/socket.hh and similar include files. 3) Convert from using XrdSysDNS to a combination of XrdNetAddr and XrdNetUtils. 4) Private security plug-ins must set the addrInfo member. 5) If one of your plug-ins relied on the host member in XrdSecEntity to contain an actual host name, it should be changed to get the actual host using the addrInfo field. Please be aware that the host member never consistently pointed to a real host name as it was sensitive to the presence of the nodnr option on the xrd.network directive. 6) All plug-ins must now contain version information. This has become mandatory. Use the XrdVERSIONINFO macro defined in XrdVersion.hh to include version information in your plug-in. xrootd-5.6.9/docs/ReleaseNotes.txt000066400000000000000000005475561457266313600171770ustar00rootroot00000000000000====== XRootD ====== Release Notes ============= ------------- Version 5.6.9 ------------- + **Minor bug fixes** **[Python]** Check list of files in prepare to ensure they are strings **[Python]** Fix iteration over a file with Python3 **[Python]** Use int for 'force' in File::Stat (#2208) **[Utils]** Correct comparison that wrongly missed reaping certain directives **[XrdCl]** Fix logic error when upgrading connections to TLS **[XrdCl]** Stop Poller before TaskManager (fixes rare crashes at shutdown) **[XrdHttpTPC]** Fix 500 server response code if X-Number-Of-Streams > 100 (issue #2186) **[XrdSciTokens]** Add stat permissions to create, modify and write operations (issue #2185) **[XrdSciTokens]** Allow creation of parent directories if necessary (#2184) **[XrdSciTokens]** Fix bug when scope includes basepath or `/` (issue #2132) + **Miscellaneous** **[Tests]** Optimize cluster configuration to speedup tests ------------- Version 5.6.8 ------------- + **Minor bug fixes** **[RPM]** Create systemd tmpfiles at post-install step **[XrdCl]** Only claim to be TLS capable if TLS initialization succeeds (issue #2020) **[XrdCl]** Only consider an endpoint TLS-enabled if the connection is encrypted** **[XrdCl]** Remove duplicates from URL list to avoid undefined behavior **[XrdHttpTPC]** Fix infinite loop when scitags packet marking is enabled (issue #2192) **[XrdPosix,XrdSecztn]** Fix build on FreeBSD (issue #2090) **[XrdTls]** Fix automatic renewal of server certificate with OpenSSL>=1.1 (issue #1678) + **Miscellaneous** **[CMake]** Use CTest module in test.cmake and optionally submit to CDash **[RPM]** Install the client as dependency of main RPM **[Server]** Fix clang compile warnings ------------- Version 5.6.7 ------------- + **Major bug fixes** **[XrdCl]** Fix crash at teardown when using copies with multiple streams (issue #2164) **[XrdSecsss]** Fix buffer overrun when serializing credentials (issue #2143) + **Minor bug fixes** **[XrdCl]** Fix TPC initialization to take into account control stream (issue #2164) **[XrdPosix]** Fix ordering of debug levels in pss.setop DebugLevel (#2183) **[XrdTpc]** Properly handle creation of packet marking handles when socket is not yet connected (#2179) + **Miscellaneous** **[XrdHeaders]** Install XrdSfsFAttr.hh as private header ------------- Version 5.6.6 ------------- + **Major bug fixes** **[XrdHttp]** Fix PostProcessHttpReq to take into account User-Agent setting (#2173) + **Minor bug fixes** **[Server]** Set TZ environment variable to avoid race conditions (issue #2107) **[XrdCl]** Treat errOperationInterrupted as a recoverable error (issue #2169) ------------- Version 5.6.5 ------------- + **Major bug fixes** **[XrdTpc]** Fix potential segmentation fault when creating packet marking handle (issue #2161) + **Minor bug fixes** **[XrdSecgsi]** Fix compilation with GCC 14 (#2165) **[XrdSys]** Include for BSD and GNU/Hurd (#2149) + **Miscellaneous** **[Server]** Align monitoring ID with HTTP (issue #2133) **[XrdCrypto]** Skip check of our standard DH parameters (issue #2162) **[XrdHttp]** Send User-Agent as part of monitoring info (#2154) ------------- Version 5.6.4 ------------- + **Major bug fixes** **[XrdHttp]** Fix segfault with macaroons (issue #2114) **[XrdPss]** Fix segfault if pss.origin uses https protocol with no port (issue #2140) + **Minor bug fixes** **[CMake]** Fix include path in XRootDConfig.cmake (#2142) **[Headers]** Fix header dependencies and missing includes/declarations (#2119) **[Server]** Initialize pidFN to pidpath base directory if an error occurs **[XrdCl]** Don't try to enable TCP_CORK in GNU/Hurd (#2115) **[XrdCl]** Reapply fix for null-characters in error output (#2138, issue #1501) **[XrdEc]** Fix alignment issues on SPARC (issue #2131) **[XrdHttp,XrdNet]** Adapt Scitag min and max value to change in spec (#2139) **[XrdPosix]** Fix compilation failure with latest MUSL libc **[XrdSciTokens]** Initialize SecEntity.addrInfo to avoid SEGV (#2128) **[XrdTls]** Switch from using a cert file to a cert chain file (issue #2126) **[XrdZip]** Support big endian architectures in XrdZip (#2145) + **Miscellaneous** **[CMake]** Install CMake config file into lib/lib64 rather than share (#2116) **[DEB/RPM]** Rewrite packaging for Debian and RHEL based distributions **[Tests]** Convert tests to GoogleTest and run without containers (#2055, GSoC 2023) **[Tests]** Other fixes and improvements to tests (#2115, #2129, #2130, #2137, #2141) + **Known Issues** **[XrdSciTokens]** In this release, as in previous ones, using scitokens with the ZTN protocol may grant more access than should be allowed by the token scopes (issue #2121). ------------- Version 5.6.3 ------------- + **Minor bug fixes** **[CMake]** Export project version in CMake config (issue #2094) **[CMake]** Find only XRootD matching XRootDConfig.cmake installation path **[Python]** Do not use PEP517 by default, not supported on CentOS 7 **[Server]** Call tzset() early to ensure thread-safety of localtime_r() and mktime() (issue #2107) **[Server]** Correct maximum exp/act value in XrdNetPMark::getEA **[Server]** Create environment file within adminpath (issue #2106) **[Server]** Fix incorrect patch for authfile parsing (issue #2088) **[Tests]** Skip server checksum query test on unsupported filesystems (issue #2096) **[XrdCl]** Return an error if xrdfs rm fails to delete any file (issue #2097) **[XrdCms]** Try to load blacklist even if some entries are invalid (issue #2092) **[XrdEc]** Wait for pipeline including XrdCl::AppendFile() to finish (issue #2050) **[XrdHttp]** Fix parsing of chunked PUT lengths (#2102, #2103) + **Miscellaneous** **[CMake]** Add extra debugging messages in XRootDConfig.cmake **[CMake]** Handle components using more standard method **[Misc]** Fix spelling errors reported by lintian (#2087) **[Python]** Convert pyxrootd installation instructions to rst **[Server]** Export ptr to full TLS context into the Xrd env **[XrdCeph]** Align CMake requirement with main CMakeLists.txt **[XrdHttp]** Implemented HTTP TPC Packet Marking (#2109) **[XrdHttp]** Parse headers provided by the client in case-insensitive way when matching header2cgi keys (#2101) **[XrdHttp]** Promote SciTag header if packet marking has been configured on the server (#2101) **[XrdSciTokens]** Use configured CA path in SciTokens plugin if supported (#2095, #2112) **[XrdTpc]** Differentiate error messages for push/pull TPC transfer modes (issue #2060) ------------- Version 5.6.2 ------------- + **Major bug fixes** **[XrdHttp]** Fix chunked PUT creating empty files (issue #2058) + **Minor bug fixes** **[CMake]** Update Findlibuuid.cmake to use correct include paths **[Python]** Fix inclusion of markdown README file in documentation (#2057) **[Server]** Align code with actual documentation for auth idspec (issue #2061) **[XrdCl]** Fix flag check for append in XrdClZipArchive **[XrdCl]** Fix promotion of root:// URLs to use TLS encryption (issue #2078) **[XrdHttp]** Correct chunked response for GET with a byte range (issue #2076) **[XrdHttp]** Refactor read issuing during GET and fix read vector too long (issue #1976) **[XrdSciTokens]** Fix logic error in user mapping (issue #2056) **[XrdSciTokens]** Update maximum header size and line length in INI files (issue #2074) **[XrdSecgsi]** Fix crash of xrdgsitest when proxy is not already set **[XrdSecztn]** Fix template for default ZTN token location (issue #2080) **[XrdTls]** Change the thread-id returned to openssl 1.0 to improve performance (issue #2084) **[XrdTls]** Insert CRLs containing critical extensions at the end of the bundle (issue #2065) + **Miscellaneous** **[CMake]** Always compile XrdOssCsi (compiled only with GCC before) **[CMake]** Hide build output for isa-l to not confuse CTest **[CMake]** Run tests in parallel and fail build when tests fail **[Python]** Allow build customization via environment variable (issue #2062) **[Python]** Check for Development.Module with CMake 3.18 and above **[Server]** Add initialiser in one of the XrdScheduler constructors (#2081) **[Server]** Default ffdest as per current pmark specification **[Server]** Export readv comma separated limits via XRD_READV_LIMITS envar **[Server]** Implement Linux epoll maxfd limit (#2063) **[XrdClHttp] Add pgWrite support to the HTTP client plugin **[XrdHttp]** Refactor request statemachine for HTTP GET requests (#2072) **[XrdTls]** Refactor CASet and CRLSet to open the output file only once before the processing ------------- Version 5.6.1 ------------- + **Minor bug fixes** **[CMake]** Fix Findlibuuid.cmake to use kernel provided uuid on macOS **[XrdCl]** Avoid race in postmaster QueryTransport **[XrdCl]** Add missing argument in call to debug log message. This fixes sporadic crashes seen in FTS when debug logging is enabled. **[XrdCrypto]** Avoid race in GetCryptoFactory + **Miscellaneous** **[CMake]** Make sure Python is required in PyPI build **[CMake]** Set RPATH that works for binaries and libraries on macOS **[CMake,Python]** Handle RPATH for Python bindings on macOS **[Python]** Use PEP517 by default when building Python bindings ------------- Version 5.6.0 ------------- + **New Features** **[CMake]** Modernization of build system, now requires CMake 3.16 **[Client]** Add xrdfs cache subcommand to allow for cache evictions **[Misc]** Add support for building with musl libc (issue #1645) **[Python]** Modernization of build system, better support for creating binary wheels, properly propagating CXXFLAGS (issues #1768, #1807, #1833, #1844, #2001, #2002) **[Python]** Better handling of unicode strings in the API (issue #2011) **[Server]** Add gsi option to display DN when it differs from entity name **[Server]** Allow generic prepare plug-in to handle large responses (issue #2023) **[Server]** Allow specfication of minimum and maximum creation mode (issue #649) **[Server]** Make maxfd be configurable (default is 256k) (issue #2010) **[Server]** Include token information in the monitoring stream (phase 1). **[Xcache]** Implement a file evict function **[Xcache,XrdCl]** Increase default number of parallel event loops to 10 (#2047) **[XrdCl]** xrdcp: number of parallel copy jobs increased from 4 to 128 **[XrdHttp]** Allow XRootD to return trailers indicating failure (#1912) **[XrdHttp]** Denote Accept-Ranges in HEAD response (issue #1889) **[XrdHttp]** Report cache object age for caching proxy mode (#1919) **[XrdPss]** Allow origin to be a directory of a locally mounted file system **[XrdSciTokens]** Implement ability to have the token username as a separate claim (#1978) **[XrdSecgsi]** Use SHA-256 for signatures, and message digest algorithm (issues #1992, #2030) **[XrdSecztn]** Allow option '-tokenlib none' to disable token validation (issue #1895) **[XrdSecztn]** Allow to point to a token file using CGI '?xrd.ztn=tokenfile' (#1926) + **Major bug fixes** **[XrdHttp]** Fix SEGV in case request has object for opaque data but no content (#2007) **[XrdSecgsi]** Fix memory leaks in GSI authentication (issue #2021) + **Minor bug fixes** **[Server]** Use correct value for testing vector size **[XrdCl]** Fix off by one error in failure recovery check in parallel operation (issue #2040) **[XrdCl]** Fix potential stream timeout when a new request is sent to an idle stream (issue #2042) **[XrdCl]** Do not enforce TLS when --notlsok option is used in combination with root:// URL. This allows falling back to e.g. Kerberos authentication on a server with ZTN plugin enabled if the client has no certificates, hence not able to use TLS (issue #2020) **[XrdEc]** Fix compilation issues and underlinking on macOS **[XrdHttp]** Fix error returned when a client provides too many range requests (issue #2003) **[XrdHttp]** Fix regression where performance markers were missing during an HTTP TPC transfer (#2017) **[XrdHttp]** Return 404 instead of 500 error code on GET request on non-existent file (issue #2018) **[XrdHttp]** Return 405 instead of 500 error code on deletion of non-empty directory (issue #1896) **[XrdHttp]** Update HTTP header handling for chunked encoding and status trailer (#2009) **[XrdTls]** Make sure TLS context is marked invalid if not properly constructed (issue #2020) **[XrdTls]** Fix build failure with latest glibc (#2012) + **Miscellaneous** **[Apps]** Make xrdcrc32c consistent with xrdadler32 (issue #2045) **[CMake]** Build option ENABLE_CRYPTO has been removed. OpenSSL is always required with XRootD 5 (issue #1827) **[CMake]** New test.cmake script added to automate configure/build/test cycle **[CMake]** Fix build with link-time optimizations on 32bit systems (issue #2032) **[docs]** Update READMEs, contribution, installation, and testing documentation **[Misc]** Fix warnings from Clang compiler (#1997) **[Misc]** Add sandboxing settings to systemd service files (initially commented out) (issue #2033) **[Server]** Also check for IPv6 ULA's to determine if an address is private **[Tests]** New script xrd-docker added to automate running of dockerized tests (#1974) **[XProtocol]** Add fallthrough statement for ENOTEMPTY errno code mapping **[XRootD] ** Update code to no longer rely on using namespace std; (needed to support C++17) **[XrdCeph]** Submodule merged back into main repository (#2008) **[XrdCeph]** Minor build system updates and integration with main repository **[XrdCrypto]** Switch to a fixed set of DH parameters compatible with older OpenSSL (issue #2014) ------------- Version 5.5.5 ------------- + **Major bug fixes** **[HTTP]** Initialize SecEntity.addrInfo to avoid SEGV (issue 1986) **[Server]** Allow XrdXrootdFile::Serialize() to be used to wait more than once (issue 1995) **[Server]** Correct file handle returned when reusing it from external table **[XrdCl]** Fix client crash on early closed connection (issue 1934) + **Minor bug fixes** **[Server]** Use correct format to print size_t (issue 1989) **[XrdApps]** Let XrdClProxyPlugin work with PgRead (issue 1993) **[XrdCl]** Avoid possibility of Channel unregistering the wrong task (issue 1883) **[XrdCl]** Fix ZipArchive issuing VectorWrite which is too large during CloseArchive (issue 2004) **[XrdSys]** Avoid memory leak when overwriting the default message for EBADE + **Miscellaneous** **[CMake]** Adjust build rules to not depend on scitokenscpp to build libXrdSecztn **[Server,XrdPosix,XrdSecgsi]** Fix compile warnings with Clang compiler **[XrdPosix]** Fix build with Clang and _FORTIFY_SOURCE enabled (issue 1975) ------------- Version 5.5.4 ------------- + **Minor bug fixes** **[SSI]** Avoid file system+SSI feature interference that caused problems **[Server]** Fix dropped connections when link is reused (issue 1928) **[Server]** Limit max number of sockets in poller (issue 1962) **[Tests]** Fix intermittent failure of SocketTest **[XrdApps]** Fix crashes and deadlocks in xrdreplay tool (issue 1937) **[XrdCl]** Remove interference between sessions in the client (issue 1942) **[XrdCrypto]** Allow ASN1 GeneralizedTime in certificates (issue 1969) **[XrdCrypto]** Fix handling of daylight savings time (issues 985, 1955) **[XrdHttp]** Avoid sending empty headers in the HTTP response **[XrdHttp]** Clear digest header between requests (issue 1956) **[XrdHttp]** Fix for HTTP requests with open-ended byte range **[XrdHttp]** Fix handling of SSL shutdown during cleanup (issue 1967) **[XrdHttp]** Fix parsing of configuration with weighted checksums (issue 1944) **[XrdHttp]** Only call ERR_print_errors() if an error occurred **[XrdPfc]** Fix HTTP request failures with direct read mode enabled **[XrdTls]** Fix intermittent clients disconnections on login (issue 1952) + **Miscellaneous** **[XrdHttp]** Map authentication failure to HTTP 401 instead of 500 **[XrdSecztn]** Enable ZTN plugin by default ------------- Version 5.5.3 ------------- + **Minor bug fixes** **[Pfc]** Add missing if-error for a rare direct-read error trace **[Server]** Avoid a race condition during deferred file close **[XrdCms]** Fix compiler warning for potentially uninitialized variables **[XrdCms]** Remove extraneous character from error output (issue 1501) **[XrdHttp]** Support user-provided script for computing checksums **[XrdPosix]** Add missing include + **Miscellaneous** **[RPM]** Add g++ as build dependency ------------- Version 5.5.2 ------------- + **Major bug fixes** **[Python]** Avoid crash during prepare call **[TLS]** XrdTlsContext: Recreate session cache in the Clone() method. **[XrdCl]** Be sure to only read the header of kXR_status messages at first **[XrdCl]** Ensure URL::GetChannelId returns an Id which can be parsed again as a url **[XrdCl]** Fix regression in ZIP CD parsing. **[XrdHttpTPC]** Fixes the HTTP TPC PULL transfer issue when authentication is necessary to perform a transfer **[XrdHttpTPC]** Fix file size request failure due to missing configuration of the server CA before calling the HEAD request **[XrdOuc]** Fix checksum verification in XrdOucPgrwUtils::csVer **[XrdSecgsi]** Fix server crash with OpenSSL 3.x (issue 1877) + **Minor bug fixes** **[Server]** Fix a short writev recovery case **[Server]** Correct formatting when displaying the SecEntity structure. **[Server]** Make sure Mkdir returns a negative code for an EEXIST error **[Server]** Follow RFC for multiple checksums in HTTP (issue 1707). **[XrdApps]** Fix small memory leak when checksum fails **[XrdCl]** xrdfs: unify rm output on error (issue 1885) **[XrdCl]** AtLeastPolicy: correctly return the status **[XrdClHttp]** Link against XrdUtils for XrdOucCRC::Calc32C symbol **[XrdHttp]** Follow RFC checksum requirements in XRootD HTTP **[XrdSecgsi]** Avoid nullptr dereference in XrdSecProtocolgsi::ClientDoInit **[XrdVoms]** Fix handling of VOMS attributes when role is empty + **Miscellaneous** **[Apps]** Cleanup xrdcp help information. **[CMake]** Use upstream FindOpenSSL.cmake **[CMake]** Make switches for server-only options dependent on XRCL_ONLY=FALSE **[Macaroons]** Implement XrdSciTokensHelper interface for macaroons **[Pss]** Convert leftover print statement to debug action **[Python]** Use setuptools over setuptools._distutils.core **[XrdCl]** xrdcp: extended # of parallel copy jobs to 128. **[XrdOssCsi]** Fix build failure with GCC 13 **[XrdPosix]** Fix build failure due to possible large memory allocation ------------- Version 5.5.1 ------------- + **Major bug fixes** **[XrdFfs]** Fix a bug in xrootdfs reported by issue #1777 **[Server]** Avoid SEGV when client tries to access file after deferred closed. **[XrdHttp]** The server certificate is renewed by the Refresh thread of the XrdTlsContext object. **[XrdHttp]** Fix a segv happening when a client sends a first line starting with a space. **[XrdTls]** Shutdown the socket if a SSL error happens when trying to accept a connection. + **Minor bug fixes** **[Apps]** Avoid SEGV when asking for help. **[XrdCl]** copy job: fix memory leak (buffers not queued on error). **[Server]** Add O_RDWR open flag when creating file to avoid fs issue. **[Server]** Properly handle opaque info for fsctl operations. **[XrdHttp]** Allow VO names with spaces and other quoted chars. **[XrdCl]** LocalFileHandler: fail gracefuly on overloaded machines. + **Miscellaneous** **[XrdCl]** Introduce new error code for handling local errors. **[XrdCl]** local file handler: obtain error code with aio_error. **[XrdCl]** xrdfs ls: sanitize ls entry. **[CMake]** Add ENABLE_ switch for scitokens and macaroons, closes #1728. **[XrdTls]** Start the CRLRefresh thread in XrdTlsContext constructor. **[XrdTls]** Changed the bit set for the activation of the Refresh thread. **[XrdTls]** The CRL refresh thread logic only starts when there is a need for it. **[XrdTls]** Free current context when a new context is generated. **[XrdHttpTpc]** Pass src size to OFS via occ.asize. ------------- Version 5.5.0 ------------- + **New Features** **[XrdApps]** Provide command line too to manipulate checksum xattr. **[XrdApps]** xrdreplay: support quoted columns **[XrdApps]** xrdrecorder: allow to set the output path using XRD_RECORDERPATH envar. **[Protocol]** Add reflink capability to protocol via kXR_open options. **[Server] Separate out authorization to overwrite data. **[Server] Allow set variable values to come from a file. **[Server]** Implement gStream to monitor all http and xroot TPC events. **[Server]** Bring packet marking up to current specification. **[Server]** Provide g-stream monitoring for Third Party Copy (TPC). **[SciTokens]** Allow the SciToken plugin to consume based on ZTN tokens. **[Server]** Report experiment and activity codes when present for monitoring. **[HTTP]** Have the XrdHttp extraction logic match GSI/ **[XrdAcc]** Make the acc subsystem aware of request-based name mapping. **[XrdFfs]** update xrootdfs to work with XrdEC faster **[Posix]** Make xrootd proxy, xrootdfs and xrdadler32 work with XrdCl+EC **[SciTokens]** Save token subject as an XrdSecEntity xattr **[Throttle]** Track maximum concurrency limits in throttle plugin **[XrdCl]** xrdfs: support multiple rm paths **[XrdCl]** record / replay plug-in **[XrdCl]** In EC, add adjustable preference to servers based on free space **[XrdCl]** Add recorder plug-in and xrdreply tool. **[XrdCl]** xrdcp --server: report IP stack to stderr. **[XrdCl]** Introduce Stream queries (IpAddr, IpStack, HostName). **[XrdCl]** Implement EC VectorRead. **[XrdFfs]** same above **[XrdVomsMapfile]** Add support for VOMS mapfile **[XrdCl/XrdEc]** Make the remote ec cfg more flexible. **[Pfc]** Implement async read and readV from the perspective of XrdOucCacheIO. + **Major bug fixes** **[Server]** Adjust for self-move behaviour changes in some compilers. **[Server]** Modify vector's size instead of capacity to avoid undef behaviour **[XrdEc]** Make sure returned read size is correct. **[XrdEc]** Reader: make sure the completion handler is called if the read is of zero size. **[XrdCl]** Avoid race condition in AsyncSocketHander on use of reader/writer objects after link is re-enabled **[XrdCl]** Set the error status if the re-connection fails early during recovery **[XrdCl]** xrdcp: don't use a common static status obj across all copy jobs. **[XrdCl]** ZIP: respect file sizes > 4GB. **[XrdCl]** Correctly calculate #pages in pgread rsp (for small rsp). **[XrdCl]** PgRead: don't exceed max iovcnt. **[XrdCl]** Avoid that pgread responses could be timedout while being processed. **[XrdCl]** Avoid situation where client does not read all of a network message. **[XrdCl]** Avoid race by using TimeOutSID in single place. **[Server]** Reset the buffer pointer after a non-aligned pgRead request. **[XrdPfc]** Count number of active reads on an PfcIO object so that POSIX AIO bailout detach can be handled correctly. **[XrdPfc]** Do early exit when prefetching of a block fails with no other subscribers. **[XrdHttp]** Map kXR_ItExists to HTTP 409. **[XrdAcc]** Fix overwrite return code. + **Minor bug fixes** **[Frm]** Fix incorrect logic in frm_admin audit space. **[Server]** Avoid SEGV during client recovery due to close waitresp. **[Server]** Allow disablement of the tardy async I/O timeout path. **[Proxy]** Allow for URLs with username. **[XrdPss]** Do not trigger DeepLocate when pss.origin is http(s) **[XrdPosix]** bug fix, report correct st_blocks in EC + **Miscellaneous** **[SciTokens]** Add addition messages and debugging. **[SciTokens]** Also grant Readdir when token grants read permission. **[Server]** Ignore -Warray-bounds warnings from stricter check in gcc 12. **[CMake]** XRootDOSDefs: Use define_default on default values **[CMake]** Add XrdOuc/XrdOucPgrwUtils.hh to private headers. **[CMake]** Change Py required version to 3. **[CI]** Add Ubunty Jammy builds. **[XrdClHttp]** Move to xrootd core. **[XrdCl]** Refactor kXR_read raw data socket readout. **[XrdCl]** Support HostList in lambda completion handlers. **[XrdCl]** Make sure FileStateHandler is preserved until all outstanding requests are resolved. **[XrdCl]** Make sure FS data are preserved until all outstanding requests are resolved. **[Crypto]** bf32: Load "legacy" provider for blowfish in openssl v3. -------------- Version 5.4.3 -------------- + **Major bug fixes** **[XrdCl]** Make sure SocketHandler does not deadlock with PollerBuiltIn. **[XrdCl]** Pass the login token from redirect rsp to login req. **[XrdCl]** Fix infinite loop when copying data from msg body to user buffer. **[XrdCl]** Make sure TPC destination is created with correct permissions. **[XrdCl]** Fix VectorRead raw data socket readout. **[XrdCl]** Make sure pgwrite MsgHandler is removed from in-queue after receiving rsp. **[XrdCl]** Handle properly out-of-order pgread rsp. **[XrdCrypto]** OpenSSL3: correctly initialize cipher with public key and DH parameters, fixes #1662 **[XrdCrypto]** bf32: respect the key length when encrypting/decrypting. **[XrdAcc]** Make the acc subsystem aware of request-based name mapping. **[XrdTpc]** [XrdTpc] Added CLOEXEC flag for curl file descriptors. **[XProtocol]** Make sure ECANCELED is translated to kXR_Cancelled. **[XrdSciTokens]** Fix memory corruption. **[Python]** Fix Python 3.10+ issues from PY_SSIZE_T_CLEAN not being set. **[XrdHttp]** Support full URIs in the GET request, fixes #1675. **[PIP]** Fix import syntax to enable shutil.which check. **[Server]** Return correct pgread offset for sync reads. + **Minor bug fixes** **[XrdHttp]** Use 405 for mkcol/mkdir EEXIST. **[XrdHttp]** Redirect PUT and POST with 307. **[XrdHttp]** Use 307 to redirect anything that is not GET. **[XrdCl]** Remove the leading ? from auto generated login token, fixes #1535. **[XrdCl]** xrdfs ls: don't use deep locate in case of data server. **[XrdSciTokens]** Also grant Readdir when token grants read permission. + **Miscellaneous** **[x509]** Allow commans in DN's. **[XrdCl]** xrdcp: turn off progress bar when not running on a terminal, closes #1608. **[XrdCl]** Simplify kXR_attn handling. **[XrdCl]** Use pgread only is server version is kXR_PROTPGRWVERSION. **[OpenSSL]** Provide OpenSSL3 compability. **[TLS]** Display all OpenSSL messages upon fatal error; fixes #1554. **[XrdPosix]** Fix a bug that return wrong info about file size in EC. **[GSI]** Increase default bits from 512 to 2048. **[PIP]** Add PIP_OPTIONS CMake option for greater control of Python bindings install. **[Python]** Remove unused Python setup files from old workflows. **[Python]** Provide cmake switch (XROOTD_PYBUILD_ENV) for setting up python build environment. **[XrdSys]** Don't abort if it looks like we're about to fork. **[Utils]** Avoid emitting fatal polling error message unless aborting. **[Server]** Avoid misleading error message due to queued but delayed event. **[Tests]** Fix strcpy overflow. **[HttpTpc]** Vector cleared after use so it can be shrunk. **[ZTN]** Point to the token via Entity.creds. **[Server]** Fix MacOS complaints about unused parameters. **[Oss]** Do not fail a mkdir if directory already exists with the same mode. **[CMake]** Add an option (FORCE_ENABLED) to fail XrootD build if explicitly enabled features can not be build. **[CMake]** Include XrdPosix in client-only builds. **[XProtocol]** Bump protocol version and pgrw version. ------------- Version 5.4.2 ------------- + **Major bug fixes** **[XrdCl]** xrdfs rm: make sure status for all files is reported. **[XrdCrypto]** Add DH_get0_p for OPENSSL_VERSION_NUMBER < 0x10100000L. -------------- Version 5.4.1 -------------- + **Major bug fixes** **[Posix]** Make sure pointer is set to 0 to avoid memory corruption. **[GSI]** Generate DH parameters on first call to XrdCryptosslCipher. **[Server]** Prevent SEGV due to missing lock call for background jobs. **[SciTokens]** Correct deletion from std::map to avoid SEGV. **[cmsd]** Avoid SEGV, avoid using pointers after deleting them. **[XrdCl]** Make sure HS wait is not handled after channel has been TTLed. **[XrdCl]** Avoid derefferencing null ptr when trasforming ChunkInfo into PageInfo. **[XrdEc]** Ensure parallel execution of Reader::Read is thread-safe. **[CMake]** Add XrdSysTrace.hh to private headers. **[PIP]** Use shutil.which over distutils.spawn.find_executable when possible. **[Posix]** Make sure pointer is set to 0 to avoid memory corruption. **[Macaroons]** Avoid undefined behaviour (e.g. SEGV) using std::vector. + **Minor bug fixes** **[SciTokens]** Regularize paths used for authorization. **[pip]** Sanitize version to be PEP 440 compliant. **[Python]** Use context manager for opening files. **[Python]** Install Python bindings with pip if available. **[RPM]** Add python2-pip to BuildRequires. **[RPM]** Add python2-pip to BuildRequires. **[Debian]** Add python3-pip, python3-setuptools as required packages. + **Miscellaneous** **[Utils]** Redefine ENODATA when missing. **[CMake]** Add support for static openssl libraries **[CI]** Add GitHub Actions based CI **[IOEvents]** Improve tracing. **[XrdCl/XrdEc]** Make XrdEc compatible with vanilla xrootd servers. **[XrdCl]** xrdfs: allow rm multiple files. -------------- Version 5.4.0 -------------- + **New Features** **[Server]** Add generic prepare plugin. **[Server]** Add new class for gathering config data from config file. **[Server]** Implement firefly network flow monitoring. **[Server]** Allow prefunctory redirect based on client's net attributes. **[Server]** Allow embedded spaces in auth id's and paths (acc.encoding). **[Server]** Allow specification of preferred bind interfaces. **[Net]** Accommodate K8s network namespaces. **[cmsd]** Allow flexible path consideration when determining affinity. **[XrdCl]** Support unaligned PgReads. **[XrdCl]** Implement PgWrite. **[XrdCl]** Implement declarative PgRead. **[XrdCl]** Implement ZipArchive::PgRead. **[XrdCl]** xrdcp: enables in-fly error correction of corrupted pages (pgread/pgwrite). **[XrdCl]** xrdcp: allow multiple --cksum options. **[XrdCl]** Add checksumming capability to default EC plugin. **[XrdCl]** SocketHandler: encapsulate the reads/writes operation in a separate class. **[XrdCl]** Allow loading XrdEc default plug-in based on cfg file. **[XrdCl/XrdEc]** Use locate to obtain EC placement gr if empty. **[XrdEc]** Allow specifying operation t/o. **[XrdEc]** Make the metadata files relocable. **[XrdEc]** Allow using the EC lib without metadata files. **[Xrdpfc]** New cmd line option. Print in json format. + **Major bug fixes** **[Server]** Correct code that frees unsent aio buffers after fatal error. **[XrdCl]** ZipArchive: always execute callbacks in fresh exec ctx. **[XrdCl]** Don't issue a close in ~File if the thread-pool has been stopped. + **Minor bug fixes** **[AIO]** Correct test whether or not to turn off aio for Xcache. **[XrdSciTokens] Avoid double slashes in the final path after rules. **[Server]** Do not hold cheksum lock across local checksum computation. **[Server]** Correct notify option handling for kXR_prepare. **[XrdCl]** Adjust timeout before retransmitting corrupted pages. **[XrdCl]** Make sure XCpSrc does not leak. **[XrdCl]** Add missing err msg if force isn't used and destination exits. **[XrdCl]** Make sure the path on kXR_open does not inlude '?' if there is no cgi. **[XrdCl]** ParallelOperation: fix race condition in AtLeast policy. **[CMake]** Install in private-devel XrdClOperationTimeout.hh & XrdClFinalOperation.hh + **Miscellaneous** **[Plugins]** Improve fault tolerance of redirlocal plugin. **[Plugins]** Improve fault tolerance of redirlocal plugin. **[XrdCmsRedirLocal]** Add localroot option for plug-in. Improve fault tolerance of RedirLocal plug-in. **[Macaroons]** Relegate debug messages to debug status. **[Xcache]** Extend pgread API to return number of corrected checksum errors. **[cmsd]** Allow more parallelism during data server selection. **[Utils]** Add utility class to handle url encoding and decoding. **[Utils]** Add generic port specification to port number utility. **[Apps]** Reintroduce the xrdprep (a.k.a xprep) command. **[Apps]** Provide cli to compute crc32c checksum (i.e. xrdcrc32c). **[Server]** Remove limit on number of args passed to a forked exec. **[Server]** Add method to create an argv list from a string. **[Server]** Replace XrdOucTrace by XrdSysTrace, part 1 (internal change). **[Server]** Use RAtomics to enhance performance where needed. **[Misc]** Implement RAtomics object all of whose operations use relaxed mem order. **[Misc]** Add XrdPosixMap.hh to private hears (i.e. developers only). **[XrdHttp]** Add func to obfuscate paths hidden into strings. **[XrdHttp]** BuffGetData: better handle the case of wait=false. **[XrdCl]** Avoid calls to PostMaster for local files. **[XrdCl]** Do read/write recovery on errSocketTimeout. **[XrdCl]** Include all chunk details in kXR_readv descripsion. **[CMake]** Find pthreads using the CMake Thread module Prefer -pthread over -lpthread if supported by the compiler. **[CMake]** Enable SSE4.2 for cmsd. **[CMake]** Add an option to build with asan. **[Docker]** Add docker imgs with centos 7/8 build env. -------------- Version 5.3.3 -------------- + **Major bug fixes** **[Server]** Use length returned by read and fix an alignment check in pgRead. **[XrdCl] Make sure timed-out responses don't end up in the in-queue. -------------- Version 5.3.2 -------------- + **Major bug fixes** **[HTTP]** Use correct flag to request creation of directory path in MKCOL. **[Server]** Avoid SEGV when previous monitory directive overridden. **[TLS]** Avoid SEGV when a refresh context cannot be cloned (rare). **[XrdCl]** Avoid calls to PostMaster for local files. **[XrdCl]** TPC: normalize preset checksum. **[XrdCl]** Make sure small compressed ZIPs are extracted at the right offset. **[XrdCl]** Add missing err msg if force isn't used and destination exits. **[SSI]** Avoid SEGV when request spans more than 1 buffer. Fixes #1518 + **Minor bug fixes** **[Server]** Close race condition between bind and the use of that bind. **[HTTP]** XML-quote error messages sent through XrdHttp. **[HTTP]** BuffGetData: better handle the case of wait=false. **[Python]** Don't use PyEval_ThreadsInitialized for python >= 3.7. **[Python]** Catch ReferenceError error when finalizing File's. **[XrdCl]** Make sure lambda wrapper does not use status obj after it's deleted. **[XrdCl]** Fix memory leak in xrdfs ls. **[XrdCl]** ZipArchive: always execute callbacks in fresh exec ctx. **[XrdCl]** Do read/write recovery on errSocketTimeout. **[XrdCl]** Avoid segv on fork prepare. **[XrdMacaroons]** Add missing includes, fixes #1468. **[CMake]** Correct the variable name from ${ZLIB_LIBRARY} to ${ZLIB_LIBRARIES}. **[CMake]** Install in private-devel XrdClOperationTimeout.hh & XrdClFinalOperation.hh, closes #1519 **[Server]** Fix an include in OssCsi. + **Miscellaneous** **[CMake]** Find pthreads using the CMake Thread module Prefer -pthread over -lpthread if supported by the compiler. **[CMake]** Use ${CMAKE_DL_LIBS} consistently. **[CMake]** Add XrdPosixMap.hh to private headers (i.e. developers only). **[CMake]** Build openssl3 files only if WITH_OPENSSL3=TRUE. **[Docker]** Add docker imgs with centos 7/8 build env. -------------- Version 5.3.1 -------------- + **Major bug fixes** **[Server]** Add missing initializer to avoid SEGV in async pgread/pgwrite. **[XrdCl]** FileStateHandler::~FileStateHandler : make sure PostMaster is not used after libXrdCl has been unloaded. + **Miscellaneous** **[PIP]** Improve the compiler/devtoolset check on RHEL7. -------------- Version 5.3.0 -------------- + **New Features** **[Server]** Provide a way to verify volumes have been mounted before use. **[Server]** Add per file stream scheduling to async I/O. **[Server]** Full comprehensive implementation?| of pgread and pgwrite. **[Server]** Add additional xrootd trace options. **[Server]** Automatically turn off async I/O for default oss plug-in. **[TPC]** Implement the reproxy option for proxies doing TPC. **[XrdOssCsi]** Report XRDOSS_HASPGRW in Features(). **[XrdCl]** Report in PageInfo the number of corrected pages. **[XrdCl]** Overload |= for Pipeline class. **[XrdCl]** Implement xrdcp --retry. **[XrdCl]** xrdcp: add --zip-append functionality. **[XrdCl]** Add ability to create file checkpoints. + **Major bug fixes** **[SSS]** Add missing initializer to avoid random sss authentication failure. **[Server]** Fix host related authorization rules that would always fail. **[Server]** Fix backward compatibility issue with oss.space directive. **[SSS]** Avoid server-side SEGV for certain configurations. **[SSS]** Correct client backward incompatibily when 'anygroup' is specified. **[XrdCl/XrdSys]i** Fail gracefuly on epoll_wait errors if in child process. **[XrdCl]** Correctly handle server disconnect when connection is encrypted. **[XrdCl]** Make sure channel is TTLed if all file objects have been destroyed. **[XrdCl]** Clear OpenSSL errors from XrdSecProtocolgsi::getCredentials(). + **Minor bug fixes** **[Server]** Re-enable POSC write recovery disabled by previous commit. **[Server]** Fix echoing config line that only has a directive. **[Server]** Make assumeV4 setting consistent (missing initializer). **[Server]** Corect copy-paste error rendering TLS RecvAll() useless. **[XrdCl]** Don't call abort on JobManager:Stop if threads are not found. **[XrdCl]** Make the wrapped lambdas work correctly with chunked responses. **[XrdCl]** Fix OperationTimeout. **[XrdCl]** Only use the passwd file if a valid entry is found. + **Miscellaneous** **[Server]** Report pgread/pgwrite statistics as regulat reads/writes. **[Server]** Avoid running exit handlers when exiting a forked process. **[Server]** Re-architect async I/O to double its capacity. **[Server]** Speed up file close. **[Server]** Add a regular condition variable object to the mix. **[Server]** Make sure to clear OpenSSL error queue after authentication. **[TLS]** Correct certfile screening to capture all files; fixes #1467 **[XrdCl]** Update TPC progress bar every 2.5s. **[XrdCl]** Allow conditional error recovery for TLS_SSL_Error. **[XrdCl]** Enable read/write recovery for tls errors. **[XrdCl]** Remove obsolete ZipArchiveReader class. **[XrdCl]** Improve error reporting. **[XrdPfc]** Use std::atomic for the XrdPgc::IO XrdOucCacheIO* m_io, remove mutex that used to guard it. **[XrdPfc]** Change default cache block size to 256kB (was 1MB). -------------- Version 5.2.0 -------------- + **New Features** **[Server]** Add ability to wrap the checksum manager plugin. **[SSI]** Allow client-side to request high resolution timestamps. **[Server]** New ztn security plugin to provide a SciTokens sentinel. **[Server]** Allow optional return of checksum in dirlist response. **[Server]** Allow additional ports to be associated with a protocol. **[Xcache]** Make TLS substitution for net cksum selectable. **[Server]** Implement an internal nano-DNS for k8s setups (mostly for SSI). **[Server]** Allow internal application push of plugin library. **[Server]** Implement checkpointed file modifications (i.e. kXR_chkpoint). **[Server]** Enhance the StatPF() method. **[Xache]** Phase 1 of checksum integrity implementation (a.k.a pgread). **[Monitoring]** Implement extensive g-stream enhancements. **[XrdCl]** Add factory method (ResponseHandler::Wrap) for creating handlers from lambdas. **[XrdCl]** Implement ReadV. **[XrdCl]** Implement default erasure coding (EC) plug-in. **[XrdCl]** Implement EC-redirect allowing the server to request using the default EC plugin for given file. **[XrdCl]** Allow callbacks on client connection failure. **[XrdCl]** Add triedrc=srverr on connection t/o. **[XrdCl]** Implement cp timeout. **[XrdCl]** xrdcp: implement xrate threshold. **[XrdCl]** Add File::TryOtherServer API. **[XrdCl]** xrdfs: cat multiple files at once, closes #1243. **[XrdCl]** Support custom config file location via envar. **[XrdCl]** xrdcp: provide posc semantics for local destinations. **[XrdCl/Python]** Add method for querying the defaults of env. **[XrdEc]** Allow user to specify CGI for data/metadata URLs. **[XrdOssCsi]** Add osscsi plugin to store page crc32c checksums. **[XrdTls]** Provide a temp CA file generator that concatenats all the CAs in a given CA directory. **[XrdTpc]** Add support for certfile directive for TPC handler. + **Major bug fixes** **[Server]** Avoid network usage rac condition during config leading to a SEGV. **[Server]** Fix backward compatibility issue for xrd.protocol. Fixes #1444 **[Server]** Bypass voms MT initialization issues. **[ZTN]** Fix fatal flaws in SciToken strip method; avoid token rejection. **[TLS]** Provide thread-safety when required to do so. **[POSIX]** Avoid returning null pointer upon failure to avoid SEGV. **[POSIX]** Correctly initialize serverless cache to avoid SEGV. **[Server]** Make sure that alignment is a power of two for posix_memalign. **[SSI]** Prevent a buffer from being recycled twice leading to random errors. **[SSI]** Avoid race condition between Finished() and Event Dispatch (causes SEGV) **[Server]** Redo checksum changes to provide backward compatibility. **[Server]** Correct version checking to prevent false negatives. **[XrdCl]** Use ntohll() for pgread offsets. **[XrdCl]** Make checksum comparison case insensitive. **[XrdCl]** Make sure ZipCache doesn't segv. **[XrdCl]** Account for the ZIP data descriptor. **[XrdCl]** ZIP: handle correctly patological case where compressed size is greater than uncompressed. **[XrdCl]** Make sure TLS session is shutdown before socket is closed. **[XrdCl]** xrdcp: clean up chunks in-fly before destroying ZipArchive object. **[POSIX]** Initialize pointer when object reused to prevent memory corruption. **[TPC]** Do not hold lock when deleting a proxy autorm TPC job that failed. **[Server]** Fix memory leak of 64MB per checksum request. Fixes #1291 **[XrdEc]** Use correct stripe/buffer size when calculating crc32c. **[XrdEc]** Guard the RedundancyProvider multiton with mutex. **[Python]** Delete chunk buffer for VectorReadInfo after copying into python string. **[XrdTpc]** Overhaul curl's usage of CAs. **[XrdTpc]** Do not modify curl handle after curl_easy_cleanup(). + **Minor bug fixes** **[Server]** Make sure to close config file continuations. **[Server]** Fix possible storage opverlay when dealing with TLS protocols. **[Server]** Prevent inadvertent double load of fixed protocol. **[POSIX]** Make sure to always return consistent stat() information. **[cmsd]** Correct write lengths to not write null bytes to env file. **[cmsd]** Correct parsing bug that results in an erroneous warning. **[Server]** Fix esoteric truncation of long dirlist. Fixes #1340 **[Server]** Do no terminate dirlist when a deleted directory is encountered. **[Server]** When requested fully verify supplied checksums in pgwrite. **[Server]** Allow negative flags in redirect response. **[Server]** Clear redirect flags if client does not support them. **[XrdCl]** Parallel: make sure AtLeast waits for all operations (failed/succeeded). **[XrdCl]** Treat session-error as success in case of File::Close. **[XrdCl]** Allow to process user plugin config if user has no password entry. **[XProtocol]** Add kXR_virtReadv code for the File::ReadV call. **[XProtocol]** Add kXR_ecRedir redirect flag. **[XProtocol]** Extend the set of abilities presented to the servet on login. **[XrdHttp]** Fix memleak in SecEntity.host. + **Miscellaneous** **[gStream]** Use different sequence number for different packet types. **[Proxy]** Turn off the fork handler to undo the new default (i.e. is on). **[Server]** Add forgotten multi-user plugin checksum support. **[Server]** Make sure atleast one export exists in the export list. **[cmsd]** Display the name of the named pipe being waited upon. **[Server]** Echo the command line into the log. **[SSI]** Allow server-side debug to be enabled via an envar. **[cmsd]** Correctly parse osslib when it have options. **[Xcache]** Allow origin location query to be refreshed. **[CMS]** Ignore stacked plugin specifications as they are not supported. **[CMake/RPM]** Enable c++14. Note: on CC7 devtoolset-7 is required in order to build XRootD (including PIP install). **[Docs]** Remove deprecated man pages. **[Systemd]** Set RestartSec=10 by default, closes #1410. **[PIP]** Use sys.executable as the interpreter in install.sh **[PIP]** Enable devtoolset on pip install on RHEL7. **[XrdCl]** Enable fork-handler by default. **[XrdSys]** Remove unused custom semaphore implementation. -------------- Version 5.1.1 -------------- + **Major bug fixes** * **[Server]** Make sure each aio request has the minimum number of aio objects available. * **[Server]** Make sure stalled aio get written with the right offset and length. * **[XrdTpc]** XrdTpc: Buffer starting at the right offset. * **[XrdHttp]** Fix chunked transfer with 100-continue. * **[XrdCl]** Avoid partially reading status message data too early. * **[XrdCl]** Enable TPC placement step for roots/xroots transfers. + **Minor bug fixes** * **[Server]** Pass username to background checksum computation. * **[XrdCl]** In TPC set oss.asize only if known. + **Miscellaneous** * **[XrdCl]** Allow file listing with the same semantic as in shell. -------------- Version 5.1.0 -------------- + **New Features** * **[SSI]** Allow client-side to request high resolution timestamps. * **[XCache]** Make TLS substitution for net cksum selectable. * **[XCache]** Support pgRead and checksums. * **[Server]** New ztn security plugin to provide a SciTokens sentinel. * **[Server]** Allow optional return of checksum in dirlist response. * **[Server]** Allow additional ports to be associated with a protocol. * **[Server]** Implement an internal nano-DNS for k8s setups (mostly for SSI). * **[Server]** Allow internal application push of plugin library. * **[Server]** Implement checkpointed file modifications (i.e. kXR_chkpoint). * **[Server]** Enhance the StatPF() method. * **[Xache]** Phase 1 of checksum integrity implementation (a.k.a pgread). * **[Monitoring]** Implement extensive g-stream enhancements. * **[XrdCl]** Don't enforce host verification for localhost, closes #1318. * **[XrdCl]** Allow for roots/xroots protocol in metalinks. * **[XrdCl]** Add write API that transfers data from a FD. * **[XrdCl]** Add write API that takes ownership of the buffer. * **[XrdCl]** Implement redirect collapsing. * **[XrdCl]** Add stream TTL callback. * **[XrdCl]** xrdcp: allow write recovery at a redirector. * **[XrdCl]** Allow negative flags in redirect port. * **[XrdCl]** Implement PgRead. * **[XrdCl]** Substitute PgRead with Read if the former is not supported. * **[XrdCl]** Add a switch for disabling IP shuffling. * **[XrdCl]** Declarative API: implement Repeat, Replace, Ignore and Stop directives. * **[XrdCl]** Allow exxternal components to register new topics in XrdCl::Log. * **[XrdCl]** xrdfs ls: add option to get checksum per entry. * **[Python]** Expose the Prepare::Evict flag, fixes #1322. * **[XrdZip]** Add general ZIP utilities. * **[XrdSys]** Add kernel buffer utility. * **[XrdZip]** Provide headers implementing LFH, CDFH, ZIP64EOCD, ZIP64EOCDL and EOCD records. * **[CMake]** Add suitable RUNPATH to build-tree library. * **[CMake]** Add 'plugins' phony target. * **[XrdSciTokens]** Add scitokens plugin. * **[XrdEc]** Introduce Intel ISAL based erasure coding library. + **Major bug fixes** * **[Server]** Make sure that alignment is a power of two for posix_memalign. * **[Server]** Redo checksum changes to provide backward compatibility. * **[Server]** Correct version checking to prevent false negatives. * **[Server]** When requested fully verify supplied checksums in pgwrite. * **[Server]** Fix memory leak of 64MB per checksum request. Fixes #1291 * **[TLS]** Provide thread-safety when required to do so. * **[POSIX]** Avoid returning null pointer upon failure to avoid SEGV. * **[POSIX]** Initialize pointer when object reused to prevent memory corruption. * **[POSIX]** Correctly initialize serverless cache to avoid SEGV. * **[XrdTpc]** Fix error handling on stream write errors. * **[XrdTpc]** Do not call Write() on flush. * **[XrdTpc]** Avoid using invalid object in multi-stream. * **[XrdTpc]** Always check and fail on error. * **[XrdTpc]** Catch all negative return values not just SFS_ERROR. * **[XrdTpc]** Do not hold lock when deleting a proxy autorm TPC job that failed. * **[XrdTpc]** When we fail the transfer, free the curl-related memory. * **[XrdHttp]** Prevent unterminated/corrupted headers from looping the server. * **[XrdHttp]** Fix empty PUT. * **[XrdHttp]** Store a new header in the headers map before mangling it. * **[PIP]** Convert egg-info into proper dist-info. * **[XrdCl]** xrdcp: fix regression in recursive copy. * **[XrdCl]** Use ntohll() for pgread offsets. * **[XrdCl]** Correctly propagate TLS error message. * **[SSI]** Prevent a buffer from being recycled twice leading to random errors. * **[SSI]** Avoid race condition between Finished() and Event Dispatch (causes SEGV) * **[XrdCl]** xrdfs xattr: fix segv if server does not support xattr. * **[XrdCl]** Make sure xrdcp does not segv on multi-chunk read from compressed ZIP. + **Minor bug fixes** * **[XrdCl]** Create file descriptors with XrdSysFD utility. * **[XrdCl]** Make sure kXR_attrVirtRdr flag is set properly. * **[XrdCl]** Make sure error message does not include a null-character. * **[XrdCl]** Fix 'xrdfs ls -l' output parsing. * **[XrdTpc]** Fix the return value of `Stream::Write` * **[XrdTpc]** Always populate error buffer with messages. * **[XrdHttp]** URI-quote outgoing opaque data for HTTP-TPC. * **[XrdHttp]** In http mode fail the read if any error comes from the socket, including timeout. * **[CMake]** Set rpath for pip build. * **[Python]** Strip the leading v from __version__. * **[POSIX]** Make sure to always return consistent stat() information. * **[cmsd]** Correct write lengths to not write null bytes to env file. * **[cmsd]** Correct parsing bug that results in an erroneous warning. * **[Server]** Fix possible storage opverlay when dealing with TLS protocols. * **[Server]** Fix esoteric truncation of long dirlist. Fixes #1340 * **[Server]** Do no terminate dirlist when a deleted directory is encountered. * **[Server]** When requested fully verify supplied checksums in pgwrite. * **[Server]** Prevent inadvertent double load of fixed protocol. * **[PIP]** Fix WHEEL file (new line char). * **[PIP]** Make sure pip list displays correct xrootd version. * **[PIP]** Include the PKG-INFO in the root dir. * **[PIP]** Make sure the right version is set in metadata. + **Miscellaneous** * **[Server]** Make sure atleast one export exists in the export list. * **[Server]** Echo the command line into the log. * **[SSI]** Allow server-side debug to be enabled via an envar. * **[Xcache]** Allow origin location query to be refreshed. * **[cmsd]** Ignore stacked plugin specifications as they are not supported. * **[cmsd]** Correctly parse osslib when it have options. * **[cmsd]** Display the name of the named pipe being waited upon. * **[XrdSciTokens]** Add XrdSciTokens submodule. * **[XrdHttp]** Bump the max size for an Http header line to 16K. * **[XrdHttp]** Per request, use curl's escaping function to match EOS practice. * **[XrdHttp]** Make "tlsreuse off" the default. * **[XrdTpc]** Factor out recursive writes. * **[XrdTpc]** Do not allow partial buffer writes. * **[XrdTpc]** Adjust buffer size based on number of streams. * **[XrdCl]** Force socket error on header integrity check failure. * **[RPM]** Add client-compat and server-compat packages, based on XRootD 4. * **[XrdCns]** Retire the component. * **[XrdCl]** Report version in the logs when the client lib is initialized. -------------- Version 5.0.3 -------------- + **Major bug fixes** **[Server]** Add missing return that breaks loading of the oss.statlib **[Server]** Prevent SEGV upon poller error on TLS links doing async I/O. **[Server]** Avoid SEGV when checksum response is invalid. **[TLS]** Avoid bad behaviour when authentication fails on a TLS connection. + **Minor bug fixes** **[Server]** Report correct owner/group in extended status. **[XrdCl]** Fix group ownership print out in extended stat. **[XrdPosix]** Fix _STAT_VER problem on Fedora rawhide. **[HTTP]** Align TPC cadir procesing with HTTP to avoid config issues. Fixes #1323 + **Miscellaneous** **[Server]** Correct TLS identification test for authentication protocol. **[Server]** Pass the environment+secEntity to checksm manager. Fixes #1294 -------------- Version 5.0.2 -------------- + **Major bug fixes** **[Server]** Avoid POSC deletion when file creation fails because it exists. **[HTTP]** Prevent secret key leakage if specified in the config file. **[Python]** Prevent deadlock in Python bindings from XRootD.client.finalize. + **Minor bug fixes** **[OFS]** Correct missparsing '+cksio' option to avoid config failure. **[XRootD]** Correct flag reset code for esoteric ssq monitor option. **[XrdCl]** Fix memory leak in copy process. + **Miscellaneous** **[Server]** Strip out explicit plugin version designation with nasty message. **[Server]** Tighten requirements to display config file. **[HTTP]** Honor the tlsreuse option. **[HTTP]** Avoid issuing confusing messages for improbable configs. **[TLS]** Really kill the session cache when asked. **[Xcache]** Correct regression that killed dca option. **[Utils]** Add SHA3 checksum to utils. **[XrdCl]** Utils::GetHostAddresses: use shuffle insted of random_shuffle. **[XrdCl]** xrdcp: short circuit file exists error. **[Debian]** Add XrdClHttp plugin to xrootd-client-plugins package. **[XrdVom]** Add symlink to libXrdHttpVOMS for backwards compatibility. -------------- Version 5.0.1 -------------- + **Major bug fixes** **[CMS]** Use correct adminpath for creating Unix socket. Fixes #1255 **[HTTP]** Correctly handle certs relative to refcount to avoid SEGV. **[HTTP]** Escape pathnames in HTTP and XML responses. **[Xcache]** Add missing initializer to avoid malloc() abort. + **Minor bug fixes** **[VOMS]** Correct use of undocumented API that caused hang w/ dbg. Fixes #1245 **[GSI]** Use the storage deallocator that matches the allocator. **[TPC]** Fix potential null pointer dereference on Push. **[XrdOuc]** Fix XrdOucUtils::Sanitize not sanitizing the first character in the string. + **Miscellaneous** **[SSI]** Forward port LSST request scaler with auto-tuning. **[HTTP]** Enable session cache by default, provide http.tlsreuse directive. **[cmsd]** Reimplement affinity algorithm using verified LSST mechanism. **[CMS]** Allow redirect to local filesystem when using HTTP. **[Mon]** Rationalize UDP packet sequence numbers. **[Server]** Adding fbuff argument to monitoring to restrict maximum size of fstream packet. **[XrdHttp/XrdVoms]** Pass parameters specified in the http.secxtractor down the chain. **[XrdCrypto/XrdHttp]** Extract DN from user (proxy, multi-proxy) certificate and properly handle the gridmap-file functionality when accessing through HTTP. **[XrdHttp]** Add "required" parameter to the http.secxtractor and http.gridmap configuration directives. **[xrootdfs]** Set XrdSecEntity.uid/gid in xrootdfs. -------------- Version 5.0.0 -------------- + **New Features** * **[gsi]** Allow keywords for options that accept numeric values. * **[gsi]** Add -authzcall option to better control authz usage. * **[XrdSec]** Add support for x509 capabilities. * **[PSS]** Allow http and https for forwarding proxies. * **[Voms]** Allow Voms plugin to work for gsi and https. * **[SSS]** Allow sss authentication protocol to clone credentials. * **[Auth]** Allow authentication protocols to require TLS. * **[Apps]** Add xrdpinls to list plugin version requirements. * **[All]** Add hardware assisted CRC32C checksum based on Mark Adler's code. * **[All]** Implement new thread-safe strerror() replacement. * **[Server]** Provide option to reduce performance impact of usage tracking. * **[Server]** Define method to get partition information; mostly for Xcache. * **[Server]** Allow display of SecEntity for https and xroot. * **[Server]** Add xrootd.tlsreuse directive for session cache control. * **[Server]** Add command line options -a and -A for adminpath default. * **[Server]** Add command line options -w and -W for homepath setting. * **[Server]** Implement kXR_pgread and kXR_status. * **[Server]** Allow XrdNetAddr to track socket dialect. * **[Server]** Add ofs.ctllib directive for FSctl plugin. * **[Server]** Provide a way to see the actual server config when running. * **[Server]i** Provide fallback when an IPv6 address is missing a ptr record. * **[Server]** Allow xrootd.fslib plugin to be generally stacked. * **[Server]** Implement a stackable post authentication plug-in (sec.entitylib). * **[Server]** Add getfile and putfile SFS interface. * **[Server]** Allow plugin stacking for most OFS plugins. * **[Server]** Add plug-in interface for performance reporting. * **[Server]** Add Features(), getFile(), and putFile() methods to SFS. * **[Server]** Implement simple g-stream monitoring for medium level repotring. * **[Server]** Trivialize OFS plugin wrapping (breaks ABI). * **[Server]** Add additional fields to he SecEntity structure (breaks ABI). * **[Server]** Allow redirector to handle kXR_dirlist location resolution. * **[Server]** Implement tcpmonlib directive for TCP socket monitring. * **[XrdCl]** Introduce recovery mechanism for declarative operations. * **[XrdCl]** Introduce policies for parallel operations (all, any, some, at least). * **[XrdCl]** xrdcp: add --notlsok and --tlsnodata options. * **[XrdCl]** xrdcp: enable roots and xroots as TLS protocols. * **[XrdCl]** xrdcp: add new --tlsmetalink option. * **[XrdCl]** xrdcp: add an option (--xattr) to preserve extended attributes. * **[XrdCl]** xrdcp: allow roots/xroots protocol. * **[XrdCl]** xrdfs: expose extended stat information (if available) in stat and ls -l. * **[XrdCl]** Add support for extended attributes. * **[XrdCl]** Log properly TLS events. * **[XrdCl]** Monitor writev requests. * **[XrdCl]** Make sure there is at least one data stream if server requests the client to encrypt only control and send data unencrypted. * **[XrdCl]** Implement TLS encryption (roots/xroots). Use async TLS, handle want read/write in an event-loop. * **[XrdCl]** Ensure only one instance of DefaultEnv exists * **[XrdCl]** Allow server to request encryption. * **[XrdCl]** Make Socket class upgradeable to TLS. * **[XrdCl]** Add support for local file rm. * **[XrdCl]** Fail all handlers in pipeline on failure. * **[XrdCl]** Implement extended attribute (xattr) support. * **[XrdCl]** Add global on connect callback API. * **[XrdCl]** Introduce PgRead/PgWrite interface. * **[XrdCl]** Add envar (XRD_TLSFALLBACK) that makes the client fallback to root if the server does not support roots. * **[XrdCl]** Take into account connection specific CGI when selecting channel. * **[XrdCl]** Set kXR_ExpTPC if connection is being established in context of TPC. * **[POSIX]** Add methods to the cache mngt objecT to get status of a cached file. * **[POSIX]** Add N2N pfn2lfn option -cachesrc to pass data source to the N2N. * **[TLS]** Add peer certificate verification. * **[TLS]** Make sure log notes the connection type. * **[TLS]** Add hostname validation. * **[TLS]** Use recommended ciphers add xrd.tlsciphers directive to override. * **[TLS]** Implement crl/ca automatic refresh. * **[TLS]** Add additional options for session caching. * **[TLS]** Add tracing capability to the TLS stack. * **[SSI]** Export request scaling interface. * **[SSI]** Add generic Control() method for future use. * **[SSI]** Simplify GetResponseData() to return void not enum (breaks ABI). * **[SSI]** Add generalized option setting method, SetConfig(). * **[Proxy]** Report features. * **[Proxy]** Allow proxy to forward xroots and roots protocols. * **[XCache]** Optionally supply source and CGI to N2N cache plugin. * **[XCache]** Provide monitoring summary statistics. * **[XCache]** If needed, delay IO destruction internally in Detach. XrdScheduler is now always available. * **[XCache]** Provide confh backward compatibility for "pss.cachelib" directive. * **[XCache]** Simplify summary statics gathering. * **[XCache]** Autoconfig oss and cmsd using pfcache export attribute. * **[Python]** Add xattr API. * **[CMSD]** Add space trace option for tracing space utilization changes. * **[FRM]** Allow frm_xfrd to split in/out copy allocation. + **Major bug fixes** * **[VOMS]** Do not touch the Entity.name field; especially converting spaces. * **[Server]** Add missing initializer to avoid TPC SEGV on busy systems. * **[Server]** Accommodate changes in epoll handling in CentOS 7.10 plus. * **[SSI]** Fix double delete due to extraneous code addition. * **[XrdCl]** Use different bit for mkpath and tpc delegation. * **[XrdCl]** tpc: don't open non-root/xroot source. * **[XrdCl]** Adjust # of strm queues when server requests additional data. * **[PFC]** Make sure scheduler pointer is always valid. * **[Proxy]** Remove offending CGI elements before passing URL. * **[XrdHttp]** Fix MKCOL response when we have an EEXIST. + **Minor bug fixes** * **[Acc]** Process compound capability rules as documented. * **[gsi]** Fail server initialization when gsi did not initialize. Fixes #1042 * **[Proxy]** Properly handle non-xroot URLs. * **[Proxy]** Correct version information. * **[cmsd]** Make sure to return correct error when creation conflict occurs. * **[Server]** Handle iovec's longer than IOV_MAX. * **[Server]** Correct action determination for debug message to avoid strife. * **[Server]** Use correct mechanism to determine DNS registration. * **[Server]** Make space usage maintenance fully thread safe. * **[Server]** Avoid duplicate usage update under certain conditions. * **[XrdCl]i** xrdcp: don't create unwanted dir if --recursive switch was used. + **Miscellaneous** * **[Acc]** Dynamically attach a sliced SecEntity as one of its object attributes. * **[Utils]** Don't hide true error when loading a plugin w/o an alternate. * **[gsi]** Make -trustdns default false (i.e. do not use DNS). * **[gsi]** Remove built-in VOMS extractor and FunLite example. * **[gsi]** Make -vomsat default "ignore". * **[Server]** Implement exchange buffering for increased performance. * **[Server]** Remove Solaris polldev support (reverts to using pollpoll). * **[Server]** Turn off forwarding if authorization has not been enabled. * **[Server]** Rationalize how pidfiles are created. * **[Server]** Make sure loginid corresponds to the POSIX.1-2008 standard. * **[Server]** Honor authentication protocol TLS requirements. * **[Server]** Turn off IP address caching when dynamic DNS is enabled. * **[Server]** Log unusual stat() errors affecting file availability. * **[Server]** Manage reference counters in a more timely fashion. * **[Server]** Fallback using known DNS name when reverse translation fails. * **[Server]** Provide interface for checksum handler to report progress. * **[Server]** Improve SFS wrapping and add "how to" documentation. * **[Server]** Add appname to SecEntity attribute set. * **[Server]** Assign a unique ID to each SecEntity instance. * **[Server]** Make cksio the default for the ofs.osslib directive. * **[Server]** Add checkpoint() method to XrdSfsFile. * **[Server]** Add kXR_ItExists error for EEXIST errno don't use kXR_BadRequest. * **[Server]** Remap kXR_BadRequest to EBADRQC. * **[Server]** Change uses of ENOSYS to ENOTSUP for consistent meaning. * **[Server]** Make all error code mappings consistent. * **[Server]** Deprecate use of '-2' option in oss.statlib. * **[Server]** Distinguish between authentication and authorization failures. * **[Server]** Allow TLS session reuse for xroots protocol. * **[SSS]** Update sss protocol for new SecEntity object. * **[SSI]** Avoid SIGABRT when init fails. Fixes 751 * **[SSI]** Use exchange buffering to avoid using memcpy for improved peformance. * **[XCache]** Provide config backward compatibility for "pss.cachelib" directive. * **[XCache]** Switch from usage of XrdFileCache prefix to XrdPfc. * **[XCache]** Cleanup statistics object and add additional fields. * **[Proxy]** Refactor proxy server caching implementation. * **[All]** Use thread-safe strerror() replacement, part 1. * **[CMake]** Move to CMake 3. * **[Protocol]** Remove unused kXR_attn subcodes. * **[Protocol]** Define the asyninfo response structure. * **[Protocol]** Add TLS-specific bits for anonymous get/putfile. * **[XrdCl]** When selecting channel take into account protocol (root vs roots). * **[XrdCl]** Use XRootDStatus instead of Status for internal workflows. * **[XrdCl]** Make sure TLS err msg are propagated to the end user. * **[XrdCl]** Log socket upgrade to TLS at Info level. * **[XrdCl]** Filter out xrdcl.* parameters from tpc.scgi. * **[XrdCl]** Widely apply PIMPL idiom. * **[XrdCl]** Review public headers. * **[XrdCl]** Simplify Channel abstraction. * **[XrdCl]** Remove OpenFlags::Append. * **[XrdCl]** Clear SSL error queue after every authentication routine. * **[XrdCl]** Preserve xrdcl.* cgi elements on redirect as they are important for the internal workflow. * **[RPM]** Remove xrdstagetool, xrd, xrdcp-old and xprep. * **[RPM]** Don't build/install cns. * **[RPM]** Remove old XRootD client. * **[RPM]** Remove XRootD 3.x.x compat package. * **[RPM]** Add symlink libXrdFileCache.so -> libXrdPfc.so. * **[XrdMacaroon]* When possible, use the built-in chaining featured in Xrootd 5. * **[XrdTpc]** Have HTTP-TPC reuse the SFS from the environment. * **[TPC]** Honor client's source protocol specification server-side. * **[Plugins]** Make sure underlying msgs get passed along. Fixes #1015 * **[XrdHttp]** Use the framework to handle OpenSSL contexts. * **[TLS]** Always configure with session cache disabled. -------------- Version 4.12.4 -------------- + **Major bug fixes** * **[XrdCl]** Fix regression in recursive copy (introduced in f6723e00). * **[VOMS]** Do not touch the Entity.name field; especially converting spaces. * **[VOMS]** Fix improper collection of multi-VO cert attributes. * **[RPM]** Refine xrootd-voms obsoletes/provides for vomsxrd. * **[RPM]** Remove libXrdSecgsiVOMS-4.so from xrootd-libs pkg. * **[pfc] Make sure v4 does not try to read v5 cinfo files. * **[XrdHttp]** Shutdown the connection in the case of an unrecognized HTTP first line. + **Minor bug fixes** * **[Server]** Make sure to sanitize username in the HTTP bridge. * **[Server]** Make sure loginid corresponds to the POSIX.1-2008 standard. + **Miscellaneous** * **[Debian]** Add XrdClHttp plugin to xrootd-client-plugins package. * **[XrdHttp]** Add "required" parameter to the http.secxtractor and http.gridmap configuration directives. * **[XrdCrypto/XrdHttp]** Extract DN from user (proxy, multi-proxy) certificate and properly handle the gridmap-file functionality when accessing through HTTP. * **[Protocol]** Add flag to say locate is for dirlist. * **[XrdCl]** Apply kXR_compress & kXR_4dirlist to deep locate if done in context of dir list. -------------- Version 4.12.3 -------------- + **Major bug fixes** * **[RPM]** xrootd-voms pkg obsoletes xrootd-voms-plugin. * **[Python]** Make sure XrdVersion.hh is generated correctly during pip install. -------------- Version 4.12.2 -------------- + **Major bug fixes** * **[XrdHttp]** Added protection against a NULL bridge instance. * **[XrdHttp]** Avoid the reqstate to go below zero in case of strange headers. * **[XrdVoms]** Set the prox field to "xrdvoms" as the extractor. * **[XrdVoms]** Fix various option parsing problems. * **[XrdVoms]** Backport a clean patched VOMS version from R5. + **Minor bug fixes** * **[Docs]** Make the xrdmapc help text and manpage match the code. * **[Docs]** Fix empty xrdmapc manpage. * **[CMake]** Fix XRootD config module. + **Miscellaneous** * **[Server]** Allow specification of whether or not dirlist is locally handled. * **[XrdVoms]** Replace git sub-module with full-fledged component of xrootd core. * **[XrdVoms]** Simplify the gropts parameter. * **[XrdVoms]** Rename XrdSecgsiVOMS-4.so to XrdVoms-4.so, create XrdSecgsiVOMS-4.so symlink for compability. -------------- Version 4.12.1 -------------- + **Major bug fixes** * **[XrdXrootdVoms]** Fix run-time lib dependencies. + **Minor bug fixes** * **[xrdcp]** Don't create unwanted dir if --recursive option was used and the source is a file. -------------- Version 4.12.0 -------------- + **New Features** * **[Server]** User redirector to find directory to be listed (R5 backport). * **[XrdCl]** More effective error recovery policy when the source is a metalink. * **[xrdcp]** Implement bandwidth limiting (--xrate option). * **[xrdcp]** Implement ability to continue interrupted transfer (--continue). * **[xrdcp/Python]** Add an option (--zip-mtln-cksum) to use the checksum from a metalink with files extracted from ZIP archives. * **[xrdcp/Python]** Automatically infer checksum type if asked to. * **[xrdcp/Python]** Add an option (--rm-bad-cksum) to automatically remove destination file if the checksum check failed. + **Major bug fixes** * **[Server]** Correct sequencing of an async close to prevent a SEGV. * **[Server]** Add missing initializer to avoid TPC SEGV on busy systems. * **[Server]** Initialize the XrdAccEntityInfo structure. * **[XrdHttp]** Fix MKCOL response when we have an EEXIST. * **[XrdHttp]** Periodically reload verify cert store. * **[XrdHttp]** Disable session cache. * **[XrdCl]** Don't set on-connection handler for local files. * **[XrdCl]** Don't set the stream error window for auth errors. * **[XrdCl]** Fix race condition resulting in dangling pointer to SidMgr. * **[XrdCl]** Make Channel operations thread-safe in respect to ForceDisconnect. + **Miscellaneous** * **[Deb]** Refine debian packaging. * **[CMake]** Repleace XRootD find module with config module. * **[RPM/CMake]** Don't build XrdCns. * **[Packaging]** Add xrootd-voms and xrootd-voms-devel packages equivalent * **[Packaging]** Add additional private headers for vomsxrd. to the vomsxrd packages. * **[Python]** Support PyPy. -------------- Version 4.11.3 -------------- + **Major bug fixes** * **[Server]** Avoid SEGV when skipping over large if/else/fi blocks. * **[XrdHttp]** Fix curl speed limit to be really around 10KB/s. * **[xrootdfs]** Make sure xrootdfs_create() checks return code of xrootdfs_do_create(). * **[XrdHttp]** Change the default ssl cipher. * **[XrdHttp]** Enable elliptic-curve support for OpenSSL 1.0.2+. * **[XrdHttp]** Use Mozilla cipher list only with OpenSSL 1.0.2+. + **Minor bug fixes** * **[XrdCl]** When xrdcp reports an error refine the message so it is clear whether the erros comes from the source or destination. * **[XrdCl]** Make sure on error local redirect is not retried infinitely. * **[XrdXrootdConfig]** Fixed wrong segsz parameter. * **[XrdHttp]** Give a chance to map the issuer name in the case of a proxy certificate (needed to accommodate systems that do user mapping, e.g. eos) * **[XrdHttp]** Fix the logic for determining SecEntity.name. * **[XrdHttp]** Don't overwrite SecEntity.name after the gridmap phase. * **[xrootdfs]** Make sure xrootdfs_create() does check the return code of xrootdfs_do_create(). + **Miscellaneous** * **[XrdSecgsi]** In the case of delegation, give client a chance to use XrdSecGSISRVNAMES to limit where it is being redirected to. * **[Python]** Make rpath relative to the module. * **[Debian]** Proper Debian pkg naming. * **[XrdCl]** Use glibc semaphores for rhel>=7. * **[Server]** Export pointers to Acc, Ofs, and Sfs plugin into the XrdEnv. * **[XrdMacaroons]** Use env ptrs to get authorize obj. -------------- Version 4.11.2 -------------- + **Major bug fixes** * **[Proxy]** Remove offending CGI elements before passing URL. * **[Server]** Accommodate changes in epoll handling in CentOS 7.10 plus. * **[SSI]** Fix double delete due to extraneous code addition. * **[XrdOuc]** Prevent seqv in XrdOucString::replace(). * **[XrdHttp]** Fix escape code for \r. * **[XrdHttp]** Fix: the url quoting function was swapping the quoting of CR and LF. * **[XrdHttp]** Fix the "100 Continue" response. * **[XrdHttp]** Prevent the logic that looks for the /n from getting confused by lines containing zeroes. * **[XrdTpc]** Properly forward existing opaque info to the OFS layer. * **[XrdTpc]** Properly format redirection URL when OFS layer returns also opaque information. * **[SSI]** Avoid SIGABRT when init fails. Fixes 751 + **Minor bug fixes** * **[Python]** Use right bdist_wheel. * **[XrdCl]** Report last error if metalink runs out of replicas, fixes #1071 + **Miscellaneous** * **[CMake]** Add XROOTD_PLUGIN_VERSION to FindXRootD.cmake. * **[XrdCl]** TPC: use separate cgi element (tpc.dlgon) to mark user intension to delegate. * **[Server]** Use new tpc.dlgon CGI token to drive tpc redirection. * **[CMake]** Add option to build only client libraries. * **[XrdHttp]** If a gridmap file has been configured, XrdHttp now prefers the already mapped username to the non-mapped one that's coming from a security extractor plugin. -------------- Version 4.11.1 -------------- + **Major bug fixes** * **[XrdSys]** Avoid deadlock on poller stop, fixes #1095 * **[XrdCl]** Recreate env r/w lock after forking. * **[XrdCl]** Use different bit for mkpath and tpc delegation. + **Minor bug fixes** * **[Server]** Use correct mechanism to determine DNS registration. * **[XrdCl]** Reflect server added cgi in tpc.scgi. * **[XrdCl]** tpc: don't open non-root/xroot source. * **[XrdCl]** Set the tpckey to 'delegate' if delegating. * **[XrdCl]** Install priv headers in correct dir. * **[CMake]** Link to ${UUID_LIBRARIES} instead of uuid. * **[Python]** Change pkg name to xrootd. + **Miscellaneous** * **[Server]** Allow redirect differentiation for delegated and undelegated TPC. * **[TPC]** Assume delegation if the tpc.scgi token is present. * **[RPM]** drop default 'all.export /tmp' in standalone cfg * **[RPM]** Package SsiLib and SsiShMap with client. -------------- Version 4.11.0 -------------- + **New Features** **[Server]** Add support for multi-vo credentials. **[Python]** Set rpath for bindings to correct xrdcl. **[Python]** Expose XrdCl::Env in Python. **[Python]** Add version attribute to xrootd module, closes #1027 **[Python]** Add cat API to FileSystem, closes #1047 **[XrdCl]** Add unique uuid to open message. closes #1037 **[XrdCl]** Support zcrc32 when extracting a file from ZIP archive. **[XrdCl]** Support decompression (inflate) of files in ZIP archives. **[XrdCl]** Select the least loaded stream for next data chunk. **[XrdCl]** xrdcp: adjust the number of parallel chunks in real time proportionally to the number of connected data streams. **[XCache]** Allow up to 512 MB blocksize. **[CMake]** Add Find XRootD CMake module. **[CMake]** Add FindLibUuid module. **[XrdCms]** Add XrdCmsRedirLocal as ofs.cmslib plugin. + **Major bug fixes** **[Xcache]** Make sure direct cache access is actually accessible. **[Xcache]** Fix missing mode setting in the data space. **[GSI]** Fix SAN check logic due to cut-paste error. **[GSI]** Fix faulty SAN wildcard check and make message clearer. **[Python]** Correctly handle install --user. **[XrdCl]** Fix dirlist response parsing, closes #1038 **[XrdCl]** Sign endsess if instructed to do so by server, fixes #1045 **[SSI]** Make sure the request sessN pointer is always valid. + **Minor bug fixes** **[TPC]** Make sure to include source CGI in a delegated transfer. **[XrdCl]** Respect target checksum vs preset source checksum. **[XrdCl]** Stat local file without opaqueinfo. **[XrdCl]** xrdcp -S option should set the number of additional data streams. **[XrdCl]** Avoid overflow of worker jobs size. + **Miscellaneous** **[Misc]** Allow client to indicate it supports local files. **[UPS]** Disable python bindings. **[XCache]** Mark File is in sync for the final sync. **[Xcache]** Enhance security of the pss.dca directive. **[RPM]** Move xrdmapc to client package. **[XrdCl]** Adjust xrdcp defaults. -------------- Version 4.10.1 -------------- + **Major bug fixes** * **[XrdCl]** Make TPC check bogus-response proof. -------------- Version 4.10.0 -------------- + **New Features** * **[POSIX]** Add methods to the cache mngt objecT to get status of a cached file. * **[Server]** Add xrd.network dyndns option for dynamic DNS handling. * **[Server]** Properly handle dynamic DNS environments. * **[Server]** Add evict option to the prepare request. * **[Server]** Allow better handling for proxy caching clusters. * **[Server]** Allow configurable posc sync level. * **[Server]** Implement framework for a kXR_prepare plug-in. * **[XrdCl]** Implement streaming dirls, closes #225 * **[XrdCl]** Retry policy changes: see details in #950. * **[XrdCl]** Add switch/flag to enforce zip archive listing. * **[XrdCl]** Preserve tried/triedrc cgi info on redirect for kXR_locate request, #closes #944 * **[XrdCl]** Implement prepare evict and expose in xrdfs. * **[XrdCl]** Expose prepare abort and query prepare. * **[XrdCl]** Add tpc.scgi if delegation is being used. * **[Python]** Expose chunked dirlist API in Python. * **[Python]** Support globbing. * **[XrdClHttp]** Add XrdClHttp submodule. * **[XCache]** Implement write-queue number of blocks / threads config options. (#963) * **[CMake]** Add switch for XrdCl builds only. + **Major bug fixes** * **[XrdCl]** Allow to cancel classical copy jobs. * **[XrdCl]** Fix race condition in error recovery. * **[XrdCl]** Prevent race condition between Request T/O and socket read. * **[XCache]** Check for errors during XrdOssFile::FSync - do not write cinfo file if it fails. * **[XCache]** Deny write access to a file in initial Prepare() call, fixes #663 * **[XCache]** Review and correct error handling in POSIX and XCache, implement XCache::Unlink() * **[XrdTpc]** Always query dual stack for HTTP TPC. * **[XrdTpc]** Do not include authz header in the filename. * **[XrdTpc]** Add curl handle cleanup on redirections or errors. * **[XrdHttp]** Include provided opaque information in the redirection. * **[XrdHttp]** Fix digest names returned to clients. * **[XrdHttp]** Fix opaque info handling on redirections. * **[XrdMacaroon]** Fix macaroon authorization config. * **[XrdSecgsi]** Make proxy cache path aware. * **[XrdSecgsi]** XrdSecProtocolgsi::Encrypt - set IV correctly and report correct size. + **Minor bug fixes** * **[XrdCl]** Use DirList with Locate and Merge flag in xrdcp * **[XrdCl]** Index substreams with right index. * **[XrdCl]** Make sure there is no race condition while loading sec protocol. * **[CMake]** Enable by default rpaths on MacOSX. * **[Server]** Use correct directory path creation flag in kXR_open (compatible). * **[Server]** Require localroot to exist and be a directory for default N2N. * **[Server]** Avoid SEGV when given an empty directory continuation. * **[SSI]** Close loophole between internal "provision" and Finished() calls. * **[POSIX]** Correct pfn to lfn translation for the simple case. * **[FRM]** Ignore the space assignment directive. * **[XCache]** Cache::LocalFilePath() only try to fill output pfn buffer if it is non-null. * **[Python]** Make sure pip install fails if install script fails. * **[Python]** Check dependencies on pip install, if some are missing list them and raise an exception. + **Miscellaneous** * **[XrdCl]** Enhance redirect back-trace output. * **[Python]** Respect pip python version when installing xrootd, fixes #955. * **[Python]** Respect pip --user switch, fixes #952 * **[CMSD]** Allow redirect when limit exceeded in sched directive. * **[Packaging]** Add support for Ubuntu disco. * **[Packaging]** Discontinue support for Ubuntu artful. * **[XrdCeph]** Replace XrdCeph source code with a submodule. * **[XCache]** More reasonable defaults for pfc.writequeue parameters. * **[CMake]** Declare CMake options, fixes #995 * **[CMake]** Set -Werror for debug and CI builds only. ------------- Version 4.9.1 ------------- + **Major bug fixes** * **[POSIX]** Fix VMP issues with POSIX preload library causing SEGV. * **[POSIX]** Additional hardening of POSIX functions for Xcache. * **[XrdCl]** Make sure local destination path is not scrambled, fixes #925 * **[XrdCl]** Fix memory leak in AsyncSocketHandler. * **[XrdCl]** Fix recursive copy. * **[XrdCl]** Make sure released SID is valid object. * **[XrdCl]** Correctly calculate the offset of the last file in ZIP64. * **[Server]** Liberalize character set allowed in login name (fixes EOS issue). * **[Server]** Save the proper ReqID for async responses on close. * **[Server]** Handle waitresp sequencing to avoid early bridge termination. * **[XrdHttp]** Only consider running plugins for new requests. * **[XrdHttp]** Handle clients that request multiple checksums. * **[XrdHttp]** Do not accept more data than specified in the HTTP request. * **[XrdHttp]** Fix high memory usage caused by cached XrdHttpProtocol objects. * **[XrdSecgsi]** Fine tune message bucket content. * **[XrdSecgsi]** Make sure options for proxy saving to file are honored. * **[XrdSecgsi]** add missing entries in gsiClientStesp and gsiServerSteps. * **[Python]** Ensure Python module doesn't deadlock/segv on exit, fixes #330 * **[XrdMacaroon]** Create macaroon with proper caveats and path. * **[Proxy]** Improve handling of failures during opening of local data and cinfo files. * **[Proxy]** Fix memory leak - block was not freed when writing to disk failed. * **[XrdTpc]** Only use Curl's low-speed-limit with libcurl v7.38 and later. + **Minor bug fixes** * **[Server]** Fix bridge waitResp issues to allow merge of pull #902. * **[Server]** Further simplify tracing, especially for var persistece. * **[Server]** Fix strlcpy for Group null-string in XrdOfsTPC. * **[XrdMacaroon]** Log user caveats with macaroon generation. * **[XrdMacaroon]** Allow admin to customize default macaroons authz. * **[XrdMacaroon]** Set the macaroons.onmissing default. * **[XrdMacaroon]** Interact with json-c appropriately for request. * **[XrdMacaroon]** Ensure we delete macaroon object after use. * **[XrdHttp]** XrdHttp does not support chunked encoding; respond appropriately. * **[POSIX]** Fix string scoping issues in debug messages. * **[XrdSecgsi]** Downgrade error msg from key file issues to DEBUG. * **[XrdSecgsi]** Improve buffer content dumps for debugging purposes. * **[XrdCl]** Use correct flag to remove handler in inqueue. * **[XrdCl]** Log properly kXR_waitresp, closes #852 * **[XrdCl]** Fail ZIP extraction if file is compressed. * **[XrdHttp]** Don't assume that a header "Depth: 1" means that the resource is a directory. * **[Python]** Expose parallel copy jobs in python bindings. * **[Python]** Expose multi-source download in python. + **Miscellaneous** * **[XrdHttp]** Add support for chunked encoding in uploads. * **[XrdHttp]** Implement RFC 3230 digests for GET requests. * **[Packaging]** Add xrootd-fuse deb package, closes #831 * **[XrdCl]** Add more descriptive error message on XrdCl::CopyProcess:run, closes #849 * **[XrdCl]** Add special logs for future debugging of BNL problem. ------------- Version 4.9.0 ------------- + **New Features** * **[XrdCl]** Provide operation pipelining API. * **[XrdCl]** Redirect traceback dump. * **[XrdCl]** Allow specifying plug-ins on protocol level * **[XrdCl]** Pass # streams to TPC destination. * **[XrdCl]** Enable proxy delegation. * **[XrdCl]** Enable state redirection for local files. * **[XrdCl]** Add an API to close connection. * **[XrdCl/Server]** Implement TPC-Lite for delegated TPC copy. * **[XrdCl/Server]** Implement kXR_writev operation. * **[Server]** Allow tpc requests at destination endpoint to be redirected. * **[Server]** Add &I= to 'u' login monitoring record. * **[Server]** Allow spaces in id's via acc.spacechar. * **[Server]** Allow space preassignment of file creation paths. * **[Server]** Identify type of IP connection in monitoring (&I=4|6 in 'u' record). * **[Server]** Enable use of delegated credentials for 3rd party copy. * **[Server]** Allow callbacks for close() and opague() requests. * **[Server]** Allow the CKS manager to run on the redirector. * **[Server]** Allow config continuation to files in a directory. * **[Server]** Add 'query confg start' to find server start time. * **[SSI]** Add ability to spread requests to increase parallism. * **[SSI]** Add monitoring capabilities to SSI. * **[SSI]** Properly handle Finished() calls prior to response posting. * **[XrdSsi]** Provide summary monitoring information to report stream. * **[TPC]** Allow number of streams to use to be passed to the server. * **[TPC]** Support Http TPC. * **[XrdHttp]** Added XrdHttp cipherlist filter string config option. * **[XrdHttp]** Map Xrd error codes to HTTP status codes * **[XrdHttp]** Add plumbing necessary for RFC3230 digest requests. * **[XrdSecgsi]** Use hostname, not reverse DNS, for address comparison. * **[XrdSecgsi]** Allow XrdSecGSITrustDNS setting to disable use of all DNS lookups. * **[XrdSecgsi]** Add option to save delegated proxies as credentials. * **[XrdSecgsi]** Improved checking of CA expiration. * **[XrdSecgsi]** Review proxy delegation mechanism. * **[XrdSecgsi]** Support signing of server DH public parameters. * **[XrdCrypto]** Allow XRootD client to accept subjectAltNames. * **[XrdMacaroons]** Macaroons plugin for XRootD. * **[Proxy]** Implement new options in pfc.diskusage for better control of purging. * **[Proxy]** Support origins that only talk http. * **[Proxy]** Add 'pss.tid' CGI element to all outgoing URL's. * **[Proxy]** Implement new options in pfc.diskusage for better control of purging. * **[POSIX]** Implement Cache Context Management plugin. * **[XrdApps]** Implement xrdqstats command to display summary monitoring. + **Major bug fixes** * **[XrdCl]** Handle properly seerver disconnect. * **[XrdCl]** apply TPC timeout to the 2nd sync, fixes #800 * **[XrdCl]** Parse correctly CDFH ZIP64 extension. * **[XrdCl]** Correct ownership of virtual redirect msg. * **[SSI]** Do not leak memory when a fatal error occurs. Fixes #775 * **[SSI]** Avoid race condition between Finish() and SetResponse(). * **[Server]** Avoid SEGV when the close operation fails. * **[Server]** Fix SEGV when using delegation and VOMSFUN with raw creds. * **[POSIX]** Avoid SEGV due to race condition between open() and close(). * **[POSIX]** Correct regression that broke POSIX preload. * **[Proxy]** Support multiple IO objects working with the same file. * **[Proxy]** Fix incorrect rc check when processing Fstat() (disabled progbar). + **Minor bug fixes** * **[Server]** Follow protocol spec when returning file compression info. * **[Server]** Fix checksumming on filesystems that don't support fattr. * **[XrdOfs]** Prevent double-parsing of parameters for ofs.ckslib. * **[XrdCl]** Correctly set path in case of state redirect. * **[XrdCl]** Correct handling of --infiles option. Fixes #779 * **[XrdCl]** Don't read past buffer when parsing protocol response. * **[XrdHttp]** Fix error codes in the case of PUT and MKDIR. * **[XrdHttp]** Support mv for files with spaces. * **[XrdHttp]** Return correct content-range header. * **[Secsss]** Pass correct parameters when registering an ID. Fixes #689 + **Miscellaneous** * **[XrdCl]** Delete channel on TTL. * **[Server]** Do not allow badly formed CGI to pollute subsequent tokens. * **[TPC]** Do not take case into account for hostname matching. * **[TPC]** Allow source protocol to be transmitted to server via tpc.spr CGI. * **[TPC]** Set envars XRD_TRACEID, XRDTPC_SPROT, and XRDTPC_TPROT * **[Proxy]** Avoid auth failure due to URL cgi directives. * **[Proxy]** Add method to cache to get local file path of cached file. * **[Proxy]** Fix wrong logic in cache purge algorithm. * **[Proxy]** Resolve conflicts in URL rewriting. * **[POSIX]** Avoid unnecessary cache open requests. * **[XrdCrypto]** Replace C-style string manipulation with C++ equivalent. * **[Secgsi]** Implement stricter version of RFC2818 (host name check) for client. * **[Secgsi]** Enable use of an unique IV in enc/dec cipher operations. * **[XrdHttp]** Break XrdHttp into a module and a utility library. * **[XrdHttp]** Expose query string in external req handler. * **[XrdThrottle]** Update the throttle plugin to propogate underlying errors. * **[XrdThrottle]** Improve chaining behavior of throttle plugin. * **[RPM]** Use OPENSSL_NO_FILENAMES flag. * **[Docs]** Update Doxyfile, closes #743 * **[All]** Place protocol definition under a modified BSD license. * **[All]** FreeBSD compatibility patch. ------------- Version 4.8.6 ------------- + **Major bug fixes** * **[XrdCl]** Make sure released SID is valid object. * **[XrdCl]** Handle properly server disconnect. ------------- Version 4.8.5 ------------- + **Major bug fixes** * **[XrdCrypto]** add protection against missing extension. * **[XrdCl]** Ensure DeepLocate counter doesn't overflow, fixes #758 * **[XrdCl]** Fix recursive copy, fixes #792 * **[XrdCl]** Correctly classify errno while reading/writing to a socket. * **[XrdCl]** Fix local checksum double I/O problem. * **[XrdHttp]** Fix callback of protocol plugin after bridge delay. * **[XrdHttp]** Obey the `Connection` request header. * **[XrdHttp]** Reset filesize when the XrdHttpReq object is used. * **[XrdHttp]** Do not increment reqstate when headers are incomplete. * **[Proxy]** Fix incorrect rc check when processing Fstat(). + **Minor bug fixes** * **[XrdCl]** xrdfs ls: fix formatting issue,fixes #823 * **[XrdCl]** If stream is broken delete the in-message. * **[XrdCrypto]** fix signatures of Export methods. + **Miscellaneous** * **[XrdCl]** Expose kXR_cancel flag in FileSystem::Prepare, closes #699 * **[XrdCl]** Specify file for extraction from ZIP archive also through opaque info. * **[XrdCl]** Document xrdcp/xrdfs return codes, closes #628 * **[XrdCl]** Explicitly log request retries, closes #690 * **[XrdCl]** Remove redundant stat from ZIP open sequence. * **[XrdCl/XrdSec]** Add CGI to specify 'sss' keytab. * **[XrdSecsss]** Add possiblity to inject an endorsement string into an SSS SecEntity object. ------------- Version 4.8.4 ------------- + **Major bug fixes** * **[XrdCrypto]** Use consistently time_t. * **[XrdCl]** Treat empty redirect response as error. + **Minor bug fixes** * **[XrdCl]** Use NEED2SECURE macro properly. + **Miscellaneous** * **[XrdSecgsi]** Improving checking of CA expiration. * **[XrdCl]** Improve TPC key generation, fixes #662 ------------- Version 4.8.3 ------------- + **Major bug fixes** * **[XrdCl]** Release SIDs on PostMaster::Send() failure. ------------- Version 4.8.2 ------------- + **Major bug fixes** * **[Proxy]** Make sure to use N2N even when only localroot specified, fixes #650. * **[Proxy]** Fully support third party copy in proxy servers. * **[Server]** Correct faulty logic for sendq backlog warning message. * **[XrdCl]** Correctly handler error/wait response to endsess request. * **[XrdCl]** MsgHandler must not be enqueued in InQueue on virtual redirect, fixes #682 * **[XrdCl]** Add ZIP64 support, fixes #402 * **[XrdHttp]** Always have OpenSSL read/write data through the XrdLink object. + **Minor bug fixes** * **[Net]** Optimize formatting corresponding to RFC 1178 and RFC 3696. * **[XrdHttp]** Fix HTTP PUT flags, fixes #637. * **[XrdHttp]** Close file handle for simple HTTP reads. * **[All]** Fix compilation with gcc 8. * **[CMake]** Make sure plugins are declared as MODULEs, fixes #653 * **[SSI]** Ruggedize server-side SSI interactions. * **[SSI]** Prevent request ID conflicts with reusable resources. * **[XrdCl]** Validate URLs comming from metalinks. + **Miscellaneous** * **[Server]** Correct lock handling in commit 2c169141. * **[Server]** Make endsess more reliable. * **[Server]** Make sure no temporary opens occur during error recovery. * **[Server]** Add method to get logging mask. * **[XrdHttp]** Support HTTP chunked transfer encoding. * **[XrdHttp]** Allow parsing of unknown HTTP verbs. * **[XrdOss]** Improve XrdPosix 'rename' POSIX compliancy * **[Proxy]** Make sure to pass through TPC requests in otgoing proxies. * **[Proxy]** Support progress bar during TPC transfers. * **[Proxy]** Do not fail a TPC fstat() due to bad timing. ------------- Version 4.8.1 ------------- + **Major bug fixes** * **[XrdCl]** Try all IP addresses in case posix connect fails. * **[XrdCl]** Fix checksuming in xrdcp for local files. * **[Py]** Make sure FileSystem::Copy returns a tuple, fixes #633. * **[Py]** Make sure empty strings are not converted to None. * **[SSI]** Unbind the request prior to teardown. * **[SSI]** Avoid SEGV when generating a request for a new TCP connection. * **[SSI]** Fix race condition that can cause a SEGV during parallel execution. * **[SSI]** Allow zero length requests to be passed to servers. Fixes #640 * **[SSI]** Make sure to avoid object refs after Finished() is called to avoid SEGV. + **Minor bug fixes** * **[XrdPosix]** Fix various memory related issues. * **[XrdCrypto]** Fix various small issues. * **[Server]** Fix overlapping string copy. Fixes #643 * **[XrdCl]** Provide compatibility between root://localfile and file://. ------------- Version 4.8.0 ------------- + **New Features** * **[XrdCl]** Local redirection and local file support. * **[XrdCl]** merge xrdfs ls results if not unique, closes #541. * **[XrdCl]** Provide client specific CGI info. * **[XrdCl]** File::WriteV implementation, closes #388. * **[XrdHttp]** Pass the HTTP verb to the external handler for path matching. * **[XrdHttp]** Allow one to access the XrdSecEntity object associated with a request. * **[XrdHttp]** Allow filtering based on HTTP verb in MatchesPath. * **[XrdHttp]** Allow overwrites to be done on PUT. * **[XrdHttp]** Allow multiple external handlers to be loaded by XrdHttp. + **Major bug fixes** * **[Server]** Correctly handle monEnt on file close to avoid SEGV. Fixes #618. * **[Server]** Poperly handle file descriptors up to 65535. Fixes #607. * **[Server]** Fix handling of >65K attached files (active links). Fixes #623. * **[Server]** Make sure doPost does not become <0 (regression introduced in 4.7.1). * **[Proxy]** Avoid SEGV when localroot specified w/o remote root. Fixes #627. * **[XrdCl]** Connection Window should be applied per IP address. Fixes #625. * **[XrdCl]** Write request and raw data with single writev, fixes #609. * **[XrdHttp]** Allow XrdSfsGetDefaultFileSystem to be called multiple times. * **[XrdHttp]** Correct external handling logic. * **[XrdSecgsi]** Use stack for proper cleaning of invalidated CRLs and CAs. + **Minor bug fixes** * **[Server]** Print error msg and close socket when a FD cannot. be handled. * **[Server]** Close additional loophole for fstream disconnect. * **[Server]** Always unhook the statistcs object from xfr monitoring if hooked. * **[Server]** Ruggedize TPC to be less sensitive to protocol violations. * **[Server]** Correct tpc directive scanning and make it more obvious. Fixes #604. * **[Server]** Enable url rewrites. Eliminates GSI roadblock. * **[Server]** Do not reference a deleted object. * **[XrdSsi]** Make sure to finalyze all requests upon disc, fixes #616. * **[XrdHttp]** Handle properly http.secretkey. * **[XrdCl]** various memory releated fixes. * **[XrdPy]** Translate binary buffers into bytes objects, closes #632 + **Miscellaneous** * **[RPM]** Add python3 sub package. * **[RPM]** Rename python sub-package, closes #614. * **[Py]** Facilitate building python bindings with wheel. ------------- Version 4.7.1 ------------- + **Major bug fixes** * **[XrdSecgsi]** Fix segv in cache checking, fixes #595 * **[XrdHttp]** Fix for the persistent connection issue. * **[XrdHttp]** Fix FD leak when a socket error is encountered. * **[XrdSsi]** Avoid race condition when response is posted. * **[XrdSsi]** Avoid state conflict when request is being processed and client asks for response. * **[XrdCl]** Prevent segv in case user has no name. * **[Server]** Close link on enable errors to prevent socket leaks. + **Minor bug fixes** * **[XrdLink]** Increment the IOSemaphore once for each waiting thread. * **[XrdHttp]** Make sure that the preexisting url tokens are properly quoted when generating a redirection. * **[XrdCl]** Fix invalid memory reads/writes when RAII finalizes mutex after the object has been deleted. + **Miscellaneous** * **[XrdCl]** Log last error in case redirect limit has been reached. * **[XrdCl]** Add option to read credentials under different fsuid/fsgid. * **[XrdCl]** Accept empty login response for protocol <= 2.8.9 (This is only to ensure compatibility with dCache, which due to its inaccurate implementation of XRoot protocol in some cases returns an empty login response for protocol version <= 2.8.9.) * **[XrdCl]** Add envar to config Nagle algorithm. * **[XrdSsi]** Reinitializ response object after Finished() so it can reused. * **[XrdHttp]** Header2cgi directive. ------------- Version 4.7.0 ------------- + **New Features** * **[Proxy]** Make cache I/O synchronization tunable. * **[Proxy]** Allow caching of S3-style objects. * **[Proxy/Posix]** Allow Name2Name to populate cache using the LFN. * **[Posix]** Enable LITE feature in Posix preload library. * **[Posix]** Implement serverless file caching (disk or memory). * **[Server]** Allow storing S3-style objects in a file system. * **[Server]** Add xrootd.fsoverload directive to handle filesystem overloads. * **[Server]** Allow port to be specified for a supervisor. * **[Server]** Add org and role types to AuthDB authorization. * **[Server]** Allow definition and test of compound authorization identifiers. * **[Server/Packaging]** Handle systemd socket inheritance. * **[XrdApps]** Add XrdClProxyPlugin implementation. * **[XrdCl]** Extreme copy implementation. * **[XrdCl]** Delegate all callbacks to the thread-pool. * **[XrdCl]** xrdfs: add recursive list, closes #421. * **[XrdCeph]** Added support for namelib in ceph plugin . * **[XrdFfs]** Implement xrootdfs_create. * **[Python]** Python 3 support in C / Python interface. * **[XrdHttp]** Make XrdHTTP able to forward HTTP requests to an external, optional plugin (conceptually similar to CGI). * **[Server]** XrdSsi V2 (scalable service interface) implementation. + **Major bug fixes** * **[XrdCl]** Avoid deadlock between FSH deletion and Tick() timeout. * **[XrdCl]** Process virtual redierections in the threadpool. * **[Xrd] Fix handling of sendfile offset argument. + **Minor bug fixes** * **[Server]** Make file locking independent of FS plugin. Fixes #533 * **[Server]** Correct debug message interval for free space report. * **[XrdCeph]** Fixed internal (f)stat so that it sets S_IFREG in returned mode. * **[XrdCeph]** properly return ENOENT when file does not exist in open for read. * **[XrdCeph]** Fixed configuration of the XrdCephOss module. * **[XrdCeph]** Fixed some resource leak when Posix_Open fails. * **[XrdFfs]** Remove default fuse argument "allow_other" as it is impossible to unset. * **[XrdFfs]** Check file descriptor before using it in xrootdfs wcache. * **[XrdFfs]** Add more error checks when creating write cache. * **[XrdFfs]** Avoid using literal 1024, replace with MAXROOTURLLEN. * **[XrdFfs]** Control allow_other by env XROOTD_NOALLOWOTHER. * **[XrdFfs]** Rewrite xrootdfs_mknod, extract low-level function. * **[XrdCl]** Check login resp size, fixes #530 * **[XrdCl]** Avoid FileStateHandler deadlock while forking. * **[XrdCl]** Handle failed stateful operations without XrdCl::File lock being locked. * **[XrdPosix]** Use strncpy when copying checksum. * **[RPM]** Fix init script bad exit code, fixes #536 * **[XrdBuffer]** Decrement total buffer count when freeing buffers. + **Miscellaneous** * **[Server]** Re-enable the oss.fdlimit directive to allow POSIX preload+xrootd. * **[Server]** Avoid thread pile-up durin slow close operations. * **[Proxy]** Simplify delayed destruction on wait vs post. * **[Posix]** Convert to using universal tracing facility. * **[CI]** Add Travis CI configuration. * **[CI]** Add .gitlab-ci.yml for gitlab CI. * **[Packaging]** Add a sample XrdHttp config file. * **[Packaging]** Make RPM version configurable by the user. * **[Packaging]** Debian packaging. * **[RPM/CMake]** Enable c++0x/c++11 by default. * **[Crypto] Remove unused crypto code. * **[XrdFileCache]** Add configuration parameter for flush frequency. * **[XrdFileCache]** Alter ram limits and blocks size parameter if caching is on the client side. * **[XrdSut]** New XrdSutCache based on XrdOucHash. * **[XrdSecgsi]** do not delete explicitely the CRL in Delete. * **[XrdSut/Crypto]** Secgsi improvements: new version of XrdSutCache, lightweith locking (PR #539). ------------- Version 4.6.1 ------------- + **Major bug fixes** * **[Server/Proxy]** Avoid SEGV when close(), closedir() returns an error. * **[cmsd]** Fix feature interaction causing improper file existence to be sent. * **[XrdCrypto/XrdSecgsi]** Make sure the CRL is loaded for the right CA. * **[XrdCrypto]** Support for OpenSSL 1.1 * **[XrdSecgsi]** do not build/package libXrdSecgsiGMAPLDAP-4.so. * **[XrdSecgsi]** Improve detection of errors when loading CRL. * **[XrdSecgsi]** Fix for valid legacy proxy detection (PR #469) * **[XrdSecgsi]** Absent CRLs not an error (#465) * **[XrdSecgsi]** Fix for CA chain verification segfault (issue #463) * **[XrdSecgsi]** Two memory leaks (PR #503) * **[XrdCl]** Make sure there is no request/response mismatch, when the retry logics tries to recover from an error. * **[XrdCl/Server]** Be case insensitive when it comes to checksum names. * **[XrdCeph]** Fix ability to read back a file written with O_RDWR flags. * **[XrdCeph]** Disable logging of every read and write operation. A proper debug-level logging would be needed instead. * **[XrdCeph]** Added statistics about read/write operations in the close log. + **Minor bug fixes** * **[XrdHttp]** Make the XrdHttpSecXtractor API backwards compatible. * **[XrdFileCache]** Make caching proxy configuration backwards compatible. * **[XrdFileCache]** Fix cache v1 to cache v2 bridge after introducing cache v2. * **[XrdSec]** Use CommonCrypto header instead of openssl for SHA on OSX. * **[XrdSeckrb5]** Fix memory leaks in client context and cache. * **[Server/Logrotate]** Make sure XRootD logrotate does not interfire with system logrotate, fixes #490 * ** [Server]** Avoid std::ABORT should a naked logfile path be specified. * **[XrdCl]** Make sure ForkHandler doesn't segv if PostMaster is null, fixes #489 * **[Packaging]** Set the working dir to /var/spool/xrootd on CC7, fixes #365 * **[Packaging]** On platforms where systemd is available, manage files in /var/run with tmpfiles.d, fixes #485 + **Miscellaneous** * **[XrdPosix]** Add new minpages option to pss.cache to support large pages. * **[XrdPosix]** Make XrdPosix.hh a public header; closes #479 * **[XrdApps]** Remove XrdClient dependency from xrdadler32. * **[Server]** Add XrdCksAssist functions to help handle XRootD checksums. * **[Server/Proxy]** Move disk sync operations out of IO::ioActive() call. * **[Server/Proxy]** Change severity IO::initLocalStat() log message. * **[XrdFileCache]** Ease development of decision plugins. * **[XrdFileCache]** Use ref-counts on File objects. ------------- Version 4.6.0 ------------- + **New Features** * **[XrdCms]** Add non-blocking sends to avoid slow links. * **[XrdFileCache]** File caching proxy V2 (and new pss async interface). + **Major bug fixes** * **[XrdCeph]** Account for return Ceph xattr return codes. * **[XrdCeph]** Fixed initialization of Ceph clusters when stripers are not used. * **[XrdCrypto]** Improved determination of X509 certificate type, including proxy version * **[XrdHttp]** Fix memory leak in Bridge protocol (affects HTTP). * **[XrdSecgsi]** Several improvements in the way CRLs are checked and reloaded. * **[XrdCl]** Protect against spurious wakeups in SyncResponseHandler. * **[XrdCl]** On read-timeout, if the stream is broken, make sure the request and its handler are not double deleted. + **Minor bug fixes** * **[XrdCl]** Check if the file was correctly closed upon ZipArchiveReader destruction. * **[Server]** Add limits for prepare requests. * **[Server]** Delete buffers when the buffer manager is deleted. Fixes #414 * **[Server]** Do not double count overlapping spaces. Fixes #425 * **[XrdHttp]** Allow unauthenticated https clients. * **[XrdHttp]** Make Xrdhttp secure by default (rejecting proxy cert in the absence of a proper SecXtractor plugin) + **Miscellaneous** * **[XrdSecgsi]** Re-activate xrdgsitest * **[RPM]** Include xrdgsitest in xrootd-client-devel package. * **[XrdFileCache]** Add example of filecache configuration. ------------- Version 4.5.0 ------------- + **New Features** * **[XrdCms]** Allow specifying a different timeout for null cached entries; fixes #413 * **[XProtocol/XrdSec/Server/XrdCl]** Implement request signing. * **[XrdCl]** Add ZIP extracting capability to xrdcp. * **[XrdCl]** Include the release number in client Login request cgi. * **[XrdCl]** Add support for spaces in file names for mv operation. + **Major bug fixes** * **[XrdCrypto/Secgsi]** Fix XrdCryptosslMsgDigest::Init ; set 'sha256' as default algorithm. * **[XrdCl]** Use posix semaphores for fedora >= 22. Disable omit-frame-ponter for gcc >= 4.9.3 if custom semaphores are used. + **Minor bug fixes** * **[XrdSecsss]** Fix memory leak in sss protocol. * **[XrdNet]** Allow hostnames to begin with a digit. * **[XrdCl]** Fix segfault in case a user cannot be mapped to a home directory. * **[XrdCl]** Make sure a socket is always associated with a proper poller object (not null). * **[XrdCl]** Fix deadlock in XrdCl::PollerBuiltIn during finalize. * **[XrdCrypto]** Do not use md5 checksum on OSX platform. + **Miscellaneous** * **[RPM]** Include xrdacctest in xrootd-server package. * **[RPM]** Add conditional BuildRequires for ceph >= 11. * **[RPM]** Use compat-openssl10-devel for fedora>=26. * **[XrdCl]** Make sure the Log class can be used by any client plugin implementation. ------------- Version 4.4.0 ------------- + **New Features** * **[Server]** Add new [no]rpipa option to xrd.network directive. * **[Server]** Allow objectid's to be specified in the authorization file. * **[Server]** Add new logging plugin interface. * **[Server]** Fixes #345 - add sid to TOD structure (ABI compliant). * **[Server]** Implement resource selection affinity (primarily for ssi). * **[XrdCl]** Add Metalink support (xrdcp & API). * **[XrdCl]** Enable metalink processing on default. * **[XrdCl]** xrdcp: use cks.type cgi tag to select the checksum type. * **[XrdCl]** Support local metalink files. * **[XrdCl]** Add support for GLFN redirector of last resort. * **[XrdCeph]** Implemented pools of ceph objects. + **Major bug fixes** * **[Posix]** Remove double unlock of a mutex. * **[Client]** Serialize security protocol manager to allow MT loads. * **[Authentication/sss]** Fix dynamic id incompatibility introduced in 4.0. * **[XtdHttp]** Removed the deprecated cipher SSLv3, in favor of TLS1.2 * **[XrdCl]** Close file on open timeout. * **[XrdCl]** Differentiate between a handshake and an xrootd request/response while processing an incoming/outgoing message. * **[XrdCl]** Fix dangling pointer issue that occurs while forking. * **[XrdCl]** Ensure DefaultEnv is finalized after last use of the object. + **Minor bug fixes** * **[Proxy]** Avoid SEGV when printing memory cache statistics. * **[Server]** Avoid XrdNetIF static initialization issues. * **[Server]** Honor DFS setting when forwarding operations. * **[Server]** Make sure lockfile time is updated in deprecated runmodeold. * **[Server]** Fixes #344 - squash path before checking for static redirect. * **[Server]** Free Entity before replacing it from the cache (memleak). * **[XrdCl]** xrdfs ls does not include opaque info in a listing. * **[XrdCl]** Eliminate unnecessary write notifications. * **[XrdCl]** Forward xrd.* parameters from the original to the redirection URL. * **[XrdCl]** Do not preset CWD in batch mode. * **[XrdCl]** Be complaint with file URI scheme. * **[XrdCl]** Fix wrong query string used for opaquefile code. * **[XrdCl]** Translate XRootD error code to errno before passing to strerror. * **[XrdCeph]** Fixed thread safety of filedescriptors in the ceph plugin. * **[XrdCeph]** Protected initialization of ioCtx object and striper objects by mutex in the ceph plugin. * **[XrdCeph]** Fixed memory corruption in asynchronous read from ceph. * **[XrdXml]** Make sure c-string buffes are properly terminated. * **[XtdHttp]** Don't trim printable characters. + **Miscellaneous** * **[XrdCl]** Change the way handlers and messages are matched (use maps). * **[Apps]** Add xrdacctest to the tools set to test access control databases. * **[Packaging/RPM]** Set max open files limit to 65k for systemd services. ------------- Version 4.3.0 ------------- + **New Features** * Add option to query network configuration via configured interfaces. * **[Proxy]** Default event loops to 3 and allow it to be set via config. * **[Server]** Let client inform redirector why it's retrying a lookup using the triedrc CGI element. * **[Server]** Add cms.cidtag directive to qualify the global cluster id (solves dpm problem). * **[Server]** Make it possible to effeciently query an external database for file existence via the statlib plug-in (largely for DPM).` * **[Server]** Allow declaring extra large I/O buffers (mostly for Ceph). * **[Server]** Allow iovec based data responses (no ABI changes). * **[Misc]** Add back trace capability to the tool set. * **[Misc]** Add generalized XML parsing ability. * **[Misc]** Add metalink parsing for future integration. * **[XrdCl]** xrdcp add env var to disable recovery * **[XrdCl]** Add support for multiple event loops. + **Major bug fixes** * **[Server]** Correct IP address matching between IPv4 and IPv6. Fixes #300. * **[Server]** Ruggedize cmsd thread synchronization during node deletion. * **[Server]** Delete extraneous semaphore wait to avoid deadlock (#290). * **[Server]** Return correct response to a delayed open. * **[Server]** Fix build of kXR_wait message in case of delayed open. * **[XrdCl]** Detect whether client is dual stacked based on outgoing connection in addition to DNS registration. * **[XrdCl]** Avoid EAGAIN loop. Fixes #303. * **[XrdCl]** Append opaque info in case we retry at a data server after being redirected. * **[XrdCl]** Fix: FileStateHandler::OnOpen seqfaults when both write timeout and OpenHandler timeout at the same time. * **[XrdCl]** Avoid SEGV when server fails after it responds waitresp. * **[XrdCl]** Continue processing remaining files after error occurrence. * **[XrdCl]** Fix for dangling pointer problem in deep locate, fixes #324 * **[Misc]** Add possibility to specify disk usage parameters in .. G, T units using XrdOuca2x::a2sz(). * **[Python]** Fix lock inversion in python bindings. * **[Python]** Check if python interpreter is still initialized. + **Minor bug fixes** * **[All]** Fix numerous issues with space reporting (spaceinfo, query space, statvfs) such a double counting, scaling, and format issues. * **[Proxy]** Do not use the ffs code path if nothing is writable. This avoids initialization failure when the origin is a large WAN cluster. * **[Server]** Be agnostc NTP defaults when rotating logs (fixes new RH7 defaults). * **[Server]** Pass correct total data length in iovec to Send(). * **[Server]** Avoid redirection loop during error recovery in a uniform cluster. * **[Server]** Make sure N2N gets configured for the cmsd when actually needed. * **[Server]** Properly handle an inifit NPROC limit. Fixes #288. * **[Server]** Make sure the cluster ID is always formatted the same way. * **[Server]** Correctly compute timeout wait. * **[Server/Logrotate]** Make sure rotating pattern is not expanded in an if statement, fixes #302 * **[Misc]** Include XrdXmlReader in the spec file. * **[Misc]** Fix bug in access statistics print. * **[Misc]** Allow conversion of decimal numbers in XrdOuca2x::a2sz() * **[XrdCl]** xrdfs prepare has to be provided with a filename, fixes #309 * **[XrdCl]** Make sure recursive copy is disallowed only for checksum with user provided value, fixes #304 * **[XrdCl]** Use the same timeout value for all close operations in xrdcp with TPC enabled. * **[XrdCeph]** Fixed race condition in multistream access to files fo CEPH + **Miscellaneous** * **[Server]** Prevent cmsd reconnect storm when things get way slow. * **[Server]** Changes to allow for Solaris compilation. * **[Server]** Changes to allow for OSX compilation. * **[Server]** Detect cyclic DNS host registration when processing '+' hosts. * **[Server]** Display manager IP addresses during '+' host resolution. * **[Util]** Avoid compiler warning about unsafe mktemp. * **[App]** Do not report expected errors as errors. * **[App]** Always show any unusual node status in the display. * **[Client/Python]** Add MANIFEST.in for python bindings. * **[XrdFileCache]** Implement blacklisting in a FileCache decision plugin. * **[XrdFileCache]** Make sure requested offset is reasonable. * **[XrdFileCache]** Return -1 and set errno when bad offset is passed in. * **[XrdFileCache]** Only generate error for negative offsets, as per posix. * **[XrdFileCache]** Add startup protection for ReadV, too. It was already there for Read. * **[XrdFileCache]** Fix bug in cache scanning; simplify deletion loop. * **[XrdFileCache]** Use bytes to calculate how many files to purge, not blocks; subtract actual size of the file, not the length of it returned by stat. * **[XrdFileCache]** In cache purge, use stat.mtime of cinfo file if last access time can not be determined from contents of cinfo file. * **[XrdFileCache]** Fix argument type from int to long long (was n_blocks, is size_in_bytes now). * **[XrdCl/XrdSys]** Use custom semaphores only for glibc<2.21. * **[XrdCl]** Remove libevent-based poller implementaion. * **[XrdCl]** Report reason for reselection via triedrc CGI element. * **[XrdClient]** Changes to allow for Fedora rawhide C++11 compilation. * **[XrdCeph]** Fixed XrdCeph compilation for C++11 enabled compilers * **[XrdCeph/CMake]** Fix for undefined symbols (link XrdUtils). ------------- Version 4.2.3 ------------- + **Major bug fixes** * **[Server]** Avoid SEGV if cmsd login fails very early. * **[Server]** Avoid SEGV when an excessively long readv vector is presented. * **[Server]** Rationalize non-specfic locate requests. * **[XrdCl]** Process waitresp synchronously via Ignore return to avoid SEGV. * **[XrdCl]** Avoid memory leak when a handler returns Ignore for a taken message. * **[XrdCl]** Fix "tried" logic by forwarding the errNo ------------- Version 4.2.2 ------------- + **Major bug fixes** * **[Proxy]** Protect forwarding proxy server from slow connections. This should fix most, if not all, SEGV's that the server encountered under heavy load. * **[Server]** Fixes #248 Prevent infinite loop when shift arg is negative. * **[Server]** Complain when passed I/O length is negative. * **[Server]** Avoid execution stall during node logout when the thread limit has been reached. * **[Server]** Make sure to capture return code for stat() to prevent random results. * **[XrdCl]** Make sure to get filestate lock during timeout processing to avoid MT intereference and possible random results. * **[XrdClient]** Restore commented out abort() when an attemp is made to index a vector outside of its current bounds (avoids random results). * **[Server/Proxy]** Delay deleting a file object if the close was not successful. This avoids deleting objects that may have pending activity resulting in an eventual SEGV. This is a bypass fix to another problem. + **Minor bug fixes** * **[Server]** Fixes #234 Properly register all components in a mkpath request. * Correctly handle copying into a non-existent directory when automatic path creation is enabled. * **[XrdCl]** xrdfs correctly handles quotations (fixes the problem with ALICE token) + **Miscellaneous** * Fixes #245 Provide compatibility when cmake version is > 3.0. * Use atomics to manipulate unlocked variable pollNum. * Bugfix: release lock when a file is closed before the prefetch thread is started. Observed with xrdcp ran without -f option and an existing local file. Fixes #239. * Protect from reads exceeding file size. Fixes #249. * Release Stream lock before invoking callbacks. Fixes #216 * TPC: Fix deadlock in case of error in the TPC authentication * Increase max size of write to disk queues. * Fix bug in endswith. Fixes #260 * XrdCeph : fixed problem with files bigger than 2GB for synchronous writes * **[XrdCl]** Change message loglevel from Error to Debug. Fixes #246. * **[XrdCl]** Fix race condition in PostMaster initialization * **[XrdCl]** Provide atomicity for PostMaster value using built-in functions * **[XrdFileCache]** fixed deadlock on immediate file close (e.g. xrdcp to non-writable output) * **[XrdFileCache]** fixed errors on some posix operations using virtual mount ------------- Version 4.2.1 ------------- + **Miscellaneous** * **[Client/Cl]** Make sure kXR_mkpath is set for classic copy jobs when the destination is xrootd (backward compatibility fix). ------------- Version 4.2.0 ------------- + **New Features** * **[Client/Python]** Integrate xrootd-python into the main package. * **[Server]** Include a Ceph OSS plug-ing. * **[Server]** Implement throttling. * **[Server]** Detect redirect loops using "tried" token. * **[Server]** Implement the "cid" option for config query to display the unique cluster ID. * **[Server]** Allow suspending and enabling remote debugging without a restart. * **[Server]** Implement black/whitelist with optional redirection. * **[Server/Proxy]** Add the xrdpfc_print tool to print the caching proxy metadata. * **[Server/PlugIns]** Provide a mechanism to pass command line arguments to plug-ins. * **[Server/PlugIns]** Provide access to the native and the active extended attribute implementation. + **Major bug fixes** * **[All]** Fix various memory access issues. * **[Server]** Fix various IPv4/IPv6 compatibility issues. * **[Server]** Avoid disabling of frm notifications due to plug-in initialization issues. * **[Server/Proxy]** Avoid holding a global lock when opening/closing files to solve timeout issues. * **[Security/GSI]** Fix reloading of CA and CRLs. + **Minor bug fixrs** * **[Server/HTTP]** Fix issues related to invalid chunk sizes. * **[Server/Proxy]** Various logic and permission processing fixes. + **Miscellaneous** * **[Client/Cl]** Make the compiler issue warnings when the return codes from the File and FileSystem methods are unchecked. (issue #188) * **[RPM]** Disable building of the compat package by default. * **[Server/Proxy]** Avoid serializing stat() via the proxy to improve performance. * **[Tests]** Factor out the common testing code from the client tests so that it can be re-used. ------------- Version 4.1.2 ------------- + **Major bug fixes** * **[Utils]** Don't confuse -I and --tpc while parsing commandline parameters for xrdcp. (issue #213) * **[Server]** Fix various IPv4/IPv6 issues. (issues #164, #227) + **Minor bug fixes** * **[Client/Cl]** Print mtime when doing xrdfs stat. * **[All]** Fix some memory access issues. (issues #186, #197, #205) * **[Server]** Recreate logfile fifo if it already exists and is a file. (issue #183) * **[Server]** Properly reset suspend state when reconnecting cmsd. (issue #218) * **[Server]** Avoid disabling async I/O when using an oss plugin that does not implement file compression. (issue #219) * **[Server]** Do not debit space when relocating a file within the same partition. * **[Server]** Fix meta-manager port directive ordering. * **[Server/Logrotate]** Do not print anything to stdout to avoid making cron send emails to admins. (issue #221) + **Miscellaneous** * **[Server/Proxy]** Disable POSC processing when a proxy plugin is loaded. ------------- Version 4.1.1 ------------- + **Major bug fixes** * **[RPM]** Remove the library patch from xrootd-config to enable multiarch installations. * **[RPM]** Move the user creation scriptlets to xrootd-server where they belong. (issue #179) * **[Server]** Fix PowerPC compilation. (issue #177) * **[Server]** Avoid the pitfalls of infinite nproc hard limit in Linux. * **[Server]** Correct flag definition to include cms plugin loading. (issue #176) + **Miscellaneous** * **[Man]** Update documentation. * **[Client/Cl]** Set the multi-protocol ability basing on an environment variable. ------------- Version 4.1.0 ------------- + **New Features** * **[Everyting]** Implement dynamic plugin shared library filename versioning to allow multiple major versions to co-exist. * **[Server]** Compelete IPv6/IPv6 and public/private network routing. * **[Server]** Allow the checksum manager to use OSS layer to access data. (issue #140) * **[Server]** Allow the definition of subordinate clusters. * **[Server]** Support multiple checksum types. Client can select non-default checksum using the "cks.type=" cgi element. * **[Server]** Provide plugin interface for handling extended attributes. * **[Server]** Add options to xrd.network to control keepalive. * **[Server]** Control core file generation via xrd.sched core directive. * **[Server]** Add pss.permit directive to restrict outbound connections for forwarding proxies. * **[Server]** Allow xrootd to handle objectid names as exports. * **[Server]** Install and package the cluster mapping utility: xrdmapc. * **[Server]** Allow the specification of xrootd.seclib default. * **[Server]** Pass along XRD_MONINFO setting and application name to monitoring. * **[Server/Proxy]** Implement a forwarding proxy option. * **[Server/Proxy]** New configuration of XrdFileCache using 'pfc.' prefix. * **[Sever/HTTP]** Support gridmap parsing. * **[Client/Cl]** Inform the server about availability of local IP address types (IPv6/IPv4, public/private) to in order to facilitate redirections. * **[Client/Cl]** Make the client send kXR_endsess request when recovering broken connection - avoids 'file already open' errors. * **[Client/Cl]** Implement TCP keep-alive support. * **[Client/Cl/xrdcp]** Optimize xrdcp uploads by compensating for latency. * **[Client/Cl/xrdcp]** Make it possible for xrdcp to run multiple transfers in parallel using the '--parallel' option. * **[Client/Cl/xrdcp]** Make it possible for xrdcp to concatenate multiple sources to stdout. * **[Client/Cl/xrdfs]** Add xrdfs locate -i option to ignore network dependencies (IPv6/IPv4). * **[Security]** Add new security framework loader to allow external pacakges that linked against security plugins to dynamically load them instead. * **[Security/sss]** Allow forwardable sss tokens when ecrypted with a forwarding key as defined by the xrdsssadmin command. * **[Plugins]** Implement generic matching rules to version check 3rd party plug-ins. * **[Packaging/RPM]** Add SystemD configuration files for RHEL7. * **[Packaging/RPM]** Introduce compat RPM packaging providing xrootd 3.3.6 daemons and libraries with the ability to switch between desired versions using the sysconfig file. * **[Packaging/RPM]** The RPM naming has been switched back to xrootd (from xrootd4). * **[Utils]** Add xrootd-config utility. + **Major bug fixes** * **[Server/HTTP]** Make it possible to handle files larger than 2GB. * **[Server]** Prevent blacklisting of all connctions when role is supervisor. * **[Server]** Fix bug in handling cms.dfs redirect verify that would keep the client is an infinite wait loop. This also affected locate requests regardless of what the redirect option was set to. * **[Server/Proxy]** Avoid SEGV when no environment has been passed in the proxy server. + **Minor bug fixes** * **[C++ API]** Provide complete portability and correct behaviour across platforms with and without Atomics. This patch does not change any ABI's. * **[Server]** Do not set *TCP_NODELAY* for unix domain sockets as this issues a nasty error message. * **[Server]** Allow cms.dfs mdhold argument to be 0 as documented. * **[Server/Plugins]** Add missing initializer to the LocInfo structure. * **[Server/Plugins]** Correct header define gaurd in XrdSfsFlags.hh. * **[Server/Proxy]** Fully support extended file system features and pass those features through a proxy server. (issue #115) * **[Client/Cl]** Remove duplicates from the HostList. * **[Client/Cl]** Fix minor atomicity issues (C++11). + **Miscellaneous** * **[Server]** Actually remove xmi plugin handling as xmilib is no longer supported. * **[Server]** Make sure to always passhrough CGI information. * **[Server]** Honor network routing when creating the client's i/f selection mask. * **[Server]** Efficiently handle replicated subscribers (i.e. managers). * **[Server/HTTP]** Remove useless loading the security framework. * **[Server/Security]** Add new NetSecurity::Authorize() method that accepts text. * **[Server/Proxy]** Properly support proxying objectids. * **[Server/Proxy]** Clean-ups in the caching proxy. ------------- Version 4.0.4 ------------- * **Major bug fixes** * **[Client/Cl]** Properly allocate buffers for error messages. (issue #136) * **[Client/Cl]** Check if there is enough data before unmarshalling. * **[Client/Cl]** Fix a memory leak in MessageUtils::WaitForResponse affecting all synchronous calls. * **[Client/Cl]** Prevent a segfault in the destructor when called after the libXrdCl library has been finalized by the linker - ROOT garbage collection. https://github.com/cms-externals/xrootd/pull/1 * **[Client/Posix]** Fix broken readdir_r() and readdir_r64() functions. * **[Server]** Use correct flag when adding a cluster. The bug made it impossible to have more than one supervisor node. * **[Server/Logrotate]** Prevent stack corruption by correctly sizing the timestamp buffer. + **Minor bug fixes** * **[Client/Cl]** Properly check if a recursive copy was requested to avoid unnecessarily stating the source. * **[Client/Cl]** Avoid inserting duplicate entries to HostList when retrying at the same server. * **[Client/Cl]** Normalize (trim leading zeroes) before comparing adler and crc checksums. (issue #139) * **[Client/Posix]** Prevent mkdir failure in a clustered environment by creating the full directory path by default. * **[Client/Possix]** Fix a memory leak when doing deep locate. * **[Server/Logrotate]** Use expect to send a ping to pipes. This prevents logrotate from hanging when nobody is listening at the other end of the pipe. * **[Authentication/Client]** Pass the external environment to the protocol manager. (issue #133) * **[Authentication/sss]** Fix a memory leak. * **[Utils]** Avoid SEGV when assigning a unix domain address to a NetAddrInfo object previously used to hold a TCP domain address. * **[Server/cmsd]** Use the same write selection rules for dfs and non-dfs environments. + **Miscellaneous** * **[Server/Logrotate]** Prevent the default configuration from sending emails to admins and from creating a new log after the old one has been rotated. (issue #135) * **[Server/SELinux]** Using expect in logrotate requires the logrotate_t context to have access to pseudoterminals and tmpfs as well as stating fifos * **[Client/Commandline Parser]** Allow local to local copy in new xrdcp but not in the old one. * **[Client/Cl]** Discard a whole cluster on failure in federation context. (issue #132) ------------- Version 4.0.3 ------------- + **Major bug fixes** * **[Server]** Make sure the network routing is honored in all cases. This fixes problems encountered by sites whose clients use a private IP address to connect to a redirector's public IP address. (issue #130) ------------- Version 4.0.2 ------------- + **Minor bug fixes** * **[Client/Cl]** Handle all non-NULL-terminated error responses correctly. * **[Client/Cl]** Release old auth buffer when reconnecting after TTL expiration. + **Miscellaneous** * **[Client/Cl]** Retry after an incomplete local write. This produces clearer error messages. Ie: "Run: [ERROR] OS Error: No space left on device" instead of: "Run: [ERROR] OS Error: Operation now in progress". * **[Client/Cl]** Don't force a server to issue a short read when fetching last data chunk. This works around issues for proxied FAX sites. ------------- Version 4.0.1 ------------- + **Major bug fixes** * **[Server]** Prohibit accessing memory via /proc using digFS. + **Minor bug fixes** * **[Server]** Prevent over-scan of the xrd.network routes option which may cause a config file error message and initialization failure. * **[Server]** Fixes to make things compile on ix86, arm and ppc64. * **[Server]** Correct protocol name supplied to monitoring for userid. * **[Server/Proxy]** Various minor fixes to caching proxy. * **[Security]** Check the length before looking inside a SUT buffer. (issue #126) * **[Client/Cl]** Check for copy source and target validity to display proper error messages. * **[Client/Cl]** Return default plug-in factory for an empty URL. (issue #120) * **[Client/Posix]** Provide full error mapping for POSIX interface. * **[All]** Remove some unnecessary commas and semicolons. (issue #121) + **Miscellaneous** * **[Server]** Pass client login information to monitoring. * **[Client/Cl]** Make xrdfs locate -h synonymous to locate -m. * **[Client/Cl]** Add -i option to xrdfs locate setting the Force flag. * **[Docs]** Various documentation updates. ------------- Version 4.0.0 ------------- + **New Features** * Supprt IPv6. Please read docs/README_IPV4_To_IPV6 for details. * Introduce the XrdFileCache library - a proxy server plugin used for caching of data into local files. * Beta support HTTP(S). * Provide protocol bridge to let other protocols use xrootd back-end plugins. * Provide full support for public/private IP networks. * Allow remote debugging via the xrootd.diglib directive. * Provide a mechanism to manually control log file rotation via -k and add support for logrotate. * Add -z option to enable high recision log file timestamps. * Define a new plug-in to allow replacement of the stat() function when used to determine exported file characteristics. This plug-in is meant to be used by tape-backed file systems that identify offline files in odd ways (e.g. GPFS). Patch assumes XRDROLE patch below. * Implement full readv-passthru for enhanced performance. * Add a disconnect record to the f-stream. * xrdcp is now the same as xrdcopy, and old xrdcp is now xrdcp-old * Make clients configurable via /etc/xrootd/client.conf and ~/.xrootd/client.conf * Implement a plug-in system for client's File and FileSystem queries. * Make it possible for 'xrdfs stat' to query for combination of flags. * Make third party copies cancellable. * Implement xrdfs spaceinfo, cat and tail commands * Terminate iddle connections after a timeout and treat timeouts on streams that should be active (because of outstanding requests with no delay times) as errors. * Implement XrdCl::File::Visa and XrdCl::File::Fcntl. * Support for full URL redirects. * File and Filesystem objects implement property system to pass custom information to and from them (including plug-ins) without breaking ABI. * Add --dynamic-src to xrdcp options to allow dynamic file copying. * Implement the directory listing in bulk. * Enable locate to return host names not just IP addreses. * Implement node blacklisting for the cmsd (see cms.blacklist directive). * Add mv command to frm_admin. * Allow query of current role and dynamic cms state via kXR_query. * Implement query config chksum to return supported chksum name. * Add version as a variable that can be returned by kXR_Qconfig. * Add sitename as an argument to kXR_Query+kXR_Qconfig. * Provide disconnect notifiation to underlying file system. * Provide the filesystem plugin a way of creating a session storage area. * Add flag to indicates a secondary copy of a file exists. * Allow testing for undefined set/env vars via if-else-fi. * Add '-L' flag to the xrootd command to allow loading a protocol library * Add flag to indicates a secondary copy of a file exists + **Bug fixes** * Fix various dead locks in the IOEvents poller. * Implement LinuxSemaphore class in order to replace buggy POSIX semaphores on Linux. * Honor the cmsd.dfs directive for locate request to avoid placing a file in ENOENT status. * Make sure that the old client runs only in IPv4 mode as mixing modes does not work for a variety of reasons. * Accept old-style as well as new-style IPv6 addresses in the sss protocol. This allows the new client to use this protocol after it implemented IPv6 support. * Prevent invalid mutex operations in auto-termination routine. * Resolve naming conflicts within the frm that resulted from the statlib plugin implementation. * Do not rely in file locking to serialize inter-thread access. This fixes the prolem of usage file drift. * Fix various parse context issues in copy config with --recursive. * Recognize object deletion in the error handling path. * Use atomic FD_CLOEXEC where available to prevent FD leaks. * Squelch casting complaints from C++11. * Make sure to return all nodes in a star locate request. * Always load protocols in the specified order. * Fix xrootdfs wcache crashing issue when using virtual file descriptor. * Fix selection of a server when a DNS entry resolves to more than one. * Correct pthread_cond_timedwait() time calculation and error handling. * Fix null insertion of hostname in error message when open fails. * Fix issues with extensions in GSI proxies * Fix problem with creation of the forwarded KRB5 ticket * Correctly handle reading of a partial readv headers (issue #45) * Make sure to propagate username and password when redirecting * Honor request timeouts when processing kXR_wait + **Miscellaneous** * XrdClient and associated commandline utilities are now obsoleted. * Propagate info about partial success from deeplocate to dirlist. * Remove perl interface. * Send timezone, country code and application name while logging in. * Change interfaces to copy process to use property system (allows for adding features without breaking the ABI). * Final change to f-stream monitoring. Replace standard deviation (sdv) calc with reporting sum of squares (ssq) counts. * Make public headers compile cleanly with -Wall -Wextra -Werror. * Support passing cert, key paths via URLs * Allow testing of undefined set/env vars via if-else-fi * Pass user environment settings settings in the login CGI * Use DNS names instead of addresses for kXR_locate when listing ------------- Version 3.3.6 ------------- + **Minor bug fixes** * Prevent SEGV when error occurs during stat (issue #70) * Prevent SEGV in redirect monitoring (issue #61) * Set reasonable linux thread limit and warn it we cannot do so. + **Miscellaneous** * Support for C++11 (narrowing fixes, unique_ptr vs. auto_ptr) * Support for CMake 2.8.12 (interface link libraries) ------------- Version 3.3.5 ------------- + **Minor bug fixes** * Fix minor Coverity issues in XrdCl * Fix a rarely occuring segfault when forking XrdCl under heavy load * Fix various issues related to group name retrieval (issues #51, #52, #53) + **Miscellaneous** * Make XrdSys/XrdSysIOEvents.hh private - could not have been used anyways * Add a sysconfig template to preload custom allocators in order to fix memory issues on RHEL6 * Allow up to 63 characters for a site name ------------- Version 3.3.4 ------------- + **Major bug fixes** * Serialize sss authentication client initialization to prevent race conditions * Actually cancel the JobManager threads while stopping it - this affected client side fork handling (new client) * Restore original meaning of -adler and -md5 to xrdcp (issue #44) + **Minor bug fixes** * Append CGI info when retrying at a server that handshaked but never respnded to the request (xrdcp) * Do socket accepts asynchronously to prevent DNS resolution from blocking accepts (issue #33) * Warn about incomplete dirlist responses (xrdfs) * Cast the utilization statistics to uint16_t before printing to print actual numbers instead of letters corresponding to ASCII codes (xrdfs) + **Miscellaneous** * When calling File::Stat use file handle instead of path * Improve handling of malformed kXR_readv responses (new client) * Explain parameters of xrdcopy --tpc (documentation, issue #46) ------------- Version 3.3.3 ------------- + **Major bug fixes** * Prevent SEGV's when reusing a recycled protocol object under certain conditions (xrootd server) * Prevent SEGV when using the -DS/-DI commandline parameters in xrdcp (issue #13) * Prevent integer overflow when calculating client recovery windows * Make sure the new client tries all available authentication protocols when connecting to a security enabled server (issue #14) * Detect buffer size mis-matches when server returned valid response with invalid size (xrdcopy) * Recognize /dev/null and /dev/zero as special files when using copy commands + **Minor bug fixes** * Prevent the new client deadlock on Solaris and MacOS when using the built-in poller and connecting to localhost (issue #5) * Compensate for ROOT garbage colletion issues when calling the new client code * Avoid favoring socket writes when using new client with the built-in poller * Strip off opaque information from dest filename when copying to local filesystem using xrdcp (issue #21) * Fix setting client timeout resolution while connecting to a server + **Miscellaneous** * Change the RPM package layout to match the one used by EPEL (issue #12) * Drop the daemon user RPMs * Allow new client connection parameters to be tweaked by connection URL CGI * Make the built-in poller default again in the new client - after resolving issue #5 ------------- Version 3.3.2 ------------- + **Major bug fixes** * Fix the opaque information setting in xrdcp using -OD (issue #1) * Fix compilation on Solaris 11 (issue #7) * Fix issues with semaphore locking during thread cancellation on MaxOSX (issue #10) * Solve locking problems in the built-in poller (issue #4) * Solve performance issues in the new client. Note: this actually changes some low level public interfaces, so the soname of libXrdCl.so has been bumped to libXrdCl.so.1. The xrootd.org RPMs also provide the old libXrdCl.so.0 in order to preserve the binary compatibility with the clients linked against it. ------------- Version 3.3.1 ------------- + **Major bug fixes** * Correct XrdClient ABI incompatibility issue introduced in 3.3.0 * Install additional private headers ------------- Version 3.3.0 ------------- + **New Features** * Stable interfaces immutable in minor releases (except XrdCl). Only public header files are installed in the usual include directory. In order to ease up transition of some clients some of the private include files are also installed in private subdirectory. * New asynchronous and thread-safe client libraries and executables (XrdCl). The ABI compatibility is not guaranteed until 4.0.0. * Build the xrootd protocol plugin as a shared library. * Add the altds directive to allow pairing a cmsd with an alternate data server. * Differentiate between packed and unpacked readv monitoring records. * Allow plugin libraries to be preloaded. This feature is only meant for MacOS. * Include optional site name in summary monitoring records. * Include optional site name in server identification record if the site name was specified on the command line (-S) or via config file (all.sitename directive). * Define a standard supported mechanism to obtain the default storage system object. * Provide an ABI-compatible interface to obtain a default cmsd client object. This patch does not change the definition of the XrdCmsClient object and is ABI compatible with all previous releases (DPM support). * Allow multiple comma separated protocols in XrdSecPROTOCOL client-side envar. This allows the client to select 1 of n protocols. * Implement new "f" stream monitoring. * Add new summary counters for readv and readv segs. * Add boiler plate comments indicating the all software is licensed under LGPL. No functional source code was modified by this patch. * Add GPL and LGPL license text. * Liberlize locking structure to prevent lock inversion relative to external locks. * Provide libevent replacement for Linux (epoll), Solaris (poll_create), and others (poll). Note: versions of Solaris less than 10 are no longer supported and they will no longer compile with this update! * Provide a libevent type replacement package. * Allow tracker files (e.g. ".fail") to be placed in a shadow directory. This is controlled by the new fdir option on the oss.xfr directive. * Allow meta-files (i.e. .fail file) to be relocated to a shadow directory using the oss.xfr directive. This avoids polluting the exported name space when an frm transfer operation fails. * Create a general place for platform dependent utility methods. * Add third party copy statistics to the summary record. * zlib compatible checksum plugin + **Major bug fixes** * Serialize access to cache entries to prevent SEGV's. * Fix the fast response queue so that it doesn't run out of response slots causing a big performance penalty. This is a high priority fix. * Properly disarm the mutex helper when the mustex object is deleted. * Use correct variable to hold osslib parameters. This patch fixes commit 2e27f87a (version checking) and without this patch makes it impossible to load an oss plug-in. * Properly check for errors when client read returns 0 and reflect true status. This only affects the Posix client interface. * Remove redundant flag indicating a running poller. This may cause the poller to never be woken up when a timeout value changes. * Fix tag in ofs statistics. It is improperly terminated and may cause certain xml parsers to fail; rendering monitoring useless. * Undo the side-effect of commit ff8bdbd6 that prevented the frm from sending stage notifications to xrootd; causing opens and xrdstagetool to hang with dynamic staging enabled. * Make sure the id buffer is large enough to hold all id combinations. * Avoid deadlock when closing a Posix File with an active preread. * For concurrent queries for the same file allow servers to respond to the query and only redirect clients to a stageable server if the file is not found. + **Minor bug fixes** * Add EPOLLRDHUP to avoid leaving sockets in CLOSE_WAIT with a one-shot poll framework. * Fully integrate checksum processing into a manager node. When configured, it does not matter whether a client directs a checksum request to a manager or a server. This also fixes bug report #93388. * Make sure to reflect proper range of errors during read/write operations. This also provides filesystem plugins full range of allowed return codes. * Initialize the rMon toggle to avoid valgrind complaint. * Fix minor issues reported by Coverity. * Make sure opendir() returns a null pointer when the directory doesn't exist. * Make sure that XrootdFS returns ENOENT when opendir() returns a null. * Make sure to use correct time to set mtime/atime after a physical reloc. * Prevent hangs when doing exterme copy from server to server. * Fix the -force option to really work for the mark subcommand. * Pass through error code returned by the N2N plug-in. This only affects the proxy server and caused feature interference. * Automatically exclude originating server/cluster on an enoent static redirect. * Correct typos XRDPSOIX envars should really be named XRDPOSIX. + **Miscellaneous** * Remove superfluous includes or other move includes to eliminate unnecessary dependencies in ".hh" files. This patch is required to create an EPEL conformable include directory. * Add port to prepare request struct as documented in 2.9.9. * Add pathid to readv request struct as documented in 2.9.9. ------------- Version 3.2.6 ------------- + **Major bug fixes** * GSI authentication: fix possible race condition while re-loading CA certificates; fix also related memory leaks. * GSI authentication: make sure the CA cache is not initialized twice (e.g. server and client inside there), and that the cache entry pointers are always initialized. * Crypto OpenSSL modules: use more appropriate way to read the RSA complete key, solving various issues for RH6 and derivations, included SL(C)6. * Make sure redirect opaque information is passed along for all filename based requests. This is required for DPM and EOS N2N services to work in all cases (most importantly, stat). * Make sure buffer ends with null byte before read suspension. This only occurs on very heavily loaded connections. * Fix the fast response queue so that it doesn't run out of response slots causing a big performance penalty. This is a high priority fix. + **Minor bug fixes** * Properly detect external process failure and report correct error status to a client. This also fixes bug report #91141. * [XRootDPosix] Make sure to use a supplied cache even when no cache directives given. * Make sure to return a usable path string via XrdOucCacheIO::Path(). * Actually support 4 different redirect destinations. + **Miscellaneous** * Transparent support for new name hashing algorithm adopted in openssl 1.0.0x (GSI authentication protocol) * Verbosity levels revised for GSI and PWD authentication protocols. * Notification of initialization option for GSI and PWD authentication protocols. * Do not repudiate file existence on an "cancelled" error during open. this patch addresses overloaded dCache pool nodes. ------------- Version 3.2.5 ------------- + **Major bug fixes** * Make realoading gridmapfile atomic (protect from segfault) * Propagate to clients proper range of errors during read/write operations * Fix segfault when handling writes to files that have not been opened ------------- Version 3.2.4 ------------- + **Major bug fixes** * Work around a dead-lock in the client fork handlers. ------------- Version 3.2.3 ------------- + **Major bug fixes** * Make sure read statistics are updated for sendfile() and mmap I/O. * Make sure refresh thread is dead before deleting deleting the keytab to avoid SEGV's. * Add missing include for compiling with gcc-4.7 (from Sebastien Binet). This patch is required for successful compilation. * Avoid segfaults when limiting number of redirections caused by failed authorization. * Avoid deadlock in the client fork handlers. + **Minor bug fixes** * Correct monitor initialization test to start monitor under all configs. * Fix a memory leak in the client handshake algorithm. + **Miscellaneous** * Make RHEL6-created SRPMs buildable on RHEL5 by forcing RPM to use MD5 digests. * Fuse: Use default TTL values for data server connection and load balance server connection. ------------- Version 3.2.2 ------------- + **Major bug fixes** * Correct test whether or not to initialize redirect monitoring. The old code never initialized it this disabling redirect monitoring. * Backport frm notification fix that stalled stage-in requests from commit 69e38cfd6b8bb024dd34f8eb28a666fbf97f346b * Prevent SEGV when xrd.monitor rbuff value not specified * Prevent xrdcp hangs when doing exterme copy from server to server. * In case of 'limited proxy' look for VOMS attributes also in the parent proxy. * Correct log processing for sites that use the root directory as the stomping ground for newly created files. ------------- Version 3.2.1 ------------- + **Major bug fixes** * Don't build sendfile support on MacOSX because it doesn't work * Prevent double-free abort when more than 16 files have been opened by a client and the client terminates the session without closing the 17th one. ------------- Version 3.2.0 ------------- + **New Features** * Retool the XrdOucCache object so that cache implementations can be implemented as plugins. * Add FSize method to the XrdOucCacheIO object to ease implementation of disk caches containing partial files. * Add the pss.cachelib directive to specify a cache plugin. * Implement ultralow overhead redirect monitoring. WARNING: ofs plugin writers will need to recompile their plugin interface to be fully compatible with this commit due to additional information passed to the ofs object "new" methods. * Allow the XrdCmsClient interface (a.k.a Finder) to be a plug-in. * Add ofs.cmslib directive to specify the XrdCmsClient plug-in. * Add new class, XrdOucCallBack, to simplify using callbacks in the XrdCmsClient plug-in. * Define the frm.all.monitor directive to enable migration, purging, and staging monitoring. This was originally part of xrootd.monitor but that just was odd. Note that the stage, purge, migr events are no longer accepted on the xrootd.monitor directive. * Collapse he staging (s) and migration (m) records into a single transfer (x) record. While not compatible, the previous implementation was new code and no one actually was capturing these records. * Implement a server identification record (=) that unquely identifies each server. The record can be sent periodically and can be used as a heartbeat. * Add -y option to xrdcp to limit number of extreme copy sources. * Uniformly pass the execution environment to all oss and cms client methods. This is largely for DPM support. WARNING: While this update is binary backwad compatible to existing oss plug-ins it is not source compatible. Plug-in writers will need to modify their oss methods to successfully compile. * Allow an automatic redirect when a file operation ends with ENOENT. Allow redirects for chsum and trunc operations. Both of the above are controlled via the xrootd.redirect directive. * Report the timezone when connecting to a [meta]manager. * Allow configuration of staging, migration, and purging events. * Allow transfer script to inject information into the monitoring stream. * Report number of attempted login, authentication failures, successful authenticated and unauthenticated logins in the summary statistics. * Indicate whether a disconnect was forced and whether it was a parallel path (as opposed to a control path) in the monitoring record. + **Major bug fixes** * Provide compatibility for sprintf() implementations that check output buffer length. This currently only affects gentoo and Ubuntu Linux. We place it in the "major" section as it causes run-time errors there. * Reinsert buffer size calculation that was mistakenly deleted. This eventually causes a SEGV when detailed monitoring is enabled. * Remove improper initialization that may cause a SEGV in the checksum manager. * Add missing initializer without which we will get a SEGV. This is a fix for the just added monitoring code. * Remove regressions that prevent a proxy cluster from being fully configured. + **Minor bug fixes** * Correct debug message frequency that caused people to think some file system partitions were being ignored. * Correct pthread Num() to return thread-specific numbers. * Make sure the sendfile interrupt counter is initialized to zero. * Make sure to honor absolute cms.space values when percentage not specified. * Prevent double user map record when monitoring when auth is configured but not actually monitored. * Take timezone changes into account when waiting for midnight. This solves the log rolling problem when changing between DST and standard time. * Make sure to cut close records for open files during a forced disconnect when monitoring file information. * Do not create meta-files or update extended attributes when placing a file into read-only space. + **Miscellaneous** * Bonjour code dropped * Complete implementation of the fstat() version of stat(). * Consistently pass the enviroment to the cms client enterface. * Make return codes consistent between synchronous & async XrdCmsClient returns. * Document the XrdCmsClient interface in the header file. * Cut close monitor records before cutting the disconnect record. * Make frm_purged and frm_xfrd use sparate log files. ------------- Version 3.1.1 ------------- + **New Features** * Compile on Solaris 11 * Add support for sending DN with monitoring information * Add possibility to switch off automatic download of CRL from the web; default is OFF; to enable it multiply by 10 the relevant CRL options (i.e. 12 and 13 are like 2 and 3 but trying download if the file is not found). * Add refresh frequency time for CRL's; default 1 day . + **Major bug fixes** * Fix various client threading issues. * [bug #87880] Properly unpack the incoming vector read data. * Rework the handshake when making a parallel connection. Previous method caused a deadlock when parallel connections were requested (e.g. xrdcp). * Add HAVE_SENDFILE definition to cmake config. All post-cmake version of xrootd until now have disabled use of sendfile() with resulting poor performance. This fix corrects this. * Don't force libXrdPss.so to be loaded for proxy managers. * Fix various CMake issues: disable library inheritance, fix underlinking problems, make sure libcom_err is present when building kerberos. * Replace non-reentrant versions of getpwxxx and getgrxxx with reentrant versions. This should prevent spurious uid/gid translations. * Fix RedHat bug #673069: Missing header files required by DPM * Don't ignore errors returned by kXR_close * Init scripts: don't change the ownership of the sysconfig files preventing the xrootd user from executing arbitrary code as root + **Minor bug fixes** * Add 'k' to the option list. It was wrongly deleted in the last option refalgamization. * Fix a typo in the specfile causing problems with multithreaded compilation. * Initialize xattr variable name so that xrdadler32 can fetch previous checksum. The error caused xrdadler32 to always recompute the checksum. * Make sure that monitor write length is really negative. * Add the oss.asize hint to the destination URL in all possible cases. * Properly print adler32 checksum in xrdcp. * When the server certificate is expired, try to renew from the same path before failing. * Get the signing certificate for the CRL from its issuer hash, which can be different from the CA hash. * Add check for the format of the downloaded CRLs: DER or PEM * Solaris init script: switch to xrootd user when invoked as root * RHEL init scripts: always create /var/run/xrootd to handle /var/run being mounted as tmpfs + **Miscellaneous** * Relax requirements on the permission mode of the x509 key files * Disable client redirections reports to the console. * Stop doing XrdFfsPosix_statall() if task queue is long. * Get rid of compiler warnings * Improve some log messages * At server startup, only initialize the CA (and CRL, if required) for the authority issuing the server certificate; additional CA's are initialized only if needed. ------------- Version 3.1.0 ------------- + **New Features** * Use CMake to build the source code and retire all the other build systems. * Add IOV as a selectable detail to xrootd.monitor directive. * Provide a mode in xrootdfs to auto-update internal list of data servers. and extend client connection TTL from one hour to infinity. * Provide virtual xattr ("xroot.cksum") to obtain checksum for consistency. * Make xrdadler32 use the new checksum format if it is set (fallback to old format otherwise). In all cases, the old format is converted to the new format whenever possible. * Enforce r/o exports in the proxy server (finally added). * Allow auto-fluching of I/O stream monitoring (default is off). Patch submitted by Matevz Tadel, UCSD. * Make proxy honor the export list at the storage layer. This allows sites to disable staging via the proxy by specifying nostage for otherwise locally stageable paths. * Do not export the stage attribute to the meta-manager unless the path is tagged with the stage+ attrbute on the export directive. * WARNING: This update makes the oss plug-in source incompatible because an additional parameter was added to the Stat() method. The update is binary compatible and so only affects sites that recompile their plug-in. * Allow the query checksum request to be issued via a proxy server. * Add a query checksum interface to the POSIX interface. * Defines the livXrdSecgsiAuthzVO plug-in to allow easy mapping from voms vo names to users and groups. The plugin is configurable at run-time. * Allow the OucErrInfo object to point to an environment. * Add method to SysDNS to format an AF_INETx address into the RFC IPV6 recommended format. * Allow pointers to be placed in the OucEnv environment table. * Extend the kXR_protocol request to allow the server to return detailed information about node's role. This is backwardly compatible. * The client uses kXR_protocol request to query for the server's role (to distinguish managers from meta managers). * The client goes back to a meta manager on authentication failure. * The client prints to stdout the redirections it gets. This behavior may be disabled by setting the XRD_PRINTREDIRECTS environment variable to 0, or, from C++ by saying: EnvPutInt( NAME_PRINT_REDIRECTS, 0 ) * Set $HOST value for possible copycmd substitution. * Phase 1 to allow for redirection monitoring. Add rbuff and redir options to the xrootd.monitor directive. * Add error, redirect, and delay counts to the xrootd protocol summary statistics. * Allow file additions/deletion to be communicated to the XrdCnsd so that is can maintain an accurate inventory. This update adds the frm.all.cnsd directive which specifies how the information is to be commuincated. * Enable cmsd monitoring. For now, only [meta]manager information is reported. * Add new repstats config directive to increase reporting detail. * New class, XrdCmsRole, to make role naming/handling consistent. * Implement the 'cms.delay qdn' directive which allows one to tell the meta-manager the minimum number of responses needed to satisfy a hold delay (i.e. fast redirect). * Accept XrdSecSSSKT envar as documented but also continue to support XrdSecsssKT for backward compatibility. * Allow servers to specify to the meta-manager what share of requests they are willing to handle. Add the 'cms.sched gsdflt' and 'cms.sched gshr' configuration directives to specify this. * Include additional information in the protocol statistics. * Resize some counters to prevent overflows. * Add the 'cms.delay qdn' directive to allow better redirection control in the future. * Allow a plugin (notably the proxy plugin) to disable async I/O. * Implement a general memory caching object. Currently, this will be used by the Posix object. * Allow optional memory caching when using the Posix library. This is primarily used by the proxy object to reduce trips to a data server when small blocks are accessed via the proxy server. This requires configuration using the new 'pss.memcache' directive. * Finally implement adding authentication information to the user monitoring record (requested by Matevz Tadel, CMS). This adds a new generic option, auth, to the xrootd.monitor directive. It needs to be specified for the authentication information to be added. This keeps backward compatibility. * Add a new method, chksum, to the standard filesystem interface. * Integrate checksums into the logical filesystem layer implementation. See the ofs.ckslib directive on how to do non-default configuration. This also added a more effecient lfn2pfn() method to the storage system. * Allow native checksums to be enabled in the xrootd layer. See the xrootd.chksum directive on how to do this. * Add checksum management to the frm_admin command. * Allow XrdOucProg to dispatch a local program as well as a process. * Allow a line to be insrerted into an XrdOucStream managed stream. * Implement native checksums usable stand-alone or as plugins. Three digests are supported: adler32, crc32, and md5. An additional digest can be added via a plugin. Also, the native digests can be over-ridden via a plugin. * In XrdSecgsi, new interface for the authorization plug-in which has now full access to the XrdSecEntity object, with the possibility to fill/modify all the fields according to the proxy chain. The plug-in is now called at the end of the all process, after a successful handshake and DN-username mapping. Implementations must contain three extern C functions; see the dummy example provided in src/XrdSecgsi/XrdSecgsiAuthzFunDN.cc. See also the header of XrdSecProtocolgsi::LoadAuthzFun. * In XrdCryptosslgsiAux, add function to extract the VOMS attributes; can be used in authz plug-ins. * In XrdSecgsi, add possibility to extract the VOMS attributes and save them in the XrdSecEntity. New switch '-vomsat:0/1 [1]'. * In 'xrdgsiproxy info' show also the VOMS attributes, if present. * Automatically build the RPM for the xrootd user when an OSG build is detected and add fedora > 15 init scripts dependencies + **Major bug fixes** * Do not close the loger's shadow file descriptor when backgrounding as this may cause random crashes later on. * Avoid SEGV by setting network pointer prior to loading the 1st protocol. * Enforce r/o path during mkdir operations. * Avoid segv when initializing the finder on a multi-core machine. * Fix incorrect lock handling for multiple waiters. * Fix possible deadlocks in XrdSutCache preventing the pwd security module to work correctly + **Minor bug fixes** * Properly handle the case when a site has an excessive number of groups assignments. * Prevent the response to a query from being truncated on the client side. * Report readv information in the detailed monitoring stream. * Correct default settings due to feature interactions after the fact. Now, oss.defaults acts as if the setting were actually specified via oss.export. * Actually use the N2N library of specified or implied via pss.localroot for proxy server interactions withthe origin (required for Atlas T2). * Use re-enterant versions of getpwuid() and getpwgid(). This is need for FUSE. * Correct bad english in a few error messages. * Set correct checksum length when converting ASCII to binary. * Allow the sss protocol to work for multi-homed hosts. * Correct definition of AtomicISM that caused the maximum link count to never be updated in the statistics. * Apply N2N mapping to source path when relocating the file. * Report correct port when locate is directly issued to a data server (before it was being reported as 0). * Make the default file system a pointer to a dynamic instance of XrdOfs instead of a global static (i.e. the Andreas Peters patch). This makes writing an ofs plugin easier. * Fix the RPM uninstall scriptlets incorrectly invoking /sbin/ldconfig. * Install XrdOlbMonPerf and netchk tools. * Fix a bug preventing the core of authentication errors to be logged to clients * In the krb5 security plugin, define KRB5CCNAME to point to the credential cache file /tmp/krb5cc_ only if this file exists and is readable. Solves an issue with credentials cached in memory (API::n). * Fix array deletion mismatches reported by cppcheck (from D. Volgyes) * Make sure that loading of XrdSecgsi.so fails if either the GMAPFun or the AuthzFun plug-ins fail to load. + **Miscellaneous** * Drop Windows support. * Code cleanup: remove XrdTokenAuthzOfs, simple tests, broken utilities, the gridftp code, krb4 and secssl plugins, obsolete documentation files * Make the loadable module extensions configurable depending on the platform (so on Linux and Solaris, dylib on MacOs) * Add new XrdVNUMBER macro. * Clean up the conditional compilation macros. * Remove compression related attributes (compchk, ssdec) and directives (compdetect) as they were never used nor fully implemented. * Remove the userprty directive. It was deprecated and never specified. * Refactor PosixPreeload and Posix libraries to prevent split initialization of the preload library which will cause failures on certain systems. * Provide automatic proxy checksum defaults when role is set to proxy. * Remove all references via extern statements to object instances. This only applies to the Xrd package. * Do not echo lines qualified by an in-line if when the if fails. * Remove the old "redirect" directive. It has passed its prime. * Remove back references to symbols defined in XrdXrootd package used by the cms client to allow for clean shared library builds. * Remove externs to XrdSecGetProtocol and XrdSecGetService from XrdSecInterface.hh to avoid having undefined references just because the include file was included somewhere. * Rename XrdNetDNS to XrdSysDNS to avoid cross-dependencies. This means that all plug-in developers will need to do the same as XrdNetDNS no longer exists. * Split XrdFrm into XrdFrm and XrdFrc. This prevents cross-dependencies in packages that use the File Residency Manager. ------------- Version 3.0.5 ------------- + **Major bug fixes** * Avoid stage failures when target file exists in purgeable or writable space. * Make sure all the threads are joined when closing a physical connection. * Fix free/delete mismatch in XrdSecProtocolgsi et al. + **Minor bug fixes** * Remove old async shutdown workaround patch introduced in Linux 2.3. The problem has been since fixed and the solution now causes problems. * Install the netchk tool ------------- Version 3.0.4 ------------- + **New features** * xrdcp now has -version parameter * xrdcp automatically ads the oss.asize hint to the url opaque data. This functionality may be disabled by setting the XrdCpSizeHint variable to 0 (XRD_XRDCPSIZEHIN in the shell). * The client will try to resolve the server hostname on every retry to enable DNS failovers. * RPM: devel package split into libs-devel, client-devel and server-devel * XrootdFS: all paramenters can be passed via command line, add -h. * Allow a plugin (notably the proxy plugin) to disable async I/O. * New class XrdSysRWLock interfacing the pthread_rwlock functionality * In XrdSecEntity: Add new fields 'creds' and 'credslen' to be filled with the raw client credentials * In XrdSutCache: Use XrdSysRWLock to coordinate concurrent access to the cache * In XrdSecgsi: - Add option to have Entity.name filled with the plain DN, instead of the DN hash, when no mapping is requested or found. - Enable cache also for authz mapping results. - Do not require the existence of a grid-mapfile if gmapopt=2 and there is at least a gmapfun or an authzfun defined. - Add example of mapping function allowing to match parts of the DN - Extend existing option 'authzpxy' to allow exporting the incoming client credentials in XrdSecEntity. + **Major bug fixes** * Async write errors are now being properly caught and reacted to. XrdClient::Close will now fail if it cannot recover from async write errors. * xrdcp prints an error message and returns failure to the shell when some of the write requests it issues fail. * libXrdPosixPreload now builds with autotools and is included into the xrootd-client RPM * RPM: FFS moved from libs to client * Properly parse oss.asize. This because a major problem when xrdcp started adding this to the url which causes the copy to fail. * Spin connection portion of proxy initialization to a background thread. This prevents init.d hangs when a redirector is not available. + **Minor bug fixes** * Test for 64-bit atomics instead 32-bit ones. Fixes build on 32-bit PowerPC. * RPM: xrootd-fuse now depends on fuse * Take correctly into accoutn summer time in calculating the time left for a proxy validity * Add support for Ubuntu 11 which uses the directory /usr/lib/`dpkg-architecture -qDEB_HOST_MULTIARCH` to store platform dependent libraries. ------------- Version 3.0.3 ------------- + **New features** * Change configure.classic to handle various versions of processors in a more sane way (this fixes several Solaris issues and atomics for i686). * Add fwdwait option to cms.request directive to allow pacing of forwarded requests (off by default). * Use native memory synchronization primitives when available when doing network I/O. This will eventually be extended to cover all other cases. * Add the qdl option to the cms.delay directive to allow changing the query window independently of the time a client is asked to wait for the query to actually complete. * Add 'pss.namelib' directive to allow proxies to pre-translate the lfn for servers that cannot do so (e.g., dCache xrootd door). * Optimize handling of shared-everything ile systems (e.g., dCache, GPFS, DFS, Lustre, etc.) in the cmsd. * Implement optional throttling for meta-manager requests in the cmsd. * New cmsd directive, cms.dfs, declares that the underlying file system is a shared-everything system (i.e., distributed file system) and allow for optimal configuration and meta-manager throttling. * Change the oss and fm components to use file extended attributes instead of meta-files. This affects copy, create, reloc, rename, and unlink in the oss layer. Migrate, purge, transfer, and most admin commands in the frm component. The 'all.export' directive now accepts the noxattr/xattr option. WARNING: If the migrator or purge options have been specified for any path in the 'all.export; directive then this change requires either the the 'oss.runmodeold' directive be added to the configuration file to provide backward compatibility or that the name and data spaces be migrated using the frm_admin command. See "Migrating tp Extended Attributes" manual for detailed information and the new 'frm_admin convert' subcommand. * Avoid physical copy if the operation can be handled using hard links. This greatly speeds up static space token reassignment. * Add platform independent interface to extended file attributes. * RPM packaging and Red Hat Enterprise Linux compatible init scripts capable of handling multiple instances of the xrootd daemons. The instances can be defined in the /etc/sysconfig/xrootd file and then handled using standard:: service xrootd start|stop|... service cmsd start|stop|... ... or handled by name:: service xrootd start instance1 instance5 * New '-s' commandline option for xrootd, cmsd, frm_purged and frm_xfrd creating a pidfile. * xrootd, cmsd, frm_purged and frm_xfrd now return failure to the shell when called with '-b' option (daemonization) and the daemon fails to initialize. * New 'EnableTCPKeepAlive' client environment option added enabling the TCP stack keep-alive functionality for the sockets. On Linux three addtional fine-tunning options are available: - TCPKeepAliveTime - interval (in seconds) between the last data packet and the first keep-alive probe - TCPKeepAliveInterval - interval (in seconds) between the probes - TCPKeepAliveProbes - number of probes lost to consider the connection broken * New functionality handling process forking. When enabled (via the 'EnableForkHandlers' env option) prior to a call to fork it shuts down all the xrootd connection management facilities (including the connections themselves) and reinitializes them after the fork both in the parent and the child process. This ensures relative fork safety provided that all the XrdClient and XrdAdmin instances are closed when the fork function is invoked. + **Major bug fixes** * Add missing braces that caused config failure in frm_admin command. * Account for correct path when -M value is zero in hpsscp command. * In XrdCryptossl, fix for thread-safeness; solves random crashes observed on the server side under high authentication frequency * In XrdOucBonjour, fix important issue with host domain name registration, preventing the correct domain to be posted. + **Minor bug fixes** * Correct file discovery propogation for proxy manager relative to meta-managers. * Correct oss partition selection algorithm to further spread out file allocation. * Allow underscores in set/setenv variables. * Add null byte after checksum value response. * Move mapping of errno to xrootd error code to the protocol package where it belongs. This also removes a cross dependency. * Correct RetToken() behaviour in the presence of multiple spaces between tokens and the previous call returned the remainder of the line (very obscure circumstances). * [bug #77535] xrdcp now returns an error to the shell when it fails to copy the file * [bug #79710] xrdcp now gracefully aborts when it encounters a corrupted local file * Reset the transaction timeout for the Query method. This fixes transaction timeout issues for clients doing only queries. * Rename variable to remove conflict between it and global of the same name. * Fix frm_admin command line option parsing so it does not trip over subcommand options. This also fixes a SEGV in MacOS when this actually happens. * Enable the '-md5' option when OpenSSL is present and xrootd is built with autotools. + **Documentation** * Added man pages for: xprep, xrd, xrdcp, xrdstagetool, xrdgsiproxy ------------- Version 3.0.2 ------------- + **Minor bug fixes** * Fix the build on Solaris 10. * Fix the build on SLC4. * Fix the out-of-the-source-tree builds with autotools. * Fix a segfault while doing a recursive copy from root:// to root://. ------------- Version 3.0.1 ------------- + **New features** * New application, cconfing, added to display configuration files relative to a host-program-instance. * New application, netchk, that tests that firewalls have been correctly setup. * New configure.classic option to allow use of stl4port library for Solaris. * New internal feature in XrdPosix library to not shadow files with actual file descriptors (used by the proxy service). This increases scalability. * Allow the xrootd server to tell the client that it is a meta-manager. * Support fo proxies generated by Globus version 4.2.1 in libXrdSecssl. + **Major bug fixes** * Change link options for xrdadler32 to not use shared libraries. The previous setup caused the command to hang upon exit. * Remove instance of XrdPosixXrootd from that same file. Including it disallows defaults from being changed. + **Minor bug fixes** * Fix XrdOucStream to not return ending "fi". * Correct network option interference -- do not turn on network nodnr option should the keepalive option be specified. * Remove duplicate option in option table used by the proxy service. * Compile on Solaris 11 Express using SunCC. * Compile on Windows using MSVC++2010. xrootd-5.6.9/docs/TESTING.md000066400000000000000000000415421457266313600154640ustar00rootroot00000000000000## Configuring and Running XRootD tests with CTest XRootD tests are divided into two main categories: unit and integration tests that can be run directly with CTest, and containerized tests that are required to be run from within a container built with docker or podman. This document describes how to run the former, that is, the tests that are run just with CTest. This document assumes you are already familiar with how to build XRootD from source. If you need instructions on how to do that, please see the [INSTALL.md](INSTALL.md) file. There you will also find a full list of optional features and which dependencies are required to enable them. ### Enabling tests during CMake configuration XRootD unit and integration tests are enabled via the CMake configuration option `-DENABLE_TESTS=ON`. Unit and integration tests may depend on CppUnit or GoogleTest (a migration from CppUnit to GoogleTest is ongoing). Therefore, the development packages for CppUnit and GoogleTest (i.e. `cppunit-devel` and `gtest-devel` on RPM-based distributions) are needed in order to enable all available tests. Here we discuss how to use the [test.cmake](../test.cmake) CTest script to run all steps to configure and build the project, then run all tests using CTest. The script [test.cmake](../test.cmake) can be generically called from the top directory of the repository as shown below ```sh xrootd $ ctest -V -S test.cmake -- Using CMake cache file config.cmake Run dashboard with model Experimental Source directory: xrootd Build directory: xrootd/build Reading ctest configuration file: xrootd/CTestConfig.cmake Site: example.cern.ch (Linux - x86_64) Build name: Linux GCC 12.3.1 RelWithDebInfo Use Experimental tag: 20230622-0712 Updating the repository: xrootd Use GIT repository type Old revision of repository is: 6fce466a5f9b369f45ef2592c2ae246de1f13103 New revision of repository is: 6fce466a5f9b369f45ef2592c2ae246de1f13103 Gathering version information (one . per revision): Configure project Each . represents 1024 bytes of output ..... Size of output: 4K Build project Each symbol represents 1024 bytes of output. '!' represents an error and '*' a warning. .................................................. Size: 49K .. Size of output: 52K 0 Compiler errors 0 Compiler warnings Test project xrootd/build Start 1: XrdCl::URLTest.LocalURLs 1/23 Test #1: XrdCl::URLTest.LocalURLs ..................... Passed 0.01 sec Start 2: XrdCl::URLTest.RemoteURLs 2/23 Test #2: XrdCl::URLTest.RemoteURLs .................... Passed 0.12 sec Start 3: XrdCl::URLTest.InvalidURLs 3/23 Test #3: XrdCl::URLTest.InvalidURLs ................... Passed 0.01 sec Start 4: XrdHttpTests.checksumHandlerTests 4/23 Test #4: XrdHttpTests.checksumHandlerTests ............ Passed 0.01 sec Start 5: XrdHttpTests.checksumHandlerSelectionTest 5/23 Test #5: XrdHttpTests.checksumHandlerSelectionTest .... Passed 0.01 sec Start 6: XrdCl::Poller 6/23 Test #6: XrdCl::Poller ................................ Passed 5.01 sec Start 7: XrdCl::Socket 7/23 Test #7: XrdCl::Socket ................................ Passed 0.02 sec Start 8: XrdCl::Utils 8/23 Test #8: XrdCl::Utils ................................. Passed 8.01 sec Start 9: XrdEc::AlignedWriteTest 9/23 Test #9: XrdEc::AlignedWriteTest ...................... Passed 0.06 sec Start 10: XrdEc::SmallWriteTest 10/23 Test #10: XrdEc::SmallWriteTest ........................ Passed 0.06 sec Start 11: XrdEc::BigWriteTest 11/23 Test #11: XrdEc::BigWriteTest .......................... Passed 0.05 sec Start 12: XrdEc::VectorReadTest 12/23 Test #12: XrdEc::VectorReadTest ........................ Passed 0.06 sec Start 13: XrdEc::IllegalVectorReadTest 13/23 Test #13: XrdEc::IllegalVectorReadTest ................. Passed 0.06 sec Start 14: XrdEc::AlignedWrite1MissingTest 14/23 Test #14: XrdEc::AlignedWrite1MissingTest .............. Passed 0.06 sec Start 15: XrdEc::AlignedWrite2MissingTest 15/23 Test #15: XrdEc::AlignedWrite2MissingTest .............. Passed 0.05 sec Start 16: XrdEc::AlignedWriteTestIsalCrcNoMt 16/23 Test #16: XrdEc::AlignedWriteTestIsalCrcNoMt ........... Passed 0.06 sec Start 17: XrdEc::SmallWriteTestIsalCrcNoMt 17/23 Test #17: XrdEc::SmallWriteTestIsalCrcNoMt ............. Passed 0.06 sec Start 18: XrdEc::BigWriteTestIsalCrcNoMt 18/23 Test #18: XrdEc::BigWriteTestIsalCrcNoMt ............... Passed 0.06 sec Start 19: XrdEc::AlignedWrite1MissingTestIsalCrcNoMt 19/23 Test #19: XrdEc::AlignedWrite1MissingTestIsalCrcNoMt ... Passed 0.06 sec Start 20: XrdEc::AlignedWrite2MissingTestIsalCrcNoMt 20/23 Test #20: XrdEc::AlignedWrite2MissingTestIsalCrcNoMt ... Passed 0.06 sec Start 21: XRootD::start 21/23 Test #21: XRootD::start ................................ Passed 0.01 sec Start 23: XRootD::smoke-test 22/23 Test #23: XRootD::smoke-test ........................... Passed 1.63 sec Start 22: XRootD::stop 23/23 Test #22: XRootD::stop ................................. Passed 1.00 sec 100% tests passed, 0 tests failed out of 23 Total Test time (real) = 16.55 sec ``` For full verbose output, use `-VV` instead of `-V`. We recommend using at least `-V` to add some verbosity. The output is too terse to be useful otherwise. ### Customizing the Build #### Selecting a build type, compile flags, optional features, etc Since the script is targeted for usage with continuous integration, it tries to load a configuration file from the `.ci` subdirectory in the source directory. The default configuration is in the `config.cmake` file. This file is used to pre-load the CMake cache. If found, it is passed to CMake during configuration via the `-C` option. This file is a CMake script that should only contain CMake `set()` commands using the `CACHE` option to populate the cache. Some effort is made to detect and use a more specific configuration file than the generic `config.cmake` that is used by default. For example, on Ubuntu, a file named `ubuntu.cmake` will be used if present. The script also tries to detect the version of the OS and use a more specific file if found for that version. For example, on Alma Linux 8, one could use `almalinux8.cmake` which would have higher precedence than `almalinux.cmake`. The default `config.cmake` file will enable as many options as possible without failing if the dependencies are not installed, so it should be sufficient in most cases. The behavior of the [test.cmake](../test.cmake) script can also be influenced by environment variables like `CC`, `CXX`, `CXXFLAGS`, `CMAKE_ARGS`, `CMAKE_GENERATOR`, `CMAKE_BUILD_PARALLEL_LEVEL`, `CTEST_PARALLEL_LEVEL`, and `CTEST_CONFIGURATION_TYPE`. These are mostly self-explanatory and can be used to override the provided defaults. For example, to build with `clang` and use `ninja` as CMake generator, one can run: ```sh xrootd $ env CC=clang CXX=clang++ CMAKE_GENERATOR=Ninja ctest -V -S test.cmake ``` For performance analysis and profiling with `perf`, we recommend building with ```sh xrootd $ CXXFLAGS='-fno-omit-frame-pointer' ctest -V -C RelWithDebInfo -S test.cmake ``` For enabling link-time optimizations (LTO), we recommend using ``` CXXFLAGS='-flto -Werror=odr -Werror=lto-type-mismatch -Werror=strict-aliasing' ``` This turns some important warnings into errors to avoid potential runtime issues with LTO. Please see GCC's manual page for descriptions of each of the warnings above. XRootD also support using address and thread sanitizers, via the options `-DENABLE_ASAN=ON` and `-DENABLE_TSAN=ON`, respectively. These should be enabled using `CMAKE_ARGS`, as shown below ```sh $ env CMAKE_ARGS="-DENABLE_TSAN=1" ctest -V -S test.cmake ``` Note that options passed by setting `CMAKE_ARGS` in the environment have higher precedence than what is in the pre-loaded cache file, so this method can be used to override the defaults without having to edit the pre-loaded cache file. #### Enabling coverage, memory checking, and static analysis The [test.cmake](../test.cmake) has are several options that allow the developer to customize the build being tested. The main options are shown in the table below: | Option | Description | |:------------------------:|:-------------------------------------------| | **-DCOVERAGE=ON** | Enables test coverage analysis with gcov | | **-DMEMCHECK=ON** | Enables memory checking with valgrind | | **-DSTATIC_ANALYSIS=ON** | Enables static analysis with clang-tidy | | **-DINSTALL=ON** | Enables an extra step to call make install | When enabling coverage, a report is generated by default in the `html/` directory inside the build directory. The results can then be viewed by opening the file `html/coverage_details.html` in a web browser. The report generation step can be disabled by passing the option `-DGCOVR=0` to the script. If `gcovr` is not found, the step will be skipped automatically. It is recommended to use a debug build to generate the coverage analysis. The CMake build type can be specified directly on the command line with the `-C` option of `ctest`. For example, to run a coverage build in debug mode, showing test output when a test failure happens, one can run: ```sh xrootd $ ctest -V --output-on-failure -C Debug -DCOVERAGE=ON -S test.cmake ``` Memory checking is performed with `valgrind` when it is enabled. In this case, the tests are run twice, once as usual, and once with `valgrind`. The output logs from running the tests with `valgrind` can be found in the build directory at `build/Testing/Temporary/MemoryChecker.<#>.log` where `<#>` corresponds to the test number as shown when running `ctest`. Static analysis requires `clang-tidy` and is enabled by setting `CMAKE_CXX_CLANG_TIDY` for the build. If `clang-tidy` is not in the standard `PATH`, then it may be necessary to set it manually instead of using the option `-DSTATIC_ANALYSIS=ON`. For the moment XRootD does not provide yet its own configuration file for `clang-tidy`, so the defaults will be used for the build. Warnings and errors coming from static analysis will be shown as part of the regular build, so it is important to enable full verbosity when enabling static analysis to be able to see the output from `clang-tidy`. The option `-DINSTALL=ON` will enable a step to perform a so-called staged installation inside the build directory. It can be used to check if the installation procedure is working as expected, by inspecting the contents of the `install/` directory inside the build directory after installation: ```sh xrootd $ ctest -DINSTALL=1 -S test.cmake Each . represents 1024 bytes of output ..... Size of output: 4K Each symbol represents 1024 bytes of output. '!' represents an error and '*' a warning. .................................................. Size: 49K .. Size of output: 52K Each symbol represents 1024 bytes of output. '!' represents an error and '*' a warning. ............................................*!.... Size: 49K . Size of output: 50K Error(s) when building project xrootd $ tree -L 3 -d build/install build/install └── usr ├── bin ├── include │   └── xrootd ├── lib │   └── python3.11 ├── lib64 └── share ├── man └── xrootd 11 directories ``` Note that, as shown above, `CTest` erroneously shows build errors when installing XRootD with this command. This is because of a deprecation warning emitted by `pip` while installing the Python bindings and can be safely ignored. ### Dependencies Required for Coverage, Memory Checking, and Static Analysis #### RPM-based distributions: RedHat, Fedora, CentOS, Alma, Rocky The [test.cmake](../test.cmake) script may also need some extra dependencies for some of its features. For memory checking, `valgrind` is needed, and for static analysis, `clang-tidy` is needed: ```sh dnf install \ clang \ clang-tools-extra \ valgrind ``` For coverage, you need to install `gcovr`. It is not available via `yum`/`dnf`, but can be installed with `pip`. See https://gcovr.com/en/stable/installation.html for more information. Dependencies to run containerized tests with `podman` on RHEL 8/9 and derivatives can be installed with `dnf groupinstall 'Container Management'`. On CentOS 7 and RHEL 7, one can use `docker` by installing it with `yum install docker`. In this case, you will need to ensure that your user is in the `docker` group so that you can run docker without using `sudo`, and that the system daemon for Docker is running. A quick test to check if everything is correctly setup is to try to start a busybox image: `docker run --rm -it busybox`. #### DEB-based distributions: Debian 11, Ubuntu 22.04 On Debian, Ubuntu, and derivatives, The extra dependencies to use with the [test.cmake](../test.cmake) script can be installed with: ```sh apt install clang clang-tidy valgrind gcovr ``` Dependencies to run containerized tests can be installed with ```sh apt install podman ``` ## Running XRootD Tests on other platforms with Docker and/or Podman If you would like to run XRootD tests on other platforms, you can use the `xrd-docker` script and associated `Dockerfile`s in the `docker/` subdirectory. The steps needed are described below. ### Create an XRootD tarball to build in the container The first thing that needs to be done is packaging a tarball with the version of XRootD to be used to build in the container image. The command `xrd-docker package` by default creates a tarball named `xrootd.tar.gz` in the current directory using the `HEAD` of the currently checked branch. We recommend changing directory to the `docker/` directory in the XRootD git repository in order to run these commands. Suppose we would like to run the tests for release v5.6.4. Then, we would run ```sh $ xrd-docker package v5.6.4 ``` to create the tarball that will be used to build the container image. The tarball created by this command is a standard tarball created with `git archive`. Inside it, the `VERSION` file contains the expanded version which is used by the new spec file to detect the version of XRootD being built. You can also create a source RPM with such tarballs, but they must be built with `rpmbuild --with git` as done in the CI builds and the `Dockerfile`s in the `docker/build/` subdirectory. ### Build the container image The next step is to build the container image for the desired OS. It can be built with either `docker` or `podman`. The `xrd-docker` script has the `build` command to facilitate this. Currently, supported OSs for building are CentOS 7, AlmaLinux 8, AlmaLinux 9, Fedora. The command to build the image is simply ```sh $ xrd-docker build ``` where `` is one of `centos7` (default), `alma8`, `alma9`, or `fedora`. The name simply chooses which `Dockerfile` is used from the `build/` directory, as they are named `Dockerfile.` for each suported OS. It is possible to add new `Dockerfile`s following this same naming scheme to support custom setups and still use `xrd-docker build` command to build an image. The images built with `xrd-docker build` are named simply `xrootd` (latest being a default tag added by docker), and an extra `xrootd:` tag is added to allow having it built for multiple OSs at the same time. The current `Dockerfile`s use the spec file and build the image using the RPM packaging workflow, installing dependencies as declared in the spec file, in the first stage, building the RPMs in a second stage, then, in a third stage starting from a fresh image, the RPMs built in stage 2 are copied over and installed with `yum` or `dnf`. #### Switching between `docker` and `podman` if both are installed The `xrd-docker` script takes either `docker` or `podman` if available, in this order. If you have only one of the two installed, everything should work without any extra setup, but if you have both installed and would like to use `podman` instead of `docker` for building the images, it can be done by exporting an environment variable: ```sh $ export DOCKER=$(command -v podman) $ xrd-docker build # uses podman from now on... ``` ### Appendix #### Setting up `podman` Unlike `docker`, `podman` may not work out of the box after installing it. If it doesn't, make sure that you have subuids and subgids setup for your user by running, for example, the two commands below: ```sh $ sudo usermod --add-subuids 1000000-1000999999 $(id -un) $ sudo usermod --add-subgids 1000000-1000999999 $(id -un) ``` You may also have to ensure that container registries in `/etc/containers/registries.conf`. Usually a usable configuration can be renamed from `/etc/containers/registries.conf.example`. Finally, you may want to try container runtimes other than the default. If you still have problems getting started, `podman`'s documentation can be found at `https://podman.io/docs`. xrootd-5.6.9/docs/man/000077500000000000000000000000001457266313600145725ustar00rootroot00000000000000xrootd-5.6.9/docs/man/CMakeLists.txt000066400000000000000000000024071457266313600173350ustar00rootroot00000000000000if( XRDCL_LIB_ONLY ) return() endif() set(MAN1PAGES xrdcp.1 xrdfs.1 xrdmapc.1 ) set(MAN8PAGES cmsd.8 frm_admin.8 frm_purged.8 frm_xfragent.8 frm_xfrd.8 mpxstats.8 xrdpfc_print.8 xrdpwdadmin.8 xrdsssadmin.8 xrootd.8 ) if ( BUILD_FUSE ) list(APPEND MAN1PAGES xrootdfs.1) endif() if ( NOT XRDCL_ONLY ) list(APPEND MAN1PAGES xrdadler32.1 xrdgsiproxy.1 xrdgsitest.1) foreach(MAN ${MAN8PAGES}) configure_file(${MAN} man8/${MAN} @ONLY) endforeach() install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/man8 DESTINATION ${CMAKE_INSTALL_MANDIR}) if( BUILD_VOMS ) list(APPEND MAN1PAGES libXrdVoms.1) endif() endif() foreach(MAN ${MAN1PAGES}) configure_file(${MAN} man1/${MAN} @ONLY) endforeach() install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/man1 DESTINATION ${CMAKE_INSTALL_MANDIR}) install(CODE "execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink xrdcp.1 xrdcopy.1 WORKING_DIRECTORY \$ENV{DESTDIR}/${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_MANDIR}/man1)") if( BUILD_VOMS ) install(CODE "execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink libXrdVoms.1 libXrdSecgsiVOMS.1 WORKING_DIRECTORY \$ENV{DESTDIR}/${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_MANDIR}/man1)") endif() xrootd-5.6.9/docs/man/cmsd.8000066400000000000000000000016661457266313600156220ustar00rootroot00000000000000.TH cmsd 8 "@XRootD_VERSION_STRING@" .SH NAME cmsd - Cluster Management Services daemon .SH SYNOPSIS .nf \fBcmsd\fR [\fIoptions\fR] \fB-c\fR \fIconfigfile\fR .fi .br .ad l .SH DESCRIPTION The \fBcmsd\fR daemon provides cluster services to xrootd data servers. Usage synopsis can be displayed by typing "\fBcmsd -h\fR". Complete HTML documentation can be found at .ce http://xrootd.org/doc/prod/cms_config.htm .SH NOTES Documentation for all components associated with \fBcmsd\fR can be found at http://xrootd.org/docs.html .SH DIAGNOSTICS Configuration errors yield an error message and a non-zero exit status. The program never exits upon success. Use the kill command or "/etc/init.d/cmsd stop" to terminate the program. .SH LICENSE License terms can be displayed by typing "\fBcmsd -H\fR". .SH SUPPORT LEVEL The \fBcmsd\fR daemon is supported by the xrootd collaboration. Contact information can be found at .ce http://xrootd.org/contact.html xrootd-5.6.9/docs/man/frm_admin.8000066400000000000000000000020121457266313600166120ustar00rootroot00000000000000.TH frm_admin 8 "@XRootD_VERSION_STRING@" .SH NAME frm_admin - administer file residency parameters .SH SYNOPSIS .nf \fBfrm_admin\fR [\fIoptions\fR] [\fIcommand\fR [\fIcmdopts\fR]] .fi .br .ad l .SH DESCRIPTION The \fBfrm_admin\fR utility displays and alters data server space attributes associated with file residency. These attributes are used by the \fBfrm_purged\fR, \fBfrm_xfrd\fR, and \fBxrootd\fR daemons to manage file residency. Usage synopsis can be displayed by typing "\fBfrm_admin -h\fR". Complete HTML documentation can be found at .ce http://xrootd.org/doc/prod/frm_config.htm .SH NOTES Documentation for all components associated with \fBfrm_admin\fR can be found at http://xrootd.org/docs.html .SH DIAGNOSTICS Configuration errors yield an error message and a non-zero exit status. .SH LICENSE License terms can be displayed by typing "\fBxrootd -H\fR". .SH SUPPORT LEVEL The \fBfrm_admin\fR command is supported by the xrootd collaboration. Contact information can be found at .ce http://xrootd.org/contact.html xrootd-5.6.9/docs/man/frm_purged.8000066400000000000000000000020621457266313600170150ustar00rootroot00000000000000.TH frm_purged 8 "@XRootD_VERSION_STRING@" .SH NAME frm_purged - File Residency Manager purge daemon .SH SYNOPSIS .nf \fBfrm_purged\fR [\fIoptions\fR] [\fIparameters\fR] .fi .br .ad l .SH DESCRIPTION The \fBfrm_purged\fR daemon automatically removes disk resident files based on a set of local rules. It is part of the File Residency Manager software suite. Usage synopsis can be displayed by typing "\fBfrm_purged -h\fR". Complete HTML documentation can be found at .ce http://xrootd.org/doc/prod/frm_config.htm .SH NOTES Documentation for all components associated with \fBfrm_purged\fR can be found at http://xrootd.org/docs.html .SH DIAGNOSTICS Configuration errors yield an error message and a non-zero exit status. The program never exits upon success. Use the kill command or "/etc/init.d/frm_purged stop" to terminate the program. .SH LICENSE License terms can be displayed by typing "\fBxrootd -H\fR". .SH SUPPORT LEVEL The \fBfrm_purged\fR daemon is supported by the xrootd collaboration. Contact information can be found at .ce http://xrootd.org/contact.html xrootd-5.6.9/docs/man/frm_xfragent.8000066400000000000000000000016551457266313600173540ustar00rootroot00000000000000.TH frm_xfragent 8 "@XRootD_VERSION_STRING@" .SH NAME frm_xfragent - File Residency Manager Transfer agent .SH SYNOPSIS .nf \fBfrm_xfragent\fR [\fIoptions\fR] .fi .br .ad l .SH DESCRIPTION The \fBfrm_xfragent\fR utility sends file transfer requests to the \fBfrm_xfrd\fR daemon. It is part of the File Residency Manager software suite. Usage synopsis can be displayed by typing "\fBfrm_xfragent -h\fR". Complete HTML documentation can be found at .ce http://xrootd.org/doc/prod/frm_config.htm .SH NOTES Documentation for all components associated with \fBfrm_xfragent\fR can be found at http://xrootd.org/docs.html .SH DIAGNOSTICS Configuration errors yield an error message and a non-zero exit status. .SH LICENSE License terms can be displayed by typing "\fBxrootd -H\fR". .SH SUPPORT LEVEL The \fBfrm_xfragent\fR daemon is supported by the xrootd collaboration. Contact information can be found at .ce http://xrootd.org/contact.html xrootd-5.6.9/docs/man/frm_xfrd.8000066400000000000000000000020041457266313600164660ustar00rootroot00000000000000.TH frm_xfrd 8 "@XRootD_VERSION_STRING@" .SH NAME frm_xfrd - File Residency Manager transfer daemon .SH SYNOPSIS .nf \fBfrm_xfrd\fR [\fIoptions\fR] .fi .br .ad l .SH DESCRIPTION The \fBfrm_xfrd\fR daemon automatically copies files in and out of the data server. It is part of the File Residency Manager software suite. Usage synopsis can be displayed by typing "\fBfrm_xfrd -h\fR". Complete HTML documentation can be found at .ce http://xrootd.org/doc/prod/frm_config.htm .SH NOTES Documentation for all components associated with \fBfrm_xfrd\fR can be found at http://xrootd.org/docs.html .SH DIAGNOSTICS Configuration errors yield an error message and a non-zero exit status. The program never exits upon success. Use the kill command or "/etc/init.d/frm_xfrd stop" to terminate the program. .SH LICENSE License terms can be displayed by typing "\fBxrootd -H\fR". .SH SUPPORT LEVEL The \fBfrm_xfrd\fR daemon is supported by the xrootd collaboration. Contact information can be found at .ce http://xrootd.org/contact.html xrootd-5.6.9/docs/man/libXrdVoms.1000066400000000000000000000105421457266313600167470ustar00rootroot00000000000000.TH libXrdVoms 1 "@XRootD_VERSION_STRING@" .SH NAME libXrdVoms - XRootD plug-in to extract VOMS attributes .SH SYNOPSIS .nf sec.protparm gsi -vomsfun:\fBlibXrdVoms.so\fR sec.protparm gsi -vomsfunparms:\fIoptions\fR .fi .SH DESCRIPTION The \fBlibXrdVoms\fR plug-in provides an implementation of the .nf int XrdSecgsiVOMSFun(XrdSecEntity &ent) int XrdSecgsiVOMSInit(const char *cfg) .fi functions making use of the official VOMS API libraries to validate and extract the VOMS attributes from a VOMS proxy. .SH OPTIONS The following options are available: \fBcertfmt={raw,pem,x509}\fR .RS 2 Certificate format: \fBraw\fR to be used with XrdCrypto tools; \fBpem\fR PEM base64 format (as in cert files); \fBx509\fR, as a STACK_OF(X509). Default: \fBraw\fR. .RE \fBgrpopt=opt\fR .RS 2 Defines how to use the group names information; \fBopt\fR is defined as \fBsel\fR * 10 + \fBwhich\fR, with \fBsel\fR either \fB0\fR (consider all the groups present in the VOMS extension) or \fB1\fR (select among those groups specified by the \fBgrps\fR option; see below); \fBwhich\fR can be either \fB0\fR (take the first one) or \fB1\fR (take the last) or \fB2\fR (take all, comma separated, and created a vertically sliced tuple; see \fBNOTES\fR below). .RE \fBgrps=grp1[,grp2,...]\fR .RS 2 Group(s) for which the information is extracted; if specified, the grpopt \fBsel\fR is set to \fB1\fR regardless of the setting; see \fBNOTES\fR below. .RE \fBvos=vo1[,vo2,...]\fR .RS 2 VOs to be considered; the first match is taken; see \fBNOTES\fR below. .RE \fBgrpfmt=fmtstring\fR, \fBrolefmt=fmtstring\fR, \fBvofmt=fmtstring\fR .RS 2 String to be used to format the content of XrdSecEntity::grps, XrdSecEntity::role, XrdSecEntity::vorg, respectively. These strings are optional and by default they are empty. .RE .RS 2 Recognized place holders in the above format strings: .RE .RS 5 .nf : role : group : VO : Full Qualified Attribute Name .fi .RE .RS 2 For example, rolefmt=|grpfmt=|vofmt=" " will inverse the group and role, and will add a space and the FQAN in the vorg field of XrdSecEntity. .RE \fBdbg\fR .RS 2 Force verbose mode. .RE Multiple options can be specified separated by '\fR|\fB'. .SH NOTES Specifying \fBgrps\fR or \fBvos\fR options forces a failure if the requested group and/or VO is not found. In this regard, this plug-in may act as a sort of authorization filter. Note that most refined authorization based on VOMS information may be achieved using the \fBlibXrdSecgsiAuthzVO\fR plug-in distributed with XRootD. Option 'all' for the group selection (which=2) will generated a vertically sliced tuple including VO, group and role fields. For example, the following VOMS attributes .nf attribute : /atlas/de/Role=production/Capability=NULL attribute : /atlas/de/Role=NULL/Capability=NULL attribute : /atlas/Role=NULL/Capability=NULL .fi would result in following content in the XrdSecEntity fields: .nf vorg: atlas atlas atlas grps: /atlas/de /atlas/de /atlas role: producton NULL NULL .fi The default XrdAcc will take its decision by checking in turn the triplets obtained slicing vertically this tuple. .SH EXAMPLES The following example shows how configure the plugin to select VO=cms, select the first group, use the PEM format for the proxy and switch on debugging; it shows also how to specify multiple options, either on the same line or on multiple lines. .RS 5 .nf sec.protparm gsi -vomsfun:libXrdVoms.so sec.protparm gsi -vomsfunparms:grpopt=0|vos=cms|certfmt=pem sec.protparm gsi -vomsfunparms:dbg .fi .SH FILES The plug-in files are .nf lib64/libXrdVoms-4.so (or lib/libXrdVoms-4.so)\fR include/xrootd/private/XrdVoms/XrdVoms.hh\fR .fi and are typically available under \fB/usr\fR. .SH ENVIRONMENT The environment \fBX509_VOMS_DIR\fR must be set to a valid directory; this is typically \fB/etc/grid-security/vomsdir\fR. .SH DIAGNOSTICS The \fBlibXrdVoms\fR plug-in requires \fBlibvomsapi.so\fR and the openssl libraries. In case of load failure it may be useful to check with ldd if all the required dependencies are correctly resolved. .SH LICENSE LGPL; see http://www.gnu.org/licenses/. .SH AUTHOR AND SUPPORT The \fBlibXrdVoms\fR plug-in has been implemented by Gerardo Ganis (Gerardo.Ganis@cern.ch). Any request for support should addressed via the project main web site .ce https://github.com/gganis/vomsxrd or via the XRootD support site .ce https://github.com/xrootd/xrootd xrootd-5.6.9/docs/man/mpxstats.8000066400000000000000000000020421457266313600165440ustar00rootroot00000000000000.TH mpxstats 8 "@XRootD_VERSION_STRING@" .SH NAME mpxstats - Multiplexing Monitor Statistics daemon .SH SYNOPSIS .nf \fBmpxstats\fR [\fIoptions\fR] \fB-p\fR \fIport\fR .fi .br .ad l .SH DESCRIPTION The \fBmpxstats\fR daemon multiplexes xrootd and cmsd monitoring data streams into a single sequential format suitable for ingestion by a monitoring agent (e.g. ganglia, MonaLisa, nagios, etc.). Usage synopsis can be displayed by typing "\fBmpxstats\fR". Complete HTML documentation can be found at .ce http://xrootd.org/doc/prod/xrd_monitoring.htm .SH NOTES Documentation for all components associated with \fBmpxstats\fR can be found at http://xrootd.org/docs.html .SH DIAGNOSTICS Errors yield an error message and a non-zero exit status. The program never exits upon success. Use the kill command to terminate the program. .SH LICENSE License terms can be displayed by typing "\fBxrootd -H\fR". .SH SUPPORT LEVEL The \fBmpxstats\fR daemon is supported by the xrootd collaboration. Contact information can be found at .ce http://xrootd.org/contact.html xrootd-5.6.9/docs/man/xrdadler32.1000066400000000000000000000020071457266313600166250ustar00rootroot00000000000000.TH xrdadler32 1 "@XRootD_VERSION_STRING@" .SH NAME xrdadler32 - compute and display an adler32 checksum .SH SYNOPSIS .nf \fBxrdadler32\fR [\fIfile\fR] .fi .br .ad l .SH DESCRIPTION The \fBxrdadler32\fR utiliity computes and displays an adler32 checksum value for a file. Usage synopsis can be displayed by typing "\fBxrdadler32 -h\fR". Currently, only the usage synopsis is available as documentation. .SH NOTES Documentation for all components associated with \fBxrdadler32\fR can be found at http://xrootd.org/docs.html .br The \fBxrdadler32\fR command uses the XRootD infrastructure to record the computed checksum in the file's extended attributes. Subsequent queries use the checksum stored in the file's attributes. .SH DIAGNOSTICS Errors yield an error message and a non-zero exit status. .SH LICENSE License terms can be displayed by typing "\fBxrootd -H\fR". .SH SUPPORT LEVEL The \fBxrdadler32\fR command is supported by the xrootd collaboration. Contact information can be found at .ce http://xrootd.org/contact.html xrootd-5.6.9/docs/man/xrdcp.1000066400000000000000000000417431457266313600160050ustar00rootroot00000000000000.TH xrdcopy 1 "@XRootD_VERSION_STRING@" .SH NAME xrdcp - copy files .SH SYNOPSIS .nf \fBxrdcp\fR [\fIoptions\fR] \fIsource\fR \fIdestination\fR \fIoptions\fR: [\fB--cksum\fR \fIargs\fR] [\fB--debug\fR \fIlvl\fR] [\fB--coerce\fR] [\fB--force\fR] [\fB--help\fR] [\fB--license\fR] [\fB--nopbar\fR] [\fB--posc\fR] [\fB--proxy \fIipaddr\fB:\fIport\fR] [\fB--recursive\fR] [\fB--retry\fR \fItime\fR] [\fB--server\fR] [\fB--silent\fR] [\fB--sources\fR \fInum\fR] [\fB--streams\fR \fInum\fR] [\fB--tpc\fR [\fBdelegate\fR] \fBfirst\fR|\fBonly\fR] [\fB--verbose\fR] [\fB--version\fR] [\fB--xrate\fR \fIrate\fR] [\fB--zip\fR \fIfile\fR] [\fB--dynamic-src\fR] [\fB--infiles\fR \fIfn\fR] [\fB--parallel\fR \fIn\fR] [\fB--allow-http\fR] [\fB--xattr\fR] [\fB--notlsok\fR] [\fB--tlsnodata\fR] [\fB--tlsmetalink\fR] [\fB--zip-mtln-cksum\fR] [\fB--rm-bad-cksum\fR] [\fB--continue\fR] [\fB--xrate-threshold\fR] [\fB--retry-policy\fR] \fIlegacy options\fR: [\fB-adler\fR] [\fB-DS\fR\fIparm string\fR] [\fB-DI\fR\fIparm number\fR] [\fB-md5\fR] [\fB-np\fR] [\fB-OD\fR\fIcgi\fR] [\fB-OS\fR\fIcgi\fR] [\fB-x\fR] .fi .br .ad l .SH DESCRIPTION The \fBxrdcp\fR utility copies one or more files from one location to another. The data source and destination may be a local or remote file or directory. Additionally, the data source may also reside on multiple servers. .SH OPTIONS \fB-C\fR | \fB--cksum\fR \fItype\fR[\fB:\fR\fIvalue\fR|\fIprint\fR|\fIsource\fR] .RS 5 obtains the checksum of \fItype\fR (i.e. adler32, crc32, md5 or zcrc32) from the source, computes the checksum at the destination, and verifies that they are the same. If \fIauto\fR is chosen as the checksum type, xrdcp will try to automatically infer the right checksum type based on source/destination configuration, source file type (metalink, ZIP), and available checksum plug-ins. If a \fIvalue\fR is specified, it is used as the source checksum. When \fIprint\fR is specified, the checksum at the destination is printed but is \fInot\fR verified. .RE \fB-d\fR | \fB--debug\fR \fIlvl\fR .RS 5 debug level: 1 (low), 2 (medium), 3 (high) .RE \fB-F\fR | \fB--coerce\fR .RS 5 ignores locking semantics on the destination file. This option may lead to file corruption if not properly used. .RE \fB-f\fR | \fB--force\fR .RS 5 re-creates a file if it is already present. .RE \fB-h\fR | \fB--help\fR .RS 5 displays usage information. .RE \fB-H\fR | \fB--license\fR .RS 5 displays license terms and conditions. .RE \fB-N\fR | \fB--nopbar\fR .RS 5 does not display the progress bar. .RE \fB-P\fR | \fB--posc\fR .RS 5 requests POSC (persist-on-successful-close) processing to create a new file. Files are automatically deleted should they not be successfully closed. .RE \fB-D\fR | \fB--proxy\fR \fIproxyaddr\fB:\fIproxyport\fR .RS 5 [NOT YET IMPLEMENTED] use \fIproxyaddr\fB:\fIproxyport\fR as a SOCKS4 proxy. Only numerical addresses are supported. .RE \fB-r\fR | \fB--recursive\fR .RS 5 recursively copy all files starting at the given source directory. .RE \fB--retry\fR .RS 5 retry failed copy-jobs. .RE \fB--server\fR .RS 5 runs as if in a server environment. Used only for server-side third party copy support. .RE \fB-s\fR | \fB--silent\fR .RS 5 neither produces summary information nor displays the progress bar. .RE \fB-y\fR | \fB--sources\fR \fInum\fR .RS 5 uses up to \fInum\fR sources to copy the file. .RE \fB-S\fR | \fB--streams\fR \fInum\fR .RS 5 use \fInum\fR parallel data streams to do the transfer (the main stream is in this case not used to carry out the transfer). The maximum value is 15. The default is 0 (i.e., use only the control stream). .RE \fB--tpc\fR [\fBdelegate\fR] \fBfirst\fR|\fBonly\fR .RS 5 copies the file from remote server to remote server using third-party-copy protocol (i.e., data flows from server to server). The source and destination servers must support third party copies. Additional security restrictions may apply and may cause the copy to fail if they cannot be satisfied. Argument '\fBfirst\fR' tries tpc and if it fails, does a normal copy; while '\fBonly\fR' fails the copy unless tpc succeeds. When '\fBdelegate\fR' is specified, the copy delegates the command issuer's credentials to the target server which uses those credentials to authenticate with the source server. Delegation is ignored if the target server is not configured to use delegated credentials. Currently, only gsi credentials can be delegated. .RE \fB-v\fR | \fB--verbose\fR .RS 5 displays summary output. .RE \fB-V\fR | \fB-version\fR .RS 5 displays version information and immediately exits. .RE \fB-z\fR | \fB--zip\fR \fIfile\fR .RS 5 copy given file from a ZIP archive (same as xrdcl.unzip opaque info). .RE \fB-X\fR | \fB--xrate\fR \fIrate\fR .RS 5 limits the copy speed to the specified \fIrate\fB. The rate may be qualified with the letter \fBk\fR, \fBm\fR, or \fBg\fR to indicate kilo, mega, or giga bytes, respectively. The option only applies when the source or destination is local. .RE \fB-X\fR | \fB--xrate-threshold\fR \fIrate\fR .RS 5 If the transfer rate drops below given threshold force the client to use different source or if no more sources are available fail the transfer. .RE \fB-Z\fR | \fB--dynamic-src\fR .RS 5 file size may change during the copy .RE \fB-I\fR | \fB--infiles\fR \fIfn\fR .RS 5 specifies the file that contains a list of input files .RE \fB--parallel\fR \fIn\fR .RS 5 number of copy jobs to be run simultaneously .RE \fB--allow-http\fR .RS 5 allow HTTP as source or destination protocol. Requires the XrdClHttp client plugin .RE \fB--xattr\fR .RS 5 preserve extended attributes .RE \fB--notlsok\fR .RS 5 If server is too old to support TLS encryption fallback to unencrypted communication. .RE \fB--tlsnodata\fR .RS 5 In case of roots/xroots protocol, encrypt only the control stream and leave the data streams unencrypted. .RE \fB--tlsmetalink .RS 5 Treat all URLs in metalink as roots/xroots. .RE \fB--zip-mtln-cksum .RS 5 Use the checksum available in a metalink file even if a file is being extracted from a ZIP archive. .RE \fB--rm-bad-cksum .RS 5 Remove the target file if checksum verification failed (enables also POSC semantics). .RE \fB--continue .RS 5 Continue copying a file from the point where the previous copy was interrupted. .SH LEGACY OPTIONS Legacy options are provided for backward compatibility. These are now deprecated and should be avoided. .RE \fB-adler\fR .RS 5 equivalent to "\fB--cksum adler32:source\fR". .RE \fB-DI\fR\fIpname numberval\fR .RS 5 set the internal parameter \fIpname\fR with the numeric value \fInumberval\fR. .RE \fB-DS\fR\fIpname stringval\fR .RS 5 set the internal parameter \fIpname\fR with the string value \fIstringval\fR. .RE \fB-md5\fR .RS 5 equivalent to "\fB--cksum md5:source\fR". .RE \fB-np\fR .RS 5 equivalent to "\fB--nopbar\fR". .RE \fB-OD\fR\fIcgi\fR .RS 5 add cgi information \fIcgi\fR to any destination xrootd URL. You should specify the opaque information directly on the destination URL. .RE \fB-OS\fR\fIcgi\fR .RS 5 add cgi information \fIcgi\fR to any source xrootd URL. .RE \fB-x\fR .RS 5 equivalent to "\fB--sources 12\fR". .RE .SH OPERANDS \fIsource\fR .RS 5 a dash (i.e. \fB-\fR) indicating stanard in, a local file, a local directory name suffixed by /, or an xrootd URL in the form of .ce 1 \fBxroot://[\fIuser\fB@\fR]\fIhost[\fB:\fIport\fR]\fB/\fIabsolutepath\fR The \fIabsolutepath\fR can be a directory. .RE \fIdestination\fR .RS 5 a dash (i.e. \fB-\fR) indicating stanard out, a local file, a local directory name suffixed by /, or an xrootd URL in the form .ce 1 \fBxroot://[\fIuser\fB@\fR]\fIhost[\fB:\fIport\fR]\fB/\fIabsolutepath\fR The \fIabsolutepath\fR can be a directory. .RE .SH ENVIRONMENT The following environment variables are supported. They apply to xrdfs and any other application using the libXrdCl library, unless specified otherwise. The text in the brackets is a name of the corresponding xrdcp commandline parameter. .br XRD_LOGLEVEL .RS 5 Determines the amount of diagnostics that should be printed. Valid values are: \fIDump\fR, \fIDebug\fR, \fIInfo\fR, \fIWarning\fR, and \fIError\fR. .RE XRD_LOGFILE .RS 5 If set, the diagnostics will be printed to the specified file instead of stderr. .RE XRD_LOGMASK .RS 5 Determines which diagnostics topics should be printed at all levels. It's a "|" separated list of topics. The first element may be "All" in which case all the topics are enabled and the subsequent elements may turn them off, or "None" in which case all the topics are disabled and the subsequent flags may turn them on. If the topic name is prefixed with "^", then it means that the topic should be disabled. If the topic name is not prefixed, then it means that the topic should be enabled. .br The log mask may as well be handled for each diagnostic level separately by setting one or more of the following variables: \fIXRD_LOGMASK_ERROR\fR, \fIXRD_LOGMASK_WARNING\fR, \fIXRD_LOGMASK_INFO\fR, \fIXRD_LOGMASK_DEBUG\fR, and \fIXRD_LOGMASK_DUMP\fR. The default for each level is "All", except for the \fIDump\fR level, where the default is "All|^PollerMsg". This means that, at the \fIDump\fR level, all the topics but "PollerMsg" are enabled. .br Available topics: AppMsg, UtilityMsg, FileMsg, PollerMsg, PostMasterMsg, XRootDTransportMsg, TaskMgrMsg, XRootDMsg, FileSystemMsg, AsyncSockMsg, TlsMsg .RE XRD_TLSDBGLVL .RS 5 Determine the debug level for the TLS component. Valid values are: \fIOFF\fR, \fICTX\fR, \fISOK\fR, \fISIO\fR, \fIALL\fR and \fIOUT\fR. The default value is \fIOFF\fR. .RE XRD_PARALLELEVTLOOP .RS 5 The number of event loops (i.e. the number of threads handling requests). Default number is 10. .RE XRD_READRECOVERY .RS 5 Determines if read recovery should be enabled or disabled (enabled by default). .RE XRD_WRITERECOVERY .RS 5 Determines if write recovery should be enabled or disabled (enabled by default). .RE XRD_OPENRECOVERY .RS 5 Determines if open recovery should be enabled or disabled for mutable (truncate or create) opens (enabled by default). .RE XRD_CONNECTIONWINDOW (-DIConnectionWindow) .RS 5 A time window for the connection establishment. A connection failure is declared if the connection is not established within the time window. If a connection failure happens earlier then another connection attempt will only be made at the beginning of the next window. .RE XRD_CONNECTIONRETRY (-DIConnectionRetry) .RS 5 Number of connection attempts that should be made (number of available connection windows) before declaring a permanent failure. .RE XRD_REQUESTTIMEOUT (-DIRequestTimeout) .RS 5 Default value for the time after which an error is declared if it was impossible to get a response to a request. .RE XRD_STREAMTIMEOUT (-DIStreamTimeout) .RS 5 Default value for the time after which a connection error is declared (and a recovery attempted) if there are unfulfilled requests and there is no socket activity or a registered wait timeout. .RE XRD_SUBSTREAMSPERCHANNEL (-DISubStreamsPerChannel) .RS 5 Number of streams per session. .RE XRD_TIMEOUTRESOLUTION (-DITimeoutResolution) .RS 5 Resolution for the timeout events. Ie. timeout events will be processed only every XRD_TIMEOUTRESOLUTION seconds. .RE XRD_STREAMERRORWINDOW (-DIStreamErrorWindow) .RS 5 Time after which the permanent failure flags are cleared out and a new connection may be attempted if needed. .RE XRD_RUNFORKHANDLER (-DIRunForkHandler) .RS 5 Determines whether the fork handlers should be enabled, making the API fork safe. .RE XRD_REDIRECTLIMIT (-DIRedirectLimit) .RS 5 Maximum number of allowed redirections. .RE XRD_NOTAUTHORIZEDRETRYLIMIT (-dNotAuthorizedRetryLimit) .RS 5 Maximum number of allowed retries at a meta-manager for not-authorized error. .RE XRD_POLLERPREFERENCE (-DSPollerPreference) .RS 5 A comma separated list of poller implementations in order of preference. The default is: built-in. .RE XRD_CLIENTMONITOR (-DSClientMonitor) .RS 5 Path to the client monitor library. .RE XRD_CLIENTMONITORPARAM (-DSClientMonitorParam) .RS 5 Additional optional parameters that will be passed to the monitoring object on initialization. .RE XRD_WORKERTHREADS (-DIWorkerThreads) .RS 5 Number of threads processing user callbacks. .RE XRD_CPPARALLELCHUNKS (-DICPParallelChunks) .RS 5 Maximum number of asynchronous requests being processed by the xrdcp command per connected channel substream (adjusted in real-time). .RE XRD_CPCHUNKSIZE (-DICPChunkSize) .RS 5 Size of a single data chunk handled by xrdcp. .RE XRD_NETWORKSTACK (-DSNetworkStack) .RS 5 The network stack that the client should use to connect to the server. Possible values are: .B IPAuto - automatically detect which IP stack to use .B IPAll - use IPv6 stack (AF_INET6 sockets) and both IPv6 and IPv4 (mapped to IPv6) addresses .B IPv6 - use only IPv6 stack and addresses .B IPv4 - use only IPv4 stack (AF_INET sockets) and addresses .B IPv4Mapped6 - use IPv6 stack and mapped IPv4 addresses .RE XRD_DATASERVERTTL (-DIDataServerTTL) .RS 5 Time period after which an idle connection to a data server should be closed. .RE XRD_LOADBALANCERTTL (-DILoadBalancerTTL) .RS 5 Time period after which an idle connection to a manager or a load balancer should be closed. .RE XRD_APPNAME (-DSAppName) .RS 5 Override the application name reported to the server. .RE XRD_PLUGINCONFDIR .RS 5 A custom location containing client plug-in config files. .RE XRD_PLUGIN .RS 5 A default client plug-in to be used. .RE XRD_CPINITTIMEOUT (-DICPInitTimeout) .RS 5 Maximum time allowed for the copy process to initialize, ie. open the source and destination files. .RE XRD_CPTPCTIMEOUT (-DICPTPCTimeout) .RS 5 Maximum time allowed for a third-party copy operation to finish. .RE XRD_TCPKEEPALIVE (-DITCPKeepAlive) .RS 5 Enable/Disable the TCP keep alive functionality .RE XRD_TCPKEEPALIVETIME (-DITCPKeepAliveTime) .RS 5 Time between last data packet sent and the first keepalive probe (Linux only) .RE XRD_TCPKEEPALIVEINTERVAL (-DITCPKeepAliveInterval) .RS 5 Interval between subsequent keepalive probes (Linux only) .RE XRD_TCPKEEPALIVEPROBES (-DITCPKeepProbes) .RS 5 Number of unacknowledged probes before considering the connection dead (Linux only) .RE XRD_METALINKPROCESSING .RS 5 Enable/Disable Metalink processing (enabled by default) .RE XRD_LOCALMETALINKFILE .RS 5 Enable/Disable local Metalink file processing (by convention the following URL schema has to be used: root://localfile//path/filename.meta4) The 'localfile' semantic is now deprecated, use file://localhost/path/filename.meta4 instead! .RE XRD_GLFNREDIRECTOR .RS 5 The redirector will be used as a last resort if the GLFN tag is specified in a Metalink file. .RE XRD_XCPBLOCKSIZE .RS 5 Maximu size of a data block assigned to a single source in case of an extreme copy transfer. .RE XRD_NODELAY .RS 5 Disables the Nagle algorithm if set to 1 (default), enables it if set to 0. .RE XRD_PREFERIPV4 .RS 5 If set the client tries first IPv4 address (turned off by default). .RE XRD_MAXMETALINKWAIT .RS 5 The maximum time in seconds a clinet can be stalled by the server if a Metalink redirector is available (defaults to 60s). .RE XRD_PRESERVELOCATETRIED .RS 5 If set to 1 XRootD client will preserve tried/triedrc cgi opaque info for kXR_locate request across redirects/retries, if set to 0 XRootD client will treat kXR_locate as any other passive request. .RE XRD_PRESERVEXATTRS .RS 5 If set to 1 (default) xrdcp will preserve file extended attribues, if set to 0 file extended attributes won't be preserved. .RE XRD_NOTLSOK .RS 5 If set to 1 and the server is to old to support TLS encryption, xrdcp will fallback to unencrypted transmission although roots/xroots protocol was used. By default set to 0. .RE XRD_TLSNODATA .RS 5 If set to 1, in case of roots/xroots protocol only the control stream will be encrypted, data streams will be left unencrypted. By default set to 0. .RE XRD_TLSMETALINK .RS 5 If set to 1 all URLs in metalink will be treated as roots/xroots. By default set to 0. .RE XRD_ZIPMTLNCKSUM .RS 5 If set to 1, use the checksum available in a metalink file even if a file is being extracted from a ZIP archive. By default set to 0; .RE XRD_CPTIMEOUT .RS 5 Timeout for a classical (not TPC) copy job. By default set to 0 (disabled); .RE XRD_CLCONFDIR .RS 5 User defined directory with config files (*.conf). .RE XRD_CLCONFFILE .RS 5 User defined config file. .RE XRD_CPRETRY .RS 5 Maximum number of times to retry failed copy-jobs (default: 0). .RE XRD_CPRETRYPOLICY .RS 5 Copy job retry policy, either force or retry (default: force). .RE XRD_CPUSEPGWRTRD .RS 5 Enable in-fly error correction of corrupted pages (default: 1). .RE .SH RETURN CODES .RE \fB50\fR : generic error (e.g. config, internal, data, OS, command line option) \fB51\fR : socket related error \fB52\fR : postmaster related error \fB53\fR : XRootD related error \fB54\fR : redirection error \fB55\fR : query response was negative (this is not an error) .SH NOTES Documentation for all components associated with \fBxrdcp\fR can be found at http://xrootd.org/docs.html .SH DIAGNOSTICS Errors yield an error message and a non-zero exit status. .SH LICENSE LGPL .SH SUPPORT LEVEL The \fBxrdcp\fR command is supported by the xrootd collaboration. Contact information can be found at: .ce http://xrootd.org/contact.html xrootd-5.6.9/docs/man/xrdfs.1000066400000000000000000000144121457266313600160040ustar00rootroot00000000000000.TH xrdfs 1 "@XRootD_VERSION_STRING@" .SH NAME xrdfs - xrootd file and directory meta-data utility .SH SYNOPSIS .nf \fBxrdfs\fR \fI[--no-cwd]\fR \fIhost[:port]\fR \fI[command [args]]\fR \fBcommand\fR: help, chmod, ls, locate, mkdir, mv, stat, statvfs, query, rm, rmdir, truncate, prepare, cat, tail, spaceinfo .fi .br .ad l .SH DESCRIPTION The \fBxrdfs\fR utility executes meta-data oriented operations (e.g., ls, mv, rm, etc.) on one or more xrootd servers. Command help is available by invoking \fBxrdfs\fR with no command line options or parameters and then typing "help" in response to the input prompt. .SH OPTIONS \fB--no-cwd\fR .RS 3 No CWD is being preset in interactive mode. .SH COMMANDS \fBchmod\fR \fIpath\fR \fI\fR .RS 3 Modify permissions of the \fIpath\fR. Permission string example: \fIrwxr-x--x\fR .RE \fBls\fR \fI[-l]\fR \fI[-u]\fR \fI[-R]\fR \fI[-D]\fR \fI[-Z]\fR \fI[-C]\fR \fI[dirname]\fR .RS 3 Get directory listing. .br \fI-l\fR stat every entry and print long listing .br \fI-u\fR print paths as URLs .br \fI-R\fR list subdirectories recursively .br \fI-D\fR show duplicate entries .br \fI-Z\fR check if file is a ZIP archive and if yes list its content .br \fI-C\fR checksum every entry .RE \fBlocate\fR \fI[-n]\fR \fI[-r]\fR \fI[-d]\fR \fI[-m]\fR \fI[-i]\fR \fI[-p]\fR \fI\fR .RS 3 Get the locations of the path. .br \fI-r\fR refresh, don't use cached locations .br \fI-n\fR make the server return the response immediately even though it may be incomplete .br \fI-d\fR do a recursive, deep locate in order to find data servers .br \fI-m\fR prefer host names to IP addresses .br \fI-i\fR ignore network dependencies (IPv6/IPv4) .br \fI-p\fR be passive: ignore tried/triedrc cgi opaque info .RE \fBmkdir\fR \fI[-p] [-m] \fR .RS 3 Creates a directory/tree of directories. .br \fI-p\fR create the entire directory tree recursively .br \fI-m\fR\fB\fR permissions for newly created directories .RE \fBmv\fR \fI \fR .RS 3 Move path1 to path2 locally on the same server. .RE \fBstat\fR \fI\fR \fI[]\fR .RS 3 Get info about the file(s) or directory(ies). .br \fI-q\fR \fB\fR Makes xrdfs return error code 55 to the shell if the requested flag combination is not present; flags may be combined together using '|' or '&' .br Available flags: \fBXBitSet\fR, \fBIsDir\fR, \fBOther\fR, \fBOffline\fR, \fBPOSCPending\fR, \fBIsReadable\fR, \fBIsWriteable\fR, \fBBackUpExists\fR .RE \fBstatvfs\fR \fI\fR .RS 3 Get info about a virtual file system. .RE \fBquery\fR \fI \fR .RS 3 Obtain server information. Query codes: .br \fIconfig\fR \fB\fR Server configuration; is one of the following: .RS 5 bind_max - the maximum number of parallel streams .br chksum - the supported checksum .br pio_max - maximum number of parallel I/O requests .br readv_ior_max - maximum size of a readv element .br readv_iov_max - maximum number of readv entries .br tpc - support for third party copies .br wan_port - the port to use for wan copies .br wan_window - the wan_port window size .br window - the tcp window size .br cms - the status of the cmsd .br role - the role in a cluster .br sitename - the site name .br version - the version of the server .br .RE \fIchecksumcancel\fR \fB\fR File checksum cancellation .br \fIchecksum\fR \fB\fR File checksum .br \fIopaque\fR \fB\fR Implementation dependent .br \fIopaquefile\fR \fB\fR Implementation dependent .br \fIspace\fR \fB\fR Logical space stats .br \fIstats\fR \fB\fR Server stats; is a list of letters indicating information to be returned: .RS 5 a - all statistics .br p - protocol statistics .br b - buffer usage statistics .br s - scheduling statistics .br d - device polling statistics .br u - usage statistics .br i - server identification .br z - synchronized statistics .br l - connection statistics .br .RE \fIxattr\fR \fB\fR Extended attributes .br \fIprepare\fR \fB [filenames]\fR Prepare request status .RE \fBrm\fR \fI\fR \fI[]\fR .RS 3 Remove a file. .RE \fBrmdir\fR \fI\fR .RS 3 Remove a directory. .RE \fBtruncate\fR \fI \fR .RS 3 Truncate a file. .RE \fBprepare\fR \fI[-c]\fR \fI[-f]\fR \fI[-s]\fR \fI[-w]\fR \fI[-e]\fR \fI[-p priority]\fR \fI[-a requestid]\fR \fIfilenames\fR .RS 3 Prepare one or more files for access. .br \fI-c\fR co-locate staged files if possible .br \fI-f\fR refresh file access time even if the location is known .br \fI-s\fR stage the files to disk if they are not online .br \fI-w\fR when files will be accessed for modification .br \fI-p\fR priority of the request, 0 (lowest) - 3 (highest) .br \fI-a\fR abort the request .br \fI-e\fR evict the file from disk cache .RE \fBcat\fR \fI[-o localfile]\fR \fIfiles\fR .RS 3 Print contents of one or mote files to stdout .br \fI-o\fR print to the specified local file .RE \fBtail\fR \fI[-c bytes] [-f]\fR \fIfile\fR .RS 3 Output last part of files to stdout. .br \fI-c\fR num_bytes out last num_bytes .br \fI-f\fR output appended data as file grows .RE \fBspaceinfo\fR \fIpath\fR .RS 3 Get space statistics for given path. .RE \fBxattr\fR \fI \fR .RS 3 Operation on extended attributes. Codes: .br \fIset\fR \fB\fR Set extended attribute; is string of form name=value .br \fIget\fR \fB\fR Get extended attribute. .br \fIdel\fR \fB\fR Delete extended attribute. .br \fIlist\fR List extended attributes. .SH RETURN CODES .RE \fB50\fR : generic error (e.g. config, internal, data, OS) \fB51\fR : socket related error \fB52\fR : postmaster related error \fB53\fR : XRootD related error \fB54\fR : redirection error \fB55\fR : query response was negative (this is not an error) .SH NOTES For the list of available environment variables please refere to xrdcopy(1) .SH DIAGNOSTICS Errors yield an error message and a non-zero exit status. .SH LICENSE LGPL .SH SUPPORT LEVEL The \fBxrdfs\fR command is supported by the XRootD Collaboration. Contact information can be found at .ce http://xrootd.org/contact.html xrootd-5.6.9/docs/man/xrdgsiproxy.1000066400000000000000000000034141457266313600172600ustar00rootroot00000000000000.TH xrdgsiproxy 1 "@XRootD_VERSION_STRING@" .SH NAME xrdgsiproxy - generate a proxy X.509 certificate .SH SYNOPSIS .nf \fBxrdgsiproxy\fR [\fB-h\fR] [\fImode\fR] [\fB-debug\fR] [\fB-f\fR [\fB-out\fR] \fIfile\fR] .fi .br .ad l .SH DESCRIPTION The \fBxrdgsiproxy\fR utility displays, creates, and destroys X.509 user proxy certificates used by the grid security infrastructure (\fBgsi\fR) security protocol. .SH OPERANDS \fImode\fR .RS 5 \fBinit\fR create proxy certificate and related proxy file .br \fBinfo\fR display content of existing proxy file .br \fBdestroy\fR delete existing proxy file .RE .br .SH OPTIONS .B -h display help .TP \fB-f\fR [\fB-out\fR] \fIfile\fR Non-standard location of proxy file .TP .BI init \0 mode \0 only: .TP \fB-certdir\fR \fIdir\fR Non-standard location of directory with information about known CAs .TP \fB-cert\fR \fIfile\fR Non-standard location of certificate for which proxies are wanted .TP \fB-key\fR \fIfile\fR Non-standard location of private key to be used to sign the proxy .TP \fB-bits\fR \fIn\fR Strength in bits of the key [default 512] .TP \fB-valid\fR \fIhh:mm\fR Time validity of the proxy certificate [default 12:00] .TP \fB-path-length\fR \fIlen\fR Max number of descendent levels below this proxy [default 0] .SH NOTES Complete HTML documentation can be found at .ce 1 http://xrootd.org/doc/prod/sec_config.htm .br Documentation for all components associated with \fBxrdgsiproxy\fR can be found at http://xrootd.org/docs.html .SH DIAGNOSTICS Errors yield an error message and a non-zero exit status. .SH LICENSE License terms can be displayed by typing "\fBxrootd -H\fR". .SH SUPPORT LEVEL The \fBxrdgsiproxy\fR command is supported by the xrootd collaboration. Contact information can be found at .ce http://xrootd.org/contact.html xrootd-5.6.9/docs/man/xrdgsitest.1000066400000000000000000000152631457266313600170630ustar00rootroot00000000000000.TH xrdgsitest 1 "@XRootD_VERSION_STRING@" .SH NAME xrdgsitest - test crypto functionality relevant for the GSI implementation .SH SYNOPSIS .nf \fBxrdgsitest\fR [\fB-h\fR, \fB--help\fR] [\fB-v\fR, \fB--verbose\fR] .fi .br .ad l .SH DESCRIPTION The \fBxrdgsitest\fR utility runs a few tests of the crypto functionality implemented in XrdCrypto relevant for the XrdSecgsi module, i.e. handling of certificates, proxies, chains, verification and similar actions. .br .SH OPTIONS .B -h, --help display help .TP .B -v, --verbose Print very detailed information about the tests. .SH FILES The program needs access to a user certificate file and its private key, and the related CA file(s); the CRL is downloaded using the information found in the CA certificate. The location of the files are the standard ones and they can modified by the standard environment variables: .TP 3 X509_USER_CERT [$HOME/.globus/usercert.pem] user certificate .TP 3 X509_USER_KEY [$HOME/.globus/userkey.pem] user private key .TP 3 X509_USER_PROXY [/tmp/x509up_u] user proxy .TP 3 X509_CERT_DIR [/etc/grid-security/certificates/] CA certificates and CRL directories .SH OUTPUT The output is a list of PASSED/FAILED test similar to .TP $ xrdgsitest .br || --------------------------------------------------------------------------------- .br || Crypto functionality tests for GSI ---------------------------------------------- .br || --------------------------------------------------------------------------------- .br || Loading EEC ............................................................. PASSED .br || Loading User Proxy ...................................................... PASSED .br || --------------------------------------------------------------------------------- .br || Recreate the proxy certificate -------------------------------------------------- .br Enter PEM pass phrase: .br || Recreating User Proxy ................................................... PASSED .br || --------------------------------------------------------------------------------- .br || Load CA certificates ------------------------------------------------------------ .br || Loading CA certificate .................................................. PASSED .br || Loading CA certificate .................................................. PASSED .br || --------------------------------------------------------------------------------- .br || Testing ParseFile --------------------------------------------------------------- .br || Chain reorder: ......................................................... PASSED .br || Chain verify: .......................................................... PASSED .br || --------------------------------------------------------------------------------- .br || Testing ExportChain ------------------------------------------------------------- .br || Attach to X509ExportChain ............................................... PASSED .br || --------------------------------------------------------------------------------- .br || Testing Chain Import ------------------------------------------------------------ .br || Chain reorder: ......................................................... PASSED .br || Chain verify: .......................................................... PASSED .br || --------------------------------------------------------------------------------- .br || Testing GSI chain import and verification --------------------------------------- .br || GSI chain verify: ...................................................... PASSED .br || --------------------------------------------------------------------------------- .br || Testing GSI chain copy ---------------------------------------------------------- .br || GSI chain verify: ...................................................... PASSED .br || --------------------------------------------------------------------------------- .br || Testing Cert verification ------------------------------------------------------- .br || verify cert: EE signed by CA ............................................ PASSED .br || verify cert: PX signed by EE ............................................ PASSED .br || verify cert: PX not signed by CA ........................................ PASSED .br || --------------------------------------------------------------------------------- .br || Testing request creation -------------------------------------------------------- .br || Creating request ........................................................ PASSED .br || --------------------------------------------------------------------------------- .br || Testing request signature ------------------------------------------------------- .br || Check proxyCertInfo extension ........................................... PASSED .br || --------------------------------------------------------------------------------- .br || Testing export of signed proxy -------------------------------------------------- .br || Saving signed proxy chain to file ....................................... PASSED .br || --------------------------------------------------------------------------------- .br || Testing CRL identification ------------------------------------------------------ .br || Check CRL distribution points extension OK .............................. PASSED .br || --------------------------------------------------------------------------------- .br || Testing CRL loading ------------------------------------------------------------- .br --2016-12-12 19:31:36-- http://cafiles.cern.ch/cafiles/crl/CERN%20Root%20Certification%20Authority%202.crl .br Resolving cafiles.cern.ch (cafiles.cern.ch)... 137.138.4.52, 2001:1458:201:96::100:26 .br Connecting to cafiles.cern.ch (cafiles.cern.ch)|137.138.4.52|:80... connected. .br HTTP request sent, awaiting response... 200 OK .br Length: 1097 (1.1K) [application/pkix-crl] .br Saving to: ‘/tmp/5168735f.0.crltmp’ .br .br /tmp/5168735f.0.crltmp 100%[========================================================================>] 1.07K --.-KB/s in 0s .br .br 2016-12-12 19:31:36 (383 MB/s) - ‘/tmp/5168735f.0.crltmp’ saved [1097/1097] .br .br || Loading CA1 crl ......................................................... PASSED .br || CRL signature OK ........................................................ PASSED .br || --------------------------------------------------------------------------------- .TP The result of each test can be interleaved with details when the verbose option is chosen. .SH LICENSE License terms can be displayed by typing "\fBxrootd -H\fR". .SH SUPPORT LEVEL The \fBxrdgsitest\fR command is supported by the xrootd collaboration. Contact information can be found at .ce http://xrootd.org/contact.html xrootd-5.6.9/docs/man/xrdmapc.1000066400000000000000000000024661457266313600163220ustar00rootroot00000000000000.TH xrdmapc 1 "@XRootD_VERSION_STRING@" .SH NAME xrdmapc - query XRootD redirector (status/subscribers/paths) .SH SYNOPSIS .nf \fBxrdmapc\fR [\fIoptions\fR] \fIhost:port\fR [\fIpath\fR] \fIoptions\fR: [\fB--help\fR] [\fB--list\fR \fIargs\fR] [\fB--quiet\fR] [\fB--verify\fR] .fi .br .ad l .SH DESCRIPTION The \fBxrdmapc\fR utility queries the XRootD redirector (status/subscribers/paths). \fIpath\fR when specified, uses : to determine the locations of path and does optional verification. .SH OPTIONS \fB-l\fR | \fB--list\fR [\fB\fR\fIall\fR|\fIm\fR|\fIs\fR] .RS 5 \&'all' lists managers and servers (default), 'm' lists only managers and 's' lists only servers. .RE \fB-q\fR | \fB--quiet\fR .RS 5 Does not print error messages to cerr; errors appear inline. .RE \fB-r\fR | \fB--refresh\fR .RS 5 Does not use cached information but will refresh the cache. .RE \fB-v\fR | \fB--verify\fR .RS 5 Verifies existence status at each server. .SH NOTES Documentation for all components associated with \fBxrdcp\fR can be found at http://xrootd.org/docs.html .SH DIAGNOSTICS Errors yield an error message and a non-zero exit status. .SH LICENSE LGPL .SH SUPPORT LEVEL The \fBxrdmapc\fR command is supported by the xrootd collaboration. Contact information can be found at: .ce http://xrootd.org/contact.html xrootd-5.6.9/docs/man/xrdpfc_print.8000066400000000000000000000040431457266313600173660ustar00rootroot00000000000000.TH xrdpfc_print 8 "@XRootD_VERSION_STRING@" .SH NAME xrdpfc_print - print content of XRootd ProxyFileCache meta data .SH SYNOPSIS .nf \fBxrdpfc_print\fR [\fIoptions\fR] \fRpath ...\fR \fIoptions\fR: [\fB-config\fR \fIfile\fR] [\fB-unit\fR \fIB|kB|MB\fR] [\fB-json\fR] [\fB-indent\fR \fIchars\fR] [\fB-verbose\fR] [\fB-help\fR] .fi .br .ad l .SH DESCRIPTION The \fBxrdpfc_print\fR prints status of downloaded file blocks and access statistics of cached files. .SH OPTIONS \fB-c\fR | \fB-config \fR .RS 5 Xrootd configuration file. Used to load non-default file system (directive ofs.osslib) and prefix for the location of cached files (directive oss.localroot). .RE \fB-v\fR | \fB-verbose\fR .RS 5 Prints state bit for each file block. .RE \fB-u\fR | \fB-unit \fR .RS 5 Specify unit for file-size and access byte-counts; buffer/block size is printed at most as kB. Downward rounding occurs for kB and MB. kB is the default. .br This option does NOT apply to JSON output format, there the unit is always B. .RE \fB-j\fR | \fB-json\fR .RS 5 Prints output in JSON format. .RE \fB-i\fR | \fB-indent \fR .RS 5 Specify alignment for JSON format. Default is -1: single line without any white-space. .RE \fB-h\fR | \fB-help\fR .RS 5 Displays usage information. .RE .RE .SH OPERANDS \fRpath\fR .RS 5 Path to a cinfo file or directory for which the information is to be printed. Path can be relative or absolute. If the path begins with root:/ the path is assumed to be a LFN and gets translated via the standard OSS rules (in the least, it gets prefixed by the oss.localroot). In this case -config option is mandatory. .RE .SH NOTES Documentation for all components associated with \fBxrdpfc_print\fR can be found at http://xrootd.org/docs.html .SH DIAGNOSTICS Errors yield an error message and a non-zero exit status. .SH LICENSE License terms can be displayed by typing "\fBxrootd -H\fR". .SH SUPPORT LEVEL The \fBxrdpfc_print\fR command is supported by the xrootd collaboration. Contact information can be found at .ce http://xrootd.org/contact.html xrootd-5.6.9/docs/man/xrdpwdadmin.8000066400000000000000000000015641457266313600172120ustar00rootroot00000000000000.TH xrdpwdadmin 8 "@XRootD_VERSION_STRING@" .SH NAME xrdpwdadmin - administer pwd security protocol passwords .SH SYNOPSIS .nf \fBxrdpwdadmin\fR [\fB-m\fR \fImode\fR] \fIcommand\fR [\fIoptions\fR] .fi .br .ad l .SH DESCRIPTION The \fBxrdpwdadmin\fR utility displays and maintains password and auto-login files used by the password (\fBpwd\fR) security protocol. Complete HTML documentation can be found at .ce http://xrootd.org/doc/prod/sec_config.htm .SH NOTES Documentation for all components associated with \fBxrdpwdadmin\fR can be found at http://xrootd.org/docs.html .SH DIAGNOSTICS Errors yield an error message and a non-zero exit status. .SH LICENSE License terms can be displayed by typing "\fBxrootd -H\fR". .SH SUPPORT LEVEL The \fBxrdpwdadmin\fR command is supported by the xrootd collaboration. Contact information can be found at .ce http://xrootd.org/contact.html xrootd-5.6.9/docs/man/xrdsssadmin.8000066400000000000000000000015361457266313600172270ustar00rootroot00000000000000.TH xrdsssadmin 8 "@XRootD_VERSION_STRING@" .SH NAME xrdsssadmin - administer simple shared secret keytables .SH SYNOPSIS .nf \fBxrdsssadmin\fR [\fIoptions\fR] \fIcommand\fR [\fIfile\fR] .fi .br .ad l .SH DESCRIPTION The \fBxrdsssadmin\fR utility displays and maintains keytables for the simple shared secret (\fBsss\fR) security protocol. Complete HTML documentation can be found at .ce http://xrootd.org/doc/prod/sec_config.htm .SH NOTES Documentation for all components associated with \fBxrdsssadmin\fR can be found at http://xrootd.org/docs.html .SH DIAGNOSTICS Errors yield an error message and a non-zero exit status. .SH LICENSE License terms can be displayed by typing "\fBxrootd -H\fR". .SH SUPPORT LEVEL The \fBxrdsssadmin\fR command is supported by the xrootd collaboration. Contact information can be found at .ce http://xrootd.org/contact.html xrootd-5.6.9/docs/man/xrootd.8000066400000000000000000000017551457266313600162120ustar00rootroot00000000000000.TH xrootd 8 "@XRootD_VERSION_STRING@" .SH NAME xrootd - eXtended ROOT daemon .SH SYNOPSIS .nf \fBxrootd\fR [\fIoptions\fR] [\fIexports\fR] .fi .br .ad l .SH DESCRIPTION The \fBxrootd\fR daemon provides LAN/WAN access to data in stand-alone nodes, clustered nodes, as well as globally federated clusters. Usage synopsis can be displayed by typing "\fBxrootd -h\fR". Complete HTML documentation can be found at .ce http://xrootd.org/doc/prod/xrd_config.htm .SH NOTES Documentation for all components associated with \fBxrootd\fR can be found at http://xrootd.org/docs.html .SH DIAGNOSTICS Configuration errors yield an error message and a non-zero exit status. The program never exits upon success. Use the kill command or "/etc/init.d/xrootd stop" to terminate the program. .SH LICENSE License terms can be displayed by typing "\fBxrootd -H\fR". .SH SUPPORT LEVEL The \fBxrootd\fR daemon is supported by the xrootd collaboration. Contact information can be found at .ce http://xrootd.org/contact.html xrootd-5.6.9/docs/man/xrootdfs.1000066400000000000000000000072551457266313600165350ustar00rootroot00000000000000.TH xrootdfs 1 "@XRootD_VERSION_STRING@" .SH NAME xrootdfs - xrootd FUSE file system daemon .SH SYNOPSIS .nf \fBxrootdfs\fR [\fIoptions\fR] \fIparameters\fR .fi .br .ad l .SH DESCRIPTION The \fBxrootdfs\fR daemon provides a file system view of an xrootd cluster using FUSE. Usage synopsis can be displayed by typing "\fBxrootdfs -h\fR". Short documentation can be found in a README file in the src/XrdFfs source directory. .SH EXAMPLES Assuming the redirector is .B rdr:port run from command line with debugging output .RS xrootdfs -d -o rdr=root://rdr:port//data,uid=daemon /mnt .RE use in /etc/fstab .RS xrootdfs /mnt fuse rdr=root://rdr:port//data,uid=daemon 1 2 .RE use with autofs .RS 1. add a line to /etc/auto.master .br /\- /etc/auto.fuse 2. create /etc/auto.fuse with the following one line .br /mnt \-fstype=fuse,uid=2,rdr=root://rdr\\:port//data :xrootdfs.sh 3. create script /usr/bin/xrootdfs.sh (make sure +x bit is set) .br #!/bin/sh .br exec /usr/bin/xrootdfs $@ >/dev/null 2>&1 .SH NOTES Documentation for all components associated with \fBxrootdfs\fR can be found at http://xrootd.org/docs.html xrootdfs allows users and administators to query and change the internal parameters on the fly via the filesystem extended attributes getfattr -n attribute_name /mount_point .br setfattr -n attribute_name [ -v value ] /mount_point attribute_name: .RS .B xroot.url: query the actual ROOT url of the file (this is an old one) .br .B xrootdfs.fs.nworkers: query or change the number of threads working in parallel on operations such as stat(), unlink()/rmdir(), readdir(), statvfs(), etc. .br .B xrootdfs.fs.dataserverlist: query or refresh the list of all data servers known to this xrootdfs instance (or "kill -USR1 pid" to refresh) .SH SECURITY By default, XrootdFS does not send individual user identity to the Xrootd storage servers. So Xrootd storage thinks that all operations from an XrootdFS instance come from the user that runs the XrootdFS instance. When the Xrootd "sss" security module (Simple Shared Security) is enabled at both XrootdFS and Xrootd storage system, XrootdFS will send individual user identity information to the Xrootd storage servers. This info can be used along with the Xrootd ACL to control file/directory access. To use "sss" security module, both Xrootd data servers and XrootdFS should be configured to use "sss" in a particular way, e.g. both sides should use a key file that contains the same key generated by the xrdsssadmin program in the following way: xrdsssadmin -k my_key_name -u anybody -g usrgroup add keyfile (change only "my_key_name" and "keyfile"). Please refer to environment variable "XrdSecsssKT" in Xrootd "Authentication & Access Control Configuration Reference" for more information on the location of the keyfile and its unix permission bits. That same document also describes the Xrootd ACL DB file. To enable "sss" with XrootdFS, use the sss=/keyfile option with XrootdFS. The following example shows how to use both unix and sss security modules with the Xrootd data servers. .RS xrootd.seclib /usr/lib64/libXrdSec.so .br sec.protocol /usr/lib64 sss -s /keyfile .br sec.protocol /usr/lib64 unix .br acc.authdb /your_xrootd_ACL_auth_db_file .br acc.authrefresh 300 .br ofs.authorize .SH DIAGNOSTICS Errors yield an error message and a non-zero exit status. The program never exits upon success. Use the umount command to terminate the program. Additional logging information can be found in syslog (/var/log/messages) .SH LICENSE License terms can be displayed by typing "\fBxrootd -H\fR". .SH SUPPORT LEVEL The \fBxrootdfs\fR daemon is supported by the xrootd collaboration. Contact information can be found at .ce http://xrootd.org/contact.html xrootd-5.6.9/gen-tarball.sh000077500000000000000000000002461457266313600156200ustar00rootroot00000000000000#!/bin/bash set -e TAG=$(printf "%s" "$(git describe "${1:-HEAD}")") NAME="xrootd-${TAG#v}" set -x git archive -9 --prefix="${NAME}/" -o "${NAME}.tar.gz" ${TAG} xrootd-5.6.9/genversion.sh000077500000000000000000000025121457266313600156050ustar00rootroot00000000000000#!/usr/bin/env bash # This script no longer generates XrdVersion.hh, but just # prints the version using the same strategy as in the module # XRootDVersion.cmake. The script will first try to use a custom # version set with the option --version or via the environment # variable XRDVERSION, then read the VERSION file and if that is # not expanded by git, git describe is used. If a bad version is # set for any reason, a fallback version is used based on a date. function usage() { cat 1>&2 <<-EOF Usage: $(basename $0) [--help|--version] --help prints this message --print-only ignored, used for backward compatibility --version VERSION sets a custom version EOF } SRC=$(dirname $0) VF=${SRC}/VERSION if [[ -n "${XRDVERSION}" ]]; then VERSION=${XRDVERSION}; elif [[ -r "${VF}" ]] && grep -vq "Format:" "${VF}"; then VERSION=$(sed -e 's/-g/+g/' "${VF}") elif git -C ${SRC} describe >/dev/null 2>&1; then VERSION=$(git -C ${SRC} describe | sed -e 's/-g/+g/') else VERSION="v5.7-rc$(date +%Y%m%d)" fi while [[ $# -gt 0 ]]; do case $1 in --help) usage exit 0 ;; --print-only) shift ;; --version) shift if [[ $# == 0 ]]; then echo "error: --version parameter needs an argument" 1>&2 fi VERSION=$1 shift ;; *) echo "warning: unknown option: $1" 1>&2 shift ;; esac done printf "${VERSION}" xrootd-5.6.9/packaging/000077500000000000000000000000001457266313600150135ustar00rootroot00000000000000xrootd-5.6.9/packaging/common/000077500000000000000000000000001457266313600163035ustar00rootroot00000000000000xrootd-5.6.9/packaging/common/client-plugin.conf.example000066400000000000000000000015271457266313600233630ustar00rootroot00000000000000#------------------------------------------------------------------------------- # Example client plug-in configuration file. Its name needs to end wiht '.conf' # in order to be taken into account while the plug-in environment is set up. # # It needs to contain at least three lines with the following key-value pairs: # 1) url - denoting a semicolon separated list of URLs the plug-in applies # to # 2) lib - path to the library containing the plug-in factory # 3) enable - telling the system whether the plugin for the specified URLs # should be processed, or disabled, if it has already been # registered by one of the previusly processed files #------------------------------------------------------------------------------- url = root://eosatlas.cern.ch:1094 lib = /usr/lib/libXrdEosClient.so enable = true xrootd-5.6.9/packaging/common/client.conf000066400000000000000000000140051457266313600204300ustar00rootroot00000000000000#------------------------------------------------------------------------------- # XRootD client configuration file # # Uncomment the line containing the variable you want to change to make it # effective. # # Settings defined in /etc/xrootd/client.conf will be overriden by the # ones defined in ~/.xrootd/client.conf or by environment variable (see # man xrdcp for details). # # All the timeout values are in seconds. #------------------------------------------------------------------------------- #------------------------------------------------------------------------------- # A time window for the connection establishment. A connection failure is # declared if the connection is not established within the time window. If # a connection failure happens earlier then another connection attempt will # only be made at the beginning of the next window. # # ConnectionWindow = 120 #------------------------------------------------------------------------------- # Number of connection attempts that should be made (number of available # connection windows) before declaring a permanent failure. # # ConnectionRetry = 5 #------------------------------------------------------------------------------- # Default value for the time after which an error is declared if it was # impossible to get a response to a request. # # RequestTimeout = 1800 #------------------------------------------------------------------------------- # Default value for the time after which a connection error is declared (and a # recovery attempted) if there are unfulfilled requests and there is no socket # activity or a registered wait timeout. # # StreamTimeout = 60 #------------------------------------------------------------------------------- # Number of streams per session. # # SubStreamsPerChannel = 1 #------------------------------------------------------------------------------- # Resolution for the timeout events. Ie. timeout events will be processed only # every TimeoutResolution seconds. # # TimeoutResolution = 15 #------------------------------------------------------------------------------- # Time after which the permanent failure flags are cleared out and a new # connection may be attempted if needed. # # StreamErrorWindow = 1800 #------------------------------------------------------------------------------- # Determines whether the fork handlers should be enabled, making the API fork # safe - this has performance implications, so probably you want to set it # on per-process level via an environment variable. # # RunForkHandler = 0 #------------------------------------------------------------------------------- # Maximum number of allowed redirections. # # RedirectLimit = 16 #------------------------------------------------------------------------------- # Number of threads processing user callbacks. # # WorkerThreads = 3 #------------------------------------------------------------------------------- # Size of a single data chunk handled by xrdcopy. # # CPChunkSize = 16777216 #------------------------------------------------------------------------------- # Maximum number of asynchronous requests being processed by the xrdcopy # command at any given time. # # CPParallelChunks = 4 #------------------------------------------------------------------------------- # Time period after which an idle connection to a data server should be closed. # # DataServerTTL = 300 #------------------------------------------------------------------------------- # Time period after which an idle connection to a manager or a load balancer # should be closed. # # LoadBalancerTTL = 1200 #------------------------------------------------------------------------------- # Maximum time allowed for the copy process to initialize, ie. open the source # and destination files. # # CPInitTimeout = 600 #------------------------------------------------------------------------------- # Maximum time allowed for a third-party copy operation to finish. # # CPTPCTimeout = 1800 #------------------------------------------------------------------------------- # Enable/Disable the TCP keep alive functionality # # TCPKeepAlive = 0 #------------------------------------------------------------------------------- # Time between last data packet sent and the first keepalive probe (Linux only) # # TCPKeepAliveTime = 7200 #------------------------------------------------------------------------------- # Interval between subsequent keepalive probes (Linux only) # # TCPKeepAliveInterval = 75 #------------------------------------------------------------------------------- # Number of unacknowledged probes before considering the connection dead # (Linux only) # # TCPKeepProbes = 9 #------------------------------------------------------------------------------- # A comma separated list of poller implementations in order of preference. # # PollerPreference = built-in #------------------------------------------------------------------------------- # The network stack that the client should use to connect to the server. # Possible values are: # # IPAuto - automatically detect which IP stack to use # IPAll - use IPv6 stack (AF_INET6 sockets) and both IPv6 and IPv4 # (mapped to IPv6) addresses # IPv6 - use only IPv6 stack and addresses # IPv4 - use only IPv4 stack (AF_INET sockets) and addresses # IPv4Mapped6 - use IPv6 stack and mapped IPv4 addresses # # NetworkStack = IPAuto #------------------------------------------------------------------------------- # Path to the client monitor library. # # ClientMonitor = #------------------------------------------------------------------------------- # Additional optional parameters that will be passed to the monitoring object # on initialization. # # ClientMonitorParam = #------------------------------------------------------------------------------- # A custom location containing client plug-in config files. # # PlugInConfDir = #------------------------------------------------------------------------------- # A default client plug-in to be used. # # PlugIn = #------------------------------------------------------------------------------- xrootd-5.6.9/packaging/common/cmsd@.service000066400000000000000000000012551457266313600207160ustar00rootroot00000000000000[Unit] Description=XRootD cmsd daemon instance %I Documentation=man:cmsd(8) Documentation=http://xrootd.org/docs.html Requires=network-online.target After=network-online.target [Service] #PrivateDevices=true #ProtectHostname=true #ProtectClock=true #ProtectKernelTunables=true #ProtectKernelModules=true #ProtectKernelLogs=true #ProtectControlGroups=true #RestrictRealtime=true ExecStart=/usr/bin/cmsd -l /var/log/xrootd/cmsd.log -c /etc/xrootd/xrootd-%i.cfg -k fifo -s /run/xrootd/cmsd-%i.pid -n %i User=xrootd Group=xrootd Type=simple Restart=on-abort RestartSec=10 KillMode=control-group LimitNOFILE=65536 WorkingDirectory=/var/spool/xrootd [Install] RequiredBy=multi-user.target xrootd-5.6.9/packaging/common/frm_purged@.service000066400000000000000000000012501457266313600221150ustar00rootroot00000000000000[Unit] Description=XRootD frm_purged daemon instance %I Documentation=man:frm_purged(8) Documentation=http://xrootd.org/docs.html Requires=network-online.target After=network-online.target [Service] #PrivateDevices=true #ProtectHostname=true #ProtectClock=true #ProtectKernelTunables=true #ProtectKernelModules=true #ProtectKernelLogs=true #ProtectControlGroups=true #RestrictRealtime=true ExecStart=/usr/bin/frm_purged -l /var/log/xrootd/frm_purged.log -c /etc/xrootd/xrootd-%i.cfg -k fifo -s /run/xrootd/frm_purged-%i.pid -n %i User=xrootd Group=xrootd Type=simple Restart=on-abort RestartSec=10 KillMode=control-group LimitNOFILE=65536 [Install] RequiredBy=multi-user.target xrootd-5.6.9/packaging/common/frm_xfrd@.service000066400000000000000000000012361457266313600215760ustar00rootroot00000000000000[Unit] Description=XRootD frm_xfrd daemon instance %I Documentation=man:frm_xrfd(8) Documentation=http://xrootd.org/docs.html Requires=network-online.target After=network-online.target [Service] #PrivateDevices=true #ProtectHostname=true #ProtectClock=true #ProtectKernelTunables=true #ProtectKernelModules=true #ProtectKernelLogs=true #ProtectControlGroups=true #RestrictRealtime=true ExecStart=/usr/bin/frm_xfrd -l /var/log/xrootd/frm_xfrd.log -c /etc/xrootd/xrootd-%i.cfg -k fifo -s /run/xrootd/frm_xfrd-%i.pid -n %i User=xrootd Group=xrootd Type=simple Restart=on-abort RestartSec=10 KillMode=control-group LimitNOFILE=65536 [Install] RequiredBy=multi-user.target xrootd-5.6.9/packaging/common/http.client.conf.example000066400000000000000000000001101457266313600230300ustar00rootroot00000000000000url = http://*;https://* lib = /usr/lib64/libXrdClHttp.so enable = true xrootd-5.6.9/packaging/common/recorder.conf000066400000000000000000000000751457266313600207610ustar00rootroot00000000000000url = * lib = /usr/lib64/libXrdClRecorder-5.so enable = falsexrootd-5.6.9/packaging/common/xrdhttp@.socket000066400000000000000000000001711457266313600213110ustar00rootroot00000000000000[Unit] Description=XrdHttp socket [Socket] ListenStream=80 Service=xrootd@%i.service [Install] WantedBy=sockets.target xrootd-5.6.9/packaging/common/xrootd-clustered.cfg000066400000000000000000000056761457266313600223110ustar00rootroot00000000000000########################################################################### # This is a very simple sample configuration file sufficient to start an # # xrootd data server using the default port 1094 and its companion cmsd. # # Trying to use the xrootd will cause the client to simply wait because # # there is no redirector and this configuration file is insufficient to # # start one. Consult the reference manuals on how to create a usable # # configuration file to completely describe a functional xrootd cluster. # # # # On start-up the xrootd will complain about not connecting to the pipe # # named '/var/spool/xrootd/.olb/olbd.admin'. This will continue until the # # cmsd starts. When the cmsd start is will say ' Waiting for primary # # server to login.' Once xrootd is started and connects to the cmsd, the # # cmsd will complain 'Unable to connect socket to localhost' because # # there is no redirector. However, this shows that xrootd and cmsd have # # been correctly installed. # # # # Note: You should always create a *single* configuration file and use it # # when starting each daemon that you need to run in the cluster! # ########################################################################### # The export directive indicates which paths are to be exported. While the # default is '/tmp', we indicate it anyway and add the 'stages attribute # to allow you to start the frm_xfrd to bring in missing files into '/tmp'. # Remove this attribute if you don't want to enable this feature. # all.export /tmp stage # The role directive tells xrootd to run as a data server as part of a # cluster. The causes the xrootd to try to contact the local cmsd which # needs to started as part of of initialization. As a side note, a # redirector would have a manager role. # all.role server # The cmsd running on a data server node needs to know where the redirector # (i.e. manager) is running. In this generic config we simply say that it # is on this host to allow initialization to succeed. However, the final # result is not practically usable. # all.manager localhost 3121 # The copycmd directive tells the frm_xfrd what to use to copy files into # an exported path with the 'stage' attribute. Here we just say this will # be '/bin/cp' to allow the frm_xfrd to actual start to show that it works. # Here missing files are created in /tmp as zero-length files. # frm.xfr.copycmd /bin/cp /dev/null $PFN # The adminpath and pidpath variables indicate where the pid and various # IPC files should be placed # all.adminpath /var/spool/xrootd all.pidpath /run/xrootd # More configuration files can be added in /etc/xrootd/config.d/ # For example /etc/xrootd/config.d/10-mygrid.cfg and # /etc/xrootd/config.d/98-mysite-specifics.cfg # continue /etc/xrootd/config.d/ xrootd-5.6.9/packaging/common/xrootd-filecache-clustered.cfg000066400000000000000000000064161457266313600242030ustar00rootroot00000000000000########################################################################### # This is a very simple sample configuration file sufficient to start an # # xrootd file caching data server using the default port 1094 and its # # companion cmsd. Trying to use the xrootd will cause the client to # # simply wait there is no redirector and this configuration file is # # insufficient to start one. Consult the reference manuals on how to # # create a usable configuration file to completely describe a functional # # xrootd cluster. # # # # On start-up the xrootd will complain about not connecting to the pipe # # named '/var/spool/xrootd/.olb/olbd.admin'. This will continue until the # # cmsd starts. When the cmsd start is will say ' Waiting for primary # # server to login.' Once xrootd is started and connects to the cmsd, the # # cmsd will complain 'Unable to connect socket to localhost' because # # there is no redirector. However, this shows that xrootd and cmsd have # # been correctly installed. # # # # Note: You should always create a *single* configuration file and use it # # when starting each daemon that you need to run in the cluster! # ########################################################################### # Tell everyone who the manager is # all.manager redirector:1213 # The redirector and all cmsd’s export /data red-only with the stage option. The stage # option requests that if the file isn’t found in the cluster the redirector should send # the client to a PFC server with enough space to cache the file. # all.export /data stage r/o # Configuration is different for the redirector, the server cmsd, and # for the server xrootd. We break those out in the if-else-fi clauses. # if redirector all.role manager # Export with stage option - if the file isn’t found in the cluster the # redirector sends the client to a PFC server with enough free space. # all.export /data stage r/o # Server’s cmsd configuration – all PFC’s are virtual data servers # else if exec cmsd all.role server # Export with stage option - this tells manager cmsd we can pull files from the origin # all.export /data stage r/o # The cmsd uses the standard oss plug-in to locate files in the cache. # oss.localroot directive should be the same as for the server. # oss.localroot /pfc-cache # Server’s xrootd configuration – all PFC’s are virtual data servers # else all.role server # For xrootd, load the proxy plugin and the disk caching plugin. # ofs.osslib libXrdPss.so pss.cachelib default # The server needs to write to disk, stage not relevant # all.export /data rw # Tell the proxy where the data is coming from (arbitrary). # pss.origin someserver.domain.org:1094 # Tell the PFC’s where the disk cache resides (arbitrary). # oss.localroot /pfc-cache # Tell the PFC’s available RAM # pfc.ram 100g fi # More configuration files can be added in /etc/xrootd/config.d/ # For example /etc/xrootd/config.d/10-mygrid.cfg and # /etc/xrootd/config.d/98-mysite-specifics.cfg # continue /etc/xrootd/config.d/ xrootd-5.6.9/packaging/common/xrootd-filecache-standalone.cfg000066400000000000000000000032021457266313600243270ustar00rootroot00000000000000########################################################################### # This is a very simple sample configuration file sufficient to start an # # xrootd file caching proxy server using the default port 1094. This # # server runs by itself (stand-alone) and does not assume it is part of a # # cluster. You can then connect to this server to access files in '/tmp'. # # Consult the the reference manuals on how to create more complicated # # configurations. # # # # On successful start-up you will see 'initialization completed' in the # # last message. You can now connect to the xrootd server. # # # # Note: You should always create a *single* configuration file for all # # daemons related to xrootd. # ########################################################################### # Allow access to path with given prefix. # all.export /test/ # Load the proxy plugin and the disk caching plugin. # ofs.osslib libXrdPss.so pss.cachelib default # Tell the proxy where the data is coming from (arbitrary). # pss.origin source.edu:1094 # Specify where the local file system name space is actually rooted. # oss.localroot /data/xrd # Tell maximum allowed RAM usage. # pfc.ram 16g # More configuration files can be added in /etc/xrootd/config.d/ # For example /etc/xrootd/config.d/10-mygrid.cfg and # /etc/xrootd/config.d/98-mysite-specifics.cfg # continue /etc/xrootd/config.d/ xrootd-5.6.9/packaging/common/xrootd-http.cfg000066400000000000000000000037021457266313600212620ustar00rootroot00000000000000########################################################################### # This is a very simple sample configuration file sufficient to start an # # xrootd data server using the default port 1094 plus http protocol on # # port 80. This server runs by itself (stand-alone) and does not assume # # it is part of a cluster. You can then connect to this server to access # # files in '/tmp'. Consult the the reference manuals on how to create # # more complicated configurations and set the host cert and key for http. # # # # On successful start-up you will see 'initialization completed' in the # # last message. You can now connect to the xrootd server. # # # # Note: You should always create a *single* configuration file for all # # daemons related to xrootd. # ########################################################################### # The export directive indicates which paths are to be exported. While the # default is '/tmp', we indicate it anyway to show you this directive. # all.export /tmp # The adminpath and pidpath variables indicate where the pid and various # IPC files should be placed # all.adminpath /var/spool/xrootd all.pidpath /run/xrootd # Load the http protocol, indicate that it should be served on port 80. # The socket bound to port 80 has to be preallocated by the systemd # xrdhttp.socket (requires systemd!). # # In order to enable the xrdhttp.socket run: # systemctl enable xrdhttp@http.socket # In order to start the xrdhttp.socket run: # systemctl start xrdhttp@http.socket # xrd.protocol XrdHttp:80 /usr/lib64/libXrdHttp.so # More configuration files can be added in /etc/xrootd/config.d/ # For example /etc/xrootd/config.d/10-mygrid.cfg and # /etc/xrootd/config.d/98-mysite-specifics.cfg # continue /etc/xrootd/config.d/ xrootd-5.6.9/packaging/common/xrootd-standalone.cfg000066400000000000000000000030661457266313600224360ustar00rootroot00000000000000########################################################################### # This is a very simple sample configuration file sufficient to start an # # xrootd data server using the default port 1094. This server runs by # # itself (stand-alone) and does not assume it is part of a cluster. You # # can then connect to this server to access files in '/tmp'. # # Consult the the reference manuals on how to create more complicated # # configurations. # # # # On successful start-up you will see 'initialization completed' in the # # last message. You can now connect to the xrootd server. # # # # Note: You should always create a *single* configuration file for all # # daemons related to xrootd. # ########################################################################### # The export directive indicates which paths are to be exported. While the # default is '/tmp', we indicate it anyway to show you this directive. # #all.export /tmp # The adminpath and pidpath variables indicate where the pid and various # IPC files should be placed # all.adminpath /var/spool/xrootd all.pidpath /run/xrootd # More configuration files can be added in /etc/xrootd/config.d/ # For example /etc/xrootd/config.d/10-mygrid.cfg and # /etc/xrootd/config.d/98-mysite-specifics.cfg # continue /etc/xrootd/config.d/ xrootd-5.6.9/packaging/common/xrootd.logrotate000066400000000000000000000011621457266313600215440ustar00rootroot00000000000000/var/log/xrootd/*/*.log /var/log/xrootd/*.log { # if the xrootd log rotate is enabled don't do anything # (by convention if xrootd log rotate is enabled the # .lock file exists) prerotate LOCK=`dirname $1`/.lock if [ -f $LOCK ]; then exit 1 else exit 0 fi endscript dateext missingok nomail nocreate rotate 100 notifempty daily compress postrotate PIPE=`dirname $1`/.`basename $1` if [ -p "$PIPE" ]; then /usr/bin/expect -c "set timeout 2; spawn /bin/sh -c \"echo ping > $PIPE\"; expect timeout { exit 1 } eof { exit 0 }" > /dev/null; fi endscript } xrootd-5.6.9/packaging/common/xrootd.te000066400000000000000000000012501457266313600201520ustar00rootroot00000000000000 policy_module(xrootd, 4) #============= logrotate_t ============== optional_policy(` gen_require(` type logrotate_t; type var_log_t; type ptmx_t; type tmpfs_t; type devpts_t; class fifo_file { write open getattr }; class chr_file { open read write ioctl getattr }; class filesystem getattr; ') allow logrotate_t var_log_t:fifo_file { write open getattr }; allow logrotate_t ptmx_t:chr_file { open read write ioctl }; allow logrotate_t tmpfs_t:filesystem getattr; allow logrotate_t devpts_t:filesystem getattr; allow logrotate_t devpts_t:chr_file { getattr open read write ioctl }; ') xrootd-5.6.9/packaging/common/xrootd@.service000066400000000000000000000012671457266313600213120ustar00rootroot00000000000000[Unit] Description=XRootD xrootd daemon instance %I Documentation=man:xrootd(8) Documentation=http://xrootd.org/docs.html Requires=network-online.target After=network-online.target [Service] #PrivateDevices=true #ProtectHostname=true #ProtectClock=true #ProtectKernelTunables=true #ProtectKernelModules=true #ProtectKernelLogs=true #ProtectControlGroups=true #RestrictRealtime=true ExecStart=/usr/bin/xrootd -l /var/log/xrootd/xrootd.log -c /etc/xrootd/xrootd-%i.cfg -k fifo -s /run/xrootd/xrootd-%i.pid -n %i User=xrootd Group=xrootd Type=simple Restart=on-abort RestartSec=10 KillMode=control-group LimitNOFILE=65536 WorkingDirectory=/var/spool/xrootd [Install] RequiredBy=multi-user.target xrootd-5.6.9/packaging/common/xrootd@.socket000066400000000000000000000001721457266313600211340ustar00rootroot00000000000000[Unit] Description=XRootD socket [Socket] ListenStream=1094 Service=xrootd@%i.service [Install] WantedBy=sockets.target xrootd-5.6.9/packaging/debian_scripts/000077500000000000000000000000001457266313600200045ustar00rootroot00000000000000xrootd-5.6.9/packaging/debian_scripts/publish_debian_cern.sh000077500000000000000000000033121457266313600243210ustar00rootroot00000000000000#!/bin/bash #------------------------------------------------------------------------------- # Publish debian artifacts on CERN Gitlab CI # Author: Jozsef Makai (11.08.2017) #------------------------------------------------------------------------------- set -e comp=$1 prefix=/eos/project/s/storage-ci/www/debian/xrootd for dist in focal jammy; do echo "Publishing for $dist"; path=$prefix/pool/$dist/$comp/x/xrootd/; mkdir -p $path; if [[ "$comp" == "master" ]]; then find ${path} -type f -name '*deb' -delete; fi cp $dist/*deb $path; mkdir -p $prefix/dists/$dist/$comp/binary-amd64/; (cd $prefix && apt-ftparchive --arch amd64 packages pool/$dist/$comp/ > dists/$dist/$comp/binary-amd64/Packages); gzip -c $prefix/dists/$dist/$comp/binary-amd64/Packages > $prefix/dists/$dist/$comp/binary-amd64/Packages.gz; components=$(find $prefix/dists/$dist/ -mindepth 1 -maxdepth 1 -type d -exec basename {} \; | tr '\n' ' ') if [ -e $prefix/dists/$dist/Release ]; then rm $prefix/dists/$dist/Release fi if [ -e $prefix/dists/$dist/InRelease ]; then rm $prefix/dists/$dist/InRelease fi if [ -e $prefix/dists/$dist/Release.gpg ]; then rm $prefix/dists/$dist/Release.gpg fi apt-ftparchive -o APT::FTPArchive::Release::Origin=CERN -o APT::FTPArchive::Release::Label=XrootD -o APT::FTPArchive::Release::Codename=$dist -o APT::FTPArchive::Release::Architectures=amd64 -o APT::FTPArchive::Release::Components="$components" release $prefix/dists/$dist/ > $prefix/dists/$dist/Release; gpg --homedir /home/stci/ --clearsign -o $prefix/dists/$dist/InRelease $prefix/dists/$dist/Release; gpg --homedir /home/stci/ -abs -o $prefix/dists/$dist/Release.gpg $prefix/dists/$dist/Release; done xrootd-5.6.9/packaging/makesrpm.sh000077500000000000000000000174301457266313600171760ustar00rootroot00000000000000#!/bin/bash #------------------------------------------------------------------------------- # Create a source RPM package # Author: Lukasz Janyst (10.03.2011) #------------------------------------------------------------------------------- #------------------------------------------------------------------------------- # Print help #------------------------------------------------------------------------------- function printHelp() { echo "Usage:" 1>&2 echo "${0} [--help] [--source PATH] [--output PATH]" 1>&2 echo " --help prints this message" 1>&2 echo " --source PATH specify the root of the source tree" 1>&2 echo " defaults to ../" 1>&2 echo " --output PATH the directory where the source rpm" 1>&2 echo " should be stored, defaulting to ." 1>&2 echo " --version VERSION the version provided by user" 1>&2 echo " --define 'MACRO EXPR'" 1>&2 } #------------------------------------------------------------------------------- # Parse the commandline, if only we could use getopt... :( #------------------------------------------------------------------------------- SOURCEPATH=$(realpath $(dirname $0)/..) OUTPUTPATH="." PRINTHELP=0 while test ${#} -ne 0; do if test x${1} = x--help; then PRINTHELP=1 elif test x${1} = x--source; then if test ${#} -lt 2; then echo "--source parameter needs an argument" 1>&2 exit 1 fi SOURCEPATH=${2} shift elif test x${1} = x--output; then if test ${#} -lt 2; then echo "--output parameter needs an argument" 1>&2 exit 1 fi OUTPUTPATH=${2} shift elif test x${1} = x--version; then if test ${#} -lt 2; then echo "--version parameter needs an argument" 1>&2 exit 1 fi VERSION="${2}" shift elif test x${1} = x--define; then if test ${#} -lt 2; then echo "--define parameter needs an argument" 1>&2 exit 1 fi USER_DEFINE="$USER_DEFINE --define \""${2}"\"" shift elif test x${1} = x-D; then if test ${#} -lt 2; then echo "-D parameter needs an argument" 1>&2 exit 1 fi USER_D="$USER_D -D \""${2}"\"" fi shift done if test $PRINTHELP -eq 1; then printHelp exit 0 fi echo "[i] Working on: $SOURCEPATH" echo "[i] Storing the output to: $OUTPUTPATH" #------------------------------------------------------------------------------- # Check if the source and the output dirs #------------------------------------------------------------------------------- if test ! -d $SOURCEPATH -o ! -r $SOURCEPATH; then echo "[!] Source path does not exist or is not readable" 1>&2 exit 2 fi if test ! -d $OUTPUTPATH -o ! -w $OUTPUTPATH; then echo "[!] Output path does not exist or is not writeable" 1>&2 exit 2 fi #------------------------------------------------------------------------------- # Check if we have all the necassary components #------------------------------------------------------------------------------- if ! command -v rpmbuild 2>/dev/null; then echo "[!] Unable to find rpmbuild, aborting..." 1>&2 exit 1 fi if ! command -v git 2>/dev/null; then echo "[!] Unable to find git, aborting..." 1>&2 exit 1 fi #------------------------------------------------------------------------------- # Check if the source is a git repository #------------------------------------------------------------------------------- if test ! -d $SOURCEPATH/.git; then echo "[!] I can only work with a git repository" 1>&2 exit 2 fi : ${VERSION:=$(git --git-dir=$SOURCEPATH/.git describe)} : ${RELEASE:=1} if test $? -ne 0; then echo "[!] Unable to figure out the version number" 1>&2 exit 4 fi echo "[i] Working with version: $VERSION" #------------------------------------------------------------------------------- # Sanitize version to work with RPMs # https://docs.fedoraproject.org/en-US/packaging-guidelines/Versioning/ #------------------------------------------------------------------------------- VERSION=${VERSION#v} # remove "v" prefix VERSION=${VERSION/-rc/~rc} # release candidates use ~ in RPMs VERSION=${VERSION/-g*/} # snapshots versions not supported well, filter out VERSION=${VERSION/-/.post} # handle git describe for post releases VERSION=${VERSION//-/.} # replace remaining dashes with dots echo "[i] RPM compliant version: $VERSION-$RELEASE" #------------------------------------------------------------------------------- # Create a tempdir and copy the files there #------------------------------------------------------------------------------- # exit on any error set -e TEMPDIR=`mktemp -d /tmp/xrootd.srpm.XXXXXXXXXX` RPMSOURCES=$TEMPDIR/rpmbuild/SOURCES mkdir -p $RPMSOURCES mkdir -p $TEMPDIR/rpmbuild/SRPMS echo "[i] Working in: $TEMPDIR" 1>&2 if test -d rhel -a -r rhel; then for i in rhel/*; do cp $i $RPMSOURCES done fi if test -d common -a -r common; then for i in common/*; do cp $i $RPMSOURCES done fi #------------------------------------------------------------------------------- # Generate the spec file #------------------------------------------------------------------------------- if test ! -r rhel/xrootd.spec.in; then echo "[!] The specfile template does not exist!" 1>&2 exit 7 fi cat rhel/xrootd.spec.in | sed "s/__VERSION__/$VERSION/" | \ sed "s/__RELEASE__/$RELEASE/" > $TEMPDIR/xrootd.spec #------------------------------------------------------------------------------- # Make a tarball of the latest commit on the branch #------------------------------------------------------------------------------- # no more exiting on error set +e CWD=$PWD cd $SOURCEPATH COMMIT=`git log --pretty=format:"%H" -1` if test $? -ne 0; then echo "[!] Unable to figure out the git commit hash" 1>&2 exit 5 fi git describe --tags --abbrev=0 --exact-match >/dev/null 2>&1 if test ${?} -eq 0; then TAG="`git describe --tags --abbrev=0 --exact-match`" fi git archive --prefix=xrootd/ --format=tar $COMMIT > $RPMSOURCES/xrootd.tar if test $? -ne 0; then echo "[!] Unable to create the source tarball" 1>&2 exit 6 fi #------------------------------------------------------------------------------- # gzip the tarball #------------------------------------------------------------------------------- gzip -9fn $RPMSOURCES/xrootd.tar #------------------------------------------------------------------------------- # Check if we need some other versions #------------------------------------------------------------------------------- OTHER_VERSIONS=`cat $TEMPDIR/xrootd.spec | \ grep -E '^Source[0-9]+:[[:space:]]*xrootd-.*.gz$' |\ awk '{ print $2; }'` for VER in $OTHER_VERSIONS; do VER=${VER/xrootd-/} VER=${VER/.tar.gz/} git archive --prefix=xrootd-$VER/ --format=tar v$VER | gzip -9fn > \ $RPMSOURCES/xrootd-$VER.tar.gz if test $? -ne 0; then echo "[!] Unable to create the source tarball" 1>&2 exit 9 fi done cd $CWD #------------------------------------------------------------------------------- # Build the source RPM #------------------------------------------------------------------------------- echo "[i] Creating the source RPM..." # Dirty, dirty hack! echo "%_sourcedir $RPMSOURCES" >> $TEMPDIR/rpmmacros eval "rpmbuild --define \"_topdir $TEMPDIR/rpmbuild\" \ --define \"%_sourcedir $RPMSOURCES\" \ --define \"%_srcrpmdir %{_topdir}/SRPMS\" \ --define \"_source_filedigest_algorithm md5\" \ --define \"_binary_filedigest_algorithm md5\" \ ${USER_DEFINE} \ ${USER_D} \ -bs $TEMPDIR/xrootd.spec > $TEMPDIR/log" if test $? -ne 0; then echo "[!] RPM creation failed" 1>&2 exit 8 fi cp $TEMPDIR/rpmbuild/SRPMS/xrootd*.src.rpm $OUTPUTPATH rm -rf $TEMPDIR echo "[i] Done." xrootd-5.6.9/packaging/rhel/000077500000000000000000000000001457266313600157455ustar00rootroot00000000000000xrootd-5.6.9/packaging/rhel/cmsd.init000066400000000000000000000014511457266313600175610ustar00rootroot00000000000000#!/bin/sh # # /etc/init.d/cmsd - Start/stop the cmsd service # # The following two lines allow this script to be managed by Fedora's # chkconfig program. # # chkconfig: - 80 30 # description: cmsd is a manager for the xrootd cluster file system. # Source function library. . /etc/rc.d/init.d/xrootd.functions if [ -e /etc/sysconfig/xrootd ]; then . /etc/sysconfig/xrootd fi COMMAND=$1 shift case "$COMMAND" in 'start') handleDaemons start cmsd $@ ;; 'stop') handleDaemons stop cmsd $@ ;; 'status') handleDaemons status cmsd $@ ;; 'reload' | 'restart') handleDaemons stop cmsd $@ handleDaemons start cmsd $@ ;; 'condrestart') handleDaemons condrestart cmsd $@ ;; *) echo "usage: $0 {start|stop|status|restart|condrestart}" ;; esac exit $? xrootd-5.6.9/packaging/rhel/frm_purged.init000066400000000000000000000015331457266313600207660ustar00rootroot00000000000000#!/bin/sh # # /etc/init.d/frm_purged - Start/stop the frm_purged service # # The following two lines allow this script to be managed by Fedora's # chkconfig program. # # chkconfig: - 80 30 # description: frm_purged manages cache eviction in the xrootd system. # Source function library. . /etc/rc.d/init.d/xrootd.functions if [ -e /etc/sysconfig/xrootd ]; then . /etc/sysconfig/xrootd fi COMMAND=$1 shift case "$COMMAND" in 'start') handleDaemons start frm_purged $@ ;; 'stop') handleDaemons stop frm_purged $@ ;; 'status') handleDaemons status frm_purged $@ ;; 'reload' | 'restart') handleDaemons stop frm_purged $@ handleDaemons start frm_purged $@ ;; 'condrestart') handleDaemons condrestart frm_purged $@ ;; *) echo "usage: $0 {start|stop|status|restart|condrestart}" ;; esac exit $? xrootd-5.6.9/packaging/rhel/frm_xfrd.init000066400000000000000000000015071457266313600204440ustar00rootroot00000000000000#!/bin/sh # # /etc/init.d/frm_xfrd - Start/stop the frm_xfrd service # # The following two lines allow this script to be managed by Fedora's # chkconfig program. # # chkconfig: - 80 30 # description: frm_xfrd is a caching daemon for the xrootd system. # Source function library. . /etc/rc.d/init.d/xrootd.functions if [ -e /etc/sysconfig/xrootd ]; then . /etc/sysconfig/xrootd fi COMMAND=$1 shift case "$COMMAND" in 'start') handleDaemons start frm_xfrd $@ ;; 'stop') handleDaemons stop frm_xfrd $@ ;; 'status') handleDaemons status frm_xfrd $@ ;; 'reload' | 'restart') handleDaemons stop frm_xfrd $@ handleDaemons start frm_xfrd $@ ;; 'condrestart') handleDaemons condrestart frm_xfrd $@ ;; *) echo "usage: $0 {start|stop|status|restart|condrestart}" ;; esac exit $? xrootd-5.6.9/packaging/rhel/xrootd.functions000066400000000000000000000212421457266313600212170ustar00rootroot00000000000000#------------------------------------------------------------------------------- # Library for handling xrootd daemons on RHEL # Author: Lukasz Janyst (09.03.2011) #------------------------------------------------------------------------------- . /etc/rc.d/init.d/functions #------------------------------------------------------------------------------- # Add the user accounts if needed and set up the directory ownership #------------------------------------------------------------------------------- function setupInstallation() { XROOTD_USER=$1 XROOTD_GROUP=$2 getent group $XROOTD_GROUP >/dev/null || groupadd -r $XROOTD_GROUP getent passwd $XROOTD_USER >/dev/null || \ useradd -r -g $XROOTD_GROUP -c "XRootD runtime user" \ -s /sbin/nologin -d /var/spool/xrootd $XROOTD_USER chown $XROOTD_USER:$XROOTD_GROUP -R /var/spool/xrootd chown $XROOTD_USER:$XROOTD_GROUP -R /var/log/xrootd } #------------------------------------------------------------------------------- # Check if a file has the right permissions #------------------------------------------------------------------------------- function checkFile() { FILE=$1 XROOTD_USER=$2 XROOTD_GROUP=$3 if test x"`stat -c %U $FILE`" != x$XROOTD_USER; then echo "$FILE has wrong user ownership: \"`stat -c %U $FILE`\" instead of \"$XROOTD_USER\"" return 1 fi } #------------------------------------------------------------------------------- # Check a directory and it's permissions #------------------------------------------------------------------------------- function checkDirectory() { DIRECTORY=$1 XROOTD_USER=$2 XROOTD_GROUP=$3 if test ! -d $DIRECTORY; then echo "Directory: $DIRECTORY does not exist" return 1 fi for i in `find $DIRECTORY`; do checkFile $i $XROOTD_USER $XROOTD_GROUP if test $? -ne 0; then return 1 fi done return 0 } #------------------------------------------------------------------------------- # Check if the installation is in a sane state #------------------------------------------------------------------------------- function checkSanity() { XROOTD_USER=$1 XROOTD_GROUP=$2 #----------------------------------------------------------------------------- # Check if the user account exist #----------------------------------------------------------------------------- getent passwd $XROOTD_USER >/dev/null if test $? -ne 0; then echo "User account for: $XROOTD_USER doesn't exist" return 1 fi getent group $XROOTD_GROUP >/dev/null if test $? -ne 0; then echo "Group account for: $XROOTD_GROUP doesn't exist" return 2 fi #----------------------------------------------------------------------------- # We need these directories to be owned by the xroot user for the init # scripts to work properly, and we can safely change the ownership if # it is wrong. #----------------------------------------------------------------------------- checkDirectory /var/spool/xrootd $XROOTD_USER $XROOTD_GROUP if test $? -ne 0; then chown $XROOTD_USER:$XROOTD_GROUP -R /var/spool/xrootd fi mkdir -p /run/xrootd chown $XROOTD_USER:$XROOTD_GROUP -R /run/xrootd checkDirectory /run/xrootd $XROOTD_USER $XROOTD_GROUP if test $? -ne 0; then chown $XROOTD_USER:$XROOTD_GROUP -R /run/xrootd fi } #------------------------------------------------------------------------------- # Start a daemon #------------------------------------------------------------------------------- function startDaemon() { ulimit -n 65536 DAEMON=$1 EX=$2 XROOTD_USER=$3 XROOTD_GROUP=$4 INSTANCE=$5 PIDFILE="/run/xrootd/$DAEMON-$INSTANCE.pid" # check sanity of the installation checkSanity $XROOTD_USER $XROOTD_GROUP if test $? -ne 0; then echo "Please run: service xrootd setup" return 1 fi echo -n "Starting xrootd ($DAEMON, $INSTANCE): " shift 5 # useful for storing coredumps cd /var/spool/xrootd daemon --user $XROOTD_USER --pidfile $PIDFILE $EX $@ -b -s $PIDFILE -n $INSTANCE RETVAL=$? echo return $RETVAL } #------------------------------------------------------------------------------- # Stop a daemon #------------------------------------------------------------------------------- function stopDaemon() { echo -n "Shutting down xrootd ($1, $5): " PIDFILE="/run/xrootd/$1-$5.pid" if [ -e $PIDFILE ]; then killproc -p $PIDFILE $1 RETVAL=$? echo return $RETVAL fi echo -n "$1-$5 not running" echo return 0 } #------------------------------------------------------------------------------- # Get the status of a daemon #------------------------------------------------------------------------------- function statusOfTheDaemon() { PIDFILE="/run/xrootd/$1-$5.pid" echo -n "[$5] " status -p $PIDFILE $1 return $RETVAL } #------------------------------------------------------------------------------- # Conditionally restart a daemon #------------------------------------------------------------------------------- function condrestartDaemon() { PIDFILE="/run/xrootd/$1-$5.pid" status -p $PIDFILE $1 > /dev/null if test $? -ne 0; then return 0 fi stopDaemon $@ if test $? -ne 0; then return 1 fi startDaemon $@ if test $? -ne 0; then return 2 fi return 0 } #------------------------------------------------------------------------------- # Do things to daemons #------------------------------------------------------------------------------- function handleDaemons() { #----------------------------------------------------------------------------- # Check if the user account is specified #----------------------------------------------------------------------------- if test x"$XROOTD_USER" = x; then XROOTD_USER="daemon" fi if test x"$XROOTD_GROUP" = x; then XROOTD_GROUP="daemon" fi #----------------------------------------------------------------------------- # Determine the command to be run #----------------------------------------------------------------------------- COMMAND=$1; shift case "$COMMAND" in 'start') CMD_HANDLER=startDaemon ;; 'stop') CMD_HANDLER=stopDaemon ;; 'status') CMD_HANDLER=statusOfTheDaemon ;; 'condrestart') CMD_HANDLER=condrestartDaemon ;; 'setup') setupInstallation $XROOTD_USER $XROOTD_GROUP return $? ;; *) echo "Unrecognized command: $COMMAND" return 1 ;; esac #----------------------------------------------------------------------------- # Select the daemon to be started #----------------------------------------------------------------------------- DAEMON=$1; shift case "$DAEMON" in 'xrootd' | 'cmsd') EXEC=/usr/bin/$DAEMON CONFIG_NAME=`echo $DAEMON | tr '[[:lower:]]' '[[:upper:]]'` ;; 'frm_purged') EXEC=/usr/bin/$DAEMON CONFIG_NAME=PURD ;; 'frm_xfrd') EXEC=/usr/bin/$DAEMON CONFIG_NAME=XFRD ;; *) echo "Unrecognized daemon: $DAEMON" return 1 ;; esac #----------------------------------------------------------------------------- # Select the instances to run #----------------------------------------------------------------------------- if test $# -eq 0; then eval INSTANCES=\$${CONFIG_NAME}_INSTANCES else INSTANCES=$@ fi eval INSTANCES_XROOTD3=\$${CONFIG_NAME}3_INSTANCES INSTANCES=`echo $INSTANCES | tr '[[:upper:]]' '[[:lower:]]'` INSTANCES_XROOTD3=`echo $INSTANCES_XROOTD3 | tr '[[:upper:]]' '[[:lower:]]'` #----------------------------------------------------------------------------- # Exec the command on the instances #----------------------------------------------------------------------------- STATUS=0 for INSTANCE in $INSTANCES; do INSTANCE_UPPER=`echo $INSTANCE | tr '[[:lower:]]' '[[:upper:]]'` eval OPTS=\$${CONFIG_NAME}_${INSTANCE_UPPER}_OPTIONS #--------------------------------------------------------------------------- # Check if the instance is xrootd3 or xrootd4, if a xrootd3 binary is # installed #--------------------------------------------------------------------------- EXEC_RUN=$EXEC if test -x $EXEC-3; then for INS in $INSTANCES_XROOTD3; do if test $INS = $INSTANCE; then EXEC_RUN=$EXEC-3 fi done fi #--------------------------------------------------------------------------- # Run the instance #--------------------------------------------------------------------------- if test x"$OPTS" = x; then continue fi $CMD_HANDLER $DAEMON $EXEC_RUN "$XROOTD_USER" "$XROOTD_GROUP" $INSTANCE "$OPTS" STATUS=$? if test $? -ne 0 && "$CMD_HANDLER" != "statusOfTheDaemon"; then STATUS=1 fi done return $STATUS } xrootd-5.6.9/packaging/rhel/xrootd.functions-slc4000066400000000000000000000204031457266313600220600ustar00rootroot00000000000000#------------------------------------------------------------------------------- # Library for handling xrootd daemons on antique RHEL # Author: Lukasz Janyst (18.03.2011) #------------------------------------------------------------------------------- . /etc/rc.d/init.d/functions #------------------------------------------------------------------------------- # Add the user accounts if needed and set up the directory ownership #------------------------------------------------------------------------------- function setupInstallation() { XROOTD_USER=$1 XROOTD_GROUP=$2 getent group $XROOTD_GROUP >/dev/null || groupadd -r $XROOTD_GROUP getent passwd $XROOTD_USER >/dev/null || \ useradd -r -g $XROOTD_GROUP -c "XRootD runtime user" \ -s /sbin/nologin -d /var/spool/xrootd $XROOTD_USER chown $XROOTD_USER:$XROOTD_GROUP -R /var/spool/xrootd chown $XROOTD_USER:$XROOTD_GROUP -R /var/log/xrootd } #------------------------------------------------------------------------------- # Check if a file has the right permissions #------------------------------------------------------------------------------- function checkFile() { FILE=$1 XROOTD_USER=$2 XROOTD_GROUP=$3 if test x"`stat -c %U $FILE`" != x$XROOTD_USER; then echo "$FILE has wrong user ownership: \"`stat -c %U $FILE`\" instead of \"$XROOTD_USER\"" return 1 fi } #------------------------------------------------------------------------------- # Check a directory and it's permissions #------------------------------------------------------------------------------- function checkDirectory() { DIRECTORY=$1 XROOTD_USER=$2 XROOTD_GROUP=$3 if test ! -d $DIRECTORY; then echo "Directory: $DIRECTORY does not exist" return 1 fi for i in `find $DIRECTORY`; do checkFile $i $XROOTD_USER $XROOTD_GROUP if test $? -ne 0; then return 1 fi done return 0 } #------------------------------------------------------------------------------- # Check if the installation is in a sane state #------------------------------------------------------------------------------- function checkSanity() { XROOTD_USER=$1 XROOTD_GROUP=$2 #----------------------------------------------------------------------------- # Check if the user account exist #----------------------------------------------------------------------------- getent passwd $XROOTD_USER >/dev/null if test $? -ne 0; then echo "User account for: $XROOTD_USER doesn't exist" return 1 fi getent group $XROOTD_GROUP >/dev/null if test $? -ne 0; then echo "Group account for: $XROOTD_GROUP doesn't exist" return 2 fi #----------------------------------------------------------------------------- # We need these directories to be owned by the xroot user for the init # scripts to work properly, and we can safely change the ownership if # it is wrong. #----------------------------------------------------------------------------- checkDirectory /var/spool/xrootd $XROOTD_USER $XROOTD_GROUP if test $? -ne 0; then chown $XROOTD_USER:$XROOTD_GROUP -R /var/spool/xrootd fi mkdir -p /run/xrootd chown $XROOTD_USER:$XROOTD_GROUP -R /run/xrootd checkDirectory /run/xrootd $XROOTD_USER $XROOTD_GROUP if test $? -ne 0; then chown $XROOTD_USER:$XROOTD_GROUP -R /run/xrootd fi } #------------------------------------------------------------------------------- # Start a daemon #------------------------------------------------------------------------------- function startDaemon() { ulimit -n 65536 DAEMON=$1 EXEC=$2 XROOTD_USER=$3 XROOTD_GROUP=$4 INSTANCE=$5 PIDFILE="/run/xrootd/$DAEMON-$INSTANCE.pid" # check sanity of the installation checkSanity $XROOTD_USER $XROOTD_GROUP if test $? -ne 0; then echo "Please run: service xrootd setup" return 1 fi echo -n "Starting xrootd ($DAEMON, $INSTANCE): " statusOfTheDaemon $@ > /dev/null if test $? -ne 0; then shift 5 # change the CWD to have some room for core dumps cd /var/spool/xrootd daemon --user $XROOTD_USER $EXEC $@ -b -s $PIDFILE -n $INSTANCE fi RETVAL=$? echo return $RETVAL } #------------------------------------------------------------------------------- # Stop a daemon #------------------------------------------------------------------------------- function stopDaemon() { echo -n "Shutting down xrootd ($1, $5): " PIDFILE="/run/xrootd/$1-$5.pid" if test -r "$PIDFILE"; then PID=`cat "$PIDFILE"` INST=`ps aux | grep $2 | grep $PID | grep $5` if test x"$INST" != x; then kill -TERM "$PID" && success || failure rm -f $PIDFILE else failure fi else failure fi RETVAL=$? echo return $RETVAL } #------------------------------------------------------------------------------- # Get the status of a daemon #------------------------------------------------------------------------------- function statusOfTheDaemon() { PIDFILE="/run/xrootd/$1-$5.pid" echo -n "[$5] " if test -r "$PIDFILE"; then PID=`cat $PIDFILE` INST=`ps aux | grep $2 | grep $PID | grep $5` if test x"$INST" != x; then echo "$1 (pid $PID) is running..." return 0 else echo "$1 is stopped" return 1 fi else echo "$1 is stopped" return 1 fi return 0 } #------------------------------------------------------------------------------- # Conditionally restart a daemon #------------------------------------------------------------------------------- function condrestartDaemon() { statusOfTheDaemon $@ > /dev/null if test $? -ne 0; then return 0 fi stopDaemon $@ if test $? -ne 0; then return 1 fi startDaemon $@ if test $? -ne 0; then return 2 fi return 0 } #------------------------------------------------------------------------------- # Do things to daemons #------------------------------------------------------------------------------- function handleDaemons() { #----------------------------------------------------------------------------- # Check if the user account is specified #----------------------------------------------------------------------------- if test x"$XROOTD_USER" = x; then XROOTD_USER="daemon" fi if test x"$XROOTD_GROUP" = x; then XROOTD_GROUP="daemon" fi #----------------------------------------------------------------------------- # Determine the command to be run #----------------------------------------------------------------------------- COMMAND=$1; shift case "$COMMAND" in 'start') CMD_HANDLER=startDaemon ;; 'stop') CMD_HANDLER=stopDaemon ;; 'status') CMD_HANDLER=statusOfTheDaemon ;; 'condrestart') CMD_HANDLER=condrestartDaemon ;; 'setup') setupInstallation $XROOTD_USER $XROOTD_GROUP return $? ;; *) echo "Unrecognized command: $COMMAND" return 1 ;; esac #----------------------------------------------------------------------------- # Select the daemon to be started #----------------------------------------------------------------------------- DAEMON=$1; shift case "$DAEMON" in 'xrootd' | 'cmsd') EXEC=/usr/bin/$DAEMON CONFIG_NAME=`echo $DAEMON | tr '[[:lower:]]' '[[:upper:]]'` ;; 'frm_purged' | 'frm_xfrd') EXEC=/usr/bin/$DAEMON CONFIG_NAME=FRMD ;; *) echo "Unrecognized daemon: $DAEMON" return 1 ;; esac #----------------------------------------------------------------------------- # Select the instances to run #----------------------------------------------------------------------------- if test $# -eq 0; then eval INSTANCES=\$${CONFIG_NAME}_INSTANCES else INSTANCES=$@ fi INSTANCES=`echo $INSTANCES | tr '[[:upper:]]' '[[:lower:]]'` #----------------------------------------------------------------------------- # Exec the command on the instances #----------------------------------------------------------------------------- STATUS=0 for INSTANCE in $INSTANCES; do INSTANCE_UPPER=`echo $INSTANCE | tr '[[:lower:]]' '[[:upper:]]'` eval OPTS=\$${CONFIG_NAME}_${INSTANCE_UPPER}_OPTIONS if test x"$OPTS" = x; then continue fi $CMD_HANDLER $DAEMON $EXEC "$XROOTD_USER" "$XROOTD_GROUP" $INSTANCE "$OPTS" if test $? -ne 0; then STATUS=1 fi done return $STATUS } xrootd-5.6.9/packaging/rhel/xrootd.init000066400000000000000000000015351457266313600201550ustar00rootroot00000000000000#!/bin/sh # # /etc/init.d/xrootd - Start/stop the xrootd services # # The following two lines allow this script to be managed by Fedora's # chkconfig program. # # chkconfig: - 80 30 # description: xrootd is a cluster file system. # Source function library. . /etc/rc.d/init.d/xrootd.functions if [ -e /etc/sysconfig/xrootd ]; then . /etc/sysconfig/xrootd fi COMMAND=$1 shift case "$COMMAND" in 'start') handleDaemons start xrootd $@ ;; 'stop') handleDaemons stop xrootd $@ ;; 'status') handleDaemons status xrootd $@ ;; 'reload' | 'restart') handleDaemons stop xrootd $@ handleDaemons start xrootd $@ ;; 'condrestart') handleDaemons condrestart xrootd $@ ;; 'setup') handleDaemons setup xrootd ;; *) echo "usage: $0 {start|stop|status|restart|condrestart|setup}" ;; esac exit $? xrootd-5.6.9/packaging/rhel/xrootd.sysconfig000066400000000000000000000066411457266313600212210ustar00rootroot00000000000000#------------------------------------------------------------------------------- # Define the instances of xrootd, cmsd and frmd here and specify the option you # need. For example, use the -d flag to send debug output to the logfile, # the options responsible for daemonizing, pidfiles and instance naming will # be appended automatically. #------------------------------------------------------------------------------- #------------------------------------------------------------------------------- # Define the user account name which will be used to start the daemons. # These may have many unexpected side effects, so be sure you know what you're # doing before playing with them. #------------------------------------------------------------------------------- XROOTD_USER=xrootd XROOTD_GROUP=xrootd #------------------------------------------------------------------------------- # Define the commandline options for the instances of the daemons. # The format is: # DAEMON_NAME_OPTIONS, where: # DAEMON - the daemon name, the valid values are: XROOTD, CMSD, XFRD and PURD # NAME - the name of the instance, any uppercase alphanumeric string # without whitespaces is valid #------------------------------------------------------------------------------- XROOTD_DEFAULT_OPTIONS="-l /var/log/xrootd/xrootd.log -c /etc/xrootd/xrootd-clustered.cfg -k fifo" #XROOTD_DEFAULT_OPTIONS="-l /var/log/xrootd/xrootd.log -c /etc/xrootd/xrootd-standalone.cfg -k fifo" CMSD_DEFAULT_OPTIONS="-l /var/log/xrootd/cmsd.log -c /etc/xrootd/xrootd-clustered.cfg -k fifo" PURD_DEFAULT_OPTIONS="-l /var/log/xrootd/purged.log -c /etc/xrootd/xrootd-clustered.cfg -k fifo" XFRD_DEFAULT_OPTIONS="-l /var/log/xrootd/xfrd.log -c /etc/xrootd/xrootd-clustered.cfg -k fifo" #------------------------------------------------------------------------------- # Names of the instances to be started by default, the case doesn't matter, # the names will be converted to lowercase automatically, use space as a # separator #------------------------------------------------------------------------------- XROOTD_INSTANCES="default" CMSD_INSTANCES="default" PURD_INSTANCES="default" XFRD_INSTANCES="default" #------------------------------------------------------------------------------- # Names of the instances that should run XRootD 3 versions of the executables. # By default XRootD 4 versions are run, however it is possible to run XRootD 3 # if the appropriate compat packages are installed. #------------------------------------------------------------------------------- #XROOTD3_INSTANCES="default" #CMSD3_INSTANCES="default" #PURD3_INSTANCES="default" #XFRD3_INSTANCES="default" #------------------------------------------------------------------------------- # Use a custom memory allocator when the one provided with the Standard C # Library (glibc) is eating up too much system memory. This is almost always the # case for RHEL >= 6, so you may consider installing jemalloc or tcmalloc and # using one of them instead. # # In order to do that, install either one of the packages: # # yum install jemalloc # for jemalloc # or # yum install gperftools-libs # for tcmalloc # # and uncomment one of the following lines. You may need to adjust the library # paths depending on the type of your system. #------------------------------------------------------------------------------- #export LD_PRELOAD=/usr/lib64/libjemalloc.so.1 #export LD_PRELOAD=/usr/lib64/libtcmalloc.so.4 xrootd-5.6.9/packaging/rhel/xrootd.tmpfiles000066400000000000000000000000401457266313600210230ustar00rootroot00000000000000d /run/xrootd - xrootd xrootd - xrootd-5.6.9/packaging/tgz/000077500000000000000000000000001457266313600156175ustar00rootroot00000000000000xrootd-5.6.9/packaging/tgz/README000066400000000000000000000037731457266313600165110ustar00rootroot00000000000000 $Id$ This directory contains start and stop scripts for the cmsd, olbd (deprecated) and xrootd, as well as a monitoring script for the cmsd when you wish to use load balanced clustering. Please note that the cmsd and olbd provide the same services but you must use one or the other everywhere. You may not mix them. The cmsd is the prefered daemon (the olbd is provided for backward compatibility). The documentation for each script is contained within the script itself. Please read the header comments in each script before using the script! The StartCMS script starts the cmsd and the StartXRD script starts xrootd. Both scripts must have a StartXRD.cf file installed in the same directory as the start script. This distribution includes a StartXRD.cf.example file to guide you through setting the various variables to the appropriate values. Please create your own StartXRD.cf file from this example file. The StartXRD.cf.example and the xrootd.cf.example files provided here use simple defaults, which allow the xrootd daemon to start locally. They are useful to have something to start with and to get inspiration about the envvar usage. To start a cmsd daemon, all you need to do is to is to fill in the name of the manager host in the places where it should go inside the xrootd.cf file. The default configuration files is provided as an easy way to have something trivial which starts immediately after having compiled the package. These scripts can also be used to set up a full cluster, but may be not adequate for particularily complex installations. In this case, please refer to the example scripts and to the documentation for support. You need not use the provided start/stop scripts. However, they are provided here as a helpful aid in managing the servers. The StartXrd.cf.example is the simple one that shoud be good enough to get you going in less that 15 minutes! Don't forget to modify your init scripts to start the servers at boot time! Full documentation can be found at http://xrootd.slac.stanford.edu/ xrootd-5.6.9/packaging/tgz/StartCMS000077500000000000000000000107561457266313600172160ustar00rootroot00000000000000#!/bin/sh # # $Id$ # # (C) 2005 by the Board of Trustees of the Leland Stanford, Jr., University # All Rights Reserved # Produced by Andrew Hanushevsky for Stanford University under contract # DE-AC03-76-SFO0515 with the Deprtment of Energy # Modified by Fabrizio Furano 12/7/2007 # Syntax: StartCMS [-c cfile] [-D] [-t] [-v] [oth] # Where: -c specifies the configuration file to be used. # -D turns on internal debugging. # -t types the commands and does not execute them # -v produces verbose output. # oth Any other options to be passed to the executable. # # Set default values from the StartXRD.cf file # . `dirname $0`/StartXRD.cf VERBOSE=0 # Set TEST to equal /bin/echo to only display lines to be executed # TEST= umask 002 ############################################################################## # s u b r o u t i n e s # ############################################################################## Debug () { if test $VERBOSE -eq 1; then echo $1 fi } MustExist () { Debug "Checking existence of $1 $3 ..." if test ! -${2} $3 ; then Notify "$1 $3 not found." fi } Writable () { Debug "Checking writability of $1 $2 ..." if test ! -w $2 ; then Notify "$1 $2 is not writable by $MYNAME." fi } Wait4File () { Debug "Checking file $2 ..." tcnt=$count until test -${1} $2 -o $tcnt -eq 0; do echo StartCMS: File $2 not found\; waiting $time seconds... sleep $time tcnt=`/usr/bin/expr $tcnt - 1` done if [ $tcnt -le 0 ]; then Notify "File $2 not found." fi } Notify () { echo StartCMS: $1 exit 4 } ############################################################################## # m a i n p r o g r a m # ############################################################################## # Pick up options # CMSOPTS= CMSPARM=$* while test -n "$1"; do if [ "$1" = "-c" ]; then if [ -z "$2" ]; then Notify "Configuration file not specified." fi if [ ! -r "$2" ]; then Notify "Configuration file '$2' not found." fi CMSCONFIGFN=$2 shift elif [ "$1" = "-D" ]; then set -x elif [ "$1" = "-t" ]; then TEST=echo elif [ "$1" = "-v" ]; then VERBOSE=1 else CMSOPTS="$CMSOPTS $1" fi shift done # Establish location of the CMS executables # XRDBASE=`(cd $XRDBASE; pwd)` Debug "XRDBASE has been set to '$XRDBASE'" if [ $? != 0 ]; then exit 4 fi CMSPROG=$XRDBASE/bin/$XRDARCH/cmsd XRDLIBBASE=$XRDBASE/lib/$XRDARCH PROGRAM=$XRDCFG/$PROGRAM if [ "$CMSCONFIGFN" = "" ]; then CMSCONFIGFN=$XRDCFG/$XRDCONFIG fi # Establish log file name # CMSLOGFILE=$XRDLOGDIR/$CMSLOGFN # Verify that we are not executing this script as root (if we are, switch) # if [ $MYNAME = root ]; then if [ $XRDUSER != root ]; then $TEST exec su $XRDUSER -c "$PROGRAM $CMSPARM" fi elif [ $MYNAME != $XRDUSER ]; then Notify "Attempt to start $CMSUSER cmsd as user $MYNAME." fi # Just in case we don't have the basic directories, try to create them # $TEST mkdir -m 0744 -p $XRDLOGDIR $CMSHOMEDIR 2> /dev/null # Verify that all required directories are present # for FN in $XRDBASE $XRDLOGDIR $CMSHOMEDIR do MustExist Directory d $FN done # Verify that all owned directories are writable # for FN in $XRDLOGDIR $CMSHOMEDIR do Writable Directory $FN done # Verify that all required readable files are present # for FN in $XRDCFG do Wait4File r $FN done # Verify that all required executable files are present # XLIST="$CMSPROG" for FN in $XLIST do Wait4File x $FN done # Export the variables required in the config file # $TEST export XRDCFG $TEST export XRDBASE $TEST export XRDLIBBASE # Set appropriate limits # $TEST ulimit -n $MAXFD $TEST ulimit -c unlimited # Add our "lib" directory to LD_LIBRARY_PATH # CMSLIBBASE=$XRDBASE/lib/$XRDARCH if [ -z "$LD_LIBRARY_PATH" ]; then LD_LIBRARY_PATH=$CMSLIBBASE else LD_LIBRARY_PATH=$CMSLIBBASE:$LD_LIBRARY_PATH fi export LD_LIBRARY_PATH # Now start the daemon # echo Starting cmsd ... $TEST cd $CMSHOMEDIR $TEST $CMSPROG $CMSOPTS -l $CMSLOGFILE -c $CMSCONFIGFN & stat=$? # Check if we were successful # if test $stat -gt 0 ; then Notify "$CMSPROG returned a status of ${stat}." fi xrootd-5.6.9/packaging/tgz/StartFRM000077500000000000000000000104441457266313600172120ustar00rootroot00000000000000#!/bin/sh # Usage: StartFRM [-d] [-f] [-p] [-t] [-x] # Where: -d turns on debugging # -f forces the start irrespective of the hold file or status, if any. # -p starts only the purge daemon # -t types the start commands and does not execute them # -x starts only the transfer daemon # No option cause everything to be started. # Set defaults #******************************************************************************# #* D e f a u l t s *# #******************************************************************************# TEST= EXITRC=0 FORCE= DOALL=1 ONLYP=0 ONLYX=0 # Source he config file to set additional options # . `dirname $0`/StartXRD.cf #******************************************************************************# #* S u b r o u t i n e s *# #******************************************************************************# Emsg () { echo StartFRM: $1 } isOK () { thePGM=$1 theHLD=$2 if [ -z "$1" -o -z "$2" ]; then REASON="$1 not configured." return 0; fi if [ ! -x $1 ]; then REASON="$1 missing." return 0 fi if [ ! -z "$FORCE" ]; then rm -f $theHLD elif [ -f $theHLD ]; then REASON="$theHLD exists." return 0 fi xPID $thePGM if [ $? -ne 0 ]; then return 0 fi return 1 } xPID () { pidPGM=$1 set -- '' set -- `ps -e -o pid -o args | grep $pidPGM | awk '$0 !~ /grep/ {print $1}'` if [ -z "$1" ]; then return 0 else REASON="already running as pid $1." fi return 1 } Xeq () { $TEST $* & if [ $? -ne 0 ]; then EXITRC=1 fi } #******************************************************************************# #* M a i n *# #******************************************************************************# # Pick up options # XRDPARMS=$* while test -n "$1"; do if [ "$1" = "-d" ]; then set -x OPTS="-d $OPTS" elif [ "$1" = "-f" ]; then FORCE=1 elif [ "$1" = "-p" ]; then DOALL=0 ONLYP=1 elif [ "$1" = "-t" ]; then TEST=echo elif [ "$1" = "-x" ]; then DOALL=0 ONLYX=1 else Emsg "Invalid option - $1." echo 'Usage: StartFRM [-d] [-f] [-p] [-t] [-x]' exit 1 fi shift done # Determine hostname and username # MYHOST=`/bin/hostname` if [ -x /usr/bin/whoami ]; then MYNAME=`/usr/bin/whoami` elif [ -x /usr/ucb/whoami ]; then MYNAME=`/usr/ucb/whoami` else Emsg "Unable to determine username." exit 1 fi # See if we should su to the right user for the frm component. On Solaris # /bin/su doesn't forward the current environment variables and one has to # su to the XRDUSER and run the Start script again. PROGRAM=$XRDCFG/$PROGRAM if [ $MYNAME = root ]; then if [ $XRDUSER != root ]; then $TEST export LD_LIBRARY_PATH $TEST exec su $XRDUSER -c "$PROGRAM $XRDPARMS" fi elif [ $MYNAME != $XRDUSER ]; then Emsg "$MYNAME cannot start $XRDUSER FRM subsystems!" fi # Now set the LD_LIBRARY_PATH variable # if [ -z "$LD_LIBRARY_PATH" ]; then LD_LIBRARY_PATH=$LDLIBPATH else LD_LIBRARY_PATH=$LDLIBPATH:$LD_LIBRARY_PATH fi export LD_LIBRARY_PATH # Now start up the purge daemon # if [ ! -z "$START_PURGE" -a "$DOALL" = "1" -o "$ONLYP" = "1" ]; then isOK $START_PURGE $HFILE_PURGE if [ $? -eq 1 ]; then Emsg "Starting $XRDUSER purge daemon. . ." $TEST cd $HOMED_PURGE Xeq "$START_PURGE $OPTS $PARMS_PURGE" else Emsg "$XRDUSER purge daemon cannot be started on $MYHOST; $REASON" EXITRC=1 fi fi # Now start up the transfer daemon # if [ "$DOALL" = "1" -o "$ONLYX" = "1" ]; then isOK $START_TRANSFER $HFILE_TRANSFER if [ $? -eq 1 ]; then Emsg "Starting $XRDUSER transfer daemon. . ." $TEST cd $HOMED_TRANSFER Xeq "$START_TRANSFER $OPTS $PARMS_TRANSFER" else Emsg "$XRDUSER transfer daemon cannot be started on $MYHOST; $REASON" EXITRC=1 fi fi exit $EXITRC xrootd-5.6.9/packaging/tgz/StartOLB000077500000000000000000000124541457266313600172050ustar00rootroot00000000000000#!/bin/sh # # $Id$ # # (C) 2005 by the Board of Trustees of the Leland Stanford, Jr., University # All Rights Reserved # Produced by Andrew Hanushevsky for Stanford University under contract # DE-AC03-76-SFO0515 with the Deprtment of Energy # Syntax: StartOLB [-c cfile] [-d] [-D] [-m] [-s] [oth] [-v] [ver] # Where: -c specifies the configuration file to be used. # -d passes the -d option to olbd # -D turns on internal debugging. # -m specifies manager mode (default in StartOLB.cf) # -s specifies server mode (default in StartOLB.cf) # -t types the commands and does not execute them # -v produces verbose output. # oth Any other single letter options to be passed to the executable. # ver is version number to use. This is the string that is put in # $OLBBASE//bin, the default is prod. # # Set default values # . $0.cf VERBOSE=0 # Set TEST to equal /bin/echo to only display lines to be executed # TEST= umask 002 ############################################################################## # s u b r o u t i n e s # ############################################################################## Debug () { if test $VERBOSE -eq 1; then /bin/echo $1 fi } MustExist () { Debug "Checking existence of $1 $3 ..." if test ! -${2} $3 ; then Notify "$1 $3 not found." fi } Writable () { Debug "Checking writability of $1 $2 ..." if test -r $2 ; then if test ! -w $2 ; then Notify "$1 $2 is not writable by $MYNAME." fi fi } Wait4File () { Debug "Checking file $2 ..." tcnt=$count until test -${1} $2 -o $tcnt -eq 0; do /bin/echo StartOLB: File $2 not found\; waiting $time seconds... /bin/sleep $time tcnt=`/usr/bin/expr $tcnt - 1` done if [ $tcnt -le 0 ]; then Notify "File $2 not found." fi } Notify () { /bin/echo StartOLB: $1 exit 4 } ############################################################################## # m a i n p r o g r a m # ############################################################################## # Pick up options # HAVEVER= OLBOPTS= OLBPARM=$* OLBMS=0 while test -n "$1"; do isopt=0 case $1 in [-]*) isopt=1; esac if [ "$isopt" = "0" -a -z "$HAVEVER" ]; then OLBVER=$1 HAVEVER=1 elif [ "$1" = "-c" ]; then if [ -z "$2" ]; then Notify "Configuration file not specified." fi if [ ! -r "$2" ]; then Notify "Configuration file '$2' not found." fi OLBCONFIGFN=$2 shift elif [ "$1" = "-D" ]; then set -x elif [ "$1" = "-m" ]; then if [ "$OLBMS" = "1" ]; then OLBMODE="-m -s" OLBTYPE='supervisor' else OLBMODE="-m" OLBTYPE='manager' fi OLBMS=1 elif [ "$1" = "-n" ]; then shift OLBOPTS="$OLBOPTS -n $1" elif [ "$1" = "-s" ]; then if [ "$OLBMS" = "1" ]; then OLBMODE="-m -s" OLBTYPE='supervisor' else OLBMODE="-s" OLBTYPE='server' fi OLBMS=1 elif [ "$1" = "-t" ]; then TEST=echo elif [ "$1" = "-v" ]; then VERBOSE=1 else OLBOPTS="$OLBOPTS $1" fi shift done # Establish location of the OLB executables # OLBBASE=`(cd $OLBBASE/$OLBVER; pwd)` if [ $? != 0 ]; then exit 4 fi OLBPROG=$OLBBASE/bin/$OLBPROG PROGRAM=$OLBBASE/etc/$PROGRAM if [ "$OLBCONFIGFN" = "" ]; then OLBCONFIGFN=$OLBBASE/etc/$OLBCONFIG fi # Establish log file name # OLBLOGFILE=$OLBLOGDIR/$OLBLOGFN if [ "x$OLBMODE" = "x-m -s" ]; then OLBLOGFILE=$OLBLOGFILE.super fi # Verify that we are not executing this script as root (if we are, switch) # if [ $MYNAME = root ]; then if [ $OLBUSER != root ]; then $TEST exec /bin/su $OLBUSER -c "$PROGRAM $OLBPARM" fi elif [ $MYNAME != $OLBUSER ]; then Notify "Attempt to start $OLBUSER OLB as user $MYNAME." fi # Verify that all required directories are present # for FN in $OLBBASE $OLBLOGDIR $OLBHOMEDIR do MustExist Directory d $FN done # Verify that all owned directories are writable # for FN in $OLBLOGDIR $OLBHOMEDIR do Writable Directory $FN done # Verify that we can overwrite the pid file # Writable File $OLBPIDFILE # Verify that all required readable files are present # for FN in $OLBCONFIGFN do Wait4File r $FN done # Verify that all required executable files are present # XLIST="$OLBPROG" for FN in $XLIST do Wait4File x $FN done # Set appropriate limits # $TEST ulimit -n $MAXFD $TEST ulimit -c unlimited # Add our "lib" directory to LD_LIBRARY_PATH # OLBLIBBASE=$OLBBASE/lib if [ -z "$LD_LIBRARY_PATH" ]; then LD_LIBRARY_PATH=$OLBLIBBASE else LD_LIBRARY_PATH=$OLBLIBBASE:$LD_LIBRARY_PATH fi export LD_LIBRARY_PATH # Now start the daemon # /bin/echo Starting $OLBTYPE OLB ... $TEST cd $OLBHOMEDIR $TEST $OLBPROG $OLBOPTS $OLBMODE -l $OLBLOGFILE -c $OLBCONFIGFN & stat=$? # Check if we were successful # if test $stat -gt 0 ; then Notify "$OLBPROG returned a status of ${stat}." fi xrootd-5.6.9/packaging/tgz/StartOLB.cf.example000066400000000000000000000040171457266313600212170ustar00rootroot00000000000000#!/bin/sh # # $Id$ # # Set 'time' to be number of seconds to wait for a required file to appear # This is only meaningful for files in AFS or NFS # time=60 # Set 'count to be the maximum number of times to wait for a required file # This is only meaningful for files in AFS or NFS # count=30 # Set OLBUSER to be the username to "su" to if the script is run as root # OLBUSER=$USER # Set OLBBASE to be the base directory where olbd version directories have been # installed. They all must be in the same base directory. # OLBBASE=/opt/xrootd # Set OLBVER to be the name of the default version directory. This directory # must be installed in the OLBBASE directory. It contains bin, etc, and lib. # OLBVER=prod # Set OLBPROG to be the name of the executable in the "bin" directory. The # start script uses '$OLBBASE/$OLBVER/bin/$OLBPROG' as the executable. # OLBPROG=olbd # Set OLBCONFIG the default config file name. Normally, this would be # '$OLBBASE/$OLBVER/etc/xrootd.cf'. # OLBCONFIGFN=$OLBBASE/$OLBVER/etc/xrootd.cf # Set 'OLBHOMEDIR' to be the working directory when olbd is started # OLBHOMEDIR=/var/adm/olbd/core # Set 'OLBLOGDIR' to be the directory where log files are placed and # Set 'OLBLOGFN' to be the base log file name. # OLBLOGDIR=/var/adm/xrootd/logs OLBLOGFN=olbdlog # Set 'OLBPIDFILE' to be where the pid file goes (it is check for w-access) # OLBPIDFILE=/tmp/olbd.pid #-#-#-#-#-#-#-#-# E N D O F C O N F I G U R A T I O N #-#-#-#-#-#-#-#-# # The following logic tries to set variables as follows: # MYOS - the current os name # MAXFD - the file descriptor limit # MYNAME - the current username # PROGRAM - the name of the start script # MYOS=`/bin/uname | /bin/awk '{print $1}'` if [ "$MYOS" = "SunOS" ]; then MAXFD=`ulimit -H -n` MYNAME=`/usr/ucb/whoami` else MAXFD=`ulimit -H -n | /bin/grep files | /bin/awk '{print $3}'` if [ "$MAXFD" = "" ]; then MAXFD=`ulimit -H -n` fi MYNAME=`/usr/bin/whoami` fi ############################################ PROGRAM=`echo $0 | /bin/awk '{n=split($0,x,"/"); print x[n]}'` xrootd-5.6.9/packaging/tgz/StartXRD000077500000000000000000000112371457266313600172240ustar00rootroot00000000000000#!/bin/sh # # $Id$ # # (C) 2003 by the Board of Trustees of the Leland Stanford, Jr., University # All Rights Reserved # Produced by Andrew Hanushevsky for Stanford University under contract # DE-AC03-76-SFO0515 with the Deprtment of Energy # Syntax: StartXRD [-c cfile] [-D] [-v] [oth] # Where: -c specifies the configuration file to be used. # -D turns on script debugging. # -f does not use an arbitrary port even when it is possible to do so. # -s is the service name whose port number is to be used by xrootd. # -v produces verbose output. # oth Any other single letter options to be passed to the executable. # # Set default values # . $0.cf VERBOSE=0 # Change TEST to be /bin/echo to see what will be executed # TEST= umask 002 ############################################################################## # s u b r o u t i n e s # ############################################################################## Debug () { if test $VERBOSE -eq 1; then echo $1 fi } MustExist () { Debug "Checking existence of $1 $3 ..." if test ! -${2} $3 ; then Notify "$1 $3 not found." fi } Writable () { Debug "Checking writability of $1 $2 ..." if test ! -w $2 ; then Notify "$1 $2 is not writable by $MYNAME." fi } Wait4File () { Debug "Checking file $2 ..." tcnt=$count until test -${1} $2 -o $tcnt -eq 0; do echo StartXRD: File $2 not found\; waiting $time seconds... sleep $time tcnt=`/usr/bin/expr $tcnt - 1` done if [ $tcnt -le 0 ]; then Notify "File $2 not found." fi } Notify () { echo StartXRD: $1 exit 4 } ############################################################################## # m a i n p r o g r a m # ############################################################################## # Pick up options # XRDOPTIONS= XRDPARMS=$* XRDPORT= XRDLOGSFX= while test -n "$1"; do if [ "$1" = "-c" ]; then if [ -z "$2" ]; then Notify "Configuration file not specified." fi if [ ! -r "$2" ]; then Notify "Configuration file '$2' not found." fi XRDCONFIGFN=$2 shift elif [ "$1" = "-D" ]; then set -x elif [ "$1" = "-t" ]; then TEST=echo elif [ "$1" = "-v" ]; then VERBOSE=1 else XRDOPTIONS="$XRDOPTIONS $1" fi shift done # Establish location of the XRD executable and libraries # XRDBASE=`(cd $XRDBASE; pwd)` Debug "XRDBASE has been set to '$XRDBASE'" if [ $? != 0 ]; then exit 4 fi XRDPROG=$XRDBASE/bin/$XRDARCH/xrootd XRDLIBBASE=$XRDBASE/lib/$XRDARCH PROGRAM=$XRDCFG/$PROGRAM if [ "$XRDCONFIGFN" = "" ]; then XRDCONFIGFN=$XRDCFG/$XRDCONFIG fi if [ -z "$LD_LIBRARY_PATH" ]; then LD_LIBRARY_PATH=$XRDLIBBASE else LD_LIBRARY_PATH=$XRDLIBBASE:$LD_LIBRARY_PATH fi export LD_LIBRARY_PATH # Verify that we are not executing this script as root (if we are, switch) # if [ $MYNAME = root ]; then if [ $XRDUSER != root ]; then $TEST export LD_LIBRARY_PATH $TEST exec su $XRDUSER -c "$PROGRAM $XRDPARMS" fi elif [ $MYNAME != $XRDUSER ]; then Notify "Attempt to start $XRDUSER XRD as user $MYNAME." fi # Establish the port number to be used. Treat this as an explicit override of the main config file. # if [ "x$XRDPORT" != "x" ]; then XRDPORT="-p $XRDPORT" fi # Set the log file name # XRDLOGFILE=$XRDLOGDIR/$XRDLOGFN # Just in case we don't have the basic directories, try to create them # $TEST mkdir -m 0744 -p $XRDLOGDIR $XRDHOMEDIR 2> /dev/null # Verify that all required directories are present # for FN in $XRDLIBBASE $XRDLOGDIR $XRDHOMEDIR do MustExist Directory d $FN done # Verify that all owned directories are writable # for FN in $XRDLOGDIR $XRDHOMEDIR do Writable Directory $FN done # Verify that all required readable files are present # for FN in $XRDCONFIGFN do Wait4File r $FN done # Verify that all required executable files are present # for FN in $XRDPROG do Wait4File x $FN done # Export the variables required in the config file # $TEST export XRDCFG $TEST export XRDBASE $TEST export XRDLIBBASE # Set appropriate limits # $TEST ulimit -n $MAXFD $TEST ulimit -c unlimited # Now start the daemon # echo Starting xrootd ... $TEST cd $XRDHOMEDIR $TEST $XRDPROG $XRDOPTIONS $XRDPORT -l $XRDLOGFILE -c $XRDCONFIGFN & stat=$? # Check if we were successful # if test $stat -gt 0 ; then Notify "$XRDPROG returned a status of ${stat}." fi xrootd-5.6.9/packaging/tgz/StartXRD.cf.example000066400000000000000000000074021457266313600212410ustar00rootroot00000000000000#!/bin/sh # # $Id$ # # Set 'time' to be number of seconds to wait for a required file to appear # This is only meaningful for files in AFS or NFS # time=60 # Set 'count' to be the maximum number of times to wait for a required file # This is only meaningful for files in AFS or NFS # count=30 # Set XRDUSER to be the username to "su" to if the script is run as root # XRDUSER=fabrizio # Set XRDBASE to be the base directory where xrootd has # been installed or compiled. # The default is the upper dir with respect to where this script resides # parent=`dirname $0` parent=`cd $parent ; pwd` XRDBASE=`dirname $parent` # In the case where configure.classic has not been invoked with the # option --no-arch-subdirs, we have arch subdirs. # Set XRDARCH to the architecture you want to start xrootd on # This default behavior takes the default architecture if it detects such a schema XRDARCH= if test -x $XRDBASE/bin/arch ; then XRDARCH=arch else if test -x $XRDBASE/bin/arch_dbg ; then XRDARCH=arch_dbg fi fi # Set XRDCFG to be the name of the default directory where config data and # scripts are to be found XRDCFG=$XRDBASE/etc # Set XRDCONFIG the default config file name. The start script uses # '$XRDCFG/$XRDCONFIG' as the configuration file. # XRDCONFIG=xrootd.cf # Set 'XRDHOMEDIR' to be the working directory when xrootd is started # Set 'CMSHOMEDIR' to be the working directory when cmsd is started # XRDHOMEDIR=$XRDBASE/core CMSHOMEDIR=$XRDBASE/core # Set 'XRDLOGDIR' to be the directory where log files are placed and # Set 'XRDLOGFN' to be the base log file name for xrootd. # Set 'CMSLOGFN' to be the base log file name for cmsd. # ---> If you want to use the automatic logfile rotation sheme, set each LOGFN # variable to ' -k num | sz[k|m|g]'. For example, a 7 day rotation # for cmslog: CMSLOGFN='cmslog -k 7'. The -k option is documented in the # XRD/Xrootd and cmsd configuration reference under command line options. # XRDLOGDIR=$XRDBASE/logs XRDLOGFN=xrdlog CMSLOGFN=cmslog #-#-#-#-#-#-#-# F I L E R E S I D E N C Y M A N A G E R #-#-#-#-#-#-#-#-# # The following sets the start-up command and command line parameters for # each daemon. Add as needed (e.g., log file options, instance name, etc) # The following are common options # LDLIBPATH=$XRDBASE/lib/$XRDARCH # Options that need to be set: # # CONFG_ -> What configuration file to use # HFILE_ -> The file who's presence prevents the component's [re]start # HOMED_ -> Where the home directory is (where core files will go) # LOGFN_ -> Where the log is to be written (full path with filename) # START_ -> What program to start # Set options for purge # HFILE_PURGE='/var/adm/frm/HOLD_PURGE' HOMED_PURGE='/var/adm/frm/core/purg' LOGFN_PURGE='/var/adm/frm/logs/purglog' START_PURGE="$XRDBASE/bin/$XRDARCH/frm_purged" PARMS_PURGE="-c $XRDCFG/$XRDCONFIG -l $LOGFN_PURGE" # Set options for xfrd # HFILE_TRANSFER='/var/adm/frm/HOLD_TRANSFER' HOMED_TRANSFER='/var/adm/frm/core/xfr' LOGFN_TRANSFER='/var/adm/frm/logs/xfrlog' START_TRANSFER="$XRDBASE/bin/$XRDARCH/frm_xfrd" PARMS_TRANSFER="-c $XRDCFG/$XRDCONFIG -l $LOGFN_TRANSFER" #-#-#-#-#-#-#-#-# E N D O F C O N F I G U R A T I O N #-#-#-#-#-#-#-#-# # The following logic tries to set variables as follows: # MYOS - the current os name # MAXFD - the file descriptor limit # MYNAME - the current username # PROGRAM - the name of the start script # MYOS=`uname | awk '{print $1}'` if [ "$MYOS" = "SunOS" ]; then MAXFD=`ulimit -H -n` MYNAME=`/usr/ucb/whoami` else if [ "$MYOS" = "Darwin" ]; then MAXFD=1024 else MAXFD=`ulimit -H -n | grep files | awk '{print $3}'` if [ "$MAXFD" = "" ]; then MAXFD=`ulimit -H -n` fi fi MYNAME=`whoami` fi ############################################ PROGRAM=`echo $0 | awk '{n=split($0,x,"/"); print x[n]}'` xrootd-5.6.9/packaging/tgz/StopCMS000077500000000000000000000057371457266313600170510ustar00rootroot00000000000000#!/bin/sh # # $Id$ # # (C) 2003 by the Board of Trustees of the Leland Stanford, Jr., University # All Rights Reserved # Produced by Andrew Hanushevsky for Stanford University under contract # DE-AC03-76-SFO0515 with the Deprtment of Energy #Syntax: StopCMS # The following snippet is from the StartXRD.cf file. # MYOS=`uname | awk '{print $1}'` if [ "$MYOS" = "SunOS" ]; then MYNAME=`/usr/ucb/whoami` else MYNAME=`whoami` fi CONDCHK=0 ############################################################################## # s u b r o u t i n e s # ############################################################################## Emsg () { echo StopCMS: $1 exit 4 } Terminate() { cmspid=$1 killpid=$2 # Verify that we can kill this process # if [ $MYNAME != root ]; then set -- `ps -p $cmspid -o user` if [ $2 != $MYNAME ]; then Emsg "User $MYNAME can't kill process $cmspid started by $2." fi fi # Now kill the process # set -- `kill -9 $killpid 2>&1` if [ $? -ne 0 ]; then shift 2 Emsg "Unable to kill process $cmspid; $*." fi } Check(){ cmspid=$1 # Check if the process is indeed dead # FOO=`ps -p $cmspid` if [ $? -eq 0 ]; then sleep 1 FOO=`ps -p $cmspid` if [ $? -eq 0 ]; then echo "pid $cmspid is still alive..." echo $FOO fi fi } ############################################################################## # m a i n p r o g r a m # ############################################################################## # Pick up options # while test -n "$1"; do if test -n "'$1'" -a "'$1'" = "'-D'"; then set -x elif test -n "'$1'" -a "'$1'" = "'-c'"; then CONDCHK=1 else echo StopCMS: Invalid option - $1. fi shift done # find the process number assigned to the CMS # set -- `ps -e -o pid -o comm | grep '.*cmsd$' | awk '{print $1}'` if [ -z "$1" ]; then msg="Unable to find CMS process number" if [ $CONDCHK -ne 1 ]; then Emsg "$msg; is it running?" fi echo StopCMS: $msg\; continuing. exit 0 fi # Kill each process that we have found # for cmspid do Terminate $cmspid $cmspid done # Make Sure they are dead # sleep 1 for cmspid do Check $cmspid done # find and kill the process numbers assigned to the performance monitor # set -- '' set -- `ps -e -o pid -o args | grep 'XrdOlbMonPerf' | grep -v grep | awk '{print $1}'` if [ ! -z "$1" ]; then for cmspid do Terminate $cmspid -$cmspid done fi set -- '' set -- `ps -e -o pid -o args | grep 'XrdCmsMonPerf' | grep -v grep | awk '{print $1}'` if [ ! -z "$1" ]; then for cmspid do Terminate $cmspid -$cmspid done fi echo StopCMS: CMS stopped. xrootd-5.6.9/packaging/tgz/StopFRM000077500000000000000000000072271457266313600170470ustar00rootroot00000000000000#!/bin/sh # Usage: StopFRM [-h] [-p] [-t] [-x] # Where: -h holds the daemon (i.e., prevents it from restarting) # -p stops only the purge daemon # -t types the stop commands and does not execute them # -x stops only the transfer daemon # No option cause everything to be started. # Set defaults #******************************************************************************# #* D e f a u l t s *# #******************************************************************************# TEST= EXITRC=0 DOALL=1 HOLD= ONLYP=0 ONLYX=0 # Source he config file to set additional options # . `dirname $0`/StartXRD.cf #******************************************************************************# #* S u b r o u t i n e s *# #******************************************************************************# Emsg () { echo StopFRM: $1 } Check(){ frmwho=$1 frmpid=$2 # Check if the process is indeed dead # FOO=`ps -p $frmpid` if [ $? -eq 0 ]; then sleep 1 FOO=`ps -p $frmpid` if [ $? -eq 0 ]; then Emsg "$XRDUSER $frmwho pid $frmpid is still alive..." echo $FOO fi fi } Kill() { frmwho=$1 frmpid=$2 # Verify that we can kill this process # if [ $MYNAME != root ]; then set -- `ps -p $frmpid -o user` if [ $2 != $MYNAME ]; then Emsg "User $MYNAME can't kill $frmwho process $frmpid started by $2." fi fi # Now kill the process # kCMD="kill -9 $frmpid" if [ -z "$TEST" ]; then $TEST $kCMD else set -- `$kCMD 2>&1` fi # Chek if we were successful # if [ $? -ne 0 ]; then shift 2 Emsg "Unable to kill $frmwho process $frmpid; $*." else Check $frmwho $frmpid fi } Stop() { frmwho=$1 frmpgm=$2 frmfnx=$3 set -- '' set -- `ps -e -o pid -o args | grep $frmpgm | awk '$0 !~ /grep/ {print $1}'` if [ -z "$1" ]; then Emsg "Unable to find $XRDUSER $frmwho daemon." else for frmpid do Kill $frmwho $frmpid done fi if [ "$HOLD" = "1" ]; then set -- `touch $frmfnx` fi } #******************************************************************************# #* M a i n *# #******************************************************************************# # Pick up options # while test -n "$1"; do if [ "$1" = "-h" ]; then HOLD=1 elif [ "$1" = "-p" ]; then DOALL=0 ONLYP=1 elif [ "$1" = "-t" ]; then TEST=echo OPTS=$OPTS" -t" elif [ "$1" = "-x" ]; then DOALL=0 ONLYX=1 else Emsg "Invalid option - $1." echo 'Usage: StopFRM [-h] [-p] [-t] [-x]' exit 1 fi shift done # Determine hostname and username # MYHOST=`/bin/hostname` if [ -x /usr/bin/whoami ]; then MYNAME=`/usr/bin/whoami` elif [ -x /usr/ucb/whoami ]; then MYNAME=`/usr/ucb/whoami` else Emsg "Unable to determine username." exit 1 fi # Now stop the purge daemon # if [ ! -z "$START_PURGE" -a "$DOALL" = "1" -o "$ONLYP" = "1" ]; then Emsg "Stopping $XRDUSER purge daemon. . ." Stop purge $START_PURGE $HFILE_PURGE fi # Now stop the transfer daemon # if [ "$DOALL" = "1" -o "$ONLYX" = "1" ]; then Emsg "Stopping $XRDUSER transfer daemon. . ." Stop transfer $START_TRANSFER $HFILE_TRANSFER fi exit $EXITRC xrootd-5.6.9/packaging/tgz/StopOLB000077500000000000000000000055361457266313600170400ustar00rootroot00000000000000#!/bin/sh # # $Id$ # # (C) 2003 by the Board of Trustees of the Leland Stanford, Jr., University # All Rights Reserved # Produced by Andrew Hanushevsky for Stanford University under contract # DE-AC03-76-SFO0515 with the Deprtment of Energy #Syntax: StopOLB # The following snippet is from the StartOLB.cf file. # MYOS=`/bin/uname | /bin/awk '{print $1}'` if [ "$MYOS" = "SunOS" ]; then MYNAME=`/usr/ucb/whoami` else MYNAME=`/usr/bin/whoami` fi CONDCHK=0 ############################################################################## # s u b r o u t i n e s # ############################################################################## Emsg () { /bin/echo StopOLB: $1 exit 4 } Terminate() { olbpid=$1 killpid=$2 # Verify that we can kill this process # if [ $MYNAME != root ]; then set -- `/bin/ps -p $olbpid -o user` if [ $2 != $MYNAME ]; then Emsg "User $MYNAME can't kill process $olbpid started by $2." fi fi # Now kill the process # set -- `/bin/kill -9 $killpid 2>&1` if [ $? -ne 0 ]; then shift 2 Emsg "Unable to kill process $olbpid; $*." fi } Check(){ olbpid=$1 # Check if the process is indeed dead # FOO=`/bin/ps -p $olbpid` if [ $? -eq 0 ]; then sleep 1 FOO=`/bin/ps -p $olbpid` if [ $? -eq 0 ]; then /bin/echo "pid $olbpid is still alive..." /bin/echo $FOO fi fi } ############################################################################## # m a i n p r o g r a m # ############################################################################## # Pick up options # while test -n "$1"; do if test -n "'$1'" -a "'$1'" = "'-d'"; then set -x elif test -n "'$1'" -a "'$1'" = "'-c'"; then CONDCHK=1 else /bin/echo StopOLB: Invalid option - $1. fi shift done # find the process number assigned to the OLB # set -- `ps -e -o pid -o comm | grep '.*olbd$' | awk '{print $1}'` if [ -z "$1" ]; then msg="Unable to find OLB process number" if [ $CONDCHK -ne 1 ]; then Emsg "$msg; is it running?" fi /bin/echo StopOLB: $msg\; continuing. exit 0 fi # Kill each process that we have found # for olbpid do Terminate $olbpid $olbpid done # Make Sure they are dead # sleep 1 for olbpid do Check $olbpid done # find and kill the process numbers assigned to the performance monitor # set -- '' set -- `ps -e -o pid -o args | grep 'XrdOlbMonPerf' | grep -v grep | awk '{print $1}'` if [ ! -z "$1" ]; then for olbpid do Terminate $olbpid -$olbpid done fi /bin/echo StopOLB: OLB stopped. xrootd-5.6.9/packaging/tgz/StopXRD000077500000000000000000000050671457266313600170600ustar00rootroot00000000000000#!/bin/sh # # $Id$ # # (C) 2003 by the Board of Trustees of the Leland Stanford, Jr., University # All Rights Reserved # Produced by Andrew Hanushevsky for Stanford University under contract # DE-AC03-76-SFO0515 with the Deprtment of Energy #Syntax: StopXRD # The following snippet is from the StartXRD.cf file. # MYOS=`uname | awk '{print $1}'` if [ "$MYOS" = "SunOS" ]; then MYNAME=`/usr/ucb/whoami` else MYNAME=`whoami` fi CONDCHK=0 ############################################################################## # s u b r o u t i n e s # ############################################################################## Debug () { if test $VERBOSE -eq 1; then echo $1 fi } Emsg () { echo StopXRD: $1 exit 4 } Terminate() { xrdpid=$1 # Verify that we can kill this process # if [ $MYNAME != root ]; then set -- `ps -p $xrdpid -o user` if [ $2 != $MYNAME ]; then Emsg "User $MYNAME can't kill process $xrdpid started by $2." fi fi # Now kill the process # set -- `kill -9 $xrdpid 2>&1` if [ $? -ne 0 ]; then shift 2 Emsg "Unable to kill process $xrdpid; $*." fi } Check(){ xrdpid=$1 # Check if the process is indeed dead # FOO=`ps -p $xrdpid` if [ $? -eq 0 ]; then sleep 1 FOO=`ps -p $xrdpid` if [ $? -eq 0 ]; then echo "pid $xrdpid is still alive..." echo $FOO fi fi } ############################################################################## # m a i n p r o g r a m # ############################################################################## # Pick up options # while test -n "$1"; do if test -n "'$1'" -a "'$1'" = "'-D'"; then set -x elif test -n "'$1'" -a "'$1'" = "'-c'"; then CONDCHK=1 else Notify "Invalid option '$1'." fi shift done # find the process number assigned to the XRD # set -- `ps -e -o pid -o comm | grep '.*xrootd$' | awk '{print $1}'` if [ -z "$1" ]; then msg="Unable to find XRD process number" if [ $CONDCHK -ne 1 ]; then Emsg "$msg; is it running?" fi echo StopXRD: $msg\; continuing. exit 0 fi # Kill each process that we have found # for xrdpid do Terminate $xrdpid done # Make Sure they are dead # sleep 1 for xrdpid do Check $xrdpid done echo StopXRD: XRD stopped. xrootd-5.6.9/packaging/tgz/xrootd.cf.example000066400000000000000000000037041457266313600211060ustar00rootroot00000000000000# # A very simple config script to start a local xrootd+cmsd node # # 20071219, Fabrizio Furano ################## # # Variables. Some of them are taken from the environment # and set by the StartXXX scripts // Leading path for the installed distribution set MyXrootdPath=$XRDBASE // Leading path for the xrootd libraries set MyXrootdLibBase=$XRDLIBBASE // where to find alternate scripts and cfg file set MyScriptsPath=$XRDCFG ################## # # # COMMON section # # # # This is a sample script, so any path is good to export all.export / r/w all.role server # Note: if-fi does not support nesting # The global meta manager (if any) should match this if metamanagerhost* all.role meta manager all.manager meta metamanagerhost 1213 fi # The manager node of the local cluster we are dealing with if managerhost* all.role manager all.manager managerhost 1213 # Uncomment and complete if you have a meta manager #all.manager meta metamanagerhost 1213 fi # All (and only) the data servers have to match this # Use the proper aggergatingn2n lib to set up a facultative prefix for paths # in the case you want to aggregate clusters with different storage path prefixes, # and making all of them subscribe to a global meta manager # Uncomment and fill the manager host name if you run a manager in your cluster if * all.role server # Preferred for clusters #xrd.port any #all.manager managerhost 1213 #oss.namelib libXrdAggregatingName2Name.so /my/funny/facultative/local/cluster/storage/prefix fi # # # XRD Daemon section # # # xrd.protocol xrootd * # # # XROOTD Section # # # xrootd.fslib $(MyXrootdLibBase)/libXrdOfs.so # # # CMSD Section # # # cms.delay servers 1 startup 10 cms.sched cpu 10 io 90 cms.perf int 3m pgm $(MyXrootdPath)/etc/XrdOlbMonPerf 110 # Specify how server are selected for file creation olb.space linger 0 recalc 10 min 1g 2g # # # OFS and OSS Section # # # oss.alloc * * 80 xrootd-5.6.9/packaging/tgz/xrootd.cf.example2000066400000000000000000000075121457266313600211710ustar00rootroot00000000000000# # $Id$ # # The following is a sample xrootd configuration. The relevant prefixes are: # # acc - Access Control # cms - Cluster Management Service # ofs - Open File System # oss - Open Storage System # sec - Security # xrd - Extended Request Daemon # xrootd - Xrootd Service #----------------------------------------------- # The Common Section (applies to every component) # # Tell everyone who is the redirection manager # all.manager kanrdr-a+ 3121 # Identify which machines will be redirecting clients # all.role manager if kanrdr-a+ # Identify which machines clients will be redirected to (need not be inclusive) # all.role server if kan0* # Finally, we need to indicate processing options by path. For instance, # which ones are exported (i.e., visible) and which ones are migratable (which # will indicate the logical paths (i.e., lfn's) that should exist in the MSS.) # all.export /store nodread mig #----------------------------------------------- # The Open Storage System Section # # Indicate where we have mounted filesystems that can be used for space # The cache directive will also be used by the cmsd. So, we need not repeat it. # oss.cache public /kanga/cache* # While we highly recommend that you avoid path prefixing, here we indicate # the actual way files are physically name (i.e., the lfn to pfn mapping). The # localroot is how we name this on the data server while remoteroot tells the # system the corresponding name in the Mass Storage System. # oss.localroot /kanga oss.remoteroot /kanga # Since we are using a Mass Storage System we need to indicate how MSS meta-data # information is retrieved (mssgwcmd) and how files are retrieved (stagecmd). # The xfr directive will limit the sumber of simultaneous stages to eight. # oss.msscmd /usr/etc/ooss/pudc -f -c /usr/etc/ooss/rxhpss.cf oss.stagecmd /usr/etc/ooss/ooss_Stage oss.xfr 8 #----------------------------------------------- # The XRD Section # # Generally, xrd defaults are fine. So, no need to change them. # #----------------------------------------------- # The Xrootd Section # # Here we load the extended file system support for xrootd # xrootd.fslib /opt/xrootd/lib/libXrdOfs.so #----------------------------------------------- # The Cluster Management Section # # Managers use the allow and sched directives. # Servers use the allow and perf directives. # # Indicate which hosts are allowed to connect to the cmsd (even if localhost) # cms.allow host kan*.slac.stanford.edu cms.allow host bbr-rdr*.slac.stanford.edu # To use load based scheduling, specify a load formula using sched # cms.sched cpu 100 # To effect load based scheduling, we must start a performance monitor # cms.perf int 180 pgm /opt/xrootd/etc/XrdOlbMonPerf 120 #----------------------------------------------- # The MPS Section # # Here code the relevant dorectives to control migration, purge, and staging # Most of the defaults are likely wrong. The typical ones to specify are: # # Where error messages go via mail and how often they should go # mps.adminuser = sys-hpss # For migration, how long to wait between runs, how long a failure is to be # recognized (after which the operation is retried). how often purge should # run, and the high/low purge thesholds. # mps.migr.rmfail_time = 129600 mps.migr.waittime = 600 mps.purg.waittime = 3600 mps.purg.lousedpct = 80 mps.purg.hiusedpct = 80 # For pre-stage, we generally want to indicate the maximum number allowed # at one time and the command to use to transfer data from the mss # mps.pstg.max_pstg_proc = 3 mps.pstg.pstgcmd = /opt/xrootd/utils/mps_Stage -i # Generally, for all component, we need to indicate who the MSS transfer user # is, the target host and port. Usually, we limit retries to two. # mps.xfrhost = babarmss mps.xfruser = objysrv mps.xfrport = 2021 mps.stage.max_retry = 2 #----------------------------------------------- xrootd-5.6.9/packaging/wheel/000077500000000000000000000000001457266313600161175ustar00rootroot00000000000000xrootd-5.6.9/packaging/wheel/publish.sh000077500000000000000000000022211457266313600201210ustar00rootroot00000000000000#!/bin/bash ./genversion.sh >| VERSION [[ -d dist ]] && rm -rf dist # Determine if wheel.bdist_wheel is available for wheel.bdist_wheel in setup.py python3 -c 'import wheel' &> /dev/null wheel_available=$? if [ "${wheel_available}" -ne "0" ]; then python3 -m pip install wheel wheel_available=$? if [ "${wheel_available}" -ne "0" ]; then echo "ERROR: Unable to find wheel and unable to install wheel with '$(command -v python3) -m pip install wheel'." echo " Please ensure wheel is available to $(command -v python3) and try again." exit 1 fi fi unset wheel_available # Determine if build is available python3 -c 'import build' &> /dev/null build_available=$? if [ "${build_available}" -ne "0" ]; then python3 -m pip install build build_available=$? fi if [ "${build_available}" -ne "0" ]; then echo "WARNING: Unable to find build and unable to install build from PyPI with '$(command -v python3) -m pip install build'." echo " Falling back to building sdist with '$(command -v python3) setup.py sdist'." python3 setup.py sdist else python3 -m build --sdist . fi unset build_available xrootd-5.6.9/pyproject.toml000066400000000000000000000001251457266313600160010ustar00rootroot00000000000000[build-system] requires = ["setuptools>=42"] build-backend = "setuptools.build_meta" xrootd-5.6.9/setup.py000066400000000000000000000102031457266313600145750ustar00rootroot00000000000000import os import platform import subprocess import sys from setuptools import setup, Extension from setuptools.command.build_ext import build_ext from subprocess import check_call, check_output try: from shutil import which except ImportError: from distutils.spawn import find_executable as which cmake = which("cmake3") or which("cmake") def get_cmake_args(): args = os.getenv('CMAKE_ARGS') if not args: return [] from shlex import split return split(args) def get_version(): try: version = open('VERSION').read().strip() if version.startswith('$'): output = check_output(['git', 'describe']) version = output.decode().strip() except: version = None pass if version is None: from datetime import date version = '5.7-rc' + date.today().strftime("%Y%m%d") if version.startswith('v'): version = version[1:] # Sanitize version to conform to PEP 440 # https://www.python.org/dev/peps/pep-0440 version = version.replace('-rc', 'rc') version = version.replace('-g', '+git.') version = version.replace('-', '.post', 1) version = version.replace('-', '.') return version class CMakeExtension(Extension): def __init__(self, name, src='.', sources=[], **kwa): Extension.__init__(self, name, sources=sources, **kwa) self.src = os.path.abspath(src) class CMakeBuild(build_ext): def build_extensions(self): if cmake is None: raise RuntimeError('Cannot find CMake executable') for ext in self.extensions: extdir = os.path.abspath(os.path.dirname(self.get_ext_fullpath(ext.name))) # Use $ORIGIN RPATH to ensure that the client library can load # libXrdCl which will be installed in the same directory. Build # with install RPATH because libraries are installed by Python/pip # not CMake, so CMake cannot fix the install RPATH later on. cmake_args = [ '-DPython_EXECUTABLE={}'.format(sys.executable), '-DCMAKE_BUILD_WITH_INSTALL_RPATH=TRUE', '-DCMAKE_ARCHIVE_OUTPUT_DIRECTORY={}/{}'.format(self.build_temp, ext.name), '-DCMAKE_LIBRARY_OUTPUT_DIRECTORY={}/{}'.format(extdir, ext.name), '-DENABLE_PYTHON=1', '-DENABLE_XRDCL=1', '-DXRDCL_LIB_ONLY=1', '-DPYPI_BUILD=1' ] if sys.platform == 'darwin': cmake_args += [ '-DCMAKE_INSTALL_RPATH=@loader_path' ] else: cmake_args += [ '-DCMAKE_INSTALL_RPATH=$ORIGIN' ] cmake_args += get_cmake_args() if not os.path.exists(self.build_temp): os.makedirs(self.build_temp) check_call([cmake, ext.src, '-B', self.build_temp] + cmake_args) check_call([cmake, '--build', self.build_temp, '--parallel']) version = get_version() setup(name='xrootd', version=version, description='eXtended ROOT daemon', author='XRootD Developers', author_email='xrootd-dev@slac.stanford.edu', url='http://xrootd.org', download_url='https://github.com/xrootd/xrootd/archive/v%s.tar.gz' % version, keywords=['XRootD', 'network filesystem'], license='LGPLv3+', long_description=open('README.md').read(), long_description_content_type='text/markdown', packages = ['XRootD', 'XRootD.client', 'pyxrootd'], package_dir = { 'pyxrootd' : 'bindings/python/src', 'XRootD' : 'bindings/python/libs', 'XRootD/client': 'bindings/python/libs/client', }, ext_modules= [ CMakeExtension('pyxrootd') ], cmdclass={ 'build_ext': CMakeBuild }, zip_safe=False, classifiers=[ "Intended Audience :: Information Technology", "Intended Audience :: Science/Research", "License :: OSI Approved :: GNU Lesser General Public License v3 or later (LGPLv3+)", "Operating System :: MacOS", "Operating System :: POSIX :: Linux", "Operating System :: Unix", "Programming Language :: C++", "Programming Language :: Python", ] ) xrootd-5.6.9/src/000077500000000000000000000000001457266313600136565ustar00rootroot00000000000000xrootd-5.6.9/src/CMakeLists.txt000066400000000000000000000033761457266313600164270ustar00rootroot00000000000000#------------------------------------------------------------------------------- # If this is a build for pip set up the rpath # (this is especially important for plugins) #------------------------------------------------------------------------------- if( PYPI_BUILD ) set( CMAKE_INSTALL_RPATH "$ORIGIN" ) set( CMAKE_INSTALL_RPATH_USE_LINK_PATH true ) endif() #------------------------------------------------------------------------------- # XRDCL_LIB_ONLY implies XRDCL_ONLY #------------------------------------------------------------------------------- if( XRDCL_LIB_ONLY ) set ( XRDCL_ONLY TRUE ) endif() #------------------------------------------------------------------------------- # Include the subcomponents #------------------------------------------------------------------------------- include( XrdUtils ) include( XrdApps ) include( XrdCrypto ) include( XrdPosix ) include( XrdSec ) include( XrdXml ) include( XrdHeaders ) include( XrdSecgsi ) include( XrdSecztn ) if( BUILD_KRB5 ) include( XrdSeckrb5 ) endif() if( BUILD_XRDEC ) include( XrdIsal ) endif() if( ENABLE_XRDCL ) add_subdirectory( XrdCl ) endif() if( BUILD_XRDCLHTTP ) add_subdirectory( XrdClHttp ) endif() if( BUILD_XRDEC ) add_subdirectory( XrdEc ) endif() if( NOT XRDCL_ONLY ) include( XrdServer ) include( XrdDaemons ) include( XrdFrm ) include( XrdFfs ) include( XrdPlugins ) include( XrdSsi ) include( XrdPfc ) include( XrdOssCsi ) if( BUILD_HTTP ) include( XrdHttp ) include( XrdTpc ) endif() if( BUILD_MACAROONS ) include( XrdMacaroons ) endif() if( BUILD_VOMS ) include( XrdVoms ) endif() if( XRDCEPH_SUBMODULE ) add_subdirectory( XrdCeph ) endif() if( BUILD_SCITOKENS ) include( XrdSciTokens ) endif() endif() xrootd-5.6.9/src/XProtocol/000077500000000000000000000000001457266313600156075ustar00rootroot00000000000000xrootd-5.6.9/src/XProtocol/README000066400000000000000000000016631457266313600164750ustar00rootroot00000000000000To insure that changes to the protocol are backward-consistent with pre-existing servers/clients, the following rules should be followed: a) Any new status/request codes need to be added to the end of the list of existing codes. b) Only reserved fields may contain new information (i.e., we can't change the definition of existing fields except to add new status bits to an existing flag). c) By extension, structure sizes or layout may not change for any existing structure passed to the server. Any other addition is fair game since the server doesn't really care. However, additions to the protocol defined here may conflict with future versions should a future version use an added definition for some other purpose. To prevent this from happening, please feedback your changes to the XRootD collaboration via https://github.com/xrootd so that your changes, upon acceptance, are reflected in the official protocol definition. xrootd-5.6.9/src/XProtocol/XProtocol.cc000066400000000000000000000260071457266313600200540ustar00rootroot00000000000000/******************************************************************************/ /* */ /* X P r o t o c o l . c c */ /* */ /* (c) 2016 by the Board of Trustees of the Leland Stanford, Jr., University */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* The XRootD protocol definition, documented in this file, is distributed */ /* under a modified BSD license and may be freely used to reimplement it. */ /* Any references to "source" in this license refers to this file or any */ /* other file that specifically contains the following license. */ /* */ /* Redistribution and use in source and binary forms, with or without */ /* modification, are permitted provided that the following conditions */ /* are met: */ /* */ /* 1. Redistributions of source code must retain the above copyright notice, */ /* this list of conditions and the following disclaimer. */ /* */ /* 2. Redistributions in binary form must reproduce the above copyright */ /* notice, this list of conditions and the following disclaimer in the */ /* documentation and/or other materials provided with the distribution. */ /* */ /* 3. Neither the name of the copyright holder nor the names of its */ /* contributors may be used to endorse or promote products derived from */ /* this software without specific prior written permission. */ /* */ /* 4. Derived software may not use the name XRootD or cmsd (regardless of */ /* capitilization) in association with the derived work if the protocol */ /* documented in this file is changed in any way. */ /* */ /* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS */ /* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT */ /* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR */ /* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT */ /* HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, */ /* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT */ /* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, */ /* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY */ /* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */ /* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE */ /* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /******************************************************************************/ #include #include #include #include #include #include #include #include #include "XProtocol/XProtocol.hh" /******************************************************************************/ /* G l o b a l s */ /******************************************************************************/ namespace { const char *errNames[kXR_ERRFENCE-kXR_ArgInvalid] = {"Invalid argument", // kXR_ArgInvalid = 3000, "Missing argument", // kXR_ArgMissing "Argument is too long", // kXR_ArgTooLong "File or object is locked", // kXR_FileLocked "File or object not open", // kXR_FileNotOpen "Filesystem error", // kXR_FSError "Invalid request", // kXR_InvalidRequest "I/O error", // kXR_IOError "Insufficient memory", // kXR_NoMemory "Insufficient space", // kXR_NoSpace "Request not authorized", // kXR_NotAuthorized "File or object not found", // kXR_NotFound "Internal server error", // kXR_ServerError "Unsupported request", // kXR_Unsupported "No servers available", // kXR_noserver "Target is not a file", // kXR_NotFile "Target is a directory", // kXR_isDirectory "Request cancelled", // kXR_Cancelled "Target exists", // kXR_ItExists "Checksum is invalid", // kXR_ChkSumErr "Request in progress", // kXR_inProgress "Quota exceeded", // kXR_overQuota "Invalid signature", // kXR_SigVerErr "Decryption failed", // kXR_DecryptErr "Server is overloaded", // kXR_Overloaded "Filesystem is read only", // kXR_fsReadOnly "Invalid payload format", // kXR_BadPayload "File attribute not found", // kXR_AttrNotFound "Operation requires TLS", // kXR_TLSRequired "No new servers for replica", // kXR_noReplicas "Authentication failed", // kXR_AuthFailed "Request is not possible", // kXR_Impossible "Conflicting request", // kXR_Conflict "Too many errors", // kXR_TooManyErrs "Request timed out" // kXR_ReqTimedOut }; const char *reqNames[kXR_REQFENCE-kXR_auth] = {"auth", "query", "chmod", "close", "dirlist", "gpfile", "protocol", "login", "mkdir", "mv", "open", "ping", "chkpoint", "read", "rm", "rmdir", "sync", "stat", "set", "write", "fattr", "prepare", "statx", "endsess", "bind", "readv", "pgwrite", "locate", "truncate", "sigver", "pgread", "writev" }; // Following value is used to determine if the error or request code is // host byte or network byte order making use of the fact each starts at 3000 // union Endianness {kXR_unt16 xyz; unsigned char Endian[2];} little = {1}; } /******************************************************************************/ /* e r r N a m e */ /******************************************************************************/ const char *XProtocol::errName(kXR_int32 errCode) { // Mangle request code if the byte orderdoesn't match our host order // if ((errCode < 0 || errCode > kXR_ERRFENCE) && little.Endian[0]) errCode = ntohl(errCode); // Validate the request code // if (errCode < kXR_ArgInvalid || errCode >= kXR_ERRFENCE) return "!undefined error"; // Return the proper table // return errNames[errCode - kXR_ArgInvalid]; } /******************************************************************************/ /* r e q N a m e */ /******************************************************************************/ const char *XProtocol::reqName(kXR_unt16 reqCode) { // Mangle request code if the byte orderdoesn't match our host order // if (reqCode > kXR_REQFENCE && little.Endian[0]) reqCode = ntohs(reqCode); // Validate the request code // if (reqCode < kXR_auth || reqCode >= kXR_REQFENCE) return "!unknown"; // Return the proper table // return reqNames[reqCode - kXR_auth]; } /******************************************************************************/ /* n v e c & v v e c o p e r a t i n s */ /******************************************************************************/ // Add an attribute name to nvec (the buffer has to be sufficiently big) // char* ClientFattrRequest::NVecInsert( const char *name, char *buffer ) { // set rc to 0 memset( buffer, 0, sizeof( kXR_unt16 ) ); buffer += sizeof( kXR_unt16 ); // copy attribute name including trailing null size_t len = strlen( name ); memcpy( buffer, name, len + 1 ); buffer += len + 1; // return memory that comes right after newly inserted nvec record return buffer; } // Add an attribute name to vvec (the buffer has to be sufficiently big) // char* ClientFattrRequest::VVecInsert( const char *value, char *buffer ) { // copy value size kXR_int32 len = strlen( value ); kXR_int32 lendat = htonl( len ); memcpy( buffer, &lendat, sizeof( kXR_int32 ) ); buffer += sizeof( kXR_int32 ); // copy value itself memcpy( buffer, value, len ); buffer += len; // return memory that comes right after newly inserted vvec entry return buffer; } // Read error code from nvec // char* ClientFattrRequest::NVecRead( char* buffer, kXR_unt16 &rc ) { memcpy(&rc, buffer, sizeof(kXR_unt16)); rc = htons( rc ); buffer += sizeof( kXR_unt16 ); return buffer; } // Read attribute name from nvec // char* ClientFattrRequest::NVecRead( char* buffer, char *&name ) { name = strdup( buffer ); buffer += strlen( name ) + 1; return buffer; } // Read value length from vvec // char* ClientFattrRequest::VVecRead( char* buffer, kXR_int32 &len ) { memcpy(&len, buffer, sizeof(kXR_int32)); len = htonl( len ); buffer += sizeof( kXR_int32 ); return buffer; } // Read attribute value from vvec // char* ClientFattrRequest::VVecRead( char* buffer, kXR_int32 len, char *&value ) { value = reinterpret_cast( malloc( len + 1 ) ); strncpy( value, buffer, len ); value[len] = 0; buffer += len; return buffer; } xrootd-5.6.9/src/XProtocol/XProtocol.hh000066400000000000000000001435711457266313600200740ustar00rootroot00000000000000#ifndef __XPROTOCOL_H #define __XPROTOCOL_H /******************************************************************************/ /* */ /* X P r o t o c o l . h h */ /* */ /* (c) 2012 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* The XRoot protocol definition, documented in this file, is distributed */ /* under a modified BSD license and may be freely used to reimplement it. */ /* Any references to "source" in this license refers to this file or any */ /* other file that specifically contains the following license. */ /* */ /* Redistribution and use in source and binary forms, with or without */ /* modification, are permitted provided that the following conditions */ /* are met: */ /* */ /* 1. Redistributions of source code must retain the above copyright notice, */ /* this list of conditions and the following disclaimer. */ /* */ /* 2. Redistributions in binary form must reproduce the above copyright */ /* notice, this list of conditions and the following disclaimer in the */ /* documentation and/or other materials provided with the distribution. */ /* */ /* 3. Neither the name of the copyright holder nor the names of its */ /* contributors may be used to endorse or promote products derived from */ /* this software without specific prior written permission. */ /* */ /* 4. Derived software may not use the name XRootD or cmsd (regardless of */ /* capitilization) in association with the derived work if the protocol */ /* documented in this file is changed in any way. */ /* */ /* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS */ /* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT */ /* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR */ /* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT */ /* HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, */ /* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT */ /* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, */ /* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY */ /* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */ /* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE */ /******************************************************************************/ #ifdef __CINT__ #define __attribute__(x) #endif #include "XProtocol/XPtypes.hh" /******************************************************************************/ /* P r o t o c o l V e r s i o n D e f i n i t i o n s */ /******************************************************************************/ // The following is the binary representation of the protocol version here. // Protocol version is repesented as three base10 digits x.y.z with x having no // upper limit (i.e. n.9.9 + 1 -> n+1.0.0). The kXR_PROTSIGNVERSION defines the // protocol version where request signing became available. // #define kXR_PROTOCOLVERSION 0x00000511 #define kXR_PROTXATTVERSION 0x00000500 #define kXR_PROTTLSVERSION 0x00000500 #define kXR_PROTPGRWVERSION 0x00000511 #define kXR_PROTSIGNVERSION 0x00000310 #define kXR_PROTOCOLVSTRING "5.1.0" /******************************************************************************/ /* C l i e n t - S e r v e r H a n d s h a k e */ /******************************************************************************/ // The fields to be sent as initial handshake // struct ClientInitHandShake { kXR_int32 first; kXR_int32 second; kXR_int32 third; kXR_int32 fourth; kXR_int32 fifth; }; // The body received after the first handshake's header // struct ServerInitHandShake { kXR_int32 msglen; kXR_int32 protover; kXR_int32 msgval; }; /******************************************************************************/ /* C l i e n t R e q u e s t s */ /******************************************************************************/ // G.Ganis: All the following structures never need padding bytes: // no need of packing options like __attribute__((packed)) // // All binary data is sent in network byte order. // Client request codes // enum XRequestTypes { kXR_1stRequest= 3000, kXR_auth = 3000, kXR_query, // 3001 kXR_chmod, // 3002 kXR_close, // 3003 kXR_dirlist, // 3004 kXR_gpfile, // 3005 was kXR_getfile kXR_protocol,// 3006 kXR_login, // 3007 kXR_mkdir, // 3008 kXR_mv, // 3009 kXR_open, // 3010 kXR_ping, // 3011 kXR_chkpoint,// 3012 was kXR_putfile kXR_read, // 3013 kXR_rm, // 3014 kXR_rmdir, // 3015 kXR_sync, // 3016 kXR_stat, // 3017 kXR_set, // 3018 kXR_write, // 3019 kXR_fattr, // 3020 was kXR_admin kXR_prepare, // 3021 kXR_statx, // 3022 kXR_endsess, // 3023 kXR_bind, // 3024 kXR_readv, // 3025 kXR_pgwrite, // 3026 was kXR_verifyw kXR_locate, // 3027 kXR_truncate,// 3028 kXR_sigver, // 3029 kXR_pgread, // 3030 was kXR_decrypt kXR_writev, // 3031 kXR_REQFENCE // Always last valid request code +1 }; // Virtual client request codes // enum XVirtRequestTypes { kXR_virtReadv = 2000 }; // All client requests use a header with the following format // struct ClientRequestHdr { kXR_char streamid[2]; kXR_unt16 requestid; kXR_char body[16]; kXR_int32 dlen; }; /******************************************************************************/ /* k X R _ a u t h R e q u e s t */ /******************************************************************************/ struct ClientAuthRequest { kXR_char streamid[2]; kXR_unt16 requestid; kXR_char reserved[12]; kXR_char credtype[4]; kXR_int32 dlen; }; /******************************************************************************/ /* k X R _ b i n d R e q u e s t */ /******************************************************************************/ struct ClientBindRequest { kXR_char streamid[2]; kXR_unt16 requestid; kXR_char sessid[16]; kXR_int32 dlen; }; /******************************************************************************/ /* k X R _ c h m o d R e q u e s t */ /******************************************************************************/ struct ClientChmodRequest { kXR_char streamid[2]; kXR_unt16 requestid; kXR_char reserved[14]; kXR_unt16 mode; // See XOpenRequestMode kXR_int32 dlen; }; /******************************************************************************/ /* k X R _ c h k p o i n t R e q u e s t */ /******************************************************************************/ struct ClientChkPointRequest { kXR_char streamid[2]; kXR_unt16 requestid; kXR_char fhandle[4]; // For Create, Delete, Query, or Restore kXR_char reserved[11]; kXR_char opcode; // One of kXR_ckpxxxx actions kXR_int32 dlen; }; // Actions // static const int kXR_ckpBegin = 0; // Begin checkpoint static const int kXR_ckpCommit = 1; // Commit changes static const int kXR_ckpQuery = 2; // Query checkpoint limits static const int kXR_ckpRollback= 3; // Rollback changes static const int kXR_ckpXeq = 4; // Execute trunc, write, or writev // The minimum size of a checkpoint data limit // static const int kXR_ckpMinMax = 104857604; // 10 MB /******************************************************************************/ /* k X R _ c l o s e R e q u e s t */ /******************************************************************************/ struct ClientCloseRequest { kXR_char streamid[2]; kXR_unt16 requestid; kXR_char fhandle[4]; kXR_char reserved[12]; kXR_int32 dlen; }; /******************************************************************************/ /* k X R _ d i r l i s t R e q u e s t */ /******************************************************************************/ enum XDirlistRequestOption { kXR_online = 1, kXR_dstat = 2, kXR_dcksm = 4 // dcksm implies dstat irrespective of dstat setting }; struct ClientDirlistRequest { kXR_char streamid[2]; kXR_unt16 requestid; kXR_char reserved[15]; kXR_char options[1]; // See XDirlistRequestOption enum kXR_int32 dlen; }; /******************************************************************************/ /* k X R _ e n d s e s s R e q u e s t */ /******************************************************************************/ struct ClientEndsessRequest { kXR_char streamid[2]; kXR_unt16 requestid; kXR_char sessid[16]; kXR_int32 dlen; }; /******************************************************************************/ /* k X R _ f a t t r R e q u e s t */ /******************************************************************************/ // kXR_fattr subcodes // enum xfaSubCode { kXR_fattrDel = 0, kXR_fattrGet = 1, kXR_fattrList = 2, kXR_fattrSet = 3, kXR_fatrrMaxSC = 3 // Highest valid subcode }; // kXR_fattr limits // enum xfaLimits { kXR_faMaxVars = 16, // Maximum variables per request kXR_faMaxNlen = 248, // Maximum length of variable name kXR_faMaxVlen = 65536 // Maximum length of variable value }; struct ClientFattrRequest { kXR_char streamid[2]; kXR_unt16 requestid; kXR_char fhandle[4]; kXR_char subcode; // See xfaSubCode enum kXR_char numattr; kXR_char options; // See valid options below kXR_char reserved[9]; kXR_int32 dlen; // Valid options: // static const int isNew = 0x01; // For set, the variable must not exist static const int aData = 0x10; // For list, return attribute value // Add an attribute name to nvec (the buffer has to be sufficiently big) // static char* NVecInsert( const char *name, char *buffer ); // Add an attribute name to vvec (the buffer has to be sufficiently big) // static char* VVecInsert( const char *value, char *buffer ); // Read error code from nvec // static char* NVecRead( char* buffer, kXR_unt16 &rc ); // Read attribute name from nvec, should be deallocated with free() // static char* NVecRead( char* buffer, char *&name ); // Read value length from vvec // static char* VVecRead( char* buffer, kXR_int32 &len ); // Read attribute value from vvec, should be deallocated with free() // static char* VVecRead( char* buffer, kXR_int32 len, char *&value ); }; /******************************************************************************/ /* k X R _ g p f i l e R e q u e s t */ /******************************************************************************/ struct ClientGPfileRequest { // ??? This is all wrong; correct when implemented kXR_char streamid[2]; kXR_unt16 requestid; // kXR_gpfile kXR_int32 options; kXR_char reserved[8]; kXR_int32 buffsz; kXR_int32 dlen; }; /******************************************************************************/ /* k X R _ l o c a t e R e q u e s t */ /******************************************************************************/ struct ClientLocateRequest { kXR_char streamid[2]; kXR_unt16 requestid; kXR_unt16 options; // See XOpenRequestOption enum tagged for locate kXR_char reserved[14]; kXR_int32 dlen; }; /******************************************************************************/ /* k X R _ l o g i n R e q u e s t */ /******************************************************************************/ // this is a bitmask enum XLoginAbility { kXR_nothing = 0, kXR_fullurl = 1, kXR_multipr = 3, kXR_readrdok = 4, kXR_hasipv64 = 8, kXR_onlyprv4 = 16, kXR_onlyprv6 = 32, kXR_lclfile = 64, kXR_redirflags = 128 }; // this iss a bitmask enum XLoginAbility2 { kXR_empty = 0, kXR_ecredir = 1 }; // this is a bitmask (note that XLoginVersion resides in lower bits) enum XLoginCapVer { kXR_lcvnone = 0, kXR_vermask = 63, kXR_asyncap = 128 }; // this is a single number that is or'd into capver as the version // enum XLoginVersion { kXR_ver000 = 0, // Old clients predating history kXR_ver001 = 1, // Generally implemented 2005 protocol kXR_ver002 = 2, // Same as 1 but adds asyncresp recognition kXR_ver003 = 3, // The 2011-2012 rewritten client kXR_ver004 = 4, // The 2016 sign-capable client kXR_ver005 = 5 // The 2019 TLS-capable client }; struct ClientLoginRequest { kXR_char streamid[2]; kXR_unt16 requestid; kXR_int32 pid; kXR_char username[8]; kXR_char ability2; // See XLoginAbility2 enum flags kXR_char ability; // See XLoginAbility enum flags kXR_char capver[1]; // See XLoginCapVer enum flags kXR_char reserved2; kXR_int32 dlen; }; /******************************************************************************/ /* k X R _ m k d i r R e q u e s t */ /******************************************************************************/ enum XMkdirOptions { kXR_mknone = 0, kXR_mkdirpath = 1 }; struct ClientMkdirRequest { kXR_char streamid[2]; kXR_unt16 requestid; kXR_char options[1]; kXR_char reserved[13]; kXR_unt16 mode; // See XOpenRequestMode kXR_int32 dlen; }; /******************************************************************************/ /* k X R _ m v R e q u e s t */ /******************************************************************************/ struct ClientMvRequest { kXR_char streamid[2]; kXR_unt16 requestid; kXR_char reserved[14]; kXR_int16 arg1len; kXR_int32 dlen; }; /******************************************************************************/ /* k X R _ o p e n R e q u e s t */ /******************************************************************************/ // OPEN MODE FOR A REMOTE FILE enum XOpenRequestMode { kXR_ur = 0x100, kXR_uw = 0x080, kXR_ux = 0x040, kXR_gr = 0x020, kXR_gw = 0x010, kXR_gx = 0x008, kXR_or = 0x004, kXR_ow = 0x002, kXR_ox = 0x001 }; enum XOpenRequestOption { kXR_compress = 0x0001, // 1 // also locate (return unique hosts) kXR_delete = 0x0002, // 2 kXR_force = 0x0004, // 4 kXR_new = 0x0008, // 8 kXR_open_read= 0x0010, // 16 kXR_open_updt= 0x0020, // 32 kXR_async = 0x0040, // 64 kXR_refresh = 0x0080, // 128 // also locate kXR_mkpath = 0x0100, // 256 kXR_prefname = 0x0100, // 256 // only locate kXR_open_apnd= 0x0200, // 512 kXR_retstat = 0x0400, // 1024 kXR_4dirlist = 0x0400, // 1024 // for locate intending a dirlist kXR_replica = 0x0800, // 2048 kXR_posc = 0x1000, // 4096 kXR_nowait = 0x2000, // 8192 // also locate kXR_seqio = 0x4000, // 16384 kXR_open_wrto= 0x8000 // 32768 }; enum XOpenRequestOption2 { kXR_dup = 0x0001, // 1 kXR_samefs = 0x0002 // 2 }; struct ClientOpenRequest { kXR_char streamid[2]; kXR_unt16 requestid; kXR_unt16 mode; kXR_unt16 options; kXR_char reserved[12]; kXR_int32 dlen; }; /******************************************************************************/ /* k X R _ p g r e a d R e q u e s t */ /******************************************************************************/ // The page size for pgread and pgwrite and the maximum transmission size // namespace XrdProto // Always use this namespace for new additions { static const int kXR_pgPageSZ = 4096; // Length of a page static const int kXR_pgPageBL = 12; // log2(page length) static const int kXR_pgUnitSZ = kXR_pgPageSZ + sizeof(kXR_unt32); static const int kXR_pgMaxEpr = 128; // Max checksum errs per request static const int kXR_pgMaxEos = 256; // Max checksum errs outstanding // kXR_pgread/write options // static const kXR_char kXR_AnyPath = 0xff; // In pathid static const int kXR_pgRetry = 0x01; // In reqflags } struct ClientPgReadRequest { kXR_char streamid[2]; kXR_unt16 requestid; kXR_char fhandle[4]; kXR_int64 offset; kXR_int32 rlen; kXR_int32 dlen; // Request data length must be 0 unless args present }; struct ClientPgReadReqArgs { kXR_char pathid; // Request data length must be 1 kXR_char reqflags; // Request data length must be 2 }; namespace { } /******************************************************************************/ /* k X R _ p r w r i t e R e q u e s t */ /******************************************************************************/ struct ClientPgWriteRequest { kXR_char streamid[2]; kXR_unt16 requestid; kXR_char fhandle[4]; kXR_int64 offset; kXR_char pathid; kXR_char reqflags; kXR_char reserved[2]; kXR_int32 dlen; // kXR_char data[dlen]; }; /******************************************************************************/ /* k X R _ p i n g R e q u e s t */ /******************************************************************************/ struct ClientPingRequest { kXR_char streamid[2]; kXR_unt16 requestid; kXR_char reserved[16]; kXR_int32 dlen; }; /******************************************************************************/ /* k X R _ p r o t o c o l R e q u e s t */ /******************************************************************************/ struct ClientProtocolRequest { kXR_char streamid[2]; kXR_unt16 requestid; kXR_int32 clientpv; // 2.9.7 or higher kXR_char flags; // 3.1.0 or higher kXR_char expect; // 4.0.0 or higher kXR_char reserved[10]; kXR_int32 dlen; enum RequestFlags { kXR_secreqs = 0x01, // Options: Return security requirements kXR_ableTLS = 0x02, // Options: Client is TLS capable kXR_wantTLS = 0x04, // Options: Change connection to use TLS kXR_bifreqs = 0x08 // Options: Return bind interface requirements }; enum ExpectFlags { kXR_ExpMask = 0x0f, // Isolate the relevant expect enumeration value kXR_ExpNone = 0x00, kXR_ExpBind = 0x01, kXR_ExpGPF = 0x02, kXR_ExpLogin = 0x03, kXR_ExpTPC = 0x04, kXR_ExpGPFA = 0x08 }; }; /******************************************************************************/ /* k X R _ p r e p a r e R e q u e s t */ /******************************************************************************/ enum XPrepRequestOption { kXR_cancel = 1, kXR_notify = 2, kXR_noerrs = 4, kXR_stage = 8, kXR_wmode = 16, kXR_coloc = 32, kXR_fresh = 64, kXR_usetcp = 128, kXR_evict = 0x0001 // optionsX: file no longer useful }; struct ClientPrepareRequest { kXR_char streamid[2]; kXR_unt16 requestid; kXR_char options; kXR_char prty; kXR_unt16 port; // 2.9.9 or higher kXR_unt16 optionX; // Extended options kXR_char reserved[10]; kXR_int32 dlen; }; /******************************************************************************/ /* k X R _ q u e r y R e q u e s t */ /******************************************************************************/ enum XQueryType { kXR_QStats = 1, kXR_QPrep = 2, kXR_Qcksum = 3, kXR_Qxattr = 4, kXR_Qspace = 5, kXR_Qckscan= 6, kXR_Qconfig= 7, kXR_Qvisa = 8, kXR_Qopaque=16, kXR_Qopaquf=32, kXR_Qopaqug=64 }; struct ClientQueryRequest { kXR_char streamid[2]; kXR_unt16 requestid; kXR_unt16 infotype; // See XQueryType enum kXR_char reserved1[2]; kXR_char fhandle[4]; kXR_char reserved2[8]; kXR_int32 dlen; }; /******************************************************************************/ /* k X R _ r e a d R e q u e s t */ /******************************************************************************/ struct ClientReadRequest { kXR_char streamid[2]; kXR_unt16 requestid; kXR_char fhandle[4]; kXR_int64 offset; kXR_int32 rlen; kXR_int32 dlen; // Optionally followed by read_args }; struct read_args { kXR_char pathid; kXR_char reserved[7]; // This struct may be followed by an array of readahead_list }; struct readahead_list { kXR_char fhandle[4]; kXR_int32 rlen; kXR_int64 offset; }; /******************************************************************************/ /* k X R _ r e a d v R e q u e s t */ /******************************************************************************/ struct ClientReadVRequest { kXR_char streamid[2]; kXR_unt16 requestid; kXR_char reserved[15]; kXR_char pathid; kXR_int32 dlen; // This struct followed by the read_list }; namespace XrdProto // Always use this namespace for new additions { struct read_list { kXR_char fhandle[4]; kXR_int32 rlen; kXR_int64 offset; }; static const int rlItemLen = sizeof(read_list); static const int maxRvecln = 16384; static const int maxRvecsz = maxRvecln/rlItemLen; } /******************************************************************************/ /* k X R _ r m R e q u e s t */ /******************************************************************************/ struct ClientRmRequest { kXR_char streamid[2]; kXR_unt16 requestid; kXR_char reserved[16]; kXR_int32 dlen; }; /******************************************************************************/ /* k X R _ r m d i r R e q u e s t */ /******************************************************************************/ struct ClientRmdirRequest { kXR_char streamid[2]; kXR_unt16 requestid; kXR_char reserved[16]; kXR_int32 dlen; }; /******************************************************************************/ /* k X R _ s e t R e q u e s t */ /******************************************************************************/ struct ClientSetRequest { kXR_char streamid[2]; kXR_unt16 requestid; kXR_char reserved[15]; kXR_char modifier; // For security purposes, should be zero kXR_int32 dlen; }; /******************************************************************************/ /* k X R _ s i g v e r R e q u e s t */ /******************************************************************************/ // Cryptography used for kXR_sigver SigverRequest::crypto enum XSecCrypto { kXR_SHA256 = 0x01, // Hash used kXR_HashMask = 0x0f, // Mak to extract the hash type kXR_rsaKey = 0x80 // The rsa key was used }; // Flags for kXR_sigver enum XSecFlags { kXR_nodata = 1 // Request payload was not hashed }; // Version number enum XSecVersion { kXR_Ver_00 = 0 }; struct ClientSigverRequest { kXR_char streamid[2]; kXR_unt16 requestid; kXR_unt16 expectrid; // Request code of subsequent request kXR_char version; // Security version being used (see XSecVersion) kXR_char flags; // One or more flags defined in enum (see XSecFlags) kXR_unt64 seqno; // Monotonically increasing number (part of hash) kXR_char crypto; // Cryptography used (see XSecCrypto) kXR_char rsvd2[3]; kXR_int32 dlen; }; /******************************************************************************/ /* k X R _ s t a t R e q u e s t */ /******************************************************************************/ enum XStatRequestOption { kXR_vfs = 1 }; struct ClientStatRequest { kXR_char streamid[2]; kXR_unt16 requestid; kXR_char options; // See XStatRequestOption kXR_char reserved[11]; kXR_char fhandle[4]; kXR_int32 dlen; }; /******************************************************************************/ /* k X R _ s y n c R e q u e s t */ /******************************************************************************/ struct ClientSyncRequest { kXR_char streamid[2]; kXR_unt16 requestid; kXR_char fhandle[4]; kXR_char reserved[12]; kXR_int32 dlen; }; /******************************************************************************/ /* k X R _ t r u n c a t e R e q u e s t */ /******************************************************************************/ struct ClientTruncateRequest { kXR_char streamid[2]; kXR_unt16 requestid; kXR_char fhandle[4]; kXR_int64 offset; kXR_char reserved[4]; kXR_int32 dlen; }; /******************************************************************************/ /* k X R _ w r i t e R e q u e s t */ /******************************************************************************/ struct ClientWriteRequest { kXR_char streamid[2]; kXR_unt16 requestid; kXR_char fhandle[4]; kXR_int64 offset; kXR_char pathid; kXR_char reserved[3]; kXR_int32 dlen; }; /******************************************************************************/ /* k X R _ w r i t e v R e q u e s t */ /******************************************************************************/ struct ClientWriteVRequest { kXR_char streamid[2]; kXR_unt16 requestid; kXR_char options; // See static const ints below kXR_char reserved[15]; kXR_int32 dlen; // This struct followed by the write_list static const kXR_int32 doSync = 0x01; }; namespace XrdProto // Always use this namespace for new additions { struct write_list { kXR_char fhandle[4]; kXR_int32 wlen; kXR_int64 offset; }; static const int wlItemLen = sizeof(write_list); static const int maxWvecln = 16384; static const int maxWvecsz = maxWvecln/wlItemLen; } /******************************************************************************/ /* U n i o n o f a l l C l i e n t R e q u e s t s */ /******************************************************************************/ typedef union { struct ClientRequestHdr header; struct ClientAuthRequest auth; struct ClientBindRequest bind; struct ClientChkPointRequest chkpoint; struct ClientChmodRequest chmod; struct ClientCloseRequest close; struct ClientDirlistRequest dirlist; struct ClientEndsessRequest endsess; struct ClientFattrRequest fattr; struct ClientGPfileRequest gpfile; struct ClientLocateRequest locate; struct ClientLoginRequest login; struct ClientMkdirRequest mkdir; struct ClientMvRequest mv; struct ClientOpenRequest open; struct ClientPgReadRequest pgread; struct ClientPgWriteRequest pgwrite; struct ClientPingRequest ping; struct ClientPrepareRequest prepare; struct ClientProtocolRequest protocol; struct ClientQueryRequest query; struct ClientReadRequest read; struct ClientReadVRequest readv; struct ClientRmRequest rm; struct ClientRmdirRequest rmdir; struct ClientSetRequest set; struct ClientSigverRequest sigver; struct ClientStatRequest stat; struct ClientSyncRequest sync; struct ClientTruncateRequest truncate; struct ClientWriteRequest write; struct ClientWriteVRequest writev; } ClientRequest; typedef union { struct ClientRequestHdr header; struct ClientSigverRequest sigver; } SecurityRequest; /******************************************************************************/ /* S e r v e r R e s p o n s e s */ /******************************************************************************/ // Nice header for the server response. // Note that the protocol specifies these values to be in network // byte order when sent // // G.Ganis: The following structures never need padding bytes: // no need of packing options // Server response codes // enum XResponseType { kXR_ok = 0, kXR_oksofar = 4000, kXR_attn, // 4001 kXR_authmore,// 4002 kXR_error, // 4003 kXR_redirect,// 4004 kXR_wait, // 4005 kXR_waitresp,// 4006 kXR_status, // 4007 kXR_noResponsesYet = 10000 }; // All serer responses start with the same header // struct ServerResponseHeader { kXR_char streamid[2]; kXR_unt16 status; kXR_int32 dlen; }; // This is a bit of wierdness held over from the very old days, sigh. // struct ServerResponseBody_Buffer { char data[4096]; }; /******************************************************************************/ /* k X R _ a t t n R e s p o n s e */ /******************************************************************************/ enum XActionCode { kXR_asyncab = 5000, // No longer supported kXR_asyncdi, // 5001 No longer supported kXR_asyncms = 5002, kXR_asyncrd, // 5003 No longer supported kXR_asyncwt, // 5004 No longer supported kXR_asyncav, // 5005 No longer supported kXR_asynunav, // 5006 No longer supported kXR_asyncgo, // 5007 No longer supported kXR_asynresp= 5008 }; struct ServerResponseBody_Attn { kXR_int32 actnum; // See XActionCode enum char parms[4096]; // Should be sufficient for every use }; struct ServerResponseBody_Attn_asyncms { kXR_int32 actnum; // XActionCode::kXR_asyncms char reserved[4]; ServerResponseHeader resphdr; char respdata[4096]; }; struct ServerResponseBody_Attn_asynresp { kXR_int32 actnum; // XActionCode::kXR_asynresp char reserved[4]; ServerResponseHeader resphdr; char respdata[4096]; }; /******************************************************************************/ /* k X R _ a u t h m o r e R e s p o n s e */ /******************************************************************************/ struct ServerResponseBody_Authmore { char data[4096]; }; /******************************************************************************/ /* k X R _ b i n d R e s p o n s e */ /******************************************************************************/ struct ServerResponseBody_Bind { kXR_char substreamid; }; /******************************************************************************/ /* k X R _ c h k p o i n t R e s p o n s e */ /******************************************************************************/ struct ServerResponseBody_ChkPoint { // Only for kXR_ckpQMax kXR_unt32 maxCkpSize; // Maximum number of bytes including overhead kXR_unt32 useCkpSize; // The number of bytes already being used }; /******************************************************************************/ /* k X R _ e r r o r R e s p o n s e */ /******************************************************************************/ enum XErrorCode { kXR_ArgInvalid = 3000, kXR_ArgMissing, // 3001 kXR_ArgTooLong, // 3002 kXR_FileLocked, // 3003 kXR_FileNotOpen, // 3004 kXR_FSError, // 3005 kXR_InvalidRequest, // 3006 kXR_IOError, // 3007 kXR_NoMemory, // 3008 kXR_NoSpace, // 3009 kXR_NotAuthorized, // 3010 kXR_NotFound, // 3011 kXR_ServerError, // 3012 kXR_Unsupported, // 3013 kXR_noserver, // 3014 kXR_NotFile, // 3015 kXR_isDirectory, // 3016 kXR_Cancelled, // 3017 kXR_ItExists, // 3018 kXR_ChkSumErr, // 3019 kXR_inProgress, // 3020 kXR_overQuota, // 3021 kXR_SigVerErr, // 3022 kXR_DecryptErr, // 3023 kXR_Overloaded, // 3024 kXR_fsReadOnly, // 3025 kXR_BadPayload, // 3026 kXR_AttrNotFound, // 3027 kXR_TLSRequired, // 3028 kXR_noReplicas, // 3029 kXR_AuthFailed, // 3030 kXR_Impossible, // 3031 kXR_Conflict, // 3032 kXR_TooManyErrs, // 3033 kXR_ReqTimedOut, // 3034 kXR_ERRFENCE, // Always last valid errcode + 1 kXR_noErrorYet = 10000 }; struct ServerResponseBody_Error { kXR_int32 errnum; // See XErrorCode enu char errmsg[4096]; // Should be sufficient for every use }; /******************************************************************************/ /* k X R _ l o g i n R e s p o n s e */ /******************************************************************************/ struct ServerResponseBody_Login { kXR_char sessid[16]; kXR_char sec[4096]; // Should be sufficient for every use }; /******************************************************************************/ /* k X R _ o p e n R e s p o n s e */ /******************************************************************************/ struct ServerResponseBody_Open { kXR_char fhandle[4]; kXR_int32 cpsize; // cpsize & cptype returned if kXR_compress *or* kXR_char cptype[4]; // kXR_retstat is specified }; // info will follow if kXR_retstat is specified /******************************************************************************/ /* k X R _ p g r e a d R e s p o n s e */ /******************************************************************************/ struct ServerResponseBody_pgRead { kXR_int64 offset; // info[]: File offset of data that follows // kXR_char data[dlen]; }; /******************************************************************************/ /* k X R _ p g w r i t e R e s p o n s e */ /******************************************************************************/ struct ServerResponseBody_pgWrite { kXR_int64 offset; // info[]: File offset of data written }; // The following structure is appended to ServerResponseBody_pgWrite if one or // more checksum errors occurred and need to be retransmitted. // struct ServerResponseBody_pgWrCSE { kXR_unt32 cseCRC; // crc32c of all following bits kXR_int16 dlFirst; // Data length at first offset in list kXR_int16 dlLast; // Data length at last offset in list // kXR_int64 bof[(dlen-8)/8]; // List of offsets of pages in error }; /******************************************************************************/ /* k X R _ p r o t o c o l R e s p o n s e */ /******************************************************************************/ // The following information is returned in the response body when kXR_bifreqs // is set in ClientProtocolRequest::flags. Note that the size of bifInfo is // is variable. This response will not be returned if there are no bif's. // Note: This structure is null byte padded to be a multiple of 8 bytes! // struct ServerResponseBifs_Protocol { kXR_char theTag; // Always the character 'B' to identify struct kXR_char rsvd; // Reserved for the future (always 0 for now) kXR_unt16 bifILen; // Length of bifInfo including null bytes. // kXR_char bifInfo[bifILen]; }; // The following information is returned in the response body when kXR_secreqs // is set in ClientProtocolRequest::flags. Note that the size of secvec is // defined by secvsz and will not be present when secvsz == 0. // struct ServerResponseSVec_Protocol { kXR_char reqindx; // Request index kXR_char reqsreq; // Request signing requirement }; struct ServerResponseReqs_Protocol { kXR_char theTag; // Always the character 'S' to identify struct kXR_char rsvd; // Reserved for the future (always 0 for now) kXR_char secver; // Security version kXR_char secopt; // Security options kXR_char seclvl; // Security level when secvsz == 0 kXR_char secvsz; // Number of items in secvec (i.e. its length/2) ServerResponseSVec_Protocol secvec; }; namespace XrdProto { typedef struct ServerResponseBifs_Protocol bifReqs; typedef struct ServerResponseReqs_Protocol secReqs; } // Options reflected in protocol response ServerResponseReqs_Protocol::secopt // #define kXR_secOData 0x01 #define kXR_secOFrce 0x02 // Security level definitions (these are predefined but can be over-ridden) // #define kXR_secNone 0 #define kXR_secCompatible 1 #define kXR_secStandard 2 #define kXR_secIntense 3 #define kXR_secPedantic 4 // Requirements one of which set in each ServerResponseReqs_Protocol::secvec // #define kXR_signIgnore 0 #define kXR_signLikely 1 #define kXR_signNeeded 2 // Version used for kXR_sigver and is set in SigverRequest::version, // ServerResponseReqs_Protocol::secver // #define kXR_secver_0 0 // KINDS of SERVERS (no longer used by new clients) // #define kXR_DataServer 1 #define kXR_LBalServer 0 // The below are defined for protocol version 2.9.7 or higher // These are the flag values in the kXR_protool response // #define kXR_isManager 0x00000002 #define kXR_isServer 0x00000001 #define kXR_attrMeta 0x00000100 #define kXR_attrProxy 0x00000200 #define kXR_attrSuper 0x00000400 #define kXR_attrVirtRdr 0x00000800 // Virtual options set on redirect // #define kXR_recoverWrts 0x00001000 #define kXR_collapseRedir 0x00002000 #define kXR_ecRedir 0x00004000 // Things the server supports // #define kXR_anongpf 0x00800000 #define kXR_supgpf 0x00400000 #define kXR_suppgrw 0x00200000 #define kXR_supposc 0x00100000 // TLS requirements // #define kXR_haveTLS 0x80000000 #define kXR_gotoTLS 0x40000000 #define kXR_tlsAny 0x1f000000 #define kXR_tlsData 0x01000000 #define kXR_tlsGPF 0x02000000 #define kXR_tlsLogin 0x04000000 #define kXR_tlsSess 0x08000000 #define kXR_tlsTPC 0x10000000 #define kXR_tlsGPFA 0x20000000 // Body for the kXR_protocol response... useful // struct ServerResponseBody_Protocol { kXR_int32 pval; kXR_int32 flags; ServerResponseReqs_Protocol secreq; // Only for V3.1.0+ && if requested }; // Handy definition of the size of the protocol response when the security // information is not present. // #define kXR_ShortProtRespLen sizeof(ServerResponseBody_Protocol)-\ sizeof(ServerResponseReqs_Protocol) /******************************************************************************/ /* k X R _ r e d i r e c t R e s p o n s e */ /******************************************************************************/ struct ServerResponseBody_Redirect { kXR_int32 port; char host[4096]; // Should be sufficient for every use }; /******************************************************************************/ /* k X R _ s t a t R e s p o n s e */ /******************************************************************************/ // The following bits are encoded in the "flags" token in the response // enum XStatRespFlags { kXR_file = 0, kXR_xset = 1, kXR_isDir = 2, kXR_other = 4, kXR_offline = 8, kXR_readable=16, kXR_writable=32, kXR_poscpend=64, kXR_bkpexist=128 }; /******************************************************************************/ /* k X R _ s t a t u s R e s p o n s e */ /******************************************************************************/ struct ServerResponseBody_Status { // Always preceeded by ServerResponseHeader kXR_unt32 crc32c; // IETF RFC 7143 standard kXR_char streamID[2]; // Identical to streamid[2] in ServerResponseHeader kXR_char requestid; // requestcode - kXR_1stRequest kXR_char resptype; // See RespType enum below kXR_char reserved[4]; kXR_int32 dlen; // kXR_char info[ServerResponseHeader::dlen-sizeof(ServerResponseBody_Status)]; // kXR_char data[dlen]; }; namespace XrdProto { enum RespType { kXR_FinalResult = 0x00, kXR_PartialResult = 0x01, kXR_ProgressInfo = 0x02 }; // This is the minimum size of ServerResponseHeader::dlen for kXR_status // static const int kXR_statusBodyLen = sizeof(ServerResponseBody_Status); } struct ServerResponseStatus { struct ServerResponseHeader hdr; struct ServerResponseBody_Status bdy; }; /******************************************************************************/ /* k X R _ w a i t R e s p o n s e */ /******************************************************************************/ struct ServerResponseBody_Wait { kXR_int32 seconds; char infomsg[4096]; // Should be sufficient for every use }; /******************************************************************************/ /* k X R _ w a i t r e s p R e s p o n s e */ /******************************************************************************/ struct ServerResponseBody_Waitresp { kXR_int32 seconds; }; /******************************************************************************/ /* U n i o n o f a l l S e r v e r R e s p o n s e s */ /******************************************************************************/ struct ServerResponse { ServerResponseHeader hdr; union { ServerResponseBody_Attn attn; ServerResponseBody_Authmore authmore; ServerResponseBody_Bind bind; ServerResponseBody_Buffer buffer; ServerResponseBody_Error error; ServerResponseBody_Login login; ServerResponseBody_Protocol protocol; ServerResponseBody_Redirect redirect; ServerResponseBody_Status status; ServerResponseBody_Wait wait; ServerResponseBody_Waitresp waitresp; } body; }; // The pgread and pgwrite do not fit the union above because they are composed // of three structs not two as all the above. So, we define the exceptions here. // struct ServerResponseV2 { ServerResponseStatus status; // status.bdy and status.hdr union { ServerResponseBody_pgRead pgread; ServerResponseBody_pgWrite pgwrite; } info; }; struct ALIGN_CHECK {char chkszreq[25-sizeof(ClientRequest)]; char chkszrsp[ 9-sizeof(ServerResponseHeader)]; }; /******************************************************************************/ /* X P r o t o c o l U t i l i t i e s */ /******************************************************************************/ #include #if defined(WIN32) #if !defined(ENOTBLK) # define ENOTBLK 15 #endif #if !defined(ETXTBSY) #define ETXTBSY 26 #endif #if !defined(ENOBUFS) #define ENOBUFS 105 #endif #if !defined(ENETUNREACH) #define ENETUNREACH 114 #endif #endif #ifndef ENOATTR #define ENOATTR ENODATA #endif #ifndef EBADRQC #define EBADRQC EBADRPC #endif #ifndef EAUTH #define EAUTH EBADE #endif struct stat; class XProtocol { public: // mapError() is the official mapping from errno to xroot protocol error. // static int mapError(int rc) {if (rc < 0) rc = -rc; switch(rc) {case ENOENT: return kXR_NotFound; case EINVAL: return kXR_ArgInvalid; case EPERM: return kXR_NotAuthorized; case EACCES: return kXR_NotAuthorized; case EIO: return kXR_IOError; case ENOMEM: return kXR_NoMemory; case ENOBUFS: return kXR_NoMemory; case ENOSPC: return kXR_NoSpace; case ENAMETOOLONG: return kXR_ArgTooLong; case ENETUNREACH: return kXR_noserver; case ENOTBLK: return kXR_NotFile; case ENOTSUP: return kXR_Unsupported; case EISDIR: return kXR_isDirectory; case ENOTEMPTY: [[fallthrough]]; // In the case one tries to delete a non-empty directory // we have decided that until the next major release // the kXR_ItExists flag will be returned case EEXIST: return kXR_ItExists; case EBADRQC: return kXR_InvalidRequest; case ETXTBSY: return kXR_inProgress; case ENODEV: return kXR_FSError; case EFAULT: return kXR_ServerError; case EDOM: return kXR_ChkSumErr; case EDQUOT: return kXR_overQuota; case EILSEQ: return kXR_SigVerErr; case ERANGE: return kXR_DecryptErr; case EUSERS: return kXR_Overloaded; case EROFS: return kXR_fsReadOnly; case ENOATTR: return kXR_AttrNotFound; case EPROTOTYPE: return kXR_TLSRequired; case EADDRNOTAVAIL: return kXR_noReplicas; case EAUTH: return kXR_AuthFailed; case EIDRM: return kXR_Impossible; case ENOTTY: return kXR_Conflict; case ETOOMANYREFS: return kXR_TooManyErrs; case ETIMEDOUT: return kXR_ReqTimedOut; case EBADF: return kXR_FileNotOpen; case ECANCELED: return kXR_Cancelled; default: return kXR_FSError; } } static int toErrno( int xerr ) { switch(xerr) {case kXR_ArgInvalid: return EINVAL; case kXR_ArgMissing: return EINVAL; case kXR_ArgTooLong: return ENAMETOOLONG; case kXR_FileLocked: return EDEADLK; case kXR_FileNotOpen: return EBADF; case kXR_FSError: return ENODEV; case kXR_InvalidRequest:return EBADRQC; case kXR_IOError: return EIO; case kXR_NoMemory: return ENOMEM; case kXR_NoSpace: return ENOSPC; case kXR_NotAuthorized: return EACCES; case kXR_NotFound: return ENOENT; case kXR_ServerError: return EFAULT; case kXR_Unsupported: return ENOTSUP; case kXR_noserver: return EHOSTUNREACH; case kXR_NotFile: return ENOTBLK; case kXR_isDirectory: return EISDIR; case kXR_Cancelled: return ECANCELED; case kXR_ItExists: return EEXIST; case kXR_ChkSumErr: return EDOM; case kXR_inProgress: return EINPROGRESS; case kXR_overQuota: return EDQUOT; case kXR_SigVerErr: return EILSEQ; case kXR_DecryptErr: return ERANGE; case kXR_Overloaded: return EUSERS; case kXR_fsReadOnly: return EROFS; case kXR_BadPayload: return EINVAL; case kXR_AttrNotFound: return ENOATTR; case kXR_TLSRequired: return EPROTOTYPE; case kXR_noReplicas: return EADDRNOTAVAIL; case kXR_AuthFailed: return EAUTH; case kXR_Impossible: return EIDRM; case kXR_Conflict: return ENOTTY; case kXR_TooManyErrs: return ETOOMANYREFS; case kXR_ReqTimedOut: return ETIMEDOUT; default: return ENOMSG; } } static const char *errName(kXR_int32 errCode); static const char *reqName(kXR_unt16 reqCode); /******************************************************************************/ /* O b s o l e t e D e f i n i t i o n s */ /******************************************************************************/ struct ServerResponseBody_Attn_asyncdi { // No longer supported kXR_int32 actnum; kXR_int32 wsec; kXR_int32 msec; }; struct ServerResponseBody_Attn_asyncrd { // No longer supported kXR_int32 actnum; kXR_int32 port; char host[4092]; }; struct ServerResponseBody_Attn_asyncwt { // No longer supported kXR_int32 actnum; kXR_int32 wsec; }; // Kind of error inside a XTNetFile's routine (temporary) // enum XReqErrorType { kGENERICERR = 0, // Generic error kREAD, // Error while reading from stream kWRITE, // Error while writing to stream kREDIRCONNECT, // Error redirecting to a given host kOK, // Everything seems ok kNOMORESTREAMS // No more available stream IDs for // async processing }; typedef kXR_int32 ServerResponseType; #define kXR_maxReqRetry 10 }; // XProtocol #endif xrootd-5.6.9/src/XProtocol/XPtypes.hh000066400000000000000000000127061457266313600175520ustar00rootroot00000000000000#ifndef __XPTYPES_H #define __XPTYPES_H /******************************************************************************/ /* */ /* X P t y p e s . h h */ /* */ /* (c) 2012 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* The XRootD protocol definition, documented in this file, is distributed */ /* under a modified BSD license and may be freely used to reimplement it. */ /* Any references to "source" in this license refers to this file or any */ /* other file that specifically contains the following license. */ /* */ /* Redistribution and use in source and binary forms, with or without */ /* modification, are permitted provided that the following conditions */ /* are met: */ /* */ /* 1. Redistributions of source code must retain the above copyright notice, */ /* this list of conditions and the following disclaimer. */ /* */ /* 2. Redistributions in binary form must reproduce the above copyright */ /* notice, this list of conditions and the following disclaimer in the */ /* documentation and/or other materials provided with the distribution. */ /* */ /* 3. Neither the name of the copyright holder nor the names of its */ /* contributors may be used to endorse or promote products derived from */ /* this software without specific prior written permission. */ /* */ /* 4. Derived software may not use the name XRootD or cmsd (regardless of */ /* capitilization) in association with the derived work if the protocol */ /* documented in this file is changed in any way. */ /* */ /* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS */ /* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT */ /* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR */ /* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT */ /* HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, */ /* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT */ /* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, */ /* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY */ /* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */ /* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE */ /******************************************************************************/ // Full range type compatibility work done by Gerardo Ganis, CERN. // Typical data types // // Only char and short are truly portable types typedef unsigned char kXR_char; typedef short kXR_int16; typedef unsigned short kXR_unt16; // Signed integer 4 bytes // #ifndef XR__INT16 # if defined(LP32) || defined(__LP32) || defined(__LP32__) || \ defined(BORLAND) # define XR__INT16 # endif #endif #ifndef XR__INT64 # if defined(ILP64) || defined(__ILP64) || defined(__ILP64__) # define XR__INT64 # endif #endif #if defined(XR__INT16) typedef long kXR_int32; typedef unsigned long kXR_unt32; #elif defined(XR__INT64) typedef int32 kXR_int32; typedef unsigned int32 kXR_unt32; #else typedef int kXR_int32; typedef unsigned int kXR_unt32; #endif // Signed integer 8 bytes // //#if defined(_WIN32) //typedef __int64 kXR_int64; //#else typedef long long kXR_int64; typedef unsigned long long kXR_unt64; //#endif #endif xrootd-5.6.9/src/XProtocol/YProtocol.hh000066400000000000000000000613711457266313600200720ustar00rootroot00000000000000#ifndef __YPROTOCOL_H #define __YPROTOCOL_H /******************************************************************************/ /* */ /* Y P r o t o c o l . h h */ /* */ /* (c) 2012 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* The XRootD protocol definition, documented in this file, is distributed */ /* under a modified BSD license and may be freely used to reimplement it. */ /* Any references to "source" in this license refers to this file or any */ /* other file that specifically contains the following license. */ /* */ /* Redistribution and use in source and binary forms, with or without */ /* modification, are permitted provided that the following conditions */ /* are met: */ /* */ /* 1. Redistributions of source code must retain the above copyright notice, */ /* this list of conditions and the following disclaimer. */ /* */ /* 2. Redistributions in binary form must reproduce the above copyright */ /* notice, this list of conditions and the following disclaimer in the */ /* documentation and/or other materials provided with the distribution. */ /* */ /* 3. Neither the name of the copyright holder nor the names of its */ /* contributors may be used to endorse or promote products derived from */ /* this software without specific prior written permission. */ /* */ /* 4. Derived software may not use the name XRootD or cmsd (regardless of */ /* capitilization) in association with the derived work if the protocol */ /* documented in this file is changed in any way. */ /* */ /* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS */ /* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT */ /* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR */ /* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT */ /* HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, */ /* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT */ /* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, */ /* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY */ /* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */ /* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE */ /******************************************************************************/ #ifdef __CINT__ #define __attribute__(x) #endif #include "XProtocol/XPtypes.hh" // We need to pack structures sent all over the net! // __attribute__((packed)) assures no padding bytes. // // Note all binary values shall be in network byte order. // // Data is serialized as explained in XrdOucPup. /******************************************************************************/ /* C o m m o n R e q u e s t S e c t i o n */ /******************************************************************************/ namespace XrdCms { static const unsigned char kYR_Version = 3; struct CmsRRHdr { kXR_unt32 streamid; // Essentially opaque kXR_char rrCode; // Request or Response code kXR_char modifier; // RR dependent kXR_unt16 datalen; }; enum CmsReqCode // Request Codes { kYR_login = 0, // Same as kYR_data kYR_chmod = 1, kYR_locate = 2, kYR_mkdir = 3, kYR_mkpath = 4, kYR_mv = 5, kYR_prepadd = 6, kYR_prepdel = 7, kYR_rm = 8, kYR_rmdir = 9, kYR_select = 10, kYR_stats = 11, kYR_avail = 12, kYR_disc = 13, kYR_gone = 14, kYR_have = 15, kYR_load = 16, kYR_ping = 17, kYR_pong = 18, kYR_space = 19, kYR_state = 20, kYR_statfs = 21, kYR_status = 22, kYR_trunc = 23, kYR_try = 24, kYR_update = 25, kYR_usage = 26, kYR_xauth = 27, kYR_MaxReq // Count of request numbers (highest + 1) }; // The hopcount is used for forwarded requests. It is incremented upon each // forwarding until it wraps to zero. At this point the forward is not done. // Forwarding applies to: chmod, have, mkdir, mkpath, mv, prepdel, rm, and // rmdir. Any other modifiers must be encoded in the low order 6 bits. // enum CmsFwdModifier { kYR_hopcount = 0xc0, kYR_hopincr = 0x40 }; enum CmsReqModifier { kYR_raw = 0x20, // Modifier: Unmarshalled data kYR_dnf = 0x10 // Modifier: mv, rm, rmdir (do not forward) }; /******************************************************************************/ /* C o m m o n R e s p o n s e S e c t i o n */ /******************************************************************************/ enum CmsRspCode // Response codes { kYR_data = 0, // Same as kYR_login kYR_error = 1, kYR_redirect= 2, kYR_wait = 3, kYR_waitresp= 4, kYR_yauth = 5 }; enum YErrorCode { kYR_ENOENT = 1, // -> ENOENT kYR_EPERM, // -> ENOENT kYR_EACCES, // -> EACCES kYR_EINVAL, // -> EINVALO kYR_EIO, // -> EIO kYR_ENOMEM, // -> ENOMEM kYR_ENOSPC, // -> ENOSPC kYR_ENAMETOOLONG, // -> ENAMETOOLONG kYR_ENETUNREACH, // -> ENETUNREACH kYR_ENOTBLK, // -> ENOTBLK kYR_EISDIR, // -> EISDIR kYR_FSError, // -> ENODEV kYR_SrvError, // -> EFAULT kYR_RWConflict, // -> EEXIST kYR_noReplicas // -> EADDRNOTAVAIL }; struct CmsResponse { CmsRRHdr Hdr; enum {kYR_async = 128 // Modifier: Reply to prev waitresp }; kXR_unt32 Val; // Port, Wait val, rc, asyncid // kXR_char Data[Hdr.datalen-4];// Target host, more data, or emessage }; /******************************************************************************/ /* a v a i l R e q u e s t */ /******************************************************************************/ // Request: avail // Respond: n/a // struct CmsAvailRequest { CmsRRHdr Hdr; // kXR_int32 diskFree; // kXR_int32 diskUtil; }; /******************************************************************************/ /* c h m o d R e q u e s t */ /******************************************************************************/ // Request: chmod // Respond: n/a // struct CmsChmodRequest { CmsRRHdr Hdr; // kXR_string Ident; // kXR_string Mode; // kXR_string Path; }; /******************************************************************************/ /* d i s c R e q u e s t */ /******************************************************************************/ // Request: disc // Respond: n/a // struct CmsDiscRequest { CmsRRHdr Hdr; }; /******************************************************************************/ /* g o n e R e q u e s t */ /******************************************************************************/ // Request: gone // Respond: n/a // struct CmsGoneRequest { CmsRRHdr Hdr; // kXR_string Path; }; /******************************************************************************/ /* h a v e R e q u e s t */ /******************************************************************************/ // Request: have // Respond: n/a // struct CmsHaveRequest { CmsRRHdr Hdr; enum {Online = 1, Pending = 2}; // Modifiers // kXR_string Path; }; /******************************************************************************/ /* l o c a t e R e q u e s t */ /******************************************************************************/ struct CmsLocateRequest { CmsRRHdr Hdr; // kXR_string Ident; // kXR_unt32 Opts; enum {kYR_refresh = 0x0001, kYR_retname = 0x0002, kYR_retuniq = 0x0004, kYR_asap = 0x0080, kYR_retipv4 = 0x0000, // Client is only IPv4 kYR_retipv46= 0x1000, // Client is IPv4 IPv6 kYR_retipv6 = 0x2000, // Client is only IPv6 kYR_retipv64= 0x3000, // Client is IPv6 IPv4 kYR_retipmsk= 0x3000, // Mask to isolate retipcxx bits kYR_retipsft= 12, // Shift to convert retipcxx bits kYR_listall = 0x4000, // List everything regardless of other settings kYR_prvtnet = 0x8000 // Client is using a private address }; // kXR_string Path; static const int RHLen =266; // Max length of each host response item }; /******************************************************************************/ /* l o g i n R e q u e s t */ /******************************************************************************/ // Request: login // Respond: xauth // login // struct CmsLoginData { kXR_unt16 Size; // Temp area for packing purposes kXR_unt16 Version; kXR_unt32 Mode; // From LoginMode kXR_int32 HoldTime; // Hold time in ms(managers) kXR_unt32 tSpace; // Tot Space GB (servers) kXR_unt32 fSpace; // Free Space MB (servers) kXR_unt32 mSpace; // Minf Space MB (servers) kXR_unt16 fsNum; // File Systems (servers /supervisors) kXR_unt16 fsUtil; // FS Utilization (servers /supervisors) kXR_unt16 dPort; // Data port (servers /supervisors) kXR_unt16 sPort; // Subs port (managers/supervisors) kXR_char *SID; // Server ID (servers/ supervisors) kXR_char *Paths; // Exported paths (servers/ supervisors) kXR_char *ifList; // Exported interfaces kXR_char *envCGI; // Exported environment enum LoginMode {kYR_director= 0x00000001, kYR_manager = 0x00000002, kYR_peer = 0x00000004, kYR_server = 0x00000008, kYR_proxy = 0x00000010, kYR_subman = 0x00000020, kYR_blredir = 0x00000040, // Supports or is bl redir kYR_suspend = 0x00000100, // Suspended login kYR_nostage = 0x00000200, // Staging unavailable kYR_trying = 0x00000400, // Extensive login retries kYR_debug = 0x80000000, kYR_share = 0x7f000000, // Mask to isolate share kYR_shift = 24, // Share shift position kYR_tzone = 0x00f80000, // Mask to isolate time zone kYR_shifttz = 19 // TZone shift position }; }; struct CmsLoginRequest { CmsRRHdr Hdr; CmsLoginData Data; }; struct CmsLoginResponse { CmsRRHdr Hdr; CmsLoginData Data; }; /******************************************************************************/ /* l o a d R e q u e s t */ /******************************************************************************/ // Request: load // Respond: n/a // struct CmsLoadRequest { CmsRRHdr Hdr; enum {cpuLoad=0, netLoad, xeqLoad, memLoad, pagLoad, dskLoad, numLoad}; // kXR_char theLoad[numload]; // kXR_int dskFree; }; /******************************************************************************/ /* m k d i r R e q u e s t */ /******************************************************************************/ // Request: mkdir // Respond: n/a // struct CmsMkdirRequest { CmsRRHdr Hdr; // kXR_string Ident; // kXR_string Mode; // kXR_string Path; }; /******************************************************************************/ /* m k p a t h R e q u e s t */ /******************************************************************************/ // Request: mkpath // Respond: n/a // struct CmsMkpathRequest { CmsRRHdr Hdr; // kXR_string Ident; // kXR_string Mode; // kXR_string Path; }; /******************************************************************************/ /* m v R e q u e s t */ /******************************************************************************/ // Request: mv // Respond: n/a // struct CmsMvRequest { CmsRRHdr Hdr; // Subject to kYR_dnf modifier! // kXR_string Ident; // kXR_string Old_Path; // kXR_string New_Path; }; /******************************************************************************/ /* p i n g R e q u e s t */ /******************************************************************************/ // Request: ping // Respond: n/a // struct CmsPingRequest { CmsRRHdr Hdr; }; /******************************************************************************/ /* p o n g R e q u e s t */ /******************************************************************************/ // Request: pong // Respond: n/a // struct CmsPongRequest { CmsRRHdr Hdr; }; /******************************************************************************/ /* p r e p a d d R e q u e s t */ /******************************************************************************/ // Request: prepadd \n // Respond: No response. // struct CmsPrepAddRequest { CmsRRHdr Hdr; // Modifier used with following options enum {kYR_stage = 0x0001, // Stage the data kYR_write = 0x0002, // Prepare for writing kYR_coloc = 0x0004, // Prepare for co-location kYR_fresh = 0x0008, // Prepare by time refresh kYR_metaman = 0x0010 // Prepare via meta-manager }; // kXR_string Ident; // kXR_string reqid; // kXR_string user; // kXR_string prty; // kXR_string mode; // kXR_string Path; // kXR_string Opaque; // Optional }; /******************************************************************************/ /* p r e p d e l R e q u e s t */ /******************************************************************************/ // Request: prepdel // Respond: No response. // struct CmsPrepDelRequest { CmsRRHdr Hdr; // kXR_string Ident; // kXR_string reqid; }; /******************************************************************************/ /* r m R e q u e s t */ /******************************************************************************/ // Request: rm // Respond: n/a // struct CmsRmRequest { CmsRRHdr Hdr; // Subject to kYR_dnf modifier! // kXR_string Ident; // kXR_string Path; }; /******************************************************************************/ /* r m d i r R e q u e s t */ /******************************************************************************/ // Request: rmdir // Respond: n/a // struct CmsRmdirRequest { CmsRRHdr Hdr; // Subject to kYR_dnf modifier! // kXR_string Ident; // kXR_string Path; }; /******************************************************************************/ /* s e l e c t R e q u e s t */ /******************************************************************************/ // Request: select[s] {c | d | m | r | w | s | t | x} [-host] // Note: selects - requests a cache refresh for // kYR_refresh - refresh file location cache // kYR_create c - file will be created // kYR_delete d - file will be created or truncated // kYR_metaop m - inod will only be modified // kYR_read r - file will only be read // kYR_replica - file will replicated // kYR_write w - file will be read and writen // kYR_stats s - only stat information will be obtained // kYR_online x - consider only online files // may be combined with kYR_stats (file must be resident) // - - the host failed to deliver the file. struct CmsSelectRequest { CmsRRHdr Hdr; // kXR_string Ident; // kXR_unt32 Opts; enum {kYR_refresh = 0x00000001, kYR_create = 0x00000002, // May combine with trunc -> delete kYR_online = 0x00000004, kYR_read = 0x00000008, // Default kYR_trunc = 0x00000010, // -> write kYR_write = 0x00000020, kYR_stat = 0x00000040, // Exclsuive kYR_metaop = 0x00000080, kYR_replica = 0x00000100, // Only in combination with create kYR_mwfiles = 0x00000200, // Multiple writables files are OK kYR_retipv4 = 0x00000000, // Client is only IPv4 kYR_retipv46= 0x00001000, // Client is IPv4 IPv6 kYR_retipv6 = 0x00002000, // Client is only IPv6 kYR_retipv64= 0x00003000, // Client is IPv6 IPv4 kYR_retipmsk= 0x00003000, // Mask to isolate retipcxx bits kYR_retipsft= 12, // Shift to convert retipcxx bits kYR_prvtnet = 0x00008000, // Client is using a private address kYR_tryMISS = 0x00000000, // Retry due to missing file (triedrc=enoent) kYR_tryIOER = 0x00010000, // Retry due to I/O error (triedrc=ioerr) kYR_tryFSER = 0x00020000, // Retry due to FS error (triedrc=fserr) kYR_trySVER = 0x00030000, // Retry due to server error (triedrc=srverr) kYR_tryMASK = 0x00030000, // Mask to isolate retry reason kYR_trySHFT = 16, // Amount to shift right kYR_tryRSEL = 0x00040000, // Retry for reselection LCL (triedrc=resel) kYR_tryRSEG = 0x00080000, // Retry for reselection GBL (triedrc=resel) kYR_tryMSRC = 0x000C0000, // Retry for multisource operation kYR_aWeak = 0x00100000, // Affinity: weak kYR_aStrong = 0x00200000, // Affinity: strong kYR_aStrict = 0x00300000, // Affinity: strict kYR_aNone = 0x00400000, // Affinity: none kYR_aSpec = 0x00700000, // Mask to test if any affinity specified kYR_aPack = 0x00300000, // Mask to test if the affinity packs choice kYR_aWait = 0x00200000 // Mask to test if the affinity must wait }; // kXR_string Path; // kXR_string Opaque; // Optional // kXR_string Host; // Optional }; /******************************************************************************/ /* s p a c e R e q u e s t */ /******************************************************************************/ // Request: space // struct CmsSpaceRequest { CmsRRHdr Hdr; }; /******************************************************************************/ /* s t a t e R e q u e s t */ /******************************************************************************/ // Request: state // struct CmsStateRequest { CmsRRHdr Hdr; // kXR_string Path; enum {kYR_refresh = 0x01, // Modifier kYR_noresp = 0x02, kYR_metaman = 0x08 }; }; /******************************************************************************/ /* s t a t f s R e q u e s t */ /******************************************************************************/ // Request: statfs // struct CmsStatfsRequest { CmsRRHdr Hdr; // Modifier used with following options // kXR_string Path; enum {kYR_qvfs = 0x0001 // Virtual file system query }; }; /******************************************************************************/ /* s t a t s R e q u e s t */ /******************************************************************************/ // Request: stats or statsz (determined by modifier) // struct CmsStatsRequest { CmsRRHdr Hdr; enum {kYR_size = 1 // Modifier }; }; /******************************************************************************/ /* s t a t u s R e q u e s t */ /******************************************************************************/ // Request: status // struct CmsStatusRequest { CmsRRHdr Hdr; enum {kYR_Stage = 0x01, kYR_noStage = 0x02, // Modifier kYR_Resume = 0x04, kYR_Suspend = 0x08, kYR_Reset = 0x10 // Exclusive }; }; /******************************************************************************/ /* t r u n c R e q u e s t */ /******************************************************************************/ // Request: trunc // Respond: n/a // struct CmsTruncRequest { CmsRRHdr Hdr; // kXR_string Ident; // kXR_string Size; // kXR_string Path; }; /******************************************************************************/ /* t r y R e q u e s t */ /******************************************************************************/ // Request: try // struct CmsTryRequest { CmsRRHdr Hdr; kXR_unt16 sLen; // This is the string length in PUP format // kYR_string {ipaddr:port}[up to STMax]; enum {kYR_permtop = 0x01 // Modifier Permanent redirect to top level }; }; /******************************************************************************/ /* u p d a t e R e q u e s t */ /******************************************************************************/ // Request: update // struct CmsUpdateRequest { CmsRRHdr Hdr; }; /******************************************************************************/ /* u s a g e R e q u e s t */ /******************************************************************************/ // Request: usage // struct CmsUsageRequest { CmsRRHdr Hdr; }; }; // namespace XrdCms #endif xrootd-5.6.9/src/Xrd/000077500000000000000000000000001457266313600144135ustar00rootroot00000000000000xrootd-5.6.9/src/Xrd/XrdBuffXL.cc000066400000000000000000000214551457266313600165350ustar00rootroot00000000000000/******************************************************************************/ /* */ /* X r d B u f f X L . c c */ /* */ /* (c) 2015 by the Board of Trustees of the Leland Stanford, Jr., University */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include #include #include #include "XrdOuc/XrdOucUtils.hh" #include "XrdSys/XrdSysPlatform.hh" #include "XrdBuffXL.hh" /******************************************************************************/ /* L o c a l V a l u e s */ /******************************************************************************/ namespace { static const int maxBuffSz = 1 << 30; //1 GB static const int iniBuffSz = 1 << (XRD_BUSHIFT+XRD_BUCKETS-1); static const int minBuffSz = 1 << (XRD_BUSHIFT+XRD_BUCKETS); static const int minBShift = (XRD_BUSHIFT+XRD_BUCKETS); static const int isBigBuff = 0x40000000; } /******************************************************************************/ /* C o n s t r u c t o r */ /******************************************************************************/ XrdBuffXL::XrdBuffXL() : bucket(0), totalo(0), pagsz(getpagesize()), slots(0), maxsz(1<<(XRD_BUSHIFT+XRD_BUCKETS-1)), totreq(0) { } /******************************************************************************/ /* I n i t */ /******************************************************************************/ void XrdBuffXL::Init(int maxMSZ) { unsigned int lg2; int chunksz; // If this is a duplicate call, delete the previous setup // if (bucket) {delete [] bucket; bucket = 0;} // Check if this is too small for us // if (maxMSZ <= iniBuffSz) {maxsz = iniBuffSz; return;} // Check if this is too large for us (1GB limit) and adjust // if (maxMSZ > maxBuffSz) maxMSZ = maxBuffSz; // Calculate how many buckets we need to have (note we trim this down // chunksz = maxMSZ >> minBShift; lg2 = XrdOucUtils::Log2(chunksz); chunksz = 1<<(lg2+minBShift); if (chunksz < maxMSZ) {lg2++; maxsz = chunksz << 1;} else maxsz = chunksz; // Allocate a bucket array // bucket = new BuckVec[lg2+1]; slots = lg2+1; } /******************************************************************************/ /* O b t a i n */ /******************************************************************************/ XrdBuffer *XrdBuffXL::Obtain(int sz) { XrdBuffer *bp; char *memp; int mk, buffSz, bindex = 0; // Make sure the request is within our limits // if (sz <= 0 || sz > maxsz) return 0; // Calculate bucket index. This is log2(shifted size) rounded up if need be. // If the shift results in zero we know the request fits in the slot 0 buffer. // mk = sz >> minBShift; if (!mk) buffSz = minBuffSz; else {bindex = XrdOucUtils::Log2(mk); buffSz = (bindex ? minBuffSz << bindex : minBuffSz); if (buffSz < sz) {bindex++; buffSz = buffSz << 1;} } if (bindex >= slots) return 0; // Should never happen! // Obtain a lock on the bucket array and try to give away an existing buffer // slotXL.Lock(); totreq++; bucket[bindex].numreq++; if ((bp = bucket[bindex].bnext)) {bucket[bindex].bnext = bp->next; bucket[bindex].numbuf--;} slotXL.UnLock(); // Check if we really allocated a buffer // if (bp) return bp; // Allocate a chunk of aligned memory // if (posix_memalign((void **)&memp, pagsz, buffSz)) return 0; // Wrap the memory with a buffer object // if (!(bp = new XrdBuffer(memp, buffSz, bindex|isBigBuff))) {free(memp); return 0;} // Update statistics // slotXL.Lock(); totalo += buffSz; totbuf++; slotXL.UnLock(); // Return the buffer // return bp; } /******************************************************************************/ /* R e c a l c */ /******************************************************************************/ int XrdBuffXL::Recalc(int sz) { int buffSz, mk, bindex = 0; // Make sure the request is within our limits // if (sz <= 0 || sz > maxsz) return 0; // Calculate bucket size corresponding to the desired size // mk = sz >> minBShift; if (!mk) buffSz = minBuffSz; else {bindex = XrdOucUtils::Log2(mk); buffSz = (bindex ? minBuffSz << bindex : minBuffSz); if (buffSz < sz) {bindex++; buffSz = buffSz << 1;} } if (bindex >= slots) return 0; // Should never happen! // All done, return the actual size we would have allocated // return buffSz; } /******************************************************************************/ /* R e l e a s e */ /******************************************************************************/ void XrdBuffXL::Release(XrdBuffer *bp) { int bindex = bp->bindex & ~isBigBuff; // Obtain a lock on the bucket array and reclaim the buffer // slotXL.Lock(); bp->next = bucket[bindex].bnext; bucket[bindex].bnext = bp; bucket[bindex].numbuf++; slotXL.UnLock(); } /******************************************************************************/ /* S t a t s */ /******************************************************************************/ int XrdBuffXL::Stats(char *buff, int blen, int do_sync) { static char statfmt[] = "%d" "%lld%d"; int nlen; // If only size wanted, return it // if (!buff) return sizeof(statfmt) + 16*3; // Return formatted stats // if (do_sync) slotXL.Lock(); nlen = snprintf(buff, blen, statfmt, totreq, totalo, totbuf); if (do_sync) slotXL.UnLock(); return nlen; } /******************************************************************************/ /* T r i m */ /******************************************************************************/ void XrdBuffXL::Trim() { XrdBuffer *bP; int n, m; // Obtain the lock // slotXL.Lock(); // Run through all our slots looking for buffers to release // for (int i = 0; i < slots; i++) {if (bucket[i].numbuf > 1 && bucket[i].numbuf > bucket[i].numreq) {n = bucket[i].numbuf - bucket[i].numreq; m = bucket[i].numbuf/2; if (m < n) n = m; while(n-- && (bP = bucket[i].bnext)) {bucket[i].bnext = bP->next; bucket[i].numbuf--; totalo -= bP->bsize; totbuf--; delete bP; } } bucket[i].numreq = 0; } // Release the lock // slotXL.UnLock(); } xrootd-5.6.9/src/Xrd/XrdBuffXL.hh000066400000000000000000000063171457266313600165470ustar00rootroot00000000000000#ifndef __XrdBuffXL_H__ #define __XrdBuffXL_H__ /******************************************************************************/ /* */ /* X r d B u f f X L . h h */ /* */ /* (c) 2015 by the Board of Trustees of the Leland Stanford, Jr., University */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include "XrdBuffer.hh" #include "XrdSys/XrdSysPthread.hh" // There should be only one instance of this class. // class XrdBuffXL { public: void Init(int maxMSZ); XrdBuffer *Obtain(int bsz); int Recalc(int bsz); void Release(XrdBuffer *bp); int MaxSize() {return maxsz;} void Trim(); int Stats(char *buff, int blen, int do_sync=0); XrdBuffXL(); ~XrdBuffXL() {} // The buffmanager is never deleted private: XrdSysMutex slotXL; struct BuckVec {XrdBuffer *bnext; int numbuf; int numreq; BuckVec() : bnext(0), numbuf(0), numreq(0) {} ~BuckVec() {} // Never gets deleted }; BuckVec *bucket; // 4M**(0 ... slots-1) sized buffers long long totalo; const int pagsz; int slots; int maxsz; int totreq; int totbuf; }; #endif xrootd-5.6.9/src/Xrd/XrdBuffer.cc000066400000000000000000000270531457266313600166200ustar00rootroot00000000000000/******************************************************************************/ /* */ /* X r d B u f f e r . c c */ /* */ /* (c) 2004 by the Board of Trustees of the Leland Stanford, Jr., University */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include #include #include #include #include "XrdOuc/XrdOucUtils.hh" #include "XrdSys/XrdSysError.hh" #include "XrdSys/XrdSysPlatform.hh" #include "XrdSys/XrdSysTimer.hh" #include "Xrd/XrdBuffer.hh" #include "Xrd/XrdBuffXL.hh" #include "Xrd/XrdTrace.hh" /******************************************************************************/ /* E x t e r n a l L i n k a g e s */ /******************************************************************************/ void *XrdReshaper(void *pp) { XrdBuffManager *bmp = (XrdBuffManager *)pp; bmp->Reshape(); return (void *)0; } /******************************************************************************/ /* G l o b a l s */ /******************************************************************************/ const char *XrdBuffManager::TraceID = "BuffManager"; namespace { static const int minBuffSz = 1 << XRD_BUSHIFT; } namespace XrdGlobal { XrdBuffXL xlBuff; extern XrdSysError Log; } using namespace XrdGlobal; /******************************************************************************/ /* C o n s t r u c t o r */ /******************************************************************************/ XrdBuffManager::XrdBuffManager(int minrst) : slots(XRD_BUCKETS), shift(XRD_BUSHIFT), pagsz(getpagesize()), maxsz(1<<(XRD_BUSHIFT+XRD_BUCKETS-1)), Reshaper(0, "buff reshaper") { // Clear everything to zero // totbuf = 0; totreq = 0; totalo = 0; totadj = 0; #ifdef _SC_PHYS_PAGES maxalo = static_cast(pagsz)/8 * static_cast(sysconf(_SC_PHYS_PAGES)); #else maxalo = 0x7ffffff; #endif rsinprog = 0; minrsw = minrst; memset(static_cast(bucket), 0, sizeof(bucket)); } /******************************************************************************/ /* D e s t r u c t o r */ /******************************************************************************/ XrdBuffManager::~XrdBuffManager() { XrdBuffer *bP; for (int i = 0; i < XRD_BUCKETS; i++) {while((bP = bucket[i].bnext)) {bucket[i].bnext = bP->next; delete bP; } bucket[i].numbuf = 0; } } /******************************************************************************/ /* I n i t */ /******************************************************************************/ void XrdBuffManager::Init() { pthread_t tid; int rc; // Start the reshaper thread // if ((rc = XrdSysThread::Run(&tid, XrdReshaper, static_cast(this), 0, "Buffer Manager reshaper"))) Log.Emsg("BuffManager", rc, "create reshaper thread"); } /******************************************************************************/ /* O b t a i n */ /******************************************************************************/ XrdBuffer *XrdBuffManager::Obtain(int sz) { XrdBuffer *bp; char *memp; int mk, pk, bindex; // Make sure the request is within our limits // if (sz <= 0) return 0; if (sz > maxsz) return xlBuff.Obtain(sz); // Calculate bucket index // mk = sz >> shift; bindex = XrdOucUtils::Log2(mk); mk = minBuffSz << bindex; if (mk < sz) {bindex++; mk = mk << 1;} if (bindex >= slots) return 0; // Should never happen! // Obtain a lock on the bucket array and try to give away an existing buffer // Reshaper.Lock(); totreq++; bucket[bindex].numreq++; if ((bp = bucket[bindex].bnext)) {bucket[bindex].bnext = bp->next; bucket[bindex].numbuf--;} Reshaper.UnLock(); // Check if we really allocated a buffer // if (bp) return bp; // Allocate a chunk of aligned memory // pk = (mk < pagsz ? mk : pagsz); if (posix_memalign((void **)&memp, pk, mk)) return 0; // Wrap the memory with a buffer object // if (!(bp = new XrdBuffer(memp, mk, bindex))) {free(memp); return 0;} // Update statistics // Reshaper.Lock(); totbuf++; if ((totalo += mk) > maxalo && !rsinprog) {rsinprog = 1; Reshaper.Signal();} Reshaper.UnLock(); return bp; } /******************************************************************************/ /* R e c a l c */ /******************************************************************************/ int XrdBuffManager::Recalc(int sz) { int mk, bindex; // Make sure the request is within our limits // if (sz <= 0) return 0; if (sz > maxsz) return xlBuff.Recalc(sz); // Calculate bucket index // mk = sz >> shift; bindex = XrdOucUtils::Log2(mk); mk = minBuffSz << bindex; if (mk < sz) {bindex++; mk = mk << 1;} if (bindex >= slots) return 0; // Should never happen! // All done, return the actual size we would have allocated // return mk; } /******************************************************************************/ /* R e l e a s e */ /******************************************************************************/ void XrdBuffManager::Release(XrdBuffer *bp) { int bindex = bp->bindex; // Check if we should release this via the big buffer object // if (bindex >= slots) {xlBuff.Release(bp); return;} // Obtain a lock on the bucket array and reclaim the buffer // Reshaper.Lock(); bp->next = bucket[bp->bindex].bnext; bucket[bp->bindex].bnext = bp; bucket[bindex].numbuf++; Reshaper.UnLock(); } /******************************************************************************/ /* R e s h a p e */ /******************************************************************************/ void XrdBuffManager::Reshape() { int i, bufprof[XRD_BUCKETS], numfreed; time_t delta, lastshape = time(0); long long memslot, memhave, memtarget = (long long)(.80*(float)maxalo); XrdSysTimer Timer; float requests, buffers; XrdBuffer *bp; // This is an endless loop to periodically reshape the buffer pool // while(1) {Reshaper.Lock(); while(Reshaper.Wait(minrsw) && totalo <= maxalo) {TRACE(MEM, "Reshaper has " <<(totalo>>10) <<"K; target " <<(memtarget>>10) <<"K");} if ((delta = (time(0) - lastshape)) < minrsw) {Reshaper.UnLock(); Timer.Wait((minrsw-delta)*1000); Reshaper.Lock(); } // We have the lock so compute the request profile // if (totreq > slots) {requests = (float)totreq; buffers = (float)totbuf; for (i = 0; i < slots; i++) {bufprof[i] = (int)(buffers*(((float)bucket[i].numreq)/requests)); bucket[i].numreq = 0; } totreq = 0; memhave = totalo; } else memhave = 0; Reshaper.UnLock(); // Reshape the buffer pool to agree with the request profile // memslot = maxsz; numfreed = 0; for (i = slots-1; i >= 0 && memhave > memtarget; i--) {Reshaper.Lock(); while(bucket[i].numbuf > bufprof[i]) if ((bp = bucket[i].bnext)) {bucket[i].bnext = bp->next; delete bp; bucket[i].numbuf--; numfreed++; memhave -= memslot; totalo -= memslot; totbuf--; } else {bucket[i].numbuf = 0; break;} Reshaper.UnLock(); memslot = memslot>>1; } // All done // totadj += numfreed; TRACE(MEM, "Pool reshaped; " <>10) <<"K; target " <<(memtarget>>10) <<"K"); lastshape = time(0); rsinprog = 0; // No need to lock, we're the only ones now setting it xlBuff.Trim(); // Trim big buffers } } /******************************************************************************/ /* S e t */ /******************************************************************************/ void XrdBuffManager::Set(int maxmem, int minw) { // Obtain a lock and set the values // Reshaper.Lock(); if (maxmem > 0) maxalo = (long long)maxmem; if (minw > 0) minrsw = minw; Reshaper.UnLock(); } /******************************************************************************/ /* S t a t s */ /******************************************************************************/ int XrdBuffManager::Stats(char *buff, int blen, int do_sync) { static char statfmt[] = "%d" "%lld%d%d%s"; char xlStats[1024]; int nlen; // If only size wanted, return it // if (!buff) return sizeof(statfmt) + 16*4 + xlBuff.Stats(0,0); // Return formatted stats // if (do_sync) Reshaper.Lock(); xlBuff.Stats(xlStats, sizeof(xlStats), do_sync); nlen = snprintf(buff,blen,statfmt,totreq,totalo,totbuf,totadj,xlStats); if (do_sync) Reshaper.UnLock(); return nlen; } xrootd-5.6.9/src/Xrd/XrdBuffer.hh000066400000000000000000000103131457266313600166210ustar00rootroot00000000000000#ifndef __XrdBuffer_H__ #define __XrdBuffer_H__ /******************************************************************************/ /* */ /* X r d B u f f e r . h h */ /* */ /* (c) 2004 by the Board of Trustees of the Leland Stanford, Jr., University */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include #include #include "XrdSys/XrdSysPthread.hh" /******************************************************************************/ /* x r d _ B u f f e r */ /******************************************************************************/ class XrdBuffer { public: char * buff; // -> buffer int bsize; // size of this buffer XrdBuffer(char *bp, int sz, int ix) {buff = bp; bsize = sz; bindex = ix; next = 0;} ~XrdBuffer() {if (buff) free(buff);} friend class XrdBuffManager; friend class XrdBuffXL; private: int bindex; XrdBuffer *next; static int pagesz; }; /******************************************************************************/ /* x r d _ B u f f M a n a g e r */ /******************************************************************************/ #define XRD_BUCKETS 12 #define XRD_BUSHIFT 10 // There should be only one instance of this class per buffer pool. // class XrdBuffManager { public: void Init(); XrdBuffer *Obtain(int bsz); int Recalc(int bsz); void Release(XrdBuffer *bp); int MaxSize() {return maxsz;} void Reshape(); void Set(int maxmem=-1, int minw=-1); int Stats(char *buff, int blen, int do_sync=0); XrdBuffManager(int minrst=20*60); ~XrdBuffManager(); // The buffmanager is never deleted private: const int slots; const int shift; const int pagsz; const int maxsz; struct {XrdBuffer *bnext; int numbuf; int numreq; } bucket[XRD_BUCKETS]; // 1K to 1<<(szshift+slots-1)M buffers int totreq; int totbuf; long long totalo; long long maxalo; int minrsw; int rsinprog; int totadj; XrdSysCondVar Reshaper; static const char *TraceID; }; #endif xrootd-5.6.9/src/Xrd/XrdConfig.cc000066400000000000000000002671351457266313600166230ustar00rootroot00000000000000/*******************************************************************************/ /* */ /* X r d C o n f i g . c c */ /* */ /* (c) 2011 by the Board of Trustees of the Leland Stanford, Jr., University */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Deprtment of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ /* The default port number comes from: 1) The command line option, 2) The config file, 3) The /etc/services file for service corresponding to the program name. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "XrdVersion.hh" #include "Xrd/XrdBuffer.hh" #include "Xrd/XrdBuffXL.hh" #include "Xrd/XrdConfig.hh" #include "Xrd/XrdInet.hh" #include "Xrd/XrdInfo.hh" #include "Xrd/XrdLink.hh" #include "Xrd/XrdLinkCtl.hh" #include "Xrd/XrdPoll.hh" #include "Xrd/XrdScheduler.hh" #include "Xrd/XrdStats.hh" #include "Xrd/XrdTrace.hh" #include "XrdNet/XrdNetAddr.hh" #include "XrdNet/XrdNetIF.hh" #include "XrdNet/XrdNetSecurity.hh" #include "XrdNet/XrdNetUtils.hh" #include "XrdOuc/XrdOuca2x.hh" #include "XrdOuc/XrdOucEnv.hh" #include "XrdOuc/XrdOucLogging.hh" #include "XrdOuc/XrdOucPinKing.hh" #include "XrdOuc/XrdOucSiteName.hh" #include "XrdOuc/XrdOucStream.hh" #include "XrdOuc/XrdOucString.hh" #include "XrdOuc/XrdOucUtils.hh" #include "XrdSys/XrdSysError.hh" #include "XrdSys/XrdSysFD.hh" #include "XrdSys/XrdSysHeaders.hh" #include "XrdSys/XrdSysLogger.hh" #include "XrdSys/XrdSysTimer.hh" #include "XrdSys/XrdSysUtils.hh" #include "XrdTcpMonPin.hh" #include "XrdTls/XrdTls.hh" #include "XrdTls/XrdTlsContext.hh" #if defined(__linux__) || defined(__GNU__) #include #endif #if defined(__linux__) #include #endif #ifdef __APPLE__ #include #endif /******************************************************************************/ /* G l o b a l O b j e c t s */ /******************************************************************************/ namespace XrdGlobal { XrdOucString totalCF; extern XrdSysLogger Logger; extern XrdSysError Log; extern XrdSysTrace XrdTrace; extern XrdScheduler Sched; extern XrdBuffManager BuffPool; extern XrdTlsContext *tlsCtx; extern XrdInet *XrdNetTCP; extern XrdBuffXL xlBuff; extern XrdTcpMonPin *TcpMonPin; extern int devNull; }; using namespace XrdGlobal; namespace XrdNetSocketCFG { extern int ka_Idle; extern int ka_Itvl; extern int ka_Icnt; }; /******************************************************************************/ /* F i l e L o c a l O b j e c t s */ /******************************************************************************/ namespace { XrdOucEnv theEnv; XrdVERSIONINFODEF(myVer, Xrd, XrdVNUMBER, XrdVERSION); bool SSLmsgs = true; void TlsError(const char *tid, const char *msg, bool sslmsg) {if (!sslmsg || SSLmsgs) XrdGlobal::Log.Emsg("TLS", tid, msg);} }; /******************************************************************************/ /* S t a t i c M e m b e r s */ /******************************************************************************/ const char *XrdConfig::TraceID = "Config"; /******************************************************************************/ /* d e f i n e s */ /******************************************************************************/ #define TS_Xeq(x,m) if (!strcmp(x,var)) return m(eDest, Config); #ifndef S_IAMB #define S_IAMB 0x1FF #endif /******************************************************************************/ /* L o c a l C l a s s e s */ /******************************************************************************/ /******************************************************************************/ /* X r d C o n f i g P r o t */ /******************************************************************************/ class XrdConfigProt { public: XrdConfigProt *Next; char *proname; char *libpath; char *parms; int numP; union {int port; int portVec[XrdProtLoad::PortoMax]; }; union {bool dotls; bool tlsVec[XrdProtLoad::PortoMax]; }; bool AddPort(int pnum, bool isTLS) {for (int i = 0; i < numP; i++) if (pnum == portVec[i]) {tlsVec[i] = isTLS; return true;} if (numP >= (XrdProtLoad::PortoMax)) return false; portVec[numP] = pnum; tlsVec[numP] = isTLS; numP++; return true; } void Reset(char *ln, char *pp, int np=-1, bool to=false) {if (libpath) free(libpath); libpath = ln; if (parms) free(parms); parms = pp; memset(portVec, 0, sizeof(portVec)); port = np; memset(tlsVec, 0, sizeof(tlsVec)); dotls = to; numP = 1; } XrdConfigProt(char *pn, char *ln, char *pp, int np=-1, bool to=false) : Next(0), proname(pn), libpath(ln), parms(pp), numP(1) {memset(portVec, 0, sizeof(portVec)); port = np; memset(tlsVec, 0, sizeof(tlsVec)); dotls = to; } ~XrdConfigProt() {free(proname); if (libpath) free(libpath); if (parms) free(parms); } }; /******************************************************************************/ /* X r d T c p M o n I n f o */ /******************************************************************************/ class XrdTcpMonInfo { public: XrdOucPinKing KingPin; XrdTcpMonInfo(const char *drctv, const char *cfn, XrdSysError &errR) : KingPin(drctv, theEnv, errR, &myVer) {theEnv.Put("configFN", cfn);} ~XrdTcpMonInfo() {} XrdOucEnv theEnv; }; /******************************************************************************/ /* C o n s t r u c t o r */ /******************************************************************************/ XrdConfig::XrdConfig() { // Preset all variables with common defaults // PortTCP = -1; PortUDP = -1; PortTLS = -1; ConfigFN = 0; tmoInfo = 0; myInsName= 0; mySitName= 0; AdminPath= strdup("/tmp"); HomePath = 0; PidPath = strdup("/tmp"); tlsCert = 0; tlsKey = 0; caDir = 0; caFile = 0; AdminMode= S_IRWXU; HomeMode = S_IRWXU; Police = 0; Net_Opts = XRDNET_KEEPALIVE; TLS_Blen = 0; // Accept OS default (leave Linux autotune in effect) TLS_Opts = XRDNET_KEEPALIVE | XRDNET_USETLS; repDest[0] = 0; repDest[1] = 0; repInt = 600; repOpts = 0; ppNet = 0; tlsOpts = 9ULL | XrdTlsContext::servr | XrdTlsContext::logVF; tlsNoVer = false; tlsNoCAD = true; NetADM = 0; coreV = 1; Specs = 0; isStrict = false; maxFD = 256*1024; // 256K default Firstcp = Lastcp = 0; ProtInfo.eDest = &Log; // Stable -> Error Message/Logging Handler ProtInfo.NetTCP = 0; // Stable -> Network Object ProtInfo.BPool = &BuffPool; // Stable -> Buffer Pool Manager ProtInfo.Sched = &Sched; // Stable -> System Scheduler ProtInfo.ConfigFN= 0; // We will fill this in later ProtInfo.Stats = 0; // We will fill this in later ProtInfo.AdmPath = AdminPath; // Stable -> The admin path ProtInfo.AdmMode = AdminMode; // Stable -> The admin path mode ProtInfo.theEnv = &theEnv; // Additional information ProtInfo.xrdFlags= 0; // Additional information ProtInfo.Format = XrdFORMATB; memset(ProtInfo.rsvd3, 0, sizeof(ProtInfo.rsvd3)); ProtInfo.WSize = 0; ProtInfo.ConnMax = -1; // Max connections (fd limit) ProtInfo.readWait = 3*1000; // Wait time for data before we reschedule ProtInfo.idleWait = 0; // Seconds connection may remain idle (0=off) ProtInfo.hailWait =30*1000; // Wait time for data before we drop connection ProtInfo.DebugON = 0; // 1 if started with -d ProtInfo.argc = 0; ProtInfo.argv = 0; ProtInfo.tlsPort = 0; ProtInfo.tlsCtx = 0; ProtInfo.totalCF = &totalCF; XrdNetAddr::SetCache(3*60*60); // Cache address resolutions for 3 hours } /******************************************************************************/ /* C o n f i g u r e */ /******************************************************************************/ int XrdConfig::Configure(int argc, char **argv) { /* Function: Establish configuration at start up time. Input: None. Output: 0 upon success or !0 otherwise. */ const char *xrdInst="XRDINSTANCE="; int retc, NoGo = 0, clPort = -1; const char *temp; char c, buff[512], *dfltProt, *libProt = 0; uid_t myUid = 0; gid_t myGid = 0; extern char *optarg; extern int optind, opterr; struct XrdOucLogging::configLogInfo LogInfo; int pipeFD[2] = {-1, -1}; const char *pidFN = 0; static const int myMaxc = 80; char **urArgv, *myArgv[myMaxc], argBuff[myMaxc*3+8]; char *argbP = argBuff, *argbE = argbP+sizeof(argBuff)-4; char *ifList = 0; int myArgc = 1, urArgc = argc, i; bool noV6, ipV4 = false, ipV6 = false, rootChk = true, optbg = false; // Reconstruct the command line so we can put it in the log // XrdOucString CmdLine(argv[0]); for (int k = 1; k < argc; k++) {CmdLine += ' '; CmdLine += argv[k];} // Obtain the program name we will be using // retc = strlen(argv[0]); while(retc--) if (argv[0][retc] == '/') break; myProg = &argv[0][retc+1]; // Setup the initial required protocol. The program name matches the protocol // name but may be arbitrarily suffixed. We need to ignore this suffix. So we // look for it here and it it exists we duplicate argv[0] (yes, loosing some // bytes - sorry valgrind) without the suffix. // {char *p = dfltProt = strdup(myProg); while(*p && (*p == '.' || *p == '-')) p++; if (*p) {char *dot = index(p, '.'), *dash = index(p, '-'); if (dot && (dot < dash || !dash)) p = dot; else if (dash) p = dash; else p = 0; if (p) *p = '\0'; if (!strcmp("xrootd", dfltProt)) dfltProt[5] = 0; else if (!strcmp("cmsd", dfltProt)) dfltProt[3] = 0; } } myArgv[0] = argv[0]; // Prescan the argument list to see if there is a passthrough option. In any // case, we will set the ephemeral argv/arg in the environment. // i = 1; while(i < argc) {if (*(argv[i]) == '-' && *(argv[i]+1) == '+') {int n = strlen(argv[i]+2), j = i+1, k = 1; if (urArgc == argc) urArgc = i; if (n) memcpy(buff, argv[i]+2, (n > 256 ? 256 : n)); strcpy(&(buff[n]), ".argv**"); while(j < argc && (*(argv[j]) != '-' || *(argv[j]+1) != '+')) j++; urArgv = new char*[j-i+1]; urArgv[0] = argv[0]; i++; while(i < j) urArgv[k++] = argv[i++]; urArgv[k] = 0; theEnv.PutPtr(buff, urArgv); strcpy(&(buff[n]), ".argc"); theEnv.PutInt(buff, static_cast(k)); } else i++; } theEnv.PutPtr("argv[0]", argv[0]); // Process the options. Note that we cannot passthrough long options or // options that take arguments because getopt permutes the arguments. // opterr = 0; if (argc > 1 && '-' == *argv[1]) while ((c = getopt(urArgc,argv,":a:A:bc:dhHI:k:l:L:n:p:P:R:s:S:vw:W:z")) && ((unsigned char)c != 0xff)) { switch(c) { case 'a': if (AdminPath) free(AdminPath); AdminPath = strdup(optarg); AdminMode = ProtInfo.AdmMode = S_IRWXU; ProtInfo.xrdFlags |= XrdProtocol_Config::admPSet; break; case 'A': if (AdminPath) free(AdminPath); AdminPath = strdup(optarg); AdminMode = ProtInfo.AdmMode = S_IRWXU | S_IRWXG; ProtInfo.xrdFlags |= XrdProtocol_Config::admPSet; break; case 'b': optbg = true; break; case 'c': if (ConfigFN) free(ConfigFN); ConfigFN = strdup(optarg); break; case 'd': XrdTrace.What |= TRACE_ALL; ProtInfo.DebugON = 1; XrdOucEnv::Export("XRDDEBUG", "1"); break; case 'h': Usage(0); break; case 'H': Usage(-1); break; case 'I': if (!strcmp("v4", optarg)) {ipV4 = true; ipV6 = false;} else if (!strcmp("v6", optarg)) {ipV4 = false; ipV6 = true;} else {Log.Emsg("Config", "Invalid -I argument -",optarg); Usage(1); } break; case 'k': if (!(LogInfo.keepV = Log.logger()->ParseKeep(optarg))) {Log.Emsg("Config","Invalid -k argument -",optarg); Usage(1); } break; case 'l': LogInfo.logArg = optarg; break; case 'L': if (!*optarg) {Log.Emsg("Config", "Protocol library path not specified."); Usage(1); } if (libProt) free(libProt); libProt = strdup(optarg); break; case 'n': myInsName = (!strcmp(optarg,"anon")||!strcmp(optarg,"default") ? 0 : optarg); break; case 'p': if ((clPort = XrdOuca2x::a2p(Log,"tcp",optarg)) < 0) Usage(1); break; case 'P': if (dfltProt) free(dfltProt); dfltProt = strdup(optarg); break; case 'R': if (!(getUG(optarg, myUid, myGid))) Usage(1); rootChk = false; break; case 's': pidFN = optarg; break; case 'S': mySitName = optarg; break; case ':': buff[0] = '-'; buff[1] = optopt; buff[2] = 0; Log.Emsg("Config", buff, "parameter not specified."); Usage(1); break; case 'v': std::cerr <= myMaxc || argbP >= argbE) {Log.Emsg("Config", "Too many command line arguments."); Usage(1); } myArgv[myArgc++] = argbP; *argbP++ = '-'; *argbP++ = optopt; *argbP++ = 0; break; } } // If an adminpath specified, make sure it's absolute // if ((ProtInfo.xrdFlags & XrdProtocol_Config::admPSet) && *AdminPath != '/') {Log.Emsg("Config", "Command line adminpath is not absolute."); exit(17); } // If an homepath specified, make sure it's absolute // if (HomePath && *HomePath != '/') {Log.Emsg("Config", "Command line home path is not absolute."); exit(17); } // If the configuration file is relative to where we are, get the absolute // path as we may be changing the home path. This also starts capturing. // if (ConfigFN) setCFG(true); // The first thing we must do is to set the correct networking mode // noV6 = XrdNetAddr::IPV4Set(); if (ipV4) XrdNetAddr::SetIPV4(); else if (ipV6){if (noV6) Log.Say("Config warning: ipV6 appears to be broken;" " forced ipV6 mode not advised!"); XrdNetAddr::SetIPV6(); } else if (noV6) Log.Say("Config warning: ipV6 is misconfigured or " "unavailable; reverting to ipV4."); // Set the site name if we have one // if (mySitName) mySitName = XrdOucSiteName::Set(mySitName, 63); // Drop into non-privileged state if so requested // if (myGid && setegid(myGid)) {Log.Emsg("Config", errno, "set effective gid"); exit(17);} if (myUid && seteuid(myUid)) {Log.Emsg("Config", errno, "set effective uid"); exit(17);} // Prohibit this program from executing as superuser unless -R was specified. // if (rootChk && geteuid() == 0) {Log.Emsg("Config", "Security reasons prohibit running as " "superuser; program is terminating."); _exit(8); } // Pass over any parameters // if (urArgc-optind+2 >= myMaxc) {Log.Emsg("Config", "Too many command line arguments."); Usage(1); } for ( ; optind < urArgc; optind++) myArgv[myArgc++] = argv[optind]; // Record the actual arguments that we will pass on // myArgv[myArgc] = 0; ProtInfo.argc = myArgc; ProtInfo.argv = myArgv; // Resolve background/foreground issues // if (optbg) { #ifdef WIN32 XrdOucUtils::Undercover(&Log, !LogInfo.logArg); #else if (pipe( pipeFD ) == -1) {Log.Emsg("Config", errno, "create a pipe"); exit(17);} XrdOucUtils::Undercover(Log, !LogInfo.logArg, pipeFD); #endif } // Get the full host name. We must define myIPAddr here because we may need to // run in v4 mode and that doesn't get set until after the options are scanned. // static XrdNetAddr *myIPAddr = new XrdNetAddr((int)0); if (!(myName = myIPAddr->Name(0, &temp))) myName = ""; // Get our IP address and FQN // ProtInfo.myName = myName; ProtInfo.myAddr = myIPAddr->SockAddr(); ProtInfo.myInst = XrdOucUtils::InstName(myInsName); ProtInfo.myProg = myProg; // Set the Environmental variable to hold the instance name // XRDINSTANCE= @ // XrdOucEnv::Export("XRDINSTANCE") // sprintf(buff,"%s%s %s@%s", xrdInst, myProg, ProtInfo.myInst, myName); myInstance = strdup(buff); putenv(myInstance); // XrdOucEnv::Export("XRDINSTANCE",...) myInstance += strlen(xrdInst); XrdOucEnv::Export("XRDHOST", myName); XrdOucEnv::Export("XRDNAME", ProtInfo.myInst); XrdOucEnv::Export("XRDPROG", myProg); // Bind the log file if we have one // if (LogInfo.logArg) {LogInfo.xrdEnv = &theEnv; LogInfo.iName = myInsName; LogInfo.cfgFn = ConfigFN; if (!XrdOucLogging::configLog(Log, LogInfo)) _exit(16); Log.logger()->AddMsg(CmdLine.c_str()); Log.logger()->AddMsg(XrdBANNER); } // We now test for host name. In theory, we should always get some kind of name. // We can't really continue without some kind of name at this point. Note that // vriable temp should still be valid from the previous NetAddr call. // if (!(*myName)) {Log.Emsg("Config", "Unable to determine host name; ", (temp ? temp : "reason unknown"), "; execution terminated."); _exit(16); } // Tell NetIF what logger to use as it's been properly setup by now. // XrdNetIF::SetMsgs(&Log); // Put out the herald // strcpy(buff, "Starting on "); retc = strlen(buff); XrdSysUtils::FmtUname(buff+retc, sizeof(buff)-retc); Log.Say(0, buff); Log.Say(0, CmdLine.c_str()); Log.Say(XrdBANNER); // Verify that we have a real name. We've had problems with people setting up // bad /etc/hosts files that can cause connection failures if "allow" is used. // Otherwise, determine our domain name. // if (!myIPAddr->isRegistered()) {Log.Emsg("Config",myName,"does not appear to be registered in the DNS."); Log.Emsg("Config","Verify that the '/etc/hosts' file is correct and " "this machine is registered in DNS."); Log.Emsg("Config", "Execution continues but connection failures may occur."); myDomain = 0; } else if (!(myDomain = index(myName, '.'))) Log.Say("Config warning: this hostname, ", myName, ", is registered without a domain qualification."); // Setup the initial required protocol. // Firstcp = Lastcp = new XrdConfigProt(strdup(dfltProt), libProt, 0); // Let start it up! // Log.Say("++++++ ", myInstance, " initialization started."); // Allocate /dev/null as we need it and can't live without it // devNull = XrdSysFD_Open("/dev/null", O_RDONLY); if (devNull < 0) {Log.Emsg("Config", errno, "open '/dev/null' which is required!"); NoGo = 1; } // Process the configuration file, if one is present // if (ConfigFN) {Log.Say("Config using configuration file ", ConfigFN); ProtInfo.ConfigFN = ConfigFN; NoGo = ConfigProc(); } if (clPort >= 0) PortTCP = clPort; if (ProtInfo.DebugON) {XrdTrace.What = TRACE_ALL; XrdSysThread::setDebug(&Log); } // Setup the admin path now // NoGo |= SetupAPath(); // If tls enabled, set it up. We skip this if we failed to avoid confusing msgs // if (!NoGo) {if (!tlsCert) ProtInfo.tlsCtx= 0; else {Log.Say("++++++ ", myInstance, " TLS initialization started."); if (SetupTLS()) {Log.Say("------ ",myInstance," TLS initialization ended."); if ((ProtInfo.tlsCtx = XrdGlobal::tlsCtx)) theEnv.PutPtr("XrdTlsContext*", XrdGlobal::tlsCtx); } else { NoGo = 1; Log.Say("------ ",myInstance," TLS initialization failed."); } } } // If there is TLS port verify that it can be used. We ignore this if we // will fail anyway so as to not issue confusing messages. // if (!NoGo) {if (PortTLS > 0 && !XrdGlobal::tlsCtx) {Log.Say("Config TLS port specification ignored; TLS not configured!"); PortTLS = -1; } else { ProtInfo.tlsCtx = XrdGlobal::tlsCtx; ProtInfo.tlsPort = (PortTLS > 0 ? PortTLS : 0); } } // Put largest buffer size in the env // theEnv.PutInt("MaxBuffSize", XrdGlobal::xlBuff.MaxSize()); // Export the network interface list at this point // if (ppNet && XrdNetIF::GetIF(ifList, 0, true)) XrdOucEnv::Export("XRDIFADDRS",ifList); // Configure network routing // if (!XrdInet::netIF.SetIF(myIPAddr, ifList)) {Log.Emsg("Config", "Unable to determine interface addresses!"); NoGo = 1; } // If we have an instance name change the working directory // if ((myInsName || HomePath) && !XrdOucUtils::makeHome(Log, myInsName, HomePath, HomeMode)) NoGo = 1; // Create the pid file // if (!PidFile(pidFN, optbg)) NoGo = 1; // Establish a manifest file for auto-collection // if (!NoGo) Manifest(pidFN); // Now initialize the protocols and other stuff // if (!NoGo) NoGo = Setup(dfltProt, libProt); // End config capture // setCFG(false); // If we have a tcpmon plug-in try loading it now. We won't do that unless // tcp monitoring was enabled by the monitoring framework. // if (tmoInfo && !NoGo) {void *theGS = theEnv.GetPtr("TcpMon.gStream*"); if (!theGS) Log.Say("Config warning: TCP monitoring not enabled; " "tcpmonlib plugin not loaded!"); else {tmoInfo->theEnv.PutPtr("TcpMon.gStream*", theGS); TcpMonPin = tmoInfo->KingPin.Load("TcpMonPin"); if (!TcpMonPin) NoGo = 1; } } // if we call this it means that the daemon has forked and we are // in the child process #ifndef WIN32 if (optbg) { int status = NoGo ? 1 : 0; if(write( pipeFD[1], &status, sizeof( status ) )) {}; close( pipeFD[1]); } #endif // All done, close the stream and return the return code. // temp = (NoGo ? " initialization failed." : " initialization completed."); sprintf(buff, "%s:%d", myInstance, PortTCP); Log.Say("------ ", buff, temp); if (LogInfo.logArg) {strcat(buff, " running "); retc = strlen(buff); XrdSysUtils::FmtUname(buff+retc, sizeof(buff)-retc); Log.logger()->AddMsg(buff); } return NoGo; } /******************************************************************************/ /* C o n f i g X e q */ /******************************************************************************/ int XrdConfig::ConfigXeq(char *var, XrdOucStream &Config, XrdSysError *eDest) { int dynamic; // Determine whether is is dynamic or not // if (eDest) dynamic = 1; else {dynamic = 0; eDest = &Log;} // Process common items // TS_Xeq("buffers", xbuf); TS_Xeq("network", xnet); TS_Xeq("sched", xsched); TS_Xeq("trace", xtrace); // Process items that can only be processed once // if (!dynamic) { TS_Xeq("adminpath", xapath); TS_Xeq("allow", xallow); TS_Xeq("homepath", xhpath); TS_Xeq("maxfd", xmaxfd); TS_Xeq("pidpath", xpidf); TS_Xeq("port", xport); TS_Xeq("protocol", xprot); TS_Xeq("report", xrep); TS_Xeq("sitename", xsit); TS_Xeq("tcpmonlib", xtcpmon); TS_Xeq("timeout", xtmo); TS_Xeq("tls", xtls); TS_Xeq("tlsca", xtlsca); TS_Xeq("tlsciphers", xtlsci); } // No match found, complain. // eDest->Say("Config warning: ignoring unknown xrd directive '",var,"'."); Config.Echo(); return 0; } /******************************************************************************/ /* P r i v a t e F u n c t i o n s */ /******************************************************************************/ /******************************************************************************/ /* A S o c k e t */ /******************************************************************************/ int XrdConfig::ASocket(const char *path, const char *fname, mode_t mode) { struct sockaddr_un unixvar; int plen = strlen(path), flen = strlen(fname); // Make sure we can fit everything in our buffer // if ((plen + flen + 3) > (int)sizeof(unixvar.sun_path)) {Log.Emsg("Config", "admin path", path, "too long"); return 1; } // *!*!* At this point we do not yet support the admin path for xrd. // sp we comment out all of the following code. /* // Construct the actual socket name // char sokpath[sizeof(Unix.sun_path)]; if (sokpath[plen-1] != '/') sokpath[plen++] = '/'; strcpy(&sokpath[plen], fname); // Create an admin network // NetADM = new XrdInet(&Log); if (myDomain) NetADM->setDomain(myDomain); // Bind the netwok to the named socket // if (!NetADM->Bind(sokpath)) return 1; // Set the mode and return // chmod(sokpath, mode); // This may fail on some platforms */ return 0; } /******************************************************************************/ /* C o n f i g P r o c */ /******************************************************************************/ int XrdConfig::ConfigProc() { char *var; int cfgFD, retc, NoGo = 0; XrdOucEnv myEnv; XrdOucStream Config(&Log, myInstance, &myEnv, "=====> "); // Try to open the configuration file. // if ( (cfgFD = open(ConfigFN, O_RDONLY, 0)) < 0) {Log.Emsg("Config", errno, "open config file", ConfigFN); return 1; } Config.Attach(cfgFD); // Now start reading records until eof. // while((var = Config.GetMyFirstWord())) if (!strncmp(var, "xrd.", 4) || !strcmp (var, "all.adminpath") || !strcmp (var, "all.pidpath") || !strcmp (var, "all.sitename" )) if (ConfigXeq(var+4, Config)) {Config.Echo(); NoGo = 1;} // Now check if any errors occurred during file i/o // if ((retc = Config.LastError())) NoGo = Log.Emsg("Config", retc, "read config file", ConfigFN); Config.Close(); // Return final return code // return NoGo; } /******************************************************************************/ /* g e t N e t */ /******************************************************************************/ XrdInet *XrdConfig::getNet(int port, bool isTLS) { int the_Opts, the_Blen; // Try to find an existing network for this port // for (int i = 0; i < (int)NetTCP.size(); i++) if (port == NetTCP[i]->Port()) return NetTCP[i]; // Create a new network for this port // XrdInet *newNet = new XrdInet(&Log, Police); NetTCP.push_back(newNet); // Set options // if (isTLS) {the_Opts = TLS_Opts; the_Blen = TLS_Blen; } else { the_Opts = Net_Opts; the_Blen = Net_Blen; } if (the_Opts || the_Blen) newNet->setDefaults(the_Opts, the_Blen); // Set the domain if we have one // if (myDomain) newNet->setDomain(myDomain); // Attempt to bind to this socket. // if (newNet->BindSD(port, "tcp") == 0) return newNet; delete newNet; return 0; } /******************************************************************************/ /* g e t U G */ /******************************************************************************/ int XrdConfig::getUG(char *parm, uid_t &newUid, gid_t &newGid) { struct passwd *pp; // Get the userid entry // if (!(*parm)) {Log.Emsg("Config", "-R user not specified."); return 0;} if (isdigit(*parm)) {if (!(newUid = atol(parm))) {Log.Emsg("Config", "-R", parm, "is invalid"); return 0;} pp = getpwuid(newUid); } else pp = getpwnam(parm); // Make sure it is valid and acceptable // if (!pp) {Log.Emsg("Config", errno, "retrieve -R user password entry"); return 0; } if (!(newUid = pp->pw_uid)) {Log.Emsg("Config", "-R", parm, "is still unacceptably a superuser!"); return 0; } newGid = pp->pw_gid; return 1; } /******************************************************************************/ /* M a n i f e s t */ /******************************************************************************/ void XrdConfig::Manifest(const char *pidfn) { const char *Slash; char envBuff[8192], pwdBuff[2048], manBuff[1024], *pidP, *sP, *xP; int envFD, envLen; // Get the current working directory // if (!getcwd(pwdBuff, sizeof(pwdBuff))) {Log.Emsg("Config", "Unable to get current working directory!"); return; } // The above is the authoratative home directory, so recorded here. // if (HomePath) free(HomePath); HomePath = strdup(pwdBuff); // Prepare for symlinks // strcpy(envBuff, ProtInfo.AdmPath); envLen = strlen(envBuff); if (envBuff[envLen-1] != '/') {envBuff[envLen] = '/'; envLen++;} strcpy(envBuff+envLen, ".xrd/"); xP = envBuff+envLen+5; // Create a symlink to the configuration file // if ((sP = getenv("XRDCONFIGFN"))) {sprintf(xP, "=/conf/%s.cf", myProg); XrdOucUtils::ReLink(envBuff, sP); } // Create a symlink to where core files will be found // sprintf(xP, "=/core/%s", myProg); XrdOucUtils::ReLink(envBuff, pwdBuff); // Create a symlink to where log files will be found // if ((sP = getenv("XRDLOGDIR"))) {sprintf(xP, "=/logs/%s", myProg); XrdOucUtils::ReLink(envBuff, sP); } // Create a symlink to out proc information (Linux only) // #ifdef __linux__ sprintf(xP, "=/proc/%s", myProg); sprintf(manBuff, "/proc/%d", getpid()); XrdOucUtils::ReLink(envBuff, manBuff); #endif // Create environment string // envLen = snprintf(envBuff, sizeof(envBuff), "pid=%d&host=%s&inst=%s&ver=%s" "&home=%s&cfgfn=%s&cwd=%s&apath=%s&logfn=%s", static_cast(getpid()), ProtInfo.myName, ProtInfo.myInst, XrdVSTRING, HomePath, (getenv("XRDCONFIGFN") ? getenv("XRDCONFIGFN") : ""), pwdBuff, ProtInfo.AdmPath, Log.logger()->xlogFN()); // Find out where we should write this // if (pidfn && (Slash = rindex(pidfn, '/'))) {strncpy(manBuff, pidfn, Slash-pidfn); pidP = manBuff+(Slash-pidfn);} else {strcpy(manBuff, ProtInfo.AdmPath); pidP = manBuff+strlen(ProtInfo.AdmPath);} // Construct the pid file name for ourselves // snprintf(pidP, sizeof(manBuff)-(pidP-manBuff), "/%s.%s.env", ProtInfo.myProg, ProtInfo.myInst); theEnv.Put("envFile", manBuff); // Open the file // if ((envFD = open(manBuff, O_WRONLY|O_CREAT|O_TRUNC, 0664)) < 0) {Log.Emsg("Config", errno, "create envfile", manBuff); return; } // Write out environmental information // if (write(envFD, envBuff, envLen) < 0) Log.Emsg("Config", errno, "write to envfile", manBuff); close(envFD); } /******************************************************************************/ /* P i d F i l e */ /******************************************************************************/ bool XrdConfig::PidFile(const char *clpFN, bool optbg) { int rc, xfd; char *ppath, buff[32], pidFN[1200]; const char *xop = 0; // If a command line pidfn was specified, we must successfully write it // if we are in background mode. Otherwise, we simply continue. // if (clpFN && !XrdOucUtils::PidFile(Log, clpFN) && optbg) return false; // Generate the old-style pidpath we will use // ppath=XrdOucUtils::genPath(PidPath,XrdOucUtils::InstName(-1)); // Create the path if it does not exist and write out the pid // if ((rc = XrdOucUtils::makePath(ppath,XrdOucUtils::pathMode))) {xop = "create"; snprintf(pidFN, sizeof(pidFN), "%s", ppath); errno = rc;} else {snprintf(pidFN, sizeof(pidFN), "%s/%s.pid", ppath, myProg); if ((xfd = open(pidFN, O_WRONLY|O_CREAT|O_TRUNC,0644)) < 0) xop = "open"; else {if (write(xfd,buff,snprintf(buff,sizeof(buff),"%d", static_cast(getpid()))) < 0) xop = "write"; close(xfd); } } // All done // free(ppath); if (xop) Log.Emsg("Config", errno, xop, pidFN); return true; } /******************************************************************************/ /* s e t C F G */ /******************************************************************************/ void XrdConfig::setCFG(bool start) { // If there is no config file there is nothing to do // if (!ConfigFN || !(*ConfigFN)) {if (ConfigFN) {free(ConfigFN); ConfigFN = 0; } return; } // If ending, post process the config capture // if (!start) {XrdOucStream::Capture((XrdOucString *)0); if (totalCF.length()) {char *temp = (char *)malloc(totalCF.length()+1); strcpy(temp, totalCF.c_str()); totalCF.resize(); totalCF = temp; free(temp); } return; } // Prefix current working directory to the config file if not absolute // if (*ConfigFN != '/') {char cwdBuff[1024]; if (getcwd(cwdBuff,sizeof(cwdBuff)-strlen(ConfigFN)-2)) {int n = strlen(cwdBuff); if (cwdBuff[n-1] != '/') cwdBuff[n++] = '/'; strcpy(cwdBuff+n, ConfigFN); free(ConfigFN); ConfigFN = strdup(cwdBuff); } } // Export result // XrdOucEnv::Export("XRDCONFIGFN", ConfigFN); // Setup capturing for the XrdOucStream that will be used by all others to // process config files. // XrdOucStream::Capture(&totalCF); totalCF.resize(1024*1024); const char *cvec[] = { "*** ", myProg, " config from '", ConfigFN, "':", 0 }; XrdOucStream::Capture(cvec); } /******************************************************************************/ /* s e t F D L */ /******************************************************************************/ int XrdConfig::setFDL() { struct rlimit rlim; char buff[100]; // Get the resource limit // if (getrlimit(RLIMIT_NOFILE, &rlim) < 0) return Log.Emsg("Config", errno, "get FD limit"); // Set the limit to the maximum allowed // if (rlim.rlim_max == RLIM_INFINITY || (isStrict && rlim.rlim_max > maxFD)) rlim.rlim_cur = maxFD; else rlim.rlim_cur = rlim.rlim_max; #if (defined(__APPLE__) && defined(MAC_OS_X_VERSION_10_5)) if (rlim.rlim_cur > OPEN_MAX) rlim.rlim_max = rlim.rlim_cur = OPEN_MAX; #endif #if defined(__linux__) // Setting a limit beyond this value on Linux is guaranteed to fail during epoll_wait() unsigned int epoll_max_fd = (INT_MAX / sizeof(struct epoll_event)); if (rlim.rlim_cur > (rlim_t)epoll_max_fd) rlim.rlim_max = rlim.rlim_cur = epoll_max_fd; #endif if (setrlimit(RLIMIT_NOFILE, &rlim) < 0) return Log.Emsg("Config", errno,"set FD limit"); // Obtain the actual limit now // if (getrlimit(RLIMIT_NOFILE, &rlim) < 0) return Log.Emsg("Config", errno, "get FD limit"); // Establish operating limit // ProtInfo.ConnMax = rlim.rlim_cur; sprintf(buff, "%d", ProtInfo.ConnMax); Log.Say("Config maximum number of connections restricted to ", buff); // Set core limit and but Solaris // #if !defined( __solaris__ ) && defined(RLIMIT_CORE) if (coreV >= 0) {if (getrlimit(RLIMIT_CORE, &rlim) < 0) Log.Emsg("Config", errno, "get core limit"); else {rlim.rlim_cur = (coreV ? rlim.rlim_max : 0); if (setrlimit(RLIMIT_CORE, &rlim) < 0) Log.Emsg("Config", errno,"set core limit"); } } #endif // The scheduler will have already set the thread limit. We just report it // #if ( defined(__linux__) || defined(__GNU__) || (defined(__FreeBSD_kernel__) && defined(__GLIBC__)) ) && defined(RLIMIT_NPROC) // Obtain the actual limit now (Scheduler construction sets this to rlim_max) // if (getrlimit(RLIMIT_NPROC, &rlim) < 0) return Log.Emsg("Config", errno, "get thread limit"); // Establish operating limit // int nthr = static_cast(rlim.rlim_cur); if (nthr < 8192 || ProtInfo.DebugON) {sprintf(buff, "%d", static_cast(rlim.rlim_cur)); Log.Say("Config maximum number of threads restricted to ", buff); } #endif return 0; } /******************************************************************************/ /* S e t u p */ /******************************************************************************/ int XrdConfig::Setup(char *dfltp, char *libProt) { XrdConfigProt *cp; int xport, protNum = 0; // Establish the FD limit // if (setFDL()) return 1; // Special handling for Linux sendfile() // #if ( defined(__linux__) || defined(__GNU__) ) && defined(TCP_CORK) { int sokFD, setON = 1; if ((sokFD = socket(PF_INET, SOCK_STREAM, 0)) >= 0) {setsockopt(sokFD, XrdNetUtils::ProtoID("tcp"), TCP_NODELAY, &setON, sizeof(setON)); if (setsockopt(sokFD, SOL_TCP, TCP_CORK, &setON, sizeof(setON)) < 0) XrdLink::sfOK = 0; close(sokFD); } } #endif // Indicate how sendfile is being handled // TRACE(NET,"sendfile " <<(XrdLink::sfOK ? "enabled." : "disabled!")); // Initialize the buffer manager // BuffPool.Init(); // Start the scheduler // Sched.Start(); // Setup the link and socket polling infrastructure // if (!XrdLinkCtl::Setup(ProtInfo.ConnMax, ProtInfo.idleWait) || !XrdPoll::Setup(ProtInfo.ConnMax)) return 1; // Determine the default port number (only for xrootd) if not specified. // if (PortTCP < 0) {if ((PortTCP = XrdNetUtils::ServPort(dfltp))) PortUDP = PortTCP; else PortTCP = -1; } // We now go through all of the protocols and get each respective port number. // cp = Firstcp; while(cp) {if (!tlsCtx) for (int i = 0; i < cp->numP; i++) {if (cp->tlsVec[i]) {Log.Emsg("Config", "protocol", cp->proname, "configured with a TLS-only port " "but TLS is not configured!"); return 1; } } xport = (cp->dotls ? PortTLS : PortTCP); ProtInfo.Port = (cp->port < 0 ? xport : cp->port); XrdOucEnv::Export("XRDPORT", ProtInfo.Port); cp->port = XrdProtLoad::Port(cp->libpath,cp->proname,cp->parms,&ProtInfo); if (cp->port < 0) return 1; for (int i = 1; i < cp->numP; i++) if (cp->port == cp->portVec[i]) cp->portVec[i] = -1; cp = cp->Next; } // Allocate the statistics object. This is akward since we only know part // of the current configuration. The object will figure this out later. // ProtInfo.Stats = new XrdStats(&Log, &Sched, &BuffPool, ProtInfo.myName, Firstcp->port, ProtInfo.myInst, ProtInfo.myProg, mySitName); // If the base protocol is xroot, then save the base port number so we can // extend the port to the http protocol should it have been loaded. That way // redirects via xroot will also work for http. // xport = (strcmp("xroot", Firstcp->proname) ? 0 : Firstcp->port); // Load the protocols. For each new protocol port number, create a new // network object to handle the port dependent communications part. All // port issues will have been resolved at this point. Note that we need // to set default network object from the first protocol before loading // any protocol in case one of them starts using the default network. // XrdInet *arbNet = 0, *theNet; while((cp = Firstcp)) {for (int i = 0; i < cp->numP; i++) {if (cp->portVec[i] < 0) continue; if (!(cp->portVec[i]) && arbNet) theNet = arbNet; else {theNet = getNet(cp->portVec[i], cp->tlsVec[i]); if (!theNet) return 1; if (!(cp->portVec[i])) arbNet = theNet; } if (i == 0) XrdNetTCP = theNet; // Avoid race condition!!! ProtInfo.Port = theNet->Port(); ProtInfo.NetTCP = theNet; ProtInfo.WSize = theNet->WSize(); TRACE(NET, cp->proname <<':' <libpath, cp->proname, cp->parms, &ProtInfo, cp->dotls); if (!protNum) return 1; } } if (!strcmp("http", cp->proname) && xport) {for (int i = 0; i < cp->numP; i++) {if (cp->portVec[i] == xport) {xport = 0; break;}} if (xport) XrdProtLoad::Port(protNum, xport, false); } Firstcp = cp->Next; delete cp; } // Leave the env port number to be the first used port number. This may // or may not be the same as the default port number. This corresponds to // the default network object. // PortTCP = ProtInfo.Port = XrdNetTCP->Port(); XrdOucEnv::Export("XRDPORT", PortTCP); // Now check if we have to setup automatic reporting // if (repDest[0] != 0 && repOpts) ProtInfo.Stats->Report(repDest, repInt, repOpts); // All done // return 0; } /******************************************************************************/ /* S e t u p A P a t h */ /******************************************************************************/ int XrdConfig::SetupAPath() { int rc; // Modify the AdminPath to account for any instance name. Note that there is // a negligible memory leak under certain path combinations. Not enough to // warrant a lot of logic to get around. // if (myInsName) ProtInfo.AdmPath = XrdOucUtils::genPath(AdminPath,myInsName); else ProtInfo.AdmPath = AdminPath; XrdOucEnv::Export("XRDADMINPATH", ProtInfo.AdmPath); AdminPath = XrdOucUtils::genPath(AdminPath, myInsName, ".xrd"); // Create the path. Only sockets are group writable but allow read access to // the path for group members. // // if ((rc = XrdOucUtils::makePath(AdminPath, AdminMode & ~S_IWGRP, true))) {Log.Emsg("Config", rc, "create admin path", AdminPath); return 1; } // Setup admin connection now // return ASocket(AdminPath, "admin", (mode_t)AdminMode); } /******************************************************************************/ /* S e t u p T L S */ /******************************************************************************/ bool XrdConfig::SetupTLS() { // Check if we should issue a verification error // if (!caDir && !caFile && !tlsNoVer) {if (tlsNoCAD) Log.Say("Config failure: the tlsca directive was not specified!"); else Log.Say("Config failure: the tlsca directive did not specify " "a certdir or certfile!"); return false; } // Set the message callback before doing anything else // XrdTls::SetMsgCB(TlsError); // Set tracing options as needed // if (TRACING((TRACE_DEBUG|TRACE_TLS))) {int tlsdbg = 0; if (TRACING(TRACE_DEBUG)) tlsdbg = XrdTls::dbgALL; else {if (TRACING(TRACE_TLSCTX)) tlsdbg |= XrdTls::dbgCTX; if (TRACING(TRACE_TLSSIO)) tlsdbg |= XrdTls::dbgSIO; if (TRACING(TRACE_TLSSOK)) tlsdbg |= XrdTls::dbgSOK; } XrdTls::SetDebug(tlsdbg, &Logger); } // Create a context // static XrdTlsContext xrdTLS(tlsCert, tlsKey, caDir, caFile, tlsOpts); // Check if all went well // if (!xrdTLS.isOK()) return false; // Set address of out TLS object in the global area // XrdGlobal::tlsCtx = &xrdTLS; return true; } /******************************************************************************/ /* U s a g e */ /******************************************************************************/ void XrdConfig::Usage(int rc) { extern const char *XrdLicense; if (rc < 0) std::cerr <] [-d] [-h] [-H] [-I {v4|v6}]\n" "[-k {n|sz|sig}] [-l [=]] [-n name] [-p ] [-P ] [-L ]\n" "[-R] [-s pidfile] [-S site] [-v] [-z] []" < 0 ? rc : 0); } /******************************************************************************/ /* x a p a t h */ /******************************************************************************/ /* Function: xapath Purpose: To parse the directive: adminpath [group] the path of the FIFO to use for admin requests. group allows group access to the admin path Note: A named socket is created //.xrd/admin Output: 0 upon success or !0 upon failure. */ int XrdConfig::xapath(XrdSysError *eDest, XrdOucStream &Config) { char *pval, *val; mode_t mode = S_IRWXU; // Get the path // pval = Config.GetWord(); if (!pval || !pval[0]) {eDest->Emsg("Config", "adminpath not specified"); return 1;} // Make sure it's an absolute path // if (*pval != '/') {eDest->Emsg("Config", "adminpath not absolute"); return 1;} // Record the path // if (AdminPath) free(AdminPath); AdminPath = strdup(pval); // Get the optional access rights // if ((val = Config.GetWord()) && val[0]) {if (!strcmp("group", val)) mode |= S_IRWXG; else {eDest->Emsg("Config", "invalid admin path modifier -", val); return 1; } } AdminMode = ProtInfo.AdmMode = mode; return 0; } /******************************************************************************/ /* x a l l o w */ /******************************************************************************/ /* Function: xallow Purpose: To parse the directive: allow {host | netgroup} The dns name of the host that is allowed to connect or the netgroup name the host must be a member of. For DNS names, a single asterisk may be specified anywhere in the name. Output: 0 upon success or !0 upon failure. */ int XrdConfig::xallow(XrdSysError *eDest, XrdOucStream &Config) { char *val; int ishost; if (!(val = Config.GetWord())) {eDest->Emsg("Config", "allow type not specified"); return 1;} if (!strcmp(val, "host")) ishost = 1; else if (!strcmp(val, "netgroup")) ishost = 0; else {eDest->Emsg("Config", "invalid allow type -", val); return 1; } if (!(val = Config.GetWord())) {eDest->Emsg("Config", "allow target name not specified"); return 1;} if (!Police) {Police = new XrdNetSecurity(); if (XrdTrace.What == TRACE_ALL) Police->Trace(&XrdTrace); } if (ishost) Police->AddHost(val); else Police->AddNetGroup(val); return 0; } /******************************************************************************/ /* x h p a t h */ /******************************************************************************/ /* Function: xhpath Purpose: To parse the directive: homepath [group] the path of the home director to be made as the cwd. group allows group access to the home path Output: 0 upon success or !0 upon failure. */ int XrdConfig::xhpath(XrdSysError *eDest, XrdOucStream &Config) { // If the command line specified he home, it cannot be undone // if (Specs & hpSpec) {eDest->Say("Config warning: command line homepath cannot be overridden."); Config.GetWord(); return 0; } // Free existing home path, if any // if (HomePath) {free(HomePath); HomePath = 0;} // Parse the home path and return success or failure // HomePath = XrdOucUtils::parseHome(*eDest, Config, HomeMode); return (HomePath ? 0 : 1); } /******************************************************************************/ /* x b u f */ /******************************************************************************/ /* Function: xbuf Purpose: To parse the directive: buffers [maxbsz ] [] maximum size of an individualbuffer. The default is 2m. Specify any value 2m < bsz <= 1g; if specified, it must appear before the and becomes optional. maximum amount of memory devoted to buffers minimum buffer reshape interval in seconds Output: 0 upon success or !0 upon failure. */ int XrdConfig::xbuf(XrdSysError *eDest, XrdOucStream &Config) { static const long long minBSZ = 1024*1024*2+1; // 2mb static const long long maxBSZ = 1024*1024*1024; // 1gb int bint = -1; long long blim; char *val; if (!(val = Config.GetWord())) {eDest->Emsg("Config", "buffer memory limit not specified"); return 1;} if (!strcmp("maxbsz", val)) {if (!(val = Config.GetWord())) {eDest->Emsg("Config", "max buffer size not specified"); return 1;} if (XrdOuca2x::a2sz(*eDest,"maxbz value",val,&blim,minBSZ,maxBSZ)) return 1; XrdGlobal::xlBuff.Init(blim); if (!(val = Config.GetWord())) return 0; } if (XrdOuca2x::a2sz(*eDest,"buffer limit value",val,&blim, (long long)1024*1024)) return 1; if ((val = Config.GetWord())) if (XrdOuca2x::a2tm(*eDest,"reshape interval", val, &bint, 300)) return 1; BuffPool.Set((int)blim, bint); return 0; } /******************************************************************************/ /* x m a x f d */ /******************************************************************************/ /* Function: xmaxfd Purpose: To parse the directive: maxfd [strict] strict when specified, the limits is always applied. Otherwise, it is only applied when rlimit is infinite. maximum number of fs that can be established. Specify a value optionally suffixed with 'k'. Output: 0 upon success or !0 upon failure. */ int XrdConfig::xmaxfd(XrdSysError *eDest, XrdOucStream &Config) { long long minV = 1024, maxV = 1024LL*1024LL; // between 1k and 1m long long fdVal; char *val; if ((val = Config.GetWord())) {if (!strcmp(val, "strict")) {isStrict = true; val = Config.GetWord(); } else isStrict = false; } if (!val) {eDest->Emsg("Config", "file descriptor limit not specified"); return 1;} if (XrdOuca2x::a2sz(*eDest,"maxfd value",val,&fdVal,minV,maxV)) return 1; maxFD = static_cast(fdVal); return 0; } /******************************************************************************/ /* x n e t */ /******************************************************************************/ /* Function: xnet Purpose: To parse directive: network [tls] [[no]keepalive] [buffsz ] [kaparms parms] [cache ] [[no]dnr] [routes [use ,]] [[no]rpipa] [[no]dyndns] : split | common | local tls parameters apply only to the tls port keepalive do [not] set the socket keepalive option. kaparms keepalive paramters as specified by parms. is the socket's send/rcv buffer size. Seconds to cache address to name resolutions. [no]dnr do [not] perform a reverse DNS lookup if not needed. routes specifies the network configuration (see reference) [no]rpipa do [not] resolve private IP addresses. [no]dyndns This network does [not] use a dynamic DNS. Output: 0 upon success or !0 upon failure. */ int XrdConfig::xnet(XrdSysError *eDest, XrdOucStream &Config) { char *val; int i, n, V_keep = -1, V_nodnr = 0, V_istls = 0, V_blen = -1, V_ct = -1; int V_assumev4 = -1, v_rpip = -1, V_dyndns = -1; long long llp; struct netopts {const char *opname; int hasarg; int opval; int *oploc; const char *etxt;} ntopts[] = { {"assumev4", 0, 1, &V_assumev4, "option"}, {"keepalive", 0, 1, &V_keep, "option"}, {"nokeepalive",0, 0, &V_keep, "option"}, {"kaparms", 4, 0, &V_keep, "option"}, {"buffsz", 1, 0, &V_blen, "network buffsz"}, {"cache", 2, 0, &V_ct, "cache time"}, {"dnr", 0, 0, &V_nodnr, "option"}, {"nodnr", 0, 1, &V_nodnr, "option"}, {"dyndns", 0, 1, &V_dyndns, "option"}, {"nodyndns", 0, 0, &V_dyndns, "option"}, {"routes", 3, 1, 0, "routes"}, {"rpipa", 0, 1, &v_rpip, "rpipa"}, {"norpipa", 0, 0, &v_rpip, "norpipa"}, {"tls", 0, 1, &V_istls, "option"} }; int numopts = sizeof(ntopts)/sizeof(struct netopts); if (!(val = Config.GetWord())) {eDest->Emsg("Config", "net option not specified"); return 1;} while (val) {for (i = 0; i < numopts; i++) if (!strcmp(val, ntopts[i].opname)) {if (!ntopts[i].hasarg) *ntopts[i].oploc = ntopts[i].opval; else {if (!(val = Config.GetWord())) {eDest->Emsg("Config", "network", ntopts[i].opname, "argument missing"); return 1; } if (ntopts[i].hasarg == 4) {if (xnkap(eDest, val)) return 1; break; } if (ntopts[i].hasarg == 3) { if (!strcmp(val, "split")) XrdNetIF::Routing(XrdNetIF::netSplit); else if (!strcmp(val, "common")) XrdNetIF::Routing(XrdNetIF::netCommon); else if (!strcmp(val, "local")) XrdNetIF::Routing(XrdNetIF::netLocal); else {eDest->Emsg("Config","Invalid routes argument -",val); return 1; } if (!(val = Config.GetWord())|| !(*val)) break; if (strcmp(val, "use")) continue; if (!(val = Config.GetWord())|| !(*val)) {eDest->Emsg("Config", "network routes i/f names " "not specified."); return 1; } if (!XrdNetIF::SetIFNames(val)) return 1; ppNet = 1; break; } if (ntopts[i].hasarg == 2) {if (XrdOuca2x::a2tm(*eDest,ntopts[i].etxt,val,&n,0)) return 1; *ntopts[i].oploc = n; } else { if (XrdOuca2x::a2sz(*eDest,ntopts[i].etxt,val,&llp,0)) return 1; *ntopts[i].oploc = (int)llp; } } break; } if (i >= numopts) eDest->Say("Config warning: ignoring invalid net option '",val,"'."); else if (!val) break; val = Config.GetWord(); } if (V_istls) {if (V_blen >= 0) TLS_Blen = V_blen; if (V_keep >= 0) TLS_Opts = (V_keep ? XRDNET_KEEPALIVE : 0); TLS_Opts |= (V_nodnr ? XRDNET_NORLKUP : 0) | XRDNET_USETLS; } else { if (V_blen >= 0) Net_Blen = V_blen; if (V_keep >= 0) Net_Opts = (V_keep ? XRDNET_KEEPALIVE : 0); Net_Opts |= (V_nodnr ? XRDNET_NORLKUP : 0); } // Turn off name chaing if not specified and dynamic dns was specified // if (V_dyndns >= 0) {if (V_dyndns && V_ct < 0) V_ct = 0; XrdNetAddr::SetDynDNS(V_dyndns != 0); } if (V_ct >= 0) XrdNetAddr::SetCache(V_ct); if (v_rpip >= 0) XrdInet::netIF.SetRPIPA(v_rpip != 0); if (V_assumev4 >= 0) XrdInet::SetAssumeV4(true); return 0; } /******************************************************************************/ /* x n k a p */ /******************************************************************************/ /* Function: xnkap Purpose: To parse the directive: kaparms idle[,itvl[,cnt]] idle Seconds the connection needs to remain idle before TCP should start sending keepalive probes. itvl Seconds between individual keepalive probes. icnt Maximum number of keepalive probes TCP should send before dropping the connection, */ int XrdConfig::xnkap(XrdSysError *eDest, char *val) { char *karg, *comma; int knum; // Get the first parameter, idle seconds // karg = val; if ((comma = index(val, ','))) {val = comma+1; *comma = 0;} else val = 0; if (XrdOuca2x::a2tm(*eDest,"kaparms idle", karg, &knum, 0)) return 1; XrdNetSocketCFG::ka_Idle = knum; // Get the second parameter, interval seconds // if (!(karg = val)) return 0; if ((comma = index(val, ','))) {val = comma+1; *comma = 0;} else val = 0; if (XrdOuca2x::a2tm(*eDest,"kaparms interval", karg, &knum, 0)) return 1; XrdNetSocketCFG::ka_Itvl = knum; // Get the third parameter, count // if (!val) return 0; if (XrdOuca2x::a2i(*eDest,"kaparms count", val, &knum, 0)) return 1; XrdNetSocketCFG::ka_Icnt = knum; // All done // return 0; } /******************************************************************************/ /* x p i d f */ /******************************************************************************/ /* Function: xpidf Purpose: To parse the directive: pidpath the path where the pid file is to be created. Output: 0 upon success or !0 upon failure. */ int XrdConfig::xpidf(XrdSysError *eDest, XrdOucStream &Config) { char *val; // Get the path // val = Config.GetWord(); if (!val || !val[0]) {eDest->Emsg("Config", "pidpath not specified"); return 1;} // Record the path // if (PidPath) free(PidPath); PidPath = strdup(val); return 0; } /******************************************************************************/ /* x p o r t */ /******************************************************************************/ /* Function: xport Purpose: To parse the directive: port [tls] [if [] [named ]] tls apply this to the tls port number of the tcp port for incoming requests list of applicable host patterns list of applicable instance names. Output: 0 upon success or !0 upon failure. */ int XrdConfig::xport(XrdSysError *eDest, XrdOucStream &Config) { int rc, istls = 0, pnum = 0; char *val, cport[32]; do {if (!(val = Config.GetWord())) {eDest->Emsg("Config", "tcp port not specified"); return 1;} if (strcmp("tls", val) || istls) break; istls = 1; } while(1); strncpy(cport, val, sizeof(cport)-1); cport[sizeof(cport)-1] = '\0'; if ((val = Config.GetWord()) && !strcmp("if", val)) if ((rc = XrdOucUtils::doIf(eDest,Config, "port directive", myName, ProtInfo.myInst, myProg)) <= 0) {if (!rc) Config.noEcho(); return (rc < 0);} if ((pnum = XrdOuca2x::a2p(*eDest, "tcp", cport)) < 0) return 1; if (istls) PortTLS = pnum; else PortTCP = PortUDP = pnum; return 0; } /******************************************************************************/ /* x p r o t */ /******************************************************************************/ /* Function: xprot Purpose: To parse the directive: protocol [tls] [:] {+port | []} tls The protocol requires tls. The name of the protocol (e.g., rootd) Port binding for the protocol, if not the default. The shared library in which it is located. A one line parameter to be passed to the protocol. Output: 0 upon success or !0 upon failure. */ int XrdConfig::xprot(XrdSysError *eDest, XrdOucStream &Config) { XrdConfigProt *cpp; char *val, *parms, *lib, proname[64], buff[2048]; int portnum = -1; bool dotls = false; do {if (!(val = Config.GetWord())) {eDest->Emsg("Config", "protocol name not specified"); return 1;} if (dotls || strcmp("tls", val)) break; dotls = true; } while(1); if (strlen(val) > sizeof(proname)-1) {eDest->Emsg("Config", "protocol name is too long"); return 1;} strcpy(proname, val); if ((val = index(proname, ':'))) {if ((portnum = XrdOuca2x::a2p(*eDest, "tcp", val+1)) < 0) return 1; else *val = '\0'; } if (!(val = Config.GetWord())) {eDest->Emsg("Config", "protocol library not specified"); return 1;} if (!strcmp("*", val)) lib = 0; else if (*val == '+') {if (strcmp(val, "+port")) {eDest->Emsg("Config","invalid library specification -",val); return 1; } if ((cpp = Firstcp)) do {if (!strcmp(proname, cpp->proname)) {if (cpp->AddPort(portnum, dotls)) return 0; eDest->Emsg("Config", "port add limit exceeded!"); return 1; } } while((cpp = cpp->Next)); eDest->Emsg("Config","protocol",proname,"not previously defined!"); return 1; } else lib = strdup(val); // If no library was specified then this is a default protocol. We must make sure // sure it is consistent with whatever default we have. // if (!lib && Firstcp && strcmp(proname, Firstcp->proname)) {char eBuff[512]; snprintf(eBuff, sizeof(eBuff), "the %s protocol is '%s' not '%s'; " "assuming you meant '%s'", (Firstcp->libpath ? "assigned" : "builtin"), Firstcp->proname, proname, Firstcp->proname); eDest->Say("Config warning: ", eBuff, " but please correct " "the following directive!"); snprintf(proname, sizeof(proname), "%s", Firstcp->proname); } *buff = 0; if (!Config.GetRest(buff, sizeof(buff))) {eDest->Emsg("Config", "Too many parms for protocol", proname); return 1; } parms = (*buff ? strdup(buff) : 0); if ((cpp = Firstcp)) do {if (!strcmp(proname, cpp->proname)) {cpp->Reset(lib, parms, portnum, dotls); return 0; } } while((cpp = cpp->Next)); cpp = new XrdConfigProt(strdup(proname), lib, parms, portnum, dotls); if (!lib) {cpp->Next = Firstcp; Firstcp = cpp; if (!Lastcp) Lastcp = cpp; } else {if (Lastcp) Lastcp->Next = cpp; else Firstcp = cpp; Lastcp = cpp; } return 0; } /******************************************************************************/ /* x r e p */ /******************************************************************************/ /* Function: xrep Purpose: To parse the directive: report [,] [every ] where a UDP based report is to be sent. It may be a or a local named UDP pipe (i.e., "/..."). A secondary destination. the reporting interval. The default is 10 minutes. What to report. "all" is the default. Output: 0 upon success or !0 upon failure. */ int XrdConfig::xrep(XrdSysError *eDest, XrdOucStream &Config) { static struct repopts {const char *opname; int opval;} rpopts[] = { {"all", XRD_STATS_ALL}, {"buff", XRD_STATS_BUFF}, {"info", XRD_STATS_INFO}, {"link", XRD_STATS_LINK}, {"poll", XRD_STATS_POLL}, {"process", XRD_STATS_PROC}, {"protocols",XRD_STATS_PROT}, {"prot", XRD_STATS_PROT}, {"sched", XRD_STATS_SCHD}, {"sgen", XRD_STATS_SGEN}, {"sync", XRD_STATS_SYNC}, {"syncwp", XRD_STATS_SYNCA} }; int i, neg, numopts = sizeof(rpopts)/sizeof(struct repopts); char *val, *cp; if (!(val = Config.GetWord())) {eDest->Emsg("Config", "report parameters not specified"); return 1;} // Cleanup to start anew // if (repDest[0]) {free(repDest[0]); repDest[0] = 0;} if (repDest[1]) {free(repDest[1]); repDest[1] = 0;} repOpts = 0; repInt = 600; // Decode the destination // if ((cp = (char *)index(val, ','))) {if (!*(cp+1)) {eDest->Emsg("Config","malformed report destination -",val); return 1;} else { repDest[1] = cp+1; *cp = '\0';} } repDest[0] = val; for (i = 0; i < 2; i++) {if (!(val = repDest[i])) break; if (*val != '/' && (!(cp = index(val, (int)':')) || !atoi(cp+1))) {eDest->Emsg("Config","report dest port missing or invalid in",val); return 1; } repDest[i] = strdup(val); } // Make sure dests differ // if (repDest[0] && repDest[1] && !strcmp(repDest[0], repDest[1])) {eDest->Emsg("Config", "Warning, report dests are identical."); free(repDest[1]); repDest[1] = 0; } // Get optional "every" // if (!(val = Config.GetWord())) {repOpts = XRD_STATS_ALL; return 0;} if (!strcmp("every", val)) {if (!(val = Config.GetWord())) {eDest->Emsg("Config", "report every value not specified"); return 1;} if (XrdOuca2x::a2tm(*eDest,"report every",val,&repInt,1)) return 1; val = Config.GetWord(); } // Get reporting options // while(val) {if (!strcmp(val, "off")) repOpts = 0; else {if ((neg = (val[0] == '-' && val[1]))) val++; for (i = 0; i < numopts; i++) {if (!strcmp(val, rpopts[i].opname)) {if (neg) repOpts &= ~rpopts[i].opval; else repOpts |= rpopts[i].opval; break; } } if (i >= numopts) eDest->Say("Config warning: ignoring invalid report option '",val,"'."); } val = Config.GetWord(); } // All done // if (!(repOpts & XRD_STATS_ALL)) repOpts = char(XRD_STATS_ALL & ~XRD_STATS_INFO); return 0; } /******************************************************************************/ /* x s c h e d */ /******************************************************************************/ /* Function: xsched Purpose: To parse directive: sched [mint ] [maxt ] [avlt ] [idle ] [stksz ] [core ] is the minimum number of threads that we need. Once this number of threads is created, it does not decrease. maximum number of threads that may be created. The actual number of threads will vary between and . Are the number of threads that must be available for immediate dispatch. These threads are never bound to a connection (i.e., made stickied). Any available threads above will be allowed to stick to a connection. asis - leave current value alone. max - set value to maximum allowed (hard limit). off - turn off core files. The time (in time spec) between checks for underused threads. Those found will be terminated. Default is 780. The thread stack size in bytes or K, M, or G. Output: 0 upon success or 1 upon failure. */ int XrdConfig::xsched(XrdSysError *eDest, XrdOucStream &Config) { char *val; long long lpp; int i, ppp = 0; int V_mint = -1, V_maxt = -1, V_idle = -1, V_avlt = -1; struct schedopts {const char *opname; int minv; int *oploc; const char *opmsg;} scopts[] = { {"stksz", 0, 0, "sched stksz"}, {"mint", 1, &V_mint, "sched mint"}, {"maxt", 1, &V_maxt, "sched maxt"}, {"avlt", 1, &V_avlt, "sched avlt"}, {"core", 1, 0, "sched core"}, {"idle", 0, &V_idle, "sched idle"} }; int numopts = sizeof(scopts)/sizeof(struct schedopts); if (!(val = Config.GetWord())) {eDest->Emsg("Config", "sched option not specified"); return 1;} while (val) {for (i = 0; i < numopts; i++) if (!strcmp(val, scopts[i].opname)) {if (!(val = Config.GetWord())) {eDest->Emsg("Config", "sched", scopts[i].opname, "value not specified"); return 1; } if (*scopts[i].opname == 'i') {if (XrdOuca2x::a2tm(*eDest, scopts[i].opmsg, val, &ppp, scopts[i].minv)) return 1; } else if (*scopts[i].opname == 'c') { if (!strcmp("asis", val)) coreV = -1; else if (!strcmp("max", val)) coreV = 1; else if (!strcmp("off", val)) coreV = 0; else {eDest->Emsg("Config","invalid sched core value -",val); return 1; } } else if (*scopts[i].opname == 's') {if (XrdOuca2x::a2sz(*eDest, scopts[i].opmsg, val, &lpp, scopts[i].minv)) return 1; XrdSysThread::setStackSize((size_t)lpp); break; } else if (XrdOuca2x::a2i(*eDest, scopts[i].opmsg, val, &ppp,scopts[i].minv)) return 1; *scopts[i].oploc = ppp; break; } if (i >= numopts) eDest->Say("Config warning: ignoring invalid sched option '",val,"'."); val = Config.GetWord(); } // Make sure specified quantities are consistent // if (V_maxt > 0) {if (V_mint > 0 && V_mint > V_maxt) {eDest->Emsg("Config", "sched mint must be less than maxt"); return 1; } if (V_avlt > 0 && V_avlt > V_maxt) {eDest->Emsg("Config", "sched avlt must be less than maxt"); return 1; } } // Establish scheduler options // Sched.setParms(V_mint, V_maxt, V_avlt, V_idle); return 0; } /******************************************************************************/ /* x s i t */ /******************************************************************************/ /* Function: xsit Purpose: To parse directive: sitename is the 1- to 15-character site name to be included in monitoring information. This can also come from the command line -N option. The first such name is used. Output: 0 upon success or 1 upon failure. */ int XrdConfig::xsit(XrdSysError *eDest, XrdOucStream &Config) { char *val; if (!(val = Config.GetWord())) {eDest->Emsg("Config", "sitename value not specified"); return 1;} if (mySitName) eDest->Emsg("Config", "sitename already specified, using '", mySitName, "'."); else mySitName = XrdOucSiteName::Set(val, 63); return 0; } /******************************************************************************/ /* x t c p m o n */ /******************************************************************************/ /* Function: xtcpmon Purpose: To parse the directive: tcpmonlib [++] [] absolute path to the tcp monitor plugin. optional parameters passed to the plugin. Output: 0 upon success or !0 upon failure. */ int XrdConfig::xtcpmon(XrdSysError *eDest, XrdOucStream &Config) { std::string path; char *val, parms[2048]; bool push = false; // Get the path or the push token // if ((val = Config.GetWord())) {if (!strcmp(val, "++")) {push = true; val = Config.GetWord(); } } // Make sure a path was specified // if (!val || !*val) {eDest->Emsg("Config", "tcpmonlib not specified"); return 1;} // Make sure the path is absolute // if (*val != '/') {eDest->Emsg("Config", "tcpmonlib path is not absolute"); return 1;} // Sequester the path as we will get additional tokens // path = val; // Record any parms // if (!Config.GetRest(parms, sizeof(parms))) {eDest->Emsg("Config", "tcpmonlib parameters too long"); return 1;} // Check if we have a plugin info object (we will need one for this) // if (!tmoInfo) tmoInfo = new XrdTcpMonInfo("xrd.tcpmonlib",ConfigFN,*eDest); // Add the plugin // tmoInfo->KingPin.Add(path.c_str(), (*parms ? parms : 0), push); // All done // return 0; } /******************************************************************************/ /* x t l s */ /******************************************************************************/ /* Function: xtls Purpose: To parse directive: tls [] [] is the the certificate file to be used. is the the private key file to be used. options: [no]detail do [not] print TLS library msgs hsto handshake timeout (default 10). Output: 0 upon success or 1 upon failure. */ int XrdConfig::xtls(XrdSysError *eDest, XrdOucStream &Config) { char *val; int num; if (!(val = Config.GetWord())) {eDest->Emsg("Config", "tls cert path not specified"); return 1;} if (*val != '/') {eDest->Emsg("Config", "tls cert path not absolute"); return 1;} if (tlsCert) free(tlsCert); tlsCert = strdup(val); if (tlsKey) free(tlsKey); tlsKey = 0; if (!(val = Config.GetWord())) return 0; if (*val == '/') {tlsKey = strdup(val); if (!(val = Config.GetWord())) return 0; } do { if (!strcmp(val, "detail")) SSLmsgs = true; else if (!strcmp(val, "nodetail")) SSLmsgs = false; else if (!strcmp(val, "hsto" )) {if (!(val = Config.GetWord())) {eDest->Emsg("Config", "tls hsto value not specified"); return 1; } if (XrdOuca2x::a2tm(*eDest,"tls hsto",val,&num,1,255)) return 1; tlsOpts = TLS_SET_HSTO(tlsOpts,num); } else {eDest->Emsg("Config", "invalid tls option -",val); return 1;} } while ((val = Config.GetWord())); return 0; } /******************************************************************************/ /* x t l s c a */ /******************************************************************************/ /* Function: xtlsca Purpose: To parse directive: tlsca noverify | [] parms: {certdir | certfile} opts: [crlcheck {all | external | last}] [log {failure | off}] [[no]proxies] [refresh t[m|h|s]] [verdepth ] noverify client's cert need not be verified. is the the certificate path or file to be used. Both a file and a directory path can be specified. crlcheck Controls internal crl checks: all applies crls to the full chain external leaves crl checking to an external plug-in last applies crl check to the last cert only log logs verification attempts: "failure" (the default) logs verification failures, while "off" logs nothing. proxies allows proxy certs while noproxies does not. the crl/ca refresh interval. the maximum certificate depth to be check. Output: 0 upon success or 1 upon failure. */ int XrdConfig::xtlsca(XrdSysError *eDest, XrdOucStream &Config) { char *val, **cadest, kword[16]; int vd, rt; bool isdir; if (!(val = Config.GetWord())) {eDest->Emsg("Config", "tlsca parameter not specified"); return 1;} tlsNoCAD = false; if (!strcmp(val, "noverify")) {tlsNoVer = true; if (caDir) {free(caDir); caDir = 0;} if (caFile) {free(caFile); caFile = 0;} return 0; } tlsNoVer = false; do {if (!strcmp(val, "proxies") || !strcmp("noproxies", val)) {if (*val == 'n') tlsOpts |= XrdTlsContext::nopxy; else tlsOpts &= ~XrdTlsContext::nopxy; continue; } if (strlen(val) >= (int)sizeof(kword)) {eDest->Emsg("Config", "Invalid tlsca parameter -", val); return 1; } strcpy(kword, val); if (!(val = Config.GetWord())) {eDest->Emsg("Config", "tlsca", kword, "value not specified"); return 1; } if ((isdir = !strcmp(kword, "certdir")) || !strcmp(kword, "certfile")) {if (*val != '/') {eDest->Emsg("Config","tlsca",kword,"path is not absolute."); return 1; } cadest = (isdir ? &caDir : &caFile); if (*cadest) free(*cadest); *cadest = strdup(val); } else if (!strcmp(kword, "crlcheck")) {tlsOpts &= ~(XrdTlsContext::crlON | XrdTlsContext::crlFC); if (!strcmp(val, "all")) tlsOpts |= XrdTlsContext::crlFC; else if (!strcmp(val, "last")) tlsOpts |= XrdTlsContext::crlON; else if ( strcmp(val, "external")) {eDest->Emsg("Config","Invalid tlsca crlcheck " " argument -",val); return 1; } } else if (!strcmp(kword, "log")) { if (!strcmp(val, "off")) tlsOpts &= ~XrdTlsContext::logVF; else if (!strcmp(val, "failure")) tlsOpts |= XrdTlsContext::logVF; else {eDest->Emsg("Config","Invalid tlsca log argument -",val); return 1; } } else if (!strcmp(kword, "refresh")) {if (XrdOuca2x::a2tm(*eDest, "tlsca refresh interval", val, &rt,1,std::min(int((XrdTlsContext::crlRF >> XrdTlsContext::crlRS) * 60),std::numeric_limits::max()))) return 1; if (rt < 60) rt = 60; else if (rt % 60) rt += 60; rt = rt/60; tlsOpts = TLS_SET_REFINT(tlsOpts,rt); } else if (!strcmp(kword, "verdepth")) {if (XrdOuca2x::a2i(*eDest,"tlsca verdepth",val,&vd,1,255)) return 1; tlsOpts = TLS_SET_VDEPTH(tlsOpts,vd); } else {eDest->Emsg("Config", "invalid tlsca option -",kword); return 1;} } while((val = Config.GetWord())); return 0; } /******************************************************************************/ /* x t l s c i */ /******************************************************************************/ /* Function: xtlsci Purpose: To parse directive: tlsciphers list of colon sperated ciphers to use. Output: 0 upon success or 1 upon failure. */ int XrdConfig::xtlsci(XrdSysError *eDest, XrdOucStream &Config) { char *val, *ciphers; if (!(val = Config.GetWord())) {eDest->Emsg("Config", "tlsciphers parameter not specified"); return 1;} ciphers = strdup(val); if ((val = Config.GetWord())) {eDest->Emsg("Config","Invalid tlsciphers argument -",val); return 1; } XrdTlsContext::SetDefaultCiphers(ciphers); return 0; } /******************************************************************************/ /* x t m o */ /******************************************************************************/ /* Function: xtmo Purpose: To parse directive: timeout [read ] [hail ] [idle ] [kill ] is the maximum number of seconds to wait for pending data to arrive before we reschedule the link (default is 5 seconds). is the maximum number of seconds to wait for the initial data after a connection (default is 30 seconds) is the minimum number of seconds a connection may remain idle before it is closed (default is 5400 = 90 minutes) is the minimum number of seconds to wait after killing a connection for it to end (default is 3 seconds) Output: 0 upon success or 1 upon failure. */ int XrdConfig::xtmo(XrdSysError *eDest, XrdOucStream &Config) { char *val; int i, ppp, rc; int V_read = -1, V_idle = -1, V_hail = -1, V_kill = -1; struct tmoopts { const char *opname; int istime; int minv; int *oploc; const char *etxt;} tmopts[] = { {"read", 1, 1, &V_read, "timeout read"}, {"hail", 1, 1, &V_hail, "timeout hail"}, {"idle", 1, 0, &V_idle, "timeout idle"}, {"kill", 1, 0, &V_kill, "timeout kill"} }; int numopts = sizeof(tmopts)/sizeof(struct tmoopts); if (!(val = Config.GetWord())) {eDest->Emsg("Config", "timeout option not specified"); return 1;} while (val) {for (i = 0; i < numopts; i++) if (!strcmp(val, tmopts[i].opname)) {if (!(val = Config.GetWord())) {eDest->Emsg("Config","timeout", tmopts[i].opname, "value not specified"); return 1; } rc = (tmopts[i].istime ? XrdOuca2x::a2tm(*eDest,tmopts[i].etxt,val,&ppp, tmopts[i].minv) : XrdOuca2x::a2i (*eDest,tmopts[i].etxt,val,&ppp, tmopts[i].minv)); if (rc) return 1; *tmopts[i].oploc = ppp; break; } if (i >= numopts) eDest->Say("Config warning: ignoring invalid timeout option '",val,"'."); val = Config.GetWord(); } // Set values and return // if (V_read > 0) ProtInfo.readWait = V_read*1000; if (V_hail >= 0) ProtInfo.hailWait = V_hail*1000; if (V_idle >= 0) ProtInfo.idleWait = V_idle; XrdLinkCtl::setKWT(V_read, V_kill); return 0; } /******************************************************************************/ /* x t r a c e */ /******************************************************************************/ /* Function: xtrace Purpose: To parse the directive: trace the blank separated list of events to trace. Trace directives are cummalative. Output: 0 upon success or 1 upon failure. */ int XrdConfig::xtrace(XrdSysError *eDest, XrdOucStream &Config) { char *val; static struct traceopts {const char *opname; int opval;} tropts[] = { {"all", TRACE_ALL}, {"off", TRACE_NONE}, {"none", TRACE_NONE}, {"conn", TRACE_CONN}, {"debug", TRACE_DEBUG}, {"mem", TRACE_MEM}, {"net", TRACE_NET}, {"poll", TRACE_POLL}, {"protocol", TRACE_PROT}, {"sched", TRACE_SCHED}, {"tls", TRACE_TLS}, {"tlsctx", TRACE_TLSCTX}, {"tlssio", TRACE_TLSSIO}, {"tlssok", TRACE_TLSSOK} }; int i, neg, trval = 0, numopts = sizeof(tropts)/sizeof(struct traceopts); if (!(val = Config.GetWord())) {eDest->Emsg("Config", "trace option not specified"); return 1;} while (val) {if (!strcmp(val, "off")) trval = 0; else {if ((neg = (val[0] == '-' && val[1]))) val++; for (i = 0; i < numopts; i++) {if (!strcmp(val, tropts[i].opname)) {if (neg) if (tropts[i].opval) trval &= ~tropts[i].opval; else trval = TRACE_ALL; else if (tropts[i].opval) trval |= tropts[i].opval; else trval = TRACE_NONE; break; } } if (i >= numopts) eDest->Say("Config warning: ignoring invalid trace option '",val,"'."); } val = Config.GetWord(); } XrdTrace.What = trval; return 0; } xrootd-5.6.9/src/Xrd/XrdConfig.hh000066400000000000000000000130351457266313600166210ustar00rootroot00000000000000#ifndef _XRD_CONFIG_H #define _XRD_CONFIG_H /******************************************************************************/ /* */ /* X r d C o n f i g . h h */ /* */ /* (C) 2004 by the Board of Trustees of the Leland Stanford, Jr., University */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Deprtment of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include "Xrd/XrdProtLoad.hh" #include "Xrd/XrdProtocol.hh" #include class XrdSysError; class XrdTcpMonInfo; class XrdNetSecurity; class XrdOucStream; class XrdInet; class XrdConfigProt; class XrdConfig { public: int Configure(int argc, char **argv); int ConfigXeq(char *var, XrdOucStream &Config, XrdSysError *eDest=0); XrdConfig(); ~XrdConfig() {} XrdProtocol_Config ProtInfo; XrdInet *NetADM; std::vector NetTCP; private: int ASocket(const char *path, const char *fname, mode_t mode); int ConfigProc(void); XrdInet *getNet(int port, bool isTLS); int getUG(char *parm, uid_t &theUid, gid_t &theGid); void Manifest(const char *pidfn); bool PidFile(const char *clpFN, bool optbg); void setCFG(bool start); int setFDL(); int Setup(char *dfltp, char *libProt); int SetupAPath(); bool SetupTLS(); void Usage(int rc); int xallow(XrdSysError *edest, XrdOucStream &Config); int xapath(XrdSysError *edest, XrdOucStream &Config); int xhpath(XrdSysError *edest, XrdOucStream &Config); int xbuf(XrdSysError *edest, XrdOucStream &Config); int xmaxfd(XrdSysError *edest, XrdOucStream &Config); int xnet(XrdSysError *edest, XrdOucStream &Config); int xnkap(XrdSysError *edest, char *val); int xlog(XrdSysError *edest, XrdOucStream &Config); int xpidf(XrdSysError *edest, XrdOucStream &Config); int xport(XrdSysError *edest, XrdOucStream &Config); int xprot(XrdSysError *edest, XrdOucStream &Config); int xrep(XrdSysError *edest, XrdOucStream &Config); int xsched(XrdSysError *edest, XrdOucStream &Config); int xsit(XrdSysError *edest, XrdOucStream &Config); int xtcpmon(XrdSysError *edest, XrdOucStream &Config); int xtls(XrdSysError *edest, XrdOucStream &Config); int xtlsca(XrdSysError *edest, XrdOucStream &Config); int xtlsci(XrdSysError *edest, XrdOucStream &Config); int xtrace(XrdSysError *edest, XrdOucStream &Config); int xtmo(XrdSysError *edest, XrdOucStream &Config); static const char *TraceID; XrdNetSecurity *Police; XrdTcpMonInfo *tmoInfo; const char *myProg; const char *myName; const char *myDomain; const char *mySitName; const char *myInsName; char *myInstance; char *AdminPath; char *HomePath; char *PidPath; char *tlsCert; char *tlsKey; char *caDir; char *caFile; char *ConfigFN; char *repDest[2]; XrdConfigProt *Firstcp; XrdConfigProt *Lastcp; int Net_Blen; int Net_Opts; int TLS_Blen; int TLS_Opts; int PortTCP; // TCP Port to listen on int PortUDP; // UDP Port to listen on (currently unsupported) int PortTLS; // TCP port to listen on for TLS connections int AdminMode; int HomeMode; int repInt; uint64_t tlsOpts; bool tlsNoVer; bool tlsNoCAD; char repOpts; char ppNet; signed char coreV; char Specs; static const int hpSpec = 0x01; bool isStrict; unsigned int maxFD; }; #endif xrootd-5.6.9/src/Xrd/XrdGlobals.cc000066400000000000000000000055241457266313600167710ustar00rootroot00000000000000/******************************************************************************/ /* */ /* X r d G l o b a l s . c c */ /* */ /* (c) 2018 by the Board of Trustees of the Leland Stanford, Jr., University */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include "Xrd/XrdBuffer.hh" #include "Xrd/XrdBuffXL.hh" #include "Xrd/XrdInet.hh" #include "Xrd/XrdScheduler.hh" #include "XrdSys/XrdSysTrace.hh" #include "XrdSys/XrdSysLogger.hh" #include "XrdSys/XrdSysError.hh" // All the things that the Xrd package requires. // class XrdTlsContext; class XrdBuffXL; namespace XrdGlobal { XrdSysLogger Logger; XrdSysError Log(&Logger, "Xrd"); XrdSysTrace XrdTrace("Xrd", &Logger); XrdScheduler Sched(&Log, &XrdTrace); XrdBuffManager BuffPool; XrdTlsContext *tlsCtx = 0; XrdInet *XrdNetTCP = 0; extern XrdBuffXL xlBuff; int devNull = -1; } xrootd-5.6.9/src/Xrd/XrdInet.cc000066400000000000000000000222701457266313600163020ustar00rootroot00000000000000/******************************************************************************/ /* */ /* X r d I n e t . c c */ /* */ /* (c) 2004 by the Board of Trustees of the Leland Stanford, Jr., University */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include #include #include #include #include #ifdef HAVE_SYSTEMD #include #include #endif #include "XrdSys/XrdSysError.hh" #include "Xrd/XrdInet.hh" #include "Xrd/XrdLinkCtl.hh" #include "Xrd/XrdTrace.hh" #include "XrdNet/XrdNetAddr.hh" #include "XrdNet/XrdNetOpts.hh" #include "XrdNet/XrdNetSecurity.hh" #ifdef HAVE_SYSTEMD #include "XrdNet/XrdNetBuffer.hh" #include "XrdNet/XrdNetSocket.hh" #endif /******************************************************************************/ /* G l o b a l s */ /******************************************************************************/ bool XrdInet::AssumeV4 = false; const char *XrdInet::TraceID = "Inet"; XrdNetIF XrdInet::netIF; /******************************************************************************/ /* A c c e p t */ /******************************************************************************/ XrdLink *XrdInet::Accept(int opts, int timeout, XrdSysSemaphore *theSem) { static const char *unk = "unkown.endpoint"; XrdNetAddr myAddr; XrdLink *lp; int anum=0, lnkopts = (opts & XRDNET_MULTREAD ? XRDLINK_RDLOCK : 0); // Perform regular accept. This will be a unique TCP socket. We loop here // until the accept succeeds as it should never fail at this stage. // while(!XrdNet::Accept(myAddr, opts | XRDNET_NORLKUP, timeout)) {if (timeout >= 0) {if (theSem) theSem->Post(); return (XrdLink *)0; } sleep(1); anum++; if (!(anum%60)) eDest->Emsg("Accept", "Unable to accept connections!"); } // If authorization was deferred, tell call we accepted the connection but // will be doing a background check on this connection. // if (theSem) theSem->Post(); if (!(netOpts & XRDNET_NORLKUP)) myAddr.Name(); // Authorize by ip address or full (slow) hostname format. We defer the check // so that the next accept can occur before we do any DNS resolution. // if (Patrol) {if (!Patrol->Authorize(myAddr)) {char ipbuff[512]; myAddr.Format(ipbuff, sizeof(ipbuff), XrdNetAddr::fmtAuto, XrdNetAddrInfo::noPort); eDest->Emsg("Accept",EACCES,"accept TCP connection from",ipbuff); close(myAddr.SockFD()); return (XrdLink *)0; } } // Allocate a new network object // if (!(lp = XrdLinkCtl::Alloc(myAddr, lnkopts))) {eDest->Emsg("Accept", ENOMEM, "allocate new link for", myAddr.Name(unk)); close(myAddr.SockFD()); } else { TRACE(NET, "Accepted connection on port " <(port); // Get correct socket type // if (*contype != 'u') PortType = SOCK_STREAM; else {PortType = SOCK_DGRAM; opts |= XRDNET_UDPSOCKET; } // For each fd, check if it is a socket that we should have bound to. Make // allowances for UDP sockets. Otherwise, make sure the socket is listening. // for (int i = 0; i < nSD; i++) {int sdFD = SD_LISTEN_FDS_START + i; if (sd_is_socket_inet(sdFD, v4Sock, PortType, -1, sdPort) > 0 || sd_is_socket_inet(sdFD, v6Sock, PortType, -1, sdPort) > 0) {iofd = sdFD; Portnum = port; XrdNetSocket::setOpts(sdFD, opts, eDest); if (PortType == SOCK_DGRAM) {BuffSize = (Windowsz ? Windowsz : XRDNET_UDPBUFFSZ); BuffQ = new XrdNetBufferQ(BuffSize); } else { if (sd_is_socket(sdFD,v4Sock,PortType,0) > 0 || sd_is_socket(sdFD,v6Sock,PortType,0) > 0) return Listen(); } return 0; } } #endif // Either we have no systemd process or no acceptable FD available. Do an // old-style bind() call to setup the socket. // return Bind(port, contype); } /******************************************************************************/ /* C o n n e c t */ /******************************************************************************/ XrdLink *XrdInet::Connect(const char *host, int port, int opts, int tmo) { static const char *unk = "unkown.endpoint"; XrdNetAddr myAddr; XrdLink *lp; int lnkopts = (opts & XRDNET_MULTREAD ? XRDLINK_RDLOCK : 0); // Try to do a connect. This will be a unique TCP socket. // if (!XrdNet::Connect(myAddr, host, port, opts, tmo)) return (XrdLink *)0; // Return a link object // if (!(lp = XrdLinkCtl::Alloc(myAddr, lnkopts))) {eDest->Emsg("Connect", ENOMEM, "allocate new link to", myAddr.Name(unk)); close(myAddr.SockFD()); } else { TRACE(NET, "Connected to " <Emsg("Bind", erc, eBuff); } // Return an error // return -erc; } /******************************************************************************/ /* S e c u r e */ /******************************************************************************/ void XrdInet::Secure(XrdNetSecurity *secp) { // If we don't have a Patrol object then use the one supplied. Otherwise // merge the supplied object into the existing object. // if (Patrol) Patrol->Merge(secp); else Patrol = secp; } xrootd-5.6.9/src/Xrd/XrdInet.hh000066400000000000000000000065651457266313600163250ustar00rootroot00000000000000#ifndef __XRD_INET_H__ #define __XRD_INET_H__ /******************************************************************************/ /* */ /* X r d I n e t . h h */ /* */ /* (c) 2004 by the Board of Trustees of the Leland Stanford, Jr., University */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include "XrdNet/XrdNet.hh" #include "XrdNet/XrdNetIF.hh" // The XrdInet class defines a generic network where we can define common // initial tcp/ip and udp operations. It is based on the generalized network // support framework. However, Accept and Connect have been augmented to // provide for more scalable communications handling. // class XrdSysError; class XrdSysSemaphore; class XrdNetSecurity; class XrdLink; class XrdInet : public XrdNet { public: XrdLink *Accept(int opts=0, int timeout=-1, XrdSysSemaphore *theSem=0); int BindSD(int port, const char *contype="tcp"); XrdLink *Connect(const char *host, int port, int opts=0, int timeout=-1); void Secure(XrdNetSecurity *secp); XrdInet(XrdSysError *erp, XrdNetSecurity *secp=0) : XrdNet(erp,0), Patrol(secp) {} ~XrdInet() {} static void SetAssumeV4(bool newVal) {AssumeV4 = newVal;} static bool GetAssumeV4() {return AssumeV4;} static XrdNetIF netIF; private: int Listen(); XrdNetSecurity *Patrol; static const char *TraceID; static bool AssumeV4; }; #endif xrootd-5.6.9/src/Xrd/XrdInfo.cc000066400000000000000000000054301457266313600162750ustar00rootroot00000000000000/******************************************************************************/ /* */ /* X r d I n f o . c c */ /* */ /* (c) 2004 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved. Scroll to end for Terms and Conditions of use */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include "Xrd/XrdInfo.hh" /******************************************************************************/ /* L i c e n s e T e r m a n d C o n d i t i o n s */ /* */ /* License terms are contained in the LICENSE file in the base directory. */ /******************************************************************************/ const char *XrdLicense = #include "../../LICENSE" ; xrootd-5.6.9/src/Xrd/XrdInfo.hh000066400000000000000000000046711457266313600163150ustar00rootroot00000000000000#ifndef __XRD_INFO_H__ #define __XRD_INFO_H__ /******************************************************************************/ /* */ /* X r d I n f o . h h */ /* */ /* (c) 2004 by the Board of Trustees of the Leland Stanford, Jr., University */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include "XrdVersion.hh" #define XrdFORMAT "2.0.0" #define XrdFORMATB 0x00000200 #define XrdBANNER "Copr. 2004-2012 Stanford University, xrd version " XrdVSTRING #endif xrootd-5.6.9/src/Xrd/XrdJob.hh000066400000000000000000000060331457266313600161260ustar00rootroot00000000000000#ifndef ___XRD_JOB_H___ #define ___XRD_JOB_H___ /******************************************************************************/ /* */ /* X r d J o b . h h */ /* */ /* (c) 2004 by the Board of Trustees of the Leland Stanford, Jr., University */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include #include #include // The XrdJob class is a super-class that is inherited by any class that needs // to schedule work on behalf of itself. The XrdJob class is optimized for // queue processing since that's where it spends a lot of time. This class // should not be depedent on any other class. class XrdJob { friend class XrdScheduler; public: XrdJob *NextJob; // -> Next job in the queue (zero if last) const char *Comment; // -> Description of work for debugging (static!) virtual void DoIt() = 0; XrdJob(const char *desc="") {Comment = desc; NextJob = 0; SchedTime = 0;} virtual ~XrdJob() {} private: time_t SchedTime; // -> Time job is to be scheduled }; #endif xrootd-5.6.9/src/Xrd/XrdLink.cc000066400000000000000000000611251457266313600163020ustar00rootroot00000000000000/******************************************************************************/ /* */ /* X r d L i n k . c c */ /* */ /* (c) 2018 by the Board of Trustees of the Leland Stanford, Jr., University */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include #include #include #include #include #if defined(__linux__) || defined(__GNU__) #include #if !defined(TCP_CORK) #undef HAVE_SENDFILE #endif #endif #ifdef HAVE_SENDFILE #ifndef __APPLE__ #if !defined(__FreeBSD__) #include #else #include #include #include #endif #endif #endif #include "XrdSys/XrdSysAtomics.hh" #include "XrdSys/XrdSysError.hh" #include "XrdSys/XrdSysFD.hh" #include "XrdSys/XrdSysPlatform.hh" #include "Xrd/XrdBuffer.hh" #include "Xrd/XrdLinkCtl.hh" #include "Xrd/XrdPoll.hh" #define TRACE_IDENT ID #include "Xrd/XrdTrace.hh" #include "XrdSys/XrdSysError.hh" #ifndef ETIME #define ETIME ETIMEDOUT #endif /******************************************************************************/ /* G l o b a l s */ /******************************************************************************/ namespace XrdGlobal { extern XrdSysError Log; }; using namespace XrdGlobal; /******************************************************************************/ /* S t a t i c M e m b e r s */ /******************************************************************************/ #if defined(HAVE_SENDFILE) bool XrdLink::sfOK = true; #else bool XrdLink::sfOK = false; #endif namespace { const char KillMax = 60; const char KillMsk = 0x7f; const char KillXwt = 0x80; const char *TraceID = "Link"; } /******************************************************************************/ /* C o n s t r u c t o r */ /******************************************************************************/ XrdLink::XrdLink(XrdLinkXeq &lxq) : XrdJob("connection"), linkXQ(lxq), HostName(0) { memset(rsvd1, 0, sizeof(rsvd1)); memset(rsvd2, 0, sizeof(rsvd2)); ResetLink(); } void XrdLink::ResetLink() { if (HostName) {free(HostName); HostName = 0;} Instance = 0; isBridged= false; isTLS = false; } /******************************************************************************/ /* A c t i v a t e */ /******************************************************************************/ bool XrdLink::Activate() { // Attach this link to a poller // return XrdPoll::Attach(linkXQ.PollInfo); } /******************************************************************************/ /* A d d r I n f o */ /******************************************************************************/ XrdNetAddrInfo *XrdLink::AddrInfo() {return linkXQ.AddrInfo();} /******************************************************************************/ /* a r m B r i d g e */ /******************************************************************************/ void XrdLink::armBridge() {isBridged = 1;} /******************************************************************************/ /* B a c k l o g */ /******************************************************************************/ int XrdLink::Backlog() {return linkXQ.Backlog();} /******************************************************************************/ /* C l i e n t */ /******************************************************************************/ int XrdLink::Client(char *nbuf, int nbsz) {return linkXQ.Client(nbuf, nbsz);} /******************************************************************************/ /* C l o s e */ /******************************************************************************/ int XrdLink::Close(bool defer) {return linkXQ.Close(defer);} /******************************************************************************/ /* Protected: D o I t */ /******************************************************************************/ void XrdLink::DoIt() {} // This is overridden by the implementation /******************************************************************************/ /* E n a b l e */ /******************************************************************************/ void XrdLink::Enable() { if (linkXQ.PollInfo.Poller) linkXQ.PollInfo.Poller->Enable(linkXQ.PollInfo); } /******************************************************************************/ /* F D n u m */ /******************************************************************************/ int XrdLink::FDnum() { return linkXQ.PollInfo.FD; } /******************************************************************************/ /* F i n d */ /******************************************************************************/ XrdLink *XrdLink::Find(int &curr, XrdLinkMatch *who) {return XrdLinkCtl::Find(curr, who);} /******************************************************************************/ /* g e t I O S t a t s */ /******************************************************************************/ int XrdLink::getIOStats(long long &inbytes, long long &outbytes, int &numstall, int &numtardy) {return linkXQ.getIOStats(inbytes, outbytes, numstall, numtardy); } /******************************************************************************/ /* g e t N a m e */ /******************************************************************************/ // Warning: curr must be set to a value of 0 or less on the initial call and // not touched therafter unless null is returned. Returns the length // the name in nbuf. // int XrdLink::getName(int &curr, char *nbuf, int nbsz, XrdLinkMatch *who) {return XrdLinkCtl::getName(curr, nbuf, nbsz, who);} /******************************************************************************/ /* g e t P e e r C e r t s */ /******************************************************************************/ XrdTlsPeerCerts *XrdLink::getPeerCerts() { return linkXQ.getPeerCerts(); } /******************************************************************************/ /* g e t P r o t o c o l */ /******************************************************************************/ XrdProtocol *XrdLink::getProtocol() {return linkXQ.getProtocol();} /******************************************************************************/ /* H o l d */ /******************************************************************************/ void XrdLink::Hold(bool lk) { (lk ? linkXQ.LinkInfo.opMutex.Lock() : linkXQ.LinkInfo.opMutex.UnLock()); } /******************************************************************************/ /* i s F l a w e d */ /******************************************************************************/ bool XrdLink::isFlawed() const {return linkXQ.LinkInfo.Etext != 0;} /******************************************************************************/ /* i s I n s t a n c e */ /******************************************************************************/ bool XrdLink::isInstance(unsigned int inst) const {return Instance == inst && linkXQ.PollInfo.FD >= 0;} /******************************************************************************/ /* N a m e */ /******************************************************************************/ const char *XrdLink::Name() const {return linkXQ.Name();} /******************************************************************************/ /* N e t A d d r */ /******************************************************************************/ const XrdNetAddr *XrdLink::NetAddr() const {return linkXQ.NetAddr();} /******************************************************************************/ /* P e e k */ /******************************************************************************/ int XrdLink::Peek(char *Buff, int Blen, int timeout) { if (isTLS) return linkXQ.TLS_Peek(Buff, Blen, timeout); else return linkXQ.Peek (Buff, Blen, timeout); } /******************************************************************************/ /* R e c v */ /******************************************************************************/ int XrdLink::Recv(char *Buff, int Blen) { if (isTLS) return linkXQ.TLS_Recv(Buff, Blen); else return linkXQ.Recv (Buff, Blen); } /******************************************************************************/ int XrdLink::Recv(char *Buff, int Blen, int timeout) { if (isTLS) return linkXQ.TLS_Recv(Buff, Blen, timeout); else return linkXQ.Recv (Buff, Blen, timeout); } /******************************************************************************/ int XrdLink::Recv(const struct iovec *iov, int iocnt, int timeout) { // Execute the send // if (isTLS) return linkXQ.TLS_Recv(iov, iocnt, timeout); else return linkXQ.Recv (iov, iocnt, timeout); } /******************************************************************************/ /* R e c v A l l */ /******************************************************************************/ int XrdLink::RecvAll(char *Buff, int Blen, int timeout) { if (isTLS) return linkXQ.TLS_RecvAll(Buff, Blen, timeout); else return linkXQ.RecvAll (Buff, Blen, timeout); } /******************************************************************************/ /* R e g i s t e r */ /******************************************************************************/ bool XrdLink::Register(const char *hName) { return linkXQ.Register(hName); } /******************************************************************************/ /* S e n d */ /******************************************************************************/ int XrdLink::Send(const char *Buff, int Blen) { if (isTLS) return linkXQ.TLS_Send(Buff, Blen); else return linkXQ.Send (Buff, Blen); } /******************************************************************************/ int XrdLink::Send(const struct iovec *iov, int iocnt, int bytes) { // Allways make sure we have a total byte count // if (!bytes) for (int i = 0; i < iocnt; i++) bytes += iov[i].iov_len; // Execute the send // if (isTLS) return linkXQ.TLS_Send(iov, iocnt, bytes); else return linkXQ.Send (iov, iocnt, bytes); } /******************************************************************************/ int XrdLink::Send(const sfVec *sfP, int sfN) { // Make sure we have valid vector count // if (sfN < 1 || sfN > XrdOucSFVec::sfMax) {Log.Emsg("Link", E2BIG, "send file to", ID); return -1; } // Do the send // if (isTLS) return linkXQ.TLS_Send(sfP, sfN); else return linkXQ.Send (sfP, sfN); } /******************************************************************************/ /* S e r i a l i z e */ /******************************************************************************/ void XrdLink::Serialize() { // This is meant to make sure that no protocol objects are refering to this // link so that we can safely run in pseudo single thread mode for critical // functions. // linkXQ.LinkInfo.opMutex.Lock(); if (linkXQ.LinkInfo.InUse <= 1) linkXQ.LinkInfo.opMutex.UnLock(); else {linkXQ.LinkInfo.doPost++; linkXQ.LinkInfo.opMutex.UnLock(); TRACEI(DEBUG, "Waiting for link serialization; use=" <Hold(true); if (!(cp = index(ID, ':')) || strncmp(lp->ID, ID, cp-ID) || strcmp(HostName, lp->Host())) {lp->Hold(false); return -EACCES; } int rc = lp->Terminate(ID, fdnum, inst); lp->Hold(false); return rc; } // At this pint, we are excuting in the context of the target link. // If this link is now dead, simply ignore the request. Typically, this // indicates a race condition that the server won. // if ( linkXQ.PollInfo.FD != fdnum || Instance != inst || !linkXQ.PollInfo.Poller || !linkXQ.getProtocol()) return -EPIPE; // Check if we have too many tries here // int wTime, killTries; killTries = linkXQ.LinkInfo.KillCnt & KillMsk; if (killTries > KillMax) return -ETIME; // Wait time increases as we have more unsuccessful kills. Update numbers. // wTime = killTries++; linkXQ.LinkInfo.KillCnt = killTries | KillXwt; // Make sure we can disable this link. If not, then force the caller to wait // a tad more than the read timeout interval. // if (!linkXQ.PollInfo.isEnabled || linkXQ.LinkInfo.InUse > 1 || linkXQ.LinkInfo.KillcvP) {wTime = wTime*2+XrdLinkCtl::waitKill; return (wTime > 60 ? 60: wTime); } // Set the pointer to our condvar. We are holding the opMutex to prevent a race. // XrdSysCondVar killDone(0); linkXQ.LinkInfo.KillcvP = &killDone; killDone.Lock(); // We can now disable the link and schedule a close // char buff[1024]; snprintf(buff, sizeof(buff), "ended by %s", owner); buff[sizeof(buff)-1] = '\0'; linkXQ.PollInfo.Poller->Disable(linkXQ.PollInfo, buff); linkXQ.LinkInfo.opMutex.UnLock(); // Now wait for the link to shutdown. This avoids lock problems. // if (killDone.Wait(int(XrdLinkCtl::killWait))) wTime += XrdLinkCtl::killWait; else wTime = -EPIPE; killDone.UnLock(); // Reobtain the opmutex so that we can zero out the pointer the condvar pntr // This is really stupid code but because we don't have a way of associating // an arbitrary mutex with a condvar. But since this code is rarely executed // the ugliness is sort of tolerable. // linkXQ.LinkInfo.opMutex.Lock(); linkXQ.LinkInfo.KillcvP = 0; linkXQ.LinkInfo.opMutex.UnLock(); // Do some tracing // TRACEI(DEBUG,"Terminate " << (wTime <= 0 ? "complete ":"timeout ") <. */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include "XrdNet/XrdNetAddr.hh" #include "XrdOuc/XrdOucSFVec.hh" #include "XrdSys/XrdSysPthread.hh" #include "Xrd/XrdJob.hh" /******************************************************************************/ /* C l a s s D e f i n i t i o n */ /******************************************************************************/ class XrdLinkMatch; class XrdLinkXeq; class XrdPollInfo; class XrdProtocol; class XrdTlsPeerCerts; class XrdTlsContext; class XrdLink : public XrdJob { public: //----------------------------------------------------------------------------- //! Activate a link by attaching it to a poller object. //! //! @return True if activation succeeded and false otherwise. //----------------------------------------------------------------------------- bool Activate(); //----------------------------------------------------------------------------- //! Obtain the address information for this link. //! //! @return Pointer to the XrdAddrInfo object. The pointer is valid while the //! end-point is connected. //----------------------------------------------------------------------------- XrdNetAddrInfo *AddrInfo(); //----------------------------------------------------------------------------- //! Obtain the number of queued async requests. //! //! @return The number of async requests queued. //----------------------------------------------------------------------------- int Backlog(); //----------------------------------------------------------------------------- //! Get a copy of the client's name as known by the link. //! //! @param buff Pointer to buffer to hold the name. //! @param blen Length of the buffer. //! //! @return !0 The length of the name in gthe buffer. //! =0 The name could not be returned. //----------------------------------------------------------------------------- int Client(char *buff, int blen); //----------------------------------------------------------------------------- //! Close the link. //! //! @param defer If true, the link is made unaccessible but the link //! object not the file descriptor is released. //! //! @return !0 An error occurred, the return value is the errno. //! =0 Action successfully completed. //----------------------------------------------------------------------------- int Close(bool defer=false); //----------------------------------------------------------------------------- //! Enable the link to field interrupts. //----------------------------------------------------------------------------- void Enable(); //----------------------------------------------------------------------------- //! Get the associated file descriptor. //! //! @return The file descriptor number. //----------------------------------------------------------------------------- int FDnum(); //----------------------------------------------------------------------------- //! Find the next link matching certain attributes. //! //! @param curr Is an internal tracking value that allows repeated calls. //! It must be set to a value of 0 or less on the initial call //! and not touched therafter unless a null pointer is returned. //! @param who If the object use to check if the link matches the wanted //! criterea (typically, client name and host name). If the //! pointer is nil, the next link is always returned. //! //! @return !0 Pointer to the link object that matches the criterea. The //! link's reference counter is increased to prevent it from //! being reused. A subsequent call will reduce the number. //! =0 No more links exist with the specified criterea. //----------------------------------------------------------------------------- static XrdLink *Find(int &curr, XrdLinkMatch *who=0); //----------------------------------------------------------------------------- //! Get I/O statistics. //! //! @param inbytes The number of bytes read. //! @param outbytes The number of bytes written. //! @param numstall The number of times the link was rescheduled due to //! unavailability. //! @param numtardy The number of times the link was delayed due to //! unavailability. //! //! @return The link's reference count. The parameters will hold the //! indicated statistic. //----------------------------------------------------------------------------- int getIOStats(long long &inbytes, long long &outbytes, int &numstall, int &numtardy); //----------------------------------------------------------------------------- //! Find the next client name matching certain attributes. //! //! @param curr Is an internal tracking value that allows repeated calls. //! It must be set to a value of 0 or less on the initial call //! and not touched therafter unless zero is returned. //! @param bname Pointer to a buffer where the name is to be returned. //! @param blen The length of the buffer. //! @param who If the object use to check if the link matches the wanted //! criterea (typically, client name and host name). If the //! pointer is nil, a match always occurs. //! //! @return !0 The length of the name placed in the buffer. //! =0 No more links exist with the specified criterea. //----------------------------------------------------------------------------- static int getName(int &curr, char *bname, int blen, XrdLinkMatch *who=0); //----------------------------------------------------------------------------- //! Get the x509 certificate information for this TLS enabled link. //! //! @return A pointer to the XrdTlsCerts object holding verified certificates //! if such certificates exist. Otherwise a nil pointer is returned. //! //! @note Used by various protocols, so XrdTlsPeerCerts is a private header. //----------------------------------------------------------------------------- XrdTlsPeerCerts *getPeerCerts(); //----------------------------------------------------------------------------- //! Obtain current protocol object pointer. //----------------------------------------------------------------------------- XrdProtocol *getProtocol(); //----------------------------------------------------------------------------- //! Lock or unlock the mutex used for control operations. //! //! @param lk When true, a lock is obtained. Otherwise it is released. //! The caller is responsible for consistency. //----------------------------------------------------------------------------- void Hold(bool lk); //----------------------------------------------------------------------------- //! Get the fully qualified name of the endpoint. //! //! @return Pointer to fully qualified host name. The contents are valid //! while the endpoint is connected. //----------------------------------------------------------------------------- const char *Host() const {return (const char *)HostName;} //----------------------------------------------------------------------------- //! Pointer to the client's link identity. //----------------------------------------------------------------------------- char *ID; // This is referenced a lot (should have been const). //----------------------------------------------------------------------------- //! Obtain the link's instance number. //! //! @return The link's instance number. //----------------------------------------------------------------------------- unsigned int Inst() const {return Instance;} //----------------------------------------------------------------------------- //! Indicate whether or not the link has an outstanding error. //! //! @return True the link has an outstanding error. //! the link has no outstanding error. //----------------------------------------------------------------------------- bool isFlawed() const; //----------------------------------------------------------------------------- //! Indicate whether or not this link is of a particular instance. //! only be used for display and not for security purposes. //! //! @param inst the expected instance number. //! //! @return true the link matches the instance number. //! false the link differs the instance number. //----------------------------------------------------------------------------- bool isInstance(unsigned int inst) const; //----------------------------------------------------------------------------- //! Obtain the domain trimmed name of the end-point. The returned value should //! only be used for display and not for security purposes. //! //! @return Pointer to the name that remains valid during the link's lifetime. //----------------------------------------------------------------------------- const char *Name() const; //----------------------------------------------------------------------------- //! Obtain the network address object for this link. The returned value is //! valid as long as the end-point is connected. Otherwise, it may change. //! //! @return Pointer to the object and remains valid during the link's lifetime. //----------------------------------------------------------------------------- const XrdNetAddr *NetAddr() const; //----------------------------------------------------------------------------- //! Issue a socket peek() and return result (do not use for TLS connections). //! //! @param buff pointer to buffer to hold data. //! @param blen length of buffer. //! @param timeout milliseconds to wait for data. A negative value waits //! forever. //! //! @return >=0 buffer holds data equal to the returned value. //! < 0 an error or timeout occurred. //----------------------------------------------------------------------------- int Peek(char *buff, int blen, int timeout=-1); //----------------------------------------------------------------------------- //! Read data from a link. Note that this call blocks until some data is //! available. Use Recv() with a timeout to avoid blocking. //! //! @param buff pointer to buffer to hold data. //! @param blen length of buffer (implies the maximum bytes wanted). //! //! @return >=0 buffer holds data equal to the returned value. //! < 0 an error occurred. //----------------------------------------------------------------------------- int Recv(char *buff, int blen); //----------------------------------------------------------------------------- //! Read data from a link. Note that this call either reads all the data wanted //! or no data if the passed timeout occurs before any data is present. //! //! @param buff pointer to buffer to hold data. //! @param blen length of buffer (implies the actual bytes wanted). //! @param timeout milliseconds to wait for data. A negative value waits //! forever. //! //! @return >=0 buffer holds data equal to the returned value. //! < 0 an error occurred. Note that a special error -ENOMSG //! is returned if poll() indicated data was present but //! no bytes were actually read. //----------------------------------------------------------------------------- int Recv(char *buff, int blen, int timeout); //----------------------------------------------------------------------------- //! Read data on a link. Note that this call either reads all the data wanted //! or no data if the passed timeout occurs before any data is present. //! //! @param iov pointer to the message vector. //! @param iocnt number of iov elements in the vector. //! @param timeout milliseconds to wait for data. A negative value waits //! forever. //! //! @return >=0 number of bytes read. //! < 0 an error occurred or when -ETIMEDOUT is returned, no data //! arrived within the timeout period. -ENOMSG is returned //! when poll indicated data was present but 0 bytes were read. //----------------------------------------------------------------------------- int Recv(const struct iovec *iov, int iocnt, int timeout); //----------------------------------------------------------------------------- //! Read data from a link. Note that this call reads as much data as it can //! or until the passed timeout has occurred. //! //! @param buff pointer to buffer to hold data. //! @param blen length of buffer (implies the maximum bytes wanted). //! @param timeout milliseconds to wait for data. A negative value waits //! forever. //! //! @return >=0 buffer holds data equal to the returned value. //! < 0 an error occurred or when -ETIMEDOUT is returned, no data //! arrived within the timeout period. -ENOMSG is returned //! when poll indicated data was present but 0 bytes were read. //----------------------------------------------------------------------------- int RecvAll(char *buff, int blen, int timeout=-1); //------------------------------------------------------------------------------ //! Register a host name with this IP address. This is not MT-safe! //! //! @param hName pointer to a true host name which should be fully qualified. //! One of the IP addresses registered to this name must //! match the IP address associated with this object. //! //! @return True: Specified name is now associated with this link. //! False: Nothing changed, registration could not be verified. //------------------------------------------------------------------------------ bool Register(const char *hName); //----------------------------------------------------------------------------- //! Send data on a link. This calls may block unless the socket was marked //! nonblocking. If a block would occur, the data is copied for later sending. //! //! @param buff pointer to buffer to send. //! @param blen length of buffer. //! //! @return >=0 number of bytes sent. //! < 0 an error or occurred. //----------------------------------------------------------------------------- int Send(const char *buff, int blen); //----------------------------------------------------------------------------- //! Send data on a link. This calls may block unless the socket was marked //! nonblocking. If a block would occur, the data is copied for later sending. //! //! @param iov pointer to the message vector. //! @param iocnt number of iov elements in the vector. //! @param bytes the sum of the sizes in the vector. //! //! @return >=0 number of bytes sent. //! < 0 an error occurred. //----------------------------------------------------------------------------- int Send(const struct iovec *iov, int iocnt, int bytes=0); //----------------------------------------------------------------------------- //! Send data on a link using sendfile(). This call always blocks until all //! data is sent. It should only be called if sfOK is true (see below). //! //! @param sdP pointer to the sendfile vector. //! @param sdn number of elements in the vector. //! //! @return >=0 number of bytes sent. //! < 0 an error occurred. //----------------------------------------------------------------------------- static bool sfOK; // True if Send(sfVec) enabled typedef XrdOucSFVec sfVec; int Send(const sfVec *sdP, int sdn); // Iff sfOK is true //----------------------------------------------------------------------------- //! Wait for all outstanding requests to be completed on the link. //----------------------------------------------------------------------------- void Serialize(); //----------------------------------------------------------------------------- //! Set an error indication on he link. //! //! @param text a message describing the error. //! //! @return =0 message set, the link is considered in error. //! -1 the message pointer was nil. //----------------------------------------------------------------------------- int setEtext(const char *text); //----------------------------------------------------------------------------- //! Set the client's link identity. //! //! @param userid pointer to the client's username. //! @param procid the client's process id (i.e. pid). //----------------------------------------------------------------------------- void setID(const char *userid, int procid); //----------------------------------------------------------------------------- //! Set the client's location. //! //! @param loc reference to the location information. //----------------------------------------------------------------------------- void setLocation(XrdNetAddrInfo::LocInfo &loc); //----------------------------------------------------------------------------- //! Set the link to be non-blocking. //! //! @return true mode has been set. //! @return false mode is not supported for this operating system. //----------------------------------------------------------------------------- bool setNB(); //----------------------------------------------------------------------------- //! Set the link's protocol. //! //! @param pp pointer to the protocol object. //! @param runit if true, starts running the protocol. //! @param push if true, pushes current protocol to be the alternate one. //! //! @return pointer to the previous protocol (may be nil). //----------------------------------------------------------------------------- XrdProtocol *setProtocol(XrdProtocol *pp, bool runit=false, bool push=false); //----------------------------------------------------------------------------- //! Set the link's protocol name. //! //! @param name name of he protocol being used. The storage must be //! valid for the duration of the program. //----------------------------------------------------------------------------- void setProtName(const char *name); //----------------------------------------------------------------------------- //! Set the link's parallel usage count. //! //! @param cnt should be 1 to increased the count and -1 to decrease it. //----------------------------------------------------------------------------- void setRef(int cnt); //----------------------------------------------------------------------------- //! Enable or disable TLS on the link. // //! @param enable if true, TLS is enabled if not already enabled. Otherwise, //! TLS is disabled and the TLS logical connection torn down. //! statistics may be contradictory as they are collected async. //! @param ctx The context to use when enabling TLS. When nil, the default //! context is used. //! //! @return True if successful, false otherwise. //----------------------------------------------------------------------------- bool setTLS(bool enable, XrdTlsContext *ctx=0); //----------------------------------------------------------------------------- //! Shutdown the link but otherwise keep it intact. //! //! @param getLock if true, the operation is performed under a lock. //----------------------------------------------------------------------------- void Shutdown(bool getLock); //----------------------------------------------------------------------------- //! Obtain link statistics. //! //! @param buff pointer to the buffer for the xml statistics. //! @param blen length of the buffer. //! @param do_sync if true, the statistics self-consistent. Otherwise, the //! statistics may be contradictory as they are collected async. //! //! @return number of bytes placed in the buffer excluding the null byte. //----------------------------------------------------------------------------- static int Stats(char *buff, int blen, bool do_sync=0); //----------------------------------------------------------------------------- //! Add all local statistics to the global counters. //! //! @param ctime if not nil, return the total connect time in seconds. //----------------------------------------------------------------------------- void syncStats(int *ctime=0); //----------------------------------------------------------------------------- //! Terminate a connection. //! //! @param owner pointer to the link ID representing a client who made //! the connection to be terminated. If nil then this is a //! request for the link to terminate another link, if possible. //! @param fdnum the file descriptor number of the link to be terminated. //! @param inst the link's instance number. //! //! @return >0 caller should wait this number of seconds and try again. //! @return =0 link terminated. //! @return <0 link could not be terminated: //! -EACCES the links was not created by the passed owner. //! -EPIPE link already being terminated. //! -ESRCH fdnum does not refer to a link. //! -ETIME unsuccessful, too many tries. //----------------------------------------------------------------------------- int Terminate(const char *owner, int fdnum, unsigned int inst); //----------------------------------------------------------------------------- //! Return the time the link was made active (i.e. time of connection). //----------------------------------------------------------------------------- time_t timeCon() const; //----------------------------------------------------------------------------- //! Return link's reference count. //----------------------------------------------------------------------------- int UseCnt() const; //----------------------------------------------------------------------------- //! Mark this link as an in-memory communications bridge (internal use only). //----------------------------------------------------------------------------- void armBridge(); //----------------------------------------------------------------------------- //! Determine if this link is a bridge. //! //! @return true this link is a bridge. //! @return false this link is a plain old link. //----------------------------------------------------------------------------- bool hasBridge() const {return isBridged;} //----------------------------------------------------------------------------- //! Determine if this link is using TLS. //! //! @return true this link is using TLS. //! @return false this link not using TLS. //----------------------------------------------------------------------------- bool hasTLS() const {return isTLS;} //----------------------------------------------------------------------------- //! Return TLS protocol version being used. //! //! @return The TLS protocol version number. If the link is not using TLS, //! a null string is returned; //----------------------------------------------------------------------------- const char *verTLS(); //----------------------------------------------------------------------------- //! Constructor //! //! @param lxq Reference to the implementation. //----------------------------------------------------------------------------- XrdLink(XrdLinkXeq &lxq); protected: ~XrdLink() {} // Is never deleted! void DoIt(); // This is an override of XrdJob::DoIt. void ResetLink(); int Wait4Data(int timeout); void *rsvd1[3]; // Reserved for future use XrdLinkXeq &linkXQ; // The implementation char *HostName; // Pointer to the hostname unsigned int Instance; // Instance number of this object bool isBridged; // If true, this link is an in-memory bridge bool isTLS; // If true, this link uses TLS for all I/O char rsvd2[2]; }; #endif xrootd-5.6.9/src/Xrd/XrdLinkCtl.cc000066400000000000000000000342721457266313600167500ustar00rootroot00000000000000/******************************************************************************/ /* */ /* X r d L i n k C t l . c c */ /* */ /* (c) 2018 by the Board of Trustees of the Leland Stanford, Jr., University */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include #include #include "Xrd/XrdInet.hh" #include "Xrd/XrdLinkCtl.hh" #include "Xrd/XrdLinkMatch.hh" #include "Xrd/XrdPoll.hh" #include "Xrd/XrdScheduler.hh" #define TRACELINK this #include "Xrd/XrdTrace.hh" #include "XrdSys/XrdSysAtomics.hh" #include "XrdSys/XrdSysLogger.hh" #include "XrdSys/XrdSysPthread.hh" /******************************************************************************/ /* G l o b a l O b j e c t s */ /******************************************************************************/ namespace XrdGlobal { extern XrdSysError Log; extern XrdScheduler Sched; extern XrdInet *XrdNetTCP; }; using namespace XrdGlobal; /******************************************************************************/ /* S t a t i c s */ /******************************************************************************/ XrdLinkCtl **XrdLinkCtl::LinkTab = 0; char *XrdLinkCtl::LinkBat = 0; unsigned int XrdLinkCtl::LinkAlloc= 0; int XrdLinkCtl::LTLast = -1; int XrdLinkCtl::maxFD = 0; XrdSysMutex XrdLinkCtl::LTMutex; short XrdLinkCtl::killWait = 3; // Kill then wait; short XrdLinkCtl::waitKill = 4; // Wait then kill const char *XrdLinkCtl::TraceID = "LinkCtl"; namespace { XrdSysMutex instMutex; unsigned int myInstance = 1; int idleCheck; int idleTicks; static const int XRDLINK_USED = 0x01; static const int XRDLINK_FREE = 0x00; class LinkScan : public XrdJob { public: void DoIt() {XrdLinkCtl::idleScan(); Sched.Schedule((XrdJob *)this, idleCheck+time(0)); } LinkScan() : XrdJob("Idle link scan") {} ~LinkScan() {} }; } /******************************************************************************/ /* A l l o c */ /******************************************************************************/ XrdLink *XrdLinkCtl::Alloc(XrdNetAddr &peer, int opts) { XrdLinkCtl *lp; char hName[1024], *unp, buff[32]; int bl, peerFD = peer.SockFD(); // Make sure that the incoming file descriptor can be handled // if (peerFD < 0 || peerFD >= maxFD) {snprintf(hName, sizeof(hName), "%d", peerFD); Log.Emsg("Link", "attempt to alloc out of range FD -",hName); return (XrdLink *)0; } // Make sure that the link slot is available // LTMutex.Lock(); if (LinkBat[peerFD]) {LTMutex.UnLock(); snprintf(hName, sizeof(hName), "%d", peerFD); Log.Emsg("Link", "attempt to reuse active link FD -",hName); return (XrdLink *)0; } // Check if we already have a link object in this slot. If not, allocate // a quantum of link objects and put them in the table. // if (!(lp = LinkTab[peerFD])) {unsigned int i; XrdLinkCtl **blp, *nlp = new XrdLinkCtl[LinkAlloc](); if (!nlp) {LTMutex.UnLock(); Log.Emsg("Link", ENOMEM, "create link"); return (XrdLink *)0; } blp = &LinkTab[peerFD/LinkAlloc*LinkAlloc]; for (i = 0; i < LinkAlloc; i++, blp++) *blp = &nlp[i]; lp = LinkTab[peerFD]; } else lp->Reset(); LinkBat[peerFD] = XRDLINK_USED; if (peerFD > LTLast) LTLast = peerFD; LTMutex.UnLock(); // Establish the instance number of this link. This is will prevent us from // sending asynchronous responses to the wrong client when the file descriptor // gets reused for connections to the same host. // instMutex.Lock(); lp->Instance = myInstance++; instMutex.UnLock(); // Establish the address and connection name of this link // peer.Format(hName, sizeof(hName), XrdNetAddr::fmtAuto, XrdNetAddr::old6Map4 | XrdNetAddr::noPort); lp->HostName = strdup(hName); lp->HNlen = strlen(hName); XrdNetTCP->Trim(hName); lp->Addr = peer; strlcpy(lp->Lname, hName, sizeof(lp->Lname)); bl = sprintf(buff, "anon.0:%d", peerFD); unp = lp->Uname + sizeof(Uname) - bl - 1; // Solaris compatibility memcpy(unp, buff, bl); lp->ID = unp; lp->PollInfo.FD = lp->LinkInfo.FD = peerFD; lp->Comment = (const char *)unp; // Set options as needed // lp->LockReads = (0 != (opts & XRDLINK_RDLOCK)); lp->KeepFD = (0 != (opts & XRDLINK_NOCLOSE)); // Update statistics and return the link. We need to actually get the stats // mutex even when using atomics because we need to use compound operations. // The atomics will keep reporters from seeing partial results. // statsMutex.Lock(); AtomicInc(LinkCountTot); // LinkCountTot++ if (LinkCountMax <= AtomicInc(LinkCount)) LinkCountMax = LinkCount; statsMutex.UnLock(); return lp; } /******************************************************************************/ /* F i n d */ /******************************************************************************/ // Warning: curr must be set to a value of 0 or less on the initial call and // not touched therafter unless a null pointer is returned. When an // actual link object pointer is returned, it's refcount is increased. // The count is automatically decreased on the next call to Find(). // XrdLink *XrdLinkCtl::Find(int &curr, XrdLinkMatch *who) { XrdLinkCtl *lp; const int MaxSeek = 16; unsigned int myINS; int i, seeklim = MaxSeek; // Do initialization // LTMutex.Lock(); if (curr >= 0 && LinkTab[curr]) LinkTab[curr]->setRef(-1); else curr = -1; // Find next matching link. Since this may take some time, we periodically // release the LTMutex lock which drives up overhead but will still allow // other critical operations to occur. // for (i = curr+1; i <= LTLast; i++) {if ((lp = LinkTab[i]) && LinkBat[i] && lp->HostName) if (!who || who->Match(lp->ID,lp->Lname-lp->ID-1,lp->HostName,lp->HNlen)) {myINS = lp->Instance; LTMutex.UnLock(); lp->setRef(1); curr = i; if (myINS == lp->Instance) return lp; LTMutex.Lock(); } if (!seeklim--) {LTMutex.UnLock(); seeklim = MaxSeek; LTMutex.Lock();} } // Done scanning the table // LTMutex.UnLock(); curr = -1; return 0; } /******************************************************************************/ /* g e t N a m e */ /******************************************************************************/ // Warning: curr must be set to a value of 0 or less on the initial call and // not touched therafter unless null is returned. Returns the length // the name in nbuf. // int XrdLinkCtl::getName(int &curr, char *nbuf, int nbsz, XrdLinkMatch *who) { XrdLinkCtl *lp; const int MaxSeek = 16; int i, ulen = 0, seeklim = MaxSeek; // Find next matching link. Since this may take some time, we periodically // release the LTMutex lock which drives up overhead but will still allow // other critical operations to occur. // LTMutex.Lock(); for (i = curr+1; i <= LTLast; i++) {if ((lp = LinkTab[i]) && LinkBat[i] && lp->HostName) if (!who || who->Match(lp->ID,lp->Lname-lp->ID-1,lp->HostName,lp->HNlen)) {ulen = lp->Client(nbuf, nbsz); LTMutex.UnLock(); curr = i; return ulen; } if (!seeklim--) {LTMutex.UnLock(); seeklim = MaxSeek; LTMutex.Lock();} } LTMutex.UnLock(); // Done scanning the table // curr = -1; return 0; } /******************************************************************************/ /* i d l e S c a n */ /******************************************************************************/ #undef TRACELINK #define TRACELINK lp void XrdLinkCtl::idleScan() { XrdLinkCtl *lp; int i, ltlast, lnum = 0, tmo = 0, tmod = 0; // Get the current link high watermark // LTMutex.Lock(); ltlast = LTLast; LTMutex.UnLock(); // Scan across all links looking for idle links. Links are never deallocated // so we don't need any special kind of lock for these // for (i = 0; i <= ltlast; i++) {if (LinkBat[i] != XRDLINK_USED || !(lp = LinkTab[i])) continue; lnum++; lp->LinkInfo.opMutex.Lock(); if (lp->isIdle) tmo++; lp->isIdle++; if ((int(lp->isIdle)) < idleTicks) {lp->LinkInfo.opMutex.UnLock(); continue;} lp->isIdle = 0; if (!(lp->PollInfo.Poller) || !(lp->PollInfo.isEnabled)) Log.Emsg("LinkScan","Link",lp->ID,"is disabled and idle."); else if (lp->LinkInfo.InUse == 1) {lp->PollInfo.Poller->Disable(lp->PollInfo, "idle timeout"); tmod++; } lp->LinkInfo.opMutex.UnLock(); } // Trace what we did // TRACE(CONN, lnum <<" links; " < 0) waitKill = static_cast(wkSec); if (kwSec > 0) killWait = static_cast(kwSec); } /******************************************************************************/ /* S e t u p */ /******************************************************************************/ int XrdLinkCtl::Setup(int maxfds, int idlewait) { int numalloc; // Compute the number of link objects we should allocate at a time. Generally, // we like to allocate 8k of them at a time but always as a power of two. // maxFD = maxfds; numalloc = 8192 / sizeof(XrdLink); LinkAlloc = 1; while((numalloc = numalloc/2)) LinkAlloc = LinkAlloc*2; TRACE(DEBUG, "Allocating " <syncStats();} } /******************************************************************************/ /* U n h o o k */ /******************************************************************************/ void XrdLinkCtl::Unhook(int fd) { // Indicate link no longer actvely neing used // LTMutex.Lock(); LinkBat[fd] = XRDLINK_FREE; if (fd == LTLast) while(LTLast && !(LinkBat[LTLast])) LTLast--; LTMutex.UnLock(); } xrootd-5.6.9/src/Xrd/XrdLinkCtl.hh000066400000000000000000000231661457266313600167620ustar00rootroot00000000000000#ifndef __XRD_LINKCTL_H__ #define __XRD_LINKCTL_H__ /******************************************************************************/ /* */ /* X r d L i n k C t l . h h */ /* */ /* (c) 2018 by the Board of Trustees of the Leland Stanford, Jr., University */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include "Xrd/XrdLinkXeq.hh" #include "XrdSys/XrdSysPthread.hh" /******************************************************************************/ /* C l a s s D e f i n i t i o n */ /******************************************************************************/ class XrdLinkMatch; class XrdLinkCtl : protected XrdLinkXeq { public: //----------------------------------------------------------------------------- //! Allocate a new link object. //! //! @param peer The connection information for the endpoint. //! @param opts Processing options: //! XRDLINK_NOCLOSE - do not close the FD upon recycling. //! XRDLINK_RDLOCK - obtain a lock prior to reading data. //! //! @return !0 The pointer to the new object. //! =0 A new link object could not be allocated. //----------------------------------------------------------------------------- #define XRDLINK_RDLOCK 0x0001 #define XRDLINK_NOCLOSE 0x0002 static XrdLink *Alloc(XrdNetAddr &peer, int opts=0); //----------------------------------------------------------------------------- //! Translate a file descriptor number to the corresponding link object. //! //! @param fd The file descriptor number. //! //! @return !0 Pointer to the link object. //! =0 The file descriptor is not associated with a link. //----------------------------------------------------------------------------- static XrdLink *fd2link(int fd) {if (fd < 0) fd = -fd; return (fd <= LTLast && LinkBat[fd] ? LinkTab[fd] : 0); } //----------------------------------------------------------------------------- //! Translate a file descriptor number and an instance to a link object. //! //! @param fd The file descriptor number. //! @param inst The file descriptor number instance number. //! //! @return !0 Pointer to the link object. //! =0 The file descriptor instance is not associated with a link. //----------------------------------------------------------------------------- static XrdLink *fd2link(int fd, unsigned int inst) {if (fd < 0) fd = -fd; if (fd <= LTLast && LinkBat[fd] && LinkTab[fd] && LinkTab[fd]->Instance == inst) return LinkTab[fd]; return (XrdLink *)0; } //----------------------------------------------------------------------------- //! Translate a file descriptor number to the corresponding PollInfo object. //! //! @param fd The file descriptor number. //! //! @return !0 Pointer to the PollInfo object. //! =0 The file descriptor is not associated with a link. //----------------------------------------------------------------------------- static XrdPollInfo *fd2PollInfo(int fd) {if (fd < 0) fd = -fd; if (fd <= LTLast && LinkBat[fd]) return &(LinkTab[fd]->PollInfo); return 0; } //----------------------------------------------------------------------------- //! Find the next link matching certain attributes. //! //! @param curr Is an internal tracking value that allows repeated calls. //! It must be set to a value of 0 or less on the initial call //! and not touched therafter unless a null pointer is returned. //! @param who If the object use to check if the link matches the wanted //! criterea (typically, client name and host name). If the //! ppointer is nil, the next link is always returned. //! //! @return !0 Pointer to the link object that matches the criterea. The //! link's reference counter is increased to prevent it from //! being reused. A subsequent call will reduce the number. //! =0 No more links exist with the specified criterea. //----------------------------------------------------------------------------- static XrdLink *Find(int &curr, XrdLinkMatch *who=0); //----------------------------------------------------------------------------- //! Find the next client name matching certain attributes. //! //! @param curr Is an internal tracking value that allows repeated calls. //! It must be set to a value of 0 or less on the initial call //! and not touched therafter unless zero is returned. //! @param bname Pointer to a buffer where the name is to be returned. //! @param blen The length of the buffer. //! @param who If the object use to check if the link matches the wanted //! criterea (typically, client name and host name). If the //! pointer is nil, a match always occurs. //! //! @return !0 The length of the name placed in the buffer. //! =0 No more links exist with the specified criterea. //----------------------------------------------------------------------------- static int getName(int &curr, char *bname, int blen, XrdLinkMatch *who=0); //----------------------------------------------------------------------------- //! Look for idle links and close hem down. //----------------------------------------------------------------------------- static void idleScan(); //----------------------------------------------------------------------------- //! Set kill constants. //! //! @param wkSec Seconds to wait for kill to happed, //! @param kwSec The minimum number of seconds to wait after killing a //! connection for it to end. //----------------------------------------------------------------------------- static void setKWT(int wkSec, int kwSec); //----------------------------------------------------------------------------- //! Setup link processing. //! //! @param maxfds The maximum number of connections to handle. //! @param idlewt The time interval to check for idle connections. //! //! @return !0 Successful. //! =0 Setup failed. //----------------------------------------------------------------------------- static int Setup(int maxfds, int idlewt); //----------------------------------------------------------------------------- //! Synchronize statustics for ll links. //----------------------------------------------------------------------------- static void SyncAll(); //----------------------------------------------------------------------------- //! Unhook a link from the active table of links. //----------------------------------------------------------------------------- static void Unhook(int fd); //----------------------------------------------------------------------------- //! Link destruction control constants //----------------------------------------------------------------------------- static short killWait; // Kill then wait; static short waitKill; // Wait then kill //----------------------------------------------------------------------------- //! Constructor //----------------------------------------------------------------------------- XrdLinkCtl() {} private: ~XrdLinkCtl() {} // Is never deleted! static XrdSysMutex LTMutex; // For the LinkTab only LTMutex->IOMutex allowed static XrdLinkCtl **LinkTab; static char *LinkBat; static unsigned int LinkAlloc; static int LTLast; static int maxFD; static const char *TraceID; }; #endif xrootd-5.6.9/src/Xrd/XrdLinkInfo.hh000066400000000000000000000066531457266313600171350ustar00rootroot00000000000000#ifndef __XRD_LINKINFO_H__ #define __XRD_LINKINFO_H__ /******************************************************************************/ /* */ /* X r d L i n k I n f o . h h */ /* */ /* (c) 2020 by the Board of Trustees of the Leland Stanford, Jr., University */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include #include #include "XrdSys/XrdSysPthread.hh" class XrdLinkInfo { public: XrdSysCondVar *KillcvP; // Protected by opMutex! XrdSysSemaphore IOSemaphore; // Serialization semaphore time_t conTime; // Unix time connected char *Etext; // -> error text, if nil then no error. XrdSysRecMutex opMutex; // Serialization mutex int InUse; // Number of threads using this object int doPost; // Number of threads waiting for serialization int FD; // File descriptor for link use (may be negative) char KillCnt; // Number of times a kill has been attempted void Reset() {KillcvP = 0; conTime = time(0); if (Etext) {free(Etext); Etext = 0;} InUse = 1; doPost = 0; FD = -1; KillCnt = 0; } XrdLinkInfo() : IOSemaphore(0, "link i/o"), Etext(0) {Reset();} ~XrdLinkInfo() {} }; #endif xrootd-5.6.9/src/Xrd/XrdLinkMatch.cc000066400000000000000000000116761457266313600172650ustar00rootroot00000000000000/******************************************************************************/ /* */ /* X r d L i n k M a t c h . c c */ /* */ /* (c) 2005 by the Board of Trustees of the Leland Stanford, Jr., University */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include "Xrd/XrdLinkMatch.hh" #include "XrdSys/XrdSysPlatform.hh" /******************************************************************************/ /* M a t c h */ /******************************************************************************/ int XrdLinkMatch::Match(const char *uname, int unlen, const char *hname, int hnlen) { // Check if we should try to match the username // if (Unamelen && (Unamelen > unlen+1 || strncmp(uname,Uname,Unamelen))) return 0; // Check if we should match the full host name // if (HnameL && !HnamelenL) return !strcmp(HnameL, hname); // Check if prefix suffix matching might succeed // if (HnamelenL > hnlen) return 0; // Check if we should match the host name prefix // if (HnameL && strncmp(HnameL, hname, HnamelenL)) return 0; // Check if we should match the host name suffix // if (!HnameR) return 1; return !strcmp(hname+hnlen-HnamelenR, hname); } /******************************************************************************/ /* S e t */ /******************************************************************************/ void XrdLinkMatch::Set(const char *target) { char *theast; // Free any existing target // if (!target || !strcmp(target, "*")) {Uname = HnameL = HnameR = 0; Unamelen = HnamelenL = HnamelenR = 0; return; } strlcpy(Buff, target, sizeof(Buff)-1); Uname = Buff; // Find the '@' as the pivot in this name // if (!(HnameL = index(Uname, '@'))) {if ((Unamelen = strlen(Uname))) {if (Uname[Unamelen-1] == '*') Unamelen--; else if (index(Uname, ':')) Uname[Unamelen++] = '@'; else if (index(Uname, '.')) Uname[Unamelen++] = ':'; else Uname[Unamelen++] = '.'; } HnameR = 0; return; } // We have a form of @ // *HnameL++ = '\0'; if ((Unamelen = strlen(Uname))) {if (Uname[Unamelen-1] == '*') Unamelen--; else if (index(Uname, ':')) Uname[Unamelen++] = '@'; else if (index(Uname, '.')) Uname[Unamelen++] = ':'; else Uname[Unamelen++] = '.'; } // The post string may have an asterisk. // if (!(theast = index(HnameL, '*'))) {HnamelenL = 0; HnameR = 0; return; } // Indicate how much of the prefix should match // *theast = '\0'; if (!(HnamelenL = strlen(HnameL))) HnameL = 0; // Indicate how much of the suffix should match // if ((HnamelenR = strlen(theast))) HnameR = theast+1; else HnameR = 0; Hnamelen = HnamelenL+HnamelenR; } xrootd-5.6.9/src/Xrd/XrdLinkMatch.hh000066400000000000000000000063031457266313600172660ustar00rootroot00000000000000#ifndef __LINK_MATCH__ #define __LINK_MATCH__ /******************************************************************************/ /* */ /* X r d L i n k M a t c h . h h */ /* */ /* (c) 2005 by the Board of Trustees of the Leland Stanford, Jr., University */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include class XrdLinkMatch { public: int Match(const char *uname, int unlen, const char *hname, int hnlen); inline int Match(const char *uname, int unlen, const char *hname) {return Match(uname, unlen, hname, strlen(hname));} // Target: [][*][@[][*][]] // void Set(const char *target); XrdLinkMatch(const char *target=0) {Uname = HnameL = HnameR = 0; Unamelen = Hnamelen = 0; if (target) Set(target); } ~XrdLinkMatch() {} private: char Buff[256]; int Unamelen; char *Uname; int HnamelenL; char *HnameL; int HnamelenR; char *HnameR; int Hnamelen; }; #endif xrootd-5.6.9/src/Xrd/XrdLinkXeq.cc000066400000000000000000001332541457266313600167630ustar00rootroot00000000000000/******************************************************************************/ /* */ /* X r d L i n k X e q . c c */ /* */ /* (c) 2018 by the Board of Trustees of the Leland Stanford, Jr., University */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include #include #include #include #include #include #include #if defined(__linux__) || defined(__GNU__) #include #if !defined(TCP_CORK) #undef HAVE_SENDFILE #endif #endif #ifdef HAVE_SENDFILE #if defined(__solaris__) || defined(__linux__) || defined(__GNU__) #include #endif #endif #include "XrdSys/XrdSysAtomics.hh" #include "XrdSys/XrdSysError.hh" #include "XrdSys/XrdSysFD.hh" #include "XrdSys/XrdSysPlatform.hh" #include "Xrd/XrdBuffer.hh" #include "Xrd/XrdLink.hh" #include "Xrd/XrdLinkCtl.hh" #include "Xrd/XrdLinkXeq.hh" #include "Xrd/XrdPoll.hh" #include "Xrd/XrdScheduler.hh" #include "Xrd/XrdSendQ.hh" #include "Xrd/XrdTcpMonPin.hh" #define TRACE_IDENT ID #include "Xrd/XrdTrace.hh" /******************************************************************************/ /* G l o b a l s */ /******************************************************************************/ namespace { int getIovMax() { int maxiov; #ifdef _SC_IOV_MAX if ((maxiov = sysconf(_SC_IOV_MAX)) > 0) return maxiov; #endif #ifdef IOV_MAX return IOV_MAX; #else return 1024; #endif } }; namespace XrdGlobal { extern XrdSysError Log; extern XrdScheduler Sched; extern XrdTlsContext *tlsCtx; XrdTcpMonPin *TcpMonPin = 0; extern int devNull; int maxIOV = getIovMax(); }; using namespace XrdGlobal; /******************************************************************************/ /* S t a t i c s */ /******************************************************************************/ const char *XrdLinkXeq::TraceID = "LinkXeq"; long long XrdLinkXeq::LinkBytesIn = 0; long long XrdLinkXeq::LinkBytesOut = 0; long long XrdLinkXeq::LinkConTime = 0; long long XrdLinkXeq::LinkCountTot = 0; int XrdLinkXeq::LinkCount = 0; int XrdLinkXeq::LinkCountMax = 0; int XrdLinkXeq::LinkTimeOuts = 0; int XrdLinkXeq::LinkStalls = 0; int XrdLinkXeq::LinkSfIntr = 0; XrdSysMutex XrdLinkXeq::statsMutex; /******************************************************************************/ /* C o n s t r u c t o r */ /******************************************************************************/ XrdLinkXeq::XrdLinkXeq() : XrdLink(*this), PollInfo((XrdLink &)*this) { XrdLinkXeq::Reset(); } void XrdLinkXeq::Reset() { memcpy(Uname+sizeof(Uname)-7, "anon.0@", 7); strcpy(Lname, "somewhere"); ID = &Uname[sizeof(Uname)-5]; Comment = ID; sendQ = 0; stallCnt = stallCntTot = 0; tardyCnt = tardyCntTot = 0; SfIntr = 0; isIdle = 0; BytesOut = BytesIn = BytesOutTot = BytesInTot = 0; LockReads= false; KeepFD = false; Protocol = 0; ProtoAlt = 0; LinkInfo.Reset(); PollInfo.Zorch(); ResetLink(); } /******************************************************************************/ /* B a c k l o g */ /******************************************************************************/ int XrdLinkXeq::Backlog() { XrdSysMutexHelper lck(wrMutex); // Return backlog information // return (sendQ ? sendQ->Backlog() : 0); } /******************************************************************************/ /* C l i e n t */ /******************************************************************************/ int XrdLinkXeq::Client(char *nbuf, int nbsz) { int ulen; // Generate full client name // if (nbsz <= 0) return 0; ulen = (Lname - ID); if ((ulen + HNlen) >= nbsz) ulen = 0; else {strncpy(nbuf, ID, ulen); strcpy(nbuf+ulen, HostName); ulen += HNlen; } return ulen; } /******************************************************************************/ /* C l o s e */ /******************************************************************************/ int XrdLinkXeq::Close(bool defer) { XrdSysMutexHelper opHelper(LinkInfo.opMutex); int csec, fd, rc = 0; // If a defer close is requested, we can close the descriptor but we must // keep the slot number to prevent a new client getting the same fd number. // Linux is peculiar in that any in-progress operations will remain in that // state even after the FD is closed unless there is some activity either on // the connection or an event occurs that causes an operation restart. We // portably solve this problem by issuing a shutdown() on the socket prior // closing it. On most platforms, this informs readers that the connection is // gone (though not on old (i.e. <= 2.3) versions of Linux, sigh). Also, if // nonblocking mode is enabled, we need to do this in a separate thread as // a shutdown may block for a pretty long time if lots\ of messages are queued. // We will ask the SendQ object to schedule the shutdown for us before it // commits suicide. // Note that we can hold the opMutex while we also get the wrMutex. // if (defer) {if (!sendQ) Shutdown(false); else {TRACEI(DEBUG, "Shutdown FD " <Terminate(this); sendQ = 0; wrMutex.UnLock(); } return 0; } // If we got here then this is not a deferred close so we just need to check // if there is a sendq appendage we need to get rid of. // if (sendQ) {wrMutex.Lock(); sendQ->Terminate(); sendQ = 0; wrMutex.UnLock(); } // Multiple protocols may be bound to this link. If it is in use, defer the // actual close until the use count drops to one. // while(LinkInfo.InUse > 1) {opHelper.UnLock(); TRACEI(DEBUG, "Close FD "< Stop getting requests, // -EINPROGRESS leave link disabled but otherwise all is well // -n Error, disable and close the link // = 0 -> OK, get next request, if allowed, o/w enable the link // > 0 -> Slow link, stop getting requests and enable the link // if (Protocol) do {rc = Protocol->Process(this);} while (!rc && Sched.canStick()); else {Log.Emsg("Link", "Dispatch on closed link", ID); return; } // Either re-enable the link and cycle back waiting for a new request, leave // disabled, or terminate the connection. // if (rc >= 0) {if (PollInfo.Poller && !PollInfo.Poller->Enable(PollInfo)) Close();} else if (rc != -EINPROGRESS) Close(); } /******************************************************************************/ /* g e t P e e r C e r t s */ /******************************************************************************/ XrdTlsPeerCerts *XrdLinkXeq::getPeerCerts() { return (isTLS ? tlsIO.getCerts(true) : 0); } /******************************************************************************/ /* P e e k */ /******************************************************************************/ int XrdLinkXeq::Peek(char *Buff, int Blen, int timeout) { XrdSysMutexHelper theMutex; struct pollfd polltab = {PollInfo.FD, POLLIN|POLLRDNORM, 0}; ssize_t mlen; int retc; // Lock the read mutex if we need to, the helper will unlock it upon exit // if (LockReads) theMutex.Lock(&rdMutex); // Wait until we can actually read something // isIdle = 0; do {retc = poll(&polltab, 1, timeout);} while(retc < 0 && errno == EINTR); if (retc != 1) {if (retc == 0) return 0; return Log.Emsg("Link", -errno, "poll", ID); } // Verify it is safe to read now // if (!(polltab.revents & (POLLIN|POLLRDNORM))) {Log.Emsg("Link", XrdPoll::Poll2Text(polltab.revents), "polling", ID); return -1; } // Do the peek. // do {mlen = recv(LinkInfo.FD, Buff, Blen, MSG_PEEK);} while(mlen < 0 && errno == EINTR); // Return the result // if (mlen >= 0) return int(mlen); Log.Emsg("Link", errno, "peek on", ID); return -1; } /******************************************************************************/ /* R e c v */ /******************************************************************************/ int XrdLinkXeq::Recv(char *Buff, int Blen) { ssize_t rlen; // Note that we will read only as much as is queued. Use Recv() with a // timeout to receive as much data as possible. // if (LockReads) rdMutex.Lock(); isIdle = 0; do {rlen = read(LinkInfo.FD, Buff, Blen);} while(rlen < 0 && errno == EINTR); if (rlen > 0) AtomicAdd(BytesIn, rlen); if (LockReads) rdMutex.UnLock(); if (rlen >= 0) return int(rlen); if (LinkInfo.FD >= 0) Log.Emsg("Link", errno, "receive from", ID); return -1; } /******************************************************************************/ int XrdLinkXeq::Recv(char *Buff, int Blen, int timeout) { XrdSysMutexHelper theMutex; struct pollfd polltab = {PollInfo.FD, POLLIN|POLLRDNORM, 0}; ssize_t rlen, totlen = 0; int retc; // Lock the read mutex if we need to, the helper will unlock it upon exit // if (LockReads) theMutex.Lock(&rdMutex); // Wait up to timeout milliseconds for data to arrive // isIdle = 0; while(Blen > 0) {do {retc = poll(&polltab,1,timeout);} while(retc < 0 && errno == EINTR); if (retc != 1) {if (retc == 0) {tardyCnt++; if (totlen) {if ((++stallCnt & 0xff) == 1) TRACEI(DEBUG,"read timed out"); AtomicAdd(BytesIn, totlen); } return int(totlen); } return (LinkInfo.FD >= 0 ? Log.Emsg("Link",-errno,"poll",ID) : -1); } // Verify it is safe to read now // if (!(polltab.revents & (POLLIN|POLLRDNORM))) {Log.Emsg("Link", XrdPoll::Poll2Text(polltab.revents), "polling", ID); return -1; } // Read as much data as you can. Note that we will force an error // if we get a zero-length read after poll said it was OK. // do {rlen = recv(LinkInfo.FD, Buff, Blen, 0);} while(rlen < 0 && errno == EINTR); if (rlen <= 0) {if (!rlen) return -ENOMSG; if (LinkInfo.FD > 0) Log.Emsg("Link", -errno, "receive from", ID); return -1; } totlen += rlen; Blen -= rlen; Buff += rlen; } AtomicAdd(BytesIn, totlen); return int(totlen); } /******************************************************************************/ int XrdLinkXeq::Recv(const struct iovec *iov, int iocnt, int timeout) { XrdSysMutexHelper theMutex; struct pollfd polltab = {PollInfo.FD, POLLIN|POLLRDNORM, 0}; int retc, rlen; // Lock the read mutex if we need to, the helper will unlock it upon exit // if (LockReads) theMutex.Lock(&rdMutex); // Wait up to timeout milliseconds for data to arrive // isIdle = 0; do {retc = poll(&polltab,1,timeout);} while(retc < 0 && errno == EINTR); if (retc != 1) {if (retc == 0) {tardyCnt++; return 0; } return (LinkInfo.FD >= 0 ? Log.Emsg("Link",-errno,"poll",ID) : -1); } // Verify it is safe to read now // if (!(polltab.revents & (POLLIN|POLLRDNORM))) {Log.Emsg("Link", XrdPoll::Poll2Text(polltab.revents), "polling", ID); return -1; } // If the iocnt is within limits then just go ahead and read once. // if (iocnt <= maxIOV) {rlen = RecvIOV(iov, iocnt); if (rlen > 0) {AtomicAdd(BytesIn, rlen);} return rlen; } // We will have to break this up into allowable segments and we need to add up // the bytes in each segment so that we know when to stop reading. // int seglen, segcnt = maxIOV, totlen = 0; do {seglen = 0; for (int i = 0; i < segcnt; i++) seglen += iov[i].iov_len; if ((rlen = RecvIOV(iov, segcnt)) < 0) return rlen; totlen += rlen; if (rlen < seglen) break; iov += segcnt; iocnt -= segcnt; if (iocnt <= maxIOV) segcnt = iocnt; } while(iocnt > 0); // All done // AtomicAdd(BytesIn, totlen); return totlen; } /******************************************************************************/ /* R e c v A l l */ /******************************************************************************/ int XrdLinkXeq::RecvAll(char *Buff, int Blen, int timeout) { struct pollfd polltab = {PollInfo.FD, POLLIN|POLLRDNORM, 0}; ssize_t rlen; int retc; // Check if timeout specified. Notice that the timeout is the max we will // for some data. We will wait forever for all the data. Yeah, it's weird. // if (timeout >= 0) {do {retc = poll(&polltab,1,timeout);} while(retc < 0 && errno == EINTR); if (retc != 1) {if (!retc) return -ETIMEDOUT; Log.Emsg("Link",errno,"poll",ID); return -1; } if (!(polltab.revents & (POLLIN|POLLRDNORM))) {Log.Emsg("Link",XrdPoll::Poll2Text(polltab.revents),"polling",ID); return -1; } } // Note that we will block until we receive all he bytes. // if (LockReads) rdMutex.Lock(); isIdle = 0; do {rlen = recv(LinkInfo.FD, Buff, Blen, MSG_WAITALL);} while(rlen < 0 && errno == EINTR); if (rlen > 0) AtomicAdd(BytesIn, rlen); if (LockReads) rdMutex.UnLock(); if (int(rlen) == Blen) return Blen; if (!rlen) {TRACEI(DEBUG, "No RecvAll() data; errno=" < 0) Log.Emsg("RecvAll", "Premature end from", ID); else if (LinkInfo.FD >= 0) Log.Emsg("Link", errno, "receive from", ID); return -1; } /******************************************************************************/ /* Protected: R e c v I O V */ /******************************************************************************/ int XrdLinkXeq::RecvIOV(const struct iovec *iov, int iocnt) { ssize_t retc = 0; // Read the data in. On some version of Unix (e.g., Linux) a readv() may // end at any time without reading all the bytes when directed to a socket. // We always return the number bytes read (or an error). The caller needs to // restart the read at the appropriate place in the iovec when more data arrives. // do {retc = readv(LinkInfo.FD, iov, iocnt);} while(retc < 0 && errno == EINTR); // Check how we completed // if (retc < 0) Log.Emsg("Link", errno, "receive from", ID); return retc; } /******************************************************************************/ /* R e g i s t e r */ /******************************************************************************/ bool XrdLinkXeq::Register(const char *hName) { // First see if we can register this name with the address object // if (!Addr.Register(hName)) return false; // Make appropriate changes here // if (HostName) free(HostName); HostName = strdup(hName); strlcpy(Lname, hName, sizeof(Lname)); return true; } /******************************************************************************/ /* S e n d */ /******************************************************************************/ int XrdLinkXeq::Send(const char *Buff, int Blen) { ssize_t retc = 0, bytesleft = Blen; // Get a lock // wrMutex.Lock(); isIdle = 0; AtomicAdd(BytesOut, Blen); // Do non-blocking writes if we are setup to do so. // if (sendQ) {retc = sendQ->Send(Buff, Blen); wrMutex.UnLock(); return retc; } // Write the data out // while(bytesleft) {if ((retc = write(LinkInfo.FD, Buff, bytesleft)) < 0) {if (errno == EINTR) continue; else break; } bytesleft -= retc; Buff += retc; } // All done // wrMutex.UnLock(); if (retc >= 0) return Blen; Log.Emsg("Link", errno, "send to", ID); return -1; } /******************************************************************************/ int XrdLinkXeq::Send(const struct iovec *iov, int iocnt, int bytes) { int retc; static int maxIOV = -1; if (maxIOV == -1) { #ifdef _SC_IOV_MAX maxIOV = sysconf(_SC_IOV_MAX); if (maxIOV == -1) #endif #ifdef IOV_MAX maxIOV = IOV_MAX; #else maxIOV = 1024; #endif } // Get a lock and assume we will be successful (statistically we are) // wrMutex.Lock(); isIdle = 0; AtomicAdd(BytesOut, bytes); // Do non-blocking writes if we are setup to do so. // if (sendQ) {retc = sendQ->Send(iov, iocnt, bytes); wrMutex.UnLock(); return retc; } // If the iocnt is within limits then just go ahead and write this out // if (iocnt <= maxIOV) {retc = SendIOV(iov, iocnt, bytes); wrMutex.UnLock(); return retc; } // We will have to break this up into allowable segments // int seglen, segcnt = maxIOV, iolen = 0; do {seglen = 0; for (int i = 0; i < segcnt; i++) seglen += iov[i].iov_len; if ((retc = SendIOV(iov, segcnt, seglen)) < 0) {wrMutex.UnLock(); return retc; } iolen += retc; iov += segcnt; iocnt -= segcnt; if (iocnt <= maxIOV) segcnt = iocnt; } while(iocnt > 0); // All done // wrMutex.UnLock(); return iolen; } /******************************************************************************/ int XrdLinkXeq::Send(const sfVec *sfP, int sfN) { #if !defined(HAVE_SENDFILE) return -1; #elif defined(__solaris__) sendfilevec_t vecSF[XrdOucSFVec::sfMax], *vecSFP = vecSF; size_t xframt, totamt, bytes = 0; ssize_t retc; int i = 0; // Construct the sendfilev() vector // for (i = 0; i < sfN; sfP++, i++) {if (sfP->fdnum < 0) {vecSF[i].sfv_fd = SFV_FD_SELF; vecSF[i].sfv_off = (off_t)sfP->buffer; } else { vecSF[i].sfv_fd = sfP->fdnum; vecSF[i].sfv_off = sfP->offset; } vecSF[i].sfv_flag = 0; vecSF[i].sfv_len = sfP->sendsz; bytes += sfP->sendsz; } totamt = bytes; // Lock the link, issue sendfilev(), and unlock the link. The documentation // is very spotty and inconsistent. We can only retry this operation under // very limited conditions. // wrMutex.Lock(); isIdle = 0; do{retc = sendfilev(LinkInfo.FD, vecSFP, sfN, &xframt); // Check if all went well and return if so (usual case) // if (xframt == bytes) {AtomicAdd(BytesOut, bytes); wrMutex.UnLock(); return totamt; } // The only one we will recover from is EINTR. We cannot legally get EAGAIN. // if (retc < 0 && errno != EINTR) break; // Try to resume the transfer // if (xframt > 0) {AtomicAdd(BytesOut, xframt); bytes -= xframt; SfIntr++; while(xframt > 0 && sfN) {if ((ssize_t)xframt < (ssize_t)vecSFP->sfv_len) {vecSFP->sfv_off += xframt; vecSFP->sfv_len -= xframt; break;} xframt -= vecSFP->sfv_len; vecSFP++; sfN--; } } } while(sfN > 0); // See if we can recover without destroying the connection // retc = (retc < 0 ? errno : ECANCELED); wrMutex.UnLock(); Log.Emsg("Link", retc, "send file to", ID); return -1; #elif defined(__linux__) || defined(__GNU__) static const int setON = 1, setOFF = 0; ssize_t retc = 0, bytesleft; off_t myOffset; int i, xfrbytes = 0, uncork = 1, xIntr = 0; // lock the link // wrMutex.Lock(); isIdle = 0; // In linux we need to cork the socket. On permanent errors we do not uncork // the socket because it will be closed in short order. // if (setsockopt(PollInfo.FD, SOL_TCP, TCP_CORK, &setON, sizeof(setON)) < 0) {Log.Emsg("Link", errno, "cork socket for", ID); uncork = 0; sfOK = 0; } // Send the header first // for (i = 0; i < sfN; sfP++, i++) {if (sfP->fdnum < 0) retc = sendData(sfP->buffer, sfP->sendsz); else {myOffset = sfP->offset; bytesleft = sfP->sendsz; while(bytesleft && (retc=sendfile(LinkInfo.FD,sfP->fdnum,&myOffset,bytesleft)) > 0) {bytesleft -= retc; xIntr++;} } if (retc < 0 && errno == EINTR) continue; if (retc <= 0) break; xfrbytes += sfP->sendsz; } // Diagnose any sendfile errors // if (retc <= 0) {if (retc == 0) errno = ECANCELED; wrMutex.UnLock(); Log.Emsg("Link", errno, "send file to", ID); return -1; } // Now uncork the socket // if (uncork && setsockopt(PollInfo.FD, SOL_TCP, TCP_CORK, &setOFF, sizeof(setOFF)) < 0) Log.Emsg("Link", errno, "uncork socket for", ID); // All done // if (xIntr > sfN) SfIntr += (xIntr - sfN); AtomicAdd(BytesOut, xfrbytes); wrMutex.UnLock(); return xfrbytes; #else return -1; #endif } /******************************************************************************/ /* Protected: s e n d D a t a */ /******************************************************************************/ int XrdLinkXeq::sendData(const char *Buff, int Blen) { ssize_t retc = 0, bytesleft = Blen; // Write the data out // while(bytesleft) {if ((retc = write(LinkInfo.FD, Buff, bytesleft)) < 0) {if (errno == EINTR) continue; else break; } bytesleft -= retc; Buff += retc; } // All done // return retc; } /******************************************************************************/ /* Protected: S e n d I O V */ /******************************************************************************/ int XrdLinkXeq::SendIOV(const struct iovec *iov, int iocnt, int bytes) { ssize_t bytesleft, n, retc = 0; const char *Buff; // Write the data out. On some version of Unix (e.g., Linux) a writev() may // end at any time without writing all the bytes when directed to a socket. // So, we attempt to resume the writev() using a combination of write() and // a writev() continuation. This approach slowly converts a writev() to a // series of writes if need be. We must do this inline because we must hold // the lock until all the bytes are written or an error occurs. // bytesleft = static_cast(bytes); while(bytesleft) {do {retc = writev(LinkInfo.FD, iov, iocnt);} while(retc < 0 && errno == EINTR); if (retc >= bytesleft || retc < 0) break; bytesleft -= retc; while(retc >= (n = static_cast(iov->iov_len))) {retc -= n; iov++; iocnt--;} Buff = (const char *)iov->iov_base + retc; n -= retc; iov++; iocnt--; while(n) {if ((retc = write(LinkInfo.FD, Buff, n)) < 0) {if (errno == EINTR) continue; else break; } n -= retc; Buff += retc; bytesleft -= retc; } if (retc < 0 || iocnt < 1) break; } // All done // if (retc >= 0) return bytes; Log.Emsg("Link", errno, "send to", ID); return -1; } /******************************************************************************/ /* s e t I D */ /******************************************************************************/ void XrdLinkXeq::setID(const char *userid, int procid) { char buff[sizeof(Uname)], *bp, *sp; int ulen; snprintf(buff, sizeof(buff), "%s.%d:%d", userid, procid, PollInfo.FD); ulen = strlen(buff); sp = buff + ulen - 1; bp = &Uname[sizeof(Uname)-1]; if (ulen > (int)sizeof(Uname)) ulen = sizeof(Uname); *bp = '@'; bp--; while(ulen--) {*bp = *sp; bp--; sp--;} ID = bp+1; Comment = (const char *)ID; // Update the ID in the TLS socket if enabled // if (isTLS) tlsIO.SetTraceID(ID); } /******************************************************************************/ /* s e t N B */ /******************************************************************************/ bool XrdLinkXeq::setNB() { // We don't support non-blocking output except for Linux at the moment // #if !defined(__linux__) return false; #else // Trace this request // TRACEI(DEBUG,"enabling non-blocking output"); // If we don't already have a sendQ object get one. This is a one-time call // so to optimize checking if this object exists we also get the opMutex.' // LinkInfo.opMutex.Lock(); if (!sendQ) {wrMutex.Lock(); sendQ = new XrdSendQ(*this, wrMutex); wrMutex.UnLock(); } LinkInfo.opMutex.UnLock(); return true; #endif } /******************************************************************************/ /* s e t P r o t o c o l */ /******************************************************************************/ XrdProtocol *XrdLinkXeq::setProtocol(XrdProtocol *pp, bool push) { // Set new protocol. // LinkInfo.opMutex.Lock(); XrdProtocol *op = Protocol; if (push) ProtoAlt = Protocol; Protocol = pp; LinkInfo.opMutex.UnLock(); return op; } /******************************************************************************/ /* s e t P r o t N a m e */ /******************************************************************************/ void XrdLinkXeq::setProtName(const char *name) { // Set the protocol name. // LinkInfo.opMutex.Lock(); Addr.SetDialect(name); LinkInfo.opMutex.UnLock(); } /******************************************************************************/ /* s e t T L S */ /******************************************************************************/ bool XrdLinkXeq::setTLS(bool enable, XrdTlsContext *ctx) { //??? // static const XrdTlsConnection::RW_Mode rwMode=XrdTlsConnection::TLS_RNB_WBL; static const XrdTlsSocket::RW_Mode rwMode=XrdTlsSocket::TLS_RBL_WBL; static const XrdTlsSocket::HS_Mode hsMode=XrdTlsSocket::TLS_HS_BLOCK; const char *eNote; XrdTls::RC rc; // If we are already in a compatible mode, we are done // if (isTLS == enable) return true; // If this is a shutdown, then do it now. // if (!enable) {tlsIO.Shutdown(); isTLS = enable; Addr.SetTLS(enable); return true; } // We want to initialize TLS, do so now. // if (!ctx) ctx = tlsCtx; eNote = tlsIO.Init(*ctx, PollInfo.FD, rwMode, hsMode, false, false, ID); // Check for errors // if (eNote) {char buff[1024]; snprintf(buff, sizeof(buff), "Unable to enable tls for %s;", ID); Log.Emsg("LinkXeq", buff, eNote); return false; } // Now we need to accept this TLS connection // std::string eMsg; rc = tlsIO.Accept(&eMsg); // Diagnose return state // if (rc != XrdTls::TLS_AOK) Log.Emsg("LinkXeq", eMsg.c_str()); else {isTLS = enable; Addr.SetTLS(enable); Log.Emsg("LinkXeq", ID, "connection upgraded to", verTLS()); } return rc == XrdTls::TLS_AOK; } /******************************************************************************/ /* S F E r r o r */ /******************************************************************************/ int XrdLinkXeq::SFError(int rc) { Log.Emsg("TLS", rc, "send file to", ID); return -1; } /******************************************************************************/ /* S h u t d o w n */ /******************************************************************************/ void XrdLinkXeq::Shutdown(bool getLock) { int temp; // Trace the entry // TRACEI(DEBUG, (getLock ? "Async" : "Sync") <<" link shutdown in progress"); // Get the lock if we need too (external entry via another thread) // if (getLock) LinkInfo.opMutex.Lock(); // If there is something to do, do it now // temp = Instance; Instance = 0; if (!KeepFD) {shutdown(PollInfo.FD, SHUT_RDWR); if (dup2(devNull, PollInfo.FD) < 0) {Instance = temp; Log.Emsg("Link", errno, "shutdown FD for", ID); } } // All done // if (getLock) LinkInfo.opMutex.UnLock(); } /******************************************************************************/ /* S t a t s */ /******************************************************************************/ int XrdLinkXeq::Stats(char *buff, int blen, bool do_sync) { static const char statfmt[] = "%d" "%d%lld%lld%lld" "%lld%d%d" "%d"; int i; // Check if actual length wanted // if (!buff) return sizeof(statfmt)+17*6; // We must synchronize the statistical counters // if (do_sync) XrdLinkCtl::SyncAll(); // Obtain lock on the stats area and format it // AtomicBeg(statsMutex); i = snprintf(buff, blen, statfmt, AtomicGet(LinkCount), AtomicGet(LinkCountMax), AtomicGet(LinkCountTot), AtomicGet(LinkBytesIn), AtomicGet(LinkBytesOut), AtomicGet(LinkConTime), AtomicGet(LinkTimeOuts), AtomicGet(LinkStalls), AtomicGet(LinkSfIntr)); AtomicEnd(statsMutex); return i; } /******************************************************************************/ /* s y n c S t a t s */ /******************************************************************************/ void XrdLinkXeq::syncStats(int *ctime) { long long tmpLL; int tmpI4; // If this is dynamic, get the opMutex lock // if (!ctime) LinkInfo.opMutex.Lock(); // Either the caller has the opMutex or this is called out of close. In either // case, we need to get the read and write mutexes; each followed by the stats // mutex. This order is important because we should not hold the stats mutex // for very long and the r/w mutexes may take a long time to acquire. If we // must maintain the link count we need to actually acquire the stats mutex as // we will be doing compound operations. Atomics are still used to keep other // threads from seeing partial results. // AtomicBeg(rdMutex); if (ctime) {*ctime = time(0) - LinkInfo.conTime; AtomicAdd(LinkConTime, *ctime); statsMutex.Lock(); if (LinkCount > 0) AtomicDec(LinkCount); statsMutex.UnLock(); } AtomicBeg(statsMutex); tmpLL = AtomicFAZ(BytesIn); AtomicAdd(LinkBytesIn, tmpLL); AtomicAdd(BytesInTot, tmpLL); tmpI4 = AtomicFAZ(tardyCnt); AtomicAdd(LinkTimeOuts, tmpI4); AtomicAdd(tardyCntTot, tmpI4); tmpI4 = AtomicFAZ(stallCnt); AtomicAdd(LinkStalls, tmpI4); AtomicAdd(stallCntTot, tmpI4); AtomicEnd(statsMutex); AtomicEnd(rdMutex); AtomicBeg(wrMutex); AtomicBeg(statsMutex); tmpLL = AtomicFAZ(BytesOut); AtomicAdd(LinkBytesOut, tmpLL); AtomicAdd(BytesOutTot, tmpLL); tmpI4 = AtomicFAZ(SfIntr); AtomicAdd(LinkSfIntr, tmpI4); AtomicEnd(statsMutex); AtomicEnd(wrMutex); // Make sure the protocol updates it's statistics as well // if (Protocol) Protocol->Stats(0, 0, 1); // All done // if (!ctime) LinkInfo.opMutex.UnLock(); } /******************************************************************************/ /* Protected: T L S _ E r r o r */ /******************************************************************************/ int XrdLinkXeq::TLS_Error(const char *act, XrdTls::RC rc) { std::string reason = XrdTls::RC2Text(rc); char msg[512]; snprintf(msg, sizeof(msg), "Unable to %s %s;", act, ID); Log.Emsg("TLS", msg, reason.c_str()); return -1; } /******************************************************************************/ /* T L S _ P e e k */ /******************************************************************************/ int XrdLinkXeq::TLS_Peek(char *Buff, int Blen, int timeout) { XrdSysMutexHelper theMutex; XrdTls::RC retc; int rc, rlen; // Lock the read mutex if we need to, the helper will unlock it upon exit // if (LockReads) theMutex.Lock(&rdMutex); // Wait until we can actually read something // isIdle = 0; if (timeout) {rc = Wait4Data(timeout); if (rc < 1) return rc; } // Do the peek and if sucessful, the number of bytes available. // retc = tlsIO.Peek(Buff, Blen, rlen); if (retc == XrdTls::TLS_AOK) return rlen; // Dianose the TLS error and return failure // return TLS_Error("peek on", retc); } /******************************************************************************/ /* T L S _ R e c v */ /******************************************************************************/ int XrdLinkXeq::TLS_Recv(char *Buff, int Blen) { XrdSysMutexHelper theMutex; XrdTls::RC retc; int rlen; // Lock the read mutex if we need to, the helper will unlock it upon exit // if (LockReads) theMutex.Lock(&rdMutex); // Note that we will read only as much as is queued. Use Recv() with a // timeout to receive as much data as possible. // isIdle = 0; retc = tlsIO.Read(Buff, Blen, rlen); if (retc != XrdTls::TLS_AOK) return TLS_Error("receive from", retc); if (rlen > 0) AtomicAdd(BytesIn, rlen); return rlen; } /******************************************************************************/ int XrdLinkXeq::TLS_Recv(char *Buff, int Blen, int timeout, bool havelock) { XrdSysMutexHelper theMutex; XrdTls::RC retc; int pend, rlen, totlen = 0; // Lock the read mutex if we need to, the helper will unlock it upon exit // if (LockReads && !havelock) theMutex.Lock(&rdMutex); // Wait up to timeout milliseconds for data to arrive // isIdle = 0; while(Blen > 0) {pend = tlsIO.Pending(true); if (!pend) pend = Wait4Data(timeout); if (pend < 1) {if (pend < 0) return -1; tardyCnt++; if (totlen) {if ((++stallCnt & 0xff) == 1) TRACEI(DEBUG,"read timed out"); AtomicAdd(BytesIn, totlen); } return totlen; } // Read as much data as you can. Note that we will force an error // if we get a zero-length read after poll said it was OK. However, // if we never read anything, then we simply return -ENOMSG to avoid // generating a "read link error" as clearly there was a hangup. // retc = tlsIO.Read(Buff, Blen, rlen); if (retc != XrdTls::TLS_AOK) {if (!totlen) return -ENOMSG; AtomicAdd(BytesIn, totlen); return TLS_Error("receive from", retc); } if (rlen <= 0) break; totlen += rlen; Blen -= rlen; Buff += rlen; } AtomicAdd(BytesIn, totlen); return totlen; } /******************************************************************************/ int XrdLinkXeq::TLS_Recv(const struct iovec *iov, int iocnt, int timeout) { XrdSysMutexHelper theMutex; char *Buff; int Blen, rlen, totlen = 0; // Lock the read mutex if we need to, the helper will unlock it upon exit // if (LockReads) theMutex.Lock(&rdMutex); // Individually process each element until we can't read any more // isIdle = 0; for (int i = 0; i < iocnt; i++) {Buff = (char *)iov[i].iov_base; Blen = iov[i].iov_len; rlen = TLS_Recv(Buff, Blen, timeout, true); if (rlen <= 0) break; totlen += rlen; if (rlen < Blen) break; } if (totlen) {AtomicAdd(BytesIn, totlen);} return totlen; } /******************************************************************************/ /* T L S _ R e c v A l l */ /******************************************************************************/ int XrdLinkXeq::TLS_RecvAll(char *Buff, int Blen, int timeout) { int retc; // Check if timeout specified. Notice that the timeout is the max we will // wait for some data. We will wait forever for all the data. Yeah, it's weird. // if (timeout >= 0) {retc = tlsIO.Pending(true); if (!retc) retc = Wait4Data(timeout); if (retc < 1) return (retc ? -1 : -ETIMEDOUT); } // Note that we will block until we receive all the bytes. // return TLS_Recv(Buff, Blen, -1); } /******************************************************************************/ /* T L S _ S e n d */ /******************************************************************************/ int XrdLinkXeq::TLS_Send(const char *Buff, int Blen) { XrdSysMutexHelper lck(wrMutex); ssize_t bytesleft = Blen; XrdTls::RC retc; int byteswritten; // Prepare to send // isIdle = 0; AtomicAdd(BytesOut, Blen); // Do non-blocking writes if we are setup to do so. // if (sendQ) return sendQ->Send(Buff, Blen); // Write the data out // while(bytesleft) {retc = tlsIO.Write(Buff, bytesleft, byteswritten); if (retc != XrdTls::TLS_AOK) return TLS_Error("send to", retc); bytesleft -= byteswritten; Buff += byteswritten; } // All done // return Blen; } /******************************************************************************/ int XrdLinkXeq::TLS_Send(const struct iovec *iov, int iocnt, int bytes) { XrdSysMutexHelper lck(wrMutex); XrdTls::RC retc; int byteswritten; // Get a lock and assume we will be successful (statistically we are). Note // that the calling interface gauranteed bytes are not zero. // isIdle = 0; AtomicAdd(BytesOut, bytes); // Do non-blocking writes if we are setup to do so. // if (sendQ) return sendQ->Send(iov, iocnt, bytes); // Write the data out. // for (int i = 0; i < iocnt; i++) {ssize_t bytesleft = iov[i].iov_len; char *Buff = (char *)iov[i].iov_base; while(bytesleft) {retc = tlsIO.Write(Buff, bytesleft, byteswritten); if (retc != XrdTls::TLS_AOK) return TLS_Error("send to", retc); bytesleft -= byteswritten; Buff += byteswritten; } } // All done // return bytes; } /******************************************************************************/ int XrdLinkXeq::TLS_Send(const sfVec *sfP, int sfN) { XrdSysMutexHelper lck(wrMutex); int bytes, buffsz, fileFD, retc; off_t offset; ssize_t totamt = 0; char myBuff[65536]; // Convert the sendfile to a regular send. The conversion is not particularly // fast and caller are advised to avoid using sendfile on TLS connections. // isIdle = 0; for (int i = 0; i < sfN; sfP++, i++) {if (!(bytes = sfP->sendsz)) continue; totamt += bytes; if (sfP->fdnum < 0) {if (!TLS_Write(sfP->buffer, bytes)) return -1; continue; } offset = sfP->offset; fileFD = sfP->fdnum; buffsz = (bytes < (int)sizeof(myBuff) ? bytes : sizeof(myBuff)); do {do {retc = pread(fileFD, myBuff, buffsz, offset);} while(retc < 0 && errno == EINTR); if (retc < 0) return SFError(errno); if (!retc) break; if (!TLS_Write(myBuff, buffsz)) return -1; offset += buffsz; bytes -= buffsz; totamt += retc; } while(bytes > 0); } // We are done // AtomicAdd(BytesOut, totamt); return totamt; } /******************************************************************************/ /* Protected: T L S _ W r i t e */ /******************************************************************************/ bool XrdLinkXeq::TLS_Write(const char *Buff, int Blen) { XrdTls::RC retc; int byteswritten; // Write the data out // while(Blen) {retc = tlsIO.Write(Buff, Blen, byteswritten); if (retc != XrdTls::TLS_AOK) {TLS_Error("write to", retc); return false; } Blen -= byteswritten; Buff += byteswritten; } // All done // return true; } /******************************************************************************/ /* v e r T L S */ /******************************************************************************/ const char *XrdLinkXeq::verTLS() { return tlsIO.Version(); } xrootd-5.6.9/src/Xrd/XrdLinkXeq.hh000066400000000000000000000156751457266313600170030ustar00rootroot00000000000000#ifndef __XRD_LINKXEQ_H__ #define __XRD_LINKXEQ_H__ /******************************************************************************/ /* */ /* X r d L i n k X e q . h h */ /* */ /* (c) 2018 by the Board of Trustees of the Leland Stanford, Jr., University */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include #include #include "Xrd/XrdLink.hh" #include "Xrd/XrdLinkInfo.hh" #include "Xrd/XrdPollInfo.hh" #include "Xrd/XrdProtocol.hh" #include "XrdNet/XrdNetAddr.hh" #include "XrdTls/XrdTls.hh" #include "XrdTls/XrdTlsSocket.hh" /******************************************************************************/ /* C l a s s D e f i n i t i o n */ /******************************************************************************/ class XrdSendQ; class XrdLinkXeq : protected XrdLink { public: inline XrdNetAddrInfo *AddrInfo() {return (XrdNetAddrInfo *)&Addr;} int Backlog(); int Client(char *buff, int blen); int Close(bool defer=false); void DoIt(); // Override int getIOStats(long long &inbytes, long long &outbytes, int &numstall, int &numtardy) { inbytes = BytesIn + BytesInTot; outbytes = BytesOut+BytesOutTot; numstall = stallCnt + stallCntTot; numtardy = tardyCnt + tardyCntTot; return LinkInfo.InUse; } XrdTlsPeerCerts *getPeerCerts(); static int getName(int &curr, char *bname, int blen, XrdLinkMatch *who=0); inline XrdProtocol *getProtocol() {return Protocol;} inline const char *Name() const {return (const char *)Lname;} inline const XrdNetAddr *NetAddr() const {return &Addr;} int Peek(char *buff, int blen, int timeout=-1); int Recv(char *buff, int blen); int Recv(char *buff, int blen, int timeout); int Recv(const struct iovec *iov, int iocnt, int timeout); int RecvAll(char *buff, int blen, int timeout=-1); bool Register(const char *hName); int Send(const char *buff, int blen); int Send(const struct iovec *iov, int iocnt, int bytes=0); int Send(const sfVec *sdP, int sdn); // Iff sfOK > 0 void setID(const char *userid, int procid); void setLocation(XrdNetAddrInfo::LocInfo &loc) {Addr.SetLocation(loc);} bool setNB(); XrdProtocol *setProtocol(XrdProtocol *pp, bool push); void setProtName(const char *name); bool setTLS(bool enable, XrdTlsContext *ctx=0); void Shutdown(bool getLock); static int Stats(char *buff, int blen, bool do_sync=false); void syncStats(int *ctime=0); int TLS_Peek(char *Buff, int Blen, int timeout); int TLS_Recv(char *Buff, int Blen); int TLS_Recv(char *Buff, int Blen, int timeout, bool havelock=false); int TLS_Recv(const struct iovec *iov, int iocnt, int timeout); int TLS_RecvAll(char *Buff, int Blen, int timeout); int TLS_Send(const char *Buff, int Blen); int TLS_Send(const struct iovec *iov, int iocnt, int bytes); int TLS_Send(const sfVec *sfP, int sfN); const char *verTLS(); XrdLinkXeq(); ~XrdLinkXeq() {} // Is never deleted! XrdLinkInfo LinkInfo; XrdPollInfo PollInfo; protected: int RecvIOV(const struct iovec *iov, int iocnt); void Reset(); int sendData(const char *Buff, int Blen); int SendIOV(const struct iovec *iov, int iocnt, int bytes); int SFError(int rc); int TLS_Error(const char *act, XrdTls::RC rc); bool TLS_Write(const char *Buff, int Blen); static const char *TraceID; // Statistical area (global and local) // static long long LinkBytesIn; static long long LinkBytesOut; static long long LinkConTime; static long long LinkCountTot; static int LinkCount; static int LinkCountMax; static int LinkTimeOuts; static int LinkStalls; static int LinkSfIntr; long long BytesIn; long long BytesInTot; long long BytesOut; long long BytesOutTot; int stallCnt; int stallCntTot; int tardyCnt; int tardyCntTot; int SfIntr; static XrdSysMutex statsMutex; // Protocol section // XrdProtocol *Protocol; // -> Protocol tied to the link XrdProtocol *ProtoAlt; // -> Alternate/stacked protocol // TLS section // XrdTlsSocket tlsIO; // Identification section // XrdNetAddr Addr; XrdSysMutex rdMutex; XrdSysMutex wrMutex; XrdSendQ *sendQ; // Protected by wrMutex && opMutex int HNlen; bool LockReads; bool KeepFD; char isIdle; char Uname[24]; // Uname and Lname must be adjacent! char Lname[256]; }; #endif xrootd-5.6.9/src/Xrd/XrdMain.cc000066400000000000000000000217251457266313600162730ustar00rootroot00000000000000/**************************************************************************************/ /* */ /* X r d M a i n . c c */ /* */ /* (c) 2004 by the Board of Trustees of the Leland Stanford, Jr., University */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ /* This is the XRootd server. The syntax is: xrootd [options] options: [-b] [-c ] [-d] [-h] [-l ] [-p ] [] Where: -b forces background execution. -c specifies the configuration file. This may also come from the XrdCONFIGFN environmental variable. -d Turns on debugging mode (equivalent to xrd.trace all) -h Displays usage line and exits. -l Specifies location of the log file. This may also come from the XrdOucLOGFILE environmental variable or from the oofs layer. By By default, error messages go to standard error. -p Is the port to use either as a service name or an actual port number. The default port is 1094. Are other protocol specific options. */ /******************************************************************************/ /* i n c l u d e f i l e s */ /******************************************************************************/ #include #include #include #include #include #include #include #include #include #include #include "Xrd/XrdConfig.hh" #include "Xrd/XrdInet.hh" #include "Xrd/XrdLink.hh" #include "Xrd/XrdProtLoad.hh" #include "Xrd/XrdScheduler.hh" #include "XrdSys/XrdSysError.hh" #include "XrdSys/XrdSysHeaders.hh" #include "XrdSys/XrdSysPthread.hh" #include "XrdSys/XrdSysUtils.hh" /******************************************************************************/ /* L o c a l C l a s s e s */ /******************************************************************************/ class XrdMain : XrdJob { public: XrdSysSemaphore *theSem; XrdProtocol *theProt; XrdInet *theNet; int thePort; static XrdConfig Config; void DoIt() {XrdLink *newlink; if ((newlink = theNet->Accept(0, -1, theSem))) newlink->setProtocol(theProt, true); } XrdMain() : XrdJob("main accept"), theSem(0), theProt(0), theNet(0), thePort(0) {} XrdMain(XrdInet *nP) : XrdJob("main accept"), theSem(0), theProt(0), theNet(nP), thePort(nP->Port()) {} ~XrdMain() {} }; XrdConfig XrdMain::Config; /******************************************************************************/ /* E x t e r n a l T h r e a d I n t e r f a c e s */ /******************************************************************************/ void *mainAccept(void *parg) { XrdMain *Parms = (XrdMain *)parg; XrdScheduler *mySched = Parms->Config.ProtInfo.Sched; XrdProtLoad ProtSelect(Parms->thePort); XrdSysSemaphore accepted(0); // Complete the parms // Parms->theSem = &accepted; Parms->theProt = (XrdProtocol *)&ProtSelect; // Simply schedule new accepts // while(1) {mySched->Schedule((XrdJob *)Parms); accepted.Wait(); } return (void *)0; } /******************************************************************************/ /* m a i n A d m i n */ /******************************************************************************/ void *mainAdmin(void *parg) { XrdMain *Parms = (XrdMain *)parg; XrdInet *NetADM = Parms->theNet; XrdLink *newlink; // static XrdProtocol_Admin ProtAdmin; int ProtAdmin; // At this point we should be able to accept new connections. Noe that we don't // support admin connections as of yet so the following code is superflous. // while(1) if ((newlink = NetADM->Accept())) {newlink->setProtocol((XrdProtocol *)&ProtAdmin); newlink->setProtName("xrdadmin"); Parms->Config.ProtInfo.Sched->Schedule((XrdJob *)newlink); } return (void *)0; } /******************************************************************************/ /* m a i n */ /******************************************************************************/ int main(int argc, char *argv[]) { XrdMain Main; pthread_t tid; char buff[128]; int i, retc; // Set TZ environment variable to read the system timezone from /etc/localtime // if it is not already set, to avoid race conditions between localtime_r() and // mktime() during the multi-threaded phase of the initialization. if (access("/etc/localtime", R_OK) == 0) setenv("TZ", ":/etc/localtime", /* overwrite */ false); // Call tzset() early to ensure thread-safety of localtime_r() and mktime(). tzset(); // Turn off sigpipe and host a variety of others before we start any threads // XrdSysUtils::SigBlock(); // Set the default stack size here. Note that on modern Linux the default // stack size is set at about 8MB. We force a lower limit to not have a huge // virtual address space mostly for core file debugging purposes. // if (sizeof(long) > 4) XrdSysThread::setStackSize((size_t)2097152, true); else XrdSysThread::setStackSize((size_t)1048576, true); // Process configuration file // if (Main.Config.Configure(argc, argv)) _exit(1); // Start the admin thread if an admin network is defined // if (Main.Config.NetADM && (retc = XrdSysThread::Run(&tid, mainAdmin, (void *)new XrdMain(Main.Config.NetADM), XRDSYSTHREAD_BIND, "Admin handler"))) {Main.Config.ProtInfo.eDest->Emsg("main", retc, "create admin thread"); _exit(3); } // At this point we should be able to accept new connections. Spawn a // thread for each network except the first. The main thread will handle // that network as some implementations require a main active thread. // for (i = 1; i < (int)Main.Config.NetTCP.size(); i++) if (Main.Config.NetTCP[i]) {XrdMain *Parms = new XrdMain(Main.Config.NetTCP[i]); sprintf(buff, "Port %d handler", Parms->thePort); //??? if (Parms->theNet == Main.Config.NetTCP[XrdProtLoad::ProtoMax]) // Parms->thePort = -(Parms->thePort); if ((retc = XrdSysThread::Run(&tid, mainAccept, (void *)Parms, XRDSYSTHREAD_BIND, strdup(buff)))) {Main.Config.ProtInfo.eDest->Emsg("main", retc, "create", buff); _exit(3); } } // Finally, start accepting connections on the main port // Main.theNet = Main.Config.NetTCP[0]; Main.thePort = Main.Config.NetTCP[0]->Port(); mainAccept((void *)&Main); // We should never get here // pthread_exit(0); } xrootd-5.6.9/src/Xrd/XrdObject.hh000066400000000000000000000125201457266313600166200ustar00rootroot00000000000000#ifndef __XRD_OBJECT_H__ #define __XRD_OBJECT_H__ /******************************************************************************/ /* */ /* X r d O b j e c t . h h */ /* */ /*(c) 2004 by the Board of Trustees of the Leland Stanford, Jr., University */ /*Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Deprtment of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include #include #include #include "Xrd/XrdJob.hh" // The classes here are templates for singly linked list handling that allows // elements to be added to either end but be removed only from the front. Most // objects in this package are managed in queues of this type. /******************************************************************************/ /* x r d _ O b j e c t */ /******************************************************************************/ template class XrdObjectQ; template class XrdObject { public: friend class XrdObjectQ; // Item() supplies the item value associated with itself (used with Next()). // T *objectItem() {return Item;} // Next() supplies the next list node. // XrdObject *nextObject() {return Next;} // Set the item pointer // void setItem(T *ival) {Item = ival;} XrdObject(T *ival=0) {Next = 0; Item = ival; QTime = 0;} ~XrdObject() {} private: XrdObject *Next; T *Item; time_t QTime; // Only used for time-managed objects }; /******************************************************************************/ /* x r d _ O b j e c t Q */ /******************************************************************************/ // Note to properly cleanup this type of queue you must call Set() at least // once to cause the time element to be sceduled. class XrdSysTrace; class XrdScheduler; template class XrdObjectQ : public XrdJob { public: inline T *Pop() {XrdObject *Node; QMutex.Lock(); if ((Node = First)) {First = First->Next; Count--;} QMutex.UnLock(); if (Node) return Node->Item; return (T *)0; } inline void Push(XrdObject *Node) {Node->QTime = Curage; QMutex.Lock(); if (Count >= MaxinQ) delete Node->Item; else {Node->Next = First; First = Node; Count++; } QMutex.UnLock(); } void Set(int inQMax, time_t agemax=1800); void Set(XrdScheduler *sp, XrdSysTrace *tp, int TraceChk=0) {Sched = sp; Trace = tp; TraceON = TraceChk;} void DoIt(); XrdObjectQ(const char *id, const char *desc) : XrdJob(desc) {Curage = Count = 0; Maxage = 0; TraceID = id; MaxinQ = 32; MininQ = 16; First = 0; } ~XrdObjectQ() {} private: XrdSysMutex QMutex; XrdObject *First; int Count; int Curage; int MininQ; int MaxinQ; time_t Maxage; XrdScheduler *Sched; XrdSysTrace *Trace; int TraceON; const char *TraceID; }; #include "Xrd/XrdObject.icc" #endif xrootd-5.6.9/src/Xrd/XrdObject.icc000066400000000000000000000104671457266313600167670ustar00rootroot00000000000000/******************************************************************************/ /* */ /* X r d O b j e c t . i c c */ /* */ /* (c) 2004 by the Board of Trustees of the Leland Stanford, Jr., University */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include "Xrd/XrdScheduler.hh" #include "XrdSys/XrdSysTrace.hh" /******************************************************************************/ /* D o I t */ /******************************************************************************/ template void XrdObjectQ::DoIt() { XrdObject *pp, *p; int oldcnt, agemax; // Lock the anchor and see if we met the threshold for deletion // QMutex.Lock(); agemax = Maxage; if ((oldcnt = Count) > MininQ) { // Prepare to scan down the queue. // if ((pp = First)) p = pp->Next; else p = 0; // Find the first object that's been idle for too long // while(p && (p->QTime >= Curage)) {pp = p; p = p->Next;} // Now delete half of the idle objects. The object queue element must be // part of the actual object being deleted for this to properly work. // if (pp) while(p) {pp->Next = p->Next; delete p->Item; Count--; p = ((pp = pp->Next) ? pp->Next : 0); } } // Increase the age and unlock the queue // Curage++; QMutex.UnLock(); // Trace as needed // if (TraceON && Trace->Tracing(TraceON)) {SYSTRACE(Trace->, 0, TraceID, 0, Comment <<" trim done; " < 0) Sched->Schedule((XrdJob *)this, agemax+time(0)); } /******************************************************************************/ /* S e t */ /******************************************************************************/ template void XrdObjectQ::Set(int inQMax, time_t agemax) { // Lock the data area and set the values // QMutex.Lock(); MaxinQ = inQMax; Maxage = agemax; if (!(MininQ = inQMax/2)) MininQ = 1; QMutex.UnLock(); // Schedule ourselves using the new values // if (agemax > 0) Sched->Schedule((XrdJob *)this, agemax+time(0)); } xrootd-5.6.9/src/Xrd/XrdPoll.cc000066400000000000000000000306121457266313600163100ustar00rootroot00000000000000/******************************************************************************/ /* */ /* X r d P o l l . c c */ /* */ /* (c) 2004 by the Board of Trustees of the Leland Stanford, Jr., University */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include #include #include "XrdSys/XrdSysError.hh" #include "XrdSys/XrdSysFD.hh" #include "XrdSys/XrdSysPlatform.hh" #include "XrdSys/XrdSysPthread.hh" #include "Xrd/XrdLink.hh" #include "Xrd/XrdProtocol.hh" #define TRACE_IDENT pInfo.Link.ID #include "Xrd/XrdTrace.hh" #if defined( __linux__ ) #include "Xrd/XrdPollE.hh" //#include "Xrd/XrdPollPoll.hh" #else #include "Xrd/XrdPollPoll.hh" #endif #include "Xrd/XrdPollInfo.hh" /******************************************************************************/ /* L o c a l C l a s s e s */ /******************************************************************************/ class XrdPoll_End : public XrdProtocol { public: void DoIt() {} XrdProtocol *Match(XrdLink *lp) {return (XrdProtocol *)0;} int Process(XrdLink *lp) {return -1;} void Recycle(XrdLink *lp, int x, const char *y) {} int Stats(char *buff, int blen, int do_sync=0) {return 0;} XrdPoll_End() : XrdProtocol("link termination") {} ~XrdPoll_End() {} }; /******************************************************************************/ /* G l o b a l D a t a */ /******************************************************************************/ XrdPoll *XrdPoll::Pollers[XRD_NUMPOLLERS] = {0, 0, 0}; XrdSysMutex XrdPoll::doingAttach; const char *XrdPoll::TraceID = "Poll"; namespace XrdGlobal { extern XrdSysError Log; extern XrdScheduler Sched; } using namespace XrdGlobal; /******************************************************************************/ /* T h r e a d S t a r t u p I n t e r f a c e */ /******************************************************************************/ struct XrdPollArg {XrdPoll *Poller; int retcode; XrdSysSemaphore PollSync; XrdPollArg() : PollSync(0, "poll sync") {} ~XrdPollArg() {} }; void *XrdStartPolling(void *parg) { struct XrdPollArg *PArg = (struct XrdPollArg *)parg; PArg->Poller->Start(&(PArg->PollSync), PArg->retcode); return (void *)0; } /******************************************************************************/ /* C o n s t r u c t o r */ /******************************************************************************/ XrdPoll::XrdPoll() { int fildes[2]; TID=0; numAttached=numEnabled=numEvents=numInterrupts=0; if (XrdSysFD_Pipe(fildes) == 0) {CmdFD = fildes[1]; ReqFD = fildes[0]; } else { CmdFD = ReqFD = -1; Log.Emsg("Poll", errno, "create poll pipe"); } PipeBuff = 0; PipeBlen = 0; PipePoll.fd = ReqFD; PipePoll.events = POLLIN | POLLRDNORM; } /******************************************************************************/ /* A t t a c h */ /******************************************************************************/ int XrdPoll__Attach(XrdLink *lp) {return lp->Activate();} int XrdPoll::Attach(XrdPollInfo &pInfo) { int i; XrdPoll *pp; // We allow only one attach at a time to simplify the processing // doingAttach.Lock(); // Find a poller with the smallest number of entries // pp = Pollers[0]; for (i = 1; i < XRD_NUMPOLLERS; i++) if (pp->numAttached > Pollers[i]->numAttached) pp = Pollers[i]; // Include this FD into the poll set of the poller // if (!pp->Include(pInfo)) {doingAttach.UnLock(); return 0;} // Complete the link setup // pInfo.Poller = pp; pp->numAttached++; doingAttach.UnLock(); TRACEI(POLL, "FD " <PID <<"; num=" <numAttached); return 1; } /******************************************************************************/ /* D e t a c h */ /******************************************************************************/ void XrdPoll::Detach(XrdPollInfo &pInfo) { XrdPoll *pp; // If link is not attached, simply return // if (!(pp = pInfo.Poller)) return; // Exclude this link from the associated poll set // pp->Exclude(pInfo); // Make sure we are consistent // doingAttach.Lock(); if (!pp->numAttached) {Log.Emsg("Poll","Underflow detaching", pInfo.Link.ID); abort();} pp->numAttached--; doingAttach.UnLock(); TRACEI(POLL, "FD " <PID <<"; num=" <numAttached); } /******************************************************************************/ /* F i n i s h */ /******************************************************************************/ int XrdPoll::Finish(XrdPollInfo &pInfo, const char *etxt) { static XrdPoll_End LinkEnd; // If this link is already scheduled for termination, ignore this call. // if (pInfo.Link.getProtocol() == &LinkEnd) {TRACEI(POLL, "Link " <PID = i; // Now start a thread to handle this poller object // PArg.Poller = Pollers[i]; PArg.retcode= 0; TRACE(POLL, "Starting poller " <TID = tid; PArg.PollSync.Wait(); if (PArg.retcode) {Log.Emsg("Poll", PArg.retcode, "start poller"); return 0; } } // All done // return 1; } /******************************************************************************/ /* S t a t s */ /******************************************************************************/ int XrdPoll::Stats(char *buff, int blen, int do_sync) { static const char statfmt[] = "%d" "%d%d%d"; int i, numatt = 0, numen = 0, numev = 0, numint = 0; XrdPoll *pp; // Return number of bytes if so wanted // if (!buff) return (sizeof(statfmt)+(4*16))*XRD_NUMPOLLERS; // Get statistics. While we wish we could honor do_sync, doing so would be // costly and hardly worth it. So, we do not include code such as: // x = pp->y; if (do_sync) while(x != pp->y) x = pp->y; tot += x; // for (i = 0; i < XRD_NUMPOLLERS; i++) {pp = Pollers[i]; numatt += pp->numAttached; numen += pp->numEnabled; numev += pp->numEvents; numint += pp->numInterrupts; } // Format and return // return snprintf(buff, blen, statfmt, numatt, numen, numev, numint); } /******************************************************************************/ /* I m p l e m e n t a t i o n S p e c i f i c s */ /******************************************************************************/ #if defined( __linux__ ) #include "Xrd/XrdPollE.icc" //#include "Xrd/XrdPollPoll.icc" #else #include "Xrd/XrdPollPoll.icc" #endif xrootd-5.6.9/src/Xrd/XrdPoll.hh000066400000000000000000000130141457266313600163170ustar00rootroot00000000000000#ifndef __XRD_POLL_H__ #define __XRD_POLL_H__ /******************************************************************************/ /* */ /* X r d P o l l . h h */ /* */ /* (c) 2004 by the Board of Trustees of the Leland Stanford, Jr., University */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include "XrdSys/XrdSysPthread.hh" #define XRD_NUMPOLLERS 3 class XrdPollInfo; class XrdSysSemaphore; class XrdPoll { public: // Attach() is called when a new link needs to be assigned to a poller // static int Attach(XrdPollInfo &pInfo); // Detach() is called when a link is being discarded // static void Detach(XrdPollInfo &pInfo); // Disable() is called when we need to mask interrupts from a link // virtual void Disable(XrdPollInfo &pInfo, const char *etxt=0) = 0; // Enable() is called when we want to receive interrupts from a link // virtual int Enable(XrdPollInfo &pInfo) = 0; // Finish() is called to allow a link to gracefully terminate when scheduled // static int Finish(XrdPollInfo &pInfo, const char *etxt=0); //Implementation supplied // Poll2Text() converts bits in an revents item to text // static char *Poll2Text(short events); // Implementation supplied // Setup() is called at config time to perform poller configuration // static int Setup(int numfd); // Implementation supplied // Start() is called via a thread for each poller that was created // virtual void Start(XrdSysSemaphore *syncp, int &rc) = 0; // Stats() is called to provide statistics on polling // static int Stats(char *buff, int blen, int do_sync=0); // Identification of the thread handling this object // int PID; // Poller ID pthread_t TID; // Thread ID // The following table reference the pollers in effect // static XrdPoll *Pollers[XRD_NUMPOLLERS]; XrdPoll(); virtual ~XrdPoll() {} protected: static const char *TraceID; // For tracing // Gets the next request on the poll pipe. This is common to all implentations. // int getRequest(); // Implementation supplied // Exclude() called to exclude a link from a poll set // virtual void Exclude(XrdPollInfo &pInfo) = 0; // Include() called to include a link in a poll set // virtual int Include(XrdPollInfo &pInfo) = 0; // newPoller() called to get a new poll object at initialization time // Even though static, an implementation must be supplied. // static XrdPoll *newPoller(int pollid, int numfd) /* = 0 */; // The following is common to all implementations // XrdSysMutex PollPipe; struct pollfd PipePoll; int CmdFD; // FD to send PipeData commands int ReqFD; // FD to recv PipeData requests struct PipeData {union {XrdSysSemaphore *theSem; struct {int fd; int ent;} Arg; } Parms; enum cmd {EnFD, DiFD, RmFD, Post}; cmd req; }; PipeData ReqBuff; char *PipeBuff; int PipeBlen; // The following are statistical counters each implementation must maintain // int numEnabled; // Count of Enable() calls int numEvents; // Count of poll fd's dispatched int numInterrupts; // Number of interrupts (e.g., signals) private: static XrdSysMutex doingAttach; int numAttached; // Number of fd's attached to poller }; #endif xrootd-5.6.9/src/Xrd/XrdPollE.hh000066400000000000000000000070211457266313600164250ustar00rootroot00000000000000#ifndef __XRD_POLLDEV_H__ #define __XRD_POLLDEV_H__ /******************************************************************************/ /* */ /* X r d P o l l D e v . h h */ /* */ /* (c) 2004 by the Board of Trustees of the Leland Stanford, Jr., University */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include "Xrd/XrdPoll.hh" #ifndef EPOLLRDHUP #define EPOLLRDHUP 0 #endif class XrdPollE : public XrdPoll { public: void Disable(XrdPollInfo &pInfo, const char *etxt=0); int Enable(XrdPollInfo &pInfo); void Start(XrdSysSemaphore *syncp, int &rc); XrdPollE(struct epoll_event *ptab, int numfd, int pfd, int wfd) : WaitFdSem(0) {PollTab = ptab; PollMax = numfd; PollDfd = pfd; WaitFd = wfd; } ~XrdPollE(); protected: void Exclude(XrdPollInfo &pInfo); int Include(XrdPollInfo &pInfo); const char *x2Text(unsigned int evf, char *buff); private: int AddWaitFd(); void HandleWaitFd(const unsigned int events); void remFD(XrdPollInfo &pInfo, unsigned int events); void Wait4Poller(); #ifdef EPOLLONESHOT static const int ePollOneShot = EPOLLONESHOT; #else static const int ePollOneShot = 0; #endif static const int ePollEvents = EPOLLIN | EPOLLHUP | EPOLLPRI | EPOLLERR | EPOLLRDHUP | ePollOneShot; struct epoll_event *PollTab; int PollDfd; int PollMax; int WaitFd; XrdSysSemaphore WaitFdSem; }; #endif xrootd-5.6.9/src/Xrd/XrdPollE.icc000066400000000000000000000341711457266313600165720ustar00rootroot00000000000000/******************************************************************************/ /* */ /* X r d P o l l E . i c c */ /* */ /* (c) 2008 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include #include #include #include #include #include "Xrd/XrdPollE.hh" #include "Xrd/XrdScheduler.hh" /******************************************************************************/ /* n e w P o l l e r */ /******************************************************************************/ XrdPoll *XrdPoll::newPoller(int pollid, int maxfd) { int pfd, wfd, bytes, alignment, pagsz = getpagesize(); struct epoll_event *pp; // Open the /dev/poll driver // #ifndef EPOLL_CLOEXEC if ((pfd = epoll_create(maxfd)) >= 0) fcntl(pfd, F_SETFD, FD_CLOEXEC); else #else if ((pfd = epoll_create1(EPOLL_CLOEXEC)) < 0) #endif {Log.Emsg("Poll", errno, "create epoll device"); return 0;} if ((wfd = eventfd(0, EFD_CLOEXEC)) < 0) {Log.Emsg("Poll", errno, "create an eventfd as the wait-poller descriptor"); close(pfd); return 0; } // Calculate the size of the poll table and allocate it // bytes = maxfd * sizeof(struct epoll_event); alignment = (bytes < pagsz ? 1024 : pagsz); if (posix_memalign((void **)&pp, alignment, bytes)) {Log.Emsg("Poll", ENOMEM, "create poll table"); close(wfd); close(pfd); return 0; } // Create new poll object // memset((void *)pp, 0, bytes); return (XrdPoll *)new XrdPollE(pp, maxfd, pfd, wfd); } /******************************************************************************/ /* D e s t r u c t o r */ /******************************************************************************/ XrdPollE::~XrdPollE() { if (PollTab) free(PollTab); if (WaitFd >= 0) close(WaitFd); if (PollDfd >= 0) close(PollDfd); } /******************************************************************************/ /* A d d W a i t F d */ /******************************************************************************/ int XrdPollE::AddWaitFd() { const unsigned int myPollEvts = EPOLLIN; struct epoll_event myEvent = {myPollEvts, {(void *)&WaitFd}}; // Add the waitfd to the poll set // if (epoll_ctl(PollDfd, EPOLL_CTL_ADD, WaitFd, &myEvent) < 0) {int rc = errno; Log.Emsg("Poll", rc, "include the wait FD in the poll set"); return rc; } return 0; } /******************************************************************************/ /* D i s a b l e */ /******************************************************************************/ void XrdPollE::Disable(XrdPollInfo &pInfo, const char *etxt) { // Simply return if the link is already disabled // if (!pInfo.isEnabled) return; // If Linux 2.6.9 we use EPOLLONESHOT to automatically disable a polled fd. // So, the Disable() method need not do anything. Prior kernels did not have // this mechanism so we need to do this manually. // #ifndef EPOLLONESHOT struct epoll_event myEvents = {0, (void *)&pInfo}; // Disable this fd. Unlike solaris, epoll_ctl() does not block when the pollfd // is being waited upon by another thread. // if (epoll_ctl(PollDfd, EPOLL_CTL_MOD, pInfo.FD, &myEvents)) {Log.Emsg("Poll", errno, "disable link", lp->ID); return;} #endif // Trace this event // pInfo.isEnabled = false; TRACEI(POLL, "Poller " < 0) {TRACEI(POLL, "Poller " <Post(); return; } // Indicate to the starting thread that all went well // retcode = 0; syncsem->Post(); // Now start dispatching links that are ready // do {do {numpolled = epoll_wait(PollDfd, PollTab, PollMax, -1);} while (numpolled < 0 && errno == EINTR); if (numpolled == 0) continue; if (numpolled < 0) {Log.Emsg("Poll", errno, "poll for events"); abort(); } numEvents += numpolled; // Checkout which links must be dispatched (no need to lock) // jfirst = jlast = 0; num2sched = 0; haveWaiters = false; waitFdEvents = 0; for (i = 0; i < numpolled; i++) {if (PollTab[i].data.ptr == &WaitFd) {haveWaiters = true; waitFdEvents = PollTab[i].events;} else if ((pInfo = (XrdPollInfo *)PollTab[i].data.ptr)) {if (!(pInfo->isEnabled) && pInfo->FD >= 0) remFD(*pInfo, PollTab[i].events); else {pInfo->isEnabled = 0; if (!(PollTab[i].events & pollOK) || (PollTab[i].events & POLLRDHUP)) Finish(*pInfo, x2Text(PollTab[i].events, eBuff)); lp = &(pInfo->Link); lp->NextJob = jfirst; jfirst = (XrdJob *)lp; if (!jlast) jlast=(XrdJob *)lp; num2sched++; #ifndef EPOLLONESHOT PollTab[i].events = 0; if (epoll_ctl(PollDfd,EPOLL_CTL_MOD,pInfo.FD,&PollTab[i])) Log.Emsg("Poll",errno,"disable link",pInfo.Link.ID); #endif } } else Log.Emsg("Poll", "null link event!!!!"); } // Schedule the polled links // if (num2sched == 1) Sched.Schedule(jfirst); else if (num2sched) Sched.Schedule(num2sched, jfirst, jlast); if (haveWaiters) HandleWaitFd(waitFdEvents); } while(1); } /******************************************************************************/ /* W a i t 4 P o l l e r */ /******************************************************************************/ void XrdPollE::Wait4Poller() { // Makes the caller wait for the polling thread to complete an event processing // loop. // if (eventfd_write(WaitFd, 1) < 0) {Log.Emsg("Poll", errno, "write to the wait-poller descriptor"); return; } WaitFdSem.Wait(); } /******************************************************************************/ /* x 2 T e x t */ /******************************************************************************/ const char *XrdPollE::x2Text(unsigned int events, char *buff) { if (events & EPOLLERR) return "socket error"; if (events & (EPOLLHUP | EPOLLRDHUP)) return "hangup"; sprintf(buff, "unusual event (%.4x)", events); return buff; } xrootd-5.6.9/src/Xrd/XrdPollInfo.hh000066400000000000000000000063541457266313600171440ustar00rootroot00000000000000#ifndef __XRD_POLLINFO_H__ #define __XRD_POLLINFO_H__ /******************************************************************************/ /* */ /* X r d P o l l I n f o . h h */ /* */ /* (c) 2018 by the Board of Trustees of the Leland Stanford, Jr., University */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ class XrdLink; class XrdPoll; struct pollfd; class XrdPollInfo { public: XrdPollInfo *Next; // Chain of links waiting for a PollPoll event XrdLink &Link; // Link associated with this object (always the same) struct pollfd *PollEnt; // Used only by PollPoll XrdPoll *Poller; // -> Poller object associated with this object int FD; // Associated target file descriptor number bool inQ; // True -> in a PollPoll event queue bool isEnabled; // True -> interrupts are enabled char rsv[2]; // Reserved for future flags void Zorch() {Next = 0; PollEnt = 0; Poller = 0; FD = -1; isEnabled = false; inQ = false; rsv[0] = 0; rsv[1] = 0; } XrdPollInfo(XrdLink &lnk) : Link(lnk) {Zorch();} ~XrdPollInfo() {} }; #endif xrootd-5.6.9/src/Xrd/XrdPollPoll.hh000066400000000000000000000062071457266313600171540ustar00rootroot00000000000000#ifndef __XRD_POLLPOLL_H__ #define __XRD_POLLPOLL_H__ /******************************************************************************/ /* */ /* X r d P o l l P o l l . h h */ /* */ /* (c) 2004 by the Board of Trustees of the Leland Stanford, Jr., University */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include "Xrd/XrdPoll.hh" class XrdPollInfo; class XrdPollPoll : XrdPoll { public: void Disable(XrdPollInfo &pInfo, const char *etxt=0); int Enable(XrdPollInfo &pInfo); void Start(XrdSysSemaphore *syncp, int &rc); XrdPollPoll(struct pollfd *pp, int numfd); ~XrdPollPoll(); protected: void doDetach(int pti); void Exclude(XrdPollInfo &pInfo); int Include(XrdPollInfo &pInfo); private: void doRequests(int maxreq); void dqLink(XrdPollInfo *pInfo); void LogEvent(int req, int pollfd, int cmdfd); void Recover(int numleft); void Restart(int ecode); struct pollfd *PollTab; //<--- int PollTNum; // PollMutex protects these elements XrdPollInfo *PollQ; //<--- XrdSysMutex PollMutex; int maxent; }; #endif xrootd-5.6.9/src/Xrd/XrdPollPoll.icc000066400000000000000000000427071457266313600173200ustar00rootroot00000000000000/******************************************************************************/ /* */ /* X r d P o l l P o l l . i c c */ /* */ /* (c) 2004 by the Board of Trustees of the Leland Stanford, Jr., University */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include #include "Xrd/XrdLinkCtl.hh" #include "Xrd/XrdPollPoll.hh" #include "Xrd/XrdScheduler.hh" /******************************************************************************/ /* n e w P o l l e r */ /******************************************************************************/ XrdPoll *XrdPoll::newPoller(int pollid, int maxfd) { int bytes, alignment, pagsz = getpagesize(); struct pollfd *pp; // Calculate the size of the poll table and allocate it // bytes = maxfd * sizeof(struct pollfd); alignment = (bytes < pagsz ? 1024 : pagsz); if (posix_memalign((void **)&pp, alignment, bytes)) {Log.Emsg("Poll", ENOMEM, "create poll table"); return 0; } // Create new poll object // memset((void *)pp, 0, bytes); return (XrdPoll *)new XrdPollPoll(pp, maxfd); } /******************************************************************************/ /* C o n s t r c u t o r */ /******************************************************************************/ XrdPollPoll::XrdPollPoll(struct pollfd *pp, int numfd) { // Initialize the standard stuff // PollTab = pp; PollTNum= 0; PollQ = 0; maxent = numfd; } /******************************************************************************/ /* D e s t r u c t o r */ /******************************************************************************/ XrdPollPoll::~XrdPollPoll() { if (PollTab) free(PollTab); } /******************************************************************************/ /* I n c l u d e */ /******************************************************************************/ int XrdPollPoll::Include(XrdPollInfo &pInfo) { struct pollfd *pfd; int ptnum; // Lock down the poll data structure // PollMutex.Lock(); // Get the next entry to be used // ptnum = 0; while((ptnum < PollTNum) && (PollTab[ptnum].fd != -1)) ptnum++; // Make sure we have enough table entries to add this link // if (ptnum > maxent) {Log.Emsg("Attach","Attach",pInfo.Link.ID,"failed; poll table overflow."); PollMutex.UnLock(); return 0; } // Initialize the polltable entry // pfd = &(PollTab[ptnum]); pfd->fd = -pInfo.FD; pfd->events = POLLIN | POLLRDNORM; pfd->revents = 0; // Record relevant information in the link // pInfo.PollEnt = pfd; if (ptnum == PollTNum) PollTNum++; // All done // PollMutex.UnLock(); return 1; } /******************************************************************************/ /* D i s a b l e */ /******************************************************************************/ void XrdPollPoll::Disable(XrdPollInfo &pInfo, const char *etxt) { XrdSysSemaphore mySem(0); PipeData cmdbuff[2]; int myerrno = 0; // Check if this link is in the pollQ. If so, remove it. // if (pInfo.inQ) dqLink(&pInfo); // Simply return if the link is already disabled // if (!pInfo.isEnabled) return; // Trace this event // TRACEI(POLL, "Poller " <Post(); // Now do the main poll loop // do {do {numpolled = poll(PollTab, PollTNum, -1);} while(numpolled < 0 && (errno == EAGAIN || errno == EINTR)); // Check if we had a polling error // if (numpolled < 0) {if (errno != EINTR) Restart(errno); else numInterrupts++; continue; } numEvents += numpolled; // Check out base poll table entry, we can do this without a lock // if (PollTab[0].revents & pollOK) {doRequests(numpolled); if (--numpolled <= 0) continue; } // Checkout which links must be dispatched (do this locked) // PollMutex.Lock(); plp = 0; nlp = PollQ; jfirst = jlast = 0; num2sched = 0; while ((pInfo = nlp) && numpolled > 0) {if ((pollevents = pInfo->PollEnt->revents)) {pInfo->PollEnt->fd = -pInfo->PollEnt->fd; if (plp) nlp = plp->Next = pInfo->Next; else nlp = PollQ = pInfo->Next; numpolled--; pInfo->inQ = false; if (!(pollevents & pollOK)) Finish(*pInfo, Poll2Text(pollevents)); lp = &(pInfo->Link); if (!(pInfo->isEnabled)) Log.Emsg("Poll", "Disabled event occurred for", lp->ID); else {pInfo->isEnabled = false; lp->NextJob = jfirst; jfirst = (XrdJob *)lp; if (!jlast) jlast=(XrdJob *)lp; num2sched++; continue; } } plp = pInfo; nlp = pInfo->Next; } if (numpolled) Recover(numpolled); PollMutex.UnLock(); // Schedule the polled links // if (num2sched == 1) Sched.Schedule(jfirst); else if (num2sched) Sched.Schedule(num2sched, jfirst, jlast); } while(1); } /******************************************************************************/ /* P r i v a t e M e t h o d s */ /******************************************************************************/ /******************************************************************************/ /* d o D e t a c h */ /******************************************************************************/ void XrdPollPoll::doDetach(int pti) { int lastent; // Get some starting values // PollMutex.Lock(); if ((lastent = PollTNum-1) < 0) {Log.Emsg("Poll","Underflow during detach"); abort();} if (pti == lastent) do {PollTNum--;} while(PollTNum && PollTab[PollTNum-1].fd == -1); PollMutex.UnLock(); } /******************************************************************************/ /* d o R e q u e s t s */ /******************************************************************************/ void XrdPollPoll::doRequests(int maxreq) { const char *act; int pti, ptfd, num2do; XrdPollInfo *piP; // To keep ourselves from being swamped, base request read-aheads on the number // of pending poll events. // num2do = (maxreq < 3 ? -1 : maxreq); // Now process all poll table manipulation requests // while(num2do-- && getRequest()) { if (ReqBuff.req == PipeData::Post) {ReqBuff.Parms.theSem->Post(); continue; } pti = ReqBuff.Parms.Arg.ent; if ((ptfd = abs(PollTab[pti].fd)) != ReqBuff.Parms.Arg.fd) {LogEvent(ReqBuff.req, PollTab[pti].fd, ReqBuff.Parms.Arg.fd); continue; } if (!(piP = XrdLinkCtl::fd2PollInfo(ptfd))) {LogEvent(ReqBuff.req, -1, ptfd); continue;} if (ReqBuff.req == PipeData::EnFD) {PollTab[pti].events = POLLIN | POLLRDNORM; PollTab[pti].fd = ptfd; piP->isEnabled = true; numEnabled++; act = " enabled fd "; } else if (ReqBuff.req == PipeData::DiFD) {PollTab[pti].fd = -ptfd; act = " disabled fd "; piP->isEnabled = false; } else if (ReqBuff.req == PipeData::RmFD) {PollTab[pti].fd = -1; doDetach(pti); act = " detached fd "; piP->isEnabled = false; } else {Log.Emsg("Poll", "Received an invalid poll pipe request"); continue; } TRACE(POLL, "Poller " <inQ = false; plp = 0; nlp = PollQ; while (nlp && (pInfo != nlp)) {plp=nlp; nlp = nlp->Next;} // If we found the link, remove it. Otherwise complain // if (nlp) {if (plp) plp->Next = nlp->Next; else PollQ = nlp->Next; PollMutex.UnLock(); } else {PollMutex.UnLock(); Log.Emsg("dqLink", "Link not found in Q", pInfo->Link.ID); } } /******************************************************************************/ /* L o g E v e n t */ /******************************************************************************/ void XrdPollPoll::LogEvent(int req, int pollfd, int cmdfd) { const char *opn, *id1, *id2; char buff[4096]; XrdLink *lp; if (ReqBuff.req == PipeData::EnFD) opn = "enable"; else if (ReqBuff.req == PipeData::DiFD) opn = "disable"; else if (ReqBuff.req == PipeData::RmFD) opn = "detach"; else opn = "???"; if (pollfd < 0) {sprintf(buff, "poll %d failed; FD %d", PID, cmdfd); Log.Emsg("Poll", opn, buff, "does not map to a link"); return; } if ((lp = XrdLinkCtl::fd2link(pollfd))) id1 = lp->ID; else id1 = "unknown"; if ((lp = XrdLinkCtl::fd2link(cmdfd))) id2 = lp->ID; else id2 = "unknown"; snprintf(buff, sizeof(buff)-1, "%d poll fd=%d (%s) not equal %s cmd fd=%d (%s).", PID, pollfd, id1, opn, cmdfd, id2); Log.Emsg("Poll", "cmd/poll mismatch:", buff); } /******************************************************************************/ /* R e c o v e r */ /******************************************************************************/ void XrdPollPoll::Recover(int numleft) { int i; XrdPollInfo *piP; // Turn off any unaccounted links // for (i = 1; i < PollTNum; i++) if (PollTab[i].revents) {if (!(piP = XrdLinkCtl::fd2PollInfo(PollTab[i].fd))) PollTab[i].fd = -1; else {piP->isEnabled = false; PollTab[i].fd = -PollTab[i].fd; Log.Emsg("Poll","Improper poll event for",piP->Link.ID); } } } /******************************************************************************/ /* R e s t a r t */ /******************************************************************************/ void XrdPollPoll::Restart(int ecode) { XrdPollInfo *pInfo; // Issue error message // TRACE(POLL, PID <<'-' <Next; pInfo->PollEnt->fd = -1; Finish(*pInfo, "Unexpected polling error"); Sched.Schedule((XrdJob *)&(pInfo->Link)); } PollMutex.UnLock(); } xrootd-5.6.9/src/Xrd/XrdProtLoad.cc000066400000000000000000000302501457266313600171240ustar00rootroot00000000000000/******************************************************************************/ /* */ /* X r d P r o t L o a d . c c */ /* */ /* (c) 2006 by the Board of Trustees of the Leland Stanford, Jr., University */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include "XrdOuc/XrdOucPinLoader.hh" #include "XrdSys/XrdSysError.hh" #include "Xrd/XrdLink.hh" #include "Xrd/XrdProtLoad.hh" #include "Xrd/XrdTrace.hh" #include "XrdVersion.hh" /******************************************************************************/ /* G l o b a l O b j e c t s */ /******************************************************************************/ XrdProtocol *XrdProtLoad::Protocol[ProtoMax] = {0}; char *XrdProtLoad::ProtName[ProtoMax] = {0}; int XrdProtLoad::ProtoCnt = 0; namespace { struct portMap {int port; short protIdx; bool protTLS; portMap(int pnum, int pidx, bool istls) : port(pnum), protIdx(pidx), protTLS(istls) {} ~portMap() {} }; std::vector portVec; char *liblist[XrdProtLoad::ProtoMax]; XrdOucPinLoader *libhndl[XrdProtLoad::ProtoMax]; const char *TraceID = "ProtLoad"; int libcnt = 0; } namespace XrdGlobal { extern XrdSysError Log; } using namespace XrdGlobal; /******************************************************************************/ /* C o n s t r u c t o r a n d D e s t r u c t o r */ /******************************************************************************/ XrdProtLoad::XrdProtLoad(int port) : XrdProtocol("protocol loader"), myPort(port) { int j = 0; bool hastls = false; // Extract out the protocols associated with this port // for (int i = 0; i < (int)portVec.size(); i++) {if (myPort == portVec[i].port) {if (portVec[i].protTLS) hastls = true; else myProt[j++] = portVec[i].protIdx; } } // Setup to handle tls protocols // if (hastls) {myProt[j++] = -1; for (int i = 0; i < (int)portVec.size(); i++) {if (myPort == portVec[i].port && portVec[i].protTLS) myProt[j++] = portVec[i].protIdx; } } // Put in an end marker // myProt[j] = -2; } XrdProtLoad::~XrdProtLoad() {} /******************************************************************************/ /* L o a d */ /******************************************************************************/ int XrdProtLoad::Load(const char *lname, const char *pname, char *parms, XrdProtocol_Config *pi, bool istls) { XrdProtocol *xp; int port = pi->Port; // Trace this load if so wanted // TRACE(DEBUG, "getting protocol object " <= ProtoMax) {Log.Emsg("Protocol", "Too many protocols have been defined."); return 0; } // Obtain an instance of this protocol // xp = getProtocol(lname, pname, parms, pi); if (!xp) {Log.Emsg("Protocol","Protocol", pname, "could not be loaded"); return 0; } // Add protocol to our table of protocols. // ProtName[ProtoCnt] = strdup(pname); Protocol[ProtoCnt] = xp; ProtoCnt++; // Map the port to this protocol // Port(ProtoCnt, port, istls); return ProtoCnt; } /******************************************************************************/ /* P o r t */ /******************************************************************************/ int XrdProtLoad::Port(const char *lname, const char *pname, char *parms, XrdProtocol_Config *pi) { int port; // Obtain the port number to be used by this protocol // port = getProtocolPort(lname, pname, parms, pi); // Trace this call if so wanted // TRACE(DEBUG, "protocol " < 0 && protIdx <= ProtoCnt && port > 0) {portMap pMap(port, protIdx-1, isTLS); portVec.push_back(pMap); TRACE(DEBUG, "enabling " <<(isTLS ? "tls port " : "port ") <setTLS(true))) {lp->setEtext("TLS negotiation failed."); return -1; } } else {i = *pVec; if ((pp = Protocol[i]->Match(lp))) break; else if (lp->isFlawed()) return -1; } pVec++; } // Verify that we have an actual protocol // if (!pp) {lp->setEtext("matching protocol not found"); return -1;} // Now attach the new protocol object to the link // lp->setProtocol(pp); lp->setProtName(ProtName[i]); // Trace this load if so wanted // TRACE(DEBUG, "matched port " <Activate()) {lp->setEtext("activation failed"); return -1;} // Take a short-cut and process the initial request as a sticky request // return pp->Process(lp); } /******************************************************************************/ /* R e c y c l e */ /******************************************************************************/ void XrdProtLoad::Recycle(XrdLink *lp, int ctime, const char *reason) { // Document non-protocol errors // if (lp && reason) Log.Emsg("Protocol", lp->ID, "terminated", reason); } /******************************************************************************/ /* S t a t i s t i c s */ /******************************************************************************/ int XrdProtLoad::Statistics(char *buff, int blen, int do_sync) { int i, k, totlen = 0; for (i = 0; i < ProtoCnt && (blen > 0 || !buff); i++) {k = Protocol[i]->Stats(buff, blen, do_sync); totlen += k; buff += k; blen -= k; } return totlen; } /******************************************************************************/ /* P r i v a t e M e t h o d s */ /******************************************************************************/ /******************************************************************************/ /* g e t P r o t o c o l */ /******************************************************************************/ extern "C" XrdProtocol *XrdgetProtocol(const char *pname, char *parms, XrdProtocol_Config *pi); XrdProtocol *XrdProtLoad::getProtocol(const char *lname, const char *pname, char *parms, XrdProtocol_Config *pi) { XrdProtocol *(*ep)(const char *, char *, XrdProtocol_Config *); const char *xname = (lname ? lname : ""); void *epvoid; int i; // If this is a builtin protocol getthe protocol object directly // if (!lname) return XrdgetProtocol(pname, parms, pi); // Find the matching library. It must be here because getPort was already called // for (i = 0; i < libcnt; i++) if (!strcmp(xname, liblist[i])) break; if (i >= libcnt) {Log.Emsg("Protocol", pname, "was lost during loading", lname); return 0; } // Obtain an instance of the protocol object and return it // if (!(epvoid = libhndl[i]->Resolve("XrdgetProtocol"))) return 0; ep = (XrdProtocol *(*)(const char*,char*,XrdProtocol_Config*))epvoid; return ep(pname, parms, pi); } /******************************************************************************/ /* g e t P r o t o c o l P o r t */ /******************************************************************************/ extern "C" int XrdgetProtocolPort(const char *pname, char *parms, XrdProtocol_Config *pi); int XrdProtLoad::getProtocolPort(const char *lname, const char *pname, char *parms, XrdProtocol_Config *pi) { static XrdVERSIONINFODEF(myVer, xrd, XrdVNUMBER, XrdVERSION); const char *xname = (lname ? lname : ""); int (*ep)(const char *, char *, XrdProtocol_Config *); void *epvoid; int i; // If this is for the builtin protocol then get the port directly // if (!lname) return XrdgetProtocolPort(pname, parms, pi); // See if the library is already opened, if not open it // for (i = 0; i < libcnt; i++) if (!strcmp(xname, liblist[i])) break; if (i >= libcnt) {if (libcnt >= ProtoMax) {Log.Emsg("Protocol", "Too many protocols have been defined."); return -1; } if (!(libhndl[i] = new XrdOucPinLoader(&Log,&myVer,"protocol",lname))) return -1; liblist[i] = strdup(xname); libcnt++; } // Get the port number to be used // if (!(epvoid = libhndl[i]->Resolve("XrdgetProtocolPort", 2))) return (pi->Port < 0 ? 0 : pi->Port); ep = (int (*)(const char*,char*,XrdProtocol_Config*))epvoid; return ep(pname, parms, pi); } xrootd-5.6.9/src/Xrd/XrdProtLoad.hh000066400000000000000000000074161457266313600171460ustar00rootroot00000000000000#ifndef __XrdProtLoad_H__ #define __XrdProtLoad_H__ /******************************************************************************/ /* */ /* X r d P r o t L o a d . h h */ /* */ /* (c) 2006 by the Board of Trustees of the Leland Stanford, Jr., University */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include "Xrd/XrdProtocol.hh" // This class load and allows the selection of the appropriate link protocol. // class XrdProtLoad : public XrdProtocol { public: void DoIt() {} static int Load(const char *lname, const char *pname, char *parms, XrdProtocol_Config *pi, bool istls); static int Port(const char *lname, const char *pname, char *parms, XrdProtocol_Config *pi); static void Port(int protIdx, int port, bool isTLS); XrdProtocol *Match(XrdLink *) {return 0;} int Process(XrdLink *lp); void Recycle(XrdLink *lp, int ctime, const char *txt); int Stats(char *buff, int blen, int do_sync=0) {return 0;} static int Statistics(char *buff, int blen, int do_sync=0); XrdProtLoad(int port=-1); ~XrdProtLoad(); static const int ProtoMax = 8; static const int PortoMax = 8; private: static XrdProtocol *getProtocol (const char *lname, const char *pname, char *parms, XrdProtocol_Config *pi); static int getProtocolPort(const char *lname, const char *pname, char *parms, XrdProtocol_Config *pi); static char *ProtName[ProtoMax]; // ->Supported protocol names static XrdProtocol *Protocol[ProtoMax]; // ->Supported protocol objects static int ProtoCnt; // Number in table (at least 1) int myPort; signed char myProt[ProtoMax+2]; // My protocols }; #endif xrootd-5.6.9/src/Xrd/XrdProtocol.hh000066400000000000000000000247471457266313600172310ustar00rootroot00000000000000#ifndef __XrdProtocol_H__ #define __XrdProtocol_H__ /******************************************************************************/ /* */ /* X r d P r o t o c o l . h h */ /* */ /*(c) 2004-18 By the Board of Trustees of the Leland Stanford, Jr., University*/ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include "Xrd/XrdJob.hh" /******************************************************************************/ /* X r d P r o t o c o l _ C o n f i g */ /******************************************************************************/ // The following class is passed to the XrdgetProtocol() and XrdgetProtocolPort() // functions to properly configure the protocol. This object is not stable and // the protocol must copy out any values it desires to keep. It may copy the // whole object using the supplied copy constructor. class XrdSysError; union XrdNetSockAddr; class XrdOucEnv; class XrdOucString; class XrdBuffManager; class XrdInet; class XrdScheduler; class XrdStats; class XrdTlsContext; struct sockaddr; class XrdProtocol_Config { public: // The following pointers may be copied; they are stable. // XrdSysError *eDest; // Stable -> Error Message/Logging Handler XrdInet *NetTCP; // Stable -> Network Object (@ XrdgetProtocol) XrdBuffManager *BPool; // Stable -> Buffer Pool Manager XrdScheduler *Sched; // Stable -> System Scheduler XrdStats *Stats; // Stable -> System Statistics (@ XrdgetProtocol) XrdOucEnv *theEnv; // Stable -> Additional environmental information void *rsvd0; // The following information must be duplicated; it is unstable. // char *ConfigFN; // -> Configuration file int Format; // Binary format of this server int Port; // Port number int WSize; // Window size for Port int rsvd1; const char *AdmPath; // Admin path int AdmMode; // Admin path mode int xrdFlags; static const int admPSet = 0x00000001; // The adminppath was set via cli const char *myInst; // Instance name const char *myName; // Host name const char *myProg; // Program name union { const XrdNetSockAddr *urAddr; // Host Address (the actual structure/union) const struct sockaddr *myAddr; // Host address }; int ConnMax; // Max connections int readWait; // Max milliseconds to wait for data int idleWait; // Max milliseconds connection may be idle int argc; // Number of arguments char **argv; // Argument array (prescreened) char DebugON; // True if started with -d option char rsvd3[7]; int hailWait; // Max milliseconds to wait for data after accept int tlsPort; // Default TLS port (0 if not specified) XrdTlsContext *tlsCtx; // Stable -> TLS Context (0 if not initialized) XrdOucString *totalCF; // Stable -> total config after full init XrdProtocol_Config(XrdProtocol_Config &rhs) =delete; XrdProtocol_Config() : rsvd0(0), rsvd1(0) {memset(rsvd3, 0, sizeof(rsvd3));} ~XrdProtocol_Config() {} }; /******************************************************************************/ /* X r d P r o t o c o l */ /******************************************************************************/ // This class is used by the Link object to process the input stream on a link. // At least one protocol object exists per Link object. Specific protocols are // derived from this pure abstract class since a link can use one of several // protocols. Indeed, startup and shutdown are handled by specialized protocols. // System configuration obtains an instance of a protocol by calling // XrdgetProtocol(), which must exist in the shared library. // This instance is used as the base pointer for Alloc(), Configure(), and // Match(). Unfortuantely, they cannot be static given the silly C++ rules. class XrdLink; class XrdProtocol : public XrdJob { public: // Match() is invoked when a new link is created and we are trying // to determine if this protocol can handle the link. It must // return a protocol object if it can and NULL (0), otherwise. // virtual XrdProtocol *Match(XrdLink *lp) = 0; // Process() is invoked when a link has data waiting to be read // virtual int Process(XrdLink *lp) = 0; // Recycle() is invoked when this object is no longer needed. The method is // passed the number of seconds the protocol was connected to the // link and the reason for the disconnection, if any. // virtual void Recycle(XrdLink *lp=0,int consec=0,const char *reason=0)=0; // Stats() is invoked when we need statistics about all instances of the // protocol. If a buffer is supplied, it must return a null // terminated string in the supplied buffer and the return value // is the number of bytes placed in the buffer defined by C99 for // snprintf(). If no buffer is supplied, the method should return // the maximum number of characters that could have been returned. // Regardless of the buffer value, if do_sync is true, the method // should include any local statistics in the global data (if any) // prior to performing any action. // virtual int Stats(char *buff, int blen, int do_sync=0) = 0; XrdProtocol(const char *jname): XrdJob(jname) {} virtual ~XrdProtocol() {} }; /******************************************************************************/ /* X r d g e t P r o t o c o l */ /******************************************************************************/ /* This extern "C" function must be defined in the shared library plug-in implementing your protocol. It is called to obtain an instance of your protocol. This allows protocols to live outside of the protocol driver (i.e., to be loaded at run-time). The call is made after the call to XrdgetProtocolPort() to determine the port to be used (see below) which allows e network object (NetTCP) to be proerly defined and it's pointer is passed in the XrdProtocol_Config object for your use. Required return values: Success: Pointer to XrdProtocol object. Failure: Null pointer (i.e. 0) which causes the program to exit. extern "C" // This is in a comment! { XrdProtocol *XrdgetProtocol(const char *protocol_name, char *parms, XrdProtocol_Config *pi) {....} } */ /******************************************************************************/ /* X r d g e t P r o t o c o l P o r t */ /******************************************************************************/ /* This extern "C" function must be defined for statically linked protocols but is optional for protocols defined as a shared library plug-in if the rules determining which port number to use is sufficient for your protocol. The function is called to obtain the actual port number to be used by the the protocol. The default port number is noted in XrdProtocol_Config Port. Initially, it has one of the fllowing values: <0 -> No port was specified. =0 -> An erbitrary port will be assigned. >0 -> This port number was specified. XrdgetProtoclPort() must return: <0 -> Failure is indicated and we terminate =0 -> Use an arbitrary port (even if this equals Port) >0 -> The returned port number must be used (even if it equals Port) When we finally call XrdgetProtocol(), the actual port number is indicated in Port and the network object is defined in NetTCP and bound to the port. Final Caveats: 1. The network object (NetTCP) is not defined until XrdgetProtocol() is called. 2. The statistics object (Stats) is not defined until XrdgetProtocol() is called. 3. When the protocol is loaded from a shared library, you need need not define XrdgetProtocolPort() if the standard port determination scheme is sufficient. extern "C" // This is in a comment! { int XrdgetProtocolPort(const char *protocol_name, char *parms, XrdProtocol_Config *pi) {....} } */ #endif xrootd-5.6.9/src/Xrd/XrdScheduler.cc000066400000000000000000000572421457266313600173300ustar00rootroot00000000000000/******************************************************************************/ /* */ /* X r d S c h e d u l e r . c c */ /* */ /* (c) 2004 by the Board of Trustees of the Leland Stanford, Jr., University */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include #include #include #include #include #include #include #ifdef __APPLE__ #include #endif #include "Xrd/XrdJob.hh" #include "Xrd/XrdScheduler.hh" #include "XrdOuc/XrdOucTrace.hh" // For ABI compatibility only! #include "XrdSys/XrdSysError.hh" #include "XrdSys/XrdSysLogger.hh" #define XRD_TRACE XrdTrace-> #include "Xrd/XrdTrace.hh" /******************************************************************************/ /* S t a t i c O b j e c t s */ /******************************************************************************/ const char *XrdScheduler::TraceID = "Sched"; /******************************************************************************/ /* L o c a l C l a s s e s */ /******************************************************************************/ class XrdSchedulerPID {public: XrdSchedulerPID *next; pid_t pid; XrdSchedulerPID(pid_t newpid, XrdSchedulerPID *prev) {next = prev; pid = newpid;} ~XrdSchedulerPID() {} }; /******************************************************************************/ /* E x t e r n a l T h r e a d I n t e r f a c e s */ /******************************************************************************/ void *XrdStartReaper(void *carg) {XrdScheduler *sp = (XrdScheduler *)carg; sp->Reaper(); return (void *)0; } void *XrdStartTSched(void *carg) {XrdScheduler *sp = (XrdScheduler *)carg; sp->TimeSched(); return (void *)0; } void *XrdStartWorking(void *carg) {XrdScheduler *sp = (XrdScheduler *)carg; sp->Run(); return (void *)0; } /******************************************************************************/ /* C o n s t r u c t o r */ /******************************************************************************/ XrdScheduler::XrdScheduler(XrdSysError *eP, XrdSysTrace *tP, int minw, int maxw, int maxi) : XrdJob("underused thread monitor"), XrdTraceOld(0), WorkAvail(0, "sched work") { Boot(eP, tP, minw, maxw, maxi); } /******************************************************************************/ XrdScheduler::XrdScheduler(XrdSysError *eP, XrdOucTrace *tP, int minw, int maxw, int maxi) : XrdJob("underused thread monitor"), XrdTraceOld(tP), WorkAvail(0, "sched work") { // Invoke the main initialization function with a new style trace object // Boot(eP, new XrdSysTrace("Xrd", eP->logger()), minw, maxw, maxi); } /******************************************************************************/ // This constructor creates a self contained scheduler. // XrdScheduler::XrdScheduler(int minw, int maxw, int maxi) : XrdJob("underused thread monitor"), XrdTraceOld(0), WorkAvail(0, "sched work") { XrdSysLogger *Logger; int eFD; // Get a file descriptor mirroring standard error // #if ( defined(__linux__) || defined(__GNU__) ) && defined(F_DUPFD_CLOEXEC) eFD = fcntl(STDERR_FILENO, F_DUPFD_CLOEXEC, 0); #else eFD = dup(STDERR_FILENO); fcntl(eFD, F_SETFD, FD_CLOEXEC); #endif // Now we need to get a logger object. We make this a real dumb one. // Logger = new XrdSysLogger(eFD, 0); XrdLog = new XrdSysError(Logger); // Now get a trace object // XrdTrace = new XrdSysTrace("Xrd", Logger); if (getenv("XRDDEBUG") != 0) XrdTrace->What = TRACE_SCHED; // Set remaining values. We do no use maximum possible threads here. // Init(minw, maxw, maxi); } /******************************************************************************/ /* Private: B o o t */ /******************************************************************************/ void XrdScheduler::Boot(XrdSysError *eP, XrdSysTrace *tP, int minw, int maxw, int maxi) { // Perform common initialization // XrdLog = eP; XrdTrace = tP; Init(minw, maxw, maxi); // Make sure we are using the maximum number of threads allowed (Linux only) // #if ( defined(__linux__) || defined(__GNU__) || (defined(__FreeBSD_kernel__) && defined(__GLIBC__)) ) && defined(RLIMIT_NPROC) struct rlimit rlim; // First determine the absolute maximum we can have // rlim_t theMax = MAX_SCHED_PROCS; int pdFD, rdsz; if ((pdFD = open("/proc/sys/kernel/pid_max", O_RDONLY)) >= 0) {char pmBuff[32]; if ((rdsz = read(pdFD, pmBuff, sizeof(pmBuff))) > 0) {rdsz = atoi(pmBuff); if (rdsz < 16384) theMax = 16384; // This is unlikely else if (rdsz < MAX_SCHED_PROCS) theMax = static_cast(rdsz-2000); } close(pdFD); } // Get the resource thread limit and set to maximum. In Linux this may be -1 // to indicate useless infnity, so we have to come up with a number, sigh. // if (!getrlimit(RLIMIT_NPROC, &rlim)) {if (rlim.rlim_max == RLIM_INFINITY || rlim.rlim_max > theMax) {rlim.rlim_cur = theMax; setrlimit(RLIMIT_NPROC, &rlim); } else { if (rlim.rlim_cur != rlim.rlim_max) {rlim.rlim_cur = rlim.rlim_max; setrlimit(RLIMIT_NPROC, &rlim); } } } // Readjust our internal maximum to be the actual maximum // if (!getrlimit(RLIMIT_NPROC, &rlim)) {if (rlim.rlim_cur == RLIM_INFINITY || rlim.rlim_cur > theMax) max_Workers = static_cast(theMax); else max_Workers = static_cast(rlim.rlim_cur); } #endif } /******************************************************************************/ /* D e s t r u c t o r */ /******************************************************************************/ XrdScheduler::~XrdScheduler() // The scheduler is never deleted! { } /******************************************************************************/ /* C a n c e l */ /******************************************************************************/ void XrdScheduler::Cancel(XrdJob *jp) { XrdJob *p, *pp = 0; // Lock the queue // TimerMutex.Lock(); // Find the matching job, if any // p = TimerQueue; while(p && p != jp) {pp = p; p = p->NextJob;} // Delete the job element // if (p) {if (pp) pp->NextJob = p->NextJob; else TimerQueue = p->NextJob; TRACE(SCHED, "time event " <Comment <<" cancelled"); } // All done // TimerMutex.UnLock(); } /******************************************************************************/ /* D o I t */ /******************************************************************************/ void XrdScheduler::DoIt() { int num_kill, num_idle; // Now check if there are too many idle threads (kill them if there are) // if (!num_JobsinQ) {DispatchMutex.Lock(); num_idle = idl_Workers; DispatchMutex.UnLock(); num_kill = num_idle - min_Workers; TRACE(SCHED, num_Workers <<" threads; " < 0) {if (num_kill > 1) num_kill = num_kill/2; SchedMutex.Lock(); num_Layoffs = num_kill; while(num_kill--) WorkAvail.Post(); SchedMutex.UnLock(); } } // Check if we should reschedule ourselves // if (max_Workidl > 0) Schedule((XrdJob *)this, max_Workidl+time(0)); } /******************************************************************************/ /* F o r k */ /******************************************************************************/ // This entry exists solely so that we can start a reaper thread for processes // pid_t XrdScheduler::Fork(const char *id) { static int retc, ReaperStarted = 0; pthread_t tid; pid_t pid; // Fork // if ((pid = fork()) < 0) {XrdLog->Emsg("Scheduler",errno,"fork to handle",id); return pid; } if (!pid) return pid; // Obtain the status of the reaper thread. // ReaperMutex.Lock(); firstPID = new XrdSchedulerPID(pid, firstPID); retc = ReaperStarted; ReaperStarted = 1; ReaperMutex.UnLock(); // Start the reaper thread if it has not started. // if (!retc) if ((retc = XrdSysThread::Run(&tid, XrdStartReaper, (void *)this, 0, "Process reaper"))) {XrdLog->Emsg("Scheduler", retc, "create reaper thread"); ReaperStarted = 0; } return pid; } /******************************************************************************/ /* R e a p e r */ /******************************************************************************/ void *XrdScheduler::Reaper() { int status; pid_t pid; XrdSchedulerPID *tp, *ptp, *xtp; #if defined(__APPLE__) && !defined(MAC_OS_X_VERSION_10_5) struct timespec ts = { 1, 0 }; #else sigset_t Sset; int signum; // Set up for signal handling. Note: main() must block this signal at start) // sigemptyset(&Sset); sigaddset(&Sset, SIGCHLD); #endif // Wait for all outstanding children // do {ReaperMutex.Lock(); tp = firstPID; ptp = 0; while(tp) {do {pid = waitpid(tp->pid, &status, WNOHANG);} while (pid < 0 && errno == EINTR); if (pid > 0) {if (TRACING(TRACE_SCHED)) traceExit(pid, status); xtp = tp; tp = tp->next; if (ptp) ptp->next = tp; else firstPID = tp; delete xtp; } else {ptp = tp; tp = tp->next;} } ReaperMutex.UnLock(); #if defined(__APPLE__) && !defined(MAC_OS_X_VERSION_10_5) // Mac OS X sigwait() is broken on <= 10.4. } while (nanosleep(&ts, 0) <= 0); #else } while(sigwait(&Sset, &signum) >= 0); #endif return (void *)0; } /******************************************************************************/ /* R u n */ /******************************************************************************/ void XrdScheduler::Run() { int waiting; XrdJob *jp; // Wait for work then do it (an endless task for a worker thread) // do {do {DispatchMutex.Lock(); idl_Workers++;DispatchMutex.UnLock(); WorkAvail.Wait(); DispatchMutex.Lock();waiting = --idl_Workers;DispatchMutex.UnLock(); SchedMutex.Lock(); if ((jp = WorkFirst)) {if (!(WorkFirst = jp->NextJob)) WorkLast = 0; if (num_JobsinQ) num_JobsinQ--; else XrdLog->Emsg("Scheduler","Job queue count underflow!"); } else { num_JobsinQ = 0; if (num_Layoffs > 0) {num_Layoffs--; if (waiting) {num_TDestroy++; num_Workers--; TRACE(SCHED, "terminating thread; workers=" <Comment) != '.') {TRACE(SCHED, "running " <Comment <<" inq=" <DoIt(); } while(1); } /******************************************************************************/ /* S c h e d u l e */ /******************************************************************************/ void XrdScheduler::Schedule(XrdJob *jp) { // Lock down our data area // SchedMutex.Lock(); // Place the request on the queue and broadcast it // jp->NextJob = 0; if (WorkFirst) {WorkLast->NextJob = jp; WorkLast = jp; } else { WorkFirst = jp; WorkLast = jp; } WorkAvail.Post(); // Calculate statistics // num_Jobs++; num_JobsinQ++; if (num_JobsinQ > max_QLength) max_QLength = num_JobsinQ; // Unlock the data area and return // SchedMutex.UnLock(); } /******************************************************************************/ void XrdScheduler::Schedule(int numjobs, XrdJob *jfirst, XrdJob *jlast) { // Lock down our data area // SchedMutex.Lock(); // Place the request list on the queue // jlast->NextJob = 0; if (WorkFirst) {WorkLast->NextJob = jfirst; WorkLast = jlast; } else { WorkFirst = jfirst; WorkLast = jlast; } // Calculate statistics // num_Jobs += numjobs; num_JobsinQ += numjobs; if (num_JobsinQ > max_QLength) max_QLength = num_JobsinQ; // Indicate number of jobs to work on // while(numjobs--) WorkAvail.Post(); // Unlock the data area and return // SchedMutex.UnLock(); } /******************************************************************************/ void XrdScheduler::Schedule(XrdJob *jp, time_t atime) { XrdJob *pp = 0, *p; // Cancel this event, if scheduled // Cancel(jp); // Lock the queue // if (TRACING(TRACE_SCHED) && *(jp->Comment) != '.') {TRACE(SCHED, "scheduling " <Comment <<" in " <SchedTime = atime; TimerMutex.Lock(); // Find the insertion point for the work element // p = TimerQueue; while(p && p->SchedTime <= atime) {pp = p; p = p->NextJob;} // Insert the job element // jp->NextJob = p; if (pp) pp->NextJob = jp; else {TimerQueue = jp; TimerRings.Signal();} // All done // TimerMutex.UnLock(); } /******************************************************************************/ /* s e t P a r m s */ /******************************************************************************/ void XrdScheduler::setParms(int minw, int maxw, int avlw, int maxi, int once) { static int isSet = 0; // Lock the data area and check for 1-time set // SchedMutex.Lock(); if (once && isSet) {SchedMutex.UnLock(); return;} isSet = 1; // get a consistent view of all the values // if (maxw <= 0) maxw = max_Workers; if (minw < 0) minw = min_Workers; if (minw > maxw) minw = maxw; if (avlw < 0) avlw = maxw/4*3; else if (avlw > maxw) avlw = maxw; // Set the values // min_Workers = minw; max_Workers = maxw; stk_Workers = maxw - avlw; if (maxi >=0) max_Workidl = maxi; // Unlock the data area // SchedMutex.UnLock(); // If we have an idle interval, schedule the idle check // if (maxi > 0) {Cancel((XrdJob *)this); Schedule((XrdJob *)this, (time_t)maxi+time(0)); } // Debug the info // TRACE(SCHED,"Set min_Workers=" <What = TRACE_SCHED; else if (XrdTraceOld) XrdTrace->What |= XrdTraceOld->What; // Start a time based scheduler // if ((retc = XrdSysThread::Run(&tid, XrdStartTSched, (void *)this, XRDSYSTHREAD_BIND, "Time scheduler"))) XrdLog->Emsg("Scheduler", retc, "create time scheduler thread"); // If we an idle interval, schedule the idle check // if (max_Workidl > 0) Schedule((XrdJob *)this, (time_t)max_Workidl+time(0)); // Start 1/3 of the minimum number of threads // if (!(numw = min_Workers/3)) numw = 2; while(numw--) hireWorker(0); // Unlock the data area // TRACE(SCHED, "Starting with " <%d" "%d%d" "%d%d" "%d%d" "%d"; // If only length wanted, do so // if (!buff) return sizeof(statfmt) + 16*8; // Get values protected by the Dispatch lock (avoid lock if no sync needed) // if (do_sync) DispatchMutex.Lock(); cnt_idl = idl_Workers; if (do_sync) DispatchMutex.UnLock(); // Get values protected by the Scheduler lock (avoid lock if no sync needed) // if (do_sync) SchedMutex.Lock(); cnt_Workers = num_Workers; cnt_Jobs = num_Jobs; cnt_JobsinQ = num_JobsinQ; xam_QLength = max_QLength; cnt_TCreate = num_TCreate; cnt_TDestroy= num_TDestroy; cnt_Limited = num_Limited; if (do_sync) SchedMutex.UnLock(); // Format the stats and return them // return snprintf(buff, blen, statfmt, cnt_Jobs, cnt_JobsinQ, xam_QLength, cnt_Workers, cnt_idl, cnt_TCreate, cnt_TDestroy, cnt_Limited); } /******************************************************************************/ /* T i m e S c h e d */ /******************************************************************************/ void XrdScheduler::TimeSched() { XrdJob *jp; int wtime; // Continuous loop until we find some work here // do {TimerMutex.Lock(); if (TimerQueue) wtime = TimerQueue->SchedTime-time(0); else wtime = 60*60; if (wtime > 0) {TimerMutex.UnLock(); TimerRings.Wait(wtime); } else { jp = TimerQueue; TimerQueue = jp->NextJob; Schedule(jp); TimerMutex.UnLock(); } } while(1); } /******************************************************************************/ /* P r i v a t e M e t h o d s */ /******************************************************************************/ /******************************************************************************/ /* h i r e W o r k e r */ /******************************************************************************/ void XrdScheduler::hireWorker(int dotrace) { pthread_t tid; int retc; // First check if we reached the maximum number of workers // SchedMutex.Lock(); if (num_Workers >= max_Workers) {num_Limited++; if ((num_Limited & 4095) == 1) XrdLog->Emsg("Scheduler","Thread limit has been reached!"); SchedMutex.UnLock(); return; } num_Workers++; num_TCreate++; SchedMutex.UnLock(); // Start a new thread. We do this without the schedMutex to avoid hang-ups. If // we can't start a new thread, we recalculate the maximum number we can. // retc = XrdSysThread::Run(&tid, XrdStartWorking, (void *)this, 0, "Worker"); // Now check the results and correct if we couldn't start the thread // if (retc) {XrdLog->Emsg("Scheduler", retc, "create worker thread"); SchedMutex.Lock(); num_Workers--; num_TCreate--; max_Workers = num_Workers; min_Workers = (max_Workers/10 ? max_Workers/10 : 1); stk_Workers = max_Workers/4*3; SchedMutex.UnLock(); } else if (dotrace) TRACE(SCHED, "Now have " <. */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include #include "XrdSys/XrdSysPthread.hh" #include "Xrd/XrdJob.hh" class XrdOucTrace; class XrdSchedulerPID; class XrdSysError; class XrdSysTrace; #define MAX_SCHED_PROCS 30000 class XrdScheduler : public XrdJob { public: int Active() {return num_Workers - idl_Workers + num_JobsinQ;} void Cancel(XrdJob *jp); inline int canStick() {return num_Workers < stk_Workers || (num_Workers-idl_Workers) < stk_Workers;} void DoIt(); pid_t Fork(const char *id); void *Reaper(); void Run(); void Schedule(XrdJob *jp); void Schedule(int num, XrdJob *jfirst, XrdJob *jlast); void Schedule(XrdJob *jp, time_t atime); void setParms(int minw, int maxw, int avlt, int maxi, int once=0); void Start(); int Stats(char *buff, int blen, int do_sync=0); void TimeSched(); // Statistical information // int num_TCreate; // Number of threads created int num_TDestroy;// Number of threads destroyed int num_Jobs; // Number of jobs scheduled int max_QLength; // Longest queue length we had int num_Limited; // Number of times max was reached // This is the preferred constructor // XrdScheduler(XrdSysError *eP, XrdSysTrace *tP, int minw=8, int maxw=8192, int maxi=780); // This constructor is only maintained for ABI compatibility and will be // removed in a future major release. While syntactically compatible the // sematics now are slightly different and tracing might not occur. // XrdScheduler(XrdSysError *eP, XrdOucTrace *tP, int minw=8, int maxw=8192, int maxi=780); // This constructor is used for a stand-alone scheduler. // XrdScheduler(int minw=3, int maxw=128, int maxi=12); ~XrdScheduler(); private: XrdSysError *XrdLog; XrdSysTrace *XrdTrace; XrdOucTrace *XrdTraceOld; // This is only used for ABI compatibility XrdSysMutex DispatchMutex; // Disp: Protects above area int idl_Workers; // Disp: Number of idle workers int min_Workers; // Sched: Min threads we need to have int max_Workers; // Sched: Max threads we can start int max_Workidl; // Sched: Max idle time for threads above min_Workers int num_Workers; // Sched: Number of threads we have int stk_Workers; // Sched: Number of sticky workers we can have int num_JobsinQ; // Sched: Number of outstanding jobs in the queue int num_Layoffs; // Sched: Number of threads to terminate XrdJob *WorkFirst; // Pending work XrdJob *WorkLast; XrdSysSemaphore WorkAvail; XrdSysMutex SchedMutex; // Protects private area XrdJob *TimerQueue; // Pending work XrdSysCondVar TimerRings; XrdSysMutex TimerMutex; // Protects scheduler area XrdSchedulerPID *firstPID; XrdSysMutex ReaperMutex; void Boot(XrdSysError *eP, XrdSysTrace *tP, int minw, int maxw, int maxi); void hireWorker(int dotrace=1); void Init(int minw, int maxw, int maxi); void Monitor(); void traceExit(pid_t pid, int status); static const char *TraceID; }; #endif xrootd-5.6.9/src/Xrd/XrdSendQ.cc000066400000000000000000000324451457266313600164220ustar00rootroot00000000000000/******************************************************************************/ /* */ /* X r d S e n d Q . c c */ /* */ /* (c) 2016 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include #include #include #include "Xrd/XrdLink.hh" #include "Xrd/XrdScheduler.hh" #include "Xrd/XrdSendQ.hh" #include "XrdSys/XrdSysError.hh" #include "XrdSys/XrdSysPthread.hh" /******************************************************************************/ /* L o c a l C l a s s e s */ /******************************************************************************/ class LinkShutdown : public XrdJob { public: virtual void DoIt() {myLink->Shutdown(true); myLink->setRef(-1); delete this; } LinkShutdown(XrdLink *link) : XrdJob("SendQ Shutdown"), myLink(link) {} virtual ~LinkShutdown() {} private: XrdLink *myLink; }; /******************************************************************************/ /* G l o b a l O b j e c t s */ /******************************************************************************/ namespace XrdGlobal { extern XrdSysError Log; extern XrdScheduler Sched; }; using namespace XrdGlobal; /******************************************************************************/ /* S t a t i c O b j e c t s */ /******************************************************************************/ unsigned int XrdSendQ::qWarn = 3; unsigned int XrdSendQ::qMax = 0xffffffff; bool XrdSendQ::qPerm = false; /******************************************************************************/ /* C o n s t r u c t o r */ /******************************************************************************/ XrdSendQ::XrdSendQ(XrdLink &lP, XrdSysMutex &mP) : XrdJob("sendQ runner"), mLink(lP), wMutex(mP), fMsg(0), lMsg(0), delQ(0), theFD(lP.FDnum()), inQ(0), qWmsg(qWarn), discards(0), active(false), terminate(false) {} /******************************************************************************/ /* D o I t */ /******************************************************************************/ void XrdSendQ::DoIt() { mBuff *theMsg; int myFD, rc; bool theEnd; // Obtain the lock // wMutex.Lock(); // Before we start check if we should delete any messages // if (delQ) {RelMsgs(delQ); delQ = 0;} // Send all queued messages (we can use a blocking send here) // while(!terminate && (theMsg = fMsg)) {if (!(fMsg = fMsg->next)) lMsg = 0; inQ--; myFD = theFD; wMutex.UnLock(); rc = send(myFD, theMsg->mData, theMsg->mLen, 0); free(theMsg); wMutex.Lock(); if (rc < 0) {Scuttle(); break;} } // Before we exit check if we should delete any messages // if (delQ) {RelMsgs(delQ); delQ = 0;} if ((theEnd = terminate) && fMsg) RelMsgs(fMsg); active = false; qWmsg = qWarn; // Release any messages that need to be released. Note that we may have been // deleted at this point so we cannot reference anything via "this" once we // unlock the mutex. We may also need to delete ourselves. // wMutex.UnLock(); if (theEnd) delete this; } /******************************************************************************/ /* Private: Q M s g */ /******************************************************************************/ bool XrdSendQ::QMsg(XrdSendQ::mBuff *theMsg) { // Check if we reached the max number of messages // if (inQ >= qMax) {discards++; if ((discards & 0xff) == 0x01) {char qBuff[80]; snprintf(qBuff, sizeof(qBuff), "%u) reached; %hu message(s) discarded!", qMax, discards); Log.Emsg("SendQ", mLink.Host(), "appears to be slow; queue limit (", qBuff); } return false; } // Add the message at the end of the queue // theMsg->next = 0; if (lMsg) lMsg->next = theMsg; else fMsg = theMsg; lMsg = theMsg; inQ++; // If there is no active thread handling this queue, schedule one // if (!active) {Sched.Schedule((XrdJob *)this); active = true; } // Check if we should issue a warning. // if (inQ >= qWmsg) {char qBuff[32]; qWmsg += qWarn; snprintf(qBuff, sizeof(qBuff), "%ud messages queued!", inQ); Log.Emsg("SendQ", mLink.Host(), "appears to be slow;", qBuff); } else { if (inQ < qWarn && qWmsg != qWarn) qWmsg = qWarn; } // All done // return true; } /******************************************************************************/ /* Private: R e l M s g s */ /******************************************************************************/ void XrdSendQ::RelMsgs(XrdSendQ::mBuff *mP) { mBuff *freeMP; while((freeMP = mP)) {mP = mP->next; free(freeMP); } } /******************************************************************************/ /* Private: S c u t t l e */ /******************************************************************************/ void XrdSendQ::Scuttle() // qMutex must be locked! { // Simply move any outsanding messages to the deletion queue // if (fMsg) {lMsg->next = delQ; delQ = fMsg; fMsg = lMsg = 0; inQ = 0; } } /******************************************************************************/ /* S e n d */ /******************************************************************************/ // Called with wMutex locked. int XrdSendQ::Send(const char *buff, int blen) { mBuff *theMsg; int bleft, bsent; // If there is an active thread handling messages then we have to queue it. // Otherwise try to send it. We need to hold the lock here to prevent messing // up the message is only part of it could be sent. This is a non-blocking call. // if (active) bleft = blen; else if ((bleft = SendNB(buff, blen)) <= 0) return (bleft ? -1 : blen); // Allocate buffer for the message // if (!(theMsg = (mBuff *)malloc(sizeof(mBuff) + bleft))) {errno = ENOMEM; return -1;} // Copy the unsent message fragment // bsent = blen - bleft; memcpy(theMsg->mData, buff+bsent, bleft); theMsg->mLen = bleft; // Queue the message. // return (QMsg(theMsg) ? blen : -1); } /******************************************************************************/ // Called with wMutex locked. int XrdSendQ::Send(const struct iovec *iov, int iovcnt, int iotot) { mBuff *theMsg; char *body; int bleft, bmore, iovX; // If there is an active thread handling messages then we have to queue it. // Otherwise try to send it. We need to hold the lock here to prevent messing // up the message is only part of it could be sent. This is a non-blocking call. // if (active) {bleft = 0; for (iovX = 0; iovX < iovcnt; iovX++) if ((bleft = iov[iovX].iov_len)) break; if (!bleft) return iotot; } else { if ((bleft = SendNB(iov, iovcnt, iotot, iovX)) <= 0) return (bleft ? -1 : 0); } // Readjust the total amount not sent based on where we stopped in the iovec. // bmore = bleft; for (int i = iovX+1; i < iovcnt; i++) bmore += iov[i].iov_len; // Copy the unsent message (for simplicity we will copy the whole iovec stop). // if (!(theMsg = (mBuff *)malloc(bmore+sizeof(mBuff)))) {errno = ENOMEM; return -1;} // Setup the message length // theMsg->mLen = bmore; // Copy the first fragment (it cannot be zero length) // body = theMsg->mData; memcpy(body, ((char *)iov[iovX].iov_base)+(iov[iovX].iov_len-bleft), bleft); body += bleft; // All remaining items // for (int i = iovX+1; i < iovcnt; i++) {if (iov[i].iov_len) {memcpy(body, iov[i].iov_base, iov[i].iov_len); body += iov[i].iov_len; } } // Queue the message. // return (QMsg(theMsg) ? iotot : 0); } /******************************************************************************/ /* S e n d N B */ /******************************************************************************/ // Called with wMutex locked. int XrdSendQ::SendNB(const char *Buff, int Blen) { #if !defined(__linux__) return -1; #else ssize_t retc = 0, bytesleft = Blen; // Write the data out // while(bytesleft) {do {retc = send(theFD, Buff, bytesleft, MSG_DONTWAIT);} while(retc < 0 && errno == EINTR); if (retc <= 0) break; bytesleft -= retc; Buff += retc; } // All done // if (retc <= 0) {if (!retc || errno == EAGAIN || retc == EWOULDBLOCK) return bytesleft; Log.Emsg("SendQ", errno, "send to", mLink.ID); return -1; } return bytesleft; #endif } /******************************************************************************/ // Called with wMutex locked. int XrdSendQ::SendNB(const struct iovec *iov, int iocnt, int bytes, int &iovX) { #if !defined(__linux__) return -1; #else char *msgP; ssize_t retc; int msgL, msgF = MSG_DONTWAIT|MSG_MORE, ioLast = iocnt-1; // Write the data out. The following code only works in Linux as we use the // new POSIX flags deined for send() which currently is only implemented in // Linux. This allows us to selectively use non-blocking I/O. // for (iovX = 0; iovX < iocnt; iovX++) {msgP = (char *)iov[iovX].iov_base; msgL = iov[iovX].iov_len; if (iovX == ioLast) msgF &= ~MSG_MORE; while(msgL) {do {retc = send(theFD, msgP, msgL, msgF);} while(retc < 0 && errno == EINTR); if (retc <= 0) {if (!retc || errno == EAGAIN || retc == EWOULDBLOCK) return msgL; Log.Emsg("SendQ", errno, "send to", mLink.ID); return -1; } msgL -= retc; } } // All done // return 0; #endif } /******************************************************************************/ /* T e r m i n a t e */ /******************************************************************************/ // This must be called with wMutex locked! void XrdSendQ::Terminate(XrdLink *lP) { // First step is to see if we need to schedule a shutdown prior to quiting // if (lP) Sched.Schedule((XrdJob *)new LinkShutdown(lP)); // If there is an active thread then we need to let the thread handle the // termination of this object. Otherwise, we can do it now. // if (active) {Scuttle(); terminate = true; theFD =-1; } else { if (fMsg) {RelMsgs(fMsg); fMsg = lMsg = 0;} if (delQ) {RelMsgs(delQ); delQ = 0;} delete this; } } xrootd-5.6.9/src/Xrd/XrdSendQ.hh000066400000000000000000000072271457266313600164340ustar00rootroot00000000000000#ifndef __XRDSENDQ__H #define __XRDSENDQ__H /******************************************************************************/ /* */ /* X r d S e n d Q . h h */ /* */ /* (c) 2016 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include #include #include "Xrd/XrdJob.hh" class XrdLink; class XrdSysMutex; class XrdSendQ : public XrdJob { public: unsigned int Backlog() {return inQ;} virtual void DoIt(); int Send(const char *buff, int blen); int Send(const struct iovec *iov, int iovcnt, int iotot); static void SetAQ(bool onoff) {qPerm = onoff;} static void SetQM(unsigned int qmVal) {qMax = qmVal;} static void SetQW(unsigned int qwVal) {qWarn = qwVal;} void Terminate(XrdLink *lP=0); XrdSendQ(XrdLink &lP, XrdSysMutex &mP); private: virtual ~XrdSendQ() {} int SendNB(const char *Buff, int Blen); int SendNB(const struct iovec *iov, int iocnt, int bytes, int &iovX); struct mBuff { mBuff *next; int mLen; char mData[4]; // Always made long enough }; bool QMsg(mBuff *theMsg); void RelMsgs(mBuff *mP); void Scuttle(); static unsigned int qWarn; static unsigned int qMax; static bool qPerm; XrdLink &mLink; XrdSysMutex &wMutex; mBuff *fMsg; mBuff *lMsg; mBuff *delQ; int theFD; unsigned int inQ; unsigned int qWmsg; unsigned short discards; bool active; bool terminate; }; #endif xrootd-5.6.9/src/Xrd/XrdStats.cc000066400000000000000000000276601457266313600165110ustar00rootroot00000000000000/******************************************************************************/ /* */ /* X r d S t a t s . c c */ /* */ /* (c) 2004 by the Board of Trustees of the Leland Stanford, Jr., University */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include #include #include #include "XrdVersion.hh" #include "Xrd/XrdBuffer.hh" #include "Xrd/XrdJob.hh" #include "Xrd/XrdLink.hh" #include "Xrd/XrdPoll.hh" #include "Xrd/XrdProtLoad.hh" #include "Xrd/XrdScheduler.hh" #include "Xrd/XrdStats.hh" #include "XrdNet/XrdNetMsg.hh" #include "XrdSys/XrdSysPlatform.hh" #include "XrdSys/XrdSysTimer.hh" /******************************************************************************/ /* S t a t i c O b j e c t s */ /******************************************************************************/ long XrdStats::tBoot = static_cast(time(0)); /******************************************************************************/ /* L o c a l C l a s s X r d S t a t s J o b */ /******************************************************************************/ class XrdStatsJob : XrdJob { public: void DoIt() {Stats->Report(); Sched->Schedule((XrdJob *)this, time(0)+iVal); } XrdStatsJob(XrdScheduler *schP, XrdStats *sP, int iV) : XrdJob("stats reporter"), Sched(schP), Stats(sP), iVal(iV) {Sched->Schedule((XrdJob *)this, time(0)+iVal);} ~XrdStatsJob() {} private: XrdScheduler *Sched; XrdStats *Stats; int iVal; }; /******************************************************************************/ /* C o n s t r c u t o r */ /******************************************************************************/ XrdStats::XrdStats(XrdSysError *eP, XrdScheduler *sP, XrdBuffManager *bP, const char *hname, int port, const char *iname, const char *pname, const char *site) { static const char *head = ""; char myBuff[1024]; XrdLog = eP; XrdSched = sP; BuffPool = bP; Hlen = sprintf(myBuff, head, hname, port, tBoot, pname, iname, static_cast(getpid()), (site ? site : "")); Head = strdup(myBuff); buff = 0; blen = 0; myHost = hname; myName = iname; myPort = port; } /******************************************************************************/ /* R e p o r t */ /******************************************************************************/ void XrdStats::Report(char **Dest, int iVal, int Opts) { static XrdNetMsg *netDest[2] = {0,0}; static int autoSync, repOpts = Opts; const char *Data; int theOpts, Dlen; // If we have dest then this is for initialization // if (Dest) // Establish up to two destinations // {if (Dest[0]) netDest[0] = new XrdNetMsg(XrdLog, Dest[0]); if (Dest[1]) netDest[1] = new XrdNetMsg(XrdLog, Dest[1]); if (!(repOpts & XRD_STATS_ALL)) repOpts |= XRD_STATS_ALL; autoSync = repOpts & XRD_STATS_SYNCA; // Get and schedule a new job to report // if (netDest[0]) new XrdStatsJob(XrdSched, this, iVal); return; } // This is a re-entry for reporting purposes, establish the sync flag // if (!autoSync || XrdSched->Active() <= 30) theOpts = repOpts; else theOpts = repOpts & ~XRD_STATS_SYNC; // Now get the statistics // statsMutex.Lock(); if ((Data = GenStats(Dlen, theOpts))) {netDest[0]->Send(Data, Dlen); if (netDest[1]) netDest[1]->Send(Data, Dlen); } statsMutex.UnLock(); } /******************************************************************************/ /* S t a t s */ /******************************************************************************/ void XrdStats::Stats(XrdStats::CallBack *cbP, int opts) { const char *info; int sz; // Lock the buffer, // statsMutex.Lock(); // Obtain the stats, if we have some, do the callback // if ((info = GenStats(sz, opts))) cbP->Info(info, sz); // Unlock the buffer // statsMutex.UnLock(); } /******************************************************************************/ /* P r i v a t e M e t h o d s */ /******************************************************************************/ /******************************************************************************/ /* G e n S t a t s */ /******************************************************************************/ const char *XrdStats::GenStats(int &rsz, int opts) // statsMutex must be locked! { static const char *sgen = "" "%d%lu%ld"; static const char *tail = ""; static const char *snul = "" ""; static const int snulsz = strlen(snul); static const int ovrhed = 256+strlen(sgen)+strlen(tail); XrdSysTimer myTimer; char *bp; int n, bl, sz, do_sync = (opts & XRD_STATS_SYNC ? 1 : 0); // If buffer is not allocated, do it now. We must defer buffer allocation // until all components that can provide statistics have been loaded // if (!(bp = buff)) {blen = InfoStats(0,0) + BuffPool->Stats(0,0) + XrdLink::Stats(0,0) + ProcStats(0,0) + XrdSched->Stats(0,0) + XrdPoll::Stats(0,0) + XrdProtLoad::Statistics(0,0) + ovrhed + Hlen; if (posix_memalign((void **)&buff, getpagesize(), blen+256)) buff = 0; if (!(bp = buff)) {rsz = snulsz; return snul;} } bl = blen; // Start the time if need be // if (opts & XRD_STATS_SGEN) myTimer.Reset(); // Insert the heading // sz = sprintf(buff, Head, static_cast(time(0))); bl -= sz; bp += sz; // Extract out the statistics, as needed // if (opts & XRD_STATS_INFO) {sz = InfoStats(bp, bl, do_sync); bp += sz; bl -= sz; } if (opts & XRD_STATS_BUFF) {sz = BuffPool->Stats(bp, bl, do_sync); bp += sz; bl -= sz; } if (opts & XRD_STATS_LINK) {sz = XrdLink::Stats(bp, bl, do_sync); bp += sz; bl -= sz; } if (opts & XRD_STATS_POLL) {sz = XrdPoll::Stats(bp, bl, do_sync); bp += sz; bl -= sz; } if (opts & XRD_STATS_PROC) {sz = ProcStats(bp, bl, do_sync); bp += sz; bl -= sz; } if (opts & XRD_STATS_PROT) {sz = XrdProtLoad::Statistics(bp, bl, do_sync); bp += sz; bl -= sz; } if (opts & XRD_STATS_SCHD) {sz = XrdSched->Stats(bp, bl, do_sync); bp += sz; bl -= sz; } if (opts & XRD_STATS_SGEN) {unsigned long totTime = 0; myTimer.Report(totTime); sz = snprintf(bp,bl,sgen,do_sync==0,totTime,static_cast(time(0))); bp += sz; bl -= sz; } sz = bp - buff; if (bl > 0) n = strlcpy(bp, tail, bl); else n = 0; rsz = sz + (n >= bl ? bl : n); return buff; } /******************************************************************************/ /* I n f o S t a t s */ /******************************************************************************/ int XrdStats::InfoStats(char *bfr, int bln, int do_sync) { static const char statfmt[] = "%s" "%d%s"; // Check if actual length wanted // if (!bfr) return sizeof(statfmt)+24 + strlen(myHost); // Format the statistics // return snprintf(bfr, bln, statfmt, myHost, myPort, myName); } /******************************************************************************/ /* P r o c S t a t s */ /******************************************************************************/ int XrdStats::ProcStats(char *bfr, int bln, int do_sync) { static const char statfmt[] = "" "%lld%lld" "%lld%lld" ""; struct rusage r_usage; long long utime_sec, utime_usec, stime_sec, stime_usec; // long long ru_maxrss, ru_majflt, ru_nswap, ru_inblock, ru_oublock; // long long ru_msgsnd, ru_msgrcv, ru_nsignals; // Check if actual length wanted // if (!bfr) return sizeof(statfmt)+16*13; // Get the statistics // if (getrusage(RUSAGE_SELF, &r_usage)) return 0; // Convert fields to correspond to the format we are using. Commented out fields // are either not uniformaly reported or are incorrectly reported making them // useless across multiple platforms. // // utime_sec = static_cast(r_usage.ru_utime.tv_sec); utime_usec = static_cast(r_usage.ru_utime.tv_usec); stime_sec = static_cast(r_usage.ru_stime.tv_sec); stime_usec = static_cast(r_usage.ru_stime.tv_usec); // ru_maxrss = static_cast(r_usage.ru_maxrss); // ru_majflt = static_cast(r_usage.ru_majflt); // ru_nswap = static_cast(r_usage.ru_nswap); // ru_inblock = static_cast(r_usage.ru_inblock); // ru_oublock = static_cast(r_usage.ru_oublock); // ru_msgsnd = static_cast(r_usage.ru_msgsnd); // ru_msgrcv = static_cast(r_usage.ru_msgrcv); // ru_nsignals = static_cast(r_usage.ru_nsignals); // Format the statistics // return snprintf(bfr, bln, statfmt, utime_sec, utime_usec, stime_sec, stime_usec // ru_maxrss, ru_majflt, ru_nswap, ru_inblock, ru_oublock, // ru_msgsnd, ru_msgrcv, ru_nsignals ); } xrootd-5.6.9/src/Xrd/XrdStats.hh000066400000000000000000000073741457266313600165230ustar00rootroot00000000000000#ifndef __XRD_STATS_H__ #define __XRD_STATS_H__ /******************************************************************************/ /* */ /* X r d S t a t s . h h */ /* */ /* (c) 2004 by the Board of Trustees of the Leland Stanford, Jr., University */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include "XrdSys/XrdSysPthread.hh" #define XRD_STATS_ALL 0x000000FF #define XRD_STATS_INFO 0x00000001 #define XRD_STATS_BUFF 0x00000002 #define XRD_STATS_LINK 0x00000004 #define XRD_STATS_POLL 0x00000008 #define XRD_STATS_PROC 0x00000010 #define XRD_STATS_PROT 0x00000020 #define XRD_STATS_SCHD 0x00000040 #define XRD_STATS_SGEN 0x00000080 #define XRD_STATS_SYNC 0x40000000 #define XRD_STATS_SYNCA 0x20000000 class XrdScheduler; class XrdBuffManager; class XrdStats { public: void Report(char **Dest=0, int iVal=600, int Opts=0); class CallBack {public: virtual void Info(const char *data, int dlen) = 0; CallBack() {} virtual ~CallBack() {} }; virtual void Stats(CallBack *InfoBack, int opts); XrdStats(XrdSysError *eP, XrdScheduler *sP, XrdBuffManager *bP, const char *hn, int port, const char *in, const char *pn, const char *sn); virtual ~XrdStats() {if (buff) free(buff);} private: const char *GenStats(int &rsz, int opts); int InfoStats(char *buff, int blen, int dosync=0); int ProcStats(char *buff, int blen, int dosync=0); static long tBoot; // Time at boot time XrdScheduler *XrdSched; XrdSysError *XrdLog; XrdBuffManager *BuffPool; XrdSysMutex statsMutex; char *buff; // Used by all callers int blen; int Hlen; char *Head; const char *myHost; const char *myName; int myPort; }; #endif xrootd-5.6.9/src/Xrd/XrdTcpMonPin.hh000066400000000000000000000115111457266313600172600ustar00rootroot00000000000000#ifndef __XRDTCPMONPIN_H__ #define __XRDTCPMONPIN_H__ /******************************************************************************/ /* */ /* X r d T c p M o n P i n . h h */ /* */ /* (c) 2020 by the Board of Trustees of the Leland Stanford, Jr., University */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ /*! This class defines the XrdLink TCP post monitoring plugin. This plugin is used to generate additional moinitoring information, as needed, when the connection is about to be closed. The plugin should use the g-stream object recorded in the passed environment. If not found, the getInstance() method should return failure as this plugin should not be loaded unless such an object exists in the environment. You get the g-stream object specific to this plugin by executing the following (assume envR is the environment): @code {.cpp} XrdXrootdGStream *gS = (XrdXrootdGStream *)envR.GetPtr("TcpMon.gStream*"); @endcode */ class XrdNetAddrInfo; class XrdTcpMonPin { public: //------------------------------------------------------------------------------ //! Produce monitoring information upon connection termination. //! //! @param netInfo Reference to the network object associated with link. //! @param lnkInfo Reference to link-specific information. //! @param liLen Byte length of lnkInfo being passed. //------------------------------------------------------------------------------ struct LinkInfo {const char *tident; //!< Pointer to the client's trace identifier int fd; //!< Socket file descriptor int consec; //!< Seconds connected long long bytesIn; //!< Bytes read from the socket long long bytesOut; //!< Bytes written to the socket }; virtual void Monitor(XrdNetAddrInfo &netInfo, LinkInfo &lnkInfo, int liLen) = 0; XrdTcpMonPin() {} virtual ~XrdTcpMonPin() {} }; /*! An instance of the plugin is obtained by the plugin manager using the XrdOucPinObject class. The most straightforward way to implement this is to inherit the XrdOucPinObject class by a class of your choosing that defines a file level object named TcpMonPin, as follows: class myPinObject : public XrdOucPinObject {public: XrdTcpMonPin *getInstance(...) {provide concrete implementation} } TcpMonPin; see XrdOucPinObject.hh for additional details and the definition of the getInstance() method. There are many other ways to accomplish this including inheriting this class along with the XrdTcpMonPin class by the implementation class. You should also specify the compilation version. That is, the XRootD version you used to compile your plug-in. Declare it as: #include "XrdVersion.hh" XrdVERSIONINFO(TcpMonPin,); where is a 1- to 15-character unquoted name identifying your plugin. */ #endif xrootd-5.6.9/src/Xrd/XrdTrace.hh000066400000000000000000000063331457266313600164550ustar00rootroot00000000000000#ifndef _XRD_TRACE_H #define _XRD_TRACE_H /******************************************************************************/ /* */ /* X r d T r a c e . h h */ /* */ /* (C) 2004 by the Board of Trustees of the Leland Stanford, Jr., University */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Deprtment of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ // Trace flags // #define TRACE_NONE 0x0000 #define TRACE_ALL 0x0fff #define TRACE_DEBUG 0x0001 #define TRACE_CONN 0x0002 #define TRACE_MEM 0x0004 #define TRACE_NET 0x0008 #define TRACE_POLL 0x0010 #define TRACE_PROT 0x0020 #define TRACE_SCHED 0x0040 #define TRACE_TLS 0x0500 #define TRACE_TLSCTX 0x0100 #define TRACE_TLSSIO 0x0200 #define TRACE_TLSSOK 0x0400 #ifndef NODEBUG #include "XrdSys/XrdSysHeaders.hh" #include "XrdSys/XrdSysTrace.hh" namespace XrdGlobal { extern XrdSysTrace XrdTrace; } #ifndef XRD_TRACE #define XRD_TRACE XrdGlobal::XrdTrace. #endif #define TRACE(act, x) \ if (XRD_TRACE What & TRACE_ ## act) {SYSTRACE(XRD_TRACE, 0, TraceID, 0, x)} #define TRACEI(act, x) \ if (XRD_TRACE What & TRACE_ ## act) \ {SYSTRACE(XRD_TRACE, TRACE_IDENT, TraceID, 0, x)} #define TRACING(x) XRD_TRACE What & x #else #define TRACE(act,x) #define TRACEI(act,x) #define TRACING(x) 0 #endif #endif xrootd-5.6.9/src/XrdAcc/000077500000000000000000000000001457266313600150225ustar00rootroot00000000000000xrootd-5.6.9/src/XrdAcc/XrdAccAccess.cc000066400000000000000000000434011457266313600176210ustar00rootroot00000000000000/******************************************************************************/ /* */ /* X r d A c c A c c e s s . c c */ /* */ /* (c) 2003 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include #include #include #include "XrdVersion.hh" #include "XrdAcc/XrdAccAccess.hh" #include "XrdAcc/XrdAccEntity.hh" #include "XrdAcc/XrdAccCapability.hh" #include "XrdAcc/XrdAccConfig.hh" #include "XrdAcc/XrdAccGroups.hh" #include "XrdNet/XrdNetAddrInfo.hh" #include "XrdOuc/XrdOucUtils.hh" #include "XrdSec/XrdSecEntityAttr.hh" #include "XrdSys/XrdSysPlugin.hh" /******************************************************************************/ /* E x t e r n a l R e f e r e n c e s */ /******************************************************************************/ extern unsigned long XrdOucHashVal2(const char *KeyVal, int KeyLen); /******************************************************************************/ /* G l o b a l C o n f i g u r a t i o n O b j e c t */ /******************************************************************************/ extern XrdAccConfig XrdAccConfiguration; /******************************************************************************/ /* Autorization Object Creation via XrdAccDefaultAuthorizeObject */ /******************************************************************************/ XrdAccAuthorize *XrdAccDefaultAuthorizeObject(XrdSysLogger *lp, const char *cfn, const char *parm, XrdVersionInfo &urVer) { static XrdVERSIONINFODEF(myVer, XrdAcc, XrdVNUMBER, XrdVERSION); static XrdSysError Eroute(lp, "acc_"); // Verify version compatibility // if (urVer.vNum != myVer.vNum && !XrdSysPlugin::VerCmp(urVer,myVer)) return 0; // Configure the authorization system // if (XrdAccConfiguration.Configure(Eroute, cfn)) return (XrdAccAuthorize *)0; // Set error object pointer // XrdAccEntity::setError(&Eroute); // All is well, return the actual pointer to the object // return (XrdAccAuthorize *)XrdAccConfiguration.Authorization; } /******************************************************************************/ /* C o n s t r u c t o r */ /******************************************************************************/ XrdAccAccess::XrdAccAccess(XrdSysError *erp) { // Get the audit option that we should use // Auditor = XrdAccAuditObject(erp); } /******************************************************************************/ /* A c c e s s */ /******************************************************************************/ XrdAccPrivs XrdAccAccess::Access(const XrdSecEntity *Entity, const char *path, const Access_Operation oper, XrdOucEnv *Env) { XrdAccGroupList *glp; XrdAccPrivCaps caps; XrdAccCapability *cp; XrdAccEntity *aeP; XrdAccEntityInfo eInfo; int plen = strlen(path); long phash = XrdOucHashVal2(path, plen); bool isuser; // Obtain an authorization entity (it will be released upon return). // XrdAccEntityInit accEntity(Entity, aeP); if (!aeP) return Access(caps, Entity, path, oper); // Setup the id (we don't need a lock for that) // std::string username; auto got_token = Entity->eaAPI->Get("request.name", username); if (got_token && !username.empty()) {eInfo.name = username.c_str(); isuser = true; } else if (Entity->name) {eInfo.name = Entity->name; isuser = (*eInfo.name != 0); } else { eInfo.name = "*"; isuser = false; } // Get a shared context for these potentially long running routines // Access_Context.Lock(xs_Shared); // Setup the host entry in the eInfo structure (it may need to be resolved) // eInfo.host = (hostRefX ? Resolve(Entity) : "?"); // Run through the exclusive list first as only one rule will apply // if (Atab.SXList) {XrdAccAccess_ID *xlP = Atab.SXList; do {int aSeq = 0; while(aeP->Next(aSeq, eInfo)) {if (xlP->Applies(eInfo)) {xlP->caps->Privs(caps, path, plen, phash); Access_Context.UnLock(xs_Shared); return Access(caps, Entity, path, oper); } } xlP = xlP->next; } while(xlP); } // Check if we really need to resolve the host name // //??? if (Atab.D_List || Atab.H_Hash || Atab.N_Hash) host = Resolve(Entity); if (!hostRefX && hostRefY) eInfo.host = Resolve(Entity); // Establish default privileges // if (Atab.Z_List) Atab.Z_List->Privs(caps, path, plen, phash); // Next add in the host domain privileges // if (Atab.D_List && (cp = Atab.D_List->Find(eInfo.host))) cp->Privs(caps, path, plen, phash); // Next add in the host-specific privileges // if (Atab.H_Hash && (cp = Atab.H_Hash->Find(eInfo.host))) cp->Privs(caps, path, plen, phash); // Now add in the netgroup privileges // if (Atab.N_Hash && *eInfo.host != '?' && (glp = XrdAccConfiguration.GroupMaster.NetGroups(eInfo.name,eInfo.host))) {char *gname; while((gname = (char *)glp->Next())) if ((cp = Atab.N_Hash->Find((const char *)gname))) cp->Privs(caps, path, plen, phash); delete glp; } // Check for user fungible privileges // if (isuser && Atab.X_List) Atab.X_List->Privs(caps, path, plen, phash, eInfo.name); // Add in specific user privileges // if (isuser && Atab.U_Hash && (cp = Atab.U_Hash->Find(eInfo.name))) cp->Privs(caps, path, plen, phash); // The following privileges are based on multiple attributes. Orgs and roles // may be repeated but groups generally will not be. // const char *vorgPrev = 0, *rolePrev = 0; int aSeq = 0; while(aeP->Next(aSeq, eInfo)) { // Add in the group privileges. // if (Atab.G_Hash && eInfo.grup && (cp = Atab.G_Hash->Find(eInfo.grup))) cp->Privs(caps, path, plen, phash); // Add in the org-specific privileges // if (Atab.O_Hash && eInfo.vorg && eInfo.vorg != vorgPrev) {vorgPrev = eInfo.vorg; if ((cp = Atab.O_Hash->Find(eInfo.vorg))) cp->Privs(caps, path, plen, phash); } // Add in the role-specific privileges // if (Atab.R_Hash && eInfo.role && eInfo.role != rolePrev) {rolePrev = eInfo.role; if ((cp = Atab.R_Hash->Find(eInfo.role))) cp->Privs(caps, path, plen, phash); } // Finally run through the inclusive list and apply all relevant rules // XrdAccAccess_ID *ylP = Atab.SYList; while (ylP) {if (ylP->Applies(eInfo)) ylP->caps->Privs(caps, path, plen, phash); ylP = ylP->next; } } // We are now done with looking at changeable data // Access_Context.UnLock(xs_Shared); // Return the privileges as needed // return Access(caps, Entity, path, oper); } /******************************************************************************/ XrdAccPrivs XrdAccAccess::Access( XrdAccPrivCaps &caps, const XrdSecEntity *Entity, const char *path, const Access_Operation oper ) { XrdAccPrivs myprivs; XrdAccAudit_Options audits = (XrdAccAudit_Options)Auditor->Auditing(); int accok; // Compute composite privileges and see if privs need to be returned // myprivs = (XrdAccPrivs)(caps.pprivs & ~caps.nprivs); if (!oper) return (XrdAccPrivs)myprivs; // Check if auditing is enabled or whether we can do a fastaroo test // if (!audits) return (XrdAccPrivs)Test(myprivs, oper); if ((accok = Test(myprivs, oper)) && !(audits & audit_grant)) return (XrdAccPrivs)accok; // Call the auditing routine and exit // return (XrdAccPrivs)Audit(accok, Entity, path, oper); } /******************************************************************************/ /* A u d i t */ /******************************************************************************/ int XrdAccAccess::Audit(const int accok, const XrdSecEntity *Entity, const char *path, const Access_Operation oper, XrdOucEnv *Env) { // Warning! This table must be in 1-to-1 correspondence with Access_Operation // static const char *Opername[] = {"any", // 0 "chmod", // 1 "chown", // 2 "create", // 3 "delete", // 4 "insert", // 5 "lock", // 6 "mkdir", // 7 "read", // 8 "readdir", // 9 "rename", // 10 "stat", // 11 "update", // 12 "excl_create", // 13 "excl_insert" // 14 }; const char *opname = (oper > AOP_LastOp ? "???" : Opername[oper]); std::string username; const char *id = "*"; auto got_token = Entity->eaAPI->Get("request.name", username); if (got_token && !username.empty()) { id = username.c_str(); } else if (Entity->name) id = Entity->name; const char *host = (Entity->host ? (const char *)Entity->host : "?"); char atype[XrdSecPROTOIDSIZE+1]; // Get the protocol type in a printable format // strncpy(atype, Entity->prot, XrdSecPROTOIDSIZE); atype[XrdSecPROTOIDSIZE] = '\0'; // Route the message appropriately // if (accok) Auditor->Grant(opname, Entity->tident, atype, id, host, path); else Auditor->Deny( opname, Entity->tident, atype, id, host, path); // All done, finally // return accok; } /******************************************************************************/ /* R e s o l v e */ /******************************************************************************/ const char *XrdAccAccess::Resolve(const XrdSecEntity *Entity) { // Make a quick test for IPv6 (as that's the future) and a minimal one for ipV4 // to see if we have to do a DNS lookup. // if (Entity->host == 0 || *(Entity->host) == '[' || isdigit(*(Entity->host))) return Entity->addrInfo->Name("?"); return Entity->host; } /******************************************************************************/ /* S w a p T a b s */ /******************************************************************************/ #define XrdAccSWAP(x) oldtab.x = Atab.x; Atab.x = newtab.x; \ newtab.x = oldtab.x; oldtab.x = 0; void XrdAccAccess::SwapTabs(struct XrdAccAccess_Tables &newtab) { struct XrdAccAccess_Tables oldtab; bool hRefX = false, hRefY = false; // Determine if we need to resolve the host name early // XrdAccAccess_ID *xlP = newtab.SXList; while(xlP) {if (xlP->host) {hRefX = true; break;} xlP = xlP->next; } // Determine if we need to resolve the hostname at all. // if (!hRefX) {if (newtab.D_List || newtab.H_Hash || newtab.N_Hash) hRefY = true; else {XrdAccAccess_ID *ylP = newtab.SYList; while (ylP) {if (ylP->host) {hRefY = true; break;} ylP = ylP->next; } } } // Get an exclusive context to change the table pointers // Access_Context.Lock(xs_Exclusive); // Save the old pointer while replacing it with the new pointer // XrdAccSWAP(D_List); XrdAccSWAP(E_List); XrdAccSWAP(G_Hash); XrdAccSWAP(H_Hash); XrdAccSWAP(N_Hash); XrdAccSWAP(O_Hash); XrdAccSWAP(R_Hash); XrdAccSWAP(S_Hash); XrdAccSWAP(T_Hash); XrdAccSWAP(U_Hash); XrdAccSWAP(X_List); XrdAccSWAP(Z_List); XrdAccSWAP(SXList); XrdAccSWAP(SYList); hostRefX = hRefX; hostRefY = hRefY; // When we set new access tables, we should purge the group cache // XrdAccConfiguration.GroupMaster.PurgeCache(); // We can now let loose new table searchers // Access_Context.UnLock(xs_Exclusive); } /******************************************************************************/ /* T e s t */ /******************************************************************************/ int XrdAccAccess::Test(const XrdAccPrivs priv,const Access_Operation oper) { // Warning! This table must be in 1-to-1 correspondence with Access_Operation // static XrdAccPrivs need[] = {XrdAccPriv_None, // 0 XrdAccPriv_Chmod, // 1 XrdAccPriv_Chown, // 2 XrdAccPriv_Create, // 3 XrdAccPriv_Delete, // 4 XrdAccPriv_Insert, // 5 XrdAccPriv_Lock, // 6 XrdAccPriv_Mkdir, // 7 XrdAccPriv_Read, // 8 XrdAccPriv_Readdir, // 9 XrdAccPriv_Rename, // 10 XrdAccPriv_Lookup, // 11 XrdAccPriv_Update, // 12 (XrdAccPrivs)0xffff, // 13 (XrdAccPrivs)0xffff // 14 }; // Note AOP_Excl* does not have a corresponding XrdAccPrivs; this is on // purpose as the Excl* privilege is not modelled within the AuditDB framework. if (oper < 0 || oper > AOP_LastOp) return 0; return (int)(need[oper] & priv) == need[oper]; } /******************************************************************************/ /* X r d A c c A c c e s s _ I D : : A p p l i e s */ /******************************************************************************/ bool XrdAccAccess_ID::Applies(const XrdAccEntityInfo &Entity) { // Check single value items in the most probable use order // if (org && (!Entity.vorg || strcmp(org, Entity.vorg))) return false; if (role && (!Entity.role || strcmp(role, Entity.role))) return false; if (grp && (!Entity.grup || strcmp(grp, Entity.grup))) return false; if (user && (!Entity.name || strcmp(user, Entity.name))) return false; // The check is more complicated as the host field may be a domain. // if (host) {const char *hName; if (*host == '.') {int eLen = strlen(Entity.host); if (eLen <= hlen) return false; hName = Entity.host + eLen - hlen; } else hName = Entity.host; if (strcmp(host, hName)) return false; } // All done, this rules applies! // return true; } xrootd-5.6.9/src/XrdAcc/XrdAccAccess.hh000066400000000000000000000175721457266313600176450ustar00rootroot00000000000000#ifndef __ACC_ACCESS__ #define __ACC_ACCESS__ /******************************************************************************/ /* */ /* X r d A c c A c c e s s . h h */ /* */ /* (c) 2003 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include "XrdAcc/XrdAccAudit.hh" #include "XrdAcc/XrdAccAuthorize.hh" #include "XrdAcc/XrdAccCapability.hh" #include "XrdSec/XrdSecEntity.hh" #include "XrdOuc/XrdOucHash.hh" #include "XrdSys/XrdSysXSLock.hh" #include "XrdSys/XrdSysPlatform.hh" /******************************************************************************/ /* S e t T a b s P a r a m e t e r */ /******************************************************************************/ struct XrdAccEntityInfo; struct XrdAccAccess_ID {char *name; char *grp; char *host; char *org; char *role; char *user; XrdAccCapability *caps; XrdAccAccess_ID *next; int rule; short hlen; short glen; bool Applies(const XrdAccEntityInfo &Entity); XrdAccAccess_ID *Export() {XrdAccAccess_ID *xID; xID = new XrdAccAccess_ID; *xID = *this; name = grp = host = org = role = user = 0; caps = 0; return xID; } XrdAccAccess_ID(const char *Name=0) : name(Name ? strdup(Name) : 0), grp(0), host(0), org(0), role(0), user(0), caps(0), next(0), rule(0), hlen(0), glen(0) {} ~XrdAccAccess_ID() {if (name) free(name); if (grp) free(grp); if (host) free(host); if (org) free(org); if (role) free(role); if (user) free(user); if (caps) delete caps; } }; struct XrdAccAccess_Tables {XrdOucHash *G_Hash; // Groups XrdOucHash *H_Hash; // Hosts XrdOucHash *N_Hash; // Netgroups XrdOucHash *O_Hash; // Organizations XrdOucHash *R_Hash; // Roles XrdOucHash *S_Hash; // Sets XrdOucHash *T_Hash; // Templates XrdOucHash *U_Hash; // Users XrdAccCapName *D_List; // Domains XrdAccCapName *E_List; // Domains (end of list) XrdAccCapability *X_List; // Fungable capbailities XrdAccCapability *Z_List; // Default capbailities XrdAccAccess_ID *SXList; // 's' exclusive list XrdAccAccess_ID *SYList; // 's' inclusive list XrdAccAccess_Tables() {G_Hash = 0; H_Hash = 0; N_Hash = 0; O_Hash = 0; R_Hash = 0; S_Hash = 0; T_Hash = 0; U_Hash = 0; D_List = 0; E_List = 0; X_List = 0; Z_List = 0; SXList = 0; SYList = 0; } ~XrdAccAccess_Tables() {if (G_Hash) delete G_Hash; if (H_Hash) delete H_Hash; if (N_Hash) delete N_Hash; if (O_Hash) delete O_Hash; if (R_Hash) delete R_Hash; if (S_Hash) delete S_Hash; //Deletes SX & SYList if (T_Hash) delete T_Hash; if (U_Hash) delete U_Hash; if (X_List) delete X_List; if (Z_List) delete Z_List; } }; /******************************************************************************/ /* X r d A c c A c c e s s */ /******************************************************************************/ class xrdOucError; class XrdAccAccess : public XrdAccAuthorize { public: friend class XrdAccConfig; XrdAccPrivs Access(const XrdSecEntity *Entity, const char *path, const Access_Operation oper, XrdOucEnv *Env=0); int Audit(const int accok, const XrdSecEntity *Entity, const char *path, const Access_Operation oper, XrdOucEnv *Env=0); static const char *Resolve(const XrdSecEntity *Entity); // SwapTabs() is used by the configuration object to establish new access // control tables. It may be called whenever the tables change. // void SwapTabs(struct XrdAccAccess_Tables &newtab); int Test(const XrdAccPrivs priv, const Access_Operation oper); XrdAccAccess(XrdSysError *erp); ~XrdAccAccess() {} // The access object is never deleted private: XrdAccPrivs Access( XrdAccPrivCaps &caps, const XrdSecEntity *Entity, const char *path, const Access_Operation oper); struct XrdAccAccess_Tables Atab; bool hostRefX; // True if we need to resolve hostname for exclusive rules bool hostRefY; // True if we need to resolve hostname for any other rules XrdSysXSLock Access_Context; XrdAccAudit *Auditor; }; #endif xrootd-5.6.9/src/XrdAcc/XrdAccAudit.cc000066400000000000000000000111761457266313600174720ustar00rootroot00000000000000/******************************************************************************/ /* */ /* X r d A c c A u d i t . c c */ /* */ /* (c) 2003 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include #include "XrdAcc/XrdAccAudit.hh" #include "XrdSys/XrdSysError.hh" /******************************************************************************/ /* C o n s t r u c t o r */ /******************************************************************************/ XrdAccAudit::XrdAccAudit(XrdSysError *erp) { // Set default // auditops = audit_none; mDest = erp; } /******************************************************************************/ /* D e n y */ /******************************************************************************/ void XrdAccAudit::Deny(const char *opname, const char *tident, const char *atype, const char *id, const char *host, const char *path) {if (auditops & audit_deny) {char buff[2048]; snprintf(buff, sizeof(buff)-1, "%s deny %s %s@%s %s %s", (tident ? tident : ""), atype, id, host, opname, path); buff[sizeof(buff)-1] = '\0'; mDest->Emsg("Audit", buff); } } /******************************************************************************/ /* G r a n t */ /******************************************************************************/ void XrdAccAudit::Grant(const char *opname, const char *tident, const char *atype, const char *id, const char *host, const char *path) {if (auditops & audit_deny) {char buff[2048]; snprintf(buff, sizeof(buff)-1, "%s grant %s %s@%s %s %s", (tident ? tident : ""), atype, id, host, opname, path); buff[sizeof(buff)-1] = '\0'; mDest->Emsg("Audit", buff); } } /******************************************************************************/ /* A u d i t O b j e c t G e n e r a t o r */ /******************************************************************************/ XrdAccAudit *XrdAccAuditObject(XrdSysError *erp) { static XrdAccAudit AuditObject(erp); // Simply return the default audit object // return &AuditObject; } xrootd-5.6.9/src/XrdAcc/XrdAccAudit.hh000066400000000000000000000122171457266313600175010ustar00rootroot00000000000000#ifndef __ACC_AUDIT__ #define __ACC_AUDIT__ /******************************************************************************/ /* */ /* X r d A c c A u d i t . h h */ /* */ /* (c) 2003 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ /******************************************************************************/ /* A u d i t _ O p t i o n s */ /******************************************************************************/ enum XrdAccAudit_Options {audit_none = 0, audit_deny = 1, audit_grant = 2, audit_all = 3 }; /******************************************************************************/ /* X r d A c c A u d i t */ /******************************************************************************/ // This class is really meant to be replaced by anyone who care about auditing. // Effective auditing is required to meet DOD class C security requirments. // This class should be placed in a shared library so that an installation can // easily replace it and routine auditsdits as needed. We supply a brain-dead // audit that simply issues a message: // deny // yymmdd hh:mm:ss acc_Audit: grant atype id@host opername path // Enabling/disabling is done via the method setAudit(). // The external routine XrdAccAuditObject() returns the real audit object // used by Access(). Developers should derive a class from this class and // return the object of there choosing up-cast to this object. See the // routine XrdAccAudit.C for the particulars. class XrdSysError; class XrdAccAudit { public: int Auditing(const XrdAccAudit_Options ops=audit_all) {return auditops & ops;} virtual void Deny(const char *opname, const char *tident, const char *atype, const char *id, const char *host, const char *path); virtual void Grant(const char *opname, const char *tident, const char *atype, const char *id, const char *host, const char *path); // setAudit() is used to set the auditing options: audit_none turns audit off // (the default), audit_deny audit access denials, audit_grant audits access // grants, and audit_all audits both. See XrdAccAudit.h for more information. // void setAudit(XrdAccAudit_Options aops) {auditops = aops;} XrdAccAudit(XrdSysError *erp); virtual ~XrdAccAudit() {} private: XrdAccAudit_Options auditops; XrdSysError *mDest; }; /******************************************************************************/ /* o o a c c _ A u d i t _ O b j e c t */ /******************************************************************************/ extern XrdAccAudit *XrdAccAuditObject(XrdSysError *erp); #endif xrootd-5.6.9/src/XrdAcc/XrdAccAuthDB.hh000066400000000000000000000122121457266313600175350ustar00rootroot00000000000000#ifndef __ACC_AUTHDB__ #define __ACC_AUTHDB__ /******************************************************************************/ /* */ /* X r d A c c A u t h D B . h h */ /* */ /* (c) 2003 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include "XrdSys/XrdSysError.hh" // This class is provided for obtaining capability information from some source. // Derive a class to provide an actual source for the information. The // interface is similar to the set/get/endpwent enumeration interface: // setDBpath() is used to establish the location of the database. // Open() establishes the start of the database operation. It also obtains // an exclusive mutex to be mt-safe. True is returned upon success. // getRec() get the next database record. It returns the record type as well // as a pointer to the record name. False is returned at the end // of the database. // getPP() gets the next path-priv or template name. It returns a pointer // to each one. True is returned until end-of-record. // Close() terminates database processing and releases the associated lock. // It also return FALSE if any errors occurred during processing. // Changed() Returns 1 id the current authorization file has changed since // the last time it was opened. /******************************************************************************/ /* D a t a b a s e R e c o r d T y p e s */ /******************************************************************************/ // The following are the 1-letter id types that we support // // g -> unix group name // h -> host name // n -> NIS netgroup name // s -> set name // t -> template name // u -> user name // The syntax for each database record is: // {| } [{ }] [...] // = [ [....]] // Continuation records are signified by an ending backslash (\). Blank records // and comments (i.e., lines with the first non-blank being a pound sign) are // allowed. Word separators may be spaces or tabs. /******************************************************************************/ /* X r d A c c A u t h D B C l a s s */ /******************************************************************************/ class XrdAccAuthDB { public: virtual int Open(XrdSysError &eroute, const char *path=0) = 0; virtual char getRec(char **recname) = 0; virtual char getID(char **id) = 0; virtual int getPP(char **path, char **priv, bool &istmplt) = 0; virtual int Close() = 0; virtual int Changed(const char *path=0) = 0; XrdAccAuthDB() {} virtual ~XrdAccAuthDB() {} }; /******************************************************************************/ /* X r d A c c X u t h D B _ O b j e c t */ /******************************************************************************/ extern XrdAccAuthDB *XrdAccAuthDBObject(XrdSysError *erp); #endif xrootd-5.6.9/src/XrdAcc/XrdAccAuthFile.cc000066400000000000000000000314071457266313600201240ustar00rootroot00000000000000/******************************************************************************/ /* */ /* X r d A c c A u t h F i l e . c c */ /* */ /* (c) 2003 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include #include #include #include #include "XrdAcc/XrdAccAuthFile.hh" /******************************************************************************/ /* X r d A c c A u t h D B _ O b j e c t */ /******************************************************************************/ XrdAccAuthDB *XrdAccAuthDBObject(XrdSysError *erp) { static XrdAccAuthFile mydatabase(erp); return (XrdAccAuthDB *)&mydatabase; } /******************************************************************************/ /* C o n s t r u c t o r */ /******************************************************************************/ XrdAccAuthFile::XrdAccAuthFile(XrdSysError *erp) { // Set starting values // authfn = 0; flags = Noflags; modtime = 0; Eroute = erp; // Setup for an error in the first record // strcpy(path_buff, "start of file"); } /******************************************************************************/ /* D e s t r u c t o r */ /******************************************************************************/ XrdAccAuthFile::~XrdAccAuthFile() { // If the file is open, close it // if (flags &isOpen) Close(); // Free the authfn string // if (authfn) free(authfn); } /******************************************************************************/ /* C h a n g e d */ /******************************************************************************/ int XrdAccAuthFile::Changed(const char *dbfn) { struct stat statbuff; // If no file here, indicate nothing changed // if (!authfn || !*authfn) return 0; // If file paths differ, indicate that something has changed // if (dbfn && strcmp(dbfn, authfn)) return 1; // Get the modification timestamp for this file // if (stat(authfn, &statbuff)) {Eroute->Emsg("AuthFile", errno, "find", authfn); return 0; } // Indicate whether or not the file has changed // return (modtime < statbuff.st_mtime); } /******************************************************************************/ /* C l o s e */ /******************************************************************************/ int XrdAccAuthFile::Close() { // Return is the file is not open // if (!(flags & isOpen)) return 1; // Close the stream // DBfile.Close(); // Unlock the protecting mutex // DBcontext.UnLock(); // Indicate file is no longer open // flags = (DBflags)(flags & ~isOpen); // Return indicator of whether we had any errors // if (flags & dbError) return 0; return 1; } /******************************************************************************/ /* g e t I D */ /******************************************************************************/ char XrdAccAuthFile::getID(char **id) { char *pp, idcode[2] = {0,0}; // If a record has not been read, return end of record (i.e., 0) // if (!(flags & inRec)) return 0; // Read the next word from the record (if none, simulate end of record) // if (!(pp = DBfile.GetWord())) {flags = (DBflags)(flags & ~inRec); return 0; } // Id's are of the form 'c', but historically they were 'c:' so we accept a // two character specification but only validate the first to be backward // compatible. // if (strlen(pp) > 2 || !index("ghoru", *pp)) {Eroute->Emsg("AuthFile", "Invalid ID sprecifier -", pp); flags = (DBflags)(flags | dbError); return 0; } idcode[0] = *pp; // Now get the actual id associated with it // if (!(pp = DBfile.GetWord())) {flags = (DBflags)(flags & ~inRec); Eroute->Emsg("AuthFile", "ID value missing after", idcode); flags = (DBflags)(flags | dbError); return 0; } // Copy the value since the stream buffer might get overlaid. // Copy(path_buff, pp, sizeof(path_buff)-1); // Return result // *id = path_buff; return idcode[0]; } /******************************************************************************/ /* g e t P P */ /******************************************************************************/ int XrdAccAuthFile::getPP(char **path, char **priv, bool &istmplt) { // char *pp, *bp; char *pp; // If a record has not been read, return end of record (i.e., 0) // if (!(flags & inRec)) return 0; // read the next word from the record (if none, simulate end of record) // if (!(pp = DBfile.GetWord())) {flags = (DBflags)(flags & ~inRec); return 0; } // Check of objectid specification // istmplt = false; *path = path_buff; if (*pp == '\\') {if (*(pp+1)) pp++; else {Eroute->Emsg("AuthFile", "Object ID missing after '\\'"); *path = 0; flags = (DBflags)(flags | dbError); } } else if (*pp != '/') istmplt = true; // Copy the value since the stream buffer might get overlaid. // // bp = Copy(path_buff, pp, sizeof(path_buff)-1); if (path) Copy(path_buff, pp, sizeof(path_buff)-1); // Check if this is really a path or a template // if (istmplt) {*priv = (char *)0; return 1;} // Verify that the path ends correctly (normally we would force a slash to // appear at the end but that prevents caps on files. So, we commented the // code out until we decide that maybe we really need to do this, sigh. // // bp--; // if (*bp != '/') {bp++; *bp = '/'; bp++; *bp = '\0';} // Get the next word which should be the privilege string // if (!(pp = DBfile.GetWord())) {flags = (DBflags)(flags & ~inRec); Eroute->Emsg("AuthFile", "Privileges missing after", path_buff); flags = (DBflags)(flags | dbError); *priv = (char *)0; return 0; } // All done here // *priv = pp; return 1; } /******************************************************************************/ /* g e t R e c */ /******************************************************************************/ char XrdAccAuthFile::getRec(char **recname) { char *pp; int idok; // Do this until we get a vlaid record // while(1) { // If we arer still in the middle of a record, flush it // if (flags & inRec) while(DBfile.GetWord()) {} else flags = (DBflags)(flags | inRec); // Get the next word, the record type // if (!(pp = DBfile.GetWord())) {*recname = (char *)0; return '\0';} // Verify the id-type // idok = 0; if (strlen(pp) == 1) switch(*pp) {case 'g': case 'h': case 's': case 'n': case 'o': case 'r': case 't': case 'u': case 'x': case '=': idok = 1; break; default: break; } // Check if the record type was valid // if (!idok) {Eroute->Emsg("AuthFile", "Invalid id type -", pp); flags = (DBflags)(flags | dbError); continue; } rectype = *pp; // Get the record name. It must exist // if (!(pp = DBfile.GetWord())) {Eroute->Emsg("AuthFile","Record name is missing after",path_buff); flags = (DBflags)(flags | dbError); continue; } // Copy the record name // Copy(recname_buff, pp, sizeof(recname_buff)); *recname = recname_buff; return rectype; } return '\0'; // Keep the compiler happy :-) } /******************************************************************************/ /* O p e n */ /******************************************************************************/ int XrdAccAuthFile::Open(XrdSysError &eroute, const char *path) { struct stat statbuff; int authFD; // Enter the DB context (serialize use of this database) // DBcontext.Lock(); Eroute = &eroute; // Use whichever path is the more recent // if (path) {if (authfn) free(authfn); authfn = strdup(path);} if( !authfn || !*authfn) return Bail(0, "Authorization file not specified."); // Get the modification timestamp for this file // if (stat(authfn, &statbuff)) return Bail(errno, "find", authfn); // Try to open the authorization file. // if ( (authFD = open(authfn, O_RDONLY, 0)) < 0) return Bail(errno,"open authorization file",authfn); // Copy in all the relevant information // modtime = statbuff.st_mtime; flags = isOpen; DBfile.SetEroute(Eroute); DBfile.Tabs(0); // Attach the file to the stream // if (DBfile.Attach(authFD)) return Bail(DBfile.LastError(), "initialize stream for", authfn); return 1; } /******************************************************************************/ /* P r i v a t e M e t h o d s */ /******************************************************************************/ /******************************************************************************/ /* B a i l */ /******************************************************************************/ int XrdAccAuthFile::Bail(int retc, const char *txt1, const char *txt2) { // This routine is typically used by open and the DBcontext lock must be held // flags = (DBflags)(flags & ~isOpen); DBcontext.UnLock(); if (retc) Eroute->Emsg("AuthFile", retc, txt1, txt2); else Eroute->Emsg("AuthFile", txt1, txt2); return 0; } /******************************************************************************/ /* C o p y */ /******************************************************************************/ // This routine is used instead of strncpy because, frankly, it's a lot smarter char *XrdAccAuthFile::Copy(char *dp, char *sp, int dplen) { // Copy one less that the size of the buffer so that we have room for null // while(--dplen && *sp) {*dp = *sp; dp++; sp++;} // Insert a null character and return a pointer to it. // *dp = '\0'; return dp; } xrootd-5.6.9/src/XrdAcc/XrdAccAuthFile.hh000066400000000000000000000070031457266313600201310ustar00rootroot00000000000000#ifndef __ACC_AUTHFILE__ #define __ACC_AUTHFILE__ /******************************************************************************/ /* */ /* X r d A c c A u t h F i l e . h h */ /* */ /* (c) 2003 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include #include #include "XrdSys/XrdSysError.hh" #include "XrdSys/XrdSysPlatform.hh" #include "XrdSys/XrdSysPthread.hh" #include "XrdOuc/XrdOucStream.hh" #include "XrdAcc/XrdAccAuthDB.hh" // This class is provided for obtaining capability information from a file. // class XrdAccAuthFile : public XrdAccAuthDB { public: int Open(XrdSysError &eroute, const char *path=0); char getRec(char **recname); char getID(char **id); int getPP(char **path, char **priv, bool &istmplt); int Close(); int Changed(const char *dbpath); XrdAccAuthFile(XrdSysError *erp); ~XrdAccAuthFile(); private: int Bail(int retc, const char *txt1, const char *txt2=0); char *Copy(char *dp, char *sp, int dplen); enum DBflags {Noflags=0, inRec=1, isOpen=2, dbError=4}; // Values combined XrdSysError *Eroute; DBflags flags; XrdOucStream DBfile; char *authfn; char rectype; time_t modtime; XrdSysMutex DBcontext; char recname_buff[MAXHOSTNAMELEN+1]; // Max record name by default char path_buff[MAXPATHLEN+2]; // Max path name }; #endif xrootd-5.6.9/src/XrdAcc/XrdAccAuthorize.hh000066400000000000000000000310761457266313600204110ustar00rootroot00000000000000#ifndef __ACC_AUTHORIZE__ #define __ACC_AUTHORIZE__ /******************************************************************************/ /* */ /* X r d A c c A u t h o r i z e . h h */ /* */ /* (c) 2000 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include "XrdAcc/XrdAccPrivs.hh" /******************************************************************************/ /* A c c e s s _ O p e r a t i o n */ /******************************************************************************/ //! The following are supported operations enum Access_Operation {AOP_Any = 0, //!< Special for getting privs AOP_Chmod = 1, //!< chmod() AOP_Chown = 2, //!< chown() AOP_Create = 3, //!< open() with create AOP_Delete = 4, //!< rm() or rmdir() AOP_Insert = 5, //!< mv() for target AOP_Lock = 6, //!< n/a AOP_Mkdir = 7, //!< mkdir() AOP_Read = 8, //!< open() r/o, prepare() AOP_Readdir = 9, //!< opendir() AOP_Rename = 10, //!< mv() for source AOP_Stat = 11, //!< exists(), stat() AOP_Update = 12, //!< open() r/w or append AOP_Excl_Create = 13, //!< open() with O_EXCL|O_CREAT AOP_Excl_Insert = 14, //!< mv() where destination doesn't exist. AOP_LastOp = 14 // For limits testing }; /******************************************************************************/ /* X r d A c c A u t h o r i z e */ /******************************************************************************/ class XrdOucEnv; class XrdSecEntity; class XrdSysLogger; class XrdAccAuthorize { public: //------------------------------------------------------------------------------ //! Check whether or not the client is permitted specified access to a path. //! //! @param Entity -> Authentication information //! @param path -> The logical path which is the target of oper //! @param oper -> The operation being attempted (see the enum above). //! If the oper is AOP_Any, then the actual privileges //! are returned and the caller may make subsequent //! tests using Test(). //! @param Env -> Environmental information at the time of the //! operation as supplied by the path CGI string. //! This is optional and the pointer may be zero. //! //! @return Permit: a non-zero value (access is permitted) //! Deny: zero (access is denied) //------------------------------------------------------------------------------ virtual XrdAccPrivs Access(const XrdSecEntity *Entity, const char *path, const Access_Operation oper, XrdOucEnv *Env=0) = 0; //------------------------------------------------------------------------------ //! Route an audit message to the appropriate audit exit routine. See //! XrdAccAudit.h for more information on how the default implementation works. //! Currently, this method is not called by the ofs but should be used by the //! implementation to record denials or grants, as warranted. //! //! @param accok -> True is access was grated; false otherwise. //! @param Entity -> Authentication information //! @param path -> The logical path which is the target of oper //! @param oper -> The operation being attempted (see above) //! @param Env -> Environmental information at the time of the //! operation as supplied by the path CGI string. //! This is optional and the pointer may be zero. //! //! @return Success: !0 information recorded. //! Failure: 0 information could not be recorded. //------------------------------------------------------------------------------ virtual int Audit(const int accok, const XrdSecEntity *Entity, const char *path, const Access_Operation oper, XrdOucEnv *Env=0) = 0; //------------------------------------------------------------------------------ //! Check whether the specified operation is permitted. //! //! @param priv -> the privileges as returned by Access(). //! @param oper -> The operation being attempted (see above) //! //! @return Permit: a non-zero value (access is permitted) //! Deny: zero (access is denied) //------------------------------------------------------------------------------ virtual int Test(const XrdAccPrivs priv, const Access_Operation oper) = 0; //------------------------------------------------------------------------------ //! Constructor //------------------------------------------------------------------------------ XrdAccAuthorize() {} //------------------------------------------------------------------------------ //! Destructor //------------------------------------------------------------------------------ virtual ~XrdAccAuthorize() {} }; /******************************************************************************/ /* X r d A c c A u t h o r i z e O b j e c t */ /******************************************************************************/ //------------------------------------------------------------------------------ //! Obtain an authorization object. //! //! XrdAccAuthorizeObject() is an extern "C" function that is called to obtain //! an instance of the auth object that will be used for all subsequent //! authorization decisions. It must be defined in the plug-in shared library. //! A second version which is used preferentially if it exists should be //! used if accessto theenvironmental pointer s needed. //! All the following extern symbols must be defined at file level! //! //! @param lp -> XrdSysLogger to be tied to an XrdSysError object for messages //! @param cfn -> The name of the configuration file //! @param parm -> Parameters specified on the authlib directive. If none it //! is zero. //! @param envP -> Pointer to environment only available for version 2. //! //! @return Success: A pointer to the authorization object. //! Failure: Null pointer which causes initialization to fail. //------------------------------------------------------------------------------ typedef XrdAccAuthorize *(*XrdAccAuthorizeObject_t)(XrdSysLogger *lp, const char *cfn, const char *parm); /*! extern "C" XrdAccAuthorize *XrdAccAuthorizeObject(XrdSysLogger *lp, const char *cfn, const char *parm) {...} */ // Alternatively: typedef XrdAccAuthorize *(*XrdAccAuthorizeObject2_t)(XrdSysLogger *lp, const char *cfn, const char *parm, XrdOucEnv *envP); /*! extern "C" XrdAccAuthorize *XrdAccAuthorizeObject2(XrdSysLogger *lp, const char *cfn, const char *parm, XrdOucEnv *envP) {...} */ //------------------------------------------------------------------------------ //! Add an authorization object as a wrapper to the existing object. //! //! XrdAccAuthorizeObjAdd() is an extern "C" function that is called to obtain //! an instance of the auth object that should wrap the existing object. The //! wrapper becomes the actual authorization object. The wrapper must be //! in the plug-in shared library, it is passed additional parameters. //! All the following extern symbols must be defined at file level! //! //! @param lp -> XrdSysLogger to be tied to an XrdSysError object for messages //! @param cfn -> The name of the configuration file //! @param parm -> Parameters specified on the authlib directive. If none it //! is zero. //! @param envP -> Environmental information and may be nil. //! @param accP -> to the existing authorization object. //! //! @return Success: A pointer to the authorization object. //! Failure: Null pointer which causes initialization to fail. //------------------------------------------------------------------------------ typedef XrdAccAuthorize *(*XrdAccAuthorizeObjAdd_t)(XrdSysLogger *lp, const char *cfn, const char *parm, XrdOucEnv *envP, XrdAccAuthorize *accP); /*! extern "C" XrdAccAuthorize *XrdAccAuthorizeObjAdd(XrdSysLogger *lp, const char *cfn, const char *parm, XrdOucEnv *envP, XrdAccAuthorize *accP) {...} */ //------------------------------------------------------------------------------ //! Specify the compilation version. //! //! Additionally, you *should* declare the xrootd version you used to compile //! your plug-in. While not currently required, it is highly recommended to //! avoid execution issues should the class definition change. Declare it as: //------------------------------------------------------------------------------ /*! #include "XrdVersion.hh" XrdVERSIONINFO(XrdAccAuthorizeObject,); where is a 1- to 15-character unquoted name identifying your plugin. For the default statically linked authorization framework, the non-extern C XrdAccDefaultAuthorizeObject() is called instead so as to not conflict with that symbol in a shared library plug-in. */ #endif xrootd-5.6.9/src/XrdAcc/XrdAccCapability.cc000066400000000000000000000166621457266313600205120ustar00rootroot00000000000000/******************************************************************************/ /* */ /* X r d A c c C a p a b i l i t y . c c */ /* */ /* (c) 2003 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include "XrdAcc/XrdAccCapability.hh" /******************************************************************************/ /* E x t e r n a l R e f e r e n c e s */ /******************************************************************************/ extern unsigned long XrdOucHashVal2(const char *KeyVal, int KeyLen); /******************************************************************************/ /* C o n s t r u c t o r */ /******************************************************************************/ XrdAccCapability::XrdAccCapability(char *pathval, XrdAccPrivCaps &privval) { int i; // Do common initialization // next = 0; ctmp = 0; priv.pprivs = privval.pprivs; priv.nprivs = privval.nprivs; plen = strlen(pathval); pins = 0; prem = 0; pkey = XrdOucHashVal2((const char *)pathval, plen); path = strdup(pathval); // Now set up for @= insertions. We do this eventhough it might never be used // for (i = 0; i < plen; i++) if (path[i] == '@' && path[i+1] == '=') {pins = i; prem = plen - i - 2; break;} } /******************************************************************************/ /* D e s t r u c t o r */ /******************************************************************************/ // This is a tricky destructor because deleting any item in the list must // delete all subsequent items in the list (but only once). // XrdAccCapability::~XrdAccCapability() { XrdAccCapability *cp, *np = next; if (path) {free(path); path = 0;} while(np) {cp = np; np = np->next; cp->next = 0; delete cp;} next = 0; } /******************************************************************************/ /* P r i v s */ /******************************************************************************/ int XrdAccCapability::Privs( XrdAccPrivCaps &pathpriv, const char *pathname, const int pathlen, const unsigned long pathhash, const char *pathsub) {XrdAccCapability *cp=this; const int psl = (pathsub ? strlen(pathsub) : 0); do {if (cp->ctmp) {if (cp->ctmp->Privs(pathpriv,pathname,pathlen,pathhash,pathsub)) return 1; } else if (pathlen >= cp->plen) if ((!pathsub && !strncmp(pathname, cp->path, cp->plen)) || (pathsub && cp->Subcomp(pathname,pathlen,pathsub,psl))) {pathpriv.pprivs = (XrdAccPrivs)(pathpriv.pprivs | cp->priv.pprivs); pathpriv.nprivs = (XrdAccPrivs)(pathpriv.nprivs | cp->priv.nprivs); return 1; } } while ((cp = cp->next)); return 0; } /******************************************************************************/ /* S u b c o m p */ /******************************************************************************/ int XrdAccCapability::Subcomp(const char *pathname, const int pathlen, const char *pathsub, const int sublen) { int ncmp; // First check if the prefix matches // if (strncmp(pathname, path, pins)) return 0; // Now, check if the substitution appears in the source path // if (strncmp(&pathname[pins], pathsub, sublen)) return 0; // Now check if we can match the tail // ncmp = pins + sublen; if ((pathlen - ncmp) < prem) return 0; // Return the results of matching the tail (prem should never be 0, but hey) // if (prem) return !strncmp(&path[pins+2], &pathname[ncmp], prem); return 1; } /******************************************************************************/ /* X r d A c c C a p N a m e */ /******************************************************************************/ /******************************************************************************/ /* D e s t r u c t o r */ /******************************************************************************/ XrdAccCapName::~XrdAccCapName() { XrdAccCapName *cp, *np = next; // Free regular storage // next = 0; if (CapName) free(CapName); if (C_List) delete C_List; // Delete list in a non-recursive way // while(np) {cp = np; np = np->next; cp->next = 0; delete cp;} } /******************************************************************************/ /* F i n d */ /******************************************************************************/ XrdAccCapability *XrdAccCapName::Find(const char *name) { int nlen = strlen(name); XrdAccCapName *ncp = this; do {if (ncp->CNlen <= nlen && !strcmp(ncp->CapName,name+(nlen - ncp->CNlen))) return ncp->C_List; ncp = ncp->next; } while(ncp); return (XrdAccCapability *)0; } xrootd-5.6.9/src/XrdAcc/XrdAccCapability.hh000066400000000000000000000134321457266313600205140ustar00rootroot00000000000000#ifndef __ACC_CAPABILITY__ #define __ACC_CAPABILITY__ /******************************************************************************/ /* */ /* X r d A c c C a p a b i l i t y . h h */ /* */ /* (c) 2003 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include #include #include "XrdAcc/XrdAccPrivs.hh" /******************************************************************************/ /* X r d A c c C a p a b i l i t y */ /******************************************************************************/ class XrdAccCapability { public: void Add(XrdAccCapability *newcap) {next = newcap;} XrdAccCapability *Next() {return next;} // Privs() searches the associated capability for a prefix matching path. If one // is found, the privileges are or'd into the passed XrdAccPrivCaps struct and // a 1 is returned. Otherwise, 0 is returned and XrdAccPrivCaps is unchanged. // int Privs( XrdAccPrivCaps &pathpriv, const char *pathname, const int pathlen, const unsigned long pathhash, const char *pathsub=0); int Privs( XrdAccPrivCaps &pathpriv, const char *pathname, const int pathlen, const char *pathsub=0) {extern unsigned long XrdOucHashVal2(const char *,int); return Privs(pathpriv, pathname, pathlen, XrdOucHashVal2(pathname,(int)pathlen),pathsub);} int Privs( XrdAccPrivCaps &pathpriv, const char *pathname, const char *pathsub=0) {extern unsigned long XrdOucHashVal2(const char *,int); int pathlen = strlen(pathname); return Privs(pathpriv, pathname, pathlen, XrdOucHashVal2(pathname, pathlen), pathsub);} int Subcomp(const char *pathname, const int pathlen, const char *pathsub, const int sublen); XrdAccCapability(char *pathval, XrdAccPrivCaps &privval); XrdAccCapability(XrdAccCapability *taddr) {next = 0; ctmp = taddr; pkey = 0; path = 0; plen = 0; pins = 0; prem = 0; } ~XrdAccCapability(); private: XrdAccCapability *next; // -> Next capability XrdAccCapability *ctmp; // -> Capability template /*----------- The below fields are valid when template is zero -----------*/ XrdAccPrivCaps priv; unsigned long pkey; char *path; int plen; int pins; // index of @= int prem; // remaining length after @= }; /******************************************************************************/ /* X r d A c c C a p N a m e */ /******************************************************************************/ class XrdAccCapName { public: void Add(XrdAccCapName *cnp) {next = cnp;} XrdAccCapability *Find(const char *name); XrdAccCapName(char *name, XrdAccCapability *cap) {next = 0; CapName = strdup(name); CNlen = strlen(name); C_List = cap; } ~XrdAccCapName(); private: XrdAccCapName *next; char *CapName; int CNlen; XrdAccCapability *C_List; }; #endif xrootd-5.6.9/src/XrdAcc/XrdAccConfig.cc000066400000000000000000001030531457266313600176250ustar00rootroot00000000000000/******************************************************************************/ /* */ /* X r d A c c C o n f i g . c c */ /* */ /* (C) 2003 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Deprtment of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include #include #include #include #include #include #include #include #include #include "XrdOuc/XrdOucLock.hh" #include "XrdOuc/XrdOucEnv.hh" #include "XrdOuc/XrdOucUri.hh" #include "XrdSys/XrdSysError.hh" #include "XrdSys/XrdSysHeaders.hh" #include "XrdOuc/XrdOucStream.hh" #include "XrdAcc/XrdAccAccess.hh" #include "XrdAcc/XrdAccAudit.hh" #include "XrdAcc/XrdAccConfig.hh" #include "XrdAcc/XrdAccGroups.hh" #include "XrdAcc/XrdAccCapability.hh" /******************************************************************************/ /* G l o b a l C o n f i g u r a t i o n O b j e c t */ /******************************************************************************/ // The following is the single configuration object. Other objects needing // access to this object should simply declare an extern to it. // XrdAccConfig XrdAccConfiguration; /******************************************************************************/ /* d e f i n e s */ /******************************************************************************/ #define TS_Xeq(x,m) if (!strcmp(x,var)) return m(Config,Eroute); #define TS_Str(x,m) if (!strcmp(x,var)) {free(m); m = strdup(val); return 0;} #define TS_Chr(x,m) if (!strcmp(x,var)) {m = val[0]; return 0;} #define TS_Bit(x,m,v) if (!strcmp(x,var)) {m |= v; return 0;} #define ACC_PGO 0x0001 /******************************************************************************/ /* E x t e r n a l F u n c t i o n s */ /******************************************************************************/ /******************************************************************************/ /* o o a c c _ C o n f i g _ R e f r e s h */ /******************************************************************************/ void *XrdAccConfig_Refresh( void *start_data ) { XrdSysError *Eroute = (XrdSysError *)start_data; // Get the number of seconds between refreshes // struct timespec naptime = {(time_t)XrdAccConfiguration.AuthRT, 0}; // Now loop until the bitter end // while(1) {nanosleep(&naptime, 0); XrdAccConfiguration.ConfigDB(1, *Eroute);} return (void *)0; } /******************************************************************************/ /* C o n s t r u c t o r */ /******************************************************************************/ XrdAccConfig::XrdAccConfig() { // Initialize path value and databse pointer to nil // dbpath = strdup("/opt/xrd/etc/Authfile"); Database = 0; Authorization = 0; spChar = 0; uriPath = false; // Establish other defaults // ConfigDefaults(); } /******************************************************************************/ /* C o n f i g u r e */ /******************************************************************************/ int XrdAccConfig::Configure(XrdSysError &Eroute, const char *cfn) { /* Function: Establish default values using a configuration file. Input: None. Output: 0 upon success or !0 otherwise. */ char *var; int retc, NoGo = 0, Cold = (Database == 0); pthread_t reftid; // Print warm-up message // Eroute.Say("++++++ Authorization system initialization started."); // Process the configuration file and authorization database // if (!(Authorization = new XrdAccAccess(&Eroute)) || (NoGo = ConfigFile(Eroute, cfn)) || (NoGo = ConfigDB(0, Eroute))) {if (Authorization) {delete Authorization, Authorization = 0;} NoGo = 1; } // Start a refresh thread unless this was a refresh thread call // if (Cold && !NoGo) {if ((retc=XrdSysThread::Run(&reftid,XrdAccConfig_Refresh,(void *)&Eroute))) Eroute.Emsg("ConfigDB",retc,"start refresh thread."); } // All done // var = (NoGo > 0 ? (char *)"failed." : (char *)"completed."); Eroute.Say("------ Authorization system initialization ", var); return (NoGo > 0); } /******************************************************************************/ /* C o n f i g D B */ /******************************************************************************/ int XrdAccConfig::ConfigDB(int Warm, XrdSysError &Eroute) { /* Function: Establish default values using a configuration file. Input: None. Output: 0 upon success or !0 otherwise. */ char buff[128]; int retc, anum = 0, NoGo = 0; struct XrdAccAccess_Tables tabs; XrdOucLock cdb_Lock(&Config_Context); // Indicate type of start we are doing // if (!Database) NoGo = !(Database = XrdAccAuthDBObject(&Eroute)); else if (Warm && !Database->Changed(dbpath)) return 0; // Try to open the authorization database // if (!Database || !Database->Open(Eroute, dbpath)) return 1; // Allocate new hash tables // if (!(tabs.G_Hash = new XrdOucHash()) || !(tabs.H_Hash = new XrdOucHash()) || !(tabs.N_Hash = new XrdOucHash()) || !(tabs.O_Hash = new XrdOucHash()) || !(tabs.R_Hash = new XrdOucHash()) || !(tabs.T_Hash = new XrdOucHash()) || !(tabs.U_Hash = new XrdOucHash()) ) {Eroute.Emsg("ConfigDB","Insufficient storage for id tables."); Database->Close(); return 1; } // Now start processing records until eof. // rulenum = 0; while((retc = ConfigDBrec(Eroute, tabs))) {NoGo |= retc < 0; anum++;} snprintf(buff, sizeof(buff), "%d auth entries processed in ", anum); Eroute.Say("Config ", buff, dbpath); // All done, close the database and return if we failed // if (!Database->Close() || NoGo) return 1; // Do final setup for special identifiers (this will correctly order them) // if (tabs.SYList) idChk(Eroute, tabs.SYList, tabs); // Set the access control tables // if (!tabs.G_Hash->Num()) {delete tabs.G_Hash; tabs.G_Hash=0;} if (!tabs.H_Hash->Num()) {delete tabs.H_Hash; tabs.H_Hash=0;} if (!tabs.N_Hash->Num()) {delete tabs.N_Hash; tabs.N_Hash=0;} if (!tabs.O_Hash->Num()) {delete tabs.O_Hash; tabs.O_Hash=0;} if (!tabs.R_Hash->Num()) {delete tabs.R_Hash; tabs.R_Hash=0;} if (!tabs.T_Hash->Num()) {delete tabs.T_Hash; tabs.T_Hash=0;} if (!tabs.U_Hash->Num()) {delete tabs.U_Hash; tabs.U_Hash=0;} Authorization->SwapTabs(tabs); // All done // return NoGo; } /******************************************************************************/ /* P r i v a t e F u n c t i o n s */ /******************************************************************************/ /******************************************************************************/ /* C o n f i g F i l e P r o c e s s i n g M e t h o d s */ /******************************************************************************/ int XrdAccConfig::ConfigFile(XrdSysError &Eroute, const char *ConfigFN) { /* Function: Establish default values using a configuration file. Input: None. Output: 1 - Processing failed. 0 - Processing completed successfully. -1 = Security is to be disabled by request. */ char *var; int cfgFD, retc, NoGo = 0, recs = 0; XrdOucEnv myEnv; XrdOucStream Config(&Eroute, getenv("XRDINSTANCE"), &myEnv, "=====> "); // If there is no config file, complain // if( !ConfigFN || !*ConfigFN) {Eroute.Emsg("Config", "Authorization configuration file not specified."); return 1; } // Check if security is to be disabled // if (!strcmp(ConfigFN, "none")) {Eroute.Emsg("Config", "Authorization system deactivated."); return -1; } // Try to open the configuration file. // if ( (cfgFD = open(ConfigFN, O_RDONLY, 0)) < 0) {Eroute.Emsg("Config", errno, "open config file", ConfigFN); return 1; } Eroute.Emsg("Config","Authorization system using configuration in",ConfigFN); // Now start reading records until eof. // ConfigDefaults(); Config.Attach(cfgFD); Config.Tabs(0); static const char *cvec[] = { "*** acc plugin config:", 0 }; Config.Capture(cvec); while((var = Config.GetMyFirstWord())) {if (!strncmp(var, "acc.", 2)) {recs++; if (ConfigXeq(var+4, Config, Eroute)) {Config.Echo(); NoGo = 1;} } } // Now check if any errors occurred during file i/o // if ((retc = Config.LastError())) NoGo = Eroute.Emsg("Config",-retc,"read config file",ConfigFN); else {char buff[128]; snprintf(buff, sizeof(buff), "%d authorization directives processed in ", recs); Eroute.Say("Config ", buff, ConfigFN); } Config.Close(); // Set external options, as needed // if (options & ACC_PGO) GroupMaster.SetOptions(Primary_Only); // All done // return NoGo; } /******************************************************************************/ /* C o n f i g D e f a u l t s */ /******************************************************************************/ void XrdAccConfig::ConfigDefaults() { AuthRT = 60*60*12; options = 0; } /******************************************************************************/ /* C o n f i g X e q */ /******************************************************************************/ int XrdAccConfig::ConfigXeq(char *var, XrdOucStream &Config, XrdSysError &Eroute) { // Fan out based on the variable // TS_Xeq("audit", xaud); TS_Xeq("authdb", xdbp); TS_Xeq("authrefresh", xart); TS_Xeq("encoding", xenc); TS_Xeq("gidlifetime", xglt); TS_Xeq("gidretran", xgrt); TS_Xeq("nisdomain", xnis); TS_Bit("pgo", options, ACC_PGO); TS_Xeq("spacechar", xspc); // No match found, complain. // Eroute.Emsg("Config", "unknown directive", var); Config.Echo(); return 1; } /******************************************************************************/ /* s u b S p a c e */ /******************************************************************************/ void XrdAccConfig::subSpace(char *id) { char *spc; while((spc = index(id, spChar))) {*spc = ' '; id = spc+1; } } /******************************************************************************/ /* x a u d */ /******************************************************************************/ /* Function: xaud Purpose: To parse the directive: audit options: deny audit access denials. grant audit access grants. none audit is disabled. Output: 0 upon success or !0 upon failure. */ int XrdAccConfig::xaud(XrdOucStream &Config, XrdSysError &Eroute) { static struct auditopts {const char *opname; int opval;} audopts[] = { {"deny", (int)audit_deny}, {"grant", (int)audit_grant} }; int i, audval = 0, numopts = sizeof(audopts)/sizeof(struct auditopts); char *val; val = Config.GetWord(); if (!val || !val[0]) {Eroute.Emsg("Config", "audit option not specified"); return 1;} while (val && val[0]) {if (!strcmp(val, "none")) audval = (int)audit_none; else for (i = 0; i < numopts; i++) {if (!strcmp(val, audopts[i].opname)) {audval |= audopts[i].opval; break;} if (i >= numopts) {Eroute.Emsg("Config","invalid audit option -",val); return 1; } } val = Config.GetWord(); } Authorization->Auditor->setAudit((XrdAccAudit_Options)audval); return 0; } /******************************************************************************/ /* x a r t */ /******************************************************************************/ /* Function: xart Purpose: To parse the directive: authrefresh minimum number of seconds between aythdb refreshes. Output: 0 upon success or !0 upon failure. */ int XrdAccConfig::xart(XrdOucStream &Config, XrdSysError &Eroute) { char *val; int reft; val = Config.GetWord(); if (!val || !val[0]) {Eroute.Emsg("Config","authrefresh value not specified");return 1;} if (XrdOuca2x::a2tm(Eroute,"authrefresh value",val,&reft,60)) return 1; AuthRT = reft; return 0; } /******************************************************************************/ /* x e n c */ /******************************************************************************/ /* Function: xenc Purpose: To parse the directive: encoding [space ] [pct path] the character that is to be considred as a space. This only applies to identifiers. Output: 0 upon success or !0 upon failure. */ int XrdAccConfig::xenc(XrdOucStream &Config, XrdSysError &Eroute) { char *val; if (!(val = Config.GetWord()) || *val == 0) {Eroute.Emsg("Config","encoding argument not specified"); return 1;} do{ if (!strcmp(val, "pct")) {if (!(val = Config.GetWord())) {Eroute.Emsg("Config","pct argument not specified"); return 1; } if (strcmp(val, "path")) {Eroute.Emsg("Config",val, "pct encoding not supported"); return 1; } uriPath = true; } else if (!strcmp(val, "space")) {if (!(val = Config.GetWord())) {Eroute.Emsg("Config","space argument not specified"); return 1; } if (strlen(val) != 1) {Eroute.Emsg("Config","invalid space argument -", val); return 1; } spChar = *val; } } while((val = Config.GetWord()) && *val); return 0; } /******************************************************************************/ /* x d b p */ /******************************************************************************/ /* Function: xdbp Purpose: To parse the directive: authdb is the path to the authorization database. Output: 0 upon success or !0 upon failure. */ int XrdAccConfig::xdbp(XrdOucStream &Config, XrdSysError &Eroute) { char *val; val = Config.GetWord(); if (!val || !val[0]) {Eroute.Emsg("Config","authdb path not specified");return 1;} dbpath = strdup(val); return 0; } /******************************************************************************/ /* x g l t */ /******************************************************************************/ /* Function: xglt Purpose: To parse the directive: gidlifetime maximum number of seconds to cache gid information. Output: 0 upon success or !0 upon failure. */ int XrdAccConfig::xglt(XrdOucStream &Config, XrdSysError &Eroute) { char *val; int reft; val = Config.GetWord(); if (!val || !val[0]) {Eroute.Emsg("Config","gidlifetime value not specified");return 1;} if (XrdOuca2x::a2tm(Eroute,"gidlifetime value",val,&reft,60)) return 1; GroupMaster.SetLifetime(reft); return 0; } /******************************************************************************/ /* x g r t */ /******************************************************************************/ /* Function: xgrt Purpose: To parse the directive: gidretran is a list of blank separated gid's that must be retranslated. Output: 0 upon success or !0 upon failure. */ int XrdAccConfig::xgrt(XrdOucStream &Config, XrdSysError &Eroute) { char *val; int gid; val = Config.GetWord(); if (!val || !val[0]) {Eroute.Emsg("Config","gidretran value not specified"); return 1;} while (val && val[0]) {if (XrdOuca2x::a2i(Eroute, "gid", val, &gid, 0)) return 1; if (GroupMaster.Retran((gid_t)gid) < 0) {Eroute.Emsg("Config", "to many gidretran gid's"); return 1;} val = Config.GetWord(); } return 0; } /******************************************************************************/ /* x n i s */ /******************************************************************************/ /* Function: xnis Purpose: To parse the directive: nisdomain the NIS domain to be used for nis look-ups. Output: 0 upon success or !0 upon failure. */ int XrdAccConfig::xnis(XrdOucStream &Config, XrdSysError &Eroute) { char *val; val = Config.GetWord(); if (!val || !val[0]) {Eroute.Emsg("Config","nisdomain value not specified");return 1;} GroupMaster.SetDomain(strdup(val)); return 0; } /******************************************************************************/ /* x s p c */ /******************************************************************************/ /* Function: xspc (deprecated and undocumented, replaced by acc.encoding). Purpose: To parse the directive: spacechar the character that is to be considred as a space. Output: 0 upon success or !0 upon failure. */ int XrdAccConfig::xspc(XrdOucStream &Config, XrdSysError &Eroute) { char *val; val = Config.GetWord(); if (!val || !val[0]) {Eroute.Emsg("Config","spacechar argument not specified");return 1;} if (strlen(val) != 1) {Eroute.Emsg("Config","invalid spacechar argument -", val);return 1;} spChar = *val; return 0; } /******************************************************************************/ /* D a t a b a s e P r o c e s s i n g */ /******************************************************************************/ /******************************************************************************/ /* C o n f i g D B r e c */ /******************************************************************************/ int XrdAccConfig::ConfigDBrec(XrdSysError &Eroute, struct XrdAccAccess_Tables &tabs) { // The following enum is here for convenience // enum DB_RecType { Group_ID = 'g', Host_ID = 'h', Netgrp_ID = 'n', Org_ID = 'o', Role_ID = 'r', Set_ID = 's', Template_ID = 't', User_ID = 'u', Xxx_ID = 'x', Def_ID = '=', No_ID = 0 }; char *authid, rtype, *path, *privs; int alluser = 0, anyuser = 0, domname = 0, NoGo = 0; DB_RecType rectype; XrdAccAccess_ID *sp = 0; XrdOucHash *hp; XrdAccGroupType gtype = XrdAccNoGroup; XrdAccPrivCaps xprivs; XrdAccCapability mycap((char *)"", xprivs), *currcap, *lastcap = &mycap; XrdAccCapName *ncp; bool istmplt, isDup, xclsv = false; // Prepare the next record in the database // if (!(rtype = Database->getRec(&authid))) return 0; rectype = (DB_RecType)rtype; // Set up to handle the particular record // switch(rectype) {case Group_ID: hp = tabs.G_Hash; gtype=XrdAccUnixGroup; if (spChar) subSpace(authid); break; case Host_ID: hp = tabs.H_Hash; domname = (authid[0] == '.'); break; case Set_ID: hp = 0; break; case Netgrp_ID: hp = tabs.N_Hash; gtype=XrdAccNetGroup; break; case Org_ID: hp = tabs.O_Hash; if (spChar) subSpace(authid); break; case Role_ID: hp = tabs.R_Hash; if (spChar) subSpace(authid); break; case Template_ID: hp = tabs.T_Hash; break; case User_ID: hp = tabs.U_Hash; alluser = (authid[0] == '*' && !authid[1]); anyuser = (authid[0] == '=' && !authid[1]); if (!alluser && !anyuser && spChar) subSpace(authid); break; case Xxx_ID: hp = 0; xclsv = true; break; case Def_ID: return idDef(Eroute, tabs, authid); break; default: char badtype[2] = {rtype, '\0'}; Eroute.Emsg("ConfigXeq", "Invalid id type -", badtype); return -1; break; } // Check if this id is already defined in the table. For 's' rules the id // must have been previously defined. // if (domname) isDup = tabs.D_List && tabs.D_List->Find((const char *)authid); else if (alluser) isDup = tabs.Z_List != 0; else if (anyuser) isDup = tabs.X_List != 0; else if (hp) isDup = hp->Find(authid) != 0; else {if (!(sp = tabs.S_Hash->Find(authid))) {Eroute.Emsg("ConfigXeq", "Missing id definition -", authid); return -1; } isDup = sp->caps != 0; sp->rule = (xclsv ? rulenum++ : -1); } if (isDup) {Eroute.Emsg("ConfigXeq", "duplicate rule for id -", authid); return -1; } // Add this ID to the appropriate group object constants table // if (gtype) GroupMaster.AddName(gtype, (const char *)authid); // Now start getting pairs until we hit the logical end // while(1) {NoGo = 0; if (!Database->getPP(&path, &privs, istmplt)) break; if (!path) continue; // Skip pathless entries NoGo = 1; if (istmplt) {if ((currcap = tabs.T_Hash->Find(path))) currcap = new XrdAccCapability(currcap); else {Eroute.Emsg("ConfigXeq", "Missing template -", path); break; } } else { if (!privs) {Eroute.Emsg("ConfigXeq", "Missing privs for path", path); break; } if (!PrivsConvert(privs, xprivs)) {Eroute.Emsg("ConfigXeq", "Invalid privs -", privs); break; } if (uriPath) {int plen = strlen(path); char *decp = (char *)alloca(plen+1); XrdOucUri::Decode(path, plen, decp); currcap = new XrdAccCapability(decp, xprivs); } else currcap = new XrdAccCapability(path, xprivs); } lastcap->Add(currcap); lastcap = currcap; } // Check if all went well // if (NoGo) return -1; // Check if any capabilities were specified // if (!mycap.Next()) {Eroute.Emsg("ConfigXeq", "no capabilities specified for", authid); return -1; } // Insert the capability into the appropriate table/list // if (sp) sp->caps = mycap.Next(); else if (domname) {if (!(ncp = new XrdAccCapName(authid, mycap.Next()))) {Eroute.Emsg("ConfigXeq","unable to add id",authid); return -1;} if (tabs.E_List) tabs.E_List->Add(ncp); else tabs.D_List = ncp; tabs.E_List = ncp; } else if (anyuser) tabs.X_List = mycap.Next(); else if (alluser) tabs.Z_List = mycap.Next(); else hp->Add(authid, mycap.Next()); // All done // mycap.Add((XrdAccCapability *)0); return 1; } /******************************************************************************/ /* Private: i d C h k */ /******************************************************************************/ void XrdAccConfig::idChk(XrdSysError &Eroute, XrdAccAccess_ID *idList, XrdAccAccess_Tables &tabs) { std::map idMap; XrdAccAccess_ID *idPN, *xList = 0, *yList = 0; // Run through the list to make everything was used. We also, sort these items // in the order the associated rule appeared. // while(idList) {idPN = idList->next; if (idList->caps == 0) Eroute.Say("Config ","Warning, unused identifier definition '", idList->name, "'."); else if (idList->rule >= 0) idMap[idList->rule] = idList; else {idList->next = yList; yList = idList;} idList = idPN; } // Place 'x' rules in the order they were used. The ;s; rules are in the // order the id's were defined which is OK because the are inclusive. // std::map::reverse_iterator rit; for (rit = idMap.rbegin(); rit != idMap.rend(); ++rit) {rit->second->next = xList; xList = rit->second; } // Set the new lists in the supplied tabs structure // tabs.SXList = xList; tabs.SYList = yList; } /******************************************************************************/ /* Private: i d D e f */ /******************************************************************************/ int XrdAccConfig::idDef(XrdSysError &Eroute, struct XrdAccAccess_Tables &tabs, const char *idName) { XrdAccAccess_ID *xID, theID(idName); char *idname, buff[80], idType; bool haveID = false, idDup = false; // Now start getting pairs until we hit the logical end // while(!idDup) {if (!(idType = Database->getID(&idname))) break; haveID = true; switch(idType) {case 'g': if (spChar) subSpace(idname); if (theID.grp) idDup = true; else{theID.grp = strdup(idname); theID.glen = strlen(idname); } break; case 'h': if (theID.host) idDup = true; else{theID.host = strdup(idname); theID.hlen = strlen(idname); } break; case 'o': if (theID.org) idDup = true; else {if (spChar) subSpace(idname); theID.org = strdup(idname); } break; case 'r': if (theID.role) idDup = true; else {if (spChar) subSpace(idname); theID.role = strdup(idname); } break; case 'u': if (theID.user) idDup = true; else {if (spChar) subSpace(idname); theID.user = strdup(idname); } break; default: snprintf(buff, sizeof(buff), "'%c: %s' for", idType, idname); Eroute.Emsg("ConfigXeq", "Invalid id selector -", buff, theID.name); return -1; break; } if (idDup) {snprintf(buff, sizeof(buff), "id selector '%c' specified twice for", idType); Eroute.Emsg("ConfigXeq", buff, theID.name); return -1; } } // Make sure some kind of id was specified // if (!haveID) {Eroute.Emsg("ConfigXeq", "No id selectors specified for", theID.name); return -1; } // Make sure this name has not been specified before // if (!tabs.S_Hash) tabs.S_Hash = new XrdOucHash; else if (tabs.S_Hash->Find(theID.name)) {Eroute.Emsg("ConfigXeq","duplicate id definition -",theID.name); return -1; } // Export the id definition and add it to the S_Hash // xID = theID.Export(); tabs.S_Hash->Add(xID->name, xID); // Place this FIFO in SYList (they reordered later based on rule usage) // xID->next = tabs.SYList; tabs.SYList = xID; // All done // return 1; } /******************************************************************************/ /* P r i v s C o n v e r t */ /******************************************************************************/ int XrdAccConfig::PrivsConvert(char *privs, XrdAccPrivCaps &ctab) { int i = 0; XrdAccPrivs ptab[] = {XrdAccPriv_None, XrdAccPriv_None}; // Speed conversion here // Convert the privs // while(*privs) {switch((XrdAccPrivSpec)(*privs)) {case All_Priv: ptab[i] = (XrdAccPrivs)(ptab[i]|XrdAccPriv_All); break; case Delete_Priv: ptab[i] = (XrdAccPrivs)(ptab[i]|XrdAccPriv_Delete); break; case Insert_Priv: ptab[i] = (XrdAccPrivs)(ptab[i]|XrdAccPriv_Insert); break; case Lock_Priv: ptab[i] = (XrdAccPrivs)(ptab[i]|XrdAccPriv_Lock); break; case Lookup_Priv: ptab[i] = (XrdAccPrivs)(ptab[i]|XrdAccPriv_Lookup); break; case Rename_Priv: ptab[i] = (XrdAccPrivs)(ptab[i]|XrdAccPriv_Rename); break; case Read_Priv: ptab[i] = (XrdAccPrivs)(ptab[i]|XrdAccPriv_Read); break; case Write_Priv: ptab[i] = (XrdAccPrivs)(ptab[i]|XrdAccPriv_Write); break; case Neg_Priv: if (i) return 0; i++; break; default: return 0; } privs++; } ctab.pprivs = ptab[0]; ctab.nprivs = ptab[1]; return 1; } xrootd-5.6.9/src/XrdAcc/XrdAccConfig.hh000066400000000000000000000131721457266313600176410ustar00rootroot00000000000000#ifndef _ACC_CONFIG_H #define _ACC_CONFIG_H /******************************************************************************/ /* */ /* X r d A c c C o n f i g . h h */ /* */ /* (C) 2003 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Deprtment of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include "XrdOuc/XrdOuca2x.hh" #include "XrdSys/XrdSysError.hh" #include "XrdOuc/XrdOucHash.hh" #include "XrdSys/XrdSysPthread.hh" #include "XrdOuc/XrdOucStream.hh" #include "XrdAcc/XrdAccAccess.hh" #include "XrdAcc/XrdAccAuthDB.hh" #include "XrdAcc/XrdAccCapability.hh" #include "XrdAcc/XrdAccGroups.hh" /******************************************************************************/ /* X r d A c c G l i s t */ /******************************************************************************/ struct XrdAccGlist { struct XrdAccGlist *next; /* Null if this is the last one */ char *name; /* -> null terminated name */ XrdAccGlist(const char *Name, struct XrdAccGlist *Next=0) {name = strdup(Name); next = Next;} ~XrdAccGlist() {if (name) free(name);} }; /******************************************************************************/ /* X r d A c c C o n f i g */ /******************************************************************************/ class XrdAccConfig { public: // Configure() is called during initialization. // int Configure(XrdSysError &Eroute, const char *cfn); // ConfigDB() simply refreshes the in-core authorization database. When the // Warm is true, a check is made whether the database actually changed and the // refresh is skipped if it has not changed. // int ConfigDB(int Warm, XrdSysError &Eroute); XrdAccAccess *Authorization; XrdAccGroups GroupMaster; int AuthRT; XrdAccConfig(); ~XrdAccConfig() {} // Configuration is never destroyed! private: struct XrdAccGlist *addGlist(gid_t Gid, const char *Gname, struct XrdAccGlist *Gnext); int ConfigDBrec(XrdSysError &Eroute, struct XrdAccAccess_Tables &tabs); void ConfigDefaults(void); int ConfigFile(XrdSysError &Eroute, const char *cfn); int ConfigXeq(char *, XrdOucStream &, XrdSysError &); void idChk(XrdSysError &Eroute, XrdAccAccess_ID *idList, XrdAccAccess_Tables &tabs); int idDef(XrdSysError &Eroute, XrdAccAccess_Tables &tabs, const char *idName); void subSpace(char *id); int PrivsConvert(char *privs, XrdAccPrivCaps &ctab); int xaud(XrdOucStream &Config, XrdSysError &Eroute); int xart(XrdOucStream &Config, XrdSysError &Eroute); int xdbp(XrdOucStream &Config, XrdSysError &Eroute); int xenc(XrdOucStream &Config, XrdSysError &Eroute); int xglt(XrdOucStream &Config, XrdSysError &Eroute); int xgrt(XrdOucStream &Config, XrdSysError &Eroute); int xnis(XrdOucStream &Cofig, XrdSysError &Eroute); int xspc(XrdOucStream &Cofig, XrdSysError &Eroute); XrdAccAuthDB *Database; char *dbpath; XrdSysMutex Config_Context; XrdSysThread Config_Refresh; int options; int rulenum; char spChar; bool uriPath; }; #endif xrootd-5.6.9/src/XrdAcc/XrdAccEntity.cc000066400000000000000000000202071457266313600176730ustar00rootroot00000000000000/******************************************************************************/ /* */ /* X r d A c c E n t i t y . c c */ /* */ /* (c) 2019 by the Board of Trustees of the Leland Stanford, Jr., University */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include #include "XrdAcc/XrdAccEntity.hh" #include "XrdOuc/XrdOucTokenizer.hh" #include "XrdSec/XrdSecEntity.hh" #include "XrdSec/XrdSecEntityAttr.hh" #include "XrdSys/XrdSysError.hh" /******************************************************************************/ /* S t a t i c M e m b e r s */ /******************************************************************************/ int XrdAccEntity::accSig = 0; namespace { XrdSysError *eDest = 0; } /******************************************************************************/ /* C o n s t r u c t o r */ /******************************************************************************/ XrdAccEntity::XrdAccEntity(const XrdSecEntity *secP, bool &aOK) : XrdSecAttr(&accSig) { EntityAttr attrInfo; int have, want = 0; // Assume all is going to be well and set our unique id. // aOK = true; // Copy out the various attributes we want to tokenize // if (secP->vorg) {vorgInfo = strdup(secP->vorg); want++;} else vorgInfo = 0; if (secP->role) {roleInfo = strdup(secP->role); want++;} else roleInfo = 0; if (secP->grps) {grpsInfo = strdup(secP->grps); want++;} else grpsInfo = 0; // If there are no attributes, then we are done. // if (!want) return; // If there is zero or one vorg and role then we can accept a short form // attribute entry. This provides not only backward compatabilty but also // takes care of the common case. // if (OneOrZero(vorgInfo, attrInfo.vorg) && OneOrZero(roleInfo, attrInfo.role)) {if (grpsInfo) {XrdOucTokenizer grpsLine(grpsInfo); grpsLine.GetLine(); while((attrInfo.grup = grpsLine.GetToken())) attrVec.push_back(attrInfo); } if (attrVec.size() == 0) attrVec.push_back(attrInfo); return; } // Tokenize each of the lists // XrdOucTokenizer vorgLine(vorgInfo); if (vorgInfo) vorgLine.GetLine(); attrInfo.vorg = 0; XrdOucTokenizer roleLine(roleInfo); if (roleInfo) roleLine.GetLine(); attrInfo.role = 0; XrdOucTokenizer grpsLine(grpsInfo); if (grpsInfo) grpsLine.GetLine(); attrInfo.grup = 0; while(true) {have = 0; if (vorgInfo && setAttr(vorgLine, attrInfo.vorg)) have++; if (roleInfo && setAttr(roleLine, attrInfo.role)) have++; if (grpsInfo && setAttr(grpsLine, attrInfo.grup)) have++; if (want != have) break; attrVec.push_back(attrInfo); } // Check if pairing was violated and indicate if so. // if (have) aOK = false; } /******************************************************************************/ /* G e t E n t i t y */ /******************************************************************************/ XrdAccEntity *XrdAccEntity::GetEntity(const XrdSecEntity *secP, bool &isNew) { XrdAccEntity *aeP; XrdSecAttr *seP; bool aOK; // If we already compiled the identity informaion, reuse it. // if ((seP = secP->eaAPI->Get(&accSig))) {isNew = false; return static_cast(seP); } // At this point we muxt create a new entity for authorization purposes and // return it if all went well. We do not attach it to its SecEntity object as // this will be done by the AccEntityInit object upon deletion to avoid // race conditions and memory leaks. This allows for parallel processing. // isNew = true; aeP = new XrdAccEntity(secP, aOK); if (aOK) return aeP; // Produce message indicating why we failed (there is only one possible reason) // if (eDest) {char eBuff[128]; snprintf(eBuff, sizeof(eBuff), "missing attrs in col %d for", static_cast(aeP->attrVec.size())); eDest->Emsg("Entity", "Unable to validate entity;", eBuff, (secP->tident ? secP->tident : "???")); } delete aeP; return 0; } /******************************************************************************/ /* Private: O n e O r Z e r o */ /******************************************************************************/ bool XrdAccEntity::OneOrZero(char *src, const char *&dest) { // If there is no source, then we are done // if (!src) {dest = 0; return true; } // Check if source has only one item; // while(*src == ' ') src++; char *sP = src; while(*src && *src != ' ') src++; char *eP = src; while(*src == ' ') src++; if (*src) return false; if (*sP) {dest = sP; *eP = 0;} else dest = 0; return true; } /******************************************************************************/ /* P u t E n t i t y */ /******************************************************************************/ void XrdAccEntity::PutEntity(const XrdSecEntity *secP) { // Add this object to the indicated SecEntity object. There may be one there // already if some other thread beat us to the punch (unlike). If there is // we simply delete ourselves to avoid a memory leak. // if (!secP->eaAPI->Add(*this)) delete this; } /******************************************************************************/ /* Private: s e t A t t r */ /******************************************************************************/ bool XrdAccEntity::setAttr(XrdOucTokenizer &tkl, const char *&dest) { const char *attr = tkl.GetToken(); if (!attr || !dest || strcmp(dest, attr)) dest = attr; return attr != 0; } /******************************************************************************/ /* s e t E r r o r */ /******************************************************************************/ void XrdAccEntity::setError(XrdSysError *erp) {eDest = erp;} xrootd-5.6.9/src/XrdAcc/XrdAccEntity.hh000066400000000000000000000124561457266313600177140ustar00rootroot00000000000000#ifndef __ACC_ENTITY_H__ #define __ACC_ENTITY_H__ /******************************************************************************/ /* */ /* X r d A c c E n t i t y . h h */ /* */ /* (c) 2019 by the Board of Trustees of the Leland Stanford, Jr., University */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include #include "XrdSec/XrdSecAttr.hh" /******************************************************************************/ /* X r d A c c E n t i t y I n f o */ /******************************************************************************/ struct XrdAccEntityInfo {const char *name; // Filled in by caller const char *host; // Filled in by caller const char *vorg; const char *role; const char *grup; XrdAccEntityInfo() : name(NULL), host(NULL), vorg(NULL), role(NULL), grup(NULL) {} ~XrdAccEntityInfo() {} }; /******************************************************************************/ /* X r d A c c E n t i t y */ /******************************************************************************/ class XrdOucTokenizer; class XrdSecEntity; class XrdSysError; class XrdAccEntity : public XrdSecAttr { public: static XrdAccEntity *GetEntity(const XrdSecEntity *secP, bool &isNew); bool Next(int &seq, XrdAccEntityInfo &info) {if (int(attrVec.size()) <= seq) return false; EntityAttr *aP = &attrVec[seq++]; info.vorg = aP->vorg; info.role = aP->role; info.grup = aP->grup; return true; } void PutEntity(const XrdSecEntity *secP); static void setError(XrdSysError *errP); private: XrdAccEntity(const XrdSecEntity *secP, bool &aOK); ~XrdAccEntity() {if (vorgInfo) free(vorgInfo); if (roleInfo) free(roleInfo); if (grpsInfo) free(grpsInfo); } bool OneOrZero(char *src, const char *&dest); bool setAttr(XrdOucTokenizer &tkl, const char *&dest); struct EntityAttr {const char *vorg; const char *role; const char *grup; EntityAttr() : vorg(NULL), role(NULL), grup(NULL) {} ~EntityAttr() {} }; std::vector attrVec; char *vorgInfo; char *roleInfo; char *grpsInfo; static int accSig; // Attribute Object Signture }; /******************************************************************************/ /* X r d A c c E n t i t y I n i t */ /******************************************************************************/ class XrdAccEntityInit { public: XrdAccEntityInit(const XrdSecEntity *secP, XrdAccEntity *&aeR) : seP(secP) {bool isNew; aeR = XrdAccEntity::GetEntity(secP, isNew); aeP = (isNew ? aeR : 0); } ~XrdAccEntityInit() {if (aeP) aeP->PutEntity(seP);} private: const XrdSecEntity *seP; XrdAccEntity *aeP; }; #endif xrootd-5.6.9/src/XrdAcc/XrdAccGroups.cc000066400000000000000000000362661457266313600177120ustar00rootroot00000000000000/******************************************************************************/ /* */ /* X r d A c c G r o u p s . c c */ /* */ /* (c) 2003 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include #include #include #include #include #include #include #include #include #include "XrdSys/XrdSysHeaders.hh" #include "XrdSys/XrdSysPlatform.hh" #include "XrdSys/XrdSysPwd.hh" #include "XrdAcc/XrdAccCapability.hh" #include "XrdAcc/XrdAccGroups.hh" #include "XrdAcc/XrdAccPrivs.hh" #ifdef MUSL int innetgr(const char *netgroup, const char *host, const char *user, const char *domain) { return 0; } #endif // Additionally, this routine does not support a user in more than // NGROUPS_MAX groups. This is a standard unix limit defined in limits.h. /******************************************************************************/ /* G l o b a l G r o u p s O b j e c t */ /******************************************************************************/ // There is only one Groups object that handles group memberships. Others // needing access to this object should declare an extern to this object. // XrdAccGroups XrdAccGroupMaster; /******************************************************************************/ /* G r o u p C o n s t r u c t i o n A r g u m e n t s */ /******************************************************************************/ struct XrdAccGroupArgs {const char *user; const char *host; int gtabi; const char *Gtab[NGROUPS_MAX]; }; /******************************************************************************/ /* C o n s t r u c t o r */ /******************************************************************************/ XrdAccGroups::XrdAccGroups() { // Do standard initialization // retrancnt = 0; HaveGroups = 0; HaveNetGroups = 0; options = No_Group_Opt; domain = 0; LifeTime = 60*60*12; } /******************************************************************************/ /* A d d N a m e */ /******************************************************************************/ char *XrdAccGroups::AddName(const XrdAccGroupType gtype, const char *name) { char *np; XrdOucHash *hp; // Prepare to add a group name // if (gtype == XrdAccNetGroup) {hp = &NetGroup_Names; HaveNetGroups = 1;} else {hp = &Group_Names; HaveGroups = 1;} // Lock the Name hash table // Group_Name_Context.Lock(); // Add a name into the name hash table. We need to only keep a single // read/only copy of the group name to speed multi-threading. // if (!(np = hp->Find(name))) {hp->Add(name, 0, 0, Hash_data_is_key); if (!(np = hp->Find(name))) std::cerr <<"XrdAccGroups: Unable to add group " <First()) glist = new XrdAccGroupList(*glist); else glist = 0; Group_Cache_Context.UnLock(); return glist; } Group_Cache_Context.UnLock(); // If the user has no password file entry, then we have no groups for user. // All code that tries to construct a group list is protected by the // Group_Build_Context mutex, obtained after we get the pwd entry. // XrdSysPwd thePwd(user, &pw); if (pw == NULL) return (XrdAccGroupList *)0; // Build first entry for the primary group. We will ignore the primary group // listing later. We do this to ensure that the user has at least one group // regardless of what the groups file actually says. // Group_Build_Context.Lock(); gtabi = addGroup(user, pw->pw_gid, 0, Gtab, 0); // Now run through all of the group entries getting the list of user's groups // Do this only when Primary_Only is not turned on (i.e., SVR5 semantics) // if (!(options & Primary_Only)) { setgrent() ; while ((gr = getgrent())) { if (pw->pw_gid == gr->gr_gid) continue; /*Already have this one.*/ for (cp = gr->gr_mem; cp && *cp; cp++) if (strcmp(*cp, user) == 0) gtabi = addGroup(user, gr->gr_gid, Dotran(gr->gr_gid,gr->gr_name), Gtab, gtabi); } endgrent(); } // All done with non mt-safe routines // Group_Build_Context.UnLock(); // Allocate a new GroupList object // glist = new XrdAccGroupList(gtabi, (const char **)Gtab); // Add this user to the group cache to speed things up the next time // Group_Cache_Context.Lock(); Group_Cache.Add(user, glist, LifeTime); Group_Cache_Context.UnLock(); // Return a copy of the group list since the original may be deleted // if (!gtabi) return (XrdAccGroupList *)0; return new XrdAccGroupList(gtabi, (const char **)Gtab); } /******************************************************************************/ /* N e t G r o u p s ( u s e r , h o s t ) */ /******************************************************************************/ XrdAccGroupList *XrdAccGroups::NetGroups(const char *user, const char *host) { XrdAccGroupList *glist; int i, j; char uh_key[MAXHOSTNAMELEN+96]; struct XrdAccGroupArgs GroupTab; int XrdAccCheckNetGroup(const char *netgroup, char *key, void *Arg); // Check if we have any Netgroups // if (!HaveNetGroups) return (XrdAccGroupList *)0; // Construct the key for this user // i = strlen(user); j = strlen(host); if (i+j+2 > (int)sizeof(uh_key)) return (XrdAccGroupList *)0; strcpy(uh_key, user); uh_key[i] = '@'; strcpy(&uh_key[i+1], host); // Check if we already have this user in the group cache. Since we may be // modifying the cache, we need to have exclusive control over it. We must // copy the group cache entry because the original may be deleted at any time. // NetGroup_Cache_Context.Lock(); if ((glist = NetGroup_Cache.Find(uh_key))) {if (glist->First()) glist = new XrdAccGroupList(*glist); else glist = 0; NetGroup_Cache_Context.UnLock(); return glist; } NetGroup_Cache_Context.UnLock(); // For each known netgroup, check to see if the user is in the netgroup. // GroupTab.user = user; GroupTab.host = host; GroupTab.gtabi = 0; Group_Name_Context.Lock(); NetGroup_Names.Apply(XrdAccCheckNetGroup, (void *)&GroupTab); Group_Name_Context.UnLock(); // Allocate a new GroupList object // glist = new XrdAccGroupList(GroupTab.gtabi, (const char **)GroupTab.Gtab); // Add this user to the group cache to speed things up the next time // NetGroup_Cache_Context.Lock(); NetGroup_Cache.Add((const char *)uh_key, glist, LifeTime); NetGroup_Cache_Context.UnLock(); // Return a copy of the group list // if (!GroupTab.gtabi) return (XrdAccGroupList *)0; return new XrdAccGroupList(GroupTab.gtabi, (const char **)GroupTab.Gtab); } /******************************************************************************/ /* P u r g e C a c h e */ /******************************************************************************/ void XrdAccGroups::PurgeCache() { // Purge the group cache // Group_Cache_Context.Lock(); Group_Cache.Purge(); Group_Cache_Context.UnLock(); // Purge the netgroup cache // NetGroup_Cache_Context.Lock(); NetGroup_Cache.Purge(); NetGroup_Cache_Context.UnLock(); } /******************************************************************************/ /* R e t r a n */ /******************************************************************************/ int XrdAccGroups::Retran(const gid_t gid) { if ((int)gid < 0) retrancnt = 0; else {if (retrancnt > (int)(sizeof(retrangid)/sizeof(gid_t))) return -1; retrangid[retrancnt++] = gid; } return 0; } /******************************************************************************/ /* P r i v a t e M e t h o d s */ /******************************************************************************/ /******************************************************************************/ /* a d d G r o u p */ /******************************************************************************/ int XrdAccGroups::addGroup(const char *user, const gid_t gid, char *gname, char **Gtab, int gtabi) { char *gp; // Check if we have room to add another group. We can squeek by such errors // because all it means is that the user normally has fewer privs (which is // not always true, sigh). // if (gtabi >= NGROUPS_MAX) {if (gtabi == NGROUPS_MAX) std::cerr <<"XrdAccGroups: More than " <gr_name; } // Check if we have this group registered. Only a handful of groups are // actually relevant. Ignore the unreferenced groups. If registered, we // need the persistent name because of multi-threading issues. // if (!(gp = Group_Names.Find(gname)) ) return gtabi; // Add the groupname to the table of groups for the user // Gtab[gtabi++] = gp; return gtabi; } /******************************************************************************/ /* D o t r a n */ /******************************************************************************/ char *XrdAccGroups::Dotran(const gid_t gid, char *gname) { int i; // See if the groupname needs to be retranslated. This is necessary // When multiple groups share the same gid due to NIS constraints. // for (i = 0; i < retrancnt; i++) if (retrangid[i] == gid) return (char *)0; return gname; } /******************************************************************************/ /* E x t e r n a l F u n c t i o n s */ /******************************************************************************/ /******************************************************************************/ /* o o a c c _ C h e c k N e t G r o u p */ /******************************************************************************/ int XrdAccCheckNetGroup(const char *netgroup, char *key, void *Arg) { struct XrdAccGroupArgs *grp = static_cast(Arg); // Check if this netgroup, user, host, domain combination exists. // if (innetgr(netgroup, (const char *)grp->host, (const char *)grp->user, XrdAccGroupMaster.Domain())) {if (grp->gtabi >= NGROUPS_MAX) {if (grp->gtabi == NGROUPS_MAX) std::cerr <<"XrdAccGroups: More than " <gtabi <<"netgroups for " <user <Gtab[grp->gtabi] = netgroup; grp->gtabi++; } return 0; } xrootd-5.6.9/src/XrdAcc/XrdAccGroups.hh000066400000000000000000000163401457266313600177130ustar00rootroot00000000000000#ifndef _ACC_GROUPS_H #define _ACC_GROUPS_H /******************************************************************************/ /* */ /* X r d A c c G r o u p s . h h */ /* */ /* (C) 2003 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Deprtment of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include #include "XrdOuc/XrdOucHash.hh" #include "XrdSys/XrdSysPthread.hh" /******************************************************************************/ /* X r d A c c G r o u p L i s t */ /******************************************************************************/ class XrdAccGroupList { public: const char *First() {return grouptab[0];} const char *Next() {if (grouptab[nextgroup]) return grouptab[nextgroup++]; return (const char *)0; } void Reset() {nextgroup = 0;} XrdAccGroupList(const int cnt=0, const char **gtable=0) {int j = (cnt > NGROUPS_MAX ? NGROUPS_MAX : cnt); if (cnt){memcpy((void *)grouptab, (const void *)gtable, (size_t)(j * sizeof(char *))); } memset((void *)&grouptab[cnt], 0, (size_t)((NGROUPS_MAX-j+1)*sizeof(char *))); nextgroup = 0; } XrdAccGroupList(XrdAccGroupList & rv) {memcpy((void *)grouptab,(const void *)rv.grouptab,sizeof(grouptab)); nextgroup = 0; } ~XrdAccGroupList() {} private: const char *grouptab[NGROUPS_MAX+1]; int nextgroup; }; /******************************************************************************/ /* G r o u p s O p t i o n s */ /******************************************************************************/ enum XrdAccGroups_Options { Primary_Only = 0x0001, Groups_Debug = 0x8000, No_Group_Opt = 0x0000 }; /******************************************************************************/ /* G r o u p T y p e s */ /******************************************************************************/ enum XrdAccGroupType {XrdAccNoGroup = 0, XrdAccUnixGroup, XrdAccNetGroup}; /******************************************************************************/ /* X r d A c c G r o u p s */ /******************************************************************************/ class XrdAccGroups { public: // Domain() returns whatever we have for the NIS domain. // const char *Domain() {return domain;} // AddName() registers a name in the static name table. This allows us to // avoid copying the strings a table points to when returning a table copy. // If the name was added successfully, a pointer to the name is returned. // Otherwise, zero is returned. // char *AddName(const XrdAccGroupType gtype, const char *name); // FindName() looks up a name in the static name table. // char *FindName(const XrdAccGroupType gtype, const char *name); // Groups() returns all of the relevant groups that a user belongs to. A // null pointer may be returned if no groups are applicable. // XrdAccGroupList *Groups(const char *user); // NetGroups() returns all of the relevant netgroups that the user/host // combination belongs to. A null pointer may be returned is no netgroups // are applicable. // XrdAccGroupList *NetGroups(const char *user, const char *host); // PurgeCache() removes all entries in the various caches. It is called // whenever a new set of access tables has been instantiated. // void PurgeCache(); // Use by the configuration object to set group id's that must be looked up. // int Retran(const gid_t gid); // Use by the configuration object to establish the netgroup domain. // void SetDomain(const char *dname) {domain = dname;} // Used by the configuration object to set the cache lifetime. // void SetLifetime(const int seconds) {LifeTime = (int)seconds;} // Used by the configuration object to set various options // void SetOptions(XrdAccGroups_Options opts) {options = opts;} XrdAccGroups(); ~XrdAccGroups() {} // The group object never gets deleted!! private: int addGroup(const char *user, const gid_t gid, char *gname, char **Gtab, int gtabi); char *Dotran(const gid_t gid, char *gname); gid_t retrangid[128]; // Up to 128 retranslatable gids int retrancnt; // Number of used entries time_t LifeTime; // Seconds we can keep something in the cache const char *domain; // NIS netgroup domain to use XrdAccGroups_Options options;// Various option values. int HaveGroups; int HaveNetGroups; XrdSysMutex Group_Build_Context, Group_Name_Context; XrdSysMutex Group_Cache_Context, NetGroup_Cache_Context; XrdOucHash NetGroup_Cache; XrdOucHash Group_Cache; XrdOucHash Group_Names; XrdOucHash NetGroup_Names; }; #endif xrootd-5.6.9/src/XrdAcc/XrdAccPrivs.hh000066400000000000000000000112671457266313600175420ustar00rootroot00000000000000#ifndef __ACC_PRIVS__ #define __ACC_PRIVS__ /******************************************************************************/ /* */ /* X r d A c c P r i v s . h h */ /* */ /* (c) 2003 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ /******************************************************************************/ /* X r d A c c P r i v s */ /******************************************************************************/ // Recognized privileges // enum XrdAccPrivs {XrdAccPriv_All = 0x07f, XrdAccPriv_Chmod = 0x063, // Insert + Open r/w + Delete XrdAccPriv_Chown = 0x063, // Insert + Open r/w + Delete XrdAccPriv_Create = 0x062, // Insert + Open r/w XrdAccPriv_Delete = 0x001, XrdAccPriv_Insert = 0x002, XrdAccPriv_Lock = 0x004, XrdAccPriv_Mkdir = 0x002, // Insert XrdAccPriv_Lookup = 0x008, XrdAccPriv_Rename = 0x010, XrdAccPriv_Read = 0x020, XrdAccPriv_Readdir= 0x020, XrdAccPriv_Write = 0x040, XrdAccPriv_Update = 0x060, XrdAccPriv_None = 0x000 }; /******************************************************************************/ /* X r d A c c P r i v S p e c */ /******************************************************************************/ // The following are the 1-letter privileges that we support. // enum XrdAccPrivSpec { All_Priv = 'a', Delete_Priv = 'd', Insert_Priv = 'i', Lock_Priv = 'k', Lookup_Priv = 'l', Rename_Priv = 'n', Read_Priv = 'r', Write_Priv = 'w', Neg_Priv = '-' }; /******************************************************************************/ /* X r d A c c P r i v C a p s */ /******************************************************************************/ struct XrdAccPrivCaps {XrdAccPrivs pprivs; // Positive privileges XrdAccPrivs nprivs; // Negative privileges XrdAccPrivCaps() {pprivs = XrdAccPriv_None; nprivs = XrdAccPriv_None; } ~XrdAccPrivCaps() {} }; #endif xrootd-5.6.9/src/XrdApps.cmake000066400000000000000000000143151457266313600162450ustar00rootroot00000000000000 #------------------------------------------------------------------------------- # Modules #------------------------------------------------------------------------------- set( LIB_XRDCL_PROXY_PLUGIN XrdClProxyPlugin-${PLUGIN_VERSION} ) set( LIB_XRDCL_RECORDER_PLUGIN XrdClRecorder-${PLUGIN_VERSION} ) #------------------------------------------------------------------------------- # Shared library version #------------------------------------------------------------------------------- set( XRD_APP_UTILS_VERSION 2.0.0 ) set( XRD_APP_UTILS_SOVERSION 2 ) #------------------------------------------------------------------------------- # This part is NOT built for XrdCl only builds #------------------------------------------------------------------------------- if( NOT XRDCL_ONLY ) #----------------------------------------------------------------------------- # xrdadler32 #----------------------------------------------------------------------------- add_executable( xrdadler32 XrdApps/Xrdadler32.cc ) target_link_libraries( xrdadler32 XrdPosix XrdUtils ZLIB::ZLIB ${CMAKE_THREAD_LIBS_INIT}) #----------------------------------------------------------------------------- # xrdcks #----------------------------------------------------------------------------- add_executable( xrdcks XrdApps/XrdCks.cc ) target_link_libraries( xrdcks XrdUtils ) #----------------------------------------------------------------------------- # xrdcrc32c #----------------------------------------------------------------------------- add_executable( xrdcrc32c XrdApps/XrdCrc32c.cc ) target_link_libraries( xrdcrc32c XrdUtils ) #----------------------------------------------------------------------------- # cconfig #----------------------------------------------------------------------------- add_executable( cconfig XrdApps/XrdAppsCconfig.cc ) target_link_libraries( cconfig XrdUtils ) #----------------------------------------------------------------------------- # mpxstats #----------------------------------------------------------------------------- add_executable( mpxstats XrdApps/XrdMpxStats.cc ) target_link_libraries( mpxstats XrdAppUtils XrdUtils ${EXTRA_LIBS} ${CMAKE_THREAD_LIBS_INIT} ${SOCKET_LIBRARY} ) #----------------------------------------------------------------------------- # xrdprep #----------------------------------------------------------------------------- add_executable( xrdprep XrdApps/XrdPrep.cc ) target_link_libraries( xrdprep XrdCl XrdUtils ) #----------------------------------------------------------------------------- # wait41 #----------------------------------------------------------------------------- add_executable( wait41 XrdApps/XrdWait41.cc ) target_link_libraries( wait41 XrdUtils ${CMAKE_THREAD_LIBS_INIT} ${EXTRA_LIBS} ) #----------------------------------------------------------------------------- # xrdacctest #----------------------------------------------------------------------------- add_executable( xrdacctest XrdApps/XrdAccTest.cc ) target_link_libraries( xrdacctest XrdServer XrdUtils ) #------------------------------------------------------------------------------- # xrdmapc #------------------------------------------------------------------------------- add_executable( xrdmapc XrdApps/XrdMapCluster.cc ) target_link_libraries( xrdmapc XrdCl XrdUtils ) #------------------------------------------------------------------------------- # xrdpinls #------------------------------------------------------------------------------- add_executable( xrdpinls XrdApps/XrdPinls.cc ) #------------------------------------------------------------------------------- # xrdqstats #------------------------------------------------------------------------------- add_executable( xrdqstats XrdApps/XrdQStats.cc ) target_link_libraries( xrdqstats XrdCl XrdAppUtils XrdUtils ${EXTRA_LIBS} ) endif() #------------------------------------------------------------------------------- # AppUtils #------------------------------------------------------------------------------- add_library( XrdAppUtils SHARED XrdApps/XrdCpConfig.cc XrdApps/XrdCpConfig.hh XrdApps/XrdCpFile.cc XrdApps/XrdCpFile.hh XrdApps/XrdMpxXml.cc XrdApps/XrdMpxXml.hh ) target_link_libraries( XrdAppUtils PRIVATE XrdUtils ) set_target_properties( XrdAppUtils PROPERTIES VERSION ${XRD_APP_UTILS_VERSION} SOVERSION ${XRD_APP_UTILS_SOVERSION} ) #------------------------------------------------------------------------------- # XrdClProxyPlugin library #------------------------------------------------------------------------------- add_library( ${LIB_XRDCL_PROXY_PLUGIN} MODULE XrdApps/XrdClProxyPlugin/ProxyPrefixPlugin.cc XrdApps/XrdClProxyPlugin/ProxyPrefixFile.cc) target_link_libraries(${LIB_XRDCL_PROXY_PLUGIN} PRIVATE XrdCl) #------------------------------------------------------------------------------- # XrdClRecorder library #------------------------------------------------------------------------------- add_library( ${LIB_XRDCL_RECORDER_PLUGIN} MODULE XrdApps/XrdClRecordPlugin/XrdClRecorderPlugin.cc ) target_link_libraries(${LIB_XRDCL_RECORDER_PLUGIN} PRIVATE XrdCl) add_executable( xrdreplay XrdApps/XrdClRecordPlugin/XrdClReplay.cc ) target_link_libraries( xrdreplay ${CMAKE_THREAD_LIBS_INIT} XrdCl XrdUtils ) #------------------------------------------------------------------------------- # Install #------------------------------------------------------------------------------- install( TARGETS XrdAppUtils ${LIB_XRDCL_PROXY_PLUGIN} ${LIB_XRDCL_RECORDER_PLUGIN} LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} ) if( NOT XRDCL_ONLY ) install( TARGETS xrdacctest xrdadler32 cconfig mpxstats wait41 xrdmapc xrdpinls xrdcrc32c xrdcks LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} ) endif() install( TARGETS xrdreplay RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} ) xrootd-5.6.9/src/XrdApps/000077500000000000000000000000001457266313600152375ustar00rootroot00000000000000xrootd-5.6.9/src/XrdApps/XrdAccTest.cc000066400000000000000000000344011457266313600175540ustar00rootroot00000000000000/******************************************************************************/ /* */ /* X r d A c c T e s t . c c */ /* */ /* (c) 2017 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include #include #include #include #include #include #include #include #include #include "XrdVersion.hh" #include "XrdAcc/XrdAccAuthorize.hh" #include "XrdAcc/XrdAccConfig.hh" #include "XrdAcc/XrdAccGroups.hh" #include "XrdAcc/XrdAccPrivs.hh" #include "XrdSys/XrdSysError.hh" #include "XrdSys/XrdSysHeaders.hh" #include "XrdSys/XrdSysLogger.hh" #include "XrdNet/XrdNetAddr.hh" #include "XrdOuc/XrdOucEnv.hh" #include "XrdOuc/XrdOucStream.hh" /******************************************************************************/ /* G l o b a l O b j e c t s */ /******************************************************************************/ char *PrivsConvert(XrdAccPrivCaps &ctab, char *buff, int blen); XrdAccAuthorize *Authorize; int extra; XrdSysLogger myLogger; XrdSysError eroute(&myLogger, "acc_"); namespace { XrdSecEntity Entity("host"); XrdNetAddr netAddr; bool v2 = false; } /******************************************************************************/ /* O p e r a t i o n T a b l e */ /******************************************************************************/ typedef struct {const char *opname; Access_Operation oper;} optab_t; optab_t optab[] = {{"?", AOP_Any}, {"cm", AOP_Chmod}, {"co", AOP_Chown}, {"cr", AOP_Create}, {"ec", AOP_Excl_Create}, {"rm", AOP_Delete}, {"lk", AOP_Lock}, {"mk", AOP_Mkdir}, {"mv", AOP_Rename}, {"ei", AOP_Excl_Insert}, {"rd", AOP_Read}, {"ls", AOP_Readdir}, {"st", AOP_Stat}, {"wr", AOP_Update} }; int opcnt = sizeof(optab)/sizeof(optab[0]); /******************************************************************************/ /* U s a g e */ /******************************************************************************/ void Usage(const char *msg) { if (msg) std::cerr <<"xrdacctest: " <] [ | ] \n\n"; std::cerr <<": -a -g -h -o -r -u \n"; std::cerr <<": [ [...]]\n"; std::cerr <<": cr - create mv - rename st - status lk - lock\n"; std::cerr <<" rd - read wr - write ls - readdir rm - remove\n"; std::cerr <<" ec - excl create ei - excl rename\n"; std::cerr <<" * - zap args ? - display privs\n"; std::cerr << std::flush; exit(msg ? 1 : 0); } /******************************************************************************/ /* S e t I D */ /******************************************************************************/ void SetID(char *&dest, char *val) { if (dest) free(dest); dest = (strcmp(val, "none") ? strdup(val) : 0); } /******************************************************************************/ /* Z a p E n t i t y */ /******************************************************************************/ void ZapEntity() { strncpy(Entity.prot, "host", sizeof(Entity.prot)); if (Entity.grps) free(Entity.grps); Entity.grps = 0; if (Entity.host) free(Entity.host); Entity.host = 0; if (Entity.vorg) free(Entity.vorg); Entity.vorg = 0; if (Entity.role) free(Entity.role); Entity.role = 0; if (Entity.name) free(Entity.name); Entity.name = 0; } /******************************************************************************/ /* m a i n */ /******************************************************************************/ int main(int argc, char **argv) { static XrdVERSIONINFODEF(myVer, XrdAccTest, XrdVNUMBER, XrdVERSION); extern int optind; extern char *optarg; extern XrdAccAuthorize *XrdAccDefaultAuthorizeObject(XrdSysLogger *lp, const char *cfn, const char *parm, XrdVersionInfo &myVer); int DoIt(int argpnt, int argc, char **argv, bool singleshot); const char *cfHost = "localhost", *cfProg = "xrootd"; char *p2l(XrdAccPrivs priv, char *buff, int blen); char *argval[32], buff[255], tident[80], c; int DoIt(int argnum, int argc, char **argv, int singleshot); XrdOucStream Command; const int maxargs = sizeof(argval)/sizeof(argval[0]); char *at, *lp, *ConfigFN = (char *)"./acc.cf"; int argnum, rc = 0; bool singleshot=false; // Print help if no args // if (argc == 1) Usage(0); Entity.addrInfo = &netAddr; sprintf(tident, "acctest.%d:0@localhost", getpid()); Entity.tident = tident; // Get all of the options. // while ((c=getopt(argc,argv,"a:c:de:g:h:o:r:u:s")) != (char)EOF) { switch(c) { case 'a': {size_t size = sizeof(Entity.prot)-1; strncpy(Entity.prot, optarg, size); Entity.prot[size] = '\0'; } v2 = true; break; case 'd': break; case 'e': Entity.ueid = atoi(optarg); v2 = true; break; case 'g': SetID(Entity.grps, optarg); v2 = true; break; case 'h': SetID(Entity.host, optarg); v2 = true; break; case 'o': SetID(Entity.vorg, optarg); v2 = true; break; case 'r': SetID(Entity.role, optarg); v2 = true; break; case 'u': SetID(Entity.name, optarg); v2 = true; break; case 'c': ConfigFN = optarg; break; case 's': singleshot = true; break; default: sprintf(buff, "-%c option is invalid.", c); Usage(buff); } } // Establish environment // if ((at = index(ConfigFN, '@'))) {*at++ = 0; if (*at) cfHost = at;} sprintf(buff, "%s anon@%s", cfProg, cfHost); XrdOucEnv::Export("XRDINSTANCE", buff); // Obtain the authorization object // if (!(Authorize = XrdAccDefaultAuthorizeObject(&myLogger, ConfigFN, 0, myVer))) {std::cerr << "testaccess: Initialization failed." <= argc) {sprintf(buff, "%s option value not specified.", opc); Usage(buff); } opv = argv[argpnt++]; if (strlen(opc) != 2) {sprintf(buff, "%s option is invalid.", opc); Usage(buff); } switch(*(opc+1)) {case 'a': {size_t size = sizeof(Entity.prot)-1; strncpy(Entity.prot, opv, size); Entity.prot[size] = '\0'; } v2 = true; break; case 'e': Entity.ueid = atoi(opv); v2 = true; break; case 'g': SetID(Entity.grps, opv); v2 = true; break; case 'h': SetID(Entity.host, opv); v2 = true; break; case 'o': SetID(Entity.vorg, opv); v2 = true; break; case 'r': SetID(Entity.role, opv); v2 = true; break; case 'u': SetID(Entity.name, opv); v2 = true; break; default: sprintf(buff, "%s option is invalid.", opc); Usage(buff); break; } } // Make sure user and host specified if v1 version being used // if (!v2) {if (argpnt >= argc) Usage("user not specified."); Entity.name = argv[argpnt++]; if (argpnt >= argc) Usage("host not specified."); Entity.host = argv[argpnt++]; } // Make sure op specified unless we are v2 // if (argpnt >= argc) {if (v2) return 0; else Usage("operation not specified."); } if (!strcmp(argv[argpnt], "*")) {ZapEntity(); return 0; } optype = cmd2op(argv[argpnt++]); // Make sure path specified // if (argpnt >= argc) Usage("path not specified."); // Set host, ignore errors // if (Entity.host) netAddr.Set(Entity.host, 0); // Process each path, as needed // while(argpnt < argc) {path = argv[argpnt++]; auth = Authorize->Access((const XrdSecEntity *)&Entity, (const char *)path, optype); if (optype != AOP_Any) result=(auth?(char *)"allowed":(char *)"denied"); else {pargs.pprivs = auth; pargs.nprivs = XrdAccPriv_None; result = PrivsConvert(pargs, buff, sizeof(buff)); } std::cout <. */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include #include #include #include #include #include "XrdNet/XrdNetAddr.hh" #include "XrdOuc/XrdOucEnv.hh" #include "XrdOuc/XrdOucNList.hh" #include "XrdOuc/XrdOucStream.hh" #include "XrdOuc/XrdOucUtils.hh" #include "XrdSys/XrdSysE2T.hh" #include "XrdSys/XrdSysError.hh" #include "XrdSys/XrdSysLogger.hh" #include "XrdSys/XrdSysHeaders.hh" #include "XrdSys/XrdSysPthread.hh" /******************************************************************************/ /* i n L i s t */ /******************************************************************************/ int inList(const char *var, const char **Vec) { int i = 0; while(Vec[i] && strcmp(Vec[i],var)) i++; return (Vec[i] != 0); } /******************************************************************************/ /* U s a g e */ /******************************************************************************/ void Usage(int rc) { std::cerr <<"\n Usage: cconfig -c [-h ] [-n ] [-x ] []" "\n: [[pfx]*] | [*[sfx]] []" < 1 && '-' == *argv[1]) while ((c = getopt(argc,argv,":c:h:n:x:")) && ((unsigned char)c != 0xff)) { switch(c) { case 'c': Cfn = optarg; break; case 'h': Host= optarg; break; case 'n': Name= optarg; break; case 'x': Xeq = optarg; break; default: sprintf(buff,"'%c'", optopt); if (c == ':') Say.Say(Pgm, buff, " value not specified."); else Say.Say(Pgm, buff, " option is invalid."); Usage(1); } } // Make sure config file has been specified // if (!Cfn) {Say.Say(Pgm, "Config file not specified."); Usage(1);} // Get full host name // if (!Host) Host = theAddr.Name(); else if (!theAddr.Set(Host,0)) Host = theAddr.Name(); if (!Host) {Say.Say(Pgm, "Unable to determine host name."); exit(3);} // Prepare all selector arguments // for (i = optind; i < argc; i++) DirQ.Replace(argv[i],0); chkQ = (DirQ.First() != 0); // Open the config file // if ( (cfgFD = open(Cfn, O_RDONLY, 0)) < 0) {Say.Say(Pgm, XrdSysE2T(errno), " opening config file ", Cfn); exit(4); } // Construct instance name and stream // Name = XrdOucUtils::InstName(Name); sprintf(buff,"%s %s@%s", Xeq, Name, Host); Config = new XrdOucStream(&Say, strdup(buff), &myEnv, ""); Config->Attach(cfgFD); // Now start reading records until eof. // while((var = Config->GetMyFirstWord())) {if (chkQ && !DirQ.Find(var)) {Config->noEcho(); continue;} if (inList(var, noSub)) {if (inList(var, slChk)) while((var = Config->GetWord()) && *var != '/') {} oldEnv = Config->SetEnv(0); if (var) Config->GetRest(buff, sizeof(buff)); Config->SetEnv(oldEnv); } else if (inList(var, ifChk)) {while((var = Config->GetWord()) && strcmp(var, "if")) {} if (var && !XrdOucUtils::doIf(&Say, *Config, "directive", Host, Name, Xeq)) {Config->noEcho(); continue;} } else Config->GetRest(buff, sizeof(buff)); Config->Echo(); } // Now check if any errors occurred during file i/o // if ((retc = Config->LastError())) {Say.Say(Pgm, XrdSysE2T(retc), " reading config file ", Cfn); retc = 8;} Config->Close(); // Should never get here // exit(retc); } xrootd-5.6.9/src/XrdApps/XrdCks.cc000066400000000000000000000144341457266313600167520ustar00rootroot00000000000000/******************************************************************************/ /* */ /* X r d C k s . c c */ /* */ /* (c) 2022 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include #include #include #include #include #include #include #include #include "XrdCks/XrdCksXAttr.hh" #include "XrdOuc/XrdOucXAttr.hh" /******************************************************************************/ /* G l o b a l D a t a */ /******************************************************************************/ XrdOucXAttr xCS; struct stat Stat; const char *csCmd; /******************************************************************************/ /* D i s p l a y */ /******************************************************************************/ void Display() { const char *stale; char csBuff[512]; // Check if the checksum is stale // if (xCS.Attr.Cks.fmTime != Stat.st_mtime) stale = " stale"; else stale = ""; // Get displayable checksum // xCS.Attr.Cks.Get(csBuff, sizeof(csBuff)); // Display the information // std::cout < [|delete]\n" " xrdcks -h\n\n" "Where: - path to the target file\n" " - checksum name (e.g. adler32)\n" " - ASCII hex of value (even number of digits)" < 1 && !strcmp(argv[1], "-h")) Usage(0); Usage(1); } // Verify the name // csName = argv[2]; if (!xCS.Attr.Cks.Set(csName)) {std::cerr <<"xrdsetcks: checksum name '"<(Stat.st_mtime); xCS.Attr.Cks.csTime = 0; if ((rc = xCS.Set(csPath))) Unable(rc); exit(0); } xrootd-5.6.9/src/XrdApps/XrdClProxyPlugin/000077500000000000000000000000001457266313600204745ustar00rootroot00000000000000xrootd-5.6.9/src/XrdApps/XrdClProxyPlugin/ProxyPrefixFile.cc000066400000000000000000000155731457266313600241150ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2017 by European Organization for Nuclear Research (CERN) // Author: Elvin Sindrilaru //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #include "ProxyPrefixFile.hh" #include #include #include #include #include #include "XrdCl/XrdClLog.hh" #include #include namespace xrdcl_proxy { //------------------------------------------------------------------------------ // Constructor //------------------------------------------------------------------------------ ProxyPrefixFile::ProxyPrefixFile(): mIsOpen(false), pFile(0) {} //------------------------------------------------------------------------------ // Destructor //------------------------------------------------------------------------------ ProxyPrefixFile::~ProxyPrefixFile() { if (pFile) { delete pFile; } } //------------------------------------------------------------------------------ // Open //------------------------------------------------------------------------------ XRootDStatus ProxyPrefixFile::Open(const std::string& url, OpenFlags::Flags flags, Access::Mode mode, ResponseHandler* handler, uint16_t timeout) { XRootDStatus st; if (mIsOpen) { st = XRootDStatus(stError, errInvalidOp); return st; } pFile = new XrdCl::File(false); std::string open_url = ConstructFinalUrl(url); st = pFile->Open(open_url, flags, mode, handler, timeout); if (st.IsOK()) { mIsOpen = true; } return st; } //------------------------------------------------------------------------------ // Get proxy prefix Url //------------------------------------------------------------------------------ std::string ProxyPrefixFile::GetPrefixUrl() const { std::string url_prefix = (getenv("XROOT_PROXY") ? getenv("XROOT_PROXY") : ""); // Try out also the lower-case one if (url_prefix.empty()) { url_prefix = (getenv("xroot_proxy") ? getenv("xroot_proxy") : ""); } return url_prefix; } //------------------------------------------------------------------------------ // Trim whitespaces from both ends for a string //------------------------------------------------------------------------------ std::string ProxyPrefixFile::trim(const std::string& in) const { std::string::const_iterator wsfront, wsback; std::string::const_reverse_iterator rwsback; wsfront = in.begin(); rwsback = in.rbegin(); while (*wsfront == ' ') { ++wsfront; } while (*rwsback == ' ') { ++rwsback; } wsback = rwsback.base(); return (wsback <= wsfront ? std::string() : std::string(wsfront, wsback)); /* TODO: To be used when C++11 is available auto wsfront = std::find_if_not(in.begin(), in.end(), [](int c) -> bool {return std::isspace(c);}); auto wsback = std::find_if_not(in.rbegin(), in.rend(), [](int c) -> bool {return std::isspace(c);}).base(); return (wsback <= wsfront ? std::string() : std::string(wsfront, wsback)); */ } //------------------------------------------------------------------------------ // Get list of domains which are NOT to be prefixed //------------------------------------------------------------------------------ std::list ProxyPrefixFile::GetExclDomains() const { std::string excl_domains = (getenv("XROOT_PROXY_EXCL_DOMAINS") ? getenv("XROOT_PROXY_EXCL_DOMAINS") : ""); if (excl_domains.empty()) { return std::list(); } char delim = ','; std::string item; std::list lst; std::stringstream ss(excl_domains); while (getline(ss, item, delim)) { lst.push_back(trim(item)); } return lst; } //------------------------------------------------------------------------------ // Construct final Url //------------------------------------------------------------------------------ std::string ProxyPrefixFile::ConstructFinalUrl(const std::string& orig_surl) const { std::string final_surl = orig_surl; std::string url_prefix = GetPrefixUrl(); XrdCl::Log* log = DefaultEnv::GetLog(); log->Debug(1, "url=%s, prefix_url=%s", orig_surl.c_str(), url_prefix.c_str()); if (!url_prefix.empty()) { bool exclude = false; std::list lst_excl = GetExclDomains(); XrdCl::URL orig_url(orig_surl); std::string orig_host = orig_url.GetHostId(); // Remove port if present size_t pos = orig_host.find(':'); if (pos != std::string::npos) { orig_host = orig_host.substr(0, pos); } orig_host = GetFqdn(orig_host); for (std::list::iterator it_excl = lst_excl.begin(); it_excl != lst_excl.end(); ++it_excl) { if (url_prefix.size() < it_excl->size()) { continue; } if (std::equal(it_excl->rbegin(), it_excl->rend(), orig_host.rbegin())) { exclude = true; break; } } if (!exclude) { final_surl.insert(0, url_prefix); } } log->Debug(1, "final_url=%s", final_surl.c_str()); return final_surl; } //------------------------------------------------------------------------------ // Get FQDN for specified host //------------------------------------------------------------------------------ std::string ProxyPrefixFile::GetFqdn(const std::string& hostname) const { XrdCl::Log* log = DefaultEnv::GetLog(); std::string fqdn = hostname; struct addrinfo hints, *info; int gai_result; memset(&hints, 0, sizeof hints); hints.ai_family = AF_UNSPEC; /*either IPV4 or IPV6*/ hints.ai_socktype = SOCK_STREAM; hints.ai_flags = AI_CANONNAME; if ((gai_result = getaddrinfo(hostname.c_str(), NULL, &hints, &info)) != 0) { log->Error(1, "getaddrinfo: %s", gai_strerror(gai_result)); return fqdn; } if (info) { fqdn = info->ai_canonname; } freeaddrinfo(info); return fqdn; } } // namespace xrdcl_proxy xrootd-5.6.9/src/XrdApps/XrdClProxyPlugin/ProxyPrefixFile.hh000066400000000000000000000301231457266313600241130ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2017 by European Organization for Nuclear Research (CERN) // Author: Elvin Sindrilaru //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #pragma once #include "XrdCl/XrdClDefaultEnv.hh" #include "XrdCl/XrdClPlugInInterface.hh" #include using namespace XrdCl; namespace xrdcl_proxy { //------------------------------------------------------------------------------ //! XrdClFile plugin that appends an URL prefix to the given URL. The URL //! prefix is set as an environment variable XRD_URL_PREFIX. //------------------------------------------------------------------------------ class ProxyPrefixFile: public XrdCl::FilePlugIn { public: //---------------------------------------------------------------------------- //! Constructor //---------------------------------------------------------------------------- ProxyPrefixFile(); //---------------------------------------------------------------------------- //! Destructor //---------------------------------------------------------------------------- virtual ~ProxyPrefixFile() override; //---------------------------------------------------------------------------- //! Open //---------------------------------------------------------------------------- virtual XRootDStatus Open(const std::string& url, OpenFlags::Flags flags, Access::Mode mode, ResponseHandler* handler, uint16_t timeout) override; //---------------------------------------------------------------------------- //! Close //---------------------------------------------------------------------------- virtual XRootDStatus Close(ResponseHandler* handler, uint16_t timeout) override { return pFile->Close(handler, timeout); } //---------------------------------------------------------------------------- //! Stat //---------------------------------------------------------------------------- virtual XRootDStatus Stat(bool force, ResponseHandler* handler, uint16_t timeout) override { return pFile->Stat(force, handler, timeout); } //---------------------------------------------------------------------------- //! Read //---------------------------------------------------------------------------- virtual XRootDStatus Read(uint64_t offset, uint32_t size, void* buffer, ResponseHandler* handler, uint16_t timeout) override { return pFile->Read(offset, size, buffer, handler, timeout); } //------------------------------------------------------------------------ //! PgRead //------------------------------------------------------------------------ virtual XRootDStatus PgRead( uint64_t offset, uint32_t size, void *buffer, ResponseHandler *handler, uint16_t timeout ) override { return pFile->PgRead(offset, size, buffer, handler, timeout); } //---------------------------------------------------------------------------- //! Write //---------------------------------------------------------------------------- virtual XRootDStatus Write(uint64_t offset, uint32_t size, const void* buffer, ResponseHandler* handler, uint16_t timeout) override { return pFile->Write(offset, size, buffer, handler, timeout); } //------------------------------------------------------------------------ //! Write //------------------------------------------------------------------------ virtual XRootDStatus Write( uint64_t offset, Buffer &&buffer, ResponseHandler *handler, uint16_t timeout = 0 ) override { return pFile->Write(offset, std::move(buffer), handler, timeout); } //------------------------------------------------------------------------ //! Write //------------------------------------------------------------------------ virtual XRootDStatus Write( uint64_t offset, uint32_t size, Optional fdoff, int fd, ResponseHandler *handler, uint16_t timeout = 0 ) override { return pFile->Write(offset, size, fdoff, fd, handler, timeout); } //------------------------------------------------------------------------ //! PgWrite //------------------------------------------------------------------------ virtual XRootDStatus PgWrite( uint64_t offset, uint32_t nbpgs, const void *buffer, std::vector &cksums, ResponseHandler *handler, uint16_t timeout ) override { return pFile->PgWrite(offset, nbpgs, buffer, cksums, handler, timeout); } //---------------------------------------------------------------------------- //! Sync //---------------------------------------------------------------------------- virtual XRootDStatus Sync(ResponseHandler* handler, uint16_t timeout) override { return pFile->Sync(handler, timeout); } //---------------------------------------------------------------------------- //! Truncate //---------------------------------------------------------------------------- virtual XRootDStatus Truncate(uint64_t size, ResponseHandler* handler, uint16_t timeout) override { return pFile->Truncate(size, handler, timeout); } //---------------------------------------------------------------------------- //! VectorRead //---------------------------------------------------------------------------- virtual XRootDStatus VectorRead(const ChunkList& chunks, void* buffer, ResponseHandler* handler, uint16_t timeout) override { return pFile->VectorRead(chunks, buffer, handler, timeout); } //------------------------------------------------------------------------ //! VectorWrite //------------------------------------------------------------------------ virtual XRootDStatus VectorWrite( const ChunkList &chunks, ResponseHandler *handler, uint16_t timeout = 0 ) override { return pFile->VectorWrite(chunks, handler, timeout); } //------------------------------------------------------------------------ //! @see XrdCl::File::WriteV //------------------------------------------------------------------------ virtual XRootDStatus WriteV( uint64_t offset, const struct iovec *iov, int iovcnt, ResponseHandler *handler, uint16_t timeout = 0 ) override { return pFile->WriteV(offset, iov, iovcnt, handler, timeout); } //---------------------------------------------------------------------------- //! Fcntl //---------------------------------------------------------------------------- virtual XRootDStatus Fcntl(const Buffer& arg, ResponseHandler* handler, uint16_t timeout) override { return pFile->Fcntl(arg, handler, timeout); } //---------------------------------------------------------------------------- //! Visa //---------------------------------------------------------------------------- virtual XRootDStatus Visa(ResponseHandler* handler, uint16_t timeout) override { return pFile->Visa(handler, timeout); } //---------------------------------------------------------------------------- //! IsOpen //---------------------------------------------------------------------------- virtual bool IsOpen() const override { return pFile->IsOpen(); } //---------------------------------------------------------------------------- //! SetProperty //---------------------------------------------------------------------------- virtual bool SetProperty(const std::string& name, const std::string& value) override { return pFile->SetProperty(name, value); } //---------------------------------------------------------------------------- //! GetProperty //---------------------------------------------------------------------------- virtual bool GetProperty(const std::string& name, std::string& value) const override { return pFile->GetProperty(name, value); } private: //---------------------------------------------------------------------------- //! Trim whitespaces from both ends of a string //! //! @return trimmed string //---------------------------------------------------------------------------- inline std::string trim(const std::string& in) const; //---------------------------------------------------------------------------- //! Get proxy prefix URL from the environment //! //! @return proxy prefix RUL //---------------------------------------------------------------------------- inline std::string GetPrefixUrl() const; //---------------------------------------------------------------------------- //! Get list of domains which are NOT to be prefixed //! //! @return list of excluded domains //---------------------------------------------------------------------------- std::list GetExclDomains() const; //---------------------------------------------------------------------------- //! Construct final URL if there is a proxy prefix URL specified and if the //! exclusion list is satisfied //! //! @param orig_url original url //! //! @return final URL //---------------------------------------------------------------------------- std::string ConstructFinalUrl(const std::string& orig_url) const; //---------------------------------------------------------------------------- //! Get FQDN for specified host //! //! @param hostname hostname without domain //! //! @return FQDN //---------------------------------------------------------------------------- std::string GetFqdn(const std::string& hostname) const; bool mIsOpen; XrdCl::File* pFile; }; } // namespace xrdcl_proxy xrootd-5.6.9/src/XrdApps/XrdClProxyPlugin/ProxyPrefixPlugin.cc000066400000000000000000000101661457266313600244650ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2017 by European Organization for Nuclear Research (CERN) // Author: Elvin Sindrilaru //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #include "ProxyPrefixPlugin.hh" #include "ProxyPrefixFile.hh" #include "XrdVersion.hh" #include "XrdCl/XrdClDefaultEnv.hh" #include "XrdCl/XrdClLog.hh" #include #include XrdVERSIONINFO(XrdClGetPlugIn, XrdClGetPlugIn) extern "C" { void* XrdClGetPlugIn(const void* arg) { const std::map* config = static_cast< const std::map* >(arg); return static_cast(new xrdcl_proxy::ProxyFactory(config)); } } namespace xrdcl_proxy { //------------------------------------------------------------------------------ // Construtor //------------------------------------------------------------------------------ ProxyFactory::ProxyFactory(const std::map* config) { XrdCl::Log* log = XrdCl::DefaultEnv::GetLog(); // If any of the parameters specific to this plugin are present then export // them as env variables to be used later on if not already set. if (config) { // When C++11 is here: // std::list lst_envs {"XROOT_PROXY", "xroot_proxy", // "XROOT_PROXY_EXCL_DOMAINS", // "xroot_proxy_excl_domains}; std::list lst_envs; lst_envs.push_back("XROOT_PROXY"); lst_envs.push_back("xroot_proxy"); lst_envs.push_back("XROOT_PROXY_EXCL_DOMAINS"); lst_envs.push_back("xroot_proxy_excl_domains"); for (std::list::iterator it_env = lst_envs.begin(); it_env != lst_envs.end(); ++it_env) { std::map::const_iterator it_map = config->find(*it_env); if (it_map != config->end() && !it_map->second.empty()) { if (setenv(it_map->first.c_str(), it_map->second.c_str(), 0)) { log->Error(1, "Failed to set env variable %s from the configuration" " file", it_map->first.c_str()); } } } } } //------------------------------------------------------------------------------ // Destructor //------------------------------------------------------------------------------ ProxyFactory::~ProxyFactory() {} //------------------------------------------------------------------------------ // Create a file plug-in for the given URL //------------------------------------------------------------------------------ XrdCl::FilePlugIn* ProxyFactory::CreateFile(const std::string& url) { return static_cast(new ProxyPrefixFile()); } //------------------------------------------------------------------------------ // Create a file system plug-in for the given URL //------------------------------------------------------------------------------ XrdCl::FileSystemPlugIn* ProxyFactory::CreateFileSystem(const std::string& url) { XrdCl::Log* log = XrdCl::DefaultEnv::GetLog(); log->Error(1, "FileSystem plugin implementation not supported"); return static_cast(0); } } // namespace xrdcl_proxy xrootd-5.6.9/src/XrdApps/XrdClProxyPlugin/ProxyPrefixPlugin.hh000066400000000000000000000052411457266313600244750ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2017 by European Organization for Nuclear Research (CERN) // Author: Elvin Sindrilaru //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #pragma once #include "XrdCl/XrdClPlugInInterface.hh" namespace xrdcl_proxy { //------------------------------------------------------------------------------ //! XrdCl proxy prefix plugin factory //------------------------------------------------------------------------------ class ProxyFactory: public XrdCl::PlugInFactory { public: //---------------------------------------------------------------------------- //! Construtor //! //! @param config map containing configuration parameters //---------------------------------------------------------------------------- ProxyFactory(const std::map* config); //---------------------------------------------------------------------------- //! Destructor //---------------------------------------------------------------------------- virtual ~ProxyFactory(); //---------------------------------------------------------------------------- //! Create a file plug-in for the given URL //---------------------------------------------------------------------------- virtual XrdCl::FilePlugIn* CreateFile(const std::string& url); //---------------------------------------------------------------------------- //! Create a file system plug-in for the given URL //---------------------------------------------------------------------------- virtual XrdCl::FileSystemPlugIn* CreateFileSystem(const std::string& url); }; } // namespace xrdcl_proxy xrootd-5.6.9/src/XrdApps/XrdClProxyPlugin/README.md000066400000000000000000000034361457266313600217610ustar00rootroot00000000000000# XrdClProxyPrefix Plugin This XRootD Client Plugin can be used to tunnel traffic through an XRootD Proxy machine. The proxy endpoint is specified as an environment variable. To enable this plugin the **XRD_PLUGIN** environment variable needs to point to the **libXrdClProxyPlugin.so** library. For example: ```bash XRD_PLUGIN=/usr/lib64/libXrdClProxyPlugin.so \ XROOT_PROXY=root://esvm000:2010// \ xrdcp -f -d 1 root://esvm000//tmp/file1.dat /tmp/dump [1.812kB/1.812kB][100%][==================================================][1.812kB/s] ``` This will first redirect the client to the XRootD server on port 2010 which is a forwarding proxy and then the request will be served by the default XRootD server on port 1094. The user can also specify a list of exclusion domains, for which the original URL will not be modified even if the plugin is enabled. For example: ```bash XRD_PLUGIN=/usr/lib64/libXrdClProxyPlugin.so \ XROOT_PROXY=root://esvm000.cern.ch:2010// \ XROOT_PROXY_EXCL_DOMAINS="some.domain, some.other.domain, cern.ch " \ xrdcp -f -d 1 root://esvm000.cern.ch//tmp/file1.dat /tmp/dump ``` This will not redirect the traffic since the original url "root://esmv000.cern.ch//" contains the "cern.ch" domain which is in the list of excluded domains. There are several environment variables that control the behaviour of this XRootD Client plugin: **XROOT_PROXY/xroot_proxy** - XRootD endpoint through which all traffic is tunnelled **XROOT_PROXY_EXCL_DOMAINS** - list of comma separated domains which are excluded from being tunnelled through the proxy endpoint **XRD_PLUGIN** - default environment variable used by the XRootD Client plugin loading mechanism which needs to point to the library implementation of the plugin xrootd-5.6.9/src/XrdApps/XrdClRecordPlugin/000077500000000000000000000000001457266313600205715ustar00rootroot00000000000000xrootd-5.6.9/src/XrdApps/XrdClRecordPlugin/README.md000066400000000000000000000340121457266313600220500ustar00rootroot00000000000000# 1 XrdClRecorder Plugin This XRootD Client Plugin can be used to record all user's actions on XrdCl::File object and store them into a csv file. Afterwards, using the xrdreplay utility the actions can be replayed preserving the original timing. The output file can be provided either using the `XRD_RECORDERPATH` environment variable or the `output` key in the plug-in configuration file (the enviroment variable takes precedence). If neither is provided the recorded actions will be stored at a default location: `/tmp/xrdrecord.csv`. Config file format: **recorder.conf:** ```bash url = * lib = /usr/lib64/libXrdClRecorder-5.so enable = true output = /tmp/out.csv # optional ``` In order to replay either do: ```bash xrdreplay /tmp/out.csv ``` or ```bash cat /tmp/out.csv | xrdreplay ``` _________________ # 2 xrdreplay tool The **xrdreplay** application provides the following operation modes: * print mode (-p) : display runtime and IO statistics for a record file * verify mode (-v) : verify the existence of the required input files for a record file * creation mode (-c,-t) : create the required input data using file creation and write the minimal required size (-c) or truncate files to the minimal required size (-t) * playback mode (default) : replay a given record file _________________ ## 2.1 print mode (-p) To display the IO statistics from a recorded file without playback do: ```bash xrdreplay -p recording.csv # ============================================= # IO Summary (print mode) # ============================================= # Sampled Runtime : 5.485724 s # Playback Speed : 1.00 # IO Volume (R) : 536.87 MB [ std:536.87 MB vec:0 B page:0 B ] # IO Volume (W) : 536.87 MB [ std:536.87 MB vec:0 B page:0 B ] # IOPS (R) : 64 [ std:64 vec:0 page:0 ] # IOPS (W) : 64 [ std:64 vec:0 page:0 ] # Files (R) : 1 # Files (W) : 1 # Datasize (R) : 536.87 MB # Datasize (W) : 536.87 MB # --------------------------------------------- # Quality Estimation # --------------------------------------------- # Synchronicity(R) : 4.55% # Synchronicity(W) : 100.00% ``` To further inspect details of the recording file (which files are accessed and IO per file) use the long format (-l) and optionally the summary option (-s): ```bash xrdreplay -p recording.csv # ----------------------------------------------------------------- # File: root://cmsserver//store/cms/higgs.root # Sync: 4.55% # Errs: 0 # ----------------------------------------------------------------- # close::texec : 0.00 s # close::tnomi : 0.00 s # open::texec : 0.00 s # open::tnomi : 0.13 s # read::texec : 0.04 s # read::tnomi : 17.02 s # stat::texec : 0.00 s # stat::tnomi : 0.00 s # close::n : 1 # open::n : 1 # openr::n : 1 # read::b : 536870912 # read::n : 64 # read::o : 536870912 # stat::n : 1 # ----------------------------------------------------------------- ... ``` All tags used in the output format are explained in the tags section. _________________ ## 2.2 verify mode (-v) To verify the availability of all input files one uses: ```bash xrdreplay -v recording.cvs ... # --------------------------------------------- # Verifying Dataset ... # ............................................. # file: root://cmsserver//store/cms/higgs.root # size: 536.87 MB [ 0 B out of 536.87 MB ] ( 0.00% ) # ---> info: file exists and has sufficient size ``` On success the shell returns 0, if there was a missing, too small or inaccessible file it returns -5 (251). ```bash Warning: xrdreplay considers a file only as an input file if it has no bytes written. ``` _________________ ## 2.3 creation mode (-c) In creation mode **xrdreplay** will create the required input files. In this context it is worthwhile to explain the --replace option, which allows to modify the input and output path used by **xrdreplay**. ### 2.3.1 using the --replace option You can use the --replace option (multiple times) to rewrite the URLs of input and output data e.g.: ```bash xrdreplay --replace root://cmsserver//store/cms/:=root://mycluster//mypath/ --replace file:/data/:=file:/gpfs/data/ -v ``` The option works in combination with all modes of **xrdreplay** e.g. you can create the required input files in a different location than given in the record file. There are two ways to create input data: * -c create files and write well defined patterns into the files to the minimum required offset given by the recorded pattern * -t create files and truncate files to the minum required offset given by the recorded pattern (files will contain 0) The truncate option might be not good to produce useful results when the storage systems supports sparse files and/or compression. Her is an example to create input data in create mode in a modified storage endpoint: ```bash xrdreplay --replace root://cmsserver//store/cms/:=root://mycluster//mypath/ -c ``` _________________ ## 2.4 playback mode (default) **xrdreplay** without print, verify or creation option will playback a recorded pattern file. By default **xrdreplay** will replay the IO trying to keep the original timing of each request. This might not be possible if responses are slower than in the original recording. It is possible to modify the playback speed using the -x speedval option. A value of 2 means to try to run the recorded pattern with double speed. A value of 0.5 means to replay the pattern at half speed. Increasing the playback speed can increase memory requirements significantly. The playback mode creates some additional output lines: ```bash xrdreplay recording.cvs # ============================================= # IO Summary # ============================================= # Total Runtime : 5.488581 s # Sampled Runtime : 5.485724 s # Playback Speed : 1.00 # IO Volume (R) : 536.87 MB [ std:536.87 MB vec:0 B page:0 B ] # IO Volume (W) : 536.87 MB [ std:536.87 MB vec:0 B page:0 B ] # IOPS (R) : 64 [ std:64 vec:0 page:0 ] # IOPS (W) : 64 [ std:64 vec:0 page:0 ] # Files (R) : 1 # Files (W) : 1 # Datasize (R) : 536.87 MB # Datasize (W) : 536.87 MB # IO BW (R) : 97.82 MB/s # IO BW (W) : 97.82 MB/s # --------------------------------------------- # Quality Estimation # --------------------------------------------- # Performance Mark : 99.95% # Gain Mark(R) : 86.69% # Gain Mark(W) : 98.29% # Synchronicity(R) : 4.55% # Synchronicity(W) : 100.00% # --------------------------------------------- # Response Errors : 0 # ============================================= ``` Most of the output fields are self-explaining. Performance Mark puts the original run-time to the achieved run-time into relation. The indicates if the IO could potentially be run faster than given by the recording (when > 100%). The Synchronicity measures the amount of IO requests within a given file are overlapping between request and response. A value of 100% indicates synchronous IO, a value towards 0 indicates asynchronous IO. This value does not measure parallelism between files. In case of IO errors you will see a response error counter != 0 and a shell return code of -5 (251). ```bash # --------------------------------------------- # Response Errors : 67 # ============================================= ``` ### 2.4.1 using the force (error suppression) mode (-f) By default **xrdreplay** will reject to replay a recording file with error responses. By using the -f flag you can force the player to run. In this case unsuccessful IO events will be skipped in the replay. _________________ ## 2.5 json output The print and playback modes support json output of results. ```bash { "iosummary": { "player::runtime": 7.8835, "player::speed": 1, "sampled::runtime": 5.48572, "volume::totalread": 536870912, "volume::totalwrite": 536870912, "volume::read": 536870912, "volume::write": 536870912, "volume::pgread": 0, "volume::pgwrite": 0, "volume::vectorread": 0, "volume::vectorwrite": 0, "iops::read": 64, "iops::write": 64, "iops::pgread": 0, "iops::pgwrite": 0, "iops::vectorread": 0, "iops::vectorwrite": 0, "files::read": 1, "files::write": 1, "bandwidth::MB::read": 68.1005, "bandwdith::MB::write": 68.1005, "performancemark": 69.5849, "gain::read":9.94212, "gain::write":42.2262 "synchronicity::read":4.54545, "synchronicity::write":100, "response::error:":0 } } ``` Also the -l and -s options support json output. _________________ ## 2.6 command line usage ``` usage: xrdreplay [-p|--print] [-c|--create-data] [t|--truncate-data] [-l|--long] [-s|--summary] [-h|--help] [-r|--replace :=] [-f|--suppress] [-v|--verify] [-x|--speed ] -h | --help : show this help -f | --suppress : force to run all IO with all successful result status - suppress all others - by default the player won't run with an unsuccessful recorded IO -p | --print : print only mode - shows all the IO for the given replay file without actually running any IO -s | --summary : print summary - shows all the aggregated IO counter summed for all files -l | --long : print long - show all file IO counter for each individual file -v | --verify : verify the existence of all input files -x | --speed : change playback speed by factor [ > 0.0 ] -r | --replace := : replace in the argument list the string with - option is usable several times e.g. to change storage prefixes or filenames [recordfilename] : if a file is given, it will be used as record input otherwise STDIN is used to read records! example: ... --replace file:://localhost:=root://xrootd.eu/ : redirect local file to remote ``` _________________ ## 2.7 xrdreplay output tags Per file (-l) or aggregated statistics (-s): | tag | description | | ------------------------ | ------------------------------------ | | `*::texec` | playback execution time of * in sec | | `*::tnomi` | recorded execution time of * in sec | | `*::tloss` | time running after recording in sec | | `*::tgain` | time gained vs recording in sec | | `*::n` | number of IO calls to * | | `*::b` | sum of bytes for IO calls to * | | `*::o` | highest file offset accessed | When * is listed, this can be any allowed operation like open,close,read,write,truncate,stat,sync,pgread,pgwrite,vectoread,vectorwrite. General output: | tag | description | json tag | | ------------------------ | ------------------------------------------ | -------------------------------------------- | | `Total Runtime` | runtime of the replayed recording | `player::runtime` | | `Sampled Runtime` | original run time of the recording | `sampled::runtime` | | `Playback Speed` | timestretch factor for the playback | `player::speed` | | `IO Volume (R)` | total data INGRESS (by read func) | `volume::totalread,read,pgread,vectoread` | | `IO Volume (W)` | total data EGRESS (by write func) | `volume::totalwrite,write,pgwrite,vectorwrite` | | `IOPS (R)` | total read IO ops (by read func) | `iops::read,pgread,vectoread` | | `IOPS (W)` | total write IO ops (by write func) | `iops::write,pgwrite,vectorwrite` | | `Files (R)` | sum files with reads | `files::read` | | `Files (W)` | sum files with writes | `files::write` | | `Datasize (R)` | sum of max file offsets in reads | `datasetsize::read` | | `Datasize (W)` | sum of max file offsets in writes | `datasetsize::write` | | `IO BW (R)` | bandwidth in MB/s from runtime for reading | `bandwidth::mb::read` | | `IO BW (W)` | bandwidth in MB/s from runtimg for writing | `bandwidth::mb::write` | | `Performance Mark` | 100 * (sampled runtime) / (total runtime) | `performancemark` | | `Gain Mark (R)` | 100 * (sampled read times) / (total rtime) | `gain::read` | | `Gain Mark (W)` | 100 * (sampled write times) / (total wtime)| `gain::write` | | `Synchronicity (R)` | 100=sync 1=async IO for reads | `synchronicity::read` | | `Syncrhonicity (W)` | 100=sync 1=async IO for writes | `synchronicity::write` | | `Response Errors` | number of IOs which were not successfull | `response::error` | _________________ ## 2.8 Memory consumption It is possible to limit the memory consumption of xrdreplay by setting the `XRD_MAXBUFFERSIZE` environment variable, following sufixes are supported: kb, mb, gb (case insensitive). xrootd-5.6.9/src/XrdApps/XrdClRecordPlugin/XrdClAction.hh000066400000000000000000000316631457266313600232750ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2017 by European Organization for Nuclear Research (CERN) // Author: Michal Simon //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. #ifndef SRC_XRDAPPS_RECORDPLUGIN_XRDCLACTION_HH_ #define SRC_XRDAPPS_RECORDPLUGIN_XRDCLACTION_HH_ #include #include #include #include #include "XrdCl/XrdClXRootDResponses.hh" namespace XrdCl { //---------------------------------------------------------------------------- //! Action //---------------------------------------------------------------------------- struct Action { //-------------------------------------------------------------------------- // Constructor // @param file : pointer to the file plug-in that recorded the action // (to be used as an ID) // @param timeout : operation timeout (common for every operation) //-------------------------------------------------------------------------- Action(void* file, uint16_t timeout) : id(reinterpret_cast(file)) , timeout(timeout) , start(std::chrono::system_clock::now()) // register the action start time { } //-------------------------------------------------------------------------- //! Record the server response / error / timeout //-------------------------------------------------------------------------- inline void RecordResult(XRootDStatus* st, AnyObject* rsp) { stop = std::chrono::system_clock::now(); // register the response time status = *st; Serialize(rsp); } //-------------------------------------------------------------------------- //! Convert timpoint to unix timestamp with ns //-------------------------------------------------------------------------- static inline double time( std::chrono::time_point tp) { auto secs = std::chrono::time_point_cast(tp); auto ns = std::chrono::time_point_cast(tp) - std::chrono::time_point_cast(secs); return secs.time_since_epoch().count() + ns.count() / 1000000000.0; } //-------------------------------------------------------------------------- //! Get curretn unix time in ns precision as a double //-------------------------------------------------------------------------- static inline double timeNow() { auto now = std::chrono::system_clock::now(); return time(now); } //-------------------------------------------------------------------------- //! Convert the action / response data into csv row //-------------------------------------------------------------------------- inline std::string ToString() { std::stringstream ss; ss << "\"" << id << "\"" << ','; ss << "\"" << Name() << "\"" << ','; double tstart = time(start); double tstop = time(stop); ss << "\"" << std::fixed << std::setprecision(9) << tstart << "\"" << ","; std::string argstr = ArgStr(); if (!argstr.empty()) argstr += ';'; ss << "\"" << argstr << timeout << "\"" << ','; ss << "\"" << std::fixed << std::setprecision(9) << tstop << "\"" << ","; auto ststr = status.ToString(); while (ststr.back() == ' ') ststr.pop_back(); ss << "\"" << ststr << "\"" << ','; ss << "\"" << serialrsp << "\"" << '\n'; return ss.str(); } //-------------------------------------------------------------------------- //! Destructor //-------------------------------------------------------------------------- virtual ~Action() {} //-------------------------------------------------------------------------- //! Action name //-------------------------------------------------------------------------- virtual std::string Name() = 0; //-------------------------------------------------------------------------- //! Convert operation arguments into a string //-------------------------------------------------------------------------- virtual std::string ArgStr() = 0; //-------------------------------------------------------------------------- //! Serialize server response //-------------------------------------------------------------------------- virtual void Serialize(AnyObject* response) {} uint64_t id; //> File object ID uint16_t timeout; //> operation timeout std::chrono::system_clock::time_point start; //> start time XRootDStatus status; //> operation status std::string serialrsp; //> serialized response std::chrono::system_clock::time_point stop; //> response time }; //---------------------------------------------------------------------------- //! Open action //---------------------------------------------------------------------------- struct OpenAction : public Action { OpenAction( void* file, const std::string& url, OpenFlags::Flags flags, Access::Mode mode, uint16_t timeout) : Action(file, timeout) , url(url) , flags(flags) , mode(mode) { } std::string Name() { return "Open"; } std::string ArgStr() { std::stringstream ss; ss << url << ';'; ss << flags << ';'; ss << mode; return ss.str(); } const std::string url; OpenFlags::Flags flags; Access::Mode mode; }; //---------------------------------------------------------------------------- //! Close action //---------------------------------------------------------------------------- struct CloseAction : public Action { CloseAction(void* file, uint16_t timeout) : Action(file, timeout) { } std::string Name() { return "Close"; } std::string ArgStr() { return {}; } }; //---------------------------------------------------------------------------- //! Stat action //---------------------------------------------------------------------------- struct StatAction : public Action { StatAction(void* file, bool force, uint16_t timeout) : Action(file, timeout) , force(force) { } std::string Name() { return "Stat"; } std::string ArgStr() { return force ? "true" : "false"; } void Serialize(AnyObject* response) { if (!response) return; StatInfo* info = nullptr; response->Get(info); std::stringstream ss; ss << std::to_string(info->GetSize()) << ';'; ss << std::to_string(info->GetFlags()) << ';'; ss << info->GetModTime() << ';'; ss << info->GetChangeTime() << ';'; ss << info->GetAccessTime() << ';'; ss << info->GetModeAsOctString() << ';'; ss << info->GetOwner() << ';'; ss << info->GetGroup() << ';'; ss << info->GetChecksum(); serialrsp = ss.str(); } bool force; }; //---------------------------------------------------------------------------- //! Read action //---------------------------------------------------------------------------- struct ReadAction : public Action { ReadAction(void* file, uint64_t offset, uint32_t size, uint16_t timeout) : Action(file, timeout) , offset(offset) , size(size) { } std::string Name() { return "Read"; } std::string ArgStr() { return std::to_string(offset) + ';' + std::to_string(size); } void Serialize(AnyObject* response) { if (!response) return; ChunkInfo* ptr = nullptr; response->Get(ptr); serialrsp = std::to_string(ptr->length); } uint64_t offset; uint32_t size; }; struct PgReadAction : public Action { PgReadAction(void* file, uint64_t offset, uint32_t size, uint16_t timeout) : Action(file, timeout) , offset(offset) , size(size) { } std::string Name() { return "PgRead"; } std::string ArgStr() { return std::to_string(offset) + ';' + std::to_string(size); } void Serialize(AnyObject* response) { if (!response) return; PageInfo* ptr = nullptr; response->Get(ptr); serialrsp = std::to_string(ptr->GetLength()) + ';' + std::to_string(ptr->GetNbRepair()); } uint64_t offset; uint32_t size; }; //---------------------------------------------------------------------------- //! Write action //---------------------------------------------------------------------------- struct WriteAction : public Action { WriteAction(void* file, uint64_t offset, uint32_t size, uint16_t timeout) : Action(file, timeout) , offset(offset) , size(size) { } std::string Name() { return "Write"; } std::string ArgStr() { return std::to_string(offset) + ';' + std::to_string(size); } uint64_t offset; uint32_t size; }; struct PgWriteAction : public Action { PgWriteAction(void* file, uint64_t offset, uint32_t size, uint16_t timeout) : Action(file, timeout) , offset(offset) , size(size) { } std::string Name() { return "PgWrite"; } std::string ArgStr() { std::stringstream ss; ss << std::to_string(offset) << ';' << std::to_string(size); return ss.str(); } uint64_t offset; uint32_t size; }; //---------------------------------------------------------------------------- //! Sync action //---------------------------------------------------------------------------- struct SyncAction : public Action { SyncAction(void* file, uint16_t timeout) : Action(file, timeout) { } std::string Name() { return "Sync"; } std::string ArgStr() { return {}; } }; //---------------------------------------------------------------------------- //! Truncate action //---------------------------------------------------------------------------- struct TruncateAction : public Action { TruncateAction(void* file, uint64_t size, uint16_t timeout) : Action(file, timeout) , size(size) { } std::string Name() { return "Truncate"; } std::string ArgStr() { return std::to_string(size); } uint32_t size; }; //---------------------------------------------------------------------------- //! VectorRead action //---------------------------------------------------------------------------- struct VectorReadAction : public Action { VectorReadAction(void* file, const ChunkList& chunks, uint16_t timeout) : Action(file, timeout) , req(chunks) { } std::string Name() { return "VectorRead"; } std::string ArgStr() { if (req.empty()) return {}; std::stringstream ss; ss << req[0].offset << ";" << req[0].length; for (size_t i = 1; i < req.size(); ++i) ss << ";" << req[i].offset << ";" << req[i].length; return ss.str(); } void Serialize(AnyObject* response) { if (!response) return; VectorReadInfo* ptr = nullptr; response->Get(ptr); std::stringstream ss; ss << ptr->GetSize(); auto& chunks = ptr->GetChunks(); for (auto& ch : chunks) ss << ';' << ch.offset << ';' << ch.length; serialrsp = ss.str(); } ChunkList req; }; //---------------------------------------------------------------------------- //! Vector Write action //---------------------------------------------------------------------------- struct VectorWriteAction : public Action { VectorWriteAction(void* file, const ChunkList& chunks, uint16_t timeout) : Action(file, timeout) , req(chunks) { } std::string Name() { return "VectorWrite"; } std::string ArgStr() { if (req.empty()) return {}; std::stringstream ss; ss << req[0].offset << ";" << req[0].length; for (size_t i = 1; i < req.size(); ++i) ss << ";" << req[i].offset << ";" << req[i].length; return ss.str(); } ChunkList req; }; //---------------------------------------------------------------------------- //! Fcntl action //---------------------------------------------------------------------------- struct FcntlAction : Action { FcntlAction(void* file, const Buffer& arg, uint16_t timeout) : Action(file, timeout) , req(arg.GetSize()) { } std::string Name() { return "Fcntl"; } std::string ArgStr() { return std::to_string(req); } void Serialize(AnyObject* response) { if (!response) return; Buffer* ptr = nullptr; response->Get(ptr); serialrsp = std::to_string(ptr->GetSize()); } uint32_t req; }; } /* namespace XrdCl */ #endif /* SRC_XRDAPPS_RECORDPLUGIN_XRDCLACTION_HH_ */ xrootd-5.6.9/src/XrdApps/XrdClRecordPlugin/XrdClActionMetrics.hh000066400000000000000000000231771457266313600246250ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2021 by European Organization for Nuclear Research (CERN) // Author: Andreas-Joachim Peters //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #include "XrdClAction.hh" #include #include #include #include #include #include #include #include #include #include #include #include #include #include namespace XrdCl { //------------------------------------------------------------------------------ //! Metrics struct storing all timing and IO information of an action //------------------------------------------------------------------------------ struct ActionMetrics { ActionMetrics() { std::string op[] = { "OpenR", "OpenW", "Open", "Read", "Write", "Stat", "Close", "PgRead", "PgWrite", "Truncate", "Sync", "VectorRead", "VectorWrite" }; for (auto& i : op) { std::string gain = i + "::tgain"; // time early for IO submission std::string loss = i + "::tloss"; // time late for IO submission std::string nomi = i + "::tnomi"; // nominal IO time std::string exec = i + "::texec"; // time to submit IO std::string meas = i + "::tmeas"; // time measured for IO to complete delays[gain] = 0; delays[loss] = 0; delays[nomi] = 0; delays[exec] = 0; delays[meas] = 0; std::string cnt = i + "::n"; // IOPS std::string vol = i + "::b"; // number of bytes std::string err = i + "::e"; // number of unsuccessful IOs std::string off = i + "::o"; // maximum offset seen ios[cnt] = 0; ios[vol] = 0; ios[err] = 0; ios[off] = 0; } ios["All::e"] = 0; // Error counter for summing over files synchronicity = 0.0; errors = 0; } std::string Dump(bool json) const { std::stringstream ss; if (!json) { ss << "# -----------------------------------------------------------------" << std::endl; if (fname != "") { ss << "# File: " << fname << std::endl; ss << "# Sync: " << std::fixed << std::setprecision(2) << synchronicity << "%" << std::endl; ss << "# Errs: " << std::fixed << errors << std::endl; } else { ss << "# Summary" << std::endl; } ss << "# -----------------------------------------------------------------" << std::endl; for (auto& i : delays) { std::string key = i.first; std::transform(key.begin(), key.end(), key.begin(), ::tolower); if (i.second) { ss << "# " << std::setw(16) << key << " : " << std::setw(16) << std::fixed << i.second << " s" << std::endl; } } for (auto& i : ios) { std::string key = i.first; std::transform(key.begin(), key.end(), key.begin(), ::tolower); if (i.second) { ss << "# " << std::setw(16) << key << " : " << std::setw(16) << i.second << std::endl; } } } else { std::string name = fname; if (fname.empty()) name = "_files_summary_"; ss << " {" << std::endl; ss << " \"name\":" << "\"" << name << "\"," << std::endl; ss << " \"synchronicity\": " << synchronicity << "," << std::endl; ss << " \"errors\": " << errors << "," << std::endl; for (auto& i : delays) { std::string key = i.first; std::transform(key.begin(), key.end(), key.begin(), ::tolower); if (i.second) { ss << " \"" << key << "\": " << i.second << "," << std::endl; } } for (auto& i : ios) { std::string key = i.first; std::transform(key.begin(), key.end(), key.begin(), ::tolower); if (i.second) { ss << " \"" << key << "\": " << i.second << "," << std::endl; } } ss.seekp(-2, std::ios_base::end); ss << "\n"; if (fname.empty()) ss << " }" << std::endl; else ss << " }," << std::endl; } return ss.str(); } size_t getIopsRead() const { auto v1 = ios.find("Read::n"); auto v2 = ios.find("PgRead::n"); auto v3 = ios.find("VectorRead::n"); return (v1->second + v2->second + v3->second); } size_t getIopsWrite() const { auto v1 = ios.find("Write::n"); auto v2 = ios.find("PgWrite::n"); auto v3 = ios.find("VectorWrite::n"); return (v1->second + v2->second + v3->second); } size_t getBytesRead() const { auto v1 = ios.find("Read::b"); auto v2 = ios.find("PgRead::b"); auto v3 = ios.find("VectorRead::b"); return (v1->second + v2->second + v3->second); } size_t getBytesWritten() const { auto v1 = ios.find("Write::b"); auto v2 = ios.find("PgWrite::b"); auto v3 = ios.find("VectorWrite::b"); return (v1->second + v2->second + v3->second); } void addDelays(const std::string& action, const std::string& field, double value) { // function called from callbacks requires a guard std::unique_lock guard(mtx); delays[action + "::" + field] += value; } void addIos(const std::string& action, const std::string& field, double value) { // function called from callbacks requires a guard std::unique_lock guard(mtx); ios[action + "::" + field] += value; if (field == "e") { ios["All::e"] += value; } } void add(const ActionMetrics& other) { for (auto& k : other.ios) { ios[k.first] += k.second; } for (auto& k : other.delays) { delays[k.first] += k.second; } errors += other.errors; auto w1 = other.ios.find("Write::b"); auto w2 = other.ios.find("PgWrite::b"); auto w3 = other.ios.find("VectorWrite::b"); if (((w1 != other.ios.end()) && w1->second) || ((w2 != other.ios.end()) && w2->second) || ((w3 != other.ios.end()) && w3->second)) { // count as EGRES aggregated_synchronicity.writes.push_back(other.synchronicity); } else { // count es INGRES aggregated_synchronicity.reads.push_back(other.synchronicity); } } static std::string humanreadable(uint64_t insize) { const uint64_t KB = 1000ll; const uint64_t MB = 1000ll * KB; const uint64_t GB = 1000ll * MB; const uint64_t TB = 1000ll * GB; const uint64_t PB = 1000ll * TB; const uint64_t EB = 1000ll * PB; std::stringstream ss; if (insize >= (10 * KB)) { if (insize >= MB) { if (insize >= GB) { if (insize >= TB) { if (insize >= PB) { if (insize >= EB) { // EB ss << std::fixed << std::setprecision(2) << (insize * 1.0 / EB) << " EB"; } else { // PB ss << std::fixed << std::setprecision(2) << (insize * 1.0 / PB) << " PB"; } } else { // TB ss << std::fixed << std::setprecision(2) << (insize * 1.0 / TB) << " TB"; } } else { // GB ss << std::fixed << std::setprecision(2) << (insize * 1.0 / GB) << " GB"; } } else { // MB ss << std::fixed << std::setprecision(2) << (insize * 1.0 / MB) << " MB"; } } else { // KB ss << std::fixed << std::setprecision(2) << (insize * 1.0 / KB) << " KB"; } } else { ss << std::fixed << insize << " B"; } return ss.str(); } std::string fname; std::string url; double synchronicity; // sync->0: async sync->100.: sync IO size_t errors; //< number of responses != SUCCESS out of the CVS file struct synchronicity_t { std::vector reads; std::vector writes; double ReadSynchronicity() const { if (reads.size()) { return accumulate(reads.begin(), reads.end(), 0.0) / reads.size(); } else { return 0; } } double WriteSynchronicity() const { if (writes.size()) { return accumulate(writes.begin(), writes.end(), 0.0) / writes.size(); } else { return 0; } } }; synchronicity_t aggregated_synchronicity; std::map ios; std::map delays; std::mutex mtx; // only required for async callbacks }; } xrootd-5.6.9/src/XrdApps/XrdClRecordPlugin/XrdClRecorder.hh000066400000000000000000000446711457266313600236300ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2017 by European Organization for Nuclear Research (CERN) // Author: Michal Simon //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. #ifndef XRDCK_RECORDER_HH_ #define XRDCK_RECORDER_HH_ #include "XrdCl/XrdClPlugInInterface.hh" #include "XrdCl/XrdClConstants.hh" #include "XrdCl/XrdClDefaultEnv.hh" #include "XrdCl/XrdClLog.hh" #include "XrdClAction.hh" #include #include namespace XrdCl { //------------------------------------------------------------------------------ //! XrdClFile plugin that arecords all user actions and server responses and //! dumps the data into a csv file. //------------------------------------------------------------------------------ class Recorder: public FilePlugIn { //---------------------------------------------------------------------------- //! Singleton object used for thread-safe writing the recored data into //! a csv file //---------------------------------------------------------------------------- class Output { public: //------------------------------------------------------------------------ //! Get instance, make sure the output file is open //! @return : single instance of Output object //------------------------------------------------------------------------ inline static Output& Instance() { Output& output = Get(); std::unique_lock lck( output.mtx ); if( !output.IsValid() ) { if( !output.Open() ) DefaultEnv::GetLog()->Error( AppMsg, "[Recorder] Failed to create the output file." ); } return output; } //------------------------------------------------------------------------ //! @return : single instance of Output object //------------------------------------------------------------------------ inline static Output& Get() { static Output output; return output; } //------------------------------------------------------------------------ // Record the user action // @param action : the action to be recorded // @return : true if the data was successful written to disk, // false otherwise //------------------------------------------------------------------------ bool Write( std::unique_ptr action ) { std::unique_lock lck( mtx ); const std::string &entry = action->ToString(); int btsWritten = 0; do { int rc = ::write( fd, entry.c_str(), entry.size() ); if( rc < 0 ) { DefaultEnv::GetLog()->Warning( AppMsg, "[Recorder] failed to record an action: %s", strerror( errno ) ); return false; } else btsWritten += rc; } while( size_t( btsWritten ) < entry.size() ); return true;; } //------------------------------------------------------------------------ //! Open the csv files //! @param path : path to the csv file //! @return : true if the file was successfully opened, false otherwise //------------------------------------------------------------------------ bool Open() { fd = open( path.c_str(), O_CREAT | O_WRONLY | O_TRUNC, 0644 ); if( fd < 0 ) DefaultEnv::GetLog()->Warning( AppMsg, "[Recorder] failed to open the output file: %s", strerror( errno ) ); return ( fd >= 0 ); } //------------------------------------------------------------------------ //! @return : true if the csv file is open, false otherwise //------------------------------------------------------------------------ inline bool IsValid() { return ( fd > 0 ); } void SetPath( const std::string &path ) { this->path = path; } private: //------------------------------------------------------------------------ //! Constructor //------------------------------------------------------------------------ Output( ) : fd( -1 ) { } //------------------------------------------------------------------------ // Deleted copy/move constructors and assignment operators //------------------------------------------------------------------------ Output( const Output& ) = delete; Output( Output&& ) = delete; Output& operator=( const Output& ) = delete; Output& operator=( Output&& ) = delete; //------------------------------------------------------------------------ //! Destructor //------------------------------------------------------------------------ ~Output() { if( fd >= 0 ) { int rc = close( fd ); if( rc < 0 ) DefaultEnv::GetLog()->Warning( AppMsg, "[Recorder] failed to close the output file: %s", strerror( errno ) ); } } std::mutex mtx; //< mutex guarding the writes int fd; //< the csv file descriptor std::string path; //< path to the csv file }; //---------------------------------------------------------------------------- //! Completion handler recording user action / server response //---------------------------------------------------------------------------- struct RecordHandler : public ResponseHandler { //-------------------------------------------------------------------------- //! Constructor //! @param output : the object handling writes to csv file //! @param action : user action to be recorded //! @param handler : user completion handler to be wrapped //-------------------------------------------------------------------------- RecordHandler( Output &output, std::unique_ptr action, ResponseHandler *handler ) : output( output ), action( std::move( action ) ), handler( handler ) { } //-------------------------------------------------------------------------- //! Handle server response //! @param status : operation status //! @param response : server response //! @param hostList : list of hosts involved in serving given request //-------------------------------------------------------------------------- void HandleResponseWithHosts( XRootDStatus *status, AnyObject *response, HostList *hostList ) { action->RecordResult( status, response ); output.Write( std::move( action ) ); handler->HandleResponseWithHosts( status, response, hostList ); delete this; } //-------------------------------------------------------------------------- //! Handle server response //! @param status : operation status //! @param response : server response //-------------------------------------------------------------------------- void HandleResponse( XRootDStatus *status, AnyObject *response ) { action->RecordResult( status, response ); output.Write( std::move( action ) ); handler->HandleResponse( status, response ); delete this; } Output &output; //< the object handling writes to csv file std::unique_ptr action; //< user action ResponseHandler *handler; //< user completion handler }; public: //---------------------------------------------------------------------------- //! Create the output csv file //! @param cfgpath : path for the file to be created //---------------------------------------------------------------------------- inline static void SetOutput( const std::string &cfgpath ) { static const std::string defaultpath = "/tmp/xrdrecord.csv"; const char *envpath = getenv( "XRD_RECORDERPATH" ); std::string path = envpath ? envpath : ( !cfgpath.empty() ? cfgpath : defaultpath ); Output::Get().SetPath( path ); } //---------------------------------------------------------------------------- //! Constructor //---------------------------------------------------------------------------- Recorder(): file( false ), output( Output::Instance() ) { } //---------------------------------------------------------------------------- //! @return : true if this is a valid instance, false otherwise //---------------------------------------------------------------------------- bool IsValid() const { return output.IsValid(); } //---------------------------------------------------------------------------- //! Destructor //---------------------------------------------------------------------------- virtual ~Recorder() { } //---------------------------------------------------------------------------- //! Open //---------------------------------------------------------------------------- virtual XRootDStatus Open(const std::string& url, OpenFlags::Flags flags, Access::Mode mode, ResponseHandler* handler, uint16_t timeout) { std::unique_ptr ptr( new OpenAction( this, url, flags, mode, timeout ) ); RecordHandler *recHandler = new RecordHandler( output, std::move( ptr ), handler ); return file.Open( url, flags, mode, recHandler, timeout ); } //---------------------------------------------------------------------------- //! Close //---------------------------------------------------------------------------- virtual XRootDStatus Close(ResponseHandler* handler, uint16_t timeout) { std::unique_ptr ptr( new CloseAction( this, timeout ) ); RecordHandler *recHandler = new RecordHandler( output, std::move( ptr ), handler ); return file.Close( recHandler, timeout ); } //---------------------------------------------------------------------------- //! Stat //---------------------------------------------------------------------------- virtual XRootDStatus Stat(bool force, ResponseHandler* handler, uint16_t timeout) { std::unique_ptr ptr( new StatAction( this, force, timeout ) ); RecordHandler *recHandler = new RecordHandler( output, std::move( ptr ), handler ); return file.Stat(force, recHandler, timeout); } //---------------------------------------------------------------------------- //! Read //---------------------------------------------------------------------------- virtual XRootDStatus Read(uint64_t offset, uint32_t size, void* buffer, ResponseHandler* handler, uint16_t timeout) { std::unique_ptr ptr( new ReadAction( this, offset, size, timeout ) ); RecordHandler *recHandler = new RecordHandler( output, std::move( ptr ), handler ); return file.Read( offset, size, buffer, recHandler, timeout ); } //---------------------------------------------------------------------------- //! Write //---------------------------------------------------------------------------- virtual XRootDStatus Write(uint64_t offset, uint32_t size, const void* buffer, ResponseHandler* handler, uint16_t timeout) { std::unique_ptr ptr( new WriteAction( this, offset, size, timeout ) ); RecordHandler *recHandler = new RecordHandler( output, std::move( ptr ), handler ); return file.Write( offset, size, buffer, recHandler, timeout ); } //------------------------------------------------------------------------ //! @see XrdCl:File PgRead //------------------------------------------------------------------------ virtual XRootDStatus PgRead( uint64_t offset, uint32_t size, void *buffer, ResponseHandler *handler, uint16_t timeout ) { std::unique_ptr ptr( new PgReadAction( this, offset, size, timeout ) ); RecordHandler *recHandler = new RecordHandler( output, std::move( ptr ), handler ); return file.PgRead( offset, size, buffer, recHandler, timeout ); } //------------------------------------------------------------------------ //! @see XrdCl::File::PgWrite //------------------------------------------------------------------------ virtual XRootDStatus PgWrite( uint64_t offset, uint32_t size, const void *buffer, std::vector &cksums, ResponseHandler *handler, uint16_t timeout ) { std::unique_ptr ptr( new PgWriteAction( this, offset, size, timeout ) ); RecordHandler *recHandler = new RecordHandler( output, std::move( ptr ), handler ); return file.PgWrite( offset, size, buffer, cksums, recHandler, timeout ); } //---------------------------------------------------------------------------- //! Sync //---------------------------------------------------------------------------- virtual XRootDStatus Sync(ResponseHandler* handler, uint16_t timeout) { std::unique_ptr ptr( new SyncAction( this, timeout ) ); RecordHandler *recHandler = new RecordHandler( output, std::move( ptr ), handler ); return file.Sync( recHandler, timeout ); } //---------------------------------------------------------------------------- //! Truncate //---------------------------------------------------------------------------- virtual XRootDStatus Truncate(uint64_t size, ResponseHandler* handler, uint16_t timeout) { std::unique_ptr ptr( new TruncateAction( this, size, timeout ) ); RecordHandler *recHandler = new RecordHandler( output, std::move( ptr ), handler ); return file.Truncate(size, recHandler, timeout); } //---------------------------------------------------------------------------- //! VectorRead //---------------------------------------------------------------------------- virtual XRootDStatus VectorRead(const ChunkList& chunks, void* buffer, ResponseHandler* handler, uint16_t timeout) { std::unique_ptr ptr( new VectorReadAction( this, chunks, timeout ) ); RecordHandler *recHandler = new RecordHandler( output, std::move( ptr ), handler ); return file.VectorRead(chunks, buffer, recHandler, timeout); } //---------------------------------------------------------------------------- //! VectorRead //---------------------------------------------------------------------------- virtual XRootDStatus VectorWrite( const ChunkList &chunks, ResponseHandler *handler, uint16_t timeout ) { std::unique_ptr ptr( new VectorWriteAction( this, chunks, timeout ) ); RecordHandler *recHandler = new RecordHandler( output, std::move( ptr ), handler ); return file.VectorWrite( chunks, recHandler, timeout ); } //---------------------------------------------------------------------------- //! Fcntl //---------------------------------------------------------------------------- virtual XRootDStatus Fcntl(const Buffer& arg, ResponseHandler* handler, uint16_t timeout) { std::unique_ptr ptr( new FcntlAction( this, arg, timeout ) ); RecordHandler *recHandler = new RecordHandler( output, std::move( ptr ), handler ); return file.Fcntl(arg, recHandler, timeout); } //---------------------------------------------------------------------------- //! Visa //---------------------------------------------------------------------------- virtual XRootDStatus Visa(ResponseHandler* handler, uint16_t timeout) { return file.Visa(handler, timeout); } //---------------------------------------------------------------------------- //! IsOpen //---------------------------------------------------------------------------- virtual bool IsOpen() const { return file.IsOpen(); } //---------------------------------------------------------------------------- //! SetProperty //---------------------------------------------------------------------------- virtual bool SetProperty(const std::string& name, const std::string& value) { return file.SetProperty(name, value); } //---------------------------------------------------------------------------- //! GetProperty //---------------------------------------------------------------------------- virtual bool GetProperty(const std::string& name, std::string& value) const { return file.GetProperty(name, value); } private: File file; //< The file object that performs the actual operation Output &output; //< The object for writing the recorded actions }; } // namespace XrdCl #endif /* XRDCK_RECORDER_HH_ */ xrootd-5.6.9/src/XrdApps/XrdClRecordPlugin/XrdClRecorderPlugin.cc000066400000000000000000000031471457266313600247660ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2017 by European Organization for Nuclear Research (CERN) // Author: Michal Simon //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #include "XrdClRecorder.hh" #include "XrdClRecorderPlugin.hh" XrdVERSIONINFO(XrdClGetPlugIn, XrdClGetPlugIn) extern "C" { void* XrdClGetPlugIn( const void* arg ) { const std::map* config = static_cast< const std::map* >(arg); return static_cast(new XrdCl::RecorderFactory( config ) ); } } xrootd-5.6.9/src/XrdApps/XrdClRecordPlugin/XrdClRecorderPlugin.hh000066400000000000000000000061521457266313600247770ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2017 by European Organization for Nuclear Research (CERN) // Author: Michal Simon //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. #pragma once #include "XrdCl/XrdClPlugInInterface.hh" namespace XrdCl { //------------------------------------------------------------------------------ //! XrdCl recorder plug-in factory //------------------------------------------------------------------------------ class RecorderFactory : public PlugInFactory { public: //---------------------------------------------------------------------------- //! Constructor //! //! @param config map containing configuration parameters //---------------------------------------------------------------------------- RecorderFactory( const std::map* config ) { if( config ) { auto itr = config->find( "output" ); Recorder::SetOutput( itr != config->end() ? itr->second : "" ); } } //---------------------------------------------------------------------------- //! Destructor //---------------------------------------------------------------------------- virtual ~RecorderFactory() { } //---------------------------------------------------------------------------- //! Create a file plug-in for the given URL //---------------------------------------------------------------------------- virtual FilePlugIn* CreateFile(const std::string& url) { std::unique_ptr ptr( new Recorder() ); if( !ptr->IsValid() ) return nullptr; return static_cast( ptr.release() ); } //---------------------------------------------------------------------------- //! Create a file system plug-in for the given URL //---------------------------------------------------------------------------- virtual FileSystemPlugIn* CreateFileSystem(const std::string& url) { Log* log = DefaultEnv::GetLog(); log->Error(1, "FileSystem plugin implementation not supported"); return static_cast(0); } }; } // namespace xrdcl_proxy xrootd-5.6.9/src/XrdApps/XrdClRecordPlugin/XrdClReplay.cc000066400000000000000000001504361457266313600233020ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2021 by European Organization for Nuclear Research (CERN) // Author: Michal Simon // Co-Author: Andreas-Joachim Peters //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #include "XrdCl/XrdClOperations.hh" #include "XrdCl/XrdClUtils.hh" #include "XrdCl/XrdClFileOperations.hh" #include "XrdSys/XrdSysPthread.hh" #include "XrdClAction.hh" #include "XrdClActionMetrics.hh" #include "XrdClReplayArgs.hh" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include namespace XrdCl { //------------------------------------------------------------------------------ //! Buffer pool - to limit memory consumption //------------------------------------------------------------------------------ class BufferPool { public: //-------------------------------------------------------------------------- //! Single instance access //-------------------------------------------------------------------------- static BufferPool& Instance() { static BufferPool instance; return instance; } //-------------------------------------------------------------------------- //! Allocate a buffer if has available memory, otherwise wait until enough //! memory has been reclaimed //-------------------------------------------------------------------------- std::shared_ptr> Allocate( size_t length ) { std::unique_lock lck( mtx ); cv.wait( lck, [this, length]{ return available >= length; } ); available -= length; BufferDeleter del; std::shared_ptr> buffer( new std::vector( length, 'A' ), del ); return buffer; } private: //-------------------------------------------------------------------------- //! Reclaim memory //-------------------------------------------------------------------------- void Reclaim( size_t length ) { std::unique_lock lck(mtx); available += length; cv.notify_all(); } //-------------------------------------------------------------------------- //! Custom deleter for shared_ptr (notifies the buffer-pool when memory //! has been reclaimed) //-------------------------------------------------------------------------- struct BufferDeleter { void operator()( std::vector *buff ) { BufferPool::Instance().Reclaim( buff->size() ); delete buff; } }; static const size_t KB = 1024; static const size_t MB = 1024 * KB; static const size_t GB = 1024 * MB; //-------------------------------------------------------------------------- //! Constructor - read out the memory limit from XRD_MAXBUFFERSIZE, if not //! set don't impose any limit //-------------------------------------------------------------------------- BufferPool() : mtx(), cv() { const char *maxsize = getenv( "XRD_MAXBUFFERSIZE" ); if( maxsize ) { size_t len = strlen( maxsize ); size_t pos; available = std::stoul( maxsize, &pos ); std::string sufix( len != pos ? maxsize + len - 2 : "" ); std::transform( sufix.begin(), sufix.end(), sufix.begin(), ::toupper ); if( !sufix.empty() ) { if( sufix == "KB" ) available *= KB; else if( sufix == "MB" ) available *= MB; else if( sufix == "GB" ) available *= GB; } return; } available = std::numeric_limits::max(); } BufferPool( const BufferPool& ) = delete; BufferPool( BufferPool&& ) = delete; BufferPool& operator=( const BufferPool& ) = delete; BufferPool& operator=( BufferPool& ) = delete; size_t available; std::mutex mtx; std::condition_variable cv; }; //------------------------------------------------------------------------------ //! Timer helper class //------------------------------------------------------------------------------ class mytimer_t { public: //-------------------------------------------------------------------------- //! Constructor (record start time) //-------------------------------------------------------------------------- mytimer_t() : start(clock_t::now()) { } //-------------------------------------------------------------------------- //! Reset the start time //-------------------------------------------------------------------------- void reset() { start = clock_t::now(); } //-------------------------------------------------------------------------- //! @return : get time elapsed from start //-------------------------------------------------------------------------- double elapsed() const { return (1.0 * (std::chrono::duration_cast(clock_t::now() - start).count()) / 1000000000.0); } private: using clock_t = std::chrono::high_resolution_clock; std::chrono::time_point start; //< registered start time }; //------------------------------------------------------------------------------ //! Barrier for synchronizing the asynchronous execution of actions //! It is actually a wrapper around semaphore. //------------------------------------------------------------------------------ class barrier_t { public: //------------------------------------------------------------------------ //! Constructor //! @param sem : the semaphore //------------------------------------------------------------------------ barrier_t(XrdSysSemaphore& sem) : sem(sem) { } //------------------------------------------------------------------------ //! Destructor //------------------------------------------------------------------------ ~barrier_t() { sem.Post(); } inline XrdSysSemaphore& get() { return sem; } private: XrdSysSemaphore& sem; //< the semaphore to be posted }; //------------------------------------------------------------------------------ //! AssureFile creates input data files on the fly if required //------------------------------------------------------------------------------ bool AssureFile(const std::string& url, uint64_t size, bool viatruncate, bool verify) { OpenFlags::Flags flags = OpenFlags::Read; Access::Mode mode = Access::None; uint16_t timeout = 60; { // deal with existing files auto file = std::make_unique(false); XRootDStatus status = file->Open(url, flags, mode, timeout); if (status.IsOK()) { StatInfo* statinfo; // file exists already, verify the size status = file->Stat(false, statinfo, timeout); if (status.IsOK()) { if (statinfo->GetSize() < size) { std::cerr << "Error: file size is not sufficient, but I won't touch the file - aborting ..."; return false; } else { std::cout << "# ---> info: file exists and has sufficient size" << std::endl; return true; } } } } if (verify) { std::cerr << "Verify: file is missing or inaccessible: " << url << std::endl; return false; } { // deal with non-existing file OpenFlags::Flags wflags = OpenFlags::New | OpenFlags::Write | OpenFlags::MakePath; Access::Mode wmode = Access::UR | Access::UW | Access::UX; auto file = std::make_unique(false); XRootDStatus status = file->Open(url, wflags, wmode, timeout); if (status.IsOK()) { if (viatruncate) { // create a file via truncation status = file->Truncate(size, timeout); if (!status.IsOK()) { std::cerr << "Error: " << status.ToString() << " - empty file might be left behind!" << std::endl; return false; } return true; } else { // create a file via writes using buffer_t = std::vector; //< data buffer buffer_t buffer(32768); size_t nbytes = 0; while (nbytes < size) { size_t towrite = size - nbytes; if (towrite > (buffer.size() * sizeof(uint64_t))) towrite = buffer.size() * sizeof(uint64_t); for (size_t i = 0; i < buffer.size(); ++i) { // we write the offset in this buffer buffer[i] = nbytes / sizeof(uint64_t) + i; } status = file->Write(nbytes, towrite, buffer.data(), timeout); if (!status.IsOK()) { std::cerr << "Error: " << status.ToString() << " - failed to write file at offset " << nbytes << " - incomplete file might be left behind!" << std::endl; return false; } nbytes += towrite; } } return true; } else { std::cerr << "Error: " << status.ToString() << " - failed to create file!" << std::endl; } } return false; } //------------------------------------------------------------------------------ //! Executes an action registered in the csv file //------------------------------------------------------------------------------ class ActionExecutor { using buffer_t = std::shared_ptr>; //< data buffer public: //-------------------------------------------------------------------------- //! Constructor //! @param file : the file that should be the context of the action //! @param action : the action to be executed //! @param args : arguments for the action //! @param orgststr : original status //! @param resp : original response //! @param duration : nominal duration of this action //-------------------------------------------------------------------------- ActionExecutor(File& file, const std::string& action, const std::string& args, const std::string& orgststr, const std::string& resp, const double& duration) : file(file) , action(action) , args(args) , orgststr(orgststr) , nominalduration(duration) { } //-------------------------------------------------------------------------- //! Execute the action //! @param ending : synchronization object for ending the execution //-------------------------------------------------------------------------- void Execute(std::shared_ptr& ending, std::shared_ptr& closing, ActionMetrics& metric, bool simulate) { if (action == "Open") // open action { std::string url; OpenFlags::Flags flags; Access::Mode mode; uint16_t timeout; std::tie(url, flags, mode, timeout) = GetOpenArgs(); std::string lmetric; if ((flags & OpenFlags::Update) || (flags & OpenFlags::Write)) { metric.ios["OpenW::n"]++; } else { metric.ios["OpenR::n"]++; } metric.ios["Open::n"]++; mytimer_t timer; if (!simulate) WaitFor(Open(file, url, flags, mode, timeout) >> [this, orgststr{ orgststr }, ending, closing, timer, &metric](XRootDStatus& s) mutable { metric.addIos("Open", "e", HandleStatus(s, orgststr, "Open")); metric.addDelays("Open", "tmeas", timer.elapsed()); ending.reset(); closing.reset(); }); else { ending.reset(); closing.reset(); } } else if (action == "Close") // close action { uint16_t timeout = GetCloseArgs(); mytimer_t timer; if (closing) { auto& sem = closing->get(); closing.reset(); sem.Wait(); } metric.ios["Close::n"]++; if (!simulate) Async(Close(file, timeout) >> [this, orgststr{ orgststr }, ending, timer, &metric](XRootDStatus& s) mutable { metric.addIos("Close", "e", HandleStatus(s, orgststr, "Close")); metric.addDelays("Close", "tmeas", timer.elapsed()); ending.reset(); }); else { ending.reset(); } } else if (action == "Stat") // stat action { bool force; uint16_t timeout; std::tie(force, timeout) = GetStatArgs(); metric.ios["Stat::n"]++; mytimer_t timer; if (!simulate) Async(Stat(file, force, timeout) >> [this, orgststr{ orgststr }, ending, closing, timer, &metric](XRootDStatus& s, StatInfo& r) mutable { metric.addIos("Stat", "e", HandleStatus(s, orgststr, "Stat")); metric.addDelays("Stat", "tmeas", timer.elapsed()); ending.reset(); closing.reset(); }); else { ending.reset(); closing.reset(); } } else if (action == "Read") // read action { uint64_t offset; buffer_t buffer; uint16_t timeout; std::tie(offset, buffer, timeout) = GetReadArgs(); metric.ios["Read::n"]++; metric.ios["Read::b"] += buffer->size(); if ((offset + buffer->size()) > metric.ios["Read::o"]) metric.ios["Read::o"] = offset + buffer->size(); mytimer_t timer; if (!simulate) Async(Read(file, offset, buffer->size(), buffer->data(), timeout) >> [buffer, orgststr{ orgststr }, ending, closing, timer, &metric](XRootDStatus& s, ChunkInfo& r) mutable { metric.addIos("Read", "e", HandleStatus(s, orgststr, "Read")); metric.addDelays("Read", "tmeas", timer.elapsed()); buffer.reset(); ending.reset(); closing.reset(); }); else { buffer.reset(); ending.reset(); closing.reset(); } } else if (action == "PgRead") // pgread action { uint64_t offset; buffer_t buffer; uint16_t timeout; std::tie(offset, buffer, timeout) = GetPgReadArgs(); metric.ios["PgRead::n"]++; metric.ios["PgRead::b"] += buffer->size(); if ((offset + buffer->size()) > metric.ios["Read::o"]) metric.ios["Read::o"] = offset + buffer->size(); mytimer_t timer; if (!simulate) Async(PgRead(file, offset, buffer->size(), buffer->data(), timeout) >> [buffer, orgststr{ orgststr }, ending, closing, timer, &metric](XRootDStatus& s, PageInfo& r) mutable { metric.addIos("PgRead", "e", HandleStatus(s, orgststr, "PgRead")); metric.addDelays("PgRead", "tmeas", timer.elapsed()); buffer.reset(); ending.reset(); closing.reset(); }); else { buffer.reset(); ending.reset(); closing.reset(); } } else if (action == "Write") // write action { uint64_t offset; buffer_t buffer; uint16_t timeout; std::tie(offset, buffer, timeout) = GetWriteArgs(); metric.ios["Write::n"]++; metric.ios["Write::b"] += buffer->size(); if ((offset + buffer->size()) > metric.ios["Write::o"]) metric.ios["Write::o"] = offset + buffer->size(); mytimer_t timer; if (!simulate) Async( Write(file, offset, buffer->size(), buffer->data(), timeout) >> [buffer, orgststr{ orgststr }, ending, closing, timer, &metric](XRootDStatus& s) mutable { metric.addIos("Write", "e", HandleStatus(s, orgststr, "Write")); metric.addDelays("Write", "tmeas", timer.elapsed()); buffer.reset(); ending.reset(); closing.reset(); }); else { buffer.reset(); ending.reset(); closing.reset(); } } else if (action == "PgWrite") // pgwrite action { uint64_t offset; buffer_t buffer; uint16_t timeout; std::tie(offset, buffer, timeout) = GetPgWriteArgs(); metric.ios["PgWrite::n"]++; metric.ios["PgWrite::b"] += buffer->size(); if ((offset + buffer->size()) > metric.ios["Write::o"]) metric.ios["Write::o"] = offset + buffer->size(); mytimer_t timer; if (!simulate) Async( PgWrite(file, offset, buffer->size(), buffer->data(), timeout) >> [buffer, orgststr{ orgststr }, ending, closing, timer, &metric](XRootDStatus& s) mutable { metric.addIos("PgWrite", "e", HandleStatus(s, orgststr, "PgWrite")); metric.addDelays("PgWrite", "tmeas", timer.elapsed()); buffer.reset(); ending.reset(); closing.reset(); }); else { buffer.reset(); ending.reset(); closing.reset(); } } else if (action == "Sync") // sync action { uint16_t timeout = GetSyncArgs(); metric.ios["Sync::n"]++; mytimer_t timer; if (!simulate) Async(Sync(file, timeout) >> [this, orgststr{ orgststr }, ending, closing, timer, &metric](XRootDStatus& s) mutable { metric.addIos("Sync", "e", HandleStatus(s, orgststr, "Sync")); metric.addDelays("Sync", "tmeas", timer.elapsed()); ending.reset(); closing.reset(); }); else { ending.reset(); closing.reset(); } } else if (action == "Truncate") // truncate action { uint64_t size; uint16_t timeout; std::tie(size, timeout) = GetTruncateArgs(); metric.ios["Truncate::n"]++; if (size > metric.ios["Truncate::o"]) metric.ios["Truncate::o"] = size; mytimer_t timer; if (!simulate) Async(Truncate(file, size, timeout) >> [this, orgststr{ orgststr }, ending, closing, timer, &metric](XRootDStatus& s) mutable { metric.addIos("Truncate", "e", HandleStatus(s, orgststr, "Truncate")); metric.addDelays("Truncate", "tmeas", timer.elapsed()); ending.reset(); closing.reset(); }); else { ending.reset(); closing.reset(); } } else if (action == "VectorRead") // vector read action { ChunkList chunks; uint16_t timeout; std::vector buffers; std::tie(chunks, timeout, buffers) = GetVectorReadArgs(); metric.ios["VectorRead::n"]++; for (auto& ch : chunks) { metric.ios["VectorRead::b"] += ch.GetLength(); if ((ch.GetOffset() + ch.GetLength()) > metric.ios["Read::o"]) metric.ios["Read::o"] = ch.GetOffset() + ch.GetLength(); } mytimer_t timer; if (!simulate) Async( VectorRead(file, chunks, timeout) >> [this, orgststr{ orgststr }, buffers, ending, closing, timer, &metric](XRootDStatus& s, VectorReadInfo& r) mutable { metric.addIos("VectorRead", "e", HandleStatus(s, orgststr, "VectorRead")); metric.addDelays("VectorRead", "tmeas", timer.elapsed()); buffers.clear(); ending.reset(); closing.reset(); }); else { buffers.clear(); ending.reset(); closing.reset(); } } else if (action == "VectorWrite") // vector write { ChunkList chunks; uint16_t timeout; std::vector buffers; std::tie(chunks, timeout, buffers) = GetVectorWriteArgs(); metric.ios["VectorWrite::n"]++; for (auto& ch : chunks) { metric.ios["VectorWrite::b"] += ch.GetLength(); if ((ch.GetOffset() + ch.GetLength()) > metric.ios["Write::o"]) metric.ios["Write::o"] = ch.GetOffset() + ch.GetLength(); } mytimer_t timer; if (!simulate) Async(VectorWrite(file, chunks, timeout) >> [this, orgststr{ orgststr }, buffers, ending, closing, timer, &metric](XRootDStatus& s) mutable { metric.addIos("VectorWrite", "e", HandleStatus(s, orgststr, "VectorWrite")); metric.addDelays("VectorWrite", "tmeas", timer.elapsed()); buffers.clear(); ending.reset(); closing.reset(); }); else { buffers.clear(); ending.reset(); closing.reset(); } } else { DefaultEnv::GetLog()->Warning(AppMsg, "Cannot replyt %s action.", action.c_str()); } } //-------------------------------------------------------------------------- //! Get nominal duration variable //-------------------------------------------------------------------------- double NominalDuration() const { return nominalduration; } //-------------------------------------------------------------------------- //! Get aciton name //-------------------------------------------------------------------------- std::string Name() const { return action; } private: //-------------------------------------------------------------------------- //! Handle response status //-------------------------------------------------------------------------- static bool HandleStatus(XRootDStatus& response, const std::string& orgstr, const std::string where="unknown") { std::string rspstr = response.ToString(); rspstr.erase(remove(rspstr.begin(), rspstr.end(), ' '), rspstr.end()); if (rspstr != orgstr) { DefaultEnv::GetLog()->Warning(AppMsg, "We were expecting status: %s, but " "received: %s from: %s", orgstr.c_str(), rspstr.c_str(), where.c_str()); return true; } else { return false; } } //-------------------------------------------------------------------------- //! Parse arguments for open //-------------------------------------------------------------------------- std::tuple GetOpenArgs() { std::vector tokens; Utils::splitString(tokens, args, ";"); if (tokens.size() != 4) throw std::invalid_argument("Failed to parse open arguments."); std::string url = tokens[0]; OpenFlags::Flags flags = static_cast(std::stoul(tokens[1])); Access::Mode mode = static_cast(std::stoul(tokens[2])); uint16_t timeout = static_cast(std::stoul(tokens[3])); return std::make_tuple(url, flags, mode, timeout); } //-------------------------------------------------------------------------- //! Parse arguments for close //-------------------------------------------------------------------------- uint16_t GetCloseArgs() { return static_cast(std::stoul(args)); } std::tuple GetStatArgs() { std::vector tokens; Utils::splitString(tokens, args, ";"); if (tokens.size() != 2) throw std::invalid_argument("Failed to parse stat arguments."); bool force = (tokens[0] == "true"); uint16_t timeout = static_cast(std::stoul(tokens[1])); return std::make_tuple(force, timeout); } //-------------------------------------------------------------------------- //! Parse arguments for read //-------------------------------------------------------------------------- std::tuple GetReadArgs() { std::vector tokens; Utils::splitString(tokens, args, ";"); if (tokens.size() != 3) throw std::invalid_argument("Failed to parse read arguments."); uint64_t offset = std::stoull(tokens[0]); uint32_t length = std::stoul(tokens[1]); auto buffer = BufferPool::Instance().Allocate( length ); uint16_t timeout = static_cast(std::stoul(tokens[2])); return std::make_tuple(offset, buffer, timeout); } //-------------------------------------------------------------------------- //! Parse arguments for pgread //-------------------------------------------------------------------------- inline std::tuple GetPgReadArgs() { return GetReadArgs(); } //-------------------------------------------------------------------------- //! Parse arguments for write //-------------------------------------------------------------------------- inline std::tuple GetWriteArgs() { return GetReadArgs(); } //-------------------------------------------------------------------------- //! Parse arguments for pgwrite //-------------------------------------------------------------------------- inline std::tuple GetPgWriteArgs() { return GetReadArgs(); } //-------------------------------------------------------------------------- //! Parse arguments for sync //-------------------------------------------------------------------------- uint16_t GetSyncArgs() { return static_cast(std::stoul(args)); } //-------------------------------------------------------------------------- //! Parse arguments for truncate //-------------------------------------------------------------------------- std::tuple GetTruncateArgs() { std::vector tokens; Utils::splitString(tokens, args, ";"); if (tokens.size() != 2) throw std::invalid_argument("Failed to parse truncate arguments."); uint64_t size = std::stoull(tokens[0]); uint16_t timeout = static_cast(std::stoul(tokens[1])); return std::make_tuple(size, timeout); } //-------------------------------------------------------------------------- //! Parse arguments for vector read //-------------------------------------------------------------------------- std::tuple> GetVectorReadArgs() { std::vector tokens; Utils::splitString(tokens, args, ";"); ChunkList chunks; chunks.reserve( tokens.size() - 1 ); std::vector buffers; buffers.reserve( tokens.size() - 1 ); for (size_t i = 0; i < tokens.size() - 1; i += 2) { uint64_t offset = std::stoull(tokens[i]); uint32_t length = std::stoul(tokens[i + 1]); auto buffer = BufferPool::Instance().Allocate( length ); chunks.emplace_back(offset, length, buffer->data()); buffers.emplace_back( std::move( buffer ) ); } uint16_t timeout = static_cast(std::stoul(tokens.back())); return std::make_tuple(std::move(chunks), timeout, std::move(buffers)); } //-------------------------------------------------------------------------- //! Parse arguments for vector write //-------------------------------------------------------------------------- inline std::tuple> GetVectorWriteArgs() { return GetVectorReadArgs(); } File& file; //< the file object const std::string action; //< the action to be executed const std::string args; //< arguments for the operation std::string orgststr; //< the original response status of the action double nominalduration; //< the original duration of execution }; //------------------------------------------------------------------------------ //! Split a row into columns //------------------------------------------------------------------------------ std::vector ToColumns( const std::string &row ) { std::vector columns; size_t quotecnt = 0; size_t pos = 0; //---------------------------------------------------------------------------- //! loop over all the columns in the row //---------------------------------------------------------------------------- while( pos != std::string::npos && pos < row.size() ) { if( row[pos] == '"' ) // we are handling a quoted column { if( quotecnt > 0 ) // this is a closing quote { if( pos + 1 < row.size() && row[pos + 1] != ',' ) // if it is not the last character in the row it should be followed by a comma throw std::runtime_error( "Parsing error: missing comma" ); --quotecnt; // strip the quote ++pos; // move to the comma or end of row continue; } else // this is a opening quote { ++quotecnt; auto b = std::next( row.begin(), pos + 1 ); // iterator to the beginning of our column size_t posend = row.find( "\",", pos + 1 ); // position of the cursor to the end of our column if( posend == std::string::npos && row[row.size() - 1] == '"' ) posend = row.size() - 1; else if( posend == std::string::npos ) throw std::runtime_error( "Parsing error: missing closing quote" ); auto e = std::next( row.begin(), posend ); // iterator to the end of our column columns.emplace_back( b, e ); // add the column to the result pos = posend; // move to the next column continue; } } else if( row[pos] == ',' ) // we are handling a column separator { if( pos + 1 < row.size() && row[pos + 1] == '"' ) // check if the column is quoted { ++pos; // if yes we will handle this with the logic reserved for quoted columns continue; } auto b = std::next( row.begin(), pos + 1 ); // iterator to the beginning of our column size_t posend = row.find( ',', pos + 1 ); // position of the cursor to the end of our column if( posend == std::string::npos ) posend = row.size(); auto e = std::next( row.begin(), posend ); // iterator to the end of our column columns.emplace_back( b, e ); // add the column to the result pos = posend; // move to the next column continue; } else if( pos == 0 ) // we are handling the 1st column if not quoted { size_t posend = row.find( ',', pos + 1 ); // position of the cursor to the end of our column if( posend == std::string::npos ) posend = row.size(); auto end = std::next( row.begin(), posend ); // iterator to the end of our column columns.emplace_back( row.begin(), end ); // add the column to the result pos = posend; // move to the next column continue; } else { throw std::runtime_error( "Parsing error: invalid input file." ); } } return columns; } //------------------------------------------------------------------------------ //! List of actions: start time - action //------------------------------------------------------------------------------ using action_list = std::multimap; //------------------------------------------------------------------------------ //! Parse input file //! @param path : path to the input csv file //------------------------------------------------------------------------------ std::unordered_map ParseInput(const std::string& path, double& t0, double& t1, std::unordered_map& filenames, std::unordered_map& synchronicity, std::unordered_map& responseerrors, const std::vector& option_regex) { std::unordered_map result; std::unique_ptr fin( path.empty() ? nullptr : new std::ifstream( path, std::ifstream::in ) ); std::istream &input = path.empty() ? std::cin : *fin; std::string line; std::unordered_map files; std::unordered_map last_stop; std::unordered_map overlaps; std::unordered_map overlaps_cnt; t0 = 10e99; t1 = 0; while (input.good()) { std::getline(input, line); if (line.empty()) continue; std::vector tokens = ToColumns( line ); if (tokens.size() == 6) tokens.emplace_back(); if (tokens.size() != 7) { throw std::invalid_argument("Invalid input file format."); } uint64_t id = std::stoull(tokens[0]); // file object ID std::string action = tokens[1]; // action name (e.g. Open) double start = std::stod(tokens[2]); // start time std::string args = tokens[3]; // operation arguments double stop = std::stod(tokens[4]); // stop time std::string status = tokens[5]; // operation status std::string resp = tokens[6]; // server response if (option_regex.size()) { for (auto& v : option_regex) { std::vector tokens; Utils::splitString(tokens, v, ":="); std::regex src(tokens[0]); if (tokens.size() != 2) { std::cerr << "Error: invalid regex for argument replacement - must be format like :=" << std::endl; exit(EINVAL); } else { // write the results to an output iterator args = std::regex_replace(args, src, tokens[1]); } } } if (start < t0) t0 = start; if (stop > t1) t1 = stop; if (!files.count(id)) { files[id] = new File(false); files[id]->SetProperty("BundledClose", "true"); filenames[files[id]] = args; filenames[files[id]].erase(args.find(";")); overlaps[id] = 0; overlaps_cnt[id] = 0; last_stop[id] = stop; } else { overlaps_cnt[id]++; if (start > last_stop[id]) { overlaps[id]++; } last_stop[id] = stop; } last_stop[id] = stop; double nominal_duration = stop - start; if (status != "[SUCCESS]") { responseerrors[files[id]]++; } else { result[files[id]].emplace( start, ActionExecutor(*files[id], action, args, status, resp, nominal_duration)); } } for (auto& it : overlaps) { // compute the synchronicity of requests synchronicity[files[it.first]] = 100.0 * (it.second / overlaps_cnt[it.first]); } return result; } //------------------------------------------------------------------------------ //! Execute list of actions against given file //! @param file : the file object //! @param actions : list of actions to be executed //! @param t0 : offset to add to each start time to determine when to ru an action //! @return : thread that will executed the list of actions //------------------------------------------------------------------------------ std::thread ExecuteActions(std::unique_ptr file, action_list&& actions, double t0, double speed, ActionMetrics& metric, bool simulate) { std::thread t( [file{ std::move(file) }, actions{ std::move(actions) }, t0, &metric, simulate, &speed]() mutable { XrdSysSemaphore endsem(0); XrdSysSemaphore closesem(0); auto ending = std::make_shared(endsem); auto closing = std::make_shared(closesem); for (auto& p : actions) { auto& action = p.second; auto tdelay = t0 ? ((p.first + t0) - XrdCl::Action::timeNow()) : 0; if (tdelay > 0) { tdelay /= speed; metric.delays[action.Name() + "::tloss"] += tdelay; std::this_thread::sleep_for(std::chrono::milliseconds((int) (tdelay * 1000))); } else { metric.delays[action.Name() + "::tgain"] += tdelay; } mytimer_t timer; action.Execute(ending, closing, metric, simulate); metric.addDelays(action.Name(), "tnomi", action.NominalDuration()); metric.addDelays(action.Name(), "texec", timer.elapsed()); } ending.reset(); closing.reset(); endsem.Wait(); file->GetProperty("LastURL", metric.url); file.reset(); }); return t; } } void usage() { std::cerr << "usage: xrdreplay [-p|--print] [-c|--create-data] [t|--truncate-data] [-l|--long] [-s|--summary] [-h|--help] [-r|--replace :=] [-f|--suppress] \n" << std::endl; std::cerr << " -h | --help : show this help" << std::endl; std::cerr << " -f | --suppress : force to run all IO with all successful result status - suppress all others" << std::endl; std::cerr << " - by default the player won't run with an unsuccessfully recorded IO" << std::endl; std::cerr << std::endl; std::cerr << " -p | --print : print only mode - shows all the IO for the given replay file without actually running any IO" << std::endl; std::cerr << " -s | --summary : print summary - shows all the aggregated IO counter summed for all files" << std::endl; std::cerr << " -l | --long : print long - show all file IO counter for each individual file" << std::endl; std::cerr << " -r | --replace := : replace in the argument list the string with " << std::endl; std::cerr << " - option is usable several times e.g. to change storage prefixes or filenames" << std::endl; std::cerr << std::endl; std::cerr << "example: ... --replace file:://localhost:=root://xrootd.eu/ : redirect local file to remote" << std::endl; std::cerr << std::endl; exit(-1); } int main(int argc, char** argv) { XrdCl::ReplayArgs opt(argc, argv); int rc = 0; try { double t0 = 0; double t1 = 0; std::unordered_map filenames; std::unordered_map synchronicity; std::unordered_map responseerrors; auto actions = XrdCl::ParseInput(opt.path(), t0, t1, filenames, synchronicity, responseerrors, opt.regex()); // parse the input file std::vector threads; std::unordered_map metrics; threads.reserve(actions.size()); double toffset = XrdCl::Action::timeNow() - t0; XrdCl::mytimer_t timer; XrdCl::ActionMetrics summetric; bool sampling_error = false; for (auto& action : actions) { metrics[action.first].fname = filenames[action.first]; metrics[action.first].synchronicity = synchronicity[action.first]; metrics[action.first].errors = responseerrors[action.first]; if (metrics[action.first].errors) { sampling_error = true; } } if (sampling_error) { std::cerr << "Warning: IO file contains unsuccessful samples!" << std::endl; if (!opt.suppress_error()) { std::cerr << "... run with [-f] or [--suppress] option to suppress unsuccessful IO events!" << std::endl; exit(-1); } } if (opt.print()) toffset = 0; // indicate not to follow timing for (auto& action : actions) { // execute list of actions against file object threads.emplace_back(ExecuteActions(std::unique_ptr(action.first), std::move(action.second), toffset, opt.speed(), metrics[action.first], opt.print())); } for (auto& t : threads) // wait until we are done t.join(); if (opt.json()) { std::cout << "{" << std::endl; if (opt.longformat()) std::cout << " \"metrics\": [" << std::endl; } for (auto& metric : metrics) { if (opt.longformat()) { std::cout << metric.second.Dump(opt.json()); } summetric.add(metric.second); } if (opt.summary()) std::cout << summetric.Dump(opt.json()); if (opt.json()) { if (opt.longformat()) std::cout << " ]," << std::endl; } double tbench = timer.elapsed(); if (opt.json()) { { std::cout << " \"iosummary\": { " << std::endl; if (!opt.print()) { std::cout << " \"player::runtime\": " << tbench << "," << std::endl; } std::cout << " \"player::speed\": " << opt.speed() << "," << std::endl; std::cout << " \"sampled::runtime\": " << t1 - t0 << "," << std::endl; std::cout << " \"volume::totalread\": " << summetric.getBytesRead() << "," << std::endl; std::cout << " \"volume::totalwrite\": " << summetric.getBytesWritten() << "," << std::endl; std::cout << " \"volume::read\": " << summetric.ios["Read::b"] << "," << std::endl; std::cout << " \"volume::write\": " << summetric.ios["Write::b"] << "," << std::endl; std::cout << " \"volume::pgread\": " << summetric.ios["PgRead::b"] << "," << std::endl; std::cout << " \"volume::pgwrite\": " << summetric.ios["PgWrite::b"] << "," << std::endl; std::cout << " \"volume::vectorread\": " << summetric.ios["VectorRead::b"] << "," << std::endl; std::cout << " \"volume::vectorwrite\": " << summetric.ios["VectorWrite::b"] << "," << std::endl; std::cout << " \"iops::read\": " << summetric.ios["Read::n"] << "," << std::endl; std::cout << " \"iops::write\": " << summetric.ios["Write::n"] << "," << std::endl; std::cout << " \"iops::pgread\": " << summetric.ios["PgRead::n"] << "," << std::endl; std::cout << " \"iops::pgwrite\": " << summetric.ios["PgRead::n"] << "," << std::endl; std::cout << " \"iops::vectorread\": " << summetric.ios["VectorRead::n"] << "," << std::endl; std::cout << " \"iops::vectorwrite\": " << summetric.ios["VectorRead::n"] << "," << std::endl; std::cout << " \"files::read\": " << summetric.ios["OpenR::n"] << "," << std::endl; std::cout << " \"files::write\": " << summetric.ios["OpenW::n"] << "," << std::endl; std::cout << " \"datasetsize::read\": " << summetric.ios["Read::o"] << "," << std::endl; std::cout << " \"datasetsize::write\": " << summetric.ios["Write::o"] << "," << std::endl; if (!opt.print()) { std::cout << " \"bandwidth::mb::read\": " << summetric.getBytesRead() / tbench / 1000000.0 << "," << std::endl; std::cout << " \"bandwdith::mb::write\": " << summetric.getBytesWritten() / tbench / 1000000.0 << "," << std::endl; std::cout << " \"performancemark\": " << (100.0 * (t1 - t0) / tbench) << "," << std::endl; std::cout << " \"gain::read\":" << (100.0 * summetric.delays["Read::tnomi"] / summetric.delays["Read::tmeas"]) << "," << std::endl; std::cout << " \"gain::write\":" << (100.0 * summetric.delays["Write::tnomi"] / summetric.delays["Write::tmeas"]) << std::endl; } std::cout << " \"synchronicity::read\":" << summetric.aggregated_synchronicity.ReadSynchronicity() << "," << std::endl; std::cout << " \"synchronicity::write\":" << summetric.aggregated_synchronicity.WriteSynchronicity() << "," << std::endl; std::cout << " \"response::error:\":" << summetric.ios["All::e"] << std::endl; std::cout << " }" << std::endl; std::cout << "}" << std::endl; } } else { std::cout << "# =============================================" << std::endl; if (!opt.print()) std::cout << "# IO Summary" << std::endl; else std::cout << "# IO Summary (print mode)" << std::endl; std::cout << "# =============================================" << std::endl; if (!opt.print()) { std::cout << "# Total Runtime : " << std::fixed << tbench << " s" << std::endl; } std::cout << "# Sampled Runtime : " << std::fixed << t1 - t0 << " s" << std::endl; std::cout << "# Playback Speed : " << std::fixed << std::setprecision(2) << opt.speed() << std::endl; std::cout << "# IO Volume (R) : " << std::fixed << XrdCl::ActionMetrics::humanreadable(summetric.getBytesRead()) << " [ std:" << XrdCl::ActionMetrics::humanreadable(summetric.ios["Read::b"]) << " vec:" << XrdCl::ActionMetrics::humanreadable(summetric.ios["VectorRead::b"]) << " page:" << XrdCl::ActionMetrics::humanreadable(summetric.ios["PgRead::b"]) << " ] " << std::endl; std::cout << "# IO Volume (W) : " << std::fixed << XrdCl::ActionMetrics::humanreadable(summetric.getBytesWritten()) << " [ std:" << XrdCl::ActionMetrics::humanreadable(summetric.ios["Write::b"]) << " vec:" << XrdCl::ActionMetrics::humanreadable(summetric.ios["VectorWrite::b"]) << " page:" << XrdCl::ActionMetrics::humanreadable(summetric.ios["PgWrite::b"]) << " ] " << std::endl; std::cout << "# IOPS (R) : " << std::fixed << summetric.getIopsRead() << " [ std:" << summetric.ios["Read::n"] << " vec:" << summetric.ios["VectorRead::n"] << " page:" << summetric.ios["PgRead::n"] << " ] " << std::endl; std::cout << "# IOPS (W) : " << std::fixed << summetric.getIopsWrite() << " [ std:" << summetric.ios["Write::n"] << " vec:" << summetric.ios["VectorWrite::n"] << " page:" << summetric.ios["PgWrite::n"] << " ] " << std::endl; std::cout << "# Files (R) : " << std::fixed << summetric.ios["OpenR::n"] << std::endl; std::cout << "# Files (W) : " << std::fixed << summetric.ios["OpenW::n"] << std::endl; std::cout << "# Datasize (R) : " << std::fixed << XrdCl::ActionMetrics::humanreadable(summetric.ios["Read::o"]) << std::endl; std::cout << "# Datasize (W) : " << std::fixed << XrdCl::ActionMetrics::humanreadable(summetric.ios["Write::o"]) << std::endl; if (!opt.print()) { std::cout << "# IO BW (R) : " << std::fixed << std::setprecision(2) << summetric.getBytesRead() / tbench / 1000000.0 << " MB/s" << std::endl; std::cout << "# IO BW (W) : " << std::fixed << std::setprecision(2) << summetric.getBytesRead() / tbench / 1000000.0 << " MB/s" << std::endl; } std::cout << "# ---------------------------------------------" << std::endl; std::cout << "# Quality Estimation" << std::endl; std::cout << "# ---------------------------------------------" << std::endl; if (!opt.print()) { std::cout << "# Performance Mark : " << std::fixed << std::setprecision(2) << (100.0 * (t1 - t0) / tbench) << "%" << std::endl; std::cout << "# Gain Mark(R) : " << std::fixed << std::setprecision(2) << (100.0 * summetric.delays["Read::tnomi"] / summetric.delays["Read::tmeas"]) << "%" << std::endl; std::cout << "# Gain Mark(W) : " << std::fixed << std::setprecision(2) << (100.0 * summetric.delays["Write::tnomi"] / summetric.delays["Write::tmeas"]) << "%" << std::endl; } std::cout << "# Synchronicity(R) : " << std::fixed << std::setprecision(2) << summetric.aggregated_synchronicity.ReadSynchronicity() << "%" << std::endl; std::cout << "# Synchronicity(W) : " << std::fixed << std::setprecision(2) << summetric.aggregated_synchronicity.WriteSynchronicity() << "%" << std::endl; if (!opt.print()) { std::cout << "# ---------------------------------------------" << std::endl; std::cout << "# Response Errors : " << std::fixed << summetric.ios["All::e"] << std::endl; std::cout << "# =============================================" << std::endl; if (summetric.ios["All::e"]) { std::cerr << "Error: replay job failed with IO errors!" << std::endl; rc = -5; } } if (opt.create() || opt.verify()) { std::cout << "# ---------------------------------------------" << std::endl; if (opt.create()) { std::cout << "# Creating Dataset ..." << std::endl; } else { std::cout << "# Verifying Dataset ..." << std::endl; } uint64_t created_sofar = 0; for (auto& metric : metrics) { if (metric.second.getBytesRead() && !metric.second.getBytesWritten()) { std::cout << "# ............................................." << std::endl; std::cout << "# file: " << metric.second.fname << std::endl; std::cout << "# size: " << XrdCl::ActionMetrics::humanreadable(metric.second.ios["Read::o"]) << " [ " << XrdCl::ActionMetrics::humanreadable(created_sofar) << " out of " << XrdCl::ActionMetrics::humanreadable(summetric.ios["Read::o"]) << " ] " << std::setprecision(2) << " ( " << 100.0 * created_sofar / summetric.ios["Read::o"] << "% )" << std::endl; if (!XrdCl::AssureFile( metric.second.fname, metric.second.ios["Read::o"], opt.truncate(), opt.verify())) { if (opt.verify()) { rc = -5; } else { std::cerr << "Error: failed to assure that file " << metric.second.fname << " is stored with a size of " << XrdCl::ActionMetrics::humanreadable(metric.second.ios["Read::o"]) << " !!!"; rc = -5; } } } } } } } catch (const std::invalid_argument& ex) { std::cout << ex.what() << std::endl; // print parsing errors return 1; } return rc; } xrootd-5.6.9/src/XrdApps/XrdClRecordPlugin/XrdClReplayArgs.hh000066400000000000000000000166121457266313600241260ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2021 by European Organization for Nuclear Research (CERN) // Author: Andreas-Joachim Peters //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #include #include #include #include #include namespace XrdCl { //------------------------------------------------------------------------------ //! Args parse for XrdClReplay //------------------------------------------------------------------------------ class ReplayArgs { public: ReplayArgs(int argc, char* argv[]) : option_long(false) , option_summary(false) , option_print(false) , option_create(false) , option_truncate(false) , option_json(false) , option_suppress_error(false) , option_verify(false) , option_speed(1.0) { while (1) { int option_index = 0; static struct option long_options[] = { { "help", no_argument, 0, 'h' }, { "print", no_argument, 0, 'p' }, { "create", no_argument, 0, 'c' }, { "truncate", no_argument, 0, 't' }, { "long", no_argument, 0, 'l' }, { "json", no_argument, 0, 'j' }, { "summary", no_argument, 0, 's' }, { "replace", required_argument, 0, 'r' }, { "suppress", no_argument, 0, 'f' }, { "verify", no_argument, 0, 'v' }, { "speed", required_argument, 0, 'x' }, { 0, 0, 0, 0 } }; int c = getopt_long(argc, argv, "vjpctshlfr:x:", long_options, &option_index); if (c == -1) break; switch (c) { case 'h': usage(); break; case 'c': option_create = true; option_print = true; // create mode requires to run in simulated mode (print) break; case 't': option_create = true; option_print = true; // truncate mode requires to run in simulated mode (print) option_truncate = true; break; case 'j': option_json = true; break; case 'p': option_print = true; break; case 's': option_summary = true; break; case 'l': option_long = true; break; case 'v': option_verify = true; break; case 'x': option_speed = std::strtod(optarg, 0); if (option_speed <= 0) { usage(); } break; case 'r': option_regex.push_back(optarg); break; case 'f': option_suppress_error = true; break; default: usage(); } } if (option_json && (option_long || option_summary)) option_long = option_summary = true; if (option_verify) { option_print = true; option_create = false; option_truncate = false; option_json = false; } if (optind < (argc - 1)) { usage(); } if (optind == (argc -1 )) { // we also accept to have no path and read from STDIN _path = argv[optind]; } } void usage() { std::cerr << "usage: xrdreplay [-p|--print] [-c|--create-data] [t|--truncate-data] [-l|--long] [-s|--summary] [-h|--help] [-r|--replace :=] [-f|--suppress] [-v|--verify] [-x|--speed ]\n" << std::endl; std::cerr << " -h | --help : show this help" << std::endl; std::cerr << " -f | --suppress : force to run all IO with all successful result status - suppress all others" << std::endl; std::cerr << " - by default the player won't run with an unsuccessful recorded IO" << std::endl; std::cerr << std::endl; std::cerr << " -p | --print : print only mode - shows all the IO for the given replay file without actually running any IO" << std::endl; std::cerr << " -s | --summary : print summary - shows all the aggregated IO counter summed for all files" << std::endl; std::cerr << " -l | --long : print long - show all file IO counter for each individual file" << std::endl; std::cerr << " -v | --verify : verify the existence of all input files" << std::endl; std::cerr << " -x | --speed : change playback speed by factor [ > 0.0 ]" << std::endl; std::cerr << " -r | --replace := : replace in the argument list the string with " << std::endl; std::cerr << " - option is usable several times e.g. to change storage prefixes or filenames" << std::endl; std::cerr << std::endl; std::cerr << " [recordfilename] : if a file is given, it will be used as record input otherwise STDIN is used to read records!" << std::endl; std::cerr << "example: ... --replace file:://localhost:=root://xrootd.eu/ : redirect local file to remote" << std::endl; std::cerr << std::endl; exit(-1); } bool longformat() { return option_long; } bool summary() { return option_summary; } bool print() { return option_print; } bool create() { return option_create; } bool truncate() { return option_truncate; } bool json() { return option_json; } bool suppress_error() { return option_suppress_error; } bool verify() { return option_verify; } double speed() { return option_speed; } std::vector& regex() { return option_regex; } std::string& path() { return _path; } private: bool option_long; bool option_summary; bool option_print; bool option_create; bool option_truncate; bool option_json; bool option_suppress_error; bool option_verify; double option_speed; std::vector option_regex; std::string _path; }; } xrootd-5.6.9/src/XrdApps/XrdCpConfig.cc000066400000000000000000001173241457266313600177240ustar00rootroot00000000000000/******************************************************************************/ /* */ /* X r d C p C o n f i g . c c */ /* */ /* (c) 2011 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include #include #include #include #include #include #include "XrdVersion.hh" #include "XrdApps/XrdCpConfig.hh" #include "XrdApps/XrdCpFile.hh" #include "XrdCks/XrdCksCalc.hh" #include "XrdCks/XrdCksManager.hh" #include "XrdOuc/XrdOucStream.hh" #include "XrdSys/XrdSysE2T.hh" #include "XrdSys/XrdSysError.hh" #include "XrdSys/XrdSysHeaders.hh" #include "XrdSys/XrdSysLogger.hh" using namespace std; /******************************************************************************/ /* D e f i n e M a c r o s */ /******************************************************************************/ #define EMSG(x) std::cerr <Next; delete pNow;} while((dP = intDefs)) {intDefs = dP->Next; delete dP;} while((dP = strDefs)) {strDefs = dP->Next; delete dP;} } /******************************************************************************/ /* C o n f i g */ /******************************************************************************/ void XrdCpConfig::Config(int aCnt, char **aVec, int opts) { extern char *optarg; extern int optind, opterr; static int pgmSet = 0; char Buff[128], *Path, opC; XrdCpFile pBase; int i, rc; // Allocate a parameter vector // if (parmVal) free(parmVal); parmVal = (char **)malloc(aCnt*sizeof(char *)); // Preset handling options // Argv = aVec; Argc = aCnt; Opts = opts; opterr = 0; optind = 1; opC = 0; // Set name of executable for error messages // if (!pgmSet) {char *Slash = rindex(aVec[0], '/'); pgmSet = 1; Pgm = (Slash ? Slash+1 : aVec[0]); Log->SetPrefix(Pgm); } // Process legacy options first before atempting normal options // do{while(optind < Argc && Legacy(optind)) {} if ((opC = getopt_long(Argc, Argv, opLetters, opVec, &i)) != (char)-1) switch(opC) {case OpCksum: defCks(optarg); break; case OpCoerce: OpSpec |= DoCoerce; break; case OpDebug: OpSpec |= DoDebug; if (!a2i(optarg, &Dlvl, 0, 3)) Usage(22); break; case OpDynaSrc: OpSpec |= DoDynaSrc; break; case OpForce: OpSpec |= DoForce; break; case OpZip: OpSpec |= DoZip; if (zipFile) free(zipFile); zipFile = strdup(optarg); break; case OpHelp: Usage(0); break; case OpIfile: if (inFile) free(inFile); inFile = strdup(optarg); OpSpec |= DoIfile; break; case OpLicense: License(); break; case OpNoPbar: OpSpec |= DoNoPbar; break; case OpNoTlsOK: OpSpec |= DoNoTlsOK; break; case OpPath: OpSpec |= DoPath; break; case OpPosc: OpSpec |= DoPosc; break; case OpProxy: OpSpec |= DoProxy; defPxy(optarg); break; case OpRecurse: OpSpec |= DoRecurse; break; case OpRecursv: OpSpec |= DoRecurse; break; case OpRetry: OpSpec |= DoRetry; if (!a2i(optarg, &Retry, 0, -1)) Usage(22); break; case OpRetryPolicy: OpSpec |= DoRetryPolicy; RetryPolicy = optarg; if( RetryPolicy != "force" && RetryPolicy != "continue" ) Usage(22); break; case OpZipAppend: OpSpec |= DoZipAppend; break; case OpServer: OpSpec |= DoServer|DoSilent|DoNoPbar|DoForce; break; case OpSilent: OpSpec |= DoSilent|DoNoPbar; break; case OpSources: OpSpec |= DoSources; if (!a2i(optarg, &nSrcs, 1, 32)) Usage(22); break; case OpStreams: OpSpec |= DoStreams; if (!a2i(optarg, &nStrm, 1, 15)) Usage(22); break; case OpTlsNoData: OpSpec |= DoTlsNoData; break; case OpTlsMLF: OpSpec |= DoTlsMLF; break; case OpTpc: OpSpec |= DoTpc; if (!strcmp("delegate", optarg)) {OpSpec|= DoTpcDlgt; if (optind >= Argc) {UMSG("Missing tpc qualifier after " "'delegate'"); } optarg = Argv[optind++]; } if (!strcmp("only", optarg)) OpSpec|= DoTpcOnly; else if (strcmp("first", optarg)) {optind--; UMSG("Invalid option, '" < 1) UMSG("Third party copy requires a single source."); // Check for conflicts with ZIP archive // if( OpSpec & DoZip & DoCksrc ) UMSG("Cannot calculate source checksum for a file in ZIP archive."); if( ( OpSpec & DoZip & DoCksum ) && !CksData.HasValue() ) UMSG("Cannot calculate source checksum for a file in ZIP archive."); // Turn off verbose if we are in server mode // if (OpSpec & DoServer) {OpSpec &= ~DoVerbose; Verbose = 0; } // Turn on auto-path creation if requested via envar // if (getenv("XRD_MAKEPATH")) OpSpec |= DoPath; // Process the destination first as it is special // dstFile = new XrdCpFile(parmVal[--parmCnt], rc); if (rc) FMSG("Invalid url, '" <Path <<"'.", 22); // Allow HTTP if XRDCP_ALLOW_HTTP is set if (getenv("XRDCP_ALLOW_HTTP")) { OpSpec |= DoAllowHttp; } // Do a protocol check // if (dstFile->Protocol != XrdCpFile::isFile && dstFile->Protocol != XrdCpFile::isStdIO && dstFile->Protocol != XrdCpFile::isXroot && (!Want(DoAllowHttp) && ((dstFile->Protocol == XrdCpFile::isHttp) || (dstFile->Protocol == XrdCpFile::isHttps)))) {FMSG(dstFile->ProtName <<"file protocol is not supported.", 22)} // Resolve this file if it is a local file // isLcl = (dstFile->Protocol == XrdCpFile::isFile) | (dstFile->Protocol == XrdCpFile::isStdIO); if (isLcl && (rc = dstFile->Resolve())) {if (rc != ENOENT || (Argc - optind - 1) > 1 || OpSpec & DoRecurse) FMSG(XrdSysE2T(rc) <<" processing " <Path, 2); } // Now pick up all the source files from the command line // pLast = &pBase; for (i = 0; i < parmCnt; i++) ProcFile(parmVal[i]); // If an input file list was specified, process it as well // if (inFile) {XrdOucStream inList(Log); char *fname; int inFD = open(inFile, O_RDONLY); if (inFD < 0) FMSG(XrdSysE2T(errno) <<" opening infiles " < 1) FMSG("Only a single source is allowed.", 2); srcFile = pBase.Next; // Check if we have an appropriate destination // if (dstFile->Protocol == XrdCpFile::isFile && (numFiles > 1 || (OpSpec & DoRecurse && srcFile->Protocol != XrdCpFile::isFile))) FMSG("Destination is neither remote nor a directory.", 2); // Do the dumb check // if (isLcl && Opts & optNoLclCp) FMSG("All files are local; use 'cp' instead!", 1); // Check for checksum spec conflicts // if (OpSpec & DoCksum) {if (CksData.Length && numFiles > 1) FMSG("Checksum with fixed value requires a single input file.", 2); if (CksData.Length && OpSpec & DoRecurse) FMSG("Checksum with fixed value conflicts with '--recursive'.", 2); } // Now extend all local sources if recursive is in effect // if (OpSpec & DoRecurse && !(Opts & optNoXtnd)) {pPrev = &pBase; pBase.Next = srcFile; while((pFile = pPrev->Next)) {if (pFile->Protocol != XrdCpFile::isDir) pPrev = pFile; else {Path = pFile->Path; pPrev->Next = pFile->Next; if (Verbose) EMSG("Indexing files in " <Extend(&pLast, numFiles, totBytes))) FMSG(XrdSysE2T(rc) <<" indexing " <Next) {pLast->Next = pPrev->Next; pPrev->Next = pFile->Next; } delete pFile; } } if (!(srcFile = pBase.Next)) FMSG("No regular files found to copy!", 2); if (Verbose) EMSG("Copying " <= " <= 0 && *val > maxv) ZMSG("'" <= " <= 0 && *val > maxv) ZMSG("'" <= " <= 0 && *val > maxv) ZMSG("'" <= " <= 0 && *val > maxv) ZMSG("'" <Init(""))) {delete CksMan; CksMan = 0; FMSG("Unable to initialize checksum processing.", 13); } } // Copy out the checksum name // n = (Colon ? Colon - opval : strlen(opval)); if (n >= XrdCksData::NameSize) UMSG("Invalid checksum type, '" <Object(CksData.Name))) UMSG("Invalid checksum type, '" <Type(CksLen); } // Reset checksum information // CksData.Length = 0; OpSpec &= ~(DoCkprt | DoCksrc | DoCksum); // Check for any additional arguments // if (Colon) {Colon++; if (!(*Colon)) UMSG(CksData.Name <<" argument missing after ':'."); if (!strcmp(Colon, "print")) OpSpec |= (DoCkprt | DoCksum); else if (!strcmp(Colon, "source")) OpSpec |= (DoCkprt | DoCksrc); else {n = strlen(Colon); if (n != CksLen*2 || !CksData.Set(Colon, n)) UMSG("Invalid " <Next = dP; intDend = dP;} } else { dP = new defVar(vName, theArg); if (!strDend) strDefs = strDend = dP; else {strDend->Next = dP; strDend = dP;} } // Convert the argument // return 2; } /******************************************************************************/ /* Private: d e f P x y */ /******************************************************************************/ void XrdCpConfig::defPxy(const char *opval) { const char *Colon = index(opval, ':'); char *eP; int n; // Make sure the host was specified // if (Colon == opval) UMSG("Proxy host not specified."); // Make sure the port was specified // if (!Colon || !(*(Colon+1))) UMSG("Proxy port not specified."); // Make sure the port is a valid number that is not too big // errno = 0; pPort = strtol(Colon+1, &eP, 10); if (errno || *eP || pPort < 1 || pPort > 65535) UMSG("Invalid proxy port, '" <= 1024; i++) inval = inval/1024; snprintf(Buff, Blen, "%lld%s", inval, sfx[i]); return Buff; } /******************************************************************************/ /* Private: L e g a c y */ /******************************************************************************/ int XrdCpConfig::Legacy(int oIndex) { extern int optind; char *oArg; int rc; // if (!Argv[oIndex]) return 0; while(oIndex < Argc && (*Argv[oIndex] != '-' || *(Argv[oIndex]+1) == '\0')) parmVal[parmCnt++] = Argv[oIndex++]; if (oIndex >= Argc) return 0; if (oIndex+1 >= Argc || *Argv[oIndex+1] == '-') oArg = 0; else oArg = Argv[oIndex+1]; if (!(rc = Legacy(Argv[oIndex], oArg))) return 0; optind = oIndex + rc; return 1; } /******************************************************************************/ int XrdCpConfig::Legacy(const char *theOp, const char *theArg) { if (!strcmp(theOp, "-adler")) return defCks("adler32:source"); if (!strncmp(theOp, "-DI", 3) || !strncmp(theOp, "-DS", 3)) return defOpt(theOp, theArg); if (!strcmp(theOp, "-extreme") || !strcmp(theOp, "-x")) {if (nSrcs <= 1) {nSrcs = dfltSrcs; OpSpec |= DoSources;} return 1; } if (!strcmp(theOp, "-np")) {OpSpec |= DoNoPbar; return 1;} if (!strcmp(theOp, "-md5")) return defCks("md5:source"); if (!strncmp(theOp,"-OD",3) || !strncmp(theOp,"-OS",3)) return defOpq(theOp); if (!strcmp(theOp, "-version")) {std::cerr <Next = pFile = new XrdCpFile(fname, rc); if (rc) FMSG("Invalid url, '" <Protocol == XrdCpFile::isFile && (rc = pFile->Resolve())) FMSG(XrdSysE2T(rc) <<" processing " <Path, 2); // Process file based on type (local or remote) // if (pFile->Protocol == XrdCpFile::isFile) totBytes += pFile->fSize; else if (pFile->Protocol == XrdCpFile::isDir) {if (!(OpSpec & DoRecurse)) FMSG(pFile->Path <<" is a directory.", 2); } else if (pFile->Protocol == XrdCpFile::isStdIO) {if (Opts & optNoStdIn) FMSG("Using stdin as a source is disallowed.", 22); if (numFiles) FMSG("Multiple sources disallowed with stdin.", 22); } else if (!((pFile->Protocol == XrdCpFile::isXroot) || (pFile->Protocol == XrdCpFile::isXroots) || (Want(DoAllowHttp) && ((pFile->Protocol == XrdCpFile::isHttp) || (pFile->Protocol == XrdCpFile::isHttps))))) {FMSG(pFile->ProtName <<" file protocol is not supported.", 22)} else if (OpSpec & DoRecurse && !(Opts & optRmtRec)) {FMSG("Recursive copy from a remote host is not supported.",22)} else isLcl = 0; // Update last pointer and we are done if this is stdin // numFiles++; pLast = pFile; } /******************************************************************************/ /* U s a g e */ /******************************************************************************/ void XrdCpConfig::Usage(int rc) { static const char *Syntax = "\n" "Usage: xrdcp [] [ [. . .]] \n"; static const char *Syntax1= "\n" "Usage: xrdcp [] \n"; static const char *Options= "\n" "Options: [--allow-http] [--cksum ] [--coerce] [--continue]\n" " [--debug ] [--dynamic-src] [--force] [--help]\n" " [--infiles ] [--license] [--nopbar] [--notlsok]\n" " [--parallel ] [--posc] [--proxy :]\n" " [--recursive] [--retry ] [--retry-policy ]\n" " [--rm-bad-cksum] [--server] [--silent] [--sources ]\n" " [--streams ] [--tlsmetalink] [--tlsnodata]\n" " [--tpc [delegate] {first|only}] [--verbose] [--version]\n" " [--xattr] [--xrate ] [--xrate-threshold ]\n" " [--zip ] [--zip-append] [--zip-mtln-cksum]\n"; static const char *Syntax2= "\n" ": [[x]root[s]://[:]/] | -"; static const char *Syntay2= "\n" ": [[x]root[s]://[:]/]"; static const char *Syntax3= "\n" ": [[x]root[s]://[:]/] | -"; static const char *Detail = "\n" "Note: using a dash (-) for uses stdin and for stdout\n\n" "-A | --allow-http allow HTTP as source or destination protocol. Requires\n" " the XrdClHttp client plugin\n" "-C | --cksum verifies the checksum at the destination as provided\n" " by the source server or locally computed. The args are\n" " [:{|print|source}]\n" " where is one of adler32, crc32, crc32c, md5,\n" " zcrc32 or auto. If 'auto' is chosen, xrdcp will try to\n" " automatically infer the right checksum type based on the\n" " source/destination configuration, source file type\n" " (e.g. metalink, ZIP), and available checksum plug-ins.\n" " If the hex value of the checksum is given, it is used.\n" " Otherwise, the server's checksum is used for remote files\n" " and computed for local files. Specifying print merely\n" " prints the checksum but does not verify it.\n" "-F | --coerce coerces the copy by ignoring file locking semantics\n" " --continue continue copying a file from the point where the previous\n" " copy was interrupted\n" "-d | --debug sets the debug level: 0 off, 1 low, 2 medium, 3 high\n" "-Z | --dynamic-src file size may change during the copy\n" "-f | --force replaces any existing output file\n" "-h | --help prints this information\n" "-I | --infiles specifies the file that contains a list of input files\n" "-H | --license prints license terms and conditions\n" "-N | --nopbar does not print the progress bar\n" " --notlsok if server is too old to support TLS encryption fallback\n" " to unencrypted communication\n" " --parallel number of files to copy at the same time\n" "-P | --posc enables persist on successful close semantics\n" "-D | --proxy : uses the specified SOCKS4 proxy connection\n" "-r | --recursive recursively copies all source files\n" "-t | --retry maximum number of times to retry failed copy-jobs\n" " --retry-policy retry policy: force or continue\n" " --rm-bad-cksum remove the target file if checksum verification failed\n" " (enables also POSC semantics)\n" " --server runs in a server environment with added operations\n" "-s | --silent produces no output other than error messages\n" "-y | --sources uses up to the number of sources specified in parallel\n" "-S | --streams copies using the specified number of TCP connections\n" " --tlsmetalink convert [x]root to [x]roots protocol in metalinks\n" "-E | --tlsnodata in case of [x]roots protocol, encrypt only the control\n" " stream and leave the data streams unencrypted\n" "-T | --tpc uses third party copy mode between the src and dest.\n" " Both the src and dest must allow tpc mode. Argument\n" " 'first' tries tpc and if it fails, does a normal copy;\n" " while 'only' fails the copy unless tpc succeeds.\n" "-v | --verbose produces more information about the copy\n" "-V | --version prints the version number\n" " --xattr preserve extended attributes\n" "-X | --xrate limits the transfer to the specified rate. You can\n" " suffix the value with 'k', 'm', or 'g'\n" " --xrate-threshold If the transfer rate drops below given threshold force\n" " the client to use different source or if no more sources\n" " are available fail the transfer. You can suffix the value\n" " with 'k', 'm', or 'g'\n" "-z | --zip treat the source as a ZIP archive containing given file\n" " --zip-append append file to existing zip archive\n" " --zip-mtln-cksum use the checksum available in a metalink file even if\n" " a file is being extracted from a ZIP archive\n" "\n" "Legacy options: [-adler] [-DI ] [-DS ] [-np]\n" " [-md5] [-OD] [-OS] [-version] [-x]"; std::cerr <<(Opts & opt1Src ? Syntax1 : Syntax) <. */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include "XrdCks/XrdCksData.hh" #include #include #include #include struct option; class XrdCks; class XrdCksCalc; class XrdCpFile; class XrdSysError; class XrdCpConfig { public: struct defVar { defVar *Next; // -> Next such definition, 0 if no more const char *vName; // -> Variable name union {const char *strVal; // -> String value if in strDefs int intVal; // Integer value if in intDefs }; defVar(const char *vn, const char *vl) : Next(0), vName(vn), strVal(vl) {} defVar(const char *vn, int vl) : Next(0), vName(vn), intVal(vl) {} }; defVar *intDefs; // -> -DI settings defVar *strDefs; // -> -DS settings const char *dstOpq; // -> -OD setting (dest opaque) const char *srcOpq; // -> -OS setting (src opaque) const char *Pgm; // -> Program name long long xRate; // -> xrate value in bytes/sec (0 if not set) long long xRateThreshold; // -> xrate threshold value in bytes/sec (0 if not set) int Parallel; // Number of simultaneous copy ops (1 to 4) char *pHost; // -> SOCKS4 proxy hname (0 if none) int pPort; // SOCKS4 proxy port long long OpSpec; // Bit mask of set options (see Doxxxx) int Dlvl; // Debug level (0 to 3) int nSrcs; // Number of sources wanted (dflt 1) int nStrm; // Number of streams wanted (dflt 1) int Retry; // Max times to retry failed copy job std::string RetryPolicy; // retry policy (to force or to continue) int Verbose; // True if --verbose specified int CksLen; // Binary length of checksum, if any int numFiles; // Number of source files long long totBytes; // Total number of bytes for local files XrdCksData CksData; // Checksum information XrdCks *CksMan; // -> Checksum manager XrdCksCalc *CksObj; // -> Cks computation object (0 if no cks) const char *CksVal; // -> Cks argument (0 if none) XrdCpFile *srcFile; // List of source files XrdCpFile *dstFile; // The destination for the copy char *zipFile; // The file name if the URL points to a ZIP archive static XrdSysError *Log; // -> Error message object std::vector AddCksVal; // -> Additional checksum argument static const uint64_t OpCksum = 'C'; // -adler -MD5 legacy -> DoCksrc static const uint64_t DoCksrc = 0x0000000000000001LL; // --cksum :source static const uint64_t DoCksum = 0x0000000000000002LL; // --cksum static const uint64_t DoCkprt = 0x0000000000000004LL; // --cksum :print static const uint64_t OpCoerce = 'F'; static const uint64_t DoCoerce = 0x0000000000000008LL; // -F | --coerce static const uint64_t OpDebug = 'd'; static const uint64_t DoDebug = 0x0000000000000010LL; // -d | --debug static const uint64_t OpForce = 'f'; static const uint64_t DoForce = 0x0000000000000020LL; // -f | --force static const uint64_t OpHelp = 'h'; static const uint64_t DoHelp = 0x0000000000000040LL; // -h | --help static const uint64_t OpIfile = 'I'; static const uint64_t DoIfile = 0x0000000000000080LL; // -I | --infiles static const uint64_t OpLicense = 'H'; // -H | --license static const uint64_t OpNoPbar = 'N'; // -N | --nopbar | -np {legacy} static const uint64_t DoNoPbar = 0x0000000000000100LL; static const uint64_t OpPosc = 'P'; static const uint64_t DoPosc = 0x0000000000000200LL; // -P | --posc static const uint64_t OpProxy = 'D'; static const uint64_t DoProxy = 0x0000000000000400LL; // -D | --proxy static const uint64_t OpRecurse = 'r'; static const uint64_t OpRecursv = 'R'; // -r | --recursive | -R {legacy} static const uint64_t DoRecurse = 0x0000000000000800LL; static const uint64_t OpRetry = 't'; static const uint64_t DoRetry = 0x0000000000001000LL; // -t | --retry static const uint64_t OpServer = 0x03; static const uint64_t DoServer = 0x0000000000002000LL; // --server static const uint64_t OpSilent = 's'; static const uint64_t DoSilent = 0x0000000000004000LL; // -s | --silent static const uint64_t OpSources = 'y'; static const uint64_t DoSources = 0x0000000000008000LL; // -y | --sources static const uint64_t OpStreams = 'S'; static const uint64_t DoStreams = 0x0000000000010000LL; // -S | --streams static const uint64_t OpTpc = 'T'; // -T | --tpc [delegate] {first | only} static const uint64_t DoTpc = 0x0000000000020000LL; // --tpc {first | only} static const uint64_t DoTpcOnly = 0x0000000000100000LL; // --tpc only static const uint64_t DoTpcDlgt = 0x0000000000800000LL; // --tpc delegate ... static const uint64_t OpVerbose = 'v'; static const uint64_t DoVerbose = 0x0000000000040000LL; // -v | --verbose static const uint64_t OpVersion = 'V'; // -V | --version static const uint64_t OpXrate = 'X'; static const uint64_t DoXrate = 0x0000000000080000LL; // -X | --xrate static const uint64_t OpParallel = 0x04; static const uint64_t DoParallel = 0x0000000000200000LL; // --parallel static const uint64_t OpDynaSrc = 'Z'; static const uint64_t DoDynaSrc = 0x0000000000400000LL; // --dynamic-src // const uint64_t DoTpcDlgt = 0x0000000000800000LL; // Marker for bit used static const uint64_t OpZip = 'z'; static const uint64_t DoZip = 0x0000000001000000LL; // -z | --zip static const uint64_t OpTlsNoData = 'E'; static const uint64_t DoTlsNoData = 0x0000000002000000LL; // -E | --tlsnodata static const uint64_t OpNoTlsOK = 0x05; static const uint64_t DoNoTlsOK = 0x0000000004000000LL; // --notlsok static const uint64_t OpTlsMLF = 0x06; static const uint64_t DoTlsMLF = 0x0000000008000000LL; // --tlsmetalink static const uint64_t OpPath = 'p'; static const uint64_t DoPath = 0x0000000010000000LL; // -p | --path static const uint64_t OpXAttr = 0x07; static const uint64_t DoXAttr = 0x0000000020000000LL; // --xattr static const uint64_t OpZipMtlnCksum = 0x08; static const uint64_t DoZipMtlnCksum = 0x0000000040000000LL; // --zip-mtln-cksum static const uint64_t OpRmOnBadCksum = 0x09; static const uint64_t DoRmOnBadCksum = 0x0000000080000000LL; // --rm-bad-cksum static const uint64_t OpContinue = 0x10; static const uint64_t DoContinue = 0x0000000100000000LL; // --continue static const uint64_t OpXrateThreshold = 0x11; static const uint64_t DoXrateThreshold = 0x0000000200000000LL; // --xrate-threshold static const uint64_t OpRetryPolicy = 0x12; static const uint64_t DoRetryPolicy = 0x0000000400000000LL; // --retry-policy static const uint64_t OpZipAppend = 0x13; static const uint64_t DoZipAppend = 0x0000000800000000LL; // --zip-append // Flag to allow the use of HTTP (and HTTPS) as source and destination // protocols. If specified, the XrdClHttp client plugin must be available // for the transfer operations to succeed. static const int OpAllowHttp = 'A'; static const int DoAllowHttp = 0x2000000; // --allow-http // Call Config with the parameters passed to main() to fill out this object. If // the method returns then no errors have been found. Otherwise, it exits. // The following options may be passed (largely to support legacy stuff): // static const int opt1Src = 0x00000001; // Only one source is allowed static const int optNoXtnd = 0x00000002; // Do not index source directories static const int optRmtRec = 0x00000004; // Allow remote recursive copy static const int optNoStdIn = 0x00000008; // Disallow '-' as src for stdin static const int optNoLclCp = 0x00000010; // Disallow local/local copy void Config(int argc, char **argv, int Opts=0); // Method to check for setting // inline int Want(uint64_t What) {return (OpSpec & What) != 0;} XrdCpConfig(const char *pgname); ~XrdCpConfig(); private: int a2i(const char *item, int *val, int minv, int maxv=-1); int a2l(const char *item, long long *val, long long minv, long long maxv=-1); int a2t(const char *item, int *val, int minv, int maxv=-1); int a2x(const char *Val, char *Buff, int Vlen); int a2z(const char *item, long long *val, long long minv, long long maxv=-1); int defCks(const char *opval); int defOpq(const char *theOp); int defOpt(const char *theOp, const char *theArg); void defPxy(const char *opval); const char *Human(long long Val, char *Buff, int Blen); int Legacy(int oIndex); int Legacy(const char *theOp, const char *theArg); void License(); const char *OpName(); void ProcFile(const char *fname); void Usage(int rc=0); static void toLower( char cstr[] ) { for( int i = 0; cstr[i]; ++i ) cstr[i] = tolower( cstr[i] ); } const char *PName; int Opts; int Argc; char **Argv; defVar *intDend; defVar *strDend; static const char *opLetters; static struct option opVec[]; static const int dfltSrcs = 12; XrdCpFile *pFile; XrdCpFile *pLast; XrdCpFile *pPrev; char *inFile; char **parmVal; int parmCnt; int isLcl; }; #endif xrootd-5.6.9/src/XrdApps/XrdCpFile.cc000066400000000000000000000157161457266313600174000ustar00rootroot00000000000000/******************************************************************************/ /* */ /* X r d C p F i l e . c c */ /* */ /* (c) 2012 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include #include #include #include #include #include "XrdApps/XrdCpFile.hh" #include "XrdOuc/XrdOucNSWalk.hh" /******************************************************************************/ /* S t a t i c M e m b e r s */ /******************************************************************************/ const char *XrdCpFile::mPfx = 0; /******************************************************************************/ /* C o n s t r u c t o r */ /******************************************************************************/ XrdCpFile::XrdCpFile(const char *FSpec, int &badURL) { static struct proto {const char *pHdr; int pHsz; PType pVal;} pTab[] = {{"xroot://", 8, isXroot}, {"xroots://", 9, isXroots}, {"root://", 7, isXroot}, {"roots://", 8, isXroots}, {"http://", 7, isHttp}, {"https://", 8, isHttps} }; static int pTnum = sizeof(pTab)/sizeof(struct proto); const char *Slash; int i; // Do some common initialization // Doff = 0; Dlen = 0; Next = 0; fSize = 0; badURL= 0; memset(ProtName, 0, sizeof(ProtName)); // Copy out the path and remove trailing slashes (except the last one) // Path = strdup(FSpec); i = strlen(Path); while(i) if (Path[i-1] != '/' || (i > 1 && Path[i-2] != '/')) break; else Path[--i] = 0; // Check for stdin stdout spec // if (!strcmp(Path, "-")) {Protocol = isStdIO; return; } // Dtermine protocol of the incoming spec // for (i = 0; i < pTnum; i++) {if (!strncmp(FSpec, pTab[i].pHdr, pTab[i].pHsz)) {Protocol = pTab[i].pVal; memcpy(ProtName, pTab[i].pHdr, pTab[i].pHsz-3); return; } } // See if this is a file // Protocol = isFile; if (!strncmp(Path, "file://", 7)) {char *pP = Path + 7; if (!strncmp(pP, "localhost", 9)) memmove( Path, pP + 9, strlen( pP + 9 ) + 1 ); else if (*pP == '/') memmove( Path, pP, strlen( pP ) + 1 ); else {Protocol = isOther; strcpy(ProtName, "remote"); return; } } // Set the default Doff and Dlen assuming non-recursive copy // if ((Slash = rindex(Path, '/'))) Dlen = Doff = Slash - Path + 1; } /******************************************************************************/ XrdCpFile::XrdCpFile(char *FSpec, struct stat &Stat, short doff, short dlen) : Next(0), Path(FSpec), Doff(doff), Dlen(dlen), Protocol(isFile), fSize(Stat.st_size) {strcpy(ProtName, "file");} /******************************************************************************/ /* E x t e n d */ /******************************************************************************/ int XrdCpFile::Extend(XrdCpFile **pLast, int &nFile, long long &nBytes) { XrdOucNSWalk nsObj(0, Path, 0, XrdOucNSWalk::retFile|XrdOucNSWalk::Recurse); XrdOucNSWalk::NSEnt *nP, *nnP; XrdCpFile *fP, *pP = this; int rc; short dlen, doff = strlen(Path); nsObj.setMsgOn(mPfx); while((nP = nsObj.Index(rc)) && rc == 0) {do {dlen = nP->Plen - doff; fP = new XrdCpFile(nP->Path, nP->Stat, doff, dlen); nFile++; nBytes += nP->Stat.st_size; nP->Path = 0; pP->Next = fP; pP = fP; nnP = nP->Next; delete nP; } while((nP = nnP)); } if (pLast) *pLast = pP; return rc; } /******************************************************************************/ /* R e s o l v e */ /******************************************************************************/ int XrdCpFile::Resolve() { struct stat Stat; // Ignore this call if this is not a file // if (Protocol != isFile) return 0; // This should exist but it might not, the caller will determine what to do // char *cgibeg = strchr( Path, '?' ); if( cgibeg ) *cgibeg = '\0'; if (stat(Path, &Stat)) return errno; if( cgibeg ) *cgibeg = '?'; // Find out what this really is // if (S_ISREG(Stat.st_mode)) fSize = Stat.st_size; else if (S_ISDIR(Stat.st_mode)) Protocol = isDir; else if (!strcmp(Path, "/dev/null")) Protocol = isDevNull; else if (!strcmp(Path, "/dev/zero")) Protocol = isDevZero; else return ENOTSUP; // All is well // return 0; } xrootd-5.6.9/src/XrdApps/XrdCpFile.hh000066400000000000000000000072161457266313600174060ustar00rootroot00000000000000#ifndef __XRDCPFILE_HH__ #define __XRDCPFILE_HH__ /******************************************************************************/ /* */ /* X r d C p F i l e . h h */ /* */ /* (c) 2012 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include class XrdCpFile { public: enum PType {isOther = 0, isDir, isFile, isStdIO, isXroot, isXroots, isHttp, isHttps, isDevNull, isDevZero }; XrdCpFile *Next; // -> Next file in list char *Path; // -> Absolute path to the file short Doff; // Offset to directory extension in Path short Dlen; // Length of directory extension (0 if none) // The length includes the trailing slash. PType Protocol; // Protocol type char ProtName[8]; // Protocol name long long fSize; // Size of file int Extend(XrdCpFile **pLast, int &nFile, long long &nBytes); int Resolve(); static void SetMsgPfx(const char *pfx) {mPfx = pfx;} XrdCpFile() : Next(0), Path(0), Doff(0), Dlen(0), Protocol(isOther), fSize(0) {*ProtName = 0;} XrdCpFile(const char *FSpec, int &badURL); XrdCpFile( char *FSpec, struct stat &Stat, short doff, short dlen); ~XrdCpFile() {if (Path) free(Path);} private: static const char *mPfx; }; #endif xrootd-5.6.9/src/XrdApps/XrdCrc32c.cc000066400000000000000000000137221457266313600172500ustar00rootroot00000000000000/******************************************************************************/ /* */ /* X r d C r c 3 2 . c c */ /* */ /* (c) 2021 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include #include #include #include #include #include #include #include "XrdOuc/XrdOucCRC.hh" #include "XrdSys/XrdSysE2T.hh" using namespace std; namespace {const char *pgm = "xrdcrc32c"; } #ifndef O_DIRECT #define O_DIRECT 0 #endif /******************************************************************************/ /* F a t a l */ /******************************************************************************/ void Fatal(const char *op, const char *target) { // Generate the message // std::cerr <<"xrdcrc32c: Unable to "< | -]\n" "\n the path to the file whose checksum if to be computed." "\n- compute checksum from data presented at standard in;" "\n example: xrdcp - | xrdcrc32c -\n" "\nopts: -d -h -n -s -x\n" "\n-d read data directly into the buffer, do not use the file cache." "\n-h display usage information (arguments ignored)." "\n-n do not end output with a newline character." "\n-s do not include file path in output result." "\n-x do not print leading zeroes in the checksum, if any." < 1 && '-' == *argv[1]) while ((c = getopt(argc,argv,"dhnsx")) && ((unsigned char)c != 0xff)) { switch(c) { case 'd': opts |= O_DIRECT; break; case 'h': Usage(0); break; case 'n': addNL = false; break; case 's': addPath = false; break; case 'x': fmt = "%x"; break; default: std::cerr < 0) {csVal = XrdOucCRC::Calc32C(buffP, bytes, csVal);} // Check if we ended with an error // if (bytes < 0) Fatal("read", fPath); // Produce the result // sprintf(csBuff, fmt, csVal); std::cout <<(char *)csBuff; if (addPath) std::cout << ' ' <. */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ /* This utility maps the connections in a cluster starting at some node. It can also, optionally, check for file existence at each point. Syntax: xrdmapc : [] */ /******************************************************************************/ /* i n c l u d e f i l e s */ /******************************************************************************/ #include #include #include #include #include #include #include #include #include #include "XProtocol/XProtocol.hh" #include "XrdCl/XrdClEnv.hh" #include "XrdCl/XrdClFileSystem.hh" #include "XrdCl/XrdClDefaultEnv.hh" #include "XrdNet/XrdNetAddr.hh" #include "XrdOuc/XrdOucHash.hh" #include "XrdSys/XrdSysHeaders.hh" /******************************************************************************/ /* L o c a l D e f i n i t i o n s */ /******************************************************************************/ #define EMSG(x) std::cerr <<"xrdmapc: " < clHash; }; /******************************************************************************/ /* M a k e U R L */ /******************************************************************************/ namespace { const char *MakeURL(const char *name, char *buff, int blen) { snprintf(buff, blen, "xroot://%s//", name); return buff; } }; /******************************************************************************/ /* M a p C o d e */ /******************************************************************************/ namespace { void MapCode(XrdCl::XRootDStatus &Status, clMap *node, bool nspl=false) { char buff[128]; node->verfile = '?'; if (Status.code == XrdCl::errErrorResponse) {switch(Status.errNo) {case kXR_FSError: node->state = " [fs error]"; break; case kXR_IOError: node->state = " [io error]"; break; case kXR_NoMemory: node->state = " [no memory]"; break; case kXR_NotAuthorized: node->state = " [not authorized]";break; case kXR_NotFound: if (nspl) node->state = " [no subscribers]"; else {node->state = ""; node->verfile = '-'; } break; case kXR_NotFile: node->state = " [not a file]"; break; default: sprintf(buff, " [xrootd error %d]", Status.errNo); node->state = strdup(buff); break; } return; } switch(Status.code) {case XrdCl::errInvalidAddr: node->state = " [invalid addr]"; break; case XrdCl::errSocketError: node->state = " [socket error]"; break; case XrdCl::errSocketTimeout: node->state = " [timeout]"; break; case XrdCl::errSocketDisconnected: node->state = " [disconnect]"; break; case XrdCl::errStreamDisconnect: node->state = " [disconnect]"; break; case XrdCl::errConnectionError: node->state = " [connect error]"; break; case XrdCl::errHandShakeFailed: node->state = " [handshake failed]"; break; case XrdCl::errLoginFailed: node->state = " [login failed]"; break; case XrdCl::errAuthFailed: node->state = " [auth failed]"; break; case XrdCl::errOperationExpired: node->state = " [op expired]"; break; case XrdCl::errRedirectLimit: node->state = " [redirect loop]"; break; default: if (!(*Status.ToStr().c_str())) sprintf(buff, " [client error %d]", Status.code); else snprintf(buff, sizeof(buff), " [%s]", Status.ToStr().c_str()); node->state = strdup(buff); break; } } }; /******************************************************************************/ /* M a p C l u s t e r */ /******************************************************************************/ namespace { void MapCluster(clMap *node, clMap *origin) { static XrdCl::OpenFlags::Flags flags = XrdCl::OpenFlags::None; char buff[2048]; XrdCl::URL theURL((const std::string)MakeURL(node->name,buff,sizeof(buff))); XrdCl::FileSystem xrdFS(theURL); XrdCl::XRootDStatus Status; XrdCl::LocationInfo *info = 0; XrdCl::LocationInfo::Iterator it; XrdCl::LocationInfo::LocationType locType; clMap *clmP, *branch; // Issue a locate // Status = xrdFS.Locate((const std::string)"*", flags, info, theTO); // Make sure all went well // if (!Status.IsOK()) {if (Status.errNo != kXR_NotFound && !doHush) EMSG("Unable to get " <name <<" subscribers; " <valid = 0; return; } // Grab all of the information // for( it = info->Begin(); it != info->End(); ++it ) {clmP = new clMap(it->GetAddress().c_str()); locType = it->GetType(); if (locType == XrdCl::LocationInfo::ServerOnline || locType == XrdCl::LocationInfo::ServerPending) {clmP->nextSrv = node->nextSrv; node->nextSrv = clmP; } else { clmP->nextMan = node->nextMan; node->nextMan = clmP; clmP->isMan = 1; } clHash.Add(clmP->key, clmP, 0, Hash_keep); } // Now map all managers // clmP = node->nextMan; while(clmP) {branch = new clMap(clmP->name); MapCluster(branch, clmP); // if (branch->nextSrv || branch->nextMan || node->valid) clmP->nextLvl = branch; // else delete branch; clmP = clmP->nextMan; } // All done // delete info; } }; /******************************************************************************/ /* M a p P a t h */ /******************************************************************************/ namespace { void MapPath(clMap *node, const char *Path, bool doRefresh=false) { XrdCl::OpenFlags::Flags flags = XrdCl::OpenFlags::None; char buff[2048]; XrdCl::URL theURL((const std::string)MakeURL(node->name,buff,sizeof(buff))); XrdCl::FileSystem xrdFS(theURL); XrdCl::XRootDStatus Status; XrdCl::LocationInfo *info = 0; XrdCl::LocationInfo::Iterator it; clMap *clmP; // Insert refresh is so wanted // if (doRefresh) flags = XrdCl::OpenFlags::Refresh; // Issue a locate // Status = xrdFS.Locate((const std::string)Path, flags, info, theTO); // Make sure all went well // if (!Status.IsOK()) {if (Status.errNo != kXR_NotFound && !doHush) EMSG("Unable to query " <name <<" about path; " <Begin(); it != info->End(); ++it ) {const char *clAddr = it->GetAddress().c_str(); if ((clmP = clHash.Find(clAddr))) {clmP->hasfile = '>'; if (clmP->isMan) MapPath(clmP, Path); } else { clmP = new clMap(clAddr); clmP->nextSrv = clLost; clLost = clmP; } } // All done here // delete info; } }; /******************************************************************************/ /* O p N a m e */ /******************************************************************************/ namespace { const char *OpName(char *Argv[]) { static char oName[4] = {'-', 0, 0, 0}; if (!optopt || optopt == '-' || *(Argv[optind-1]+1) == '-') return Argv[optind-1]; oName[1] = optopt; return oName; } }; /******************************************************************************/ /* P a t h C h k */ /******************************************************************************/ namespace { void PathChk(clMap *node) { char buff[2048]; XrdCl::URL theURL((const std::string)MakeURL(node->name,buff,sizeof(buff))); XrdCl::FileSystem xrdFS(theURL); XrdCl::XRootDStatus Status; XrdCl::StatInfo *info = 0; // Issue a stat for the file // Status = xrdFS.Stat((const std::string)Path, info); // Make sure all went well // if (!Status.IsOK()) MapCode(Status, node); else node->verfile = '+'; // All done here // delete info; } }; /******************************************************************************/ /* P r i n t M a p */ /******************************************************************************/ namespace { void PrintMap(clMap *clmP, int lvl) { clMap *clnow; const char *pfx = ""; char *pfxbuff = 0; int n; // Compute index spacing // if ((n = lvl*5)) {pfxbuff = (char *)malloc(n+1); memset(pfxbuff, ' ', n); pfxbuff[n] = 0; pfx = pfxbuff; } // Print all of the servers first // if (listSrv) {clnow = clmP->nextSrv; while(clnow) {if (doVerify) PathChk(clnow); if (lvl) {pfxbuff[1] = clnow->hasfile; pfxbuff[2] = clnow->verfile; } std::cout <<' ' <name <state <nextSrv; } } // Now recursively print the managers // if (listMan) {clnow = clmP->nextMan; if (lvl) pfxbuff[2] = ' '; while(clnow) {if (lvl) pfxbuff[1] = clnow->hasfile; std::cout <name <state <valid && clnow->nextLvl) PrintMap(clnow->nextLvl,lvl+1); clnow = clnow->nextMan; } } // All done // if (lvl) free(pfxbuff); } }; /******************************************************************************/ /* S e t E n v */ /******************************************************************************/ namespace { int cwValue = 10; int crValue = 0; int trValue = 5; void SetEnv() { XrdCl::Env *env = XrdCl::DefaultEnv::GetEnv(); env->PutInt("ConnectionWindow", cwValue); env->PutInt("ConnectionRetry", crValue); env->PutInt("TimeoutResolution",trValue); } }; /******************************************************************************/ /* U s a g e */ /******************************************************************************/ namespace { void Usage(const char *emsg) { if (emsg) EMSG(emsg); std::cerr <<"Usage: xrdmapc [] : []\n" <<": [--help] [--list {all|m|s}] [--quiet] [--refresh] [--verify]" < existence status at each server.\n" " when specified, uses : to determine the locations\n" " of path and does optional verification." <= argc) Usage("Initial node not specified."); // Establish starting point // if ((eMsg = sPoint.Set(argv[optind]))) {EMSG("Unable to validate initial node; " <name <state <name <<" referred to the following unconnected node:" <name <nextSrv; } } // All done // exit(0); } xrootd-5.6.9/src/XrdApps/XrdMpxStats.cc000066400000000000000000000243371457266313600200200ustar00rootroot00000000000000/******************************************************************************/ /* */ /* X r d M p x S t a t s . c c */ /* */ /* (c) 2009 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include #include #include #include "XrdApps/XrdMpxXml.hh" #include "XrdNet/XrdNetAddr.hh" #include "XrdNet/XrdNetOpts.hh" #include "XrdNet/XrdNetSocket.hh" #include "XrdSys/XrdSysError.hh" #include "XrdSys/XrdSysLogger.hh" #include "XrdSys/XrdSysHeaders.hh" #include "XrdSys/XrdSysPlatform.hh" #include "XrdSys/XrdSysPthread.hh" /******************************************************************************/ /* G l o b a l V a r i a b l e s */ /******************************************************************************/ namespace XrdMpx { XrdSysLogger Logger; XrdSysError Say(&Logger, "mpxstats"); static const int addSender = 0x0001; int Opts; }; using namespace XrdMpx; /******************************************************************************/ /* X r d M p x O u t */ /******************************************************************************/ class XrdMpxOut { public: struct statsBuff {statsBuff *Next; XrdNetSockAddr From; int Dlen; char Data[8190]; char Pad[2]; }; void Add(statsBuff *sbP); statsBuff *getBuff(); void *Run(XrdMpxXml *xP); XrdMpxOut() : Ready(0), inQ(0), Free(0) {} ~XrdMpxOut() {} private: XrdSysMutex myMutex; XrdSysSemaphore Ready; statsBuff *inQ; statsBuff *Free; }; /******************************************************************************/ /* X r d M p x O u t : : A d d */ /******************************************************************************/ void XrdMpxOut::Add(statsBuff *sbP) { // Add this to the queue and signal the processing thread // myMutex.Lock(); sbP->Next = inQ; inQ = sbP; Ready.Post(); myMutex.UnLock(); } /******************************************************************************/ /* X r d M p x O u t : : g e t B u f f */ /******************************************************************************/ XrdMpxOut::statsBuff *XrdMpxOut::getBuff() { statsBuff *sbP; // Use an available buffer or allocate one // myMutex.Lock(); if ((sbP = Free)) Free = sbP->Next; else sbP = new statsBuff; myMutex.UnLock(); return sbP; } /******************************************************************************/ /* X r d M p x O u t : : R u n */ /******************************************************************************/ void *XrdMpxOut::Run(XrdMpxXml *xP) { XrdNetAddr theAddr; const char *Host = 0; char *bP, obuff[sizeof(statsBuff)*2]; statsBuff *sbP; int wLen, rc; // Simply loop formating and outputing the buffers // while(1) {Ready.Wait(); myMutex.Lock(); if ((sbP = inQ)) inQ = sbP->Next; myMutex.UnLock(); if (!sbP) continue; if (xP) {if (!(Opts & addSender)) Host = 0; else if (theAddr.Set(&(sbP->From.Addr))) Host = 0; else Host = theAddr.Name(); wLen = xP->Format(Host, sbP->Data, obuff); bP = obuff; } else { bP = sbP->Data; *(bP + sbP->Dlen) = '\n'; wLen = sbP->Dlen+1; } while(wLen > 0) {do {rc = write(STDOUT_FILENO, bP, wLen);} while(rc < 0 && errno == EINTR); wLen -= rc; bP += rc; } myMutex.Lock(); sbP->Next = Free; Free = sbP; myMutex.UnLock(); } // Should never get here // return (void *)0; } /******************************************************************************/ /* G l o b a l O b j e c t s */ /******************************************************************************/ namespace XrdMpx { XrdMpxOut statsQ; }; /******************************************************************************/ /* T h r e a d I n t e r f a c e s */ /******************************************************************************/ void *mainOutput(void *parg) { XrdMpxXml *xP = static_cast(parg); return statsQ.Run(xP); } /******************************************************************************/ /* U s a g e */ /******************************************************************************/ void Usage(int rc) { std::cerr <<"\nUsage: mpxstats [-f {cgi|flat|xml}] -p [-s]" < 1 && '-' == *argv[1]) while ((c = getopt(argc,argv,"df:p:s")) && ((unsigned char)c != 0xff)) { switch(c) { case 'd': Debug = true; break; case 'f': if (!strcmp(optarg, "cgi" )) fType = XrdMpxXml::fmtCGI; else if (!strcmp(optarg, "flat")) fType = XrdMpxXml::fmtFlat; else if (!strcmp(optarg, "xml" )) fType = XrdMpxXml::fmtXML; else {Say.Emsg(":", "Invalid format - ", optarg); Usage(1);} break; case 'h': Usage(0); break; case 'p': if (!(Port = atoi(optarg))) {Say.Emsg(":", "Invalid port number - ", optarg); Usage(1);} break; case 's': Opts |= addSender; break; default: sprintf(buff,"'%c'", optopt); if (c == ':') Say.Emsg(":", buff, "value not specified."); else Say.Emsg(0, buff, "option is invalid"); Usage(1); break; } } // Make sure port has been specified // if (!Port) {Say.Emsg(":", "Port has not been specified."); Usage(1);} // Turn off sigpipe and host a variety of others before we start any threads // signal(SIGPIPE, SIG_IGN); // Solaris optimization sigemptyset(&myset); sigaddset(&myset, SIGPIPE); sigaddset(&myset, SIGCHLD); pthread_sigmask(SIG_BLOCK, &myset, NULL); // Set the default stack size here // if (sizeof(long) > 4) XrdSysThread::setStackSize((size_t)1048576); else XrdSysThread::setStackSize((size_t)786432); // Create a UDP socket and bind it to a port // if (mySocket.Open(0, Port, XRDNET_SERVER|XRDNET_UDPSOCKET, 0) < 0) {Say.Emsg(":", -mySocket.LastError(), "create udp socket"); exit(4);} udpFD = mySocket.Detach(); // Establish format // if (fType != XrdMpxXml::fmtXML) xP = new XrdMpxXml(fType, Debug); // Now run a thread to output whatever we get // if ((retc = XrdSysThread::Run(&tid, mainOutput, (void *)xP, XRDSYSTHREAD_BIND, "Output"))) {Say.Emsg(":", retc, "create output thread"); exit(4);} // Now simply wait for the messages // fromLen = sizeof(sbP->From); while(1) {sbP = statsQ.getBuff(); retc = recvfrom(udpFD, sbP->Data, sizeof(sbP->Data), 0, &sbP->From.Addr, &fromLen); if (retc < 0) {Say.Emsg(":", retc, "recv udp message"); exit(8);} sbP->Dlen = retc; statsQ.Add(sbP); } // Should never get here // return 0; } xrootd-5.6.9/src/XrdApps/XrdMpxXml.cc000066400000000000000000000366511457266313600174640ustar00rootroot00000000000000/******************************************************************************/ /* */ /* X r d M p x X m l . c c */ /* */ /* (c) 2018 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include #include #include #include #include #include #include #include "XrdApps/XrdMpxXml.hh" #include "XrdOuc/XrdOucTokenizer.hh" using namespace std; /******************************************************************************/ /* v n M a p D e f i n i t i o n */ /******************************************************************************/ namespace { struct vCmp {bool operator()(const char *a, const char *b) const {return strcmp(a,b) < 0;} }; std::map vnMap { {"src", "Server location:"}, {"tod", "~Statistics:"}, {"tos", "~Server started: "}, {"pgm", "Server program: "}, {"ins", "Server instance:"}, {"pid", "Server process: "}, {"site", "Server sitename: "}, {"ver", "Server version: "}, {"info.host", "Host name:"}, {"info.port", "Port:"}, {"info.name", "Instance name:"}, {"buff.reqs", "Buffer requests:"}, {"buff.mem", "Buffer bytes:"}, {"buff.buffs", "Buffer count:"}, {"buff.adj", "Buffer adjustments:"}, {"buff.xlreqs", "Buffer XL requests:"}, {"buff.xlmem", "Buffer XL bytes:"}, {"buff.xlbuffs", "Buffer XL count:"}, {"link.num", "Current connections:"}, {"link.maxn", "Maximum connections:"}, {"link.tot", "Overall connections:"}, {"link.in", "Bytes received:"}, {"link.out", "Bytes sent:"}, {"link.ctime", "Total connect seconds:"}, {"link.tmo", "Read request timeouts:"}, {"link.stall", "Number of partial reads:"}, {"link.sfps", "Number of partial sends:"}, {"poll.att", "Poll sockets:"}, {"poll.en", "Poll enables:"}, {"poll.ev", "Poll events: "}, {"poll.int", "Poll events unsolicited:"}, {"proc.usr.s", "Seconds user time:"}, {"proc.usr.u", "Micros user time:"}, {"proc.sys.s", "Seconds sys time:"}, {"proc.sys.u", "Micros sys time:"}, {"xrootd.num", "XRootD protocol loads:"}, {"xrootd.ops.open", "XRootD opens:"}, {"xrootd.ops.rf", "XRootD cache refreshes:"}, {"xrootd.ops.rd", "XRootD reads:"}, {"xrootd.ops.pr", "XRootD preads:"}, {"xrootd.ops.rv", "XRootD readv's:"}, {"xrootd.ops.rs", "XRootD readv segments:"}, {"xrootd.ops.wr", "XRootD writes:"}, {"xrootd.ops.sync", "XRootD syncs:"}, {"xrootd.ops.getf", "XRootD getfiles:"}, {"xrootd.ops.putf", "XRootD putfiles:"}, {"xrootd.ops.misc", "XRootD misc requests:"}, {"xrootd.sig.ok", "XRootD ok signatures:"}, {"xrootd.sig.bad", "XRootD bad signatures:"}, {"xrootd.sig.ign", "XRootD ign signatures:"}, {"xrootd.aio.num", "XRootD aio requests:"}, {"xrootd.aio.max", "XRootD aio max requests:"}, {"xrootd.aio.rej", "XRootD aio rejections:"}, {"xrootd.err", "XRootD request failures:"}, {"xrootd.rdr", "XRootD request redirects:"}, {"xrootd.dly", "XRootD request delays:"}, {"xrootd.lgn.num", "XRootD login total count:"}, {"xrootd.lgn.af", "XRootD login auths bad: "}, {"xrootd.lgn.au", "XRootD login auths good: "}, {"xrootd.lgn.ua", "XRootD login auths none: "}, {"ofs.role", "Server role:"}, {"ofs.opr", "Ofs reads:"}, {"ofs.opw", "Ofs writes:"}, {"ofs.opp", "POSC files now open:"}, {"ofs.ups", "POSC files deleted:"}, {"ofs.han", "Ofs handles:"}, {"ofs.rdr", "Ofs redirects:"}, {"ofs.bxq", "Ofs background tasks:"}, {"ofs.rep", "Ofs callbacks:"}, {"ofs.err", "Ofs errors:"}, {"ofs.dly", "Ofs delays:"}, {"ofs.sok", "Ofs ok events:"}, {"ofs.ser", "Ofs bad events:"}, {"ofs.tpc.grnt", "TPC grants:"}, {"ofs.tpc.deny", "TPC denials:"}, {"ofs.tpc.err", "TPC errors:"}, {"ofs.tpc.exp", "TPC expires:"}, {"oss.paths", "Oss exports:"}, {"oss.space", "Oss space:"}, {"sched.jobs", "Tasks scheduled: "}, {"sched.inq", "Tasks now queued:"}, {"sched.maxinq", "Max tasks queued:"}, {"sched.threads", "Threads in pool:"}, {"sched.idle", "Threads idling: "}, {"sched.tcr", "Threads created:"}, {"sched.tde", "Threads deleted:"}, {"sched.tlimr", "Threads unavail:"}, {"sgen.as", "Unsynchronized stats:"}, {"sgen.et", "Mills to collect stats:"}, {"sgen.toe", "~Time when stats collected:"}, {"ssi.err", "SSI errors:"}, {"ssi.req.bytes", "Request total bytes:"}, {"ssi.req.maxsz", "Request largest size:"}, {"ssi.req.ab", "Request aborts:"}, {"ssi.req.al", "Request alerts:"}, {"ssi.req.bnd", "Requests now bound:"}, {"ssi.req.can", "Requests cancelled:"}, {"ssi.req.cnt", "Request total count:"}, {"ssi.req.fin", "Requests finished:"}, {"ssi.req.finf", "Requests forced off:"}, {"ssi.req.gets", "Request retrieved:"}, {"ssi.req.perr", "Request prep errors:"}, {"ssi.req.proc", "Requests started:"}, {"ssi.req.rdr", "Requests redirected:"}, {"ssi.req.relb", "Request buff releases:"}, {"ssi.req.dly", "Requests delayed:"}, {"ssi.rsp.bad", "Response violations:"}, {"ssi.rsp.cbk", "Response callbacks:"}, {"ssi.rsp.data", "Responses as data:"}, {"ssi.rsp.errs", "Responses as errors:"}, {"ssi.rsp.file", "Responses as files:"}, {"ssi.rsp.rdy", "Responses without delay:"}, {"ssi.rsp.str", "Responses as streams:"}, {"ssi.rsp.unr", "Responses with delay:"}, {"ssi.rsp.mdb", "Response metadata bytes:"}, {"ssi.res.add", "Resources added:"}, {"ssi.res.rem", "Resources removed:"} }; } /******************************************************************************/ /* L o c a l C l a s s e s */ /******************************************************************************/ /******************************************************************************/ /* X r d M p x V a r */ /******************************************************************************/ class XrdMpxVar { public: int Pop(const char *vName); int Push(const char *vName); void Reset() {vEnd = vBuff; vNum = -1; *vBuff = 0;} const char *Var() {return vBuff;} XrdMpxVar(bool dbg=false) : vFence(vBuff + sizeof(vBuff) - 1), Debug(dbg) {Reset();} ~XrdMpxVar() {} private: static const int vMax = 15; char *vEnd, *vFence, *vStack[vMax+1], vBuff[1024]; int vNum; bool Debug; }; /******************************************************************************/ /* X r d M p x V a r : : P o p */ /******************************************************************************/ int XrdMpxVar::Pop(const char *vName) { if (Debug) std::cerr <<"Pop: " <<(vName ? vName : "") <<"; var=" <' // if (!(lP = (char *)index(lP, '>'))) return xmlErr("Invalid xml stream: ", ibuff); *lP++ = '\n'; // Now make the input tokenizable // while(*lP) {if (*lP == '>' || (*lP == '<' && *(lP+1) == '/')) *lP = ' '; lP++; } // The first token better be '' in xml stream."); getVars(Data, vTail); if (vTail[0].Data) oP = Add(oP, vTail[0].Name, vTail[0].Data); if (*(oP-1) == '&') oP--; *oP++ = '\n'; return oP - obuff; } /******************************************************************************/ /* X r d M p x X m l : : A d d */ /******************************************************************************/ char *XrdMpxXml::Add(char *Buff, const char *Var, const char *Val) { char tmBuff[256]; if (noZed && !strcmp("0", Val)) return Buff; if (doV2T) {std::map::iterator it; it = vnMap.find(Var); if (it != vnMap.end()) {Var = it->second; if (*Var == '~') {time_t tod = atoi(Val); Var++; if (tod) {struct tm *tInfo = localtime(&tod); strftime(tmBuff, sizeof(tmBuff), "%a %F %T", tInfo); Val = tmBuff; } } } } strcpy(Buff, Var); Buff += strlen(Var); *Buff++ = vSep; strcpy(Buff, Val); Buff += strlen(Val); *Buff++ = vSfx; return Buff; } /******************************************************************************/ /* */ /* X r d M p x X m l : : g e t V a r s */ /* */ /******************************************************************************/ void XrdMpxXml::getVars(XrdOucTokenizer &Data, VarInfo Var[]) { char *tVar, *tVal; int i; // Initialize the data pointers to null // i = 0; while(Var[i].Name) Var[i++].Data = 0; // Get all of the variables/values and return where possible // while((tVar = Data.GetToken()) && *tVar != '<' && *tVar != '/') {if (!(tVal = (char *)index(tVar, '='))) continue; *tVal++ = '\0'; if (*tVal == '"') {tVal++, i = strlen(tVal); if (*(tVal+i-1) == '"') *(tVal+i-1) = '\0'; } i = 0; while(Var[i].Name) {if (!strcmp(Var[i].Name, tVar)) {Var[i].Data = tVal; break;} else i++; } } if (tVar && (*tVar == '<' || *tVar == '/')) Data.RetToken(); } /******************************************************************************/ /* X r d M p x X m l : : x m l E r r */ /******************************************************************************/ int XrdMpxXml::xmlErr(const char *t1, const char *t2, const char *t3) { std::cerr <<"XrdMpxXml: " <. */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ class XrdOucTokenizer; class XrdMpxXml { public: enum fmtType {fmtCGI, fmtFlat, fmtText, fmtXML}; int Format(const char *Host, char *ibuff, char *obuff); XrdMpxXml(fmtType ft, bool nz=false, bool dbg=false) : fType(ft), Debug(dbg), noZed(nz) {if (ft == fmtCGI) {vSep = '='; vSfx = '&';} else {vSep = ' '; vSfx = '\n';} doV2T = ft == fmtText; } ~XrdMpxXml() {} private: struct VarInfo {const char *Name; char *Data; }; char *Add(char *Buff, const char *Var, const char *Val); void getVars(XrdOucTokenizer &Data, VarInfo Var[]); int xmlErr(const char *t1, const char *t2=0, const char *t3=0); fmtType fType; char vSep; char vSfx; bool Debug; bool noZed; bool doV2T; }; #endif xrootd-5.6.9/src/XrdApps/XrdPinls.cc000066400000000000000000000122531457266313600173140ustar00rootroot00000000000000/******************************************************************************/ /* */ /* X r d P i n l s . c c */ /* */ /* (c) 2009 by the Board of Trustees of the Leland Stanford, Jr., University */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ /* This utility prints plugin version requirements. Syntax: xrdpinls */ /******************************************************************************/ /* i n c l u d e f i l e s */ /******************************************************************************/ #include #include #include #include #include #include "XrdVersionPlugin.hh" /******************************************************************************/ /* L o c a l O b j e c t s */ /******************************************************************************/ namespace { struct cmp_str { bool operator()(char const *a, char const *b) const { return strcmp(a, b) < 0; } }; } /******************************************************************************/ /* D i s p l a y */ /******************************************************************************/ void Display(const char *drctv, XrdVersionPlugin *vP) { const char *vType = "Unknown"; char buff[80]; // First determine what kind of rule this is // if (vP->vProcess == XrdVERSIONPLUGIN_DoNotChk) vType = "Untested"; else if (vP->vProcess == XrdVERSIONPLUGIN_Optional) vType = "Optional"; else if (vP->vProcess == XrdVERSIONPLUGIN_Required) vType = "Required"; // Establish minimum version // if (vP->vMinLow < 0) snprintf(buff, sizeof(buff), "%2d.x ", vP->vMajLow); else snprintf(buff, sizeof(buff), "%2d.%-2d", vP->vMajLow, vP->vMinLow); // Output the line // std::cout <= "< vRules; std::map dRules; std::map::iterator itD, itV; int i; // Map all of plugin rules by plugin object creator // i = 0; while(vInfo[i].pName) {vRules[vInfo[i].pName] = &vInfo[i]; i++; } // Now for each directive, find the matching rule // i = 0; while(dInfo[i].dName) {itV = vRules.find(dInfo[i].pName); dRules[dInfo[i].dName] = (itV != dRules.end() ? itV->second : 0); i++; } // Now display the results // for (itD = dRules.begin(); itD != dRules.end(); itD++) {if (itD->second) Display(itD->first, itD->second); else std::cout <<"No version rule present for " <first <. */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include #include #include #include #include #include #include #include #include #include "XrdOuc/XrdOucEnv.hh" #include "XrdSys/XrdSysE2T.hh" #include "XrdSys/XrdSysPlatform.hh" #include "XrdCl/XrdClFileSystem.hh" using namespace XrdCl; /******************************************************************************/ /* L o c a l D e f i n e s */ /******************************************************************************/ #define EMSG(x) std::cerr <<"xrdprep: "<= 0 && *val > maxv) {EMSG(emsg<<" may not be greater than "< fList; FILE *Stream = 0; const char *msgArgs[] = {"execute", "prepare"}; char Target[512]; char *inFile = 0; PrepareFlags::Flags Opts = PrepareFlags::None; int rc, Prty = 0, Debug = 0; char c, lastOpt = 0; bool isQuery = false, needHandle = false; // See simple help is needed // if (argc <= 1) Usage(0); // Process the options // opterr = 0; if (argc > 1 && '-' == *argv[1]) while ((c = getopt(argc,argv,"d:Ef:p:sStw")) && ((unsigned char)c != 0xff)) { switch(c) { case 'd': if (!GetNum("debug level", optarg, &Debug, 0, 5)) exit(1); break; case 'E': lastOpt = c; Opts |= PrepareFlags::Evict; break; case 'f': inFile = optarg; break; case 'p': lastOpt = c; if (!GetNum("priority", optarg, &Prty, 0, 3)) exit(1); break; case 's': lastOpt = c; Opts |= PrepareFlags::Stage; break; case 'S': lastOpt = c; Opts |=(PrepareFlags::Stage|PrepareFlags::Colocate); break; case 't': lastOpt = c; Opts |= PrepareFlags::Fresh; break; case 'w': lastOpt = c; Opts |= PrepareFlags::WriteMode; break; default: EMSG("Invalid option '-"<= argc || !isalnum(*argv[optind])) {EMSG("target host name not specified"); Usage(1); } // Grab the host name or address // strcpy(Target, "root://"); strcat(Target, argv[optind]); optind++; // If we need a handle then make sure we have one // if (needHandle) {if (optind >= argc || *argv[optind] == '/') {EMSG(msgArgs[0]<<" prepare request handle not specified"); Usage(1); } } // Pre-process any command line paths at this point // std::string strArg; int totArgLen = 0; for (int i = optind; i < argc; i++) {strArg = argv[i]; totArgLen += strArg.size() + 1; fList.push_back(strArg); } // If an infile was specified, make sure we can open it // if (inFile) {if (!(Stream = fopen(inFile, "r"))) {EMSG("Unable to open "< 0) {const char *dbg[] = {"Info","Warning","Error","Debug","Dump"}; if (Debug > 5) Debug = 5; XrdOucEnv::Export("XRD_LOGLEVEL", dbg[Debug-1]); } // Get an instance of the file system // FileSystem Admin(Target); // Issue the relevant operation // Buffer *response = 0; XRootDStatus st; if (!isQuery) st = Admin.Prepare(fList, Opts, uint8_t(Prty), response); else {Buffer qryArgs(totArgLen); char *bP = qryArgs.GetBuffer(); for (int i = 0; i < (int)fList.size(); i++) {strcpy(bP, fList[i].c_str()); bP += fList[i].size(); *bP++ = '\n'; } *(bP-1) = 0; st = Admin.Query(QueryCode::Prepare, qryArgs, response); } // Check if all went well // if (!st.IsOK()) {std::string estr = st.ToStr(); const char *einfo, *etxt = estr.c_str(); if (!(einfo = rindex(etxt, ']'))) einfo = etxt; else {einfo++; while(*einfo && *einfo == ' ') einfo++; } EMSG("Unable to "<ToString(); const char *xx = rstr.c_str(); if (*xx) std::cout << xx << std::endl; delete response; // All done // exit(0); } xrootd-5.6.9/src/XrdApps/XrdQStats.cc000066400000000000000000000204521457266313600174460ustar00rootroot00000000000000/******************************************************************************/ /* */ /* X r d Q S t a t s . c c */ /* */ /* (c) 2018 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include #include #include #include #include "XrdApps/XrdMpxXml.hh" #include "XrdCl/XrdClBuffer.hh" #include "XrdCl/XrdClFileSystem.hh" #include "XrdCl/XrdClURL.hh" #include "XrdCl/XrdClXRootDResponses.hh" using namespace std; /******************************************************************************/ /* F a t a l */ /******************************************************************************/ void Fatal(const XrdCl::XRootDStatus &Status) { std::string eText; // If this is an xrootd error then get the xrootd generated error // eText = (Status.code == XrdCl::errErrorResponse ? eText = Status.GetErrorMessage() : Status.ToStr()); // Blither and exit // std::cerr <<"xrdqstats: Unable to obtain statistic - " <[:]\n" "\nopts: -f {cgi|flat|xml} -h -i -n -s what -z\n" "\n-f specify display format (default is wordy text format)." "\n-i number of seconds to wait before between redisplays, default 10." "\n-n number of redisplays; if -s > 0 and -n unspecified goes forever." "\n-z does not display items with a zero value (wordy text only).\n" "\nwhat: one or more of the following letters to select statistics:" "\na - All (default) b - Buffer usage d - Device polling" "\ni - Identification c - Connections p - Protocols" "\ns - Scheduling u - Usage data z - Synchronized info" < 1 && '-' == *argv[1]) while ((c = getopt(argc,argv,valOpts)) && ((unsigned char)c != 0xff)) { switch(c) { case 'd': Debug = true; break; case 'f': if (!strcmp(optarg, "cgi" )) fType = XrdMpxXml::fmtCGI; else if (!strcmp(optarg, "flat")) fType = XrdMpxXml::fmtFlat; else if (!strcmp(optarg, "xml" )) fType = XrdMpxXml::fmtXML; else {std::cerr <= argc) {std::cerr <GetBuffer() <Format(0, theStats->GetBuffer(), obuff); char *bP = obuff; while(wLen > 0) {do {rc = write(STDOUT_FILENO, bP, wLen);} while(rc < 0 && errno == EINTR); wLen -= rc; bP += rc; } } delete theStats; if (WTime) sleep(WTime); if (Count) std::cout <<"\n"; } // All done // return 0; } xrootd-5.6.9/src/XrdApps/XrdWait41.cc000066400000000000000000000243231457266313600173010ustar00rootroot00000000000000/******************************************************************************/ /* */ /* X r d W a i t 4 1 . c c */ /* */ /* (c) 2009 by the Board of Trustees of the Leland Stanford, Jr., University */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ /* This unitily waits for the first of n file locks. The syntax is: wait41 [ [. . .]] */ /******************************************************************************/ /* i n c l u d e f i l e s */ /******************************************************************************/ #include #include #include #include #include #include #include #include #include #include #include #include #include "XrdOuc/XrdOucTList.hh" #include "XrdSys/XrdSysE2T.hh" #include "XrdSys/XrdSysHeaders.hh" #include "XrdSys/XrdSysPlatform.hh" #include "XrdSys/XrdSysPthread.hh" /******************************************************************************/ /* L o c a l C l a s s e s */ /******************************************************************************/ class XrdW41Gate { public: static void Serialize(XrdOucTList *gfP, int Wait=1); static int Wait41(XrdOucTList *fP); XrdW41Gate() {} ~XrdW41Gate() {} private: static XrdSysMutex gateMutex; static XrdSysSemaphore gateSem; static int gateOpen; }; XrdSysMutex XrdW41Gate::gateMutex; XrdSysSemaphore XrdW41Gate::gateSem(0); int XrdW41Gate::gateOpen = 0; class XrdW41Dirs { public: static XrdOucTList *Expand(const char *Path, XrdOucTList *ptl); }; /******************************************************************************/ /* E x t e r n a l T h r e a d I n t e r f a c e s */ /******************************************************************************/ namespace XrdWait41 { void *GateWait(void *parg) { XrdOucTList *fP = (XrdOucTList *)parg; // Serialize // XrdW41Gate::Serialize(fP); return (void *)0; } } using namespace XrdWait41; /******************************************************************************/ /* m a i n */ /******************************************************************************/ int main(int argc, char *argv[]) { sigset_t myset; XrdOucTList *gateFiles = 0; struct stat Stat; const char *eText; char buff[8]; int i; // Turn off sigpipe and host a variety of others before we start any threads // signal(SIGPIPE, SIG_IGN); // Solaris optimization sigemptyset(&myset); sigaddset(&myset, SIGPIPE); sigaddset(&myset, SIGCHLD); pthread_sigmask(SIG_BLOCK, &myset, NULL); // Set the default stack size here // if (sizeof(long) > 4) XrdSysThread::setStackSize((size_t)1048576); else XrdSysThread::setStackSize((size_t)786432); // Construct a list of files. For each directory, expand that to a list // for (i = 1; i < argc; i++) {if (stat(argv[i], &Stat)) {eText = XrdSysE2T(errno); std::cerr <<"wait41: " <d_name, ".") || !strcmp(dp->d_name, "..")) continue; strcpy(sfxDir, dp->d_name); if (stat(buff, &Stat)) {eText = XrdSysE2T(errno); std::cerr <<"wait41: " <val, Act, &lock_args);} while(rc == -1 && errno == EINTR); // Determine result // if (rc != -1) rc = 0; else {rc = errno; std::cerr <<"Serialize: " <text <val); else gateOpen = 1; gateSem.Post(); gateMutex.UnLock(); } /******************************************************************************/ /* W a i t 4 1 */ /******************************************************************************/ int XrdW41Gate::Wait41(XrdOucTList *gfP) { static const int AMode = S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH; pthread_t tid; const char *eTxt; int rc, Num = 0; // Run through the chain of files setting up a wait. We try to do a fast // redispatch in case we get a lock early. // while(gfP) {if (Num) {gateMutex.Lock(); if (gateOpen) {gateMutex.UnLock(); return 1;} gateMutex.UnLock(); } if ((gfP->val = open(gfP->text, O_CREAT|O_RDWR, AMode)) < 0) {eTxt = XrdSysE2T(errno); std::cerr <<"Wait41: " <text <text <val); } else Num++; gfP = gfP->next; } // At this point we will have to wait for the lock if we have any threads // while(Num--) {gateSem.Wait(); gateMutex.Lock(); if (gateOpen) {gateMutex.UnLock(); return 1;} gateMutex.UnLock(); } // No such luck, every thread failed // return 0; } xrootd-5.6.9/src/XrdApps/Xrdadler32.cc000066400000000000000000000221231457266313600174600ustar00rootroot00000000000000/******************************************************************************/ /* */ /* X r d a d l e r 3 2 . c c */ /* */ /* (c) 2009 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ /* Produced by Wei Yang for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ /************************************************************************/ /* Calculating Adler32 checksum of a local unix file (including stdin) */ /* and file on a remote xrootd data server. Support using XROOTD_VMP. */ /************************************************************************/ #define _FILE_OFFSET_BITS 64 #include #include #include #include #include #include #include #include #if defined(__linux__) || defined(__GNU__) || (defined(__FreeBSD_kernel__) && defined(__GLIBC__)) #include #endif #include #include "XrdPosix/XrdPosixXrootd.hh" #include "XrdPosix/XrdPosixXrootdPath.hh" #include "XrdOuc/XrdOucString.hh" #include "XrdCks/XrdCksXAttr.hh" #include "XrdOuc/XrdOucXAttr.hh" void fSetXattrAdler32(const char *path, int fd, const char* attr, char *value) { XrdOucXAttr xCS; struct stat st; if (fstat(fd, &st) || strlen(value) != 8) return; if (!xCS.Attr.Cks.Set("adler32") || !xCS.Attr.Cks.Set(value,8)) return; xCS.Attr.Cks.fmTime = static_cast(st.st_mtime); xCS.Attr.Cks.csTime = static_cast(time(0) - st.st_mtime); xCS.Set("", fd); // Remove any old attribute at this point // #if defined(__linux__) || defined(__GNU__) || (defined(__FreeBSD_kernel__) && defined(__GLIBC__)) fremovexattr(fd, attr); #elif defined(__solaris__) int attrfd; attrfd = openat(fd, attr, O_XATTR|O_RDONLY); if (attrfd >= 0) {unlinkat(attrfd, attr, 0); close(attrfd);} #endif } int fGetXattrAdler32(int fd, const char* attr, char *value) { struct stat st; char mtime[12], attr_val[25], *p; int rc; if (fstat(fd, &st)) return 0; sprintf(mtime, "%ld", st.st_mtime); #if defined(__linux__) || defined(__GNU__) || (defined(__FreeBSD_kernel__) && defined(__GLIBC__)) rc = fgetxattr(fd, attr, attr_val, 25); #elif defined(__solaris__) int attrfd; attrfd = openat(fd, attr, O_XATTR|O_RDONLY); if (attrfd < 0) return(0); rc = read(attrfd, attr_val, 25); close(attrfd); #else return(0); #endif if (rc == -1 || attr_val[8] != ':') return(0); attr_val[8] = '\0'; attr_val[rc] = '\0'; p = attr_val + 9; if (strcmp(p, mtime)) return(0); strcpy(value, attr_val); return(strlen(value)); } int fGetXattrAdler32(const char *path, int fd, const char* attr, char *value) { XrdOucXAttr xCS; struct stat st; if (!xCS.Attr.Cks.Set("adler32") || xCS.Get(path, fd) <= 0 || strcmp(xCS.Attr.Cks.Name, "adler32")) {int rc = fGetXattrAdler32(fd, attr, value); if (rc == 8) fSetXattrAdler32(path, fd, attr, value); return rc; } if (fstat(fd, &st) || xCS.Attr.Cks.fmTime != static_cast(st.st_mtime)) return 0; xCS.Attr.Cks.Get(value, 9); return 8; } /* the rooturl should point to the data server, not redirector */ char getchksum(const char *rooturl, char *chksum) { char csBuff[256]; int csLen; // Obtain the checksum (this is the default checksum) // csLen = XrdPosixXrootd::Getxattr(rooturl, "xroot.cksum", csBuff, sizeof(csBuff)); if (csLen < 0) return -1; if (csLen == 0) return 0; // Server doesn't have the checksum // Verify that the checksum returned is "adler32" // if (strncmp("adler32 ", csBuff, 8)) return 0; // Return the checksum value (this is really bad code) // strcpy(chksum, csBuff+8); return strlen(csBuff+8); } #define N 1024*1024 /* reading block size */ int main(int argc, char *argv[]) { char path[2048], chksum[128], buf[N], adler_str[9]; const char attr[] = "user.checksum.adler32"; struct stat stbuf; int fd, len, rc; uLong adler; adler = adler32(0L, Z_NULL, 0); if (argc == 2 && ! strcmp(argv[1], "-h")) { printf("Usage: %s file. Calculating adler32 checksum of a given file.\n", argv[0]); printf("A file can be local file, stdin (if omitted), or root URL (including via XROOTD_VMP)\n"); return 0; } path[0] = '\0'; if (argc > 1) /* trying to convert to root URL */ { if (!strncmp(argv[1], "root://", 7)) strcpy(path, argv[1]); else {XrdPosixXrootPath xrdPath; xrdPath.URL(argv[1], path, sizeof(path)); } } if (argc == 1 || path[0] == '\0') { /* this is a local file */ if (argc > 1) { strcpy(path, argv[1]); rc = stat(path, &stbuf); if (rc != 0 || ! S_ISREG(stbuf.st_mode) || (fd = open(path,O_RDONLY)) < 0) { printf("Error_accessing %s\n", path); return 1; } else /* see if the adler32 is saved in attribute already */ if (fGetXattrAdler32(path, fd, attr, adler_str) == 8) { printf("%s %s\n", adler_str, path); return 0; } } else { fd = STDIN_FILENO; strcpy(path, "-"); } while ( (len = read(fd, buf, N)) > 0 ) adler = adler32(adler, (const Bytef*)buf, len); if (fd != STDIN_FILENO) { /* try saving adler32 to attribute before close() */ sprintf(adler_str, "%08lx", adler); fSetXattrAdler32(path, fd, attr, adler_str); close(fd); } printf("%08lx %s\n", adler, path); return 0; } else { /* this is a Xrootd file */ if (getchksum(path, chksum) > 0) { /* server implements checksum */ printf("%s %s\n", chksum, argv[1]); return (strcmp(chksum, "Error_accessing:") ? 0 : 1); } else { /* need to read the file and calculate */ XrdPosixXrootd myPFS(-8, 8, 1); rc = XrdPosixXrootd::Stat(path, &stbuf); if (rc != 0 || ! S_ISREG(stbuf.st_mode) || (fd = XrdPosixXrootd::Open(path, O_RDONLY, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH)) < 0) { printf("Error_accessing: %s\n", argv[1]); return 1; } //!!! TO DO: Remove testing of totbytes once XrdClEC read regression is fixed off_t totbytes = 0; while ( totbytes < stbuf.st_size && (len = XrdPosixXrootd::Read(fd, buf, N)) > 0 ) { adler = adler32(adler, (const Bytef*)buf, (len < (stbuf.st_size - totbytes)? len : stbuf.st_size - totbytes )); totbytes += len; } XrdPosixXrootd::Close(fd); printf("%08lx %s\n", adler, argv[1]); return 0; } } } xrootd-5.6.9/src/XrdBwm/000077500000000000000000000000001457266313600150615ustar00rootroot00000000000000xrootd-5.6.9/src/XrdBwm/XrdBwm.cc000066400000000000000000001124001457266313600165710ustar00rootroot00000000000000/******************************************************************************/ /* */ /* X r d B w m . c c */ /* */ /* (c) 2008 by the Board of Trustees of the Leland Stanford, Jr., University */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Deprtment of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include #include #include #include #include #include #include #include "XrdVersion.hh" #include "XrdBwm/XrdBwm.hh" #include "XrdBwm/XrdBwmTrace.hh" #include "XrdAcc/XrdAccAuthorize.hh" #include "XrdNet/XrdNetAddr.hh" #include "XrdOuc/XrdOucEnv.hh" #include "XrdOuc/XrdOucUtils.hh" #include "XrdOuc/XrdOucTrace.hh" #include "XrdSec/XrdSecEntity.hh" #include "XrdSfs/XrdSfsAio.hh" #include "XrdSfs/XrdSfsInterface.hh" #include "XrdSys/XrdSysError.hh" #include "XrdSys/XrdSysHeaders.hh" #include "XrdSys/XrdSysLogger.hh" #include "XrdSys/XrdSysPlatform.hh" #include "XrdSys/XrdSysPthread.hh" /******************************************************************************/ /* E r r o r R o u t i n g O b j e c t */ /******************************************************************************/ XrdSysError BwmEroute(0); XrdOucTrace BwmTrace(&BwmEroute); /******************************************************************************/ /* S t a t i c O b j e c t s */ /******************************************************************************/ XrdBwmHandle *XrdBwm::dummyHandle; /******************************************************************************/ /* F i l e S y s t e m O b j e c t */ /******************************************************************************/ XrdVERSIONINFO(XrdSfsGetFileSystem,XrdBwm); XrdBwm XrdBwmFS; /******************************************************************************/ /* X r d B w m C o n s t r u c t o r */ /******************************************************************************/ XrdBwm::XrdBwm() { XrdNetAddr myAddr(0); char buff[256], *bp; int myPort, i; // Establish defaults // Authorization = 0; Authorize = 0; AuthLib = 0; AuthParm = 0; Logger = 0; PolLib = 0; PolParm = 0; PolSlotsIn = 1; PolSlotsOut = 1; // Obtain port number we will be using // myPort = (bp = getenv("XRDPORT")) ? strtol(bp, (char **)NULL, 10) : 0; // Establish our hostname and address // myAddr.Port(myPort); HostName = strdup(myAddr.Name("*unknown*")); myAddr.Format(buff, sizeof(buff), XrdNetAddr::fmtAdv6, XrdNetAddr::old6Map4); locResp = strdup(buff); locRlen = strlen(buff); for (i = 0; HostName[i] && HostName[i] != '.'; i++) {} HostName[i] = '\0'; HostPref = strdup(HostName); HostName[i] = '.'; myDomain = &HostName[i+1]; myDomLen = strlen(myDomain); myVersion = &XrdVERSIONINFOVAR(XrdSfsGetFileSystem); // Set the configuration file name abd dummy handle // ConfigFN = 0; dummyHandle = XrdBwmHandle::Alloc("*", "/", "?", "?", 0); } /******************************************************************************/ /* X r d B w m F i l e C o n s t r u c t o r */ /******************************************************************************/ XrdBwmFile::XrdBwmFile(const char *user, int monid) : XrdSfsFile(user, monid) { oh = XrdBwm::dummyHandle; tident = (user ? user : ""); } /******************************************************************************/ /* G e t F i l e S y s t e m */ /******************************************************************************/ extern "C" { XrdSfsFileSystem *XrdSfsGetFileSystem(XrdSfsFileSystem *native_fs, XrdSysLogger *lp, const char *configfn) { // Do the herald thing // BwmEroute.SetPrefix("bwm_"); BwmEroute.logger(lp); BwmEroute.Say("Copr. 2008 Stanford University, Bwm Version " XrdVSTRING); // Initialize the subsystems // XrdBwmFS.ConfigFN = (configfn && *configfn ? strdup(configfn) : 0); if ( XrdBwmFS.Configure(BwmEroute) ) return 0; // All done, we can return the callout vector to these routines. // return &XrdBwmFS; } } /******************************************************************************/ /* */ /* D i r e c t o r y O b j e c t I n t e r f a c e s */ /* */ /******************************************************************************/ /******************************************************************************/ /* o p e n */ /******************************************************************************/ int XrdBwmDirectory::open(const char *dir_path, // In const XrdSecEntity *client, // In const char *info) // In /* Function: Open the directory `path' and prepare for reading. Input: path - The fully qualified name of the directory to open. client - Authentication credentials, if any. info - Opaque information to be used as seen fit. Output: Returns SFS_OK upon success, otherwise SFS_ERROR. Notes: 1. Currently, function not supported. */ { // Return an error // return XrdBwmFS.Emsg("opendir", error, ENOTDIR, "open directory", dir_path); } /******************************************************************************/ /* n e x t E n t r y */ /******************************************************************************/ const char *XrdBwmDirectory::nextEntry() /* Function: Read the next directory entry. Input: n/a Output: n/a */ { // Return an error // XrdBwmFS.Emsg("readdir", error, EBADF, "read directory"); return 0; } /******************************************************************************/ /* c l o s e */ /******************************************************************************/ int XrdBwmDirectory::close() /* Function: Close the directory object. Input: n/a Output: Returns SFS_OK upon success and SFS_ERROR upon failure. */ { // Return an error // XrdBwmFS.Emsg("closedir", error, EBADF, "close directory"); return SFS_ERROR; } /******************************************************************************/ /* */ /* F i l e O b j e c t I n t e r f a c e s */ /* */ /******************************************************************************/ /******************************************************************************/ /* o p e n */ /******************************************************************************/ int XrdBwmFile::open(const char *path, // In XrdSfsFileOpenMode open_mode, // In mode_t Mode, // In const XrdSecEntity *client, // In const char *info) // In /* Function: Open the file `path' in the mode indicated by `open_mode'. Input: path - The fully qualified name of the file to open. The path must start with "/_bwm_" and the lfn that will eventually be opened start at the next slash. open_mode - One of the following flag values: SFS_O_RDONLY - Open file for reading. SFS_O_WRONLY - Open file for writing. n/a SFS_O_RDWR - Open file for update n/a SFS_O_CREAT - Create the file open in RW mode n/a SFS_O_TRUNC - Trunc the file open in RW mode n/a Mode - The Posix access mode bits to be assigned to the file. These bits are ignored. client - Authentication credentials, if any. info - Opaque information: bwm.src= bwm.dst= Output: Returns SFS_OK upon success, otherwise SFS_ERROR is returned. */ { EPNAME("open"); XrdBwmHandle *hP; int incoming; const char *miss, *theUsr, *theSrc, *theDst=0, *theLfn=0, *lclNode, *rmtNode; XrdOucEnv Open_Env(info); // Trace entry // ZTRACE(calls,std::hex <Access(client, path, AOP_Update, &Open_Env)) return XrdBwmFS.Emsg("open", error, EACCES, "open", path); // Make sure that all of the relevant information is present // if (!(theSrc = Open_Env.Get("bwm.src"))) miss = "bwm.src"; else if (!(theDst = Open_Env.Get("bwm.dst"))) miss = "bwm.dst"; else if (!(theLfn = index(path+1,'/')) || !(*(theLfn+1))) miss = "lfn"; else miss = 0; if (miss) return XrdBwmFS.Emsg("open", error, miss, "open", path); theUsr = error.getErrUser(); // Determine the direction of flow // if (XrdOucUtils::endsWith(theSrc,XrdBwmFS.myDomain,XrdBwmFS.myDomLen)) {incoming = 0; lclNode = theSrc; rmtNode = theDst;} else if (XrdOucUtils::endsWith(theDst,XrdBwmFS.myDomain,XrdBwmFS.myDomLen)) {incoming = 1; lclNode = theDst; rmtNode = theSrc;} else return XrdBwmFS.Emsg("open", error, EREMOTE, "open", path); // Get a handle for this file. // if (!(hP = XrdBwmHandle::Alloc(theUsr,theLfn,lclNode,rmtNode,incoming))) return XrdBwmFS.Stall(error, 13, path); // All done // XrdBwmFS.ocMutex.Lock(); oh = hP; XrdBwmFS.ocMutex.UnLock(); return SFS_OK; } /******************************************************************************/ /* c l o s e */ /******************************************************************************/ int XrdBwmFile::close() // In /* Function: Close the file object. Input: n/a Output: Returns SFS_OK upon success and SFS_ERROR upon failure. */ { EPNAME("close"); XrdBwmHandle *hP; // Trace the call // FTRACE(calls, "close" <Name()); // Verify the handle (we briefly maintain a global lock) // XrdBwmFS.ocMutex.Lock(); if (oh == XrdBwm::dummyHandle) {XrdBwmFS.ocMutex.UnLock(); return SFS_OK;} hP = oh; oh = XrdBwm::dummyHandle; XrdBwmFS.ocMutex.UnLock(); // Now retire it and possibly return the token // hP->Retire(); // All done // return SFS_OK; } /******************************************************************************/ /* f c t l */ /******************************************************************************/ int XrdBwmFile::fctl(const int cmd, const char *args, XrdOucErrInfo &out_error) /* Function: perform request control operation. Input: cmd - The operation: SFS_FCTL_GETFD - not supported. SFS_FCTL_STATV - returns visa information args - Dependent on the cmd. out_error - Place where response goes. Output: Returns SFS_OK upon success and SFS_ERROR o/w. */ { // Make sure the file is open // if (oh == XrdBwm::dummyHandle) return XrdBwmFS.Emsg("fctl", out_error, EBADF, "fctl file"); // Scan through the fctl operations // switch(cmd) {case SFS_FCTL_GETFD: out_error.setErrInfo(-1,""); return SFS_OK; case SFS_FCTL_STATV: return oh->Activate(out_error); default: break; } // Invalid fctl // out_error.setErrInfo(EINVAL, "invalid fctl command"); return SFS_ERROR; } /******************************************************************************/ /* r e a d */ /******************************************************************************/ int XrdBwmFile::read(XrdSfsFileOffset offset, // In XrdSfsXferSize blen) // In /* Function: Preread `blen' bytes at `offset' Input: offset - The absolute byte offset at which to start the read. blen - The amount to preread. Output: Returns SFS_OK upon success and SFS_ERROR o/w. */ { EPNAME("read"); // Perform required tracing // FTRACE(calls,"preread " <Result = this->read((XrdSfsFileOffset)aiop->sfsAio.aio_offset, (char *)aiop->sfsAio.aio_buf, (XrdSfsXferSize)aiop->sfsAio.aio_nbytes); aiop->doneRead(); return 0; } /******************************************************************************/ /* w r i t e */ /******************************************************************************/ XrdSfsXferSize XrdBwmFile::write(XrdSfsFileOffset offset, // In const char *buff, // Out XrdSfsXferSize blen) // In /* Function: Write `blen' bytes at `offset' from 'buff' and return the actual number of bytes written. Input: offset - The absolute byte offset at which to start the write. buff - Address of the buffer from which to get the data. blen - The size of the buffer. This is the maximum number of bytes that will be written to 'fd'. Output: Returns the number of bytes written upon success and SFS_ERROR o/w. Notes: 1. An error return may be delayed until the next write(), close(), or sync() call. 2. Currently, we do not accept write activated commands. */ { EPNAME("write"); // Perform any required tracing // FTRACE(calls, blen <<"@" <Result = this->write((XrdSfsFileOffset)aiop->sfsAio.aio_offset, (char *)aiop->sfsAio.aio_buf, (XrdSfsXferSize)aiop->sfsAio.aio_nbytes); aiop->doneWrite(); return 0; } /******************************************************************************/ /* g e t M m a p */ /******************************************************************************/ int XrdBwmFile::getMmap(void **Addr, off_t &Size) // Out /* Function: Return memory mapping for file, if any. Output: Addr - Address of memory location Size - Size of the file or zero if not memory mapped. Returns SFS_OK upon success and SFS_ERROR upon failure. */ { // Mapping is not supported // *Addr= 0; Size = 0; return SFS_OK; } /******************************************************************************/ /* s t a t */ /******************************************************************************/ int XrdBwmFile::stat(struct stat *buf) // Out /* Function: Return file status information Input: buf - The stat structiure to hold the results Output: Returns SFS_OK upon success and SFS_ERROR upon failure. */ { EPNAME("fstat"); static unsigned int myInode = 0; union {long long Fill; int Xor[2]; XrdBwmFile *fP; dev_t Num; } theDev; // Perform any required tracing // FTRACE(calls, FName()); // Develop the device number // theDev.Fill = 0; theDev.fP = this; theDev.Xor[0] ^= theDev.Xor[1]; // Fill out the stat structure for this pseudo file // memset(buf, 0, sizeof(struct stat)); buf->st_ino = myInode++; buf->st_dev = theDev.Num; buf->st_blksize = 4096; buf->st_mode = S_IFBLK; return SFS_OK; } /******************************************************************************/ /* s y n c */ /******************************************************************************/ int XrdBwmFile::sync() // In /* Function: Commit all unwritten bytes to physical media. Input: n/a Output: Returns SFS_OK upon success and SFS_ERROR upon failure. */ { EPNAME("sync"); // Perform any required tracing // FTRACE(calls,""); // We always succeed // return SFS_OK; } /******************************************************************************/ /* s y n c A I O */ /******************************************************************************/ // For now, reverts to synchronous case // int XrdBwmFile::sync(XrdSfsAio *aiop) { aiop->Result = this->sync(); aiop->doneWrite(); return 0; } /******************************************************************************/ /* t r u n c a t e */ /******************************************************************************/ int XrdBwmFile::truncate(XrdSfsFileOffset flen) // In /* Function: Set the length of the file object to 'flen' bytes. Input: flen - The new size of the file. Output: Returns SFS_OK upon success and SFS_ERROR upon failure. Notes: 1. Truncate is not supported. */ { EPNAME("trunc"); // Lock the file handle and perform any tracing // FTRACE(calls, "len=" <Name()); } /******************************************************************************/ /* g e t C X i n f o */ /******************************************************************************/ int XrdBwmFile::getCXinfo(char cxtype[4], int &cxrsz) /* Function: Set the length of the file object to 'flen' bytes. Input: n/a Output: cxtype - Compression algorithm code cxrsz - Compression region size Returns SFS_OK upon success and SFS_ERROR upon failure. */ { // Indicate not compressed // cxrsz = 0; cxtype[0] = cxtype[1] = cxtype[2] = cxtype[3] = 0; return SFS_OK; } /******************************************************************************/ /* */ /* F i l e S y s t e m O b j e c t I n t e r f a c e s */ /* */ /******************************************************************************/ /******************************************************************************/ /* c h m o d */ /******************************************************************************/ int XrdBwm::chmod(const char *path, // In XrdSfsMode Mode, // In XrdOucErrInfo &einfo, // Out const XrdSecEntity *client, // In const char *info) // In /* Function: Change the mode on a file or directory. Input: path - Is the fully qualified name of the file to be removed. einfo - Error information object to hold error details. client - Authentication credentials, if any. info - Opaque information to be used as seen fit. Output: Returns SFS_OK upon success and SFS_ERROR upon failure. */ { // Return an error // return XrdBwmFS.Emsg("chmod", einfo, ENOTSUP, "change", path); } /******************************************************************************/ /* e x i s t s */ /******************************************************************************/ int XrdBwm::exists(const char *path, // In XrdSfsFileExistence &file_exists, // Out XrdOucErrInfo &einfo, // Out const XrdSecEntity *client, // In const char *info) // In /* Function: Determine if file 'path' actually exists. Input: path - Is the fully qualified name of the file to be tested. file_exists - Is the address of the variable to hold the status of 'path' when success is returned. The values may be: XrdSfsFileExistsIsDirectory - file not found but path is valid. XrdSfsFileExistsIsFile - file found. XrdSfsFileExistsIsNo - neither file nor directory. einfo - Error information object holding the details. client - Authentication credentials, if any. info - Opaque information to be used as seen fit. Output: Returns SFS_OK upon success and SFS_ERROR upon failure. Notes: When failure occurs, 'file_exists' is not modified. */ { file_exists=XrdSfsFileExistNo; return SFS_OK; } /******************************************************************************/ /* f s c t l */ /******************************************************************************/ int XrdBwm::fsctl(const int cmd, const char *args, XrdOucErrInfo &einfo, const XrdSecEntity *client) /* Function: Perform filesystem operations: Input: cmd - Operation command (currently supported): None. arg - Command dependent argument: - STATXV: The file handle einfo - Error/Response information structure. client - Authentication credentials, if any. Output: Returns SFS_OK upon success and SFS_ERROR upon failure. */ { // Operation is not supported // return XrdBwmFS.Emsg("fsctl", einfo, ENOTSUP, "fsctl", args); } /******************************************************************************/ /* g e t V e r s i o n */ /******************************************************************************/ const char *XrdBwm::getVersion() {return XrdVERSION;} /******************************************************************************/ /* m k d i r */ /******************************************************************************/ int XrdBwm::mkdir(const char *path, // In XrdSfsMode Mode, // In XrdOucErrInfo &einfo, // Out const XrdSecEntity *client, // In const char *info) // In /* Function: Create a directory entry. Input: path - Is the fully qualified name of the file to be removed. Mode - Is the POSIX mode value the directory is to have. Additionally, Mode may contain SFS_O_MKPTH if the full dircectory path should be created. einfo - Error information object to hold error details. client - Authentication credentials, if any. info - Opaque information to be used as seen fit. Output: Returns SFS_OK upon success and SFS_ERROR upon failure. */ { // Return an error // return XrdBwmFS.Emsg("mkdir", einfo, ENOTSUP, "mkdir", path); } /******************************************************************************/ /* p r e p a r e */ /******************************************************************************/ int XrdBwm::prepare( XrdSfsPrep &pargs, // In XrdOucErrInfo &out_error, // Out const XrdSecEntity *client) // In { return 0; } /******************************************************************************/ /* r e m o v e */ /******************************************************************************/ int XrdBwm::remove(const char type, // In const char *path, // In XrdOucErrInfo &einfo, // Out const XrdSecEntity *client, // In const char *info) // In /* Function: Delete a file from the namespace and release it's data storage. Input: type - 'f' for file and 'd' for directory. path - Is the fully qualified name of the file to be removed. einfo - Error information object to hold error details. client - Authentication credentials, if any. info - Opaque information to be used as seen fit. Output: Returns SFS_OK upon success and SFS_ERROR upon failure. */ { // Return an error // return XrdBwmFS.Emsg("remove", einfo, ENOTSUP, "remove", path); } /******************************************************************************/ /* r e n a m e */ /******************************************************************************/ int XrdBwm::rename(const char *old_name, // In const char *new_name, // In XrdOucErrInfo &einfo, //Out const XrdSecEntity *client, // In const char *infoO, // In const char *infoN) // In /* Function: Renames a file with name 'old_name' to 'new_name'. Input: old_name - Is the fully qualified name of the file to be renamed. new_name - Is the fully qualified name that the file is to have. einfo - Error information structure, if an error occurs. client - Authentication credentials, if any. infoO - old_name opaque information to be used as seen fit. infoN - new_name opaque information to be used as seen fit. Output: Returns SFS_OK upon success and SFS_ERROR upon failure. */ { // Return an error // return XrdBwmFS.Emsg("rename", einfo, ENOTSUP, "rename", old_name); } /******************************************************************************/ /* s t a t */ /******************************************************************************/ int XrdBwm::stat(const char *path, // In struct stat *buf, // Out XrdOucErrInfo &einfo, // Out const XrdSecEntity *client, // In const char *info) // In /* Function: Return file status information Input: path - The path for which status is wanted buf - The stat structure to hold the results einfo - Error information structure, if an error occurs. client - Authentication credentials, if any. info - opaque information to be used as seen fit. Output: Returns SFS_OK upon success and SFS_ERROR upon failure. */ { // Return an error // return XrdBwmFS.Emsg("stat", einfo, ENOTSUP, "locate", path); } /******************************************************************************/ int XrdBwm::stat(const char *path, // In mode_t &mode, // Out XrdOucErrInfo &einfo, // Out const XrdSecEntity *client, // In const char *info) // In /* Function: Return file status information (resident files only) Input: path - The path for which status is wanted mode - The stat mode entry (faked -- do not trust it) einfo - Error information structure, if an error occurs. client - Authentication credentials, if any. info - opaque information to be used as seen fit. Output: Always returns SFS_ERROR if a delay needs to be imposed. Otherwise, SFS_OK is returned and mode is appropriately, if inaccurately, set. If file residency cannot be determined, mode is set to -1. */ { // Return an error // return XrdBwmFS.Emsg("stat", einfo, ENOTSUP, "locate", path); } /******************************************************************************/ /* t r u n c a t e */ /******************************************************************************/ int XrdBwm::truncate(const char *path, // In XrdSfsFileOffset Size, // In XrdOucErrInfo &einfo, // Out const XrdSecEntity *client, // In const char *info) // In /* Function: Change the mode on a file or directory. Input: path - Is the fully qualified name of the file to be removed. Size - the size the file should have. einfo - Error information object to hold error details. client - Authentication credentials, if any. info - Opaque information to be used as seen fit. Output: Returns SFS_OK upon success and SFS_ERROR upon failure. */ { // Return an error // return XrdBwmFS.Emsg("truncate", einfo, ENOTSUP, "truncate", path); } /******************************************************************************/ /* E m s g */ /******************************************************************************/ int XrdBwm::Emsg(const char *pfx, // Message prefix value XrdOucErrInfo &einfo, // Place to put text & error code int ecode, // The error code const char *op, // Operation being performed const char *target) // The target (e.g., fname) { const char *etext; char buffer[MAXPATHLEN+80]; // Get the reason for the error // if (ecode < 0) ecode = -ecode; etext = BwmEroute.ec2text(ecode); // Format the error message // snprintf(buffer,sizeof(buffer),"Unable to %s %s; %s", op, target, etext); // Print it out if debugging is enabled // #ifndef NODEBUG BwmEroute.Emsg(pfx, einfo.getErrUser(), buffer); #endif // Place the error message in the error object and return // einfo.setErrInfo(ecode, buffer); return SFS_ERROR; } /******************************************************************************/ int XrdBwm::Emsg(const char *pfx, // Message prefix value XrdOucErrInfo &einfo, // Place to put text & error code const char *item, // What is missing const char *op, // Operation being performed const char *target) // The target (e.g., fname) { char buffer[MAXPATHLEN+80]; // Format the error message // snprintf(buffer,sizeof(buffer),"Unable to %s %s; %s missing", op, target, item); // Print it out if debugging is enabled // #ifndef NODEBUG BwmEroute.Emsg(pfx, einfo.getErrUser(), buffer); #endif // Place the error message in the error object and return // einfo.setErrInfo(EINVAL, buffer); return SFS_ERROR; } /******************************************************************************/ /* S t a l l */ /******************************************************************************/ int XrdBwm::Stall(XrdOucErrInfo &einfo, // Error text & code int stime, // Seconds to stall const char *path) // The path to stall on { EPNAME("Stall") #ifndef NODEBUG const char *tident = einfo.getErrUser(); #endif // Trace the stall // ZTRACE(delay, "Stall " <. */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include #include #include "XrdBwm/XrdBwmHandle.hh" #include "XrdSys/XrdSysPthread.hh" #include "XrdSfs/XrdSfsInterface.hh" class XrdOucEnv; class XrdSysError; class XrdSysLogger; class XrdOucStream; class XrdSfsAio; struct XrdVersionInfo; /******************************************************************************/ /* X r d B w m D i r e c t o r y */ /******************************************************************************/ class XrdBwmDirectory : public XrdSfsDirectory { public: int open(const char *dirName, const XrdSecEntity *client, const char *opaque = 0); const char *nextEntry(); int close(); inline void copyError(XrdOucErrInfo &einfo) {einfo = error;} const char *FName() {return "";} XrdBwmDirectory(const char *user, int monid) : XrdSfsDirectory(user, monid), tident(user ? user : "") {} virtual ~XrdBwmDirectory() {} protected: const char *tident; }; /******************************************************************************/ /* X r d B w m F i l e */ /******************************************************************************/ class XrdBwmFile : public XrdSfsFile { public: int open(const char *fileName, XrdSfsFileOpenMode openMode, mode_t createMode, const XrdSecEntity *client, const char *opaque = 0); int close(); using XrdSfsFile::fctl; int fctl(const int cmd, const char *args, XrdOucErrInfo &out_error); const char *FName() {return (oh ? oh->Name() : "?");} int getMmap(void **Addr, off_t &Size); int read(XrdSfsFileOffset fileOffset, // Preread only XrdSfsXferSize amount); XrdSfsXferSize read(XrdSfsFileOffset fileOffset, char *buffer, XrdSfsXferSize buffer_size); int read(XrdSfsAio *aioparm); XrdSfsXferSize write(XrdSfsFileOffset fileOffset, const char *buffer, XrdSfsXferSize buffer_size); int write(XrdSfsAio *aioparm); int sync(); int sync(XrdSfsAio *aiop); int stat(struct stat *buf); int truncate(XrdSfsFileOffset fileOffset); int getCXinfo(char cxtype[4], int &cxrsz); XrdBwmFile(const char *user, int monid); virtual ~XrdBwmFile() {if (oh) close();} protected: const char *tident; private: XrdBwmHandle *oh; }; /******************************************************************************/ /* C l a s s X r d B w m */ /******************************************************************************/ class XrdAccAuthorize; class XrdBwmLogger; class XrdBwmPolicy; class XrdBwm : public XrdSfsFileSystem { friend class XrdBwmDirectory; friend class XrdBwmFile; public: // Object allocation // XrdSfsDirectory *newDir(char *user=0, int monid=0) {return (XrdSfsDirectory *)new XrdBwmDirectory(user,monid);} XrdSfsFile *newFile(char *user=0, int monid=0) {return (XrdSfsFile *)new XrdBwmFile(user,monid);} // Other functions // int chmod(const char *Name, XrdSfsMode Mode, XrdOucErrInfo &out_error, const XrdSecEntity *client, const char *opaque = 0); int exists(const char *fileName, XrdSfsFileExistence &exists_flag, XrdOucErrInfo &out_error, const XrdSecEntity *client, const char *opaque = 0); int fsctl(const int cmd, const char *args, XrdOucErrInfo &out_error, const XrdSecEntity *client); int getStats(char *buff, int blen) {return 0;} const char *getVersion(); int mkdir(const char *dirName, XrdSfsMode Mode, XrdOucErrInfo &out_error, const XrdSecEntity *client, const char *opaque = 0); int prepare( XrdSfsPrep &pargs, XrdOucErrInfo &out_error, const XrdSecEntity *client = 0); int rem(const char *path, XrdOucErrInfo &out_error, const XrdSecEntity *client, const char *info = 0) {return remove('f', path, out_error, client, info);} int remdir(const char *dirName, XrdOucErrInfo &out_error, const XrdSecEntity *client, const char *info = 0) {return remove('d',dirName,out_error,client,info);} int rename(const char *oldFileName, const char *newFileName, XrdOucErrInfo &out_error, const XrdSecEntity *client, const char *infoO = 0, const char *infoN = 0); int stat(const char *Name, struct stat *buf, XrdOucErrInfo &out_error, const XrdSecEntity *client, const char *opaque = 0); int stat(const char *Name, mode_t &mode, XrdOucErrInfo &out_error, const XrdSecEntity *client, const char *opaque = 0); int truncate(const char *Name, XrdSfsFileOffset fileOffset, XrdOucErrInfo &out_error, const XrdSecEntity *client = 0, const char *opaque = 0); // Management functions // virtual int Configure(XrdSysError &); XrdBwm(); virtual ~XrdBwm() {} // Too complicate to delete :-) /******************************************************************************/ /* C o n f i g u r a t i o n V a l u e s */ /******************************************************************************/ XrdVersionInfo *myVersion; // ->Version compiled with char *ConfigFN; // ->Configuration filename char *HostName; // ->Our hostname char *HostPref; // ->Our hostname with domain removed char *myDomain; // ->Our domain name int myDomLen; // char Authorize; char Reserved[7]; /******************************************************************************/ /* P r o t e c t e d I t e m s */ /******************************************************************************/ protected: virtual int ConfigXeq(char *var, XrdOucStream &, XrdSysError &); int Emsg(const char *, XrdOucErrInfo &, int, const char *, const char *y=""); int Emsg(const char *, XrdOucErrInfo &, const char *, const char *, const char *y=""); int Stall(XrdOucErrInfo &, int, const char *); /******************************************************************************/ /* P r i v a t e C o n f i g u r a t i o n */ /******************************************************************************/ private: XrdAccAuthorize *Authorization; // ->Authorization Service char *AuthLib; // ->Authorization Library char *AuthParm; // ->Authorization Parameters XrdBwmLogger *Logger; // ->Logger XrdBwmPolicy *Policy; // ->Policy char *PolLib; char *PolParm; char *locResp; // ->Locate Response int locRlen; // Length of locResp; int PolSlotsIn; int PolSlotsOut; static XrdBwmHandle *dummyHandle; XrdSysMutex ocMutex; // Global mutex for open/close /******************************************************************************/ /* O t h e r D a t a */ /******************************************************************************/ int remove(const char type, const char *path, XrdOucErrInfo &out_error, const XrdSecEntity *client, const char *opaque); // Function used during Configuration // int setupAuth(XrdSysError &); int setupPolicy(XrdSysError &); int xalib(XrdOucStream &, XrdSysError &); int xlog(XrdOucStream &, XrdSysError &); int xpol(XrdOucStream &, XrdSysError &); int xtrace(XrdOucStream &, XrdSysError &); }; #endif xrootd-5.6.9/src/XrdBwm/XrdBwmConfig.cc000066400000000000000000000363771457266313600177410ustar00rootroot00000000000000/******************************************************************************/ /* */ /* X r d B w m C o n f i g . c c */ /* */ /* (C) 2010 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Deprtment of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include #include #include #include #include #include #include #include "XrdBwm/XrdBwm.hh" #include "XrdBwm/XrdBwmLogger.hh" #include "XrdBwm/XrdBwmPolicy.hh" #include "XrdBwm/XrdBwmPolicy1.hh" #include "XrdBwm/XrdBwmTrace.hh" #include "XrdOuc/XrdOuca2x.hh" #include "XrdOuc/XrdOucEnv.hh" #include "XrdOuc/XrdOucPinLoader.hh" #include "XrdSys/XrdSysError.hh" #include "XrdSys/XrdSysHeaders.hh" #include "XrdOuc/XrdOucStream.hh" #include "XrdOuc/XrdOucTrace.hh" #include "XrdAcc/XrdAccAuthorize.hh" /******************************************************************************/ /* d e f i n e s */ /******************************************************************************/ #define TS_Xeq(x,m) if (!strcmp(x,var)) return m(Config,Eroute); #define TS_Str(x,m) if (!strcmp(x,var)) {free(m); m = strdup(val); return 0;} #define TS_PList(x,m) if (!strcmp(x,var)) \ {m.Insert(new XrdOucPList(val,1)); return 0;} #define TS_Chr(x,m) if (!strcmp(x,var)) {m = val[0]; return 0;} #define TS_Bit(x,m,v) if (!strcmp(x,var)) {m |= v; Config.Echo(); return 0;} #define Max(x,y) (x > y ? x : y) /******************************************************************************/ /* C o n f i g u r e */ /******************************************************************************/ int XrdBwm::Configure(XrdSysError &Eroute) { /* Function: Establish default values using a configuration file. Input: None. Output: 0 upon success or !0 otherwise. */ char *var; int cfgFD, retc, NoGo = 0; XrdOucEnv myEnv; XrdOucStream Config(&Eroute, getenv("XRDINSTANCE"), &myEnv, "=====> "); // Print warm-up message // Eroute.Say("++++++ Bwm initialization started."); // Get the debug level from the command line // if (getenv("XRDDEBUG")) BwmTrace.What = TRACE_ALL; // If there is no config file, return with the defaults sets. // if( !ConfigFN || !*ConfigFN) Eroute.Emsg("Config", "Configuration file not specified."); else { // Try to open the configuration file. // if ( (cfgFD = open(ConfigFN, O_RDONLY, 0)) < 0) return Eroute.Emsg("Config", errno, "open config file", ConfigFN); Config.Attach(cfgFD); static const char *cvec[] = { "*** bwm ofs plugin config:", 0 }; Config.Capture(cvec); // Now start reading records until eof. // while((var = Config.GetMyFirstWord())) {if (!strncmp(var, "bwm.", 4)) if (ConfigXeq(var+4,Config,Eroute)) {Config.Echo();NoGo=1;} } // Now check if any errors occurred during file i/o // if ((retc = Config.LastError())) NoGo = Eroute.Emsg("Config", -retc, "read config file", ConfigFN); Config.Close(); } // Determine whether we should initialize authorization // if (Authorize) NoGo |= setupAuth(Eroute); // Establish scheduling policy // if (PolLib) NoGo |= setupPolicy(Eroute); else Policy = new XrdBwmPolicy1(PolSlotsIn, PolSlotsOut); // Start logger object // if (!NoGo && Logger) NoGo = Logger->Start(&Eroute); // Inform the handle of the policy and logger // if (!NoGo) XrdBwmHandle::setPolicy(Policy, Logger); // All done // Eroute.Say("------ Bwm initialization ", (NoGo ? "failed." : "completed.")); return NoGo; } /******************************************************************************/ /* p r i v a t e f u n c t i o n s */ /******************************************************************************/ /******************************************************************************/ /* C o n f i g X e q */ /******************************************************************************/ int XrdBwm::ConfigXeq(char *var, XrdOucStream &Config, XrdSysError &Eroute) { TS_Bit("authorize", Authorize, 1); TS_Xeq("authlib", xalib); TS_Xeq("log", xlog); TS_Xeq("policy", xpol); TS_Xeq("trace", xtrace); // No match found, complain. // Eroute.Say("Config warning: ignoring unknown directive '",var,"'."); Config.Echo(); return 0; } /******************************************************************************/ /* x a l i b */ /******************************************************************************/ /* Function: xalib Purpose: To parse the directive: authlib [] the path of the authorization library to be used. optional parms to be passed Output: 0 upon success or !0 upon failure. */ int XrdBwm::xalib(XrdOucStream &Config, XrdSysError &Eroute) { char *val, parms[1024]; // Get the path // if (!(val = Config.GetWord()) || !val[0]) {Eroute.Emsg("Config", "authlib not specified"); return 1;} // Record the path // if (AuthLib) free(AuthLib); AuthLib = strdup(val); // Record any parms // if (!Config.GetRest(parms, sizeof(parms))) {Eroute.Emsg("Config", "authlib parameters too long"); return 1;} if (AuthParm) free(AuthParm); AuthParm = (*parms ? strdup(parms) : 0); return 0; } /******************************************************************************/ /* x l o g */ /******************************************************************************/ /* Function: xlog Purpose: Parse directive: log {* | <|prog> | <>path>} - is the program to execute and dynamically feed messages about the indicated events. Messages are piped to prog. - is the udp named socket to receive the message. The server creates the path if it's not present. If is an asterisk, then messages are written to standard log file. Output: 0 upon success or !0 upon failure. */ int XrdBwm::xlog(XrdOucStream &Config, XrdSysError &Eroute) { char *val, parms[1024]; if (!(val = Config.GetWord())) {Eroute.Emsg("Config", "log parameters not specified"); return 1;} // Get the remaining parameters // Config.RetToken(); if (!Config.GetRest(parms, sizeof(parms))) {Eroute.Emsg("Config", "log parameters too long"); return 1;} val = (*parms == '|' ? parms+1 : parms); // Create a log object // if (Logger) delete Logger; Logger = new XrdBwmLogger(val); // All done // return 0; } /******************************************************************************/ /* x p o l */ /******************************************************************************/ /* Function: xpol Purpose: To parse the directive: policy args Args: {maxslots | lib []} maximum number of slots available. if preceeded by lib, the path of the policy library to be used; otherwise, the file that describes policy. optional parms to be passed Output: 0 upon success or !0 upon failure. */ int XrdBwm::xpol(XrdOucStream &Config, XrdSysError &Eroute) { char *val, parms[2048]; int pl; // Get next token // if (!(val = Config.GetWord()) || !val[0]) {Eroute.Emsg("Config", "policy not specified"); return 1;} // Start afresh // if (PolLib) {free(PolLib); PolLib = 0;} if (PolParm) {free(PolParm); PolParm = 0;} PolSlotsIn = PolSlotsOut = 0; // If the word maxslots then this is a simple policy // if (!strcmp("maxslots", val)) {if (!(val = Config.GetWord()) || !val[0]) {Eroute.Emsg("Config", "policy in slots not specified"); return 1;} if (XrdOuca2x::a2i(Eroute,"policy in slots",val,&pl,0,32767)) return 1; PolSlotsIn = pl; if (!(val = Config.GetWord()) || !val[0]) {Eroute.Emsg("Config", "policy out slots not specified"); return 1;} if (XrdOuca2x::a2i(Eroute,"policy out slots",val,&pl,0,32767)) return 1; PolSlotsOut = pl; return 0; } // Make sure the word is lib // if (strcmp("lib", val)) {Eroute.Emsg("Config", "invalid policy keyword -", val); return 1;} if (!(val = Config.GetWord()) || !val[0]) {Eroute.Emsg("Config", "policy library not specified"); return 1;} // Set the library // PolLib = strdup(val); // Get any parameters // if (!Config.GetRest(parms, sizeof(parms))) {Eroute.Emsg("Config", "policy lib parameters too long"); return 1;} PolParm = (*parms ? strdup(parms) : 0); // All done // return 0; } /******************************************************************************/ /* x t r a c e */ /******************************************************************************/ /* Function: xtrace Purpose: To parse the directive: trace the blank separated list of events to trace. Trace directives are cummalative. Output: 0 upon success or !0 upon failure. */ int XrdBwm::xtrace(XrdOucStream &Config, XrdSysError &Eroute) { static struct traceopts {const char *opname; int opval;} tropts[] = { {"all", TRACE_ALL}, {"calls", TRACE_calls}, {"debug", TRACE_debug}, {"delay", TRACE_delay}, {"sched", TRACE_sched}, {"tokens", TRACE_tokens} }; int i, neg, trval = 0, numopts = sizeof(tropts)/sizeof(struct traceopts); char *val; if (!(val = Config.GetWord())) {Eroute.Emsg("Config", "trace option not specified"); return 1;} while (val) {if (!strcmp(val, "off")) trval = 0; else {if ((neg = (val[0] == '-' && val[1]))) val++; for (i = 0; i < numopts; i++) {if (!strcmp(val, tropts[i].opname)) {if (neg) trval &= ~tropts[i].opval; else trval |= tropts[i].opval; break; } } if (i >= numopts) Eroute.Say("Config warning: ignoring invalid trace option '",val,"'."); } val = Config.GetWord(); } BwmTrace.What = trval; // All done // return 0; } /******************************************************************************/ /* s e t u p A u t h */ /******************************************************************************/ int XrdBwm::setupAuth(XrdSysError &Eroute) { extern XrdAccAuthorize *XrdAccDefaultAuthorizeObject(XrdSysLogger *lp, const char *cfn, const char *parm, XrdVersionInfo &); XrdOucPinLoader *myLib; XrdAccAuthorize *(*ep)(XrdSysLogger *, const char *, const char *); // Authorization comes from the library or we use the default // if (!AuthLib) return 0 == (Authorization = XrdAccDefaultAuthorizeObject (Eroute.logger(),ConfigFN,AuthParm,*myVersion)); // Create a pluin object (we will throw this away without deletion because // the library must stay open but we never want to reference it again). // if (!(myLib = new XrdOucPinLoader(&Eroute, myVersion, "authlib", AuthLib))) return 1; // Now get the entry point of the object creator // ep = (XrdAccAuthorize *(*)(XrdSysLogger *, const char *, const char *)) (myLib->Resolve("XrdAccAuthorizeObject")); if (!ep) return 1; // Get the Object now // if (!(Authorization = ep(Eroute.logger(), ConfigFN, AuthParm))) myLib->Unload(); delete myLib; return (Authorization == 0); } /******************************************************************************/ /* s e t u p P o l i c y */ /******************************************************************************/ int XrdBwm::setupPolicy(XrdSysError &Eroute) { XrdOucPinLoader myLib(&Eroute, myVersion, "policylib", PolLib); XrdBwmPolicy *(*ep)(XrdSysLogger *, const char *, const char *); // Now get the entry point of the object creator // ep = (XrdBwmPolicy *(*)(XrdSysLogger *, const char *, const char *)) (myLib.Resolve("XrdBwmPolicyObject")); if (!ep) {myLib.Unload(); return 1;} // Get the Object now // if (!(Policy = ep(Eroute.logger(), ConfigFN, PolParm))) myLib.Unload(); return (Policy == 0); } xrootd-5.6.9/src/XrdBwm/XrdBwmHandle.cc000066400000000000000000000334371457266313600177210ustar00rootroot00000000000000/******************************************************************************/ /* */ /* X r d B w m H a n d l e . c c */ /* */ /* (c) 2008 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include #include "XrdBwm/XrdBwmHandle.hh" #include "XrdBwm/XrdBwmLogger.hh" #include "XrdBwm/XrdBwmTrace.hh" #include "XrdSfs/XrdSfsInterface.hh" #include "XrdSys/XrdSysError.hh" #include "XrdSys/XrdSysPlatform.hh" #include "XProtocol/XProtocol.hh" /******************************************************************************/ /* S t a t i c O b j e c t s */ /******************************************************************************/ XrdBwmLogger *XrdBwmHandle::Logger = 0; XrdBwmPolicy *XrdBwmHandle::Policy = 0; XrdBwmHandle *XrdBwmHandle::Free = 0; unsigned int XrdBwmHandle::numQueued = 0; extern XrdSysError BwmEroute; /******************************************************************************/ /* L o c a l C l a s s e s */ /******************************************************************************/ class XrdBwmHandleCB : public XrdOucEICB, public XrdOucErrInfo { public: static XrdBwmHandleCB *Alloc() {XrdBwmHandleCB *mP; xMutex.Lock(); if (!(mP = Free)) mP = new XrdBwmHandleCB; else Free = mP->Next; xMutex.UnLock(); return mP; } void Done(int &Results, XrdOucErrInfo *eInfo, const char *Path=0) {xMutex.Lock(); Next = Free; Free = this; xMutex.UnLock(); } int Same(unsigned long long arg1, unsigned long long arg2) {return 0;} XrdBwmHandleCB() : Next(0) {} ~XrdBwmHandleCB() {} private: XrdBwmHandleCB *Next; static XrdSysMutex xMutex; static XrdBwmHandleCB *Free; }; XrdSysMutex XrdBwmHandleCB::xMutex; XrdBwmHandleCB *XrdBwmHandleCB::Free = 0; /******************************************************************************/ /* E x t e r n a l L i n k a g e s */ /******************************************************************************/ void *XrdBwmHanXeq(void *pp) { return XrdBwmHandle::Dispatch(); } /******************************************************************************/ /* c l a s s X r d B w m H a n d l e */ /******************************************************************************/ /******************************************************************************/ /* A c t i v a t e */ /******************************************************************************/ #define tident Parms.Tident int XrdBwmHandle::Activate(XrdOucErrInfo &einfo) { EPNAME("Activate"); XrdSysMutexHelper myHelper(hMutex); char *rBuff; int rSize, rc; // Check the status of this request. // if (Status != Idle) {if (Status == Scheduled) einfo.setErrInfo(kXR_inProgress, "Request already scheduled."); else einfo.setErrInfo(kXR_InvalidRequest, "Visa already issued."); return SFS_ERROR; } // Try to schedule this request. // qTime = time(0); rBuff = einfo.getMsgBuff(rSize); if (!(rc = Policy->Schedule(rBuff, rSize, Parms))) return SFS_ERROR; // If resource immediately available, let client run // if (rc > 0) {rHandle = rc; Status = Dispatched; rTime = time(0); ZTRACE(sched,"Run " < ") < ") <Parms.Tident = theUsr; // Always available hP->Parms.Lfn = strdup(thePath); hP->Parms.LclNode = strdup(LclNode); hP->Parms.RmtNode = strdup(RmtNode); hP->Parms.Direction = (Incoming ? XrdBwmPolicy::Incoming : XrdBwmPolicy::Outgoing); hP->Status = Idle; hP->qTime = 0; hP->rTime = 0; hP->xSize = 0; hP->xTime = 0; } // All done // return hP; } /******************************************************************************/ /* private A l l o c # 2 */ /******************************************************************************/ XrdBwmHandle *XrdBwmHandle::Alloc(XrdBwmHandle *old_hP) { static const int minAlloc = 4096/sizeof(XrdBwmHandle); static XrdSysMutex aMutex; XrdBwmHandle *hP; // No handle currently in the table. Get a new one off the free list or // return one to the free list. // aMutex.Lock(); if (old_hP) {old_hP->Next = Free; Free = old_hP; hP = 0;} else {if (!Free && (hP = new XrdBwmHandle[minAlloc])) {int i = minAlloc; while(i--) {hP->Next = Free; Free = hP; hP++;}} if ((hP = Free)) Free = hP->Next; } aMutex.UnLock(); return hP; } /******************************************************************************/ /* D i s p a t c h */ /******************************************************************************/ #define tident hP->Parms.Tident void *XrdBwmHandle::Dispatch() { EPNAME("Dispatch"); XrdBwmHandleCB *erP = XrdBwmHandleCB::Alloc(); XrdBwmHandle *hP; char *RespBuff; int RespSize, readyH, Result, Err; // Dispatch ready requests in an endless loop // do { // Setup buffer // RespBuff = erP->getMsgBuff(RespSize); *RespBuff = '\0'; erP->setErrCode(0); // Get next ready request and test if it ended with an error // if ((Err = (readyH = Policy->Dispatch(RespBuff, RespSize)) < 0)) readyH = -readyH; // Find the matching handle // if (!(hP = refHandle(readyH))) {sprintf(RespBuff, "%d", readyH); BwmEroute.Emsg("Dispatch", "Lost handle from", RespBuff); if (!Err) Policy->Done(readyH); continue; } // Lock the handle and make sure it can be dispatched // hP->hMutex.Lock(); if (hP->Status != Scheduled) {BwmEroute.Emsg("Dispatch", "ref to unscheduled handle", hP->Parms.Tident, hP->Parms.Lfn); if (!Err) Policy->Done(readyH); } else { hP->myEICB.Wait(); hP->rTime = time(0); erP->setErrCB((XrdOucEICB *)erP, hP->ErrCBarg); if (Err) {hP->Status = Idle; Result = SFS_ERROR;} else {hP->Status = Dispatched; erP->setErrCode(strlen(RespBuff)); Result = (*RespBuff ? SFS_DATA : SFS_OK); } ZTRACE(sched,(Err?"Err ":"Run ") <Parms.Lfn <<' ' <Parms.LclNode <<(hP->Parms.Direction == XrdBwmPolicy::Incoming ? " <- ":" -> ") <Parms.RmtNode); hP->ErrCB->Done(Result, (XrdOucErrInfo *)erP); erP = XrdBwmHandleCB::Alloc(); } hP->hMutex.UnLock(); } while(1); // Keep the compiler happy // return (void *)0; } #undef tident /******************************************************************************/ /* private r e f H a n d l e */ /******************************************************************************/ XrdBwmHandle *XrdBwmHandle::refHandle(int refID, XrdBwmHandle *hP) { static XrdSysMutex tMutex; static struct {XrdBwmHandle *First; XrdBwmHandle *Last; } hTab[256] = {{0,0}}; XrdBwmHandle *pP = 0; int i = refID % 256; // If we have a handle passed, add the handle to the table // tMutex.Lock(); if (hP) {hP->Next = 0; if (hTab[i].Last) {hTab[i].Last->Next = hP; hTab[i].Last = hP;} else {hTab[i].First = hTab[i].Last = hP; hP->Next = 0;} numQueued++; } else { hP = hTab[i].First; while(hP && hP->rHandle != refID) {pP = hP; hP = hP->Next;} if (hP) {if (pP) pP->Next = hP->Next; else hTab[i].First = hP->Next; if (hTab[i].Last == hP) hTab[i].Last = pP; numQueued--; } } tMutex.UnLock(); // All done. // return hP; } /******************************************************************************/ /* public R e t i r e */ /******************************************************************************/ // The handle must be locked upon entry! It is unlocked upon exit. void XrdBwmHandle::Retire() { XrdSysMutexHelper myHelper(hMutex); // Get the global lock as the links field can only be manipulated with it. // If not idle, cancel the resource. If scheduled, remove it from the table. // if (Status != Idle) {Policy->Done(rHandle); if (Status == Scheduled && !refHandle(rHandle, this)) BwmEroute.Emsg("Retire", "Lost handle to", Parms.Tident, Parms.Lfn); Status = Idle; rHandle = 0; } // If we have a logger, then log this event // if (Logger && qTime) {XrdBwmLogger::Info myInfo; myInfo.Tident = Parms.Tident; myInfo.Lfn = Parms.Lfn; myInfo.lclNode = Parms.LclNode; myInfo.rmtNode = Parms.RmtNode; myInfo.ATime = qTime; myInfo.BTime = rTime; myInfo.CTime = time(0); myInfo.Size = xSize; myInfo.ESec = xTime; myInfo.Flow = (Parms.Direction == XrdBwmPolicy::Incoming ? 'I':'O'); Policy->Status(myInfo.numqIn, myInfo.numqOut, myInfo.numqXeq); Logger->Event(myInfo); } // Free storage appendages and recycle handle // if (Parms.Lfn) {free(Parms.Lfn); Parms.Lfn = 0;} if (Parms.LclNode) {free(Parms.LclNode); Parms.LclNode = 0;} if (Parms.RmtNode) {free(Parms.RmtNode); Parms.RmtNode = 0;} Alloc(this); } /******************************************************************************/ /* s e t P o l i c y */ /******************************************************************************/ int XrdBwmHandle::setPolicy(XrdBwmPolicy *pP, XrdBwmLogger *lP) { pthread_t tid; int rc, startThread = (Policy == 0); // Set the policy and then start a thread to do dispatching if we have none // Policy = pP; if (startThread) if ((rc = XrdSysThread::Run(&tid, XrdBwmHanXeq, (void *)0, 0, "Handle Dispatcher"))) {BwmEroute.Emsg("setPolicy", rc, "create handle dispatch thread"); return 1; } // All done // Logger = lP; return 0; } xrootd-5.6.9/src/XrdBwm/XrdBwmHandle.hh000066400000000000000000000104261457266313600177240ustar00rootroot00000000000000#ifndef __BWM_HANDLE__ #define __BWM_HANDLE__ /******************************************************************************/ /* */ /* X r d B w m H a n d l e . h h */ /* */ /* (c) 2008 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include "XrdBwm/XrdBwmPolicy.hh" #include "XrdOuc/XrdOucErrInfo.hh" #include "XrdSys/XrdSysPthread.hh" class XrdBwmLogger; class XrdBwmHandle { public: enum HandleState {Idle = 0, Scheduled, Dispatched}; HandleState Status; int Activate(XrdOucErrInfo &einfo); static XrdBwmHandle *Alloc(const char *theUsr, const char *thePath, const char *lclNode, const char *rmtNode, int Incoming); static void *Dispatch(); inline const char *Name() {return Parms.Lfn;} void Retire(); static int setPolicy(XrdBwmPolicy *pP, XrdBwmLogger *lP); XrdBwmHandle() : Status(Idle), Next(0), qTime(0), rTime(0), xSize(0), xTime(0) {} ~XrdBwmHandle() {} private: static XrdBwmHandle *Alloc(XrdBwmHandle *oldHandle=0); static XrdBwmHandle *refHandle(int refID, XrdBwmHandle *hP=0); static XrdBwmPolicy *Policy; static XrdBwmLogger *Logger; static XrdBwmHandle *Free; // List of free handles static unsigned int numQueued; XrdSysMutex hMutex; XrdBwmPolicy::SchedParms Parms; XrdBwmHandle *Next; XrdOucEICB *ErrCB; unsigned long long ErrCBarg; time_t qTime; time_t rTime; long long xSize; long xTime; int rHandle; class theEICB : public XrdOucEICB { public: void Done(int &Result, XrdOucErrInfo *eInfo, const char *Path=0) {mySem.Post();} int Same(unsigned long long arg1, unsigned long long arg2) {return arg1 == arg2;} void Wait() {mySem.Wait();} theEICB() : mySem(0) {} virtual ~theEICB() {} private: XrdSysSemaphore mySem; } myEICB; }; #endif xrootd-5.6.9/src/XrdBwm/XrdBwmLogger.cc000066400000000000000000000254321457266313600177410ustar00rootroot00000000000000/******************************************************************************/ /* */ /* X r d B w m L o g g e r . c c */ /* */ /* (c) 2008 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include #include #include #include #include "XrdBwm/XrdBwmLogger.hh" #include "XrdSys/XrdSysError.hh" #include "XrdOuc/XrdOucProg.hh" #include "XrdOuc/XrdOucStream.hh" #include "XrdNet/XrdNetOpts.hh" #include "XrdNet/XrdNetSocket.hh" #include "XrdSys/XrdSysPlatform.hh" /******************************************************************************/ /* L o c a l C l a s s e s */ /******************************************************************************/ class XrdBwmLoggerMsg { public: static const int msgSize = 2048; XrdBwmLoggerMsg *next; char Text[msgSize]; int Tlen; XrdBwmLoggerMsg() : next(0), Tlen(0) {} ~XrdBwmLoggerMsg() {} }; /******************************************************************************/ /* E x t e r n a l L i n k a g e s */ /******************************************************************************/ void *XrdBwmLoggerSend(void *pp) { XrdBwmLogger *lP = (XrdBwmLogger *)pp; lP->sendEvents(); return (void *)0; } /******************************************************************************/ /* C o n s t r u c t o r */ /******************************************************************************/ XrdBwmLogger::XrdBwmLogger(const char *Target) { // Set common variables // theTarget = strdup(Target); eDest = 0; theProg = 0; msgFirst = msgLast = msgFree = 0; tid = 0; msgFD = 0; endIT = 0; theEOL= '\n'; msgsInQ = 0; } /******************************************************************************/ /* D e s t r u c t o r */ /******************************************************************************/ XrdBwmLogger::~XrdBwmLogger() { XrdBwmLoggerMsg *tp; // Kill the notification thread. This may cause a msg block to be orphaned // but, in practice, this object does not really get deleted after being // started. So, the problem is moot. // endIT = 1; if (tid) XrdSysThread::Kill(tid); // Release all queued message bocks // qMut.Lock(); while ((tp = msgFirst)) {msgFirst = tp->next; delete tp;} if (theTarget) free(theTarget); if (msgFD >= 0) close(msgFD); if (theProg) delete theProg; qMut.UnLock(); // Release all free message blocks // fMut.Lock(); while ((tp = msgFree)) {msgFree = tp->next; delete tp;} fMut.UnLock(); } /******************************************************************************/ /* E v e n t */ /******************************************************************************/ void XrdBwmLogger::Event(Info &eInfo) { static int warnings = 0; XrdBwmLoggerMsg *tp; // Get a message block // if (!(tp = getMsg())) {if ((++warnings & 0xff) == 1) eDest->Emsg("Notify", "Ran out of logger message objects;", eInfo.Tident, "event not logged."); return; } // Format the message // tp->Tlen = snprintf(tp->Text, XrdBwmLoggerMsg::msgSize, "%s%s" "%s%s%c" "%ld%ld%ld" "%d%d%d" "%lld%d%c", eInfo.Tident, eInfo.Lfn, eInfo.lclNode, eInfo.rmtNode, eInfo.Flow, eInfo.ATime, eInfo.BTime, eInfo.CTime, eInfo.numqIn, eInfo.numqOut, eInfo.numqXeq, eInfo.Size, eInfo.ESec, theEOL); // Either log this or put the message on the queue and return // tp->next = 0; qMut.Lock(); if (msgLast) {msgLast->next = tp; msgLast = tp;} else msgFirst = msgLast = tp; qMut.UnLock(); qSem.Post(); } /******************************************************************************/ /* s e n d E v e n t s */ /******************************************************************************/ void XrdBwmLogger::sendEvents(void) { XrdBwmLoggerMsg *tp; const char *theData[2] = {0,0}; int theDlen[2] = {0,0}; // This is an endless loop that just gets things off the event queue and // send them out. This allows us to only hang a simgle thread should the // receiver get blocked, instead of the whole process. // while(1) {qSem.Wait(); qMut.Lock(); if (endIT) break; if ((tp = msgFirst) && !(msgFirst = tp->next)) msgLast = 0; qMut.UnLock(); if (tp) {if (!theProg) Feed(tp->Text, tp->Tlen); else {theData[0] = tp->Text; theDlen[0] = tp->Tlen; theProg->Feed(theData, theDlen); } retMsg(tp); } } qMut.UnLock(); } /******************************************************************************/ /* S t a r t */ /******************************************************************************/ int XrdBwmLogger::Start(XrdSysError *eobj) { int rc; // Set the error object pointer // eDest = eobj; // Check if we need to create a socket to a path // if (!strcmp("*", theTarget)) {msgFD = -1; theEOL = '\0';} else if (*theTarget == '>') {XrdNetSocket *msgSock; if (!(msgSock = XrdNetSocket::Create(eobj, theTarget+1, 0, 0660, XRDNET_FIFO))) return -1; msgFD = msgSock->Detach(); delete msgSock; } else {// Allocate a new program object if we don't have one // if (theProg) return 0; theProg = new XrdOucProg(eobj); // Setup the program // if (theProg->Setup(theTarget, eobj)) return -1; if ((rc = theProg->Start())) {eobj->Emsg("Logger", rc, "start event collector"); return -1;} } // Now start a thread to get messages and send them to the collector // if ((rc = XrdSysThread::Run(&tid, XrdBwmLoggerSend, static_cast(this), 0, "Log message sender"))) {eobj->Emsg("Logger", rc, "create log message sender thread"); return -1; } // All done // return 0; } /******************************************************************************/ /* P r i v a t e M e t h o d s */ /******************************************************************************/ /******************************************************************************/ /* F e e d */ /******************************************************************************/ int XrdBwmLogger::Feed(const char *data, int dlen) { int retc; // Send message to the log if need be // if (msgFD < 0) {eDest->Say("", data); return 0;} // Write the data. since this is a udp socket all the data goes or none does // do { retc = write(msgFD, (const void *)data, (size_t)dlen);} while (retc < 0 && errno == EINTR); if (retc < 0) {eDest->Emsg("Feed", errno, "write to logger socket", theTarget); return -1; } // All done // return 0; } /******************************************************************************/ /* g e t M s g */ /******************************************************************************/ XrdBwmLoggerMsg *XrdBwmLogger::getMsg() { XrdBwmLoggerMsg *tp; // Lock the free queue // fMut.Lock(); // Get message object but don't give out too many // if (msgsInQ >= maxmInQ) tp = 0; else {if ((tp = msgFree)) msgFree = tp->next; else tp = new XrdBwmLoggerMsg(); msgsInQ++; } // Unlock and return result // fMut.UnLock(); return tp; } /******************************************************************************/ /* r e t M s g */ /******************************************************************************/ void XrdBwmLogger::retMsg(XrdBwmLoggerMsg *tp) { // Lock the free queue, return message, unlock the queue // fMut.Lock(); tp->next = msgFree; msgFree = tp; msgsInQ--; fMut.UnLock(); } xrootd-5.6.9/src/XrdBwm/XrdBwmLogger.hh000066400000000000000000000072611457266313600177530ustar00rootroot00000000000000#ifndef __XRDBWMLOGGER_H__ #define __XRDBWMLOGGER_H__ /******************************************************************************/ /* */ /* X r d B w m L o g g e r . h h */ /* */ /* (c) 2008 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include "XrdSys/XrdSysPthread.hh" class XrdBwmLoggerMsg; class XrdOucProg; class XrdSysError; class XrdBwmLogger { public: struct Info {const char *Tident; const char *Lfn; const char *lclNode; const char *rmtNode; time_t ATime; // Arrival time_t BTime; // Begin time_t CTime; // Complete int numqIn; int numqOut; int numqXeq; long long Size; int ESec; char Flow; // I or O }; void Event(Info &eInfo); const char *Prog() {return theTarget;} void sendEvents(void); int Start(XrdSysError *eobj); XrdBwmLogger(const char *Target); ~XrdBwmLogger(); private: int Feed(const char *data, int dlen); XrdBwmLoggerMsg *getMsg(); void retMsg(XrdBwmLoggerMsg *tp); pthread_t tid; char *theTarget; XrdSysError *eDest; XrdOucProg *theProg; XrdSysMutex qMut; XrdSysSemaphore qSem; XrdBwmLoggerMsg *msgFirst; XrdBwmLoggerMsg *msgLast; XrdSysMutex fMut; XrdBwmLoggerMsg *msgFree; int msgFD; int endIT; int msgsInQ; static const int maxmInQ = 256; char theEOL; }; #endif xrootd-5.6.9/src/XrdBwm/XrdBwmPolicy.hh000066400000000000000000000206601457266313600177710ustar00rootroot00000000000000#ifndef __BWM_POLICY__ #define __BWM_POLICY__ /******************************************************************************/ /* */ /* X r d B w m P o l i c y . h h */ /* */ /* (c) 2008 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ class XrdBwmPolicy { public: /* General note: Each request is to be identified by an int-sized handle. The value of the handle is unique with respect to all of the requests that are active and queued. Once a request leaves the system (i.e., cancelled or released) the handle may be re-used. Handle signs are immaterial. That is the property "n == abs(-n) == " always must hold. Note that Schedule() uses negative handles to merely indicate queuing. */ /* Dispatch() returns the handle of the next request that may become active because the resources are now available or that must be terminated because resources are not available. The returned value must have the the following property: "Dispatch() == abs(Schedule()) == ". Hence, the handle returned by Dispatch() must be one previously returned by Schedule() that was negative to indicate that the request was queued. The sign of the returned handle indicates success or failure: returns < 0: The associated previously scheduled request cannot obtain the resource. RespBuff, of size RespSize, should contain null terminated text describing the failure. Done() will not called for the returned handle. returns >= 0: The associated previously scheduled request can now be dispatched as resources are available. RespBuff, of size RespSize, should contain any visa information, as an ASCII null terminated string to be sent to client. If none, it should contain a null string (i.e., zero byte). Done() will be called for the returned handle when the resource is no longer needed. Dispatch() blocks until a request is ready or has failed. */ virtual int Dispatch(char *RespBuff, int RespSize) = 0; /* Done() indicates that the resources with a previous request associated with the handle, as returned by Dispatch() and Schedule(). When Done() is called with a handle referring to a queued request, the request should be cancelled and removed from the queue. If the handle refers to an active request (i.e., a non-negative one that was returned by Dispatch()), the resources associated with the dispatched request are no longer needed and are to be made available to another request. The value returned by Done() indicates what happened: returns < 0: The queued request was cancelled. returns = 0: No request matching the handle was found. returns > 0: The resources associated with the dispatched request returned. The handle itself may be a positive or negative, as returned by Dispatch() and Schedule(). Note that "n == abs(-n) == ", so the sign of the handle should be immaterial to Done(). Negative handles returned by Dispatch() indicate failure and thus Done() will not be called for such handles. Handles returned by Schedule() may be postive or negative. */ virtual int Done(int rHandle) = 0; /* Schedule() is invoked when the caller wishes to obtain resources controlled by the policy. The caller passes a pointer to a response buffer, RespBuff, of size contained in RespSize, to hold hold any response. Additionally. a reference to the SchedParms struct that contains information about the nature of the request. Schedule() may choose to immediately allow the resourse to be used, fail the request, or to defer the request. This is indicated by the returned int, as follows: returns < 0: The request has been queued. The returned value is the handle for the request and is to be used as the argument to Done() to cancel the queued request. returns = 0: The request failed. The RespBuff should contain any error text or a null byte if no text is present. returns > 0: The request succeeded and the resource can be used. The returned value is the handle for the request and is to be used as the argument to Done() to release the associated request resource. RespBuff should contain any visa information, as an ASCII null terminated string to be sent to client. If none, it must contain a null string (i.e., zero byte). */ enum Flow {Incoming = 0, Outgoing}; struct SchedParms { const char *Tident; // In: -> Client's trace identity char *Lfn; // In: -> Logical File Name char *LclNode; // In: -> Local node involved in the request char *RmtNode; // In: -> Remote node involved in the request Flow Direction; // In: -> Data flow relative to Lclpoint (see enum) }; virtual int Schedule(char *RespBuff, int RespSize, SchedParms &Parms) = 0; /* Status() returns the number of requests as three items via parameters: numqIn - Number of incoming data requests queued numqOut - Number of outgoing data requests queued numXeq - Number of requests that are active (in or out). */ virtual void Status(int &numqIn, int &numqOut, int &numXeq) = 0; XrdBwmPolicy() {} virtual ~XrdBwmPolicy() {} }; /******************************************************************************/ /* X r d B w m P o l i c y O b j e c t */ /******************************************************************************/ class XrdSysLogger; /* XrdBwmPolicyObject() is called to obtain an instance of the policy object that will be used for all subsequent policy scheduling requests. If it returns a null pointer; initialization fails and the program exits. The args are: lp -> XrdSysLogger to be tied to an XrdSysError object for messages cfn -> The name of the configuration file parm -> Parameters specified on the policy lib directive. If none it's zero. */ extern "C" XrdBwmPolicy *XrdBwmPolicyObject(XrdSysLogger *lp, const char *cfn, const char *parm); #endif xrootd-5.6.9/src/XrdBwm/XrdBwmPolicy1.cc000066400000000000000000000136241457266313600200420ustar00rootroot00000000000000/******************************************************************************/ /* */ /* X r d B w m P o l i c y 1 . c c */ /* */ /* (c) 2008 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include "XrdBwm/XrdBwmPolicy1.hh" /******************************************************************************/ /* C o n s t r u c t o r */ /******************************************************************************/ XrdBwmPolicy1::XrdBwmPolicy1(int inslots, int outslots) { // Initialize values // theQ[In ].maxSlots = theQ[In ].curSlots = inslots; theQ[Out].maxSlots = theQ[Out].curSlots = outslots; theQ[Xeq].maxSlots = theQ[Xeq].curSlots = 0; refID = 1; } /******************************************************************************/ /* D i s p a t c h */ /******************************************************************************/ int XrdBwmPolicy1::Dispatch(char *RespBuff, int RespSize) { refReq *rP; int rID; // Obtain mutex and check if we have any queued requests // do {pMutex.Lock(); if ((rP = theQ[In].Next()) || (rP = theQ[Out].Next())) {theQ[Xeq].Add(rP); rID = rP->refID; *RespBuff = '\0'; pMutex.UnLock(); return rID; } pMutex.UnLock(); pSem.Wait(); } while(1); // Should never get here // strcpy(RespBuff, "Fatal logic error!"); return 0; } /******************************************************************************/ /* D o n e */ /******************************************************************************/ int XrdBwmPolicy1::Done(int rHandle) { refReq *rP; int rc; // Make sure we have a positive value here // if (rHandle < 0) rHandle = -rHandle; // Remove the element from whichever queue it is in // pMutex.Lock(); if ((rP = theQ[Xeq].Yank(rHandle))) {if (theQ[rP->Way].curSlots++ == 0) pSem.Post(); rc = 1; } else { if ((rP=theQ[In].Yank(rHandle)) || (rP=theQ[Out].Yank(rHandle))) rc = -1; else rc = 0; } pMutex.UnLock(); // delete the element and return // if (rP) delete rP; return rc; } /******************************************************************************/ /* S c h e d u l e */ /******************************************************************************/ int XrdBwmPolicy1::Schedule(char *RespBuff, int RespSize, SchedParms &Parms) { static const char *theWay[] = {"Incoming", "Outgoing"}; refReq *rP; int myID; // Get the global lock and generate a reference ID // *RespBuff = '\0'; pMutex.Lock(); myID = ++refID; rP = new refReq(myID, Parms.Direction); // Check if we can immediately schedule this requestor must defer it // if (theQ[rP->Way].curSlots > 0) {theQ[rP->Way].curSlots--; theQ[Xeq].Add(rP); } else if (theQ[rP->Way].maxSlots) {theQ[rP->Way].Add(rP); myID = -myID;} else {strcpy(RespBuff, theWay[rP->Way]); strcat(RespBuff, " requests are not allowed."); delete rP; myID = 0; } // All done // pMutex.UnLock(); return myID; } /******************************************************************************/ /* S t a t u s */ /******************************************************************************/ void XrdBwmPolicy1::Status(int &numqIn, int &numqOut, int &numXeq) { // Get the global lock and return the values // pMutex.Lock(); numqIn = theQ[In ].Num; numqOut = theQ[Out].Num; numXeq = theQ[Xeq].Num; pMutex.UnLock(); } xrootd-5.6.9/src/XrdBwm/XrdBwmPolicy1.hh000066400000000000000000000105151457266313600200500ustar00rootroot00000000000000#ifndef __BWM_POLICY1_HH__ #define __BWM_POLICY1_HH__ /******************************************************************************/ /* */ /* X r d B w m P o l i c y 1 . h h */ /* */ /* (c) 2008 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include "XrdBwm/XrdBwmPolicy.hh" #include "XrdSys/XrdSysPthread.hh" class XrdBwmPolicy1 : public XrdBwmPolicy { public: int Dispatch(char *RespBuff, int RespSize); int Done(int rHandle); int Schedule(char *RespBuff, int RespSize, SchedParms &Parms); void Status(int &numqIn, int &numqOut, int &numXeq); XrdBwmPolicy1(int inslots, int outslots); ~XrdBwmPolicy1() {} enum Flow {In = 0, Out = 1, Xeq = 2, IOX = 3}; struct refReq {refReq *Next; int refID; Flow Way; refReq(int id, XrdBwmPolicy::Flow xF) : Next(0), refID(id), Way(xF == XrdBwmPolicy::Incoming ? In : Out) {} ~refReq() {} }; private: class refSch {public: refReq *First; refReq *Last; int Num; int curSlots; int maxSlots; void Add(refReq *rP) {if ((rP->Next = Last)) Last = rP; else First= Last = rP; Num++; } refReq *Next() {refReq *rP; if ((rP = First) && curSlots) {if (!(First = First->Next)) Last = 0; Num--; curSlots--; } return rP; } refReq *Yank(int rID) {refReq *pP = 0, *rP = First; while(rP && rID != rP->refID) {pP = rP; rP = rP->Next;} if (rP) {if (pP) pP->Next = rP->Next; else First = rP->Next; if (rP == Last) Last = pP; Num--; } return rP; } refSch() : First(0), Last(0), Num(0) {} ~refSch() {} // Never deleted! } theQ[IOX]; XrdSysSemaphore pSem; XrdSysMutex pMutex; int refID; }; #endif xrootd-5.6.9/src/XrdBwm/XrdBwmTrace.hh000066400000000000000000000065011457266313600175660ustar00rootroot00000000000000#ifndef ___BWM_TRACE_H___ #define ___BWM_TRACE_H___ /******************************************************************************/ /* */ /* X r d B w m T r a c e . h h */ /* */ /* (C) 2008 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Deprtment of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #ifndef NODEBUG #include "XrdSys/XrdSysHeaders.hh" #include "XrdOuc/XrdOucTrace.hh" extern XrdOucTrace BwmTrace; #define GTRACE(act) BwmTrace.What & TRACE_ ## act #define TRACES(x) \ {BwmTrace.Beg(epname,tident); std::cerr <> /etc/yum.repos.d/ceph.repo - echo -e '[xrootd-testing]\nname=XRootD Testing repository\nbaseurl=http://xrootd.org/binaries/testing/slc/7/$basearch http://xrootd.cern.ch/sw/repos/testing/slc/7/$basearch\ngpgcheck=1\nenabled=1\nprotect=0\ngpgkey=http://xrootd.cern.ch/sw/releases/RPM-GPG-KEY.txt\n' >> /etc/yum.repos.d/xrootd-testing.repo - echo -e '[xrootd-stable]\nname=XRootD Stable repository\nbaseurl=http://xrootd.org/binaries/stable/slc/7/$basearch http://xrootd.cern.ch/sw/repos/stable/slc/7/$basearch\ngpgcheck=1\nenabled=1\nprotect=0\ngpgkey=http://xrootd.cern.ch/sw/releases/RPM-GPG-KEY.txt\n' >> /etc/yum.repos.d/xrootd-stable.repo - yum-builddep --setopt=cern*.exclude=xrootd* --nogpgcheck -y *.src.rpm - rpmbuild --rebuild --define "_rpmdir RPMS/" --define "_build_name_fmt %%{NAME}-%%{VERSION}-%%{RELEASE}.%%{ARCH}.rpm" -D "dist .el7" *.src.rpm - repo=/eos/project/s/storage-ci/www/xrootd/ceph-release/cc-7/x86_64/ - sudo -u stci -H mkdir -p $repo - sudo -u stci -H cp *.src.rpm $repo - sudo -u stci -H cp RPMS/* $repo - sudo -u stci -H createrepo --update -q $repo tags: - docker_node only: - tags except: - schedules weekly:cc7:ceph: stage: build:rpm image: gitlab-registry.cern.ch/linuxsupport/cc7-base script: - yum install --nogpg -y cmake3 make gcc-c++ rpm-build which git yum-plugin-priorities sssd-client sudo createrepo - cd packaging/ - echo -e '[ceph]\nname=ceph\nbaseurl=http://linuxsoft.cern.ch/mirror/download.ceph.com/rpm-nautilus/el7/x86_64/\npriority=4\ngpgcheck=0\nenabled=1\n' >> /etc/yum.repos.d/ceph.repo - echo -e '[xrootd-experimental]\nname=XRootD Experimental repository\nbaseurl=http://storage-ci.web.cern.ch/storage-ci/xrootd/experimental/epel-7/$basearch\ngpgcheck=1\nenabled=1\nprotect=0\n' >> /etc/yum.repos.d/xrootd-experimental.repo - yum clean all - version=$(yum info xrootd-devel | grep Version | cut -d':' -f2 | tr -d "[:blank:]") - release=$(yum info xrootd-devel | grep Release | cut -d':' -f2 | tr -d "[:blank:]") - release=${release%.el7.cern} - ./makesrpm.sh --version "$version-$release" - yum-builddep --setopt=cern*.exclude=xrootd* --nogpgcheck -y *.src.rpm - rpmbuild --rebuild --define "_rpmdir RPMS/" --define "_build_name_fmt %%{NAME}-%%{VERSION}-%%{RELEASE}.%%{ARCH}.rpm" *.src.rpm tags: - docker_node only: - schedules build:cc7:ceph: stage: build:rpm image: gitlab-registry.cern.ch/linuxsupport/cc7-base script: - yum install --nogpg -y cmake3 make gcc-c++ rpm-build which git yum-plugin-priorities sssd-client sudo createrepo - cd packaging/ - echo -e '[ceph]\nname=ceph\nbaseurl=http://linuxsoft.cern.ch/mirror/download.ceph.com/rpm-nautilus/el7/x86_64/\npriority=4\ngpgcheck=0\nenabled=1\n' >> /etc/yum.repos.d/ceph.repo - echo -e '[xrootd-experimental]\nname=XRootD Experimental repository\nbaseurl=http://storage-ci.web.cern.ch/storage-ci/xrootd/experimental/epel-7/$basearch\ngpgcheck=1\nenabled=1\nprotect=0\n' >> /etc/yum.repos.d/xrootd-experimental.repo - yum clean all - version=$(yum info xrootd-devel | grep Version | cut -d':' -f2 | tr -d "[:blank:]") - release=$(yum info xrootd-devel | grep Release | cut -d':' -f2 | tr -d "[:blank:]") - release=${release%.el7.cern} - ./makesrpm.sh --version "$version-$release" - yum-builddep --setopt=cern*.exclude=xrootd* --nogpgcheck -y *.src.rpm - rpmbuild --rebuild --define "_rpmdir RPMS/" --define "_build_name_fmt %%{NAME}-%%{VERSION}-%%{RELEASE}.%%{ARCH}.rpm" *.src.rpm - path=/eos/project/s/storage-ci/www/xrootd/ceph/cc-7/x86_64/$(date +'%Y%m%d') - sudo -u stci -H mkdir -p $path; - sudo -u stci -H find ${path} -type f -name '*.rpm' -delete; - sudo -u stci -H cp RPMS/* $path; - sudo -u stci -H createrepo --update -q $path; tags: - docker_node only: - master except: - tags xrootd-5.6.9/src/XrdCeph/.travis.yml000066400000000000000000000006221457266313600173240ustar00rootroot00000000000000sudo: false dist: trusty addons: apt: packages: - libxml2-dev - libcppunit-dev language: cpp compiler: - clang - gcc script: - mkdir build - pushd build - cmake -DCMAKE_INSTALL_PREFIX=$HOME/xrootd -DENABLE_TESTS=1 .. - make - make install - popd #after_script: # - pushd build # - ./tests/common/text-runner ./tests/XrdClTests/libXrdClTests.so 'All Tests' # - popd xrootd-5.6.9/src/XrdCeph/CMakeLists.txt000066400000000000000000000033461457266313600177610ustar00rootroot00000000000000#------------------------------------------------------------------------------- # Project description #------------------------------------------------------------------------------- cmake_minimum_required(VERSION 3.16...3.25) project( xrootd-ceph ) set( CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/src ${PROJECT_SOURCE_DIR}/cmake ) if( NOT XRDCEPH_SUBMODULE ) if(NOT (CMAKE_VERSION VERSION_LESS "3.1")) cmake_policy(SET CMP0054 OLD) endif() endif() include( XRootDUtils ) CheckBuildDirectory() include( XRootDOSDefs ) include( XRootDDefaults ) include( XRootDFindLibs ) add_definitions( -DXRDPLUGIN_SOVERSION="${PLUGIN_VERSION}" ) #------------------------------------------------------------------------------- # Generate the version header #------------------------------------------------------------------------------- if( NOT XRDCEPH_SUBMODULE ) execute_process( COMMAND ${CMAKE_SOURCE_DIR}/genversion.sh --print-only ${CMAKE_SOURCE_DIR} OUTPUT_VARIABLE XROOTD_VERSION OUTPUT_STRIP_TRAILING_WHITESPACE ) add_custom_target( XrdVersion.hh ${CMAKE_SOURCE_DIR}/genversion.sh ${CMAKE_SOURCE_DIR} ) # sigh, yet another ugly hack :( macro( add_library _target ) _add_library( ${_target} ${ARGN} ) add_dependencies( ${_target} XrdVersion.hh ) endmacro() macro( add_executable _target ) _add_executable( ${_target} ${ARGN} ) add_dependencies( ${_target} XrdVersion.hh ) endmacro() endif() #------------------------------------------------------------------------------- # Build in subdirectories #------------------------------------------------------------------------------- add_subdirectory( src ) if( BUILD_TESTS ) ENABLE_TESTING() add_subdirectory( tests ) endif() include( XRootDSummary ) xrootd-5.6.9/src/XrdCeph/COPYING000066400000000000000000001045131457266313600162520ustar00rootroot00000000000000 GNU GENERAL PUBLIC LICENSE Version 3, 29 June 2007 Copyright (C) 2007 Free Software Foundation, Inc. Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. Preamble The GNU General Public License is a free, copyleft license for software and other kinds of works. The licenses for most software and other practical works are designed to take away your freedom to share and change the works. By contrast, the GNU General Public License is intended to guarantee your freedom to share and change all versions of a program--to make sure it remains free software for all its users. We, the Free Software Foundation, use the GNU General Public License for most of our software; it applies also to any other work released this way by its authors. You can apply it to your programs, too. When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for them if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs, and that you know you can do these things. To protect your rights, we need to prevent others from denying you these rights or asking you to surrender the rights. Therefore, you have certain responsibilities if you distribute copies of the software, or if you modify it: responsibilities to respect the freedom of others. For example, if you distribute copies of such a program, whether gratis or for a fee, you must pass on to the recipients the same freedoms that you received. You must make sure that they, too, receive or can get the source code. And you must show them these terms so they know their rights. Developers that use the GNU GPL protect your rights with two steps: (1) assert copyright on the software, and (2) offer you this License giving you legal permission to copy, distribute and/or modify it. For the developers' and authors' protection, the GPL clearly explains that there is no warranty for this free software. For both users' and authors' sake, the GPL requires that modified versions be marked as changed, so that their problems will not be attributed erroneously to authors of previous versions. Some devices are designed to deny users access to install or run modified versions of the software inside them, although the manufacturer can do so. This is fundamentally incompatible with the aim of protecting users' freedom to change the software. The systematic pattern of such abuse occurs in the area of products for individuals to use, which is precisely where it is most unacceptable. Therefore, we have designed this version of the GPL to prohibit the practice for those products. If such problems arise substantially in other domains, we stand ready to extend this provision to those domains in future versions of the GPL, as needed to protect the freedom of users. Finally, every program is threatened constantly by software patents. States should not allow patents to restrict development and use of software on general-purpose computers, but in those that do, we wish to avoid the special danger that patents applied to a free program could make it effectively proprietary. To prevent this, the GPL assures that patents cannot be used to render the program non-free. The precise terms and conditions for copying, distribution and modification follow. TERMS AND CONDITIONS 0. Definitions. "This License" refers to version 3 of the GNU General Public License. "Copyright" also means copyright-like laws that apply to other kinds of works, such as semiconductor masks. "The Program" refers to any copyrightable work licensed under this License. Each licensee is addressed as "you". "Licensees" and "recipients" may be individuals or organizations. To "modify" a work means to copy from or adapt all or part of the work in a fashion requiring copyright permission, other than the making of an exact copy. The resulting work is called a "modified version" of the earlier work or a work "based on" the earlier work. A "covered work" means either the unmodified Program or a work based on the Program. To "propagate" a work means to do anything with it that, without permission, would make you directly or secondarily liable for infringement under applicable copyright law, except executing it on a computer or modifying a private copy. Propagation includes copying, distribution (with or without modification), making available to the public, and in some countries other activities as well. To "convey" a work means any kind of propagation that enables other parties to make or receive copies. Mere interaction with a user through a computer network, with no transfer of a copy, is not conveying. An interactive user interface displays "Appropriate Legal Notices" to the extent that it includes a convenient and prominently visible feature that (1) displays an appropriate copyright notice, and (2) tells the user that there is no warranty for the work (except to the extent that warranties are provided), that licensees may convey the work under this License, and how to view a copy of this License. If the interface presents a list of user commands or options, such as a menu, a prominent item in the list meets this criterion. 1. Source Code. The "source code" for a work means the preferred form of the work for making modifications to it. "Object code" means any non-source form of a work. A "Standard Interface" means an interface that either is an official standard defined by a recognized standards body, or, in the case of interfaces specified for a particular programming language, one that is widely used among developers working in that language. The "System Libraries" of an executable work include anything, other than the work as a whole, that (a) is included in the normal form of packaging a Major Component, but which is not part of that Major Component, and (b) serves only to enable use of the work with that Major Component, or to implement a Standard Interface for which an implementation is available to the public in source code form. A "Major Component", in this context, means a major essential component (kernel, window system, and so on) of the specific operating system (if any) on which the executable work runs, or a compiler used to produce the work, or an object code interpreter used to run it. The "Corresponding Source" for a work in object code form means all the source code needed to generate, install, and (for an executable work) run the object code and to modify the work, including scripts to control those activities. However, it does not include the work's System Libraries, or general-purpose tools or generally available free programs which are used unmodified in performing those activities but which are not part of the work. For example, Corresponding Source includes interface definition files associated with source files for the work, and the source code for shared libraries and dynamically linked subprograms that the work is specifically designed to require, such as by intimate data communication or control flow between those subprograms and other parts of the work. The Corresponding Source need not include anything that users can regenerate automatically from other parts of the Corresponding Source. The Corresponding Source for a work in source code form is that same work. 2. Basic Permissions. All rights granted under this License are granted for the term of copyright on the Program, and are irrevocable provided the stated conditions are met. This License explicitly affirms your unlimited permission to run the unmodified Program. The output from running a covered work is covered by this License only if the output, given its content, constitutes a covered work. This License acknowledges your rights of fair use or other equivalent, as provided by copyright law. You may make, run and propagate covered works that you do not convey, without conditions so long as your license otherwise remains in force. You may convey covered works to others for the sole purpose of having them make modifications exclusively for you, or provide you with facilities for running those works, provided that you comply with the terms of this License in conveying all material for which you do not control copyright. Those thus making or running the covered works for you must do so exclusively on your behalf, under your direction and control, on terms that prohibit them from making any copies of your copyrighted material outside their relationship with you. Conveying under any other circumstances is permitted solely under the conditions stated below. Sublicensing is not allowed; section 10 makes it unnecessary. 3. Protecting Users' Legal Rights From Anti-Circumvention Law. No covered work shall be deemed part of an effective technological measure under any applicable law fulfilling obligations under article 11 of the WIPO copyright treaty adopted on 20 December 1996, or similar laws prohibiting or restricting circumvention of such measures. When you convey a covered work, you waive any legal power to forbid circumvention of technological measures to the extent such circumvention is effected by exercising rights under this License with respect to the covered work, and you disclaim any intention to limit operation or modification of the work as a means of enforcing, against the work's users, your or third parties' legal rights to forbid circumvention of technological measures. 4. Conveying Verbatim Copies. You may convey verbatim copies of the Program's source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice; keep intact all notices stating that this License and any non-permissive terms added in accord with section 7 apply to the code; keep intact all notices of the absence of any warranty; and give all recipients a copy of this License along with the Program. You may charge any price or no price for each copy that you convey, and you may offer support or warranty protection for a fee. 5. Conveying Modified Source Versions. You may convey a work based on the Program, or the modifications to produce it from the Program, in the form of source code under the terms of section 4, provided that you also meet all of these conditions: a) The work must carry prominent notices stating that you modified it, and giving a relevant date. b) The work must carry prominent notices stating that it is released under this License and any conditions added under section 7. This requirement modifies the requirement in section 4 to "keep intact all notices". c) You must license the entire work, as a whole, under this License to anyone who comes into possession of a copy. This License will therefore apply, along with any applicable section 7 additional terms, to the whole of the work, and all its parts, regardless of how they are packaged. This License gives no permission to license the work in any other way, but it does not invalidate such permission if you have separately received it. d) If the work has interactive user interfaces, each must display Appropriate Legal Notices; however, if the Program has interactive interfaces that do not display Appropriate Legal Notices, your work need not make them do so. A compilation of a covered work with other separate and independent works, which are not by their nature extensions of the covered work, and which are not combined with it such as to form a larger program, in or on a volume of a storage or distribution medium, is called an "aggregate" if the compilation and its resulting copyright are not used to limit the access or legal rights of the compilation's users beyond what the individual works permit. Inclusion of a covered work in an aggregate does not cause this License to apply to the other parts of the aggregate. 6. Conveying Non-Source Forms. You may convey a covered work in object code form under the terms of sections 4 and 5, provided that you also convey the machine-readable Corresponding Source under the terms of this License, in one of these ways: a) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by the Corresponding Source fixed on a durable physical medium customarily used for software interchange. b) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by a written offer, valid for at least three years and valid for as long as you offer spare parts or customer support for that product model, to give anyone who possesses the object code either (1) a copy of the Corresponding Source for all the software in the product that is covered by this License, on a durable physical medium customarily used for software interchange, for a price no more than your reasonable cost of physically performing this conveying of source, or (2) access to copy the Corresponding Source from a network server at no charge. c) Convey individual copies of the object code with a copy of the written offer to provide the Corresponding Source. This alternative is allowed only occasionally and noncommercially, and only if you received the object code with such an offer, in accord with subsection 6b. d) Convey the object code by offering access from a designated place (gratis or for a charge), and offer equivalent access to the Corresponding Source in the same way through the same place at no further charge. You need not require recipients to copy the Corresponding Source along with the object code. If the place to copy the object code is a network server, the Corresponding Source may be on a different server (operated by you or a third party) that supports equivalent copying facilities, provided you maintain clear directions next to the object code saying where to find the Corresponding Source. Regardless of what server hosts the Corresponding Source, you remain obligated to ensure that it is available for as long as needed to satisfy these requirements. e) Convey the object code using peer-to-peer transmission, provided you inform other peers where the object code and Corresponding Source of the work are being offered to the general public at no charge under subsection 6d. A separable portion of the object code, whose source code is excluded from the Corresponding Source as a System Library, need not be included in conveying the object code work. A "User Product" is either (1) a "consumer product", which means any tangible personal property which is normally used for personal, family, or household purposes, or (2) anything designed or sold for incorporation into a dwelling. In determining whether a product is a consumer product, doubtful cases shall be resolved in favor of coverage. For a particular product received by a particular user, "normally used" refers to a typical or common use of that class of product, regardless of the status of the particular user or of the way in which the particular user actually uses, or expects or is expected to use, the product. A product is a consumer product regardless of whether the product has substantial commercial, industrial or non-consumer uses, unless such uses represent the only significant mode of use of the product. "Installation Information" for a User Product means any methods, procedures, authorization keys, or other information required to install and execute modified versions of a covered work in that User Product from a modified version of its Corresponding Source. The information must suffice to ensure that the continued functioning of the modified object code is in no case prevented or interfered with solely because modification has been made. If you convey an object code work under this section in, or with, or specifically for use in, a User Product, and the conveying occurs as part of a transaction in which the right of possession and use of the User Product is transferred to the recipient in perpetuity or for a fixed term (regardless of how the transaction is characterized), the Corresponding Source conveyed under this section must be accompanied by the Installation Information. But this requirement does not apply if neither you nor any third party retains the ability to install modified object code on the User Product (for example, the work has been installed in ROM). The requirement to provide Installation Information does not include a requirement to continue to provide support service, warranty, or updates for a work that has been modified or installed by the recipient, or for the User Product in which it has been modified or installed. Access to a network may be denied when the modification itself materially and adversely affects the operation of the network or violates the rules and protocols for communication across the network. Corresponding Source conveyed, and Installation Information provided, in accord with this section must be in a format that is publicly documented (and with an implementation available to the public in source code form), and must require no special password or key for unpacking, reading or copying. 7. Additional Terms. "Additional permissions" are terms that supplement the terms of this License by making exceptions from one or more of its conditions. Additional permissions that are applicable to the entire Program shall be treated as though they were included in this License, to the extent that they are valid under applicable law. If additional permissions apply only to part of the Program, that part may be used separately under those permissions, but the entire Program remains governed by this License without regard to the additional permissions. When you convey a copy of a covered work, you may at your option remove any additional permissions from that copy, or from any part of it. (Additional permissions may be written to require their own removal in certain cases when you modify the work.) You may place additional permissions on material, added by you to a covered work, for which you have or can give appropriate copyright permission. Notwithstanding any other provision of this License, for material you add to a covered work, you may (if authorized by the copyright holders of that material) supplement the terms of this License with terms: a) Disclaiming warranty or limiting liability differently from the terms of sections 15 and 16 of this License; or b) Requiring preservation of specified reasonable legal notices or author attributions in that material or in the Appropriate Legal Notices displayed by works containing it; or c) Prohibiting misrepresentation of the origin of that material, or requiring that modified versions of such material be marked in reasonable ways as different from the original version; or d) Limiting the use for publicity purposes of names of licensors or authors of the material; or e) Declining to grant rights under trademark law for use of some trade names, trademarks, or service marks; or f) Requiring indemnification of licensors and authors of that material by anyone who conveys the material (or modified versions of it) with contractual assumptions of liability to the recipient, for any liability that these contractual assumptions directly impose on those licensors and authors. All other non-permissive additional terms are considered "further restrictions" within the meaning of section 10. If the Program as you received it, or any part of it, contains a notice stating that it is governed by this License along with a term that is a further restriction, you may remove that term. If a license document contains a further restriction but permits relicensing or conveying under this License, you may add to a covered work material governed by the terms of that license document, provided that the further restriction does not survive such relicensing or conveying. If you add terms to a covered work in accord with this section, you must place, in the relevant source files, a statement of the additional terms that apply to those files, or a notice indicating where to find the applicable terms. Additional terms, permissive or non-permissive, may be stated in the form of a separately written license, or stated as exceptions; the above requirements apply either way. 8. Termination. You may not propagate or modify a covered work except as expressly provided under this License. Any attempt otherwise to propagate or modify it is void, and will automatically terminate your rights under this License (including any patent licenses granted under the third paragraph of section 11). However, if you cease all violation of this License, then your license from a particular copyright holder is reinstated (a) provisionally, unless and until the copyright holder explicitly and finally terminates your license, and (b) permanently, if the copyright holder fails to notify you of the violation by some reasonable means prior to 60 days after the cessation. Moreover, your license from a particular copyright holder is reinstated permanently if the copyright holder notifies you of the violation by some reasonable means, this is the first time you have received notice of violation of this License (for any work) from that copyright holder, and you cure the violation prior to 30 days after your receipt of the notice. Termination of your rights under this section does not terminate the licenses of parties who have received copies or rights from you under this License. If your rights have been terminated and not permanently reinstated, you do not qualify to receive new licenses for the same material under section 10. 9. Acceptance Not Required for Having Copies. You are not required to accept this License in order to receive or run a copy of the Program. Ancillary propagation of a covered work occurring solely as a consequence of using peer-to-peer transmission to receive a copy likewise does not require acceptance. However, nothing other than this License grants you permission to propagate or modify any covered work. These actions infringe copyright if you do not accept this License. Therefore, by modifying or propagating a covered work, you indicate your acceptance of this License to do so. 10. Automatic Licensing of Downstream Recipients. Each time you convey a covered work, the recipient automatically receives a license from the original licensors, to run, modify and propagate that work, subject to this License. You are not responsible for enforcing compliance by third parties with this License. An "entity transaction" is a transaction transferring control of an organization, or substantially all assets of one, or subdividing an organization, or merging organizations. If propagation of a covered work results from an entity transaction, each party to that transaction who receives a copy of the work also receives whatever licenses to the work the party's predecessor in interest had or could give under the previous paragraph, plus a right to possession of the Corresponding Source of the work from the predecessor in interest, if the predecessor has it or can get it with reasonable efforts. You may not impose any further restrictions on the exercise of the rights granted or affirmed under this License. For example, you may not impose a license fee, royalty, or other charge for exercise of rights granted under this License, and you may not initiate litigation (including a cross-claim or counterclaim in a lawsuit) alleging that any patent claim is infringed by making, using, selling, offering for sale, or importing the Program or any portion of it. 11. Patents. A "contributor" is a copyright holder who authorizes use under this License of the Program or a work on which the Program is based. The work thus licensed is called the contributor's "contributor version". A contributor's "essential patent claims" are all patent claims owned or controlled by the contributor, whether already acquired or hereafter acquired, that would be infringed by some manner, permitted by this License, of making, using, or selling its contributor version, but do not include claims that would be infringed only as a consequence of further modification of the contributor version. For purposes of this definition, "control" includes the right to grant patent sublicenses in a manner consistent with the requirements of this License. Each contributor grants you a non-exclusive, worldwide, royalty-free patent license under the contributor's essential patent claims, to make, use, sell, offer for sale, import and otherwise run, modify and propagate the contents of its contributor version. In the following three paragraphs, a "patent license" is any express agreement or commitment, however denominated, not to enforce a patent (such as an express permission to practice a patent or covenant not to sue for patent infringement). To "grant" such a patent license to a party means to make such an agreement or commitment not to enforce a patent against the party. If you convey a covered work, knowingly relying on a patent license, and the Corresponding Source of the work is not available for anyone to copy, free of charge and under the terms of this License, through a publicly available network server or other readily accessible means, then you must either (1) cause the Corresponding Source to be so available, or (2) arrange to deprive yourself of the benefit of the patent license for this particular work, or (3) arrange, in a manner consistent with the requirements of this License, to extend the patent license to downstream recipients. "Knowingly relying" means you have actual knowledge that, but for the patent license, your conveying the covered work in a country, or your recipient's use of the covered work in a country, would infringe one or more identifiable patents in that country that you have reason to believe are valid. If, pursuant to or in connection with a single transaction or arrangement, you convey, or propagate by procuring conveyance of, a covered work, and grant a patent license to some of the parties receiving the covered work authorizing them to use, propagate, modify or convey a specific copy of the covered work, then the patent license you grant is automatically extended to all recipients of the covered work and works based on it. A patent license is "discriminatory" if it does not include within the scope of its coverage, prohibits the exercise of, or is conditioned on the non-exercise of one or more of the rights that are specifically granted under this License. You may not convey a covered work if you are a party to an arrangement with a third party that is in the business of distributing software, under which you make payment to the third party based on the extent of your activity of conveying the work, and under which the third party grants, to any of the parties who would receive the covered work from you, a discriminatory patent license (a) in connection with copies of the covered work conveyed by you (or copies made from those copies), or (b) primarily for and in connection with specific products or compilations that contain the covered work, unless you entered into that arrangement, or that patent license was granted, prior to 28 March 2007. Nothing in this License shall be construed as excluding or limiting any implied license or other defenses to infringement that may otherwise be available to you under applicable patent law. 12. No Surrender of Others' Freedom. If conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot convey a covered work so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may not convey it at all. For example, if you agree to terms that obligate you to collect a royalty for further conveying from those to whom you convey the Program, the only way you could satisfy both those terms and this License would be to refrain entirely from conveying the Program. 13. Use with the GNU Affero General Public License. Notwithstanding any other provision of this License, you have permission to link or combine any covered work with a work licensed under version 3 of the GNU Affero General Public License into a single combined work, and to convey the resulting work. The terms of this License will continue to apply to the part which is the covered work, but the special requirements of the GNU Affero General Public License, section 13, concerning interaction through a network will apply to the combination as such. 14. Revised Versions of this License. The Free Software Foundation may publish revised and/or new versions of the GNU General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. Each version is given a distinguishing version number. If the Program specifies that a certain numbered version of the GNU General Public License "or any later version" applies to it, you have the option of following the terms and conditions either of that numbered version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of the GNU General Public License, you may choose any version ever published by the Free Software Foundation. If the Program specifies that a proxy can decide which future versions of the GNU General Public License can be used, that proxy's public statement of acceptance of a version permanently authorizes you to choose that version for the Program. Later license versions may give you additional or different permissions. However, no additional obligations are imposed on any author or copyright holder as a result of your choosing to follow a later version. 15. Disclaimer of Warranty. THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 16. Limitation of Liability. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. 17. Interpretation of Sections 15 and 16. If the disclaimer of warranty and limitation of liability provided above cannot be given local legal effect according to their terms, reviewing courts shall apply local law that most closely approximates an absolute waiver of all civil liability in connection with the Program, unless a warranty or assumption of liability accompanies a copy of the Program in return for a fee. END OF TERMS AND CONDITIONS How to Apply These Terms to Your New Programs If you develop a new program, and you want it to be of the greatest possible use to the public, the best way to achieve this is to make it free software which everyone can redistribute and change under these terms. To do so, attach the following notices to the program. It is safest to attach them to the start of each source file to most effectively state the exclusion of warranty; and each file should have at least the "copyright" line and a pointer to where the full notice is found. Copyright (C) This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see . Also add information on how to contact you by electronic and paper mail. If the program does terminal interaction, make it output a short notice like this when it starts in an interactive mode: Copyright (C) This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. This is free software, and you are welcome to redistribute it under certain conditions; type `show c' for details. The hypothetical commands `show w' and `show c' should show the appropriate parts of the General Public License. Of course, your program's commands might be different; for a GUI interface, you would use an "about box". You should also get your employer (if you work as a programmer) or school, if any, to sign a "copyright disclaimer" for the program, if necessary. For more information on this, and how to apply and follow the GNU GPL, see . The GNU General Public License does not permit incorporating your program into proprietary programs. If your program is a subroutine library, you may consider it more useful to permit linking proprietary applications with the library. If this is what you want to do, use the GNU Lesser General Public License instead of this License. But first, please read . xrootd-5.6.9/src/XrdCeph/COPYING.BSD000066400000000000000000000053731457266313600166650ustar00rootroot00000000000000******************************************************************************** *Prior to September 2nd, 2012 the XRootD software suite was licensed under a * *modified BSD license shown below. This applies to all code that was in the * *XRootD git repository prior to that date. All code is now licensed under LGPL.* * See files LICENSE, COPYING.LGPL, and COPYING for license details. * ******************************************************************************** Copyright (c) 2005-2012, Board of Trustees of the Leland Stanford, Jr. University. Produced under contract DE-AC02-76-SF00515 with the US Department of Energy. All rights reserved. Conditions of Use Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: a. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. b. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. c. Neither the name of the Leland Stanford, Jr. University nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. d. Products derived from this software that do not adhere to the xrootd or cmsd protocol specifications may not use the acronyms 'cmsd', 'Scalla', 'xroot', and 'xrootd', regardless of capitalization, to describe such derivative works. DISCLAIMER THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 'AS IS' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. xrootd-5.6.9/src/XrdCeph/COPYING.LGPL000066400000000000000000000167431457266313600170160ustar00rootroot00000000000000 GNU LESSER GENERAL PUBLIC LICENSE Version 3, 29 June 2007 Copyright (C) 2007 Free Software Foundation, Inc. Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. This version of the GNU Lesser General Public License incorporates the terms and conditions of version 3 of the GNU General Public License, supplemented by the additional permissions listed below. 0. Additional Definitions. As used herein, "this License" refers to version 3 of the GNU Lesser General Public License, and the "GNU GPL" refers to version 3 of the GNU General Public License. "The Library" refers to a covered work governed by this License, other than an Application or a Combined Work as defined below. An "Application" is any work that makes use of an interface provided by the Library, but which is not otherwise based on the Library. Defining a subclass of a class defined by the Library is deemed a mode of using an interface provided by the Library. A "Combined Work" is a work produced by combining or linking an Application with the Library. The particular version of the Library with which the Combined Work was made is also called the "Linked Version". The "Minimal Corresponding Source" for a Combined Work means the Corresponding Source for the Combined Work, excluding any source code for portions of the Combined Work that, considered in isolation, are based on the Application, and not on the Linked Version. The "Corresponding Application Code" for a Combined Work means the object code and/or source code for the Application, including any data and utility programs needed for reproducing the Combined Work from the Application, but excluding the System Libraries of the Combined Work. 1. Exception to Section 3 of the GNU GPL. You may convey a covered work under sections 3 and 4 of this License without being bound by section 3 of the GNU GPL. 2. Conveying Modified Versions. If you modify a copy of the Library, and, in your modifications, a facility refers to a function or data to be supplied by an Application that uses the facility (other than as an argument passed when the facility is invoked), then you may convey a copy of the modified version: a) under this License, provided that you make a good faith effort to ensure that, in the event an Application does not supply the function or data, the facility still operates, and performs whatever part of its purpose remains meaningful, or b) under the GNU GPL, with none of the additional permissions of this License applicable to that copy. 3. Object Code Incorporating Material from Library Header Files. The object code form of an Application may incorporate material from a header file that is part of the Library. You may convey such object code under terms of your choice, provided that, if the incorporated material is not limited to numerical parameters, data structure layouts and accessors, or small macros, inline functions and templates (ten or fewer lines in length), you do both of the following: a) Give prominent notice with each copy of the object code that the Library is used in it and that the Library and its use are covered by this License. b) Accompany the object code with a copy of the GNU GPL and this license document. 4. Combined Works. You may convey a Combined Work under terms of your choice that, taken together, effectively do not restrict modification of the portions of the Library contained in the Combined Work and reverse engineering for debugging such modifications, if you also do each of the following: a) Give prominent notice with each copy of the Combined Work that the Library is used in it and that the Library and its use are covered by this License. b) Accompany the Combined Work with a copy of the GNU GPL and this license document. c) For a Combined Work that displays copyright notices during execution, include the copyright notice for the Library among these notices, as well as a reference directing the user to the copies of the GNU GPL and this license document. d) Do one of the following: 0) Convey the Minimal Corresponding Source under the terms of this License, and the Corresponding Application Code in a form suitable for, and under terms that permit, the user to recombine or relink the Application with a modified version of the Linked Version to produce a modified Combined Work, in the manner specified by section 6 of the GNU GPL for conveying Corresponding Source. 1) Use a suitable shared library mechanism for linking with the Library. A suitable mechanism is one that (a) uses at run time a copy of the Library already present on the user's computer system, and (b) will operate properly with a modified version of the Library that is interface-compatible with the Linked Version. e) Provide Installation Information, but only if you would otherwise be required to provide such information under section 6 of the GNU GPL, and only to the extent that such information is necessary to install and execute a modified version of the Combined Work produced by recombining or relinking the Application with a modified version of the Linked Version. (If you use option 4d0, the Installation Information must accompany the Minimal Corresponding Source and Corresponding Application Code. If you use option 4d1, you must provide the Installation Information in the manner specified by section 6 of the GNU GPL for conveying Corresponding Source.) 5. Combined Libraries. You may place library facilities that are a work based on the Library side by side in a single library together with other library facilities that are not Applications and are not covered by this License, and convey such a combined library under terms of your choice, if you do both of the following: a) Accompany the combined library with a copy of the same work based on the Library, uncombined with any other library facilities, conveyed under the terms of this License. b) Give prominent notice with the combined library that part of it is a work based on the Library, and explaining where to find the accompanying uncombined form of the same work. 6. Revised Versions of the GNU Lesser General Public License. The Free Software Foundation may publish revised and/or new versions of the GNU Lesser General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. Each version is given a distinguishing version number. If the Library as you received it specifies that a certain numbered version of the GNU Lesser General Public License "or any later version" applies to it, you have the option of following the terms and conditions either of that published version or of any later version published by the Free Software Foundation. If the Library as you received it does not specify a version number of the GNU Lesser General Public License, you may choose any version of the GNU Lesser General Public License ever published by the Free Software Foundation. If the Library as you received it specifies that a proxy can decide whether future versions of the GNU Lesser General Public License shall apply, that proxy's public statement of acceptance of any version is permanent authorization for you to choose that version for the Library. xrootd-5.6.9/src/XrdCeph/Doxyfile000066400000000000000000000256441457266313600167340ustar00rootroot00000000000000# Doxyfile 1.3.7 #--------------------------------------------------------------------------- # Project related configuration options #--------------------------------------------------------------------------- PROJECT_NAME = xrootd PROJECT_NUMBER = OUTPUT_DIRECTORY = doxydoc CREATE_SUBDIRS = NO OUTPUT_LANGUAGE = English USE_WINDOWS_ENCODING = NO BRIEF_MEMBER_DESC = YES REPEAT_BRIEF = YES ABBREVIATE_BRIEF = ALWAYS_DETAILED_SEC = NO INLINE_INHERITED_MEMB = NO FULL_PATH_NAMES = NO STRIP_FROM_PATH = STRIP_FROM_INC_PATH = SHORT_NAMES = NO JAVADOC_AUTOBRIEF = NO MULTILINE_CPP_IS_BRIEF = NO DETAILS_AT_TOP = NO INHERIT_DOCS = YES DISTRIBUTE_GROUP_DOC = NO TAB_SIZE = 8 ALIASES = OPTIMIZE_OUTPUT_FOR_C = NO OPTIMIZE_OUTPUT_JAVA = NO SUBGROUPING = YES #--------------------------------------------------------------------------- # Build related configuration options #--------------------------------------------------------------------------- EXTRACT_ALL = YES EXTRACT_PRIVATE = YES EXTRACT_STATIC = YES EXTRACT_LOCAL_CLASSES = YES EXTRACT_LOCAL_METHODS = NO HIDE_UNDOC_MEMBERS = NO HIDE_UNDOC_CLASSES = NO HIDE_FRIEND_COMPOUNDS = NO HIDE_IN_BODY_DOCS = NO INTERNAL_DOCS = NO CASE_SENSE_NAMES = YES HIDE_SCOPE_NAMES = NO SHOW_INCLUDE_FILES = YES INLINE_INFO = YES SORT_MEMBER_DOCS = YES SORT_BRIEF_DOCS = NO SORT_BY_SCOPE_NAME = NO GENERATE_TODOLIST = YES GENERATE_TESTLIST = YES GENERATE_BUGLIST = YES GENERATE_DEPRECATEDLIST= YES ENABLED_SECTIONS = MAX_INITIALIZER_LINES = 30 SHOW_USED_FILES = YES #--------------------------------------------------------------------------- # configuration options related to warning and progress messages #--------------------------------------------------------------------------- QUIET = NO WARNINGS = YES WARN_IF_UNDOCUMENTED = YES WARN_IF_DOC_ERROR = YES WARN_FORMAT = "$file:$line: $text" WARN_LOGFILE = #--------------------------------------------------------------------------- # configuration options related to the input files #--------------------------------------------------------------------------- INPUT = \ src/XrdSec/XrdSecEntity.hh \ src/XrdSec/XrdSecInterface.hh \ src/Xrd/XrdJob.hh \ src/Xrd/XrdBuffer.hh \ src/Xrd/XrdScheduler.hh \ src/Xrd/XrdLink.hh \ src/Xrd/XrdLinkMatch.hh \ src/Xrd/XrdProtocol.hh \ src/XrdXrootd/XrdXrootdMonData.hh \ src/XrdVersionPlugin.hh \ src/XrdCks/XrdCksData.hh \ src/XrdCks/XrdCksManager.hh \ src/XrdCks/XrdCksCalc.hh \ src/XrdCks/XrdCks.hh \ src/XrdSfs/XrdSfsInterface.hh \ src/XrdSfs/XrdSfsAio.hh \ src/XrdNet/XrdNet.hh \ src/XrdNet/XrdNetCmsNotify.hh \ src/XrdNet/XrdNetConnect.hh \ src/XrdNet/XrdNetOpts.hh \ src/XrdNet/XrdNetSocket.hh \ src/XrdSys/XrdSysError.hh \ src/XrdSys/XrdSysPlatform.hh \ src/XrdSys/XrdSysLogger.hh \ src/XrdSys/XrdSysPthread.hh \ src/XrdSys/XrdSysTimer.hh \ src/XrdSys/XrdSysHeaders.hh \ src/XrdSys/XrdSysDNS.hh \ src/XrdSys/XrdSysXSLock.hh \ src/XrdSys/XrdSysIOEvents.hh \ src/XrdSys/XrdSysAtomics.hh \ src/XrdSys/XrdSysPlugin.hh \ src/XrdSys/XrdSysSemWait.hh \ src/XrdClient/XrdClientConst.hh \ src/XrdClient/XrdClientVector.hh \ src/XrdClient/XrdClientAbs.hh \ src/XrdClient/XrdClientAbsMonIntf.hh \ src/XrdClient/XrdClient.hh \ src/XrdClient/XrdClientUnsolMsg.hh \ src/XrdClient/XrdClientAdmin.hh \ src/XrdClient/XrdClientUrlSet.hh \ src/XrdClient/XrdClientUrlInfo.hh \ src/XrdClient/XrdClientEnv.hh \ src/XrdOuc/XrdOucRash.hh \ src/XrdOuc/XrdOucStream.hh \ src/XrdOuc/XrdOuca2x.hh \ src/XrdOuc/XrdOucTrace.hh \ src/XrdOuc/XrdOucCRC.hh \ src/XrdOuc/XrdOucErrInfo.hh \ src/XrdOuc/XrdOucDLlist.hh \ src/XrdOuc/XrdOucCache.hh \ src/XrdOuc/XrdOucTList.hh \ src/XrdOuc/XrdOucName2Name.hh \ src/XrdOuc/XrdOucTable.hh \ src/XrdOuc/XrdOucIOVec.hh \ src/XrdOuc/XrdOucCallBack.hh \ src/XrdOuc/XrdOucEnum.hh \ src/XrdOuc/XrdOucLock.hh \ src/XrdOuc/XrdOucTokenizer.hh \ src/XrdOuc/XrdOucEnv.hh \ src/XrdOuc/XrdOucString.hh \ src/XrdOuc/XrdOucChain.hh \ src/XrdOuc/XrdOucUtils.hh \ src/XrdOuc/XrdOucHash.hh \ src/XrdOss/XrdOss.hh \ src/XrdOss/XrdOssStatInfo.hh \ src/XrdOss/XrdOssDefaultSS.hh \ src/XrdPosix/XrdPosixXrootd.hh \ src/XrdPosix/XrdPosixExtern.hh \ src/XrdPosix/XrdPosixXrootdPath.hh \ src/XrdPosix/XrdPosixOsDep.hh \ src/XrdPosix/XrdPosixCallBack.hh \ src/XrdAcc/XrdAccAuthorize.hh \ src/XrdAcc/XrdAccPrivs.hh \ src/XProtocol/XPtypes.hh \ src/XProtocol/XProtocol.hh \ src/XrdCms/XrdCmsClient.hh \ src/XrdCl/XrdClEnv.hh \ src/XrdCl/XrdClPostMaster.hh \ src/XrdCl/XrdClFileSystem.hh \ src/XrdCl/XrdClPostMasterInterfaces.hh \ src/XrdCl/XrdClBuffer.hh \ src/XrdCl/XrdClConstants.hh \ src/XrdCl/XrdClCopyProcess.hh \ src/XrdCl/XrdClDefaultEnv.hh \ src/XrdCl/XrdClMessage.hh \ src/XrdCl/XrdClMonitor.hh \ src/XrdCl/XrdClStatus.hh \ src/XrdCl/XrdClTransportManager.hh \ src/XrdCl/XrdClURL.hh \ src/XrdCl/XrdClAnyObject.hh \ src/XrdCl/XrdClXRootDResponses.hh \ src/XrdCl/XrdClFile.hh \ src/XrdFileCache/XrdFileCacheDecision.hh src/XrdFileCache/XrdFileCacheFile.hh src/XrdFileCache/XrdFileCache.hh src/XrdFileCache/XrdFileCacheInfo.hh src/XrdFileCache/XrdFileCacheIOEntireFile.hh src/XrdFileCache/XrdFileCacheIOFileBlock.hh src/XrdFileCache/XrdFileCacheIO.hh src/XrdFileCache/XrdFileCachePrint.hh src/XrdFileCache/XrdFileCacheStats.hh src/XrdFileCache/XrdFileCacheTrace.hh FILE_PATTERNS = *.hh RECURSIVE = YES EXCLUDE = EXCLUDE_SYMLINKS = NO EXCLUDE_PATTERNS = EXAMPLE_PATH = EXAMPLE_PATTERNS = EXAMPLE_RECURSIVE = NO IMAGE_PATH = INPUT_FILTER = FILTER_SOURCE_FILES = NO #--------------------------------------------------------------------------- # configuration options related to source browsing #--------------------------------------------------------------------------- SOURCE_BROWSER = NO INLINE_SOURCES = NO STRIP_CODE_COMMENTS = YES REFERENCED_BY_RELATION = YES REFERENCES_RELATION = YES VERBATIM_HEADERS = YES #--------------------------------------------------------------------------- # configuration options related to the alphabetical class index #--------------------------------------------------------------------------- ALPHABETICAL_INDEX = NO COLS_IN_ALPHA_INDEX = 5 IGNORE_PREFIX = #--------------------------------------------------------------------------- # configuration options related to the HTML output #--------------------------------------------------------------------------- GENERATE_HTML = YES HTML_OUTPUT = html HTML_FILE_EXTENSION = .html HTML_HEADER = HTML_FOOTER = HTML_STYLESHEET = HTML_ALIGN_MEMBERS = YES GENERATE_HTMLHELP = NO CHM_FILE = HHC_LOCATION = GENERATE_CHI = NO BINARY_TOC = NO TOC_EXPAND = NO DISABLE_INDEX = NO ENUM_VALUES_PER_LINE = 4 GENERATE_TREEVIEW = NO TREEVIEW_WIDTH = 250 #--------------------------------------------------------------------------- # configuration options related to the LaTeX output #--------------------------------------------------------------------------- GENERATE_LATEX = NO LATEX_OUTPUT = latex LATEX_CMD_NAME = latex MAKEINDEX_CMD_NAME = makeindex COMPACT_LATEX = NO PAPER_TYPE = a4wide EXTRA_PACKAGES = LATEX_HEADER = PDF_HYPERLINKS = NO USE_PDFLATEX = NO LATEX_BATCHMODE = NO LATEX_HIDE_INDICES = NO #--------------------------------------------------------------------------- # configuration options related to the RTF output #--------------------------------------------------------------------------- GENERATE_RTF = NO RTF_OUTPUT = rtf COMPACT_RTF = NO RTF_HYPERLINKS = NO RTF_STYLESHEET_FILE = RTF_EXTENSIONS_FILE = #--------------------------------------------------------------------------- # configuration options related to the man page output #--------------------------------------------------------------------------- GENERATE_MAN = NO MAN_OUTPUT = man MAN_EXTENSION = .3 MAN_LINKS = NO #--------------------------------------------------------------------------- # configuration options related to the XML output #--------------------------------------------------------------------------- GENERATE_XML = NO XML_OUTPUT = xml XML_SCHEMA = XML_DTD = XML_PROGRAMLISTING = YES #--------------------------------------------------------------------------- # configuration options for the AutoGen Definitions output #--------------------------------------------------------------------------- GENERATE_AUTOGEN_DEF = NO #--------------------------------------------------------------------------- # configuration options related to the Perl module output #--------------------------------------------------------------------------- GENERATE_PERLMOD = NO PERLMOD_LATEX = NO PERLMOD_PRETTY = YES PERLMOD_MAKEVAR_PREFIX = #--------------------------------------------------------------------------- # Configuration options related to the preprocessor #--------------------------------------------------------------------------- ENABLE_PREPROCESSING = YES MACRO_EXPANSION = NO EXPAND_ONLY_PREDEF = NO SEARCH_INCLUDES = YES INCLUDE_PATH = INCLUDE_FILE_PATTERNS = PREDEFINED = EXPAND_AS_DEFINED = SKIP_FUNCTION_MACROS = YES #--------------------------------------------------------------------------- # Configuration::additions related to external references #--------------------------------------------------------------------------- TAGFILES = GENERATE_TAGFILE = ALLEXTERNALS = NO EXTERNAL_GROUPS = YES PERL_PATH = /usr/bin/perl #--------------------------------------------------------------------------- # Configuration options related to the dot tool #--------------------------------------------------------------------------- CLASS_DIAGRAMS = YES HIDE_UNDOC_RELATIONS = YES HAVE_DOT = YES CLASS_GRAPH = YES COLLABORATION_GRAPH = YES UML_LOOK = NO TEMPLATE_RELATIONS = NO INCLUDE_GRAPH = YES INCLUDED_BY_GRAPH = YES CALL_GRAPH = NO GRAPHICAL_HIERARCHY = YES DOT_IMAGE_FORMAT = png DOT_PATH = DOTFILE_DIRS = MAX_DOT_GRAPH_WIDTH = 1024 MAX_DOT_GRAPH_HEIGHT = 1024 MAX_DOT_GRAPH_DEPTH = 0 GENERATE_LEGEND = YES DOT_CLEANUP = YES #--------------------------------------------------------------------------- # Configuration::additions related to the search engine #--------------------------------------------------------------------------- SEARCHENGINE = NO xrootd-5.6.9/src/XrdCeph/LICENSE000066400000000000000000000026301457266313600162210ustar00rootroot00000000000000"Copyright (c) 2005-2012, Board of Trustees of the Leland Stanford, Jr. University.\n" "Produced under contract DE-AC02-76-SF00515 with the US Department of Energy. \n" "All rights reserved. The copyright holder's institutional names may not be used to\n" "endorse or promote products derived from this software without specific prior \n" "written permission.\n\n" "This file is part of the XRootD software suite. \n\n" "XRootD is free software: you can redistribute it and/or modify it under the terms \n" "of the GNU Lesser General Public License as published by the Free Software \n" "Foundation, either version 3 of the License, or (at your option) any later version.\n\n" "XRootD is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;\n" "without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR \n" "PURPOSE. See the GNU Lesser General Public License for more details. \nn" "You should have received a copy of the GNU Lesser General Public License along \n" "with XRootD in a file called COPYING.LESSER (LGPL license) and file COPYING (GPL \n" "license). If not, see .\n\n" "Prior to September 2nd, 2012 the XRootD software suite was licensed under a\n" "modified BSD license (see file COPYING.BSD). This applies to all code that\n" "was in the XRootD git repository prior to that date.\n" xrootd-5.6.9/src/XrdCeph/README000066400000000000000000000031321457266313600160720ustar00rootroot00000000000000 -------------------------------------------------------------------------------- _ _ ______ _____ \ \ / (_____ \ _ (____ \ \ \/ / _____) ) ___ ___ | |_ _ \ \ ) ( (_____ ( / _ \ / _ \| _)| | | | / /\ \ | | |_| | |_| | |__| |__/ / /_/ \_\ |_|\___/ \___/ \___)_____/ -------------------------------------------------------------------------------- 0. xrootd-ceph is a OSS layer XRootD plug-in for interfacing with Ceph storage platform. The plug-in has to be build against respective Ceph version, the repository can be found at: https://download.ceph.com/rpm-{ceph-release}/{distro}/$basearch 1. S U P P O R T E D O P E R A T I N G S Y S T E M S XRootD is supported on the following platforms: * RedHat Enterprise Linux 7 and derivatives (Scientific Linux) compiled with gcc 2. B U I L D I N S T R U C T I O N S 2.1 Build system xrootd-ceph uses CMake to handle the build process. Please use CMake version 3 or greater (e.g. cmake3). 2.2 Build steps * Create an empty build directory: mkdir build cd build * Ensure that the correct plugin version number is set in cmake/XRootDDefaults.cmake: if( NOT XRDCEPH_SUBMODULE ) define_default( PLUGIN_VERSION 5 ) endif() * Generate the build system files using cmake, ie: cmake /path/to/the/xrootd-ceph/source -DCMAKE_INSTALL_PREFIX=/opt/xrootd * Build the source: make * Install the shared libraries: make install xrootd-5.6.9/src/XrdCeph/VERSION_INFO000066400000000000000000000001741457266313600171000ustar00rootroot00000000000000RefNames: (HEAD -> master, tag: v5.6.9, gitlab/master, amadio/master) ShortHash: 619e93fe8 Date: 2024-03-08 17:31:28 +0100 xrootd-5.6.9/src/XrdCeph/cmake/000077500000000000000000000000001457266313600162735ustar00rootroot00000000000000xrootd-5.6.9/src/XrdCeph/cmake/FindCppUnit.cmake000066400000000000000000000012671457266313600214660ustar00rootroot00000000000000# Try to find CPPUnit # Once done, this will define # # CPPUNIT_FOUND - system has cppunit # CPPUNIT_INCLUDE_DIRS - the cppunit include directories # CPPUNIT_LIBRARIES - cppunit libraries directories find_path( CPPUNIT_INCLUDE_DIRS cppunit/ui/text/TestRunner.h HINTS ${CPPUNIT_DIR} $ENV{CPPUNIT_DIR} /usr /opt PATH_SUFFIXES include ) find_library( CPPUNIT_LIBRARIES cppunit HINTS ${CPPUNIT_DIR} $ENV{CPPUNIT_DIR} /usr /opt PATH_SUFFIXES lib ) set(CPPUNIT_INCLUDE_DIRS ${CPPUNIT_INCLUDE_DIR}) set(CPPUNIT_LIBRARIES ${CPPUNIT_LIBRARY}) include(FindPackageHandleStandardArgs) find_package_handle_standard_args(CppUnit DEFAULT_MSG CPPUNIT_INCLUDE_DIRS CPPUNIT_LIBRARIES) xrootd-5.6.9/src/XrdCeph/cmake/FindXRootD.cmake000066400000000000000000000015101457266313600212520ustar00rootroot00000000000000# Try to find XRootD # Once done, this will define # # XROOTD_FOUND - system has xrootd # XROOTD_INCLUDE_DIRS - the xrootd include directories # XROOTD_LIBRARIES - xrootd libraries directories if( XRDCEPH_SUBMODULE ) set( XROOTD_INCLUDE_DIRS ${CMAKE_SOURCE_DIR}/src ) set( XROOTD_LIBRARIES XrdUtils ) else() find_path( XROOTD_INCLUDE_DIRS XrdSfs/XrdSfsAio.hh HINTS ${XROOTD_DIR} $ENV{XROOTD_DIR} /usr /opt PATH_SUFFIXES include/xrootd ) find_library( XROOTD_LIBRARIES XrdUtils HINTS ${XROOTD_DIR} $ENV{XROOTD_DIR} /usr /opt PATH_SUFFIXES lib ) endif() set(XROOTD_INCLUDE_DIR ${XROOTD_INCLUDE_DIRS}) set(XROOTD_LIBRARY ${XROOTD_LIBRARIES}) include(FindPackageHandleStandardArgs) find_package_handle_standard_args(XRootD DEFAULT_MSG XROOTD_INCLUDE_DIRS XROOTD_LIBRARIES) xrootd-5.6.9/src/XrdCeph/cmake/Findceph.cmake000066400000000000000000000013451457266313600210200ustar00rootroot00000000000000# - Find ceph # # RADOS_INCLUDE_DIR - location of the ceph-devel header files for rados # RADOS_LIBS - list of rados libraries, with full path # RADOS_FOUND find_path( RADOS_INCLUDE_DIR radosstriper/libradosstriper.hpp HINTS ${CEPH_DIR} $ENV{CEPH_DIR} /usr /opt PATH_SUFFIXES include ) find_library( RADOSSTRIPER_LIB NAMES radosstriper HINTS ${CEPH_DIR} $ENV{CEPH_DIR} /usr /opt PATH_SUFFIXES lib ) find_library( RADOS_LIB NAMES rados HINTS ${CEPH_DIR} $ENV{CEPH_DIR} /usr /opt PATH_SUFFIXES lib ) set(RADOS_LIBS ${RADOS_LIB} ${RADOSSTRIPER_LIB}) include(FindPackageHandleStandardArgs) find_package_handle_standard_args(ceph DEFAULT_MSG RADOS_INCLUDE_DIR RADOS_LIBS) xrootd-5.6.9/src/XrdCeph/cmake/GNUInstallDirs.cmake000066400000000000000000000155571457266313600221140ustar00rootroot00000000000000# - Define GNU standard installation directories # Provides install directory variables as defined for GNU software: # http://www.gnu.org/prep/standards/html_node/Directory-Variables.html # Inclusion of this module defines the following variables: # CMAKE_INSTALL_ - destination for files of a given type # CMAKE_INSTALL_FULL_ - corresponding absolute path # where is one of: # BINDIR - user executables (bin) # SBINDIR - system admin executables (sbin) # LIBEXECDIR - program executables (libexec) # SYSCONFDIR - read-only single-machine data (etc) # SHAREDSTATEDIR - modifiable architecture-independent data (com) # LOCALSTATEDIR - modifiable single-machine data (var) # LIBDIR - object code libraries (lib or lib64) # INCLUDEDIR - C header files (include) # OLDINCLUDEDIR - C header files for non-gcc (/usr/include) # DATAROOTDIR - read-only architecture-independent data root (share) # DATADIR - read-only architecture-independent data (DATAROOTDIR) # INFODIR - info documentation (DATAROOTDIR/info) # LOCALEDIR - locale-dependent data (DATAROOTDIR/locale) # MANDIR - man documentation (DATAROOTDIR/man) # DOCDIR - documentation root (DATAROOTDIR/doc/PROJECT_NAME) # Each CMAKE_INSTALL_ value may be passed to the DESTINATION options of # install() commands for the corresponding file type. If the includer does # not define a value the above-shown default will be used and the value will # appear in the cache for editing by the user. # Each CMAKE_INSTALL_FULL_ value contains an absolute path constructed # from the corresponding destination by prepending (if necessary) the value # of CMAKE_INSTALL_PREFIX. #============================================================================= # Copyright 2011 Nikita Krupen'ko # Copyright 2011 Kitware, Inc. # # Distributed under the OSI-approved BSD License (the "License"); # see accompanying file Copyright.txt for details. # # This software is distributed WITHOUT ANY WARRANTY; without even the # implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. # See the License for more information. #============================================================================= # (To distribute this file outside of CMake, substitute the full # License text for the above reference.) # Installation directories # if(NOT DEFINED CMAKE_INSTALL_BINDIR) set(CMAKE_INSTALL_BINDIR "bin" CACHE PATH "user executables (bin)") endif() if(NOT DEFINED CMAKE_INSTALL_SBINDIR) set(CMAKE_INSTALL_SBINDIR "sbin" CACHE PATH "system admin executables (sbin)") endif() if(NOT DEFINED CMAKE_INSTALL_LIBEXECDIR) set(CMAKE_INSTALL_LIBEXECDIR "libexec" CACHE PATH "program executables (libexec)") endif() if(NOT DEFINED CMAKE_INSTALL_SYSCONFDIR) set(CMAKE_INSTALL_SYSCONFDIR "etc" CACHE PATH "read-only single-machine data (etc)") endif() if(NOT DEFINED CMAKE_INSTALL_SHAREDSTATEDIR) set(CMAKE_INSTALL_SHAREDSTATEDIR "com" CACHE PATH "modifiable architecture-independent data (com)") endif() if(NOT DEFINED CMAKE_INSTALL_LOCALSTATEDIR) set(CMAKE_INSTALL_LOCALSTATEDIR "var" CACHE PATH "modifiable single-machine data (var)") endif() if(NOT DEFINED CMAKE_INSTALL_LIBDIR) set(_LIBDIR_DEFAULT "lib") # Override this default 'lib' with 'lib64' iff: # - we are on Linux system but NOT cross-compiling # - we are NOT on debian # - we are on a 64 bits system # reason is: amd64 ABI: http://www.x86-64.org/documentation/abi.pdf # Note that the future of multi-arch handling may be even # more complicated than that: http://wiki.debian.org/Multiarch if(CMAKE_SYSTEM_NAME MATCHES "Linux" AND NOT CMAKE_CROSSCOMPILING AND NOT EXISTS "/etc/debian_version") if(NOT DEFINED CMAKE_SIZEOF_VOID_P) message(AUTHOR_WARNING "Unable to determine default CMAKE_INSTALL_LIBDIR directory because no target architecture is known. " "Please enable at least one language before including GNUInstallDirs.") else() if("${CMAKE_SIZEOF_VOID_P}" EQUAL "8") set(_LIBDIR_DEFAULT "lib64") endif() endif() endif() set(CMAKE_INSTALL_LIBDIR "${_LIBDIR_DEFAULT}" CACHE PATH "object code libraries (${_LIBDIR_DEFAULT})") endif() if(NOT DEFINED CMAKE_INSTALL_INCLUDEDIR) set(CMAKE_INSTALL_INCLUDEDIR "include" CACHE PATH "C header files (include)") endif() if(NOT DEFINED CMAKE_INSTALL_OLDINCLUDEDIR) set(CMAKE_INSTALL_OLDINCLUDEDIR "/usr/include" CACHE PATH "C header files for non-gcc (/usr/include)") endif() if(NOT DEFINED CMAKE_INSTALL_DATAROOTDIR) set(CMAKE_INSTALL_DATAROOTDIR "share" CACHE PATH "read-only architecture-independent data root (share)") endif() #----------------------------------------------------------------------------- # Values whose defaults are relative to DATAROOTDIR. Store empty values in # the cache and store the defaults in local variables if the cache values are # not set explicitly. This auto-updates the defaults as DATAROOTDIR changes. if(NOT CMAKE_INSTALL_DATADIR) set(CMAKE_INSTALL_DATADIR "" CACHE PATH "read-only architecture-independent data (DATAROOTDIR)") set(CMAKE_INSTALL_DATADIR "${CMAKE_INSTALL_DATAROOTDIR}") endif() if(NOT CMAKE_INSTALL_INFODIR) set(CMAKE_INSTALL_INFODIR "" CACHE PATH "info documentation (DATAROOTDIR/info)") set(CMAKE_INSTALL_INFODIR "${CMAKE_INSTALL_DATAROOTDIR}/info") endif() if(NOT CMAKE_INSTALL_LOCALEDIR) set(CMAKE_INSTALL_LOCALEDIR "" CACHE PATH "locale-dependent data (DATAROOTDIR/locale)") set(CMAKE_INSTALL_LOCALEDIR "${CMAKE_INSTALL_DATAROOTDIR}/locale") endif() if(NOT CMAKE_INSTALL_MANDIR) set(CMAKE_INSTALL_MANDIR "" CACHE PATH "man documentation (DATAROOTDIR/man)") set(CMAKE_INSTALL_MANDIR "${CMAKE_INSTALL_DATAROOTDIR}/man") endif() if(NOT CMAKE_INSTALL_DOCDIR) set(CMAKE_INSTALL_DOCDIR "" CACHE PATH "documentation root (DATAROOTDIR/doc/PROJECT_NAME)") set(CMAKE_INSTALL_DOCDIR "${CMAKE_INSTALL_DATAROOTDIR}/doc/${PROJECT_NAME}") endif() #----------------------------------------------------------------------------- mark_as_advanced( CMAKE_INSTALL_BINDIR CMAKE_INSTALL_SBINDIR CMAKE_INSTALL_LIBEXECDIR CMAKE_INSTALL_SYSCONFDIR CMAKE_INSTALL_SHAREDSTATEDIR CMAKE_INSTALL_LOCALSTATEDIR CMAKE_INSTALL_LIBDIR CMAKE_INSTALL_INCLUDEDIR CMAKE_INSTALL_OLDINCLUDEDIR CMAKE_INSTALL_DATAROOTDIR CMAKE_INSTALL_DATADIR CMAKE_INSTALL_INFODIR CMAKE_INSTALL_LOCALEDIR CMAKE_INSTALL_MANDIR CMAKE_INSTALL_DOCDIR ) # Result directories # foreach(dir BINDIR SBINDIR LIBEXECDIR SYSCONFDIR SHAREDSTATEDIR LOCALSTATEDIR LIBDIR INCLUDEDIR OLDINCLUDEDIR DATAROOTDIR DATADIR INFODIR LOCALEDIR MANDIR DOCDIR ) if(NOT IS_ABSOLUTE ${CMAKE_INSTALL_${dir}}) set(CMAKE_INSTALL_FULL_${dir} "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_${dir}}") else() set(CMAKE_INSTALL_FULL_${dir} "${CMAKE_INSTALL_${dir}}") endif() endforeach() xrootd-5.6.9/src/XrdCeph/cmake/XRootDDefaults.cmake000066400000000000000000000010351457266313600221430ustar00rootroot00000000000000#------------------------------------------------------------------------------- # Define the default build parameters #------------------------------------------------------------------------------- if( "${CMAKE_BUILD_TYPE}" STREQUAL "" ) if( Solaris AND NOT SUNCC_CAN_DO_OPTS ) set( CMAKE_BUILD_TYPE Debug ) else() set( CMAKE_BUILD_TYPE RelWithDebInfo ) endif() endif() if( NOT XRDCEPH_SUBMODULE ) define_default( PLUGIN_VERSION 5 ) endif() define_default( ENABLE_TESTS FALSE ) define_default( ENABLE_CEPH TRUE ) xrootd-5.6.9/src/XrdCeph/cmake/XRootDFindLibs.cmake000066400000000000000000000006301457266313600220660ustar00rootroot00000000000000#------------------------------------------------------------------------------- # Find the required libraries #------------------------------------------------------------------------------- find_package( XRootD REQUIRED ) find_package( ceph REQUIRED ) if( ENABLE_TESTS ) find_package( CppUnit ) if( CPPUNIT_FOUND ) set( BUILD_TESTS TRUE ) else() set( BUILD_TESTS FALSE ) endif() endif() xrootd-5.6.9/src/XrdCeph/cmake/XRootDOSDefs.cmake000066400000000000000000000034011457266313600215160ustar00rootroot00000000000000#------------------------------------------------------------------------------- # Define the OS variables #------------------------------------------------------------------------------- include( CheckCXXSourceRuns ) add_definitions( -D_LARGEFILE_SOURCE -D_LARGEFILE64_SOURCE -D_FILE_OFFSET_BITS=64 ) set( LIBRARY_PATH_PREFIX "lib" ) #------------------------------------------------------------------------------- # GCC #------------------------------------------------------------------------------- if( CMAKE_COMPILER_IS_GNUCXX ) set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11" ) set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -Werror" ) set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-parameter" ) # gcc 4.1 is retarded execute_process( COMMAND ${CMAKE_C_COMPILER} -dumpversion OUTPUT_VARIABLE GCC_VERSION ) if( (GCC_VERSION VERSION_GREATER 4.1 OR GCC_VERSION VERSION_EQUAL 4.1) AND GCC_VERSION VERSION_LESS 4.2 ) set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-strict-aliasing" ) endif() # for 4.9.3 or greater the 'omit-frame-pointer' # interfears with custom semaphore implementation if( (GCC_VERSION VERSION_GREATER 4.9.2) AND (USE_LIBC_SEMAPHORE EQUAL 0) ) set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-omit-frame-pointer" ) endif() # gcc 6.0 is more pedantic if( GCC_VERSION VERSION_GREATER 6.0 OR GCC_VERSION VERSION_EQUAL 6.0 ) set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=misleading-indentation" ) endif() endif() #------------------------------------------------------------------------------- # Linux #------------------------------------------------------------------------------- set( Linux TRUE ) include( GNUInstallDirs ) add_definitions( -D__linux__=1 ) set( EXTRA_LIBS rt ) xrootd-5.6.9/src/XrdCeph/cmake/XRootDSummary.cmake000066400000000000000000000020471457266313600220350ustar00rootroot00000000000000#------------------------------------------------------------------------------- # Print the configuration summary #------------------------------------------------------------------------------- set( TRUE_VAR TRUE ) component_status( CEPH TRUE_VAR CEPH_FOUND ) component_status( XROOTD TRUE_VAR XROOTD_FOUND ) component_status( TESTS BUILD_TESTS CPPUNIT_FOUND ) if(CMAKE_SOURCE_DIR STREQUAL PROJECT_SOURCE_DIR) message( STATUS "----------------------------------------" ) message( STATUS "Installation path: " ${CMAKE_INSTALL_PREFIX} ) message( STATUS "C Compiler: " ${CMAKE_C_COMPILER} ) message( STATUS "C++ Compiler: " ${CMAKE_CXX_COMPILER} ) message( STATUS "Build type: " ${CMAKE_BUILD_TYPE} ) message( STATUS "Plug-in version: " ${PLUGIN_VERSION} ) message( STATUS "" ) message( STATUS "CEPH: " ${STATUS_CEPH} ) message( STATUS "XRootD: " ${STATUS_XROOTD} ) message( STATUS "Tests: " ${STATUS_TESTS} ) message( STATUS "----------------------------------------" ) endif() xrootd-5.6.9/src/XrdCeph/cmake/XRootDUtils.cmake000066400000000000000000000026111457266313600214750ustar00rootroot00000000000000 #------------------------------------------------------------------------------- # Add a compiler define flag if a variable is defined #------------------------------------------------------------------------------- macro( define_default variable value ) if( NOT DEFINED ${variable} ) set( ${variable} ${value} ) endif() endmacro() macro( component_status name flag found ) if( ${flag} AND ${found} ) set( STATUS_${name} "yes" ) elseif( ${flag} AND NOT ${found} ) set( STATUS_${name} "libs not found" ) else() set( STATUS_${name} "disabled" ) endif() endmacro() #------------------------------------------------------------------------------- # Detect in source builds #------------------------------------------------------------------------------- function( CheckBuildDirectory ) # Get Real Paths of the source and binary directories get_filename_component( srcdir "${CMAKE_SOURCE_DIR}" REALPATH ) get_filename_component( bindir "${CMAKE_BINARY_DIR}" REALPATH ) # Check for in-source builds if( ${srcdir} STREQUAL ${bindir} ) message( FATAL_ERROR "XRootD cannot be built in-source! " "Please run cmake outside the " "source directory and be sure to remove " "CMakeCache.txt or CMakeFiles if they " "exist in the source directory." ) endif() endfunction() xrootd-5.6.9/src/XrdCeph/docs/000077500000000000000000000000001457266313600161435ustar00rootroot00000000000000xrootd-5.6.9/src/XrdCeph/docs/PreReleaseNotes.txt000066400000000000000000000000721457266313600217430ustar00rootroot00000000000000====== XRootD ====== Prerelease Notes ================ xrootd-5.6.9/src/XrdCeph/docs/ReleaseNotes.txt000066400000000000000000002504651457266313600213110ustar00rootroot00000000000000====== XRootD ====== Release Notes ============= ------------- Version 4.8.0 ------------- + **New Features** * **[XrdCl]** Local redirection and local file support. * **[XrdCl]** merge xrdfs ls results if not unique, closes #541. * **[XrdCl]** Provide client specific CGI info. * **[XrdCl]** File::WriteV implementation, closes #388. * **[XrdHttp]** Pass the HTTP verb to the external handler for path matching. * **[XrdHttp]** Allow one to access the XrdSecEntity object associated with a request. * **[XrdHttp]** Allow filtering based on HTTP verb in MatchesPath. * **[XrdHttp]** Allow overwrites to be done on PUT. * **[XrdHttp]** Allow multiple external handlers to be loaded by XrdHttp. + **Major bug fixes** * **[Server]** Correctly handle monEnt on file close to avoid SEGV. Fixes #618. * **[Server]** Poperly handle file descriptors up to 65535. Fixes #607. * **[Server]** Fix handling of >65K attached files (active links). Fixes #623. * **[Server]** Make sure doPost does not become <0 (regression introduced in 4.7.1). * **[Proxy]** Avoid SEGV when localroot specified w/o remote root. Fixes #627. * **[XrdCl]** Connection Window should be applied per IP address. Fixes #625. * **[XrdCl]** Write request and raw data with single writev, fixes #609. * **[XrdHttp]** Allow XrdSfsGetDefaultFileSystem to be called multiple times. * **[XrdHttp]** Correct external handling logic. * **[XrdSecgsi]** Use stack for proper cleaning of invalidated CRLs and CAs. + **Minor bug fixes** * **[Server]** Print error msg and close socket when a FD cannot. be handled. * **[Server]** Close additional loophole for fstream disconnect. * **[Server]** Always unhook the statistcs object from xfr monitoring if hooked. * **[Server]** Ruggedize TPC to be less sensitive to protocol violations. * **[Server]** Correct tpc directive scanning and make it more obvious. Fixes #604. * **[Server]** Enable url rewrites. Eliminates GSI roadblock. * **[Server]** Do not reference a deleted object. * **[XrdSsi]** Make sure to finalyze all requests upon disc, fixes #616. * **[XrdHttp]** Handle properly http.secretkey. * **[XrdCl]** various memory releated fixes. * **[XrdPy]** Translate binary buffers into bytes objects, closes #632 + **Miscellaneous** * **[RPM]** Add python3 sub package. * **[RPM]** Rename python sub-package, closes #614. * **[Py]** Facilitate building python bindings with wheel. ------------- Version 4.7.1 ------------- + **Major bug fixes** * **[XrdSecgsi]** Fix segv in cache checking, fixes #595 * **[XrdHttp]** Fix for the persistent connection issue. * **[XrdHttp]** Fix FD leak when a socket error is encountered. * **[XrdSsi]** Avoid race condition when response is posted. * **[XrdSsi]** Avoid state conflict when request is being processed and client asks for response. * **[XrdCl]** Prevent segv in case user has no name. * **[Server]** Close link on enable errors to prevent socket leaks. + **Minor bug fixes** * **[XrdLink]** Increment the IOSemaphore once for each waiting thread. * **[XrdHttp]** Make sure that the preexisting url tokens are properly quoted when generating a redirection. * **[XrdCl]** Fix invalid memory reads/writes when RAII finalizes mutex after the object has been deleted. + **Miscellaneous** * **[XrdCl]** Log last error in case redirect limit has been reached. * **[XrdCl]** Add option to read credentials under different fsuid/fsgid. * **[XrdCl]** Accept empty login response for protocol <= 2.8.9 (This is only to ensure compatibility with dCache, which due to its inaccurate implementation of XRoot protocol in some cases returns an empty login response for protocol version <= 2.8.9.) * **[XrdCl]** Add envar to config Nagle algorithm. * **[XrdSsi]** Reinitializ response object after Finished() so it can reused. * **[XrdHttp]** Header2cgi directive. ------------- Version 4.7.0 ------------- + **New Features** * **[Proxy]** Make cache I/O synchronization tunable. * **[Proxy]** Allow caching of S3-style objects. * **[Proxy/Posix]** Allow Name2Name to populate cache using the LFN. * **[Posix]** Enable LITE feature in Posix preload library. * **[Posix]** Implement serverless file caching (disk or memory). * **[Server]** Allow storing S3-style objects in a file system. * **[Server]** Add xrootd.fsoverload directive to handle filesystem overloads. * **[Server]** Allow port to be specified for a supervisor. * **[Server]** Add org and role types to AuthDB authorization. * **[Server]** Allow definition and test of compound authorization identifiers. * **[Server/Packaging]** Handle systemd socket inheritance. * **[XrdApps]** Add XrdClProxyPlugin implementation. * **[XrdCl]** Extreme copy implementation. * **[XrdCl]** Delegate all callbacks to the thread-pool. * **[XrdCl]** xrdfs: add recursive list, closes #421. * **[XrdCeph]** Added support for namelib in ceph plugin . * **[XrdFfs]** Implement xrootdfs_create. * **[Python]** Python 3 support in C / Python interface. * **[XrdHttp]** Make XrdHTTP able to forward HTTP requests to an external, optional plugin (conceptually similar to CGI). * **[Server]** XrdSsi V2 (scalable service interface) implementation. + **Major bug fixes** * **[XrdCl]** Avoid deadlock between FSH deletion and Tick() timeout. * **[XrdCl]** Process virtual redierections in the threadpool. * **[Xrd] Fix handling of sendfile offset argument. + **Minor bug fixes** * **[Server]** Make file locking independent of FS plugin. Fixes #533 * **[Server]** Correct debug message interval for free space report. * **[XrdCeph]** Fixed internal (f)stat so that it sets S_IFREG in returned mode. * **[XrdCeph]** properly return ENOENT when file does not exist in open for read. * **[XrdCeph]** Fixed configuration of the XrdCephOss module. * **[XrdCeph]** Fixed some resource leak when Posix_Open fails. * **[XrdFfs]** Remove default fuse argument "allow_other" as it is impossible to unset. * **[XrdFfs]** Check file descriptor before using it in xrootdfs wcache. * **[XrdFfs]** Add more error checks when creating write cache. * **[XrdFfs]** Avoid using literal 1024, replace with MAXROOTURLLEN. * **[XrdFfs]** Control allow_other by env XROOTD_NOALLOWOTHER. * **[XrdFfs]** Rewrite xrootdfs_mknod, extract low-level function. * **[XrdCl]** Check login resp size, fixes #530 * **[XrdCl]** Avoid FileStateHandler deadlock while forking. * **[XrdCl]** Handle failed stateful operations without XrdCl::File lock being locked. * **[XrdPosix]** Use strncpy when copying checksum. * **[RPM]** Fix init script bad exit code, fixes #536 * **[XrdBuffer]** Decrement total buffer count when freeing buffers. + **Miscellaneous** * **[Server]** Re-enable the oss.fdlimit directive to allow POSIX preload+xrootd. * **[Server]** Avoid thread pile-up durin slow close operations. * **[Proxy]** Simplify delayed destruction on wait vs post. * **[Posix]** Convert to using universal tracing facility. * **[CI]** Add Travis CI configuration. * **[CI]** Add .gitlab-ci.yml for gitlab CI. * **[Packaging]** Add a sample XrdHttp config file. * **[Packaging]** Make RPM version configurable by the user. * **[Packaging]** Debian packaging. * **[RPM/CMake]** Enable c++0x/c++11 by default. * **[Crypto] Remove unused crypto code. * **[XrdFileCache]** Add configuration parameter for flush frequency. * **[XrdFileCache]** Alter ram limits and blocks size parameter if caching is on the client side. * **[XrdSut]** New XrdSutCache based on XrdOucHash. * **[XrdSecgsi]** do not delete explicitely the CRL in Delete. * **[XrdSut/Crypto]** Secgsi improvements: new version of XrdSutCache, lightweith locking (PR #539). ------------- Version 4.6.1 ------------- + **Major bug fixes** * **[Server/Proxy]** Avoid SEGV when close(), closedir() returns an error. * **[cmsd]** Fix feature interaction causing improper file existence to be sent. * **[XrdCrypto/XrdSecgsi]** Make sure the CRL is loaded for the right CA. * **[XrdCrypto]** Support for OpenSSL 1.1 * **[XrdSecgsi]** do not build/package libXrdSecgsiGMAPLDAP-4.so. * **[XrdSecgsi]** Improve detection of errors when loading CRL. * **[XrdSecgsi]** Fix for valid legacy proxy detection (PR #469) * **[XrdSecgsi]** Absent CRLs not an error (#465) * **[XrdSecgsi]** Fix for CA chain verification segfault (issue #463) * **[XrdSecgsi]** Two memory leaks (PR #503) * **[XrdCl]** Make sure there is no request/response mismatch, when the retry logics tries to recover from an error. * **[XrdCl/Server]** Be case insensitive when it comes to checksum names. * **[XrdCeph]** Fix ability to read back a file written with O_RDWR flags. * **[XrdCeph]** Disable logging of every read and write operation. A proper debug-level logging would be needed instead. * **[XrdCeph]** Added statistics about read/write operations in the close log. + **Minor bug fixes** * **[XrdHttp]** Make the XrdHttpSecXtractor API backwards compatible. * **[XrdFileCache]** Make caching proxy configuration backwards compatible. * **[XrdFileCache]** Fix cache v1 to cache v2 bridge after introducing cache v2. * **[XrdSec]** Use CommonCrypto header instead of openssl for SHA on OSX. * **[XrdSeckrb5]** Fix memory leaks in client context and cache. * **[Server/Logrotate]** Make sure XRootD logrotate does not interfire with system logrotate, fixes #490 * ** [Server]** Avoid std::ABORT should a naked logfile path be specified. * **[XrdCl]** Make sure ForkHandler doesn't segv if PostMaster is null, fixes #489 * **[Packaging]** Set the working dir to /var/spool/xrootd on CC7, fixes #365 * **[Packaging]** On platforms where systemd is available, manage files in /var/run with tmpfiles.d, fixes #485 + **Miscellaneous** * **[XrdPosix]** Add new minpages option to pss.cache to support large pages. * **[XrdPosix]** Make XrdPosix.hh a public header; closes #479 * **[XrdApps]** Remove XrdClient dependency from xrdadler32. * **[Server]** Add XrdCksAssist functions to help handle XRootD checksums. * **[Server/Proxy]** Move disk sync operations out of IO::ioActive() call. * **[Server/Proxy]** Change severity IO::initLocalStat() log message. * **[XrdFileCache]** Ease development of decision plugins. * **[XrdFileCache]** Use ref-counts on File objects. ------------- Version 4.6.0 ------------- + **New Features** * **[XrdCms]** Add non-blocking sends to avoid slow links. * **[XrdFileCache]** File caching proxy V2 (and new pss async interface). + **Major bug fixes** * **[XrdCeph]** Account for return Ceph xattr return codes. * **[XrdCeph]** Fixed initialization of Ceph clusters when stripers are not used. * **[XrdCrypto]** Improved determination of X509 certificate type, including proxy version * **[XrdHttp]** Fix memory leak in Bridge protocol (affects HTTP). * **[XrdSecgsi]** Several improvements in the way CRLs are checked and reloaded. * **[XrdCl]** Protect against spurious wakeups in SyncResponseHandler. * **[XrdCl]** On read-timeout, if the stream is broken, make sure the request and its handler are not double deleted. + **Minor bug fixes** * **[XrdCl]** Check if the file was correctly closed upon ZipArchiveReader destruction. * **[Server]** Add limits for prepare requests. * **[Server]** Delete buffers when the buffer manager is deleted. Fixes #414 * **[Server]** Do not double count overlapping spaces. Fixes #425 * **[XrdHttp]** Allow unauthenticated https clients. * **[XrdHttp]** Make Xrdhttp secure by default (rejecting proxy cert in the absence of a proper SecXtractor plugin) + **Miscellaneous** * **[XrdSecgsi]** Re-activate xrdgsitest * **[RPM]** Include xrdgsitest in xrootd-client-devel package. * **[XrdFileCache]** Add example of filecache configuration. ------------- Version 4.5.0 ------------- + **New Features** * **[XrdCms]** Allow specifying a different timeout for null cached entries; fixes #413 * **[XProtocol/XrdSec/Server/XrdCl]** Implement request signing. * **[XrdCl]** Add ZIP extracting capability to xrdcp. * **[XrdCl]** Include the release number in client Login request cgi. * **[XrdCl]** Add support for spaces in file names for mv operation. + **Major bug fixes** * **[XrdCrypto/Secgsi]** Fix XrdCryptosslMsgDigest::Init ; set 'sha256' as default algorithm. * **[XrdCl]** Use posix semaphores for fedora >= 22. Disable omit-frame-ponter for gcc >= 4.9.3 if custom semaphores are used. + **Minor bug fixes** * **[XrdSecsss]** Fix memory leak in sss protocol. * **[XrdNet]** Allow hostnames to begin with a digit. * **[XrdCl]** Fix segfault in case a user cannot be mapped to a home directory. * **[XrdCl]** Make sure a socket is always associated with a proper poller object (not null). * **[XrdCl]** Fix deadlock in XrdCl::PollerBuiltIn during finalize. * **[XrdCrypto]** Do not use md5 checksum on OSX platform. + **Miscellaneous** * **[RPM]** Include xrdacctest in xrootd-server package. * **[RPM]** Add conditional BuildRequires for ceph >= 11. * **[RPM]** Use compat-openssl10-devel for fedora>=26. * **[XrdCl]** Make sure the Log class can be used by any client plugin implementation. ------------- Version 4.4.0 ------------- + **New Features** * **[Server]** Add new [no]rpipa option to xrd.network directive. * **[Server]** Allow objectid's to be specified in the authorization file. * **[Server]** Add new logging plugin interface. * **[Server]** Fixes #345 - add sid to TOD structure (ABI compliant). * **[Server]** Implement resource selection affinity (primarily for ssi). * **[XrdCl]** Add Metalink support (xrdcp & API). * **[XrdCl]** Enable metalink processing on default. * **[XrdCl]** xrdcp: use cks.type cgi tag to select the checksum type. * **[XrdCl]** Support local metalink files. * **[XrdCl]** Add support for GLFN redirector of last resort. * **[XrdCeph]** Implemented pools of ceph objects. + **Major bug fixes** * **[Posix]** Remove double unlock of a mutex. * **[Client]** Serialize security protocol manager to allow MT loads. * **[Authentication/sss]** Fix dynamic id incompatibility introduced in 4.0. * **[XtdHttp]** Removed the deprecated cipher SSLv3, in favor of TLS1.2 * **[XrdCl]** Close file on open timeout. * **[XrdCl]** Differentiate between a handshake and an xrootd request/response while processing an incoming/outgoing message. * **[XrdCl]** Fix dangling pointer issue that occurs while forking. * **[XrdCl]** Ensure DefaultEnv is finalized after last use of the object. + **Minor bug fixes** * **[Proxy]** Avoid SEGV when printing memory cache statistics. * **[Server]** Avoid XrdNetIF static initialization issues. * **[Server]** Honor DFS setting when forwarding operations. * **[Server]** Make sure lockfile time is updated in deprecated runmodeold. * **[Server]** Fixes #344 - squash path before checking for static redirect. * **[Server]** Free Entity before replacing it from the cache (memleak). * **[XrdCl]** xrdfs ls does not include opaque info in a listing. * **[XrdCl]** Eliminate unnecessary write notifications. * **[XrdCl]** Forward xrd.* parameters from the original to the redirection URL. * **[XrdCl]** Do not preset CWD in batch mode. * **[XrdCl]** Be complaint with file URI scheme. * **[XrdCl]** Fix wrong query string used for opaquefile code. * **[XrdCl]** Translate XRootD error code to errno before passing to strerror. * **[XrdCeph]** Fixed thread safety of filedescriptors in the ceph plugin. * **[XrdCeph]** Protected initialization of ioCtx object and striper objects by mutex in the ceph plugin. * **[XrdCeph]** Fixed memory corruption in asynchronous read from ceph. * **[XrdXml]** Make sure c-string buffes are properly terminated. * **[XtdHttp]** Don't trim printable characters. + **Miscellaneous** * **[XrdCl]** Change the way handlers and messages are matched (use maps). * **[Apps]** Add xrdacctest to the tools set to test access control databases. * **[Packaging/RPM]** Set max open files limit to 65k for systemd services. ------------- Version 4.3.0 ------------- + **New Features** * Add option to query network configuration via configured interfaces. * **[Proxy]** Default event loops to 3 and allow it to be set via config. * **[Server]** Let client inform redirector why it's retrying a lookup using the triedrc CGI element. * **[Server]** Add cms.cidtag directive to qualify the global cluster id (solves dpm problem). * **[Server]** Make it possible to effeciently query an external database for file existence via the statlib plug-in (largely for DPM).` * **[Server]** Allow declaring extra large I/O buffers (mostly for Ceph). * **[Server]** Allow iovec based data responses (no ABI changes). * **[Misc]** Add back trace capability to the tool set. * **[Misc]** Add generalized XML parsing ability. * **[Misc]** Add metalink parsing for future integration. * **[XrdCl]** xrdcp add env var to disable recovery * **[XrdCl]** Add support for multiple event loops. + **Major bug fixes** * **[Server]** Correct IP address matching between IPv4 and IPv6. Fixes #300. * **[Server]** Ruggedize cmsd thread synchronization during node deletion. * **[Server]** Delete extraneous semaphore wait to avoid deadlock (#290). * **[Server]** Return correct response to a delayed open. * **[Server]** Fix build of kXR_wait message in case of delayed open. * **[XrdCl]** Detect whether client is dual stacked based on outgoing connection in addition to DNS registration. * **[XrdCl]** Avoid EAGAIN loop. Fixes #303. * **[XrdCl]** Append opaque info in case we retry at a data server after being redirected. * **[XrdCl]** Fix: FileStateHandler::OnOpen seqfaults when both write timeout and OpenHandler timeout at the same time. * **[XrdCl]** Avoid SEGV when server fails after it responds waitresp. * **[XrdCl]** Continue processing remaining files after error occurrence. * **[XrdCl]** Fix for dangling pointer problem in deep locate, fixes #324 * **[Misc]** Add possibility to specify disk usage parameters in .. G, T units using XrdOuca2x::a2sz(). * **[Python]** Fix lock inversion in python bindings. * **[Python]** Check if python interpreter is still initialized. + **Minor bug fixes** * **[All]** Fix numerous issues with space reporting (spaceinfo, query space, statvfs) such a double counting, scaling, and format issues. * **[Proxy]** Do not use the ffs code path if nothing is writable. This avoids initialization failure when the origin is a large WAN cluster. * **[Server]** Be agnostc NTP defaults when rotating logs (fixes new RH7 defaults). * **[Server]** Pass correct total data length in iovec to Send(). * **[Server]** Avoid redirection loop during error recovery in a uniform cluster. * **[Server]** Make sure N2N gets configured for the cmsd when actually needed. * **[Server]** Properly handle an inifit NPROC limit. Fixes #288. * **[Server]** Make sure the cluster ID is always formatted the same way. * **[Server]** Correctly compute timeout wait. * **[Server/Logrotate]** Make sure rotating pattern is not expanded in an if statement, fixes #302 * **[Misc]** Include XrdXmlReader in the spec file. * **[Misc]** Fix bug in access statistics print. * **[Misc]** Allow conversion of decimal numbers in XrdOuca2x::a2sz() * **[XrdCl]** xrdfs prepare has to be provided with a filename, fixes #309 * **[XrdCl]** Make sure recursive copy is disallowed only for checksum with user provided value, fixes #304 * **[XrdCl]** Use the same timeout value for all close operations in xrdcp with TPC enabled. * **[XrdCeph]** Fixed race condition in multistream access to files fo CEPH + **Miscellaneous** * **[Server]** Prevent cmsd reconnect storm when things get way slow. * **[Server]** Changes to allow for Solaris compilation. * **[Server]** Changes to allow for OSX compilation. * **[Server]** Detect cyclic DNS host registration when processing '+' hosts. * **[Server]** Display manager IP addresses during '+' host resolution. * **[Util]** Avoid compiler warning about unsafe mktemp. * **[App]** Do not report expected errors as errors. * **[App]** Always show any unusual node status in the display. * **[Client/Python]** Add MANIFEST.in for python bindings. * **[XrdFileCache]** Implement blacklisting in a FileCache decision plugin. * **[XrdFileCache]** Make sure requested offset is reasonable. * **[XrdFileCache]** Return -1 and set errno when bad offset is passed in. * **[XrdFileCache]** Only generate error for negative offsets, as per posix. * **[XrdFileCache]** Add startup protection for ReadV, too. It was already there for Read. * **[XrdFileCache]** Fix bug in cache scanning; simplify deletion loop. * **[XrdFileCache]** Use bytes to calculate how many files to purge, not blocks; subtract actual size of the file, not the length of it returned by stat. * **[XrdFileCache]** In cache purge, use stat.mtime of cinfo file if last access time can not be determined from contents of cinfo file. * **[XrdFileCache]** Fix argument type from int to long long (was n_blocks, is size_in_bytes now). * **[XrdCl/XrdSys]** Use custom semaphores only for glibc<2.21. * **[XrdCl]** Remove libevent-based poller implementaion. * **[XrdCl]** Report reason for reselection via triedrc CGI element. * **[XrdClient]** Changes to allow for Fedora rawhide C++11 compilation. * **[XrdCeph]** Fixed XrdCeph compilation for C++11 enabled compilers * **[XrdCeph/CMake]** Fix for undefined symbols (link XrdUtils). ------------- Version 4.2.3 ------------- + **Major bug fixes** * **[Server]** Avoid SEGV if cmsd login fails very early. * **[Server]** Avoid SEGV when an excessively long readv vector is presented. * **[Server]** Rationalize non-specfic locate requests. * **[XrdCl]** Process waitresp synchronously via Ignore return to avoid SEGV. * **[XrdCl]** Avoid memory leak when a handler returns Ignore for a taken message. * **[XrdCl]** Fix "tried" logic by forwarding the errNo ------------- Version 4.2.2 ------------- + **Major bug fixes** * **[Proxy]** Protect forwarding proxy server from slow connections. This should fix most, if not all, SEGV's that the server encountered under heavy load. * **[Server]** Fixes #248 Prevent infinite loop when shift arg is negative. * **[Server]** Complain when passed I/O length is negative. * **[Server]** Avoid execution stall during node logout when the thread limit has been reached. * **[Server]** Make sure to capture return code for stat() to prevent random results. * **[XrdCl]** Make sure to get filestate lock during timeout processing to avoid MT intereference and possible random results. * **[XrdClient]** Restore commented out abort() when an attemp is made to index a vector outside of its current bounds (avoids random results). * **[Server/Proxy]** Delay deleting a file object if the close was not successful. This avoids deleting objects that may have pending activity resulting in an eventual SEGV. This is a bypass fix to another problem. + **Minor bug fixes** * **[Server]** Fixes #234 Properly register all components in a mkpath request. * Correctly handle copying into a non-existent directory when automatic path creation is enabled. * **[XrdCl]** xrdfs correctly handles quotations (fixes the problem with ALICE token) + **Miscellaneous** * Fixes #245 Provide compatibility when cmake version is > 3.0. * Use atomics to manipulate unlocked variable pollNum. * Bugfix: release lock when a file is closed before the prefetch thread is started. Observed with xrdcp ran without -f option and an existing local file. Fixes #239. * Protect from reads exceeding file size. Fixes #249. * Release Stream lock before invoking callbacks. Fixes #216 * TPC: Fix deadlock in case of error in the TPC authentication * Increase max size of write to disk queues. * Fix bug in endswith. Fixes #260 * XrdCeph : fixed problem with files bigger than 2GB for synchronous writes * **[XrdCl]** Change message loglevel from Error to Debug. Fixes #246. * **[XrdCl]** Fix race condition in PostMaster initialization * **[XrdCl]** Provide atomicity for PostMaster value using built-in functions * **[XrdFileCache]** fixed deadlock on immediate file close (e.g. xrdcp to non-writable output) * **[XrdFileCache]** fixed errors on some posix operations using virtual mount ------------- Version 4.2.1 ------------- + **Miscellaneous** * **[Client/Cl]** Make sure kXR_mkpath is set for classic copy jobs when the destination is xrootd (backward compatibility fix). ------------- Version 4.2.0 ------------- + **New Features** * **[Client/Python]** Integrate xrootd-python into the main package. * **[Server]** Include a Ceph OSS plug-ing. * **[Server]** Implement throttling. * **[Server]** Detect redirect loops using "tried" token. * **[Server]** Implement the "cid" option for config query to display the unique cluster ID. * **[Server]** Allow suspending and enabling remote debugging without a restart. * **[Server]** Implement black/whitelist with optional redirection. * **[Server/Proxy]** Add the xrdpfc_print tool to print the caching proxy metadata. * **[Server/PlugIns]** Provide a mechanism to pass command line arguments to plug-ins. * **[Server/PlugIns]** Provide access to the native and the active extended attribute implementation. + **Major bug fixes** * **[All]** Fix various memory access issues. * **[Server]** Fix various IPv4/IPv6 compatibility issues. * **[Server]** Avoid disabling of frm notifications due to plug-in initialization issues. * **[Server/Proxy]** Avoid holding a global lock when opening/closing files to solve timeout issues. * **[Security/GSI]** Fix reloading of CA and CRLs. + **Minor bug fixrs** * **[Server/HTTP]** Fix issues related to invalid chunk sizes. * **[Server/Proxy]** Various logic and permission processing fixes. + **Miscellaneous** * **[Client/Cl]** Make the compiler issue warnings when the return codes from the File and FileSystem methods are unchecked. (issue #188) * **[RPM]** Disable building of the compat package by default. * **[Server/Proxy]** Avoid serializing stat() via the proxy to improve performance. * **[Tests]** Factor out the common testing code from the client tests so that it can be re-used. ------------- Version 4.1.2 ------------- + **Major bug fixes** * **[Utils]** Don't confuse -I and --tpc while parsing commandline parameters for xrdcp. (issue #213) * **[Server]** Fix various IPv4/IPv6 issues. (issues #164, #227) + **Minor bug fixes** * **[Client/Cl]** Print mtime when doing xrdfs stat. * **[All]** Fix some memory access issues. (issues #186, #197, #205) * **[Server]** Recreate logfile fifo if it already exists and is a file. (issue #183) * **[Server]** Properly reset suspend state when reconnecting cmsd. (issue #218) * **[Server]** Avoid disabling async I/O when using an oss plugin that does not implement file compression. (issue #219) * **[Server]** Do not debit space when relocating a file within the same partition. * **[Server]** Fix meta-manager port directive ordering. * **[Server/Logrotate]** Do not print anything to stdout to avoid making cron send emails to admins. (issue #221) + **Miscellaneous** * **[Server/Proxy]** Disable POSC processing when a proxy plugin is loaded. ------------- Version 4.1.1 ------------- + **Major bug fixes** * **[RPM]** Remove the library patch from xrootd-config to enable multiarch installations. * **[RPM]** Move the user creation scriptlets to xrootd-server where they belong. (issue #179) * **[Server]** Fix PowerPC compilation. (issue #177) * **[Server]** Avoid the pitfalls of infinite nproc hard limit in Linux. * **[Server]** Correct flag definition to include cms plugin loading. (issue #176) + **Miscellaneous** * **[Man]** Update documentation. * **[Client/Cl]** Set the multi-protocol ability basing on an environment variable. ------------- Version 4.1.0 ------------- + **New Features** * **[Everyting]** Implement dynamic plugin shared library filename versioning to allow multiple major versions to co-exist. * **[Server]** Compelete IPv6/IPv6 and public/private network routing. * **[Server]** Allow the checksum manager to use OSS layer to access data. (issue #140) * **[Server]** Allow the definition of subordinate clusters. * **[Server]** Support multiple checksum types. Client can select non-default checksum using the "cks.type=" cgi element. * **[Server]** Provide plugin interface for handling extended attributes. * **[Server]** Add options to xrd.network to control keepalive. * **[Server]** Control core file generation via xrd.sched core directive. * **[Server]** Add pss.permit directive to restrict outbound connections for forwarding proxies. * **[Server]** Allow xrootd to handle objectid names as exports. * **[Server]** Install and package the cluster mapping utility: xrdmapc. * **[Server]** Allow the specification of xrootd.seclib default. * **[Server]** Pass along XRD_MONINFO setting and application name to monitoring. * **[Server/Proxy]** Implement a forwarding proxy option. * **[Server/Proxy]** New configuration of XrdFileCache using 'pfc.' prefix. * **[Sever/HTTP]** Support gridmap parsing. * **[Client/Cl]** Inform the server about availability of local IP address types (IPv6/IPv4, public/private) to in order to facilitate redirections. * **[Client/Cl]** Make the client send kXR_endsess request when recovering broken connection - avoids 'file already open' errors. * **[Client/Cl]** Implement TCP keep-alive support. * **[Client/Cl/xrdcp]** Optimize xrdcp uploads by compensating for latency. * **[Client/Cl/xrdcp]** Make it possible for xrdcp to run multiple transfers in parallel using the '--parallel' option. * **[Client/Cl/xrdcp]** Make it possible for xrdcp to concatenate multiple sources to stdout. * **[Client/Cl/xrdfs]** Add xrdfs locate -i option to ignore network dependencies (IPv6/IPv4). * **[Security]** Add new security framework loader to allow external pacakges that linked against security plugins to dynamically load them instead. * **[Security/sss]** Allow forwardable sss tokens when ecrypted with a forwarding key as defined by the xrdsssadmin command. * **[Plugins]** Implement generic matching rules to version check 3rd party plug-ins. * **[Packaging/RPM]** Add SystemD configuration files for RHEL7. * **[Packaging/RPM]** Introduce compat RPM packaging providing xrootd 3.3.6 deamons and libraries with the ability to switch between desired versions using the sysconfig file. * **[Packaging/RPM]** The RPM naming has been switched back to xrootd (from xrootd4). * **[Utils]** Add xrootd-config utility. + **Major bug fixes** * **[Server/HTTP]** Make it possible to handle files larger than 2GB. * **[Server]** Prevent blacklisting of all connctions when role is supervisor. * **[Server]** Fix bug in handling cms.dfs redirect verify that would keep the client is an infinite wait loop. This also affected locate requests regardless of what the redirect option was set to. * **[Server/Proxy]** Avoid SEGV when no environment has been passed in the proxy server. + **Minor bug fixes** * **[C++ API]** Provide complete portability and correct behaviour across platforms with and without Atomics. This patch does not change any ABI's. * **[Server]** Do not set *TCP_NODELAY* for unix domain sockets as this issues a nasty error message. * **[Server]** Allow cms.dfs mdhold argument to be 0 as documented. * **[Server/Plugins]** Add missing initializer to the LocInfo structure. * **[Server/Plugins]** Correct header define gaurd in XrdSfsFlags.hh. * **[Server/Proxy]** Fully support extended file system features and pass those features through a proxy server. (issue #115) * **[Client/Cl]** Remove duplicates from the HostList. * **[Client/Cl]** Fix minor atomicity issues (C++11). + **Miscellaneous** * **[Server]** Actually remove xmi plugin handling as xmilib is no longer supported. * **[Server]** Make sure to always passhrough CGI information. * **[Server]** Honor network routing when creating the client's i/f selection mask. * **[Server]** Efficiently handle replicated subscribers (i.e. managers). * **[Server/HTTP]** Remove useless loading the security framework. * **[Server/Security]** Add new NetSecurity::Authorize() method that accepts text. * **[Server/Proxy]** Properly support proxying objectids. * **[Server/Proxy]** Clean-ups in the caching proxy. ------------- Version 4.0.4 ------------- * **Major bug fixes** * **[Client/Cl]** Properly allocate buffers for error messages. (issue #136) * **[Client/Cl]** Check if there is enough data before unmarshalling. * **[Client/Cl]** Fix a memory leak in MessageUtils::WaitForResponse affecting all synchronous calls. * **[Client/Cl]** Prevent a segfault in the destructor when called after the libXrdCl library has been finalized by the linker - ROOT garbage collection. https://github.com/cms-externals/xrootd/pull/1 * **[Client/Posix]** Fix broken readdir_r() and readdir_r64() functions. * **[Server]** Use correct flag when adding a cluster. The bug made it impossible to have more than one supervisor node. * **[Server/Logrotate]** Prevent stack corruption by correctly sizing the timestamp buffer. + **Minor bug fixes** * **[Client/Cl]** Properly check if a recursive copy was requested to avoid unnecessarily stating the source. * **[Client/Cl]** Avoid inserting duplicate entries to HostList when retrying at the same server. * **[Client/Cl]** Normalize (trim leading zeroes) before comparing adler and crc checksums. (issue #139) * **[Client/Posix]** Prevent mkdir failure in a clustered environment by creating the full directory path by default. * **[Client/Possix]** Fix a memory leak when doing deep locate. * **[Server/Logrotate]** Use expect to send a ping to pipes. This prevents logrotate from hanging when nobody is listening at the other end of the pipe. * **[Authentication/Client]** Pass the external environment to the protocol manager. (issue #133) * **[Authentication/sss]** Fix a memory leak. * **[Utils]** Avoid SEGV when assigning a unix domain address to a NetAddrInfo object previously used to hold a TCP domain address. * **[Server/cmsd]** Use the same write selection rules for dfs and non-dfs environments. + **Miscellaneous** * **[Server/Logrotate]** Prevent the default configuration from sending emails to admins and from creating a new log after the old one has been rotated. (issue #135) * **[Server/SELinux]** Using expect in logrotate requires the logrotate_t context to have access to pseudoterminals and tmpfs as well as stating fifos * **[Client/Commandline Parser]** Allow local to local copy in new xrdcp but not in the old one. * **[Client/Cl]** Discard a whole cluster on failure in federation context. (issue #132) ------------- Version 4.0.3 ------------- + **Major bug fixes** * **[Server]** Make sure the network routing is honored in all cases. This fixes problems encountered by sites whose clients use a private IP address to connect to a redirector's public IP address. (issue #130) ------------- Version 4.0.2 ------------- + **Minor bug fixes** * **[Client/Cl]** Handle all non-NULL-terminated error responses correctly. * **[Client/Cl]** Release old auth buffer when reconnecting after TTL expiration. + **Miscellaneous** * **[Client/Cl]** Retry after an incomplete local write. This produces clearer error messages. Ie: "Run: [ERROR] OS Error: No space left on device" instead of: "Run: [ERROR] OS Error: Operation now in progress". * **[Client/Cl]** Don't force a server to issue a short read when fetching last data chunk. This works around issues for proxied FAX sites. ------------- Version 4.0.1 ------------- + **Major bug fixes** * **[Server]** Prohibit accessing memory via /proc using digFS. + **Minor bug fixes** * **[Server]** Prevent over-scan of the xrd.network routes option which may cause a config file error message and initialization failure. * **[Server]** Fixes to make things compile on ix86, arm and ppc64. * **[Server]** Correct protocol name supplied to monitoring for userid. * **[Server/Proxy]** Various minor fixes to caching proxy. * **[Security]** Check the length before looking inside a SUT buffer. (issue #126) * **[Client/Cl]** Check for copy source and target validity to display proper error messages. * **[Client/Cl]** Return default plug-in factory for an empty URL. (issue #120) * **[Client/Posix]** Provide full error mapping for POSIX interface. * **[All]** Remove some unnecessary commas and semicolons. (issue #121) + **Miscellaneous** * **[Server]** Pass client login information to monitoring. * **[Client/Cl]** Make xrdfs locate -h synonymous to locate -m. * **[Client/Cl]** Add -i option to xrdfs locate setting the Force flag. * **[Docs]** Various documentation updates. ------------- Version 4.0.0 ------------- + **New Features** * Supprt IPv6. Please read docs/README_IPV4_To_IPV6 for details. * Introduce the XrdFileCache library - a proxy server plugin used for caching of data into local files. * Beta support HTTP(S). * Provide protocol bridge to let other protocols use xrootd back-end plugins. * Provide full support for public/private IP networks. * Allow remote debugging via the xrootd.diglib directive. * Provide a mechanism to manually control log file rotation via -k and add support for logrotate. * Add -z option to enable high recision log file timestamps. * Define a new plug-in to allow replacement of the stat() function when used to determine exported file characteristics. This plug-in is meant to be used by tape-backed file systems that identify offline files in odd ways (e.g. GPFS). Patch assumes XRDROLE patch below. * Implement full readv-passthru for enhanced performance. * Add a disconnect record to the f-stream. * xrdcp is now the same as xrdcopy, and old xrdcp is now xrdcp-old * Make clients configurable via /etc/xrootd/client.conf and ~/.xrootd/client.conf * Implement a plug-in system for client's File and FileSystem queries. * Make it possible for 'xrdfs stat' to query for combination of flags. * Make third party copies cancellable. * Implement xrdfs spaceinfo, cat and tail commands * Terminate iddle connections after a timeout and treat timeouts on streams that should be active (because of outstanding requests with no delay times) as errors. * Implement XrdCl::File::Visa and XrdCl::File::Fcntl. * Support for full URL redirects. * File and Filesystem objects implement property system to pass custom information to and from them (including plug-ins) without breaking ABI. * Add --dynamic-src to xrdcp options to allow dynamic file copying. * Implement the directory listing in bulk. * Enable locate to return host names not just IP addreses. * Implement node blacklisting for the cmsd (see cms.blacklist directive). * Add mv command to frm_admin. * Allow query of current role and dynamic cms state via kXR_query. * Implement query config chksum to return supported chksum name. * Add version as a variable that can be returned by kXR_Qconfig. * Add sitename as an argument to kXR_Query+kXR_Qconfig. * Provide disconnect notifiation to underlying file system. * Provide the filesystem plugin a way of creating a session storage area. * Add flag to indicates a secondary copy of a file exists. * Allow testing for undefined set/env vars via if-else-fi. * Add '-L' flag to the xrootd command to allow loading a protocol library * Add flag to indicates a secondary copy of a file exists + **Bug fixes** * Fix various dead locks in the IOEvents poller. * Implement LinuxSemaphore class in order to replace buggy POSIX semaphores on Linux. * Honor the cmsd.dfs directive for locate request to avoid placing a file in ENOENT status. * Make sure that the old client runs only in IPv4 mode as mixing modes does not work for a variety of reasons. * Accept old-style as well as new-style IPv6 addresses in the sss protocol. This allows the new client to use this protocol after it implemented IPv6 support. * Prevent invalid mutex operations in auto-termination routine. * Resolve naming conflicts within the frm that resulted from the statlib plugin implementation. * Do not rely in file locking to serialize inter-thread access. This fixes the prolem of usage file drift. * Fix various parse context issues in copy config with --recursive. * Recognize object deletion in the error handling path. * Use atomic FD_CLOEXEC where available to prevent FD leaks. * Squelch casting complaints from C++11. * Make sure to return all nodes in a star locate request. * Always load protocols in the specified order. * Fix xrootdfs wcache crashing issue when using virtual file descriptor. * Fix selection of a server when a DNS entry resolves to more than one. * Correct pthread_cond_timedwait() time calculation and error handling. * Fix null insertion of hostname in error message when open fails. * Fix issues with extensions in GSI proxies * Fix problem with creation of the forwarded KRB5 ticket * Correctly handle reading of a partial readv headers (issue #45) * Make sure to propagate username and password when redirecting * Honor request timeouts when processing kXR_wait + **Miscellaneous** * XrdClient and associated commandline utilities are now obsoleted. * Propagate info about partial success from deeplocate to dirlist. * Remove perl interface. * Send timezone, country code and application name while logging in. * Change interfaces to copy process to use property system (allows for adding features without breaking the ABI). * Final change to f-stream monitoring. Replace standard deviation (sdv) calc with reporting sum of squares (ssq) counts. * Make public headers compile cleanly with -Wall -Wextra -Werror. * Support passing cert, key paths via URLs * Allow testing of undefined set/env vars via if-else-fi * Pass user environment settings settings in the login CGI * Use DNS names instead of addresses for kXR_locate when listing ------------- Version 3.3.6 ------------- + **Minor bug fixes** * Prevent SEGV when error occurs during stat (issue #70) * Prevent SEGV in redirect monitoring (issue #61) * Set reasonable linux thread limit and warn it we cannot do so. + **Miscellaneous** * Support for C++11 (narrowing fixes, unique_ptr vs. auto_ptr) * Support for CMake 2.8.12 (interface link libraries) ------------- Version 3.3.5 ------------- + **Minor bug fixes** * Fix minor Coverity issues in XrdCl * Fix a rarely occuring segfault when forking XrdCl under heavy load * Fix various issues related to group name retrieval (issues #51, #52, #53) + **Miscellaneous** * Make XrdSys/XrdSysIOEvents.hh private - could not have been used anyways * Add a sysconfig template to preload custom allocators in order to fix memory issues on RHEL6 * Allow up to 63 characters for a site name ------------- Version 3.3.4 ------------- + **Major bug fixes** * Serialize sss authentication client initialization to prevent race conditions * Actually cancel the JobManager threads while stopping it - this affected client side fork handling (new client) * Restore original meaning of -adler and -md5 to xrdcp (issue #44) + **Minor bug fixes** * Append CGI info when retrying at a server that handshaked but never respnded to the request (xrdcp) * Do socket accepts asynchronously to prevent DNS resolution from blocking accepts (issue #33) * Warn about incomplete dirlist responses (xrdfs) * Cast the utilization statistics to uint16_t before printing to print actual numbers instead of letters corresponding to ASCII codes (xrdfs) + **Miscellaneous** * When calling File::Stat use file handle instead of path * Improve handling of malformed kXR_readv responses (new client) * Explain parameters of xrdcopy --tpc (documentation, issue #46) ------------- Version 3.3.3 ------------- + **Major bug fixes** * Prevent SEGV's when reusing a recycled protocol object under certain conditions (xrootd server) * Prevent SEGV when using the -DS/-DI commandline parameters in xrdcp (issue #13) * Prevent integer overflow when calculating client recovery windows * Make sure the new client tries all available authentication protocols when connecting to a security enabled server (issue #14) * Detect buffer size mis-matches when server returned valid response with invalid size (xrdcopy) * Recognize /dev/null and /dev/zero as special files when using copy commands + **Minor bug fixes** * Prevent the new client deadlock on Solaris and MacOS when using the built-in poller and connecting to localhost (issue #5) * Compensate for ROOT garbage colletion issues when calling the new client code * Avoid favoring socket writes when using new client with the built-in poller * Strip off opaque information from dest filename when copying to local filesystem using xrdcp (issue #21) * Fix setting client timeout resolution while connecting to a server + **Miscellaneous** * Change the RPM package layout to match the one used by EPEL (issue #12) * Drop the daemon user RPMs * Allow new client connection parameters to be tweaked by connection URL CGI * Make the built-in poller default again in the new client - after resolving issue #5 ------------- Version 3.3.2 ------------- + **Major bug fixes** * Fix the opaque information setting in xrdcp using -OD (issue #1) * Fix compilation on Solaris 11 (issue #7) * Fix issues with semaphore locking during thread cancellation on MaxOSX (issue #10) * Solve locking problems in the built-in poller (issue #4) * Solve performance issues in the new client. Note: this actually changes some low level public interfaces, so the soname of libXrdCl.so has been bumped to libXrdCl.so.1. The xrootd.org RPMs also provide the old libXrdCl.so.0 in order to preserve the binary compatibility with the clients linked against it. ------------- Version 3.3.1 ------------- + **Major bug fixes** * Correct XrdClient ABI incompatibility issue introduced in 3.3.0 * Install additional private headers ------------- Version 3.3.0 ------------- + **New Features** * Stable interfaces immutable in minor releases (except XrdCl). Only public header files are installed in the usual include directory. In order to ease up transition of some clients some of the private include files are also installed in private subdirectory. * New asynchronous and thread-safe client libraries and executables (XrdCl). The ABI compatibility is not guaranteed until 4.0.0. * Build the xrootd protocol plugin as a shared library. * Add the altds directive to allow pairing a cmsd with an alternate data server. * Differentiate between packed and unpacked readv monitoring records. * Allow plugin libraries to be preloaded. This feature is only meant for MacOS. * Include optional site name in summary monitoring records. * Include optional site name in server identification record if the site name was specified on the command line (-S) or via config file (all.sitename directive). * Define a standard supported mechanism to obtain the default storage system object. * Provide an ABI-compatible interface to obtain a default cmsd client object. This patch does not change the definition of the XrdCmsClient object and is ABI compatible with all previous releases (DPM support). * Allow multiple comma separated protocols in XrdSecPROTOCOL client-side envar. This allows the client to select 1 of n protocols. * Implement new "f" stream monitoring. * Add new summary counters for readv and readv segs. * Add boiler plate comments indicating the all software is licensed under LGPL. No functional source code was modified by this patch. * Add GPL and LGPL license text. * Liberlize locking structure to prevent lock inversion relative to external locks. * Provide libevent replacement for Linux (epoll), Solaris (poll_create), and others (poll). Note: versions of Solaris less than 10 are no longer supported and they will no longer compile with this update! * Provide a libevent type replacement package. * Allow tracker files (e.g. ".fail") to be placed in a shadow directory. This is controlled by the new fdir option on the oss.xfr directive. * Allow meta-files (i.e. .fail file) to be relocated to a shadow directory using the oss.xfr directive. This avoids polluting the exported name space when an frm transfer operation fails. * Create a general place for platform dependent utility methods. * Add third party copy statistics to the summary record. * zlib compatible checksum plugin + **Major bug fixes** * Serialize access to cache entries to prevent SEGV's. * Fix the fast response queue so that it doesn't run out of response slots causing a big performance penalty. This is a high priority fix. * Properly disarm the mutex helper when the mustex object is deleted. * Use correct variable to hold osslib parameters. This patch fixes commit 2e27f87a (version checking) and without this patch makes it impossible to load an oss plug-in. * Properly check for errors when client read returns 0 and reflect true status. This only affects the Posix client interface. * Remove redundant flag indicating a running poller. This may cause the poller to never be woken up when a timeout value changes. * Fix tag in ofs statistics. It is improperly terminated and may cause certain xml parsers to fail; rendering monitoring useless. * Undo the side-effect of commit ff8bdbd6 that prevented the frm from sending stage notifications to xrootd; causing opens and xrdstagetool to hang with dynamic staging enabled. * Make sure the id buffer is large enough to hold all id combinations. * Avoid deadlock when closing a Posix File with an active preread. * For concurrent queries for the same file allow servers to respond to the query and only redirect clients to a stageable server if the file is not found. + **Minor bug fixes** * Add EPOLLRDHUP to avoid leaving sockets in CLOSE_WAIT with a one-shot poll framework. * Fully integrate checksum processing into a manager node. When configured, it does not matter whether a client directs a checksum request to a manager or a server. This also fixes bug report #93388. * Make sure to reflect proper range of errors during read/write operations. This also provides filesystem plugins full range of allowed return codes. * Initialize the rMon toggle to avoid valgrind complaint. * Fix minor issues reported by Coverity. * Make sure opendir() returns a null pointer when the directory doesn't exist. * Make sure that XrootdFS returns ENOENT when opendir() returns a null. * Make sure to use correct time to set mtime/atime after a physical reloc. * Prevent hangs when doing exterme copy from server to server. * Fix the -force option to really work for the mark subcommand. * Pass through error code returned by the N2N plug-in. This only affects the proxy server and caused feature interference. * Automatically exclude originating server/cluster on an enoent static redirect. * Correct typos XRDPSOIX envars should really be named XRDPOSIX. + **Miscellaneous** * Remove superfluous includes or other move includes to eliminate unnecessary dependencies in ".hh" files. This patch is required to create an EPEL conformable include directory. * Add port to prepare request struct as documented in 2.9.9. * Add pathid to readv request struct as documented in 2.9.9. ------------- Version 3.2.6 ------------- + **Major bug fixes** * GSI authentication: fix possible race condition while re-loading CA certificates; fix also related memory leaks. * GSI authentication: make sure the CA cache is not initialized twice (e.g. server and client inside there), and that the cache entry pointers are always initialized. * Crypto OpenSSL modules: use more appropriate way to read the RSA complete key, solving various issues for RH6 and derivations, included SL(C)6. * Make sure redirect opaque information is passed along for all filename based requests. This is required for DPM and EOS N2N services to work in all cases (most importantly, stat). * Make sure buffer ends with null byte before read suspension. This only occurs on very heavily loaded connections. * Fix the fast response queue so that it doesn't run out of response slots causing a big performance penalty. This is a high priority fix. + **Minor bug fixes** * Properly detect external process failure and report correct error status to a client. This also fixes bug report #91141. * [XRootDPosix] Make sure to use a supplied cache even when no cache directives given. * Make sure to return a usable path string via XrdOucCacheIO::Path(). * Actually support 4 different redirect destinations. + **Miscellaneous** * Transparent support for new name hashing algorithm adopted in openssl 1.0.0x (GSI authentication protocol) * Verbosity levels revised for GSI and PWD authentication protocols. * Notification of initialization option for GSI and PWD authentication protocols. * Do not repudiate file existence on an "cancelled" error during open. this patch addresses overloaded dCache pool nodes. ------------- Version 3.2.5 ------------- + **Major bug fixes** * Make realoading gridmapfile atomic (protect from segfault) * Propagate to clients proper range of errors during read/write operations * Fix segfault when handling writes to files that have not been opened ------------- Version 3.2.4 ------------- + **Major bug fixes** * Work around a dead-lock in the client fork handlers. ------------- Version 3.2.3 ------------- + **Major bug fixes** * Make sure read statistics are updated for sendfile() and mmap I/O. * Make sure refresh thread is dead before deleting deleting the keytab to avoid SEGV's. * Add missing include for compiling with gcc-4.7 (from Sebastien Binet). This patch is required for successful compilation. * Avoid segfaults when limiting number of redirections caused by failed authorization. * Avoid deadlock in the client fork handlers. + **Minor bug fixes** * Correct monitor initialization test to start monitor under all configs. * Fix a memory leak in the client handshake algorithm. + **Miscellaneous** * Make RHEL6-created SRPMs buildable on RHEL5 by forcing RPM to use MD5 digests. * Fuse: Use default TTL values for data server connection and load balance server connection. ------------- Version 3.2.2 ------------- + **Major bug fixes** * Correct test whether or not to initialize redirect monitoring. The old code never initialized it this disabling redirect monitoring. * Backport frm notification fix that stalled stage-in requests from commit 69e38cfd6b8bb024dd34f8eb28a666fbf97f346b * Prevent SEGV when xrd.monitor rbuff value not specified * Prevent xrdcp hangs when doing exterme copy from server to server. * In case of 'limited proxy' look for VOMS attributes also in the parent proxy. * Correct log processing for sites that use the root directory as the stomping ground for newly created files. ------------- Version 3.2.1 ------------- + **Major bug fixes** * Don't build sendfile support on MacOSX because it doesn't work * Prevent double-free abort when more than 16 files have been opened by a client and the client terminates the session without closing the 17th one. ------------- Version 3.2.0 ------------- + **New Features** * Retool the XrdOucCache object so that cache implementations can be implemented as plugins. * Add FSize method to the XrdOucCacheIO object to ease implementation of disk caches containing partial files. * Add the pss.cachelib directive to specify a cache plugin. * Implement ultralow overhead redirect monitoring. WARNING: ofs plugin writers will need to recompile their plugin interface to be fully compatible with this commit due to additional information passed to the ofs object "new" methods. * Allow the XrdCmsClient interface (a.k.a Finder) to be a plug-in. * Add ofs.cmslib directive to specify the XrdCmsClient plug-in. * Add new class, XrdOucCallBack, to simplify using callbacks in the XrdCmsClient plug-in. * Define the frm.all.monitor directive to enable migration, purging, and staging monitoring. This was originally part of xrootd.monitor but that just was odd. Note that the stage, purge, migr events are no longer accepted on the xrootd.monitor directive. * Collapse he staging (s) and migration (m) records into a single transfer (x) record. While not compatible, the previous implementation was new code and no one actually was capturing these records. * Implement a server identification record (=) that unquely identifies each server. The record can be sent periodically and can be used as a heartbeat. * Add -y option to xrdcp to limit number of extreme copy sources. * Uniformly pass the execution environment to all oss and cms client methods. This is largely for DPM support. WARNING: While this update is binary backwad compatible to existing oss plug-ins it is not source compatible. Plug-in writers will need to modify their oss methods to successfully compile. * Allow an automatic redirect when a file operation ends with ENOENT. Allow redirects for chsum and trunc operations. Both of the above are controlled via the xrootd.redirect directive. * Report the timezone when connecting to a [meta]manager. * Allow configuration of staging, migration, and purging events. * Allow transfer script to inject information into the monitoring stream. * Report number of attempted login, authentication failures, successful authenticated and unauthenticated logins in the summary statistics. * Indicate whether a disconnect was forced and whether it was a parallel path (as opposed to a control path) in the monitoring record. + **Major bug fixes** * Provide compatibility for sprintf() implementations that check output buffer length. This currently only affects gentoo and Ubuntu Linux. We place it in the "major" section as it causes run-time errors there. * Reinsert buffer size calculation that was mistakenly deleted. This eventually causes a SEGV when detailed monitoring is enabled. * Remove improper initialization that may cause a SEGV in the checksum manager. * Add missing initializer without which we will get a SEGV. This is a fix for the just added monitoring code. * Remove regressions that prevent a proxy cluster from being fully configured. + **Minor bug fixes** * Correct debug message frequency that caused people to think some file system partitions were being ignored. * Correct pthread Num() to return thread-specific numbers. * Make sure the sendfile interrupt counter is initialized to zero. * Make sure to honor absolute cms.space values when percentage not specified. * Prevent double user map record when monitoring when auth is configured but not actually monitored. * Take timezone changes into account when waiting for midnight. This solves the log rolling problem when changing between DST and standard time. * Make sure to cut close records for open files during a forced disconnect when monitoring file information. * Do not create meta-files or update extended attributes when placing a file into read-only space. + **Miscellaneous** * Bonjour code dropped * Complete implementation of the fstat() version of stat(). * Consistently pass the enviroment to the cms client enterface. * Make return codes consistent between synchronous & async XrdCmsClient returns. * Document the XrdCmsClient interface in the header file. * Cut close monitor records before cutting the disconnect record. * Make frm_purged and frm_xfrd use sparate log files. ------------- Version 3.1.1 ------------- + **New Features** * Compile on Solaris 11 * Add support for sending DN with monitoring information * Add possibility to switch off automatic download of CRL from the web; default is OFF; to enable it multiply by 10 the relevant CRL options (i.e. 12 and 13 are like 2 and 3 but trying download if the file is not found). * Add refresh frequency time for CRL's; default 1 day . + **Major bug fixes** * Fix various client threading issues. * [bug #87880] Properly unpack the incoming vector read data. * Rework the handshake when making a parallel connection. Previous method caused a deadlock when parallel connections were requested (e.g. xrdcp). * Add HAVE_SENDFILE definition to cmake config. All post-cmake version of xrootd until now have disabled use of sendfile() with resulting poor performance. This fix corrects this. * Don't force libXrdPss.so to be loaded for proxy managers. * Fix various CMake issues: disable library inheritance, fix underlinking problems, make sure libcom_err is present when building kerberos. * Replace non-reentrant versions of getpwxxx and getgrxxx with reentrant versions. This should prevent spurious uid/gid translations. * Fix RedHat bug #673069: Missing header files required by DPM * Don't ignore errors returned by kXR_close * Init scripts: don't change the ownership of the sysconfig files preventing the xrootd user from executing arbitrary code as root + **Minor bug fixes** * Add 'k' to the option list. It was wrongly deleted in the last option refalgamization. * Fix a typo in the specfile causing problems with multithreaded compilation. * Initialize xattr variable name so that xrdadler32 can fetch previous checksum. The error caused xrdadler32 to always recompute the checksum. * Make sure that monitor write length is really negative. * Add the oss.asize hint to the destination URL in all possible cases. * Properly print adler32 checksum in xrdcp. * When the server certificate is expired, try to renew from the same path before failing. * Get the signing certificate for the CRL from its issuer hash, which can be different from the CA hash. * Add check for the format of the downloaded CRLs: DER or PEM * Solaris init script: switch to xrootd user when invoked as root * RHEL init scripts: always create /var/run/xrootd to handle /var/run being mounted as tmpfs + **Miscellaneous** * Relax requirements on the permission mode of the x509 key files * Disable client redirections reports to the console. * Stop doing XrdFfsPosix_statall() if task queue is long. * Get rid of compiler warnings * Improve some log messages * At server startup, only initialize the CA (and CRL, if required) for the authority issuing the server certificate; additional CA's are initialized only if needed. ------------- Version 3.1.0 ------------- + **New Features** * Use CMake to build the source code and retire all the other build systems. * Add IOV as a selectable detail to xrootd.monitor directive. * Provide a mode in xrootdfs to auto-update internal list of data servers. and extend client connection TTL from one hour to infinity. * Provide virtual xattr ("xroot.cksum") to obtain checksum for consistency. * Make xrdadler32 use the new checksum format if it is set (fallback to old format otherwise). In all cases, the old format is converted to the new format whenever possible. * Enforce r/o exports in the proxy server (finally added). * Allow auto-fluching of I/O stream monitoring (default is off). Patch submitted by Matevz Tadel, UCSD. * Make proxy honor the export list at the storage layer. This allows sites to disable staging via the proxy by specifying nostage for otherwise locally stageable paths. * Do not export the stage attribute to the meta-manager unless the path is tagged with the stage+ attrbute on the export directive. * WARNING: This update makes the oss plug-in source incompatible because an additional parameter was added to the Stat() method. The update is binary compatible and so only affects sites that recompile their plug-in. * Allow the query checksum request to be issued via a proxy server. * Add a query checksum interface to the POSIX interface. * Defines the livXrdSecgsiAuthzVO plug-in to allow easy mapping from voms vo names to users and groups. The plugin is configurable at run-time. * Allow the OucErrInfo object to point to an environment. * Add method to SysDNS to format an AF_INETx address into the RFC IPV6 recommended format. * Allow pointers to be placed in the OucEnv environment table. * Extend the kXR_protocol request to allow the server to return detailed information about node's role. This is backwardly compatible. * The client uses kXR_protocol request to query for the server's role (to distinguish managers from meta managers). * The client goes back to a meta manager on authentication failure. * The client prints to stdout the redirections it gets. This behavior may be disabled by setting the XRD_PRINTREDIRECTS environment variable to 0, or, from C++ by saying: EnvPutInt( NAME_PRINT_REDIRECTS, 0 ) * Set $HOST value for possible copycmd substitution. * Phase 1 to allow for redirection monitoring. Add rbuff and redir options to the xrootd.monitor directive. * Add error, redirect, and delay counts to the xrootd protocol summary statistics. * Allow file additions/deletion to be communicated to the XrdCnsd so that is can maintain an accurate inventory. This update adds the frm.all.cnsd directive which specifies how the information is to be commuincated. * Enable cmsd monitoring. For now, only [meta]manager information is reported. * Add new repstats config directive to increase reporting detail. * New class, XrdCmsRole, to make role naming/handling consistent. * Implement the 'cms.delay qdn' directive which allows one to tell the meta-manager the minimum number of responses needed to satisfy a hold delay (i.e. fast redirect). * Accept XrdSecSSSKT envar as documented but also continue to support XrdSecsssKT for backward compatibility. * Allow servers to specify to the meta-manager what share of requests they are willing to handle. Add the 'cms.sched gsdflt' and 'cms.sched gshr' configuration directives to specify this. * Include additional information in the protocol statistics. * Resize some counters to prevent overflows. * Add the 'cms.delay qdn' directive to allow better redirection control in the future. * Allow a plugin (notably the proxy plugin) to disable async I/O. * Implement a general memory caching object. Currently, this will be used by the Posix object. * Allow optional memory caching when using the Posix library. This is primarily used by the proxy object to reduce trips to a data server when small blocks are accessed via the proxy server. This requires configuration using the new 'pss.memcache' directive. * Finally implement adding authentication information to the user monitoring record (requested by Matevz Tadel, CMS). This adds a new generic option, auth, to the xrootd.monitor directive. It needs to be specified for the authentication information to be added. This keeps backward compatibility. * Add a new method, chksum, to the standard filesystem interface. * Integrate checksums into the logical filesystem layer implementation. See the ofs.ckslib directive on how to do non-default configuration. This also added a more effecient lfn2pfn() method to the storage system. * Allow native checksums to be enabled in the xrootd layer. See the xrootd.chksum directive on how to do this. * Add checksum management to the frm_admin command. * Allow XrdOucProg to dispatch a local program as well as a process. * Allow a line to be insrerted into an XrdOucStream managed stream. * Implement native checksums usable stand-alone or as plugins. Three digests are supported: adler32, crc32, and md5. An additional digest can be added via a plugin. Also, the native digests can be over-ridden via a plugin. * In XrdSecgsi, new interface for the authorization plug-in which has now full access to the XrdSecEntity object, with the possibility to fill/modify all the fields according to the proxy chain. The plug-in is now called at the end of the all process, after a successful handshake and DN-username mapping. Implementations must contain three extern C functions; see the dummy example provided in src/XrdSecgsi/XrdSecgsiAuthzFunDN.cc. See also the header of XrdSecProtocolgsi::LoadAuthzFun. * In XrdCryptosslgsiAux, add function to extract the VOMS attributes; can be used in authz plug-ins. * In XrdSecgsi, add possibility to extract the VOMS attributes and save them in the XrdSecEntity. New switch '-vomsat:0/1 [1]'. * In 'xrdgsiproxy info' show also the VOMS attributes, if present. * Automatically build the RPM for the xrootd user when an OSG build is detected and add fedora > 15 init scripts dependencies + **Major bug fixes** * Do not close the loger's shadow file descriptor when backgrounding as this may cause random crashes later on. * Avoid SEGV by setting network pointer prior to loading the 1st protocol. * Enforce r/o path during mkdir operations. * Avoid segv when initializing the finder on a multi-core machine. * Fix incorrect lock handling for multiple waiters. * Fix possible deadlocks in XrdSutCache preventing the pwd security module to work correctly + **Minor bug fixes** * Properly handle the case when a site has an excessive number of groups assignments. * Prevent the response to a query from being truncated on the client side. * Report readv information in the detailed monitoring stream. * Correct default settings due to feature interactions after the fact. Now, oss.defaults acts as if the setting were actually specified via oss.export. * Actually use the N2N library of specified or implied via pss.localroot for proxy server interactions withthe origin (required for Atlas T2). * Use re-enterant versions of getpwuid() and getpwgid(). This is need for FUSE. * Correct bad english in a few error messages. * Set correct checksum length when converting ASCII to binary. * Allow the sss protocol to work for multi-homed hosts. * Correct definition of AtomicISM that caused the maximum link count to never be updated in the statistics. * Apply N2N mapping to source path when relocating the file. * Report correct port when locate is directly issued to a data server (before it was being reported as 0). * Make the default file system a pointer to a dynamic instance of XrdOfs instead of a global static (i.e. the Andreas Peters patch). This makes writing an ofs plugin easier. * Fix the RPM uninstall scriptlets incorrectly invoking /sbin/ldconfig. * Install XrdOlbMonPerf and netchk tools. * Fix a bug preventing the core of authentication errors to be logged to clients * In the krb5 security plugin, define KRB5CCNAME to point to the credential cache file /tmp/krb5cc_ only if this file exists and is readable. Solves an issue with credentials cached in memory (API::n). * Fix array deletion mismatches reported by cppcheck (from D. Volgyes) * Make sure that loading of XrdSecgsi.so fails if either the GMAPFun or the AuthzFun plug-ins fail to load. + **Miscellaneous** * Drop Windows support. * Code cleanup: remove XrdTokenAuthzOfs, simple tests, broken utilities, the gridftp code, krb4 and secssl plugins, obsolete documentation files * Make the loadable module extensions configurable depending on the platform (so on Linux and Solaris, dylib on MacOs) * Add new XrdVNUMBER macro. * Clean up the conditional compilation macros. * Remove compression related attributes (compchk, ssdec) and directives (compdetect) as they were never used nor fully implemented. * Remove the userprty directive. It was deprecated and never specified. * Refactor PosixPreeload and Posix libraries to prevent split initialization of the preload library which will cause failures on certain systems. * Provide automatic proxy checksum defaults when role is set to proxy. * Remove all references via extern statements to object instances. This only applies to the Xrd package. * Do not echo lines qualified by an in-line if when the if fails. * Remove the old "redirect" directive. It has passed its prime. * Remove back references to symbols defined in XrdXrootd package used by the cms client to allow for clean shared library builds. * Remove externs to XrdSecGetProtocol and XrdSecGetService from XrdSecInterface.hh to avoid having undefined references just because the include file was included somewhere. * Rename XrdNetDNS to XrdSysDNS to avoid cross-dependencies. This means that all plug-in developers will need to do the same as XrdNetDNS no longer exists. * Split XrdFrm into XrdFrm and XrdFrc. This prevents cross-dependencies in packages that use the File Residency Manager. ------------- Version 3.0.5 ------------- + **Major bug fixes** * Avoid stage failures when target file exists in purgeable or writable space. * Make sure all the threads are joined when closing a physical connection. * Fix free/delete mismatch in XrdSecProtocolgsi et al. + **Minor bug fixes** * Remove old async shutdown workaround patch introduced in Linux 2.3. The problem has been since fixed and the solution now causes problems. * Install the netchk tool ------------- Version 3.0.4 ------------- + **New features** * xrdcp now has -version parameter * xrdcp automatically ads the oss.asize hint to the url opaque data. This functionality may be disabled by setting the XrdCpSizeHint variable to 0 (XRD_XRDCPSIZEHIN in the shell). * The client will try to resolve the server hostname on every retry to enable DNS failovers. * RPM: devel package split into libs-devel, client-devel and server-devel * XrootdFS: all paramenters can be passed via command line, add -h. * Allow a plugin (notably the proxy plugin) to disable async I/O. * New class XrdSysRWLock interfacing the pthread_rwlock functionality * In XrdSecEntity: Add new fields 'creds' and 'credslen' to be filled with the raw client credentials * In XrdSutCache: Use XrdSysRWLock to coordinate concurrent access to the cache * In XrdSecgsi: - Add option to have Entity.name filled with the plain DN, instead of the DN hash, when no mapping is requested or found. - Enable cache also for authz mapping results. - Do not require the existence of a grid-mapfile if gmapopt=2 and there is at least a gmapfun or an authzfun defined. - Add example of mapping function allowing to match parts of the DN - Extend existing option 'authzpxy' to allow exporting the incoming client credentials in XrdSecEntity. + **Major bug fixes** * Async write errors are now being properly caught and reacted to. XrdClient::Close will now fail if it cannot recover from async write errors. * xrdcp prints an error message and returns failure to the shell when some of the write requests it issues fail. * libXrdPosixPreload now builds with autotools and is included into the xrootd-client RPM * RPM: FFS moved from libs to client * Properly parse oss.asize. This because a major problem when xrdcp started adding this to the url which causes the copy to fail. * Spin connection portion of proxy initialization to a background thread. This prevents init.d hangs when a redirector is not available. + **Minor bug fixes** * Test for 64-bit atomics instead 32-bit ones. Fixes build on 32-bit PowerPC. * RPM: xrootd-fuse now depends on fuse * Take correctly into accoutn summer time in calculating the time left for a proxy validity * Add support for Ubuntu 11 which uses the directory /usr/lib/`dpkg-architecture -qDEB_HOST_MULTIARCH` to store platform dependent libraries. ------------- Version 3.0.3 ------------- + **New features** * Change configure.classic to handle various versions of processors in a more sane way (this fixes several Solaris issues and atomics for i686). * Add fwdwait option to cms.request directive to allow pacing of forwarded requests (off by default). * Use native memory synchronization primitives when available when doing network I/O. This will eventually be extended to cover all other cases. * Add the qdl option to the cms.delay directive to allow changing the query window independently of the time a client is asked to wait for the query to actually complete. * Add 'pss.namelib' directive to allow proxies to pre-translate the lfn for servers that cannot do so (e.g., dCache xrootd door). * Optimize handling of shared-everything ile systems (e.g., dCache, GPFS, DFS, Lustre, etc.) in the cmsd. * Implement optional throttling for meta-manager requests in the cmsd. * New cmsd directive, cms.dfs, declares that the underlying file system is a shared-everything system (i.e., distributed file system) and allow for optimal configuration and meta-manager throttling. * Change the oss and fm components to use file extended attributes instead of meta-files. This affects copy, create, reloc, rename, and unlink in the oss layer. Migrate, purge, transfer, and most admin commands in the frm component. The 'all.export' directive now accepts the noxattr/xattr option. WARNING: If the migrator or purge options have been specified for any path in the 'all.export; directive then this change requires either the the 'oss.runmodeold' directive be added to the configuration file to provide backward compatibility or that the name and data spaces be migrated using the frm_admin command. See "Migrating tp Extended Attributes" manual for detailed information and the new 'frm_admin convert' subcommand. * Avoid physical copy if the operation can be handled using hard links. This greatly speeds up static space token reassignment. * Add platform independent interface to extended file attributes. * RPM packaging and Red Hat Enterprise Linux compatible init scripts capable of handling multiple instances of the xrootd daemons. The instances can be defined in the /etc/sysconfig/xrootd file and then handled using standard:: service xrootd start|stop|... service cmsd start|stop|... ... or handled by name:: service xrootd start instance1 instance5 * New '-s' commandline option for xrootd, cmsd, frm_purged and frm_xfrd creating a pidfile. * xrootd, cmsd, frm_purged and frm_xfrd now return failure to the shell when called with '-b' option (daemonization) and the daemon fails to initialize. * New 'EnableTCPKeepAlive' client environment option added enabling the TCP stack keep-alive functionality for the sockets. On Linux three addtional fine-tunning options are available: - TCPKeepAliveTime - interval (in seconds) between the last data packet and the first keep-alive probe - TCPKeepAliveInterval - interval (in seconds) between the probes - TCPKeepAliveProbes - number of probes lost to consider the connection broken * New functionality handling process forking. When enabled (via the 'EnableForkHandlers' env option) prior to a call to fork it shuts down all the xrootd connection management facilities (including the connections themselves) and reinitializes them after the fork both in the parent and the child process. This ensures relative fork safety provided that all the XrdClient and XrdAdmin instances are closed when the fork function is invoked. + **Major bug fixes** * Add missing braces that caused config failure in frm_admin command. * Account for correct path when -M value is zero in hpsscp command. * In XrdCryptossl, fix for thread-safeness; solves random crashes observed on the server side under high authentication frequency * In XrdOucBonjour, fix important issue with host domain name registration, preventing the correct domain to be posted. + **Minor bug fixes** * Correct file discovery propogation for proxy manager relative to meta-managers. * Correct oss partition selection algorithm to further spread out file allocation. * Allow underscores in set/setenv variables. * Add null byte after checksum value response. * Move mapping of errno to xrootd error code to the protocol package where it belongs. This also removes a cross dependency. * Correct RetToken() behaviour in the presence of multiple spaces between tokens and the previous call returned the remainder of the line (very obscure circumstances). * [bug #77535] xrdcp now returns an error to the shell when it fails to copy the file * [bug #79710] xrdcp now gracefully aborts when it encounters a corrupted local file * Reset the transaction timeout for the Query method. This fixes transaction timeout issues for clients doing only queries. * Rename variable to remove conflict between it and global of the same name. * Fix frm_admin command line option parsing so it does not trip over subcommand options. This also fixes a SEGV in MacOS when this actually happens. * Enable the '-md5' option when OpenSSL is present and xrootd is built with autotools. + **Documentation** * Added man pages for: xprep, xrd, xrdcp, xrdstagetool, xrdgsiproxy ------------- Version 3.0.2 ------------- + **Minor bug fixes** * Fix the build on Solaris 10. * Fix the build on SLC4. * Fix the out-of-the-source-tree builds with autotools. * Fix a segfault while doing a recursive copy from root:// to root://. ------------- Version 3.0.1 ------------- + **New features** * New application, cconfing, added to display configuration files relative to a host-program-instance. * New application, netchk, that tests that firewalls have been correctly setup. * New configure.classic option to allow use of stl4port library for Solaris. * New internal feature in XrdPosix library to not shadow files with actual file descriptors (used by the proxy service). This increases scalability. * Allow the xrootd server to tell the client that it is a meta-manager. * Support fo proxies generated by Globus version 4.2.1 in libXrdSecssl. + **Major bug fixes** * Change link options for xrdadler32 to not use shared libraries. The previous setup caused the command to hang upon exit. * Remove instance of XrdPosixXrootd from that same file. Including it disallows defaults from being changed. + **Minor bug fixes** * Fix XrdOucStream to not return ending "fi". * Correct network option interference -- do not turn on network nodnr option should the keepalive option be specified. * Remove duplicate option in option table used by the proxy service. * Compile on Solaris 11 Express using SunCC. * Compile on Windows using MSVC++2010. xrootd-5.6.9/src/XrdCeph/packaging/000077500000000000000000000000001457266313600171375ustar00rootroot00000000000000xrootd-5.6.9/src/XrdCeph/packaging/debian/000077500000000000000000000000001457266313600203615ustar00rootroot00000000000000xrootd-5.6.9/src/XrdCeph/packaging/debian/compat000066400000000000000000000000031457266313600215600ustar00rootroot0000000000000010 xrootd-5.6.9/src/XrdCeph/packaging/debian/control000066400000000000000000000043521457266313600217700ustar00rootroot00000000000000Source: xrootd Maintainer: Jozsef Makai Section: misc Priority: optional Standards-Version: 3.9.3 Build-Depends: debhelper (>= 9), cmake (>=3.3.0), zlib1g-dev, libfuse-dev, python-dev, libssl-dev, libxml2-dev, ncurses-dev, libkrb5-dev, libreadline-dev, libsystemd-dev, selinux-policy-dev, systemd Homepage: https://github.com/xrootd/xrootd Vcs-Git: https://github.com/xrootd/xrootd.git Vcs-Browser: https://github.com/xrootd/xrootd Package: xrootd-libs Architecture: any Description: This package contains libraries used by the xrootd servers and clients. Package: xrootd-devel Architecture: any Depends: ${shlibs:Depends}, xrootd-libs (=${binary:Version}) Description: This package contains header files and development libraries for xrootd development. Package: xrootd-client-libs Architecture: any Depends: ${shlibs:Depends}, xrootd-libs (=${binary:Version}) Description: This package contains libraries used by xrootd clients. Package: xrootd-client-devel Architecture: any Depends: ${shlibs:Depends}, xrootd-devel (=${binary:Version}), xrootd-client-libs (=${binary:Version}) Description: This package contains header files and development libraries for xrootd client development. Package: xrootd-client Architecture: any Depends: ${shlibs:Depends}, libxml2, xrootd-libs (=${binary:Version}), xrootd-client-libs (=${binary:Version}) Description: This package contains the command line tools used to communicate with xrootd servers. Package: xrootd-private-devel Architecture: any Depends: ${shlibs:Depends}, xrootd-libs (=${binary:Version}) Description: This package contains some private xrootd headers. The use of these headers is strongly discouraged. Backward compatibility between versions is not guaranteed for these headers. Package: xrootd-server-libs Architecture: any Depends: ${shlibs:Depends}, xrootd-libs (=${binary:Version}), xrootd-client-libs (=${binary:Version}) Description: This package contains libraries used by xrootd servers. Package: xrootd-server-devel Architecture: any Depends: ${shlibs:Depends}, xrootd-devel (=${binary:Version}), xrootd-client-devel (=${binary:Version}), xrootd-server-libs (=${binary:Version}) Description: This package contains header files and development libraries for xrootd server development. xrootd-5.6.9/src/XrdCeph/packaging/debian/copyright000066400000000000000000000026301457266313600223150ustar00rootroot00000000000000"Copyright (c) 2005-2012, Board of Trustees of the Leland Stanford, Jr. University.\n" "Produced under contract DE-AC02-76-SF00515 with the US Department of Energy. \n" "All rights reserved. The copyright holder's institutional names may not be used to\n" "endorse or promote products derived from this software without specific prior \n" "written permission.\n\n" "This file is part of the XRootD software suite. \n\n" "XRootD is free software: you can redistribute it and/or modify it under the terms \n" "of the GNU Lesser General Public License as published by the Free Software \n" "Foundation, either version 3 of the License, or (at your option) any later version.\n\n" "XRootD is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;\n" "without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR \n" "PURPOSE. See the GNU Lesser General Public License for more details. \nn" "You should have received a copy of the GNU Lesser General Public License along \n" "with XRootD in a file called COPYING.LESSER (LGPL license) and file COPYING (GPL \n" "license). If not, see .\n\n" "Prior to September 2nd, 2012 the XRootD software suite was licensed under a\n" "modified BSD license (see file COPYING.BSD). This applies to all code that\n" "was in the XRootD git repository prior to that date.\n" xrootd-5.6.9/src/XrdCeph/packaging/debian/rules000077500000000000000000000010321457266313600214350ustar00rootroot00000000000000#!/usr/bin/make -f %: dh $@ --builddirectory=build --destdir=deb_packages override_dh_auto_configure: dh_auto_configure -- -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_INSTALL_LIBDIR=lib/$(shell dpkg-architecture -qDEB_HOST_MULTIARCH) override_dh_install: install -D -m 644 packaging/common/client.conf deb_packages/etc/xrootd/client.conf install -D -m 644 packaging/common/client-plugin.conf.example deb_packages/etc/xrootd/client.plugins.d/client-plugin.conf.example dh_install --sourcedir=deb_packages xrootd-5.6.9/src/XrdCeph/packaging/debian/source/000077500000000000000000000000001457266313600216615ustar00rootroot00000000000000xrootd-5.6.9/src/XrdCeph/packaging/debian/source/format000066400000000000000000000000141457266313600230670ustar00rootroot000000000000003.0 (quilt) xrootd-5.6.9/src/XrdCeph/packaging/debian/xrootd-client-devel.install000066400000000000000000000003461457266313600256440ustar00rootroot00000000000000usr/bin/xrdgsitest usr/lib/*/libXrdCl.so usr/lib/*/libXrdClient.so usr/lib/*/libXrdFfs.so usr/lib/*/libXrdPosix.so usr/share/man/man1/xrdgsitest.1* usr/include/xrootd/XrdCl usr/include/xrootd/XrdClient usr/include/xrootd/XrdPosix xrootd-5.6.9/src/XrdCeph/packaging/debian/xrootd-client-libs.install000066400000000000000000000003751457266313600255000ustar00rootroot00000000000000usr/lib/*/libXrdCl.so.2* usr/lib/*/libXrdClient.so.2* usr/lib/*/libXrdFfs.so.2* usr/lib/*/libXrdPosix.so.2* usr/lib/*/libXrdPosixPreload.so.1* usr/lib/*/libXrdPosixPreload.so etc/xrootd/client.plugins.d/client-plugin.conf.example etc/xrootd/client.conf xrootd-5.6.9/src/XrdCeph/packaging/debian/xrootd-client-libs.postinst000066400000000000000000000000261457266313600257060ustar00rootroot00000000000000#!/bin/bash ldconfig xrootd-5.6.9/src/XrdCeph/packaging/debian/xrootd-client-libs.postrm000066400000000000000000000000261457266313600253470ustar00rootroot00000000000000#!/bin/bash ldconfig xrootd-5.6.9/src/XrdCeph/packaging/debian/xrootd-client.install000066400000000000000000000006461457266313600245520ustar00rootroot00000000000000usr/bin/xprep usr/bin/xrd usr/bin/xrdadler32 usr/bin/xrdcopy usr/bin/xrdcp usr/bin/xrdcp-old usr/bin/xrdfs usr/bin/xrdgsiproxy usr/bin/xrdstagetool usr/share/man/man1/xprep.1* usr/share/man/man1/xrd.1* usr/share/man/man1/xrdadler32.1* usr/share/man/man1/xrdcopy.1* usr/share/man/man1/xrdcp.1* usr/share/man/man1/xrdcp-old.1* usr/share/man/man1/xrdfs.1* usr/share/man/man1/xrdgsiproxy.1* usr/share/man/man1/xrdstagetool.1* xrootd-5.6.9/src/XrdCeph/packaging/debian/xrootd-devel.install000066400000000000000000000006331457266313600243670ustar00rootroot00000000000000usr/bin/xrootd-config usr/include/xrootd/XProtocol usr/include/xrootd/Xrd usr/include/xrootd/XrdCks usr/include/xrootd/XrdNet usr/include/xrootd/XrdOuc usr/include/xrootd/XrdSec usr/include/xrootd/XrdSys usr/include/xrootd/XrdVersion.hh usr/include/xrootd/XrdXml/XrdXmlReader.hh usr/lib/*/libXrdAppUtils.so usr/lib/*/libXrdCrypto.so usr/lib/*/libXrdCryptoLite.so usr/lib/*/libXrdUtils.so usr/lib/*/libXrdXml.so xrootd-5.6.9/src/XrdCeph/packaging/debian/xrootd-libs.install000066400000000000000000000004071457266313600242200ustar00rootroot00000000000000usr/lib/*/libXrdAppUtils.so.1* usr/lib/*/libXrdClProxyPlugin-4.so usr/lib/*/libXrdCks*-4.so usr/lib/*/libXrdCrypto.so.1* usr/lib/*/libXrdCryptoLite.so.1* usr/lib/*/libXrdCryptossl-4.so usr/lib/*/libXrdSec*-4.so usr/lib/*/libXrdUtils.so.* usr/lib/*/libXrdXml.so.* xrootd-5.6.9/src/XrdCeph/packaging/debian/xrootd-libs.postinst000066400000000000000000000000261457266313600244320ustar00rootroot00000000000000#!/bin/bash ldconfig xrootd-5.6.9/src/XrdCeph/packaging/debian/xrootd-libs.postrm000066400000000000000000000000261457266313600240730ustar00rootroot00000000000000#!/bin/bash ldconfig xrootd-5.6.9/src/XrdCeph/packaging/debian/xrootd-private-devel.install000066400000000000000000000001211457266313600260270ustar00rootroot00000000000000usr/include/xrootd/private usr/lib/*/libXrdSsiLib.so usr/lib/*/libXrdSsiShMap.so xrootd-5.6.9/src/XrdCeph/packaging/debian/xrootd-server-devel.install000066400000000000000000000003321457266313600256670ustar00rootroot00000000000000usr/include/xrootd/XrdAcc usr/include/xrootd/XrdCms usr/include/xrootd/XrdFileCache usr/include/xrootd/XrdOss usr/include/xrootd/XrdSfs usr/include/xrootd/XrdXrootd usr/include/xrootd/XrdHttp usr/lib/*/libXrdServer.so xrootd-5.6.9/src/XrdCeph/packaging/debian/xrootd-server-libs.install000066400000000000000000000006231457266313600255240ustar00rootroot00000000000000usr/lib/*/libXrdBwm-4.so usr/lib/*/libXrdPss-4.so usr/lib/*/libXrdXrootd-4.so usr/lib/*/libXrdFileCache-4.so usr/lib/*/libXrdBlacklistDecision-4.so usr/lib/*/libXrdHttp-4.so usr/lib/*/libXrdN2No2p-4.so usr/lib/*/libXrdOssSIgpfsT-4.so usr/lib/*/libXrdServer.so.* usr/lib/*/libXrdSsi-4.so usr/lib/*/libXrdSsiLib.so.* usr/lib/*/libXrdSsiLog-4.so usr/lib/*/libXrdSsiShMap.so.* usr/lib/*/libXrdThrottle-4.so xrootd-5.6.9/src/XrdCeph/packaging/debian/xrootd-server-libs.postinst000066400000000000000000000000261457266313600257360ustar00rootroot00000000000000#!/bin/bash ldconfig xrootd-5.6.9/src/XrdCeph/packaging/debian/xrootd-server-libs.postrm000066400000000000000000000000261457266313600253770ustar00rootroot00000000000000#!/bin/bash ldconfig xrootd-5.6.9/src/XrdCeph/packaging/debian_scripts/000077500000000000000000000000001457266313600221305ustar00rootroot00000000000000xrootd-5.6.9/src/XrdCeph/packaging/debian_scripts/publish_debian_cern.sh000077500000000000000000000033151457266313600264500ustar00rootroot00000000000000#!/bin/bash #------------------------------------------------------------------------------- # Publish debian artifacts on CERN Gitlab CI # Author: Jozsef Makai (11.08.2017) #------------------------------------------------------------------------------- set -e comp=$1 prefix=/eos/project/s/storage-ci/www/debian/xrootd for dist in artful xenial; do echo "Publishing for $dist"; path=$prefix/pool/$dist/$comp/x/xrootd/; mkdir -p $path; if [[ "$comp" == "master" ]]; then find ${path} -type f -name '*deb' -delete; fi cp $dist/*deb $path; mkdir -p $prefix/dists/$dist/$comp/binary-amd64/; (cd $prefix && apt-ftparchive --arch amd64 packages pool/$dist/$comp/ > dists/$dist/$comp/binary-amd64/Packages); gzip -c $prefix/dists/$dist/$comp/binary-amd64/Packages > $prefix/dists/$dist/$comp/binary-amd64/Packages.gz; components=$(find $prefix/dists/$dist/ -mindepth 1 -maxdepth 1 -type d -exec basename {} \; | tr '\n' ' ') if [ -e $prefix/dists/$dist/Release ]; then rm $prefix/dists/$dist/Release fi if [ -e $prefix/dists/$dist/InRelease ]; then rm $prefix/dists/$dist/InRelease fi if [ -e $prefix/dists/$dist/Release.gpg ]; then rm $prefix/dists/$dist/Release.gpg fi apt-ftparchive -o APT::FTPArchive::Release::Origin=CERN -o APT::FTPArchive::Release::Label=XrootD -o APT::FTPArchive::Release::Codename=artful -o APT::FTPArchive::Release::Architectures=amd64 -o APT::FTPArchive::Release::Components="$components" release $prefix/dists/$dist/ > $prefix/dists/$dist/Release; gpg --homedir /home/stci/ --clearsign -o $prefix/dists/$dist/InRelease $prefix/dists/$dist/Release; gpg --homedir /home/stci/ -abs -o $prefix/dists/$dist/Release.gpg $prefix/dists/$dist/Release; done xrootd-5.6.9/src/XrdCeph/packaging/makesrpm.sh000077500000000000000000000177511457266313600213300ustar00rootroot00000000000000#!/bin/bash #------------------------------------------------------------------------------- # Create a source RPM package # Author: Lukasz Janyst (10.03.2011) #------------------------------------------------------------------------------- RCEXP='^[0-9]+\.[0-9]+\.[0-9]+\-rc.*$' CERNEXP='^[0-9]+\.[0-9]+\.[0-9]+\-[0-9]+\.CERN.*$' #------------------------------------------------------------------------------- # Find a program #------------------------------------------------------------------------------- function findProg() { for prog in $@; do if test -x "`which $prog 2>/dev/null`"; then echo $prog break fi done } #------------------------------------------------------------------------------- # Print help #------------------------------------------------------------------------------- function printHelp() { echo "Usage:" 1>&2 echo "${0} [--help] [--source PATH] [--output PATH]" 1>&2 echo " --help prints this message" 1>&2 echo " --source PATH specify the root of the source tree" 1>&2 echo " defaults to ../" 1>&2 echo " --output PATH the directory where the source rpm" 1>&2 echo " should be stored, defaulting to ." 1>&2 echo " --version VERSION the version provided by user" 1>&2 echo " --define 'MACRO EXPR'" 1>&2 } #------------------------------------------------------------------------------- # Parse the commandline, if only we could use getopt... :( #------------------------------------------------------------------------------- SOURCEPATH="../" OUTPUTPATH="." PRINTHELP=0 while test ${#} -ne 0; do if test x${1} = x--help; then PRINTHELP=1 elif test x${1} = x--source; then if test ${#} -lt 2; then echo "--source parameter needs an argument" 1>&2 exit 1 fi SOURCEPATH=${2} shift elif test x${1} = x--output; then if test ${#} -lt 2; then echo "--output parameter needs an argument" 1>&2 exit 1 fi OUTPUTPATH=${2} shift elif test x${1} = x--version; then if test ${#} -lt 2; then echo "--version parameter needs an argument" 1>&2 exit 1 fi USER_VERSION="--version ${2}" shift elif test x${1} = x--define; then if test ${#} -lt 2; then echo "--define parameter needs an argument" 1>&2 exit 1 fi USER_DEFINE="$USER_DEFINE --define \""${2}"\"" shift fi shift done if test $PRINTHELP -eq 1; then printHelp exit 0 fi echo "[i] Working on: $SOURCEPATH" echo "[i] Storing the output to: $OUTPUTPATH" #------------------------------------------------------------------------------- # Check if the source and the output dirs #------------------------------------------------------------------------------- if test ! -d $SOURCEPATH -o ! -r $SOURCEPATH; then echo "[!] Source path does not exist or is not readable" 1>&2 exit 2 fi if test ! -d $OUTPUTPATH -o ! -w $OUTPUTPATH; then echo "[!] Output path does not exist or is not writeable" 1>&2 exit 2 fi #------------------------------------------------------------------------------- # Check if we have all the necassary components #------------------------------------------------------------------------------- if test x`findProg rpmbuild` = x; then echo "[!] Unable to find rpmbuild, aborting..." 1>&2 exit 1 fi if test x`findProg git` = x; then echo "[!] Unable to find git, aborting..." 1>&2 exit 1 fi #------------------------------------------------------------------------------- # Check if the source is a git repository #------------------------------------------------------------------------------- if test ! -d $SOURCEPATH/.git; then echo "[!] I can only work with a git repository" 1>&2 exit 2 fi #------------------------------------------------------------------------------- # Check the version number #------------------------------------------------------------------------------- if test ! -x $SOURCEPATH/genversion.sh; then echo "[!] Unable to find the genversion script" 1>&2 exit 3 fi VERSION=`$SOURCEPATH/genversion.sh --print-only $USER_VERSION $SOURCEPATH 2>/dev/null` if test $? -ne 0; then echo "[!] Unable to figure out the version number" 1>&2 exit 4 fi echo "[i] Working with version: $VERSION" if test x${VERSION:0:1} = x"v"; then VERSION=${VERSION:1} fi #------------------------------------------------------------------------------- # Deal with release candidates #------------------------------------------------------------------------------- RELEASE=1 if test x`echo $VERSION | egrep $RCEXP` != x; then RELEASE=0.`echo $VERSION | sed 's/.*-rc/rc/'` VERSION=`echo $VERSION | sed 's/-rc.*//'` fi #------------------------------------------------------------------------------- # Deal with CERN releases #------------------------------------------------------------------------------- if test x`echo $VERSION | egrep $CERNEXP` != x; then RELEASE=`echo $VERSION | sed 's/.*-//'` VERSION=`echo $VERSION | sed 's/-.*\.CERN//'` fi #------------------------------------------------------------------------------- # In case of user version check if the release number has been provided #------------------------------------------------------------------------------- if test x"$USER_VERSION" != x; then TMP=`echo $VERSION | sed 's#.*-##g'` if test $TMP != $VERSION; then RELEASE=$TMP VERSION=`echo $VERSION | sed 's#-[^-]*$##'` fi fi VERSION=`echo $VERSION | sed 's/-/./g'` echo "[i] RPM compliant version: $VERSION-$RELEASE" #------------------------------------------------------------------------------- # Create a tempdir and copy the files there #------------------------------------------------------------------------------- # exit on any error set -e TEMPDIR=`mktemp -d /tmp/xrootd-ceph.srpm.XXXXXXXXXX` RPMSOURCES=$TEMPDIR/rpmbuild/SOURCES mkdir -p $RPMSOURCES mkdir -p $TEMPDIR/rpmbuild/SRPMS echo "[i] Working in: $TEMPDIR" 1>&2 if test -d rhel -a -r rhel; then for i in rhel/*; do cp $i $RPMSOURCES done fi if test -d common -a -r common; then for i in common/*; do cp $i $RPMSOURCES done fi #------------------------------------------------------------------------------- # Generate the spec file #------------------------------------------------------------------------------- if test ! -r rhel/xrootd-ceph.spec.in; then echo "[!] The specfile template does not exist!" 1>&2 exit 7 fi cat rhel/xrootd-ceph.spec.in | sed "s/__VERSION__/$VERSION/" | \ sed "s/__RELEASE__/$RELEASE/" > $TEMPDIR/xrootd-ceph.spec #------------------------------------------------------------------------------- # Make a tarball of the latest commit on the branch #------------------------------------------------------------------------------- # no more exiting on error set +e CWD=$PWD cd $SOURCEPATH COMMIT=`git log --pretty=format:"%H" -1` if test $? -ne 0; then echo "[!] Unable to figure out the git commit hash" 1>&2 exit 5 fi git archive --prefix=xrootd-ceph/ --format=tar $COMMIT | gzip -9fn > \ $RPMSOURCES/xrootd-ceph.tar.gz if test $? -ne 0; then echo "[!] Unable to create the source tarball" 1>&2 exit 6 fi cd $CWD #------------------------------------------------------------------------------- # Build the source RPM #------------------------------------------------------------------------------- echo "[i] Creating the source RPM..." # Dirty, dirty hack! echo "%_sourcedir $RPMSOURCES" >> $TEMPDIR/rpmmacros eval "rpmbuild --define \"_topdir $TEMPDIR/rpmbuild\" \ --define \"%_sourcedir $RPMSOURCES\" \ --define \"%_srcrpmdir %{_topdir}/SRPMS\" \ --define \"_source_filedigest_algorithm md5\" \ --define \"_binary_filedigest_algorithm md5\" \ ${USER_DEFINE} \ -bs $TEMPDIR/xrootd-ceph.spec > $TEMPDIR/log" if test $? -ne 0; then echo "[!] RPM creation failed" 1>&2 exit 8 fi cp $TEMPDIR/rpmbuild/SRPMS/xrootd-ceph*.src.rpm $OUTPUTPATH rm -rf $TEMPDIR echo "[i] Done." xrootd-5.6.9/src/XrdCeph/packaging/rhel/000077500000000000000000000000001457266313600200715ustar00rootroot00000000000000xrootd-5.6.9/src/XrdCeph/packaging/rhel/xrootd-ceph.spec.in000066400000000000000000000110011457266313600235770ustar00rootroot00000000000000#------------------------------------------------------------------------------- # Helper macros #------------------------------------------------------------------------------- %if %{?rhel:1}%{!?rhel:0} %if %{rhel} >= 7 %define use_systemd 1 %else %define use_systemd 0 %endif %else %if %{?fedora}%{!?fedora:0} >= 19 %define use_systemd 1 %else %define use_systemd 0 %endif %endif %if %{?fedora}%{!?fedora:0} >= 22 %define use_libc_semaphore 1 %else %define use_libc_semaphore 0 %endif %if %{?_with_ceph11:1}%{!?_with_ceph11:0} %define _with_ceph 1 %endif %if %{?rhel:1}%{!?rhel:0} %if %{rhel} > 7 %define use_cmake3 0 %else %define use_cmake3 1 %endif %else %define use_cmake3 0 %endif #------------------------------------------------------------------------------- # Package definitions #------------------------------------------------------------------------------- Name: xrootd-ceph Epoch: 1 Version: __VERSION__ Release: __RELEASE__%{?dist}%{?_with_clang:.clang} Summary: CEPH plug-in for XRootD Group: System Environment/Daemons License: LGPLv3+ URL: http://xrootd.org/ # git clone http://xrootd.org/repo/xrootd.git xrootd # cd xrootd # git-archive master | gzip -9 > ~/rpmbuild/SOURCES/xrootd.tgz Source0: xrootd-ceph.tar.gz BuildRoot: %{_tmppath}/%{name}-root %if %{use_cmake3} BuildRequires: cmake3 %else BuildRequires: cmake %endif %if %{?_with_tests:1}%{!?_with_tests:0} BuildRequires: cppunit-devel %endif BuildRequires: librados-devel = 2:14.2.15 BuildRequires: libradosstriper-devel = 2:14.2.15 %if %{?_with_clang:1}%{!?_with_clang:0} BuildRequires: clang %endif BuildRequires: xrootd-server-devel%{?_isa} = %{epoch}:%{version}-%{release} BuildRequires: xrootd-private-devel%{?_isa} = %{epoch}:%{version}-%{release} BuildRequires: xrootd-libs%{?_isa} = %{epoch}:%{version}-%{release} BuildRequires: xrootd-server-libs%{?_isa} = %{epoch}:%{version}-%{release} BuildRequires: xrootd-client-libs%{?_isa} = %{epoch}:%{version}-%{release} Requires: xrootd-server-libs%{?_isa} = %{epoch}:%{version}-%{release} Requires: xrootd-client-libs%{?_isa} = %{epoch}:%{version}-%{release} Requires: xrootd-libs%{?_isa} = %{epoch}:%{version}-%{release} %description The xrootd-ceph is an OSS layer plug-in for the XRootD server for interfacing with the Ceph storage platform. #------------------------------------------------------------------------------- # Build instructions #------------------------------------------------------------------------------- %prep %setup -c -n xrootd-ceph %build cd xrootd-ceph %if %{?_with_clang:1}%{!?_with_clang:0} export CC=clang export CXX=clang++ %endif mkdir build pushd build %if %{use_cmake3} cmake3 \ %else cmake \ %endif -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=RelWithDebInfo \ %if %{?_with_tests:1}%{!?_with_tests:0} -DENABLE_TESTS=TRUE \ %else -DENABLE_TESTS=FALSE \ %endif ../ make -i VERBOSE=1 %{?_smp_mflags} popd #------------------------------------------------------------------------------- # Installation #------------------------------------------------------------------------------- %install rm -rf $RPM_BUILD_ROOT #------------------------------------------------------------------------------- # Install 4.x.y #------------------------------------------------------------------------------- pushd xrootd-ceph pushd build make install DESTDIR=$RPM_BUILD_ROOT popd # ceph posix unversioned so rm -f $RPM_BUILD_ROOT%{_libdir}/libXrdCephPosix.so %clean rm -rf $RPM_BUILD_ROOT #------------------------------------------------------------------------------- # Files #------------------------------------------------------------------------------- %files %defattr(-,root,root,-) %{_libdir}/libXrdCeph-5.so %{_libdir}/libXrdCephXattr-5.so %{_libdir}/libXrdCephPosix.so* %if %{?_with_tests:1}%{!?_with_tests:0} %files tests %defattr(-,root,root,-) %{_libdir}/libXrdCephTests*.so %endif #------------------------------------------------------------------------------- # Changelog #------------------------------------------------------------------------------- %changelog * Wed Dec 16 2020 George Patargias - updated version for librados-devel and libradosstriper-devel to 14.2.15 following the recent upgrade on external Echo gateways - fixed version in xrootd-ceph shared libraries * Mon Mar 02 2020 Michal Simon - fixed RPM dependencies * Thu Mar 08 2018 Michal Simon - initial release xrootd-5.6.9/src/XrdCeph/src/000077500000000000000000000000001457266313600160025ustar00rootroot00000000000000xrootd-5.6.9/src/XrdCeph/src/CMakeLists.txt000066400000000000000000000005251457266313600205440ustar00rootroot00000000000000 #------------------------------------------------------------------------------- # Include the subcomponents #------------------------------------------------------------------------------- include_directories( ${CMAKE_CURRENT_SOURCE_DIR} ) if( XRDCEPH_SUBMODULE ) add_compile_definitions( XRDCEPH_SUBMODULE ) endif() include( XrdCeph ) xrootd-5.6.9/src/XrdCeph/src/XrdCeph.cmake000066400000000000000000000045011457266313600203410ustar00rootroot00000000000000include_directories( ${XROOTD_INCLUDE_DIR} ) include_directories( ${RADOS_INCLUDE_DIR} ) include_directories( ${CMAKE_SOURCE_DIR}/src ) #------------------------------------------------------------------------------- # XrdCephPosix library version #------------------------------------------------------------------------------- set( XRD_CEPH_POSIX_VERSION 0.0.1 ) set( XRD_CEPH_POSIX_SOVERSION 0 ) #------------------------------------------------------------------------------- # The XrdCephPosix library #------------------------------------------------------------------------------- add_library( XrdCephPosix SHARED XrdCeph/XrdCephPosix.cc XrdCeph/XrdCephPosix.hh ) # needed during the transition between ceph giant and ceph hammer # for object listing API set_property(SOURCE XrdCeph/XrdCephPosix.cc PROPERTY COMPILE_FLAGS " -Wno-deprecated-declarations") target_link_libraries( XrdCephPosix PRIVATE ${XROOTD_LIBRARIES} ${RADOS_LIBS} ) set_target_properties( XrdCephPosix PROPERTIES VERSION ${XRD_CEPH_POSIX_VERSION} SOVERSION ${XRD_CEPH_POSIX_SOVERSION} ) #------------------------------------------------------------------------------- # The XrdCeph module #------------------------------------------------------------------------------- set( LIB_XRD_CEPH XrdCeph-${PLUGIN_VERSION} ) add_library( ${LIB_XRD_CEPH} MODULE XrdCeph/XrdCephOss.cc XrdCeph/XrdCephOss.hh XrdCeph/XrdCephOssFile.cc XrdCeph/XrdCephOssFile.hh XrdCeph/XrdCephOssDir.cc XrdCeph/XrdCephOssDir.hh ) target_link_libraries( ${LIB_XRD_CEPH} PRIVATE ${XROOTD_LIBRARIES} XrdCephPosix ) #------------------------------------------------------------------------------- # The XrdCephXattr module #------------------------------------------------------------------------------- set( LIB_XRD_CEPH_XATTR XrdCephXattr-${PLUGIN_VERSION} ) add_library( ${LIB_XRD_CEPH_XATTR} MODULE XrdCeph/XrdCephXAttr.cc XrdCeph/XrdCephXAttr.hh ) target_link_libraries( ${LIB_XRD_CEPH_XATTR} PRIVATE ${XROOTD_LIBRARIES} XrdCephPosix ) #------------------------------------------------------------------------------- # Install #------------------------------------------------------------------------------- install( TARGETS ${LIB_XRD_CEPH} ${LIB_XRD_CEPH_XATTR} XrdCephPosix LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} ) xrootd-5.6.9/src/XrdCeph/src/XrdCeph/000077500000000000000000000000001457266313600173375ustar00rootroot00000000000000xrootd-5.6.9/src/XrdCeph/src/XrdCeph/XrdCephOss.cc000066400000000000000000000202501457266313600216670ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2014-2015 by European Organization for Nuclear Research (CERN) // Author: Sebastien Ponce //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #include #include #include #include "XrdCeph/XrdCephPosix.hh" #include "XrdOuc/XrdOucEnv.hh" #include "XrdSys/XrdSysError.hh" #include "XrdOuc/XrdOucTrace.hh" #include "XrdOuc/XrdOucStream.hh" #include "XrdOuc/XrdOucName2Name.hh" #ifdef XRDCEPH_SUBMODULE #include "XrdOuc/XrdOucN2NLoader.hh" #else #include "private/XrdOuc/XrdOucN2NLoader.hh" #endif #include "XrdVersion.hh" #include "XrdCeph/XrdCephOss.hh" #include "XrdCeph/XrdCephOssDir.hh" #include "XrdCeph/XrdCephOssFile.hh" XrdVERSIONINFO(XrdOssGetStorageSystem, XrdCephOss); XrdSysError XrdCephEroute(0); XrdOucTrace XrdCephTrace(&XrdCephEroute); // log wrapping function to be used by ceph_posix interface char g_logstring[1024]; static void logwrapper(char *format, va_list argp) { vsnprintf(g_logstring, 1024, format, argp); XrdCephEroute.Say(g_logstring); } /// pointer to library providing Name2Name interface. 0 be default /// populated in case of ceph.namelib entry in the config file /// used in XrdCephPosix extern XrdOucName2Name *g_namelib; extern "C" { XrdOss* XrdOssGetStorageSystem(XrdOss* native_oss, XrdSysLogger* lp, const char* config_fn, const char* parms) { // Do the herald thing XrdCephEroute.SetPrefix("ceph_"); XrdCephEroute.logger(lp); XrdCephEroute.Say("++++++ CERN/IT-DSS XrdCeph"); // set parameters try { ceph_posix_set_defaults(parms); } catch (std::exception &e) { XrdCephEroute.Say("CephOss loading failed with exception. Check the syntax of parameters : ", parms); return 0; } // deal with logging ceph_posix_set_logfunc(logwrapper); return new XrdCephOss(config_fn, XrdCephEroute); } } XrdCephOss::XrdCephOss(const char *configfn, XrdSysError &Eroute) { Configure(configfn, Eroute); } XrdCephOss::~XrdCephOss() { ceph_posix_disconnect_all(); } // declared and used in XrdCephPosix.cc extern unsigned int g_maxCephPoolIdx; int XrdCephOss::Configure(const char *configfn, XrdSysError &Eroute) { int NoGo = 0; XrdOucEnv myEnv; XrdOucStream Config(&Eroute, getenv("XRDINSTANCE"), &myEnv, "=====> "); // If there is no config file, nothing to be done if (configfn && *configfn) { // Try to open the configuration file. int cfgFD; if ((cfgFD = open(configfn, O_RDONLY, 0)) < 0) { Eroute.Emsg("Config", errno, "open config file", configfn); return 1; } Config.Attach(cfgFD); // Now start reading records until eof. char *var; while((var = Config.GetMyFirstWord())) { if (!strncmp(var, "ceph.nbconnections", 18)) { var = Config.GetWord(); if (var) { unsigned long value = strtoul(var, 0, 10); if (value > 0 and value <= 100) { g_maxCephPoolIdx = value; } else { Eroute.Emsg("Config", "Invalid value for ceph.nbconnections in config file (must be between 1 and 100)", configfn, var); return 1; } } else { Eroute.Emsg("Config", "Missing value for ceph.nbconnections in config file", configfn); return 1; } } if (!strncmp(var, "ceph.namelib", 12)) { var = Config.GetWord(); if (var) { // Warn in case parameters were givne char parms[1040]; if (!Config.GetRest(parms, sizeof(parms)) || parms[0]) { Eroute.Emsg("Config", "namelib parameters will be ignored"); } // Load name lib XrdOucN2NLoader n2nLoader(&Eroute,configfn,NULL,NULL,NULL); g_namelib = n2nLoader.Load(var, XrdVERSIONINFOVAR(XrdOssGetStorageSystem), NULL); if (!g_namelib) { Eroute.Emsg("Config", "Unable to load library given in ceph.namelib : %s", var); } } else { Eroute.Emsg("Config", "Missing value for ceph.namelib in config file", configfn); return 1; } } } // Now check if any errors occurred during file i/o int retc = Config.LastError(); if (retc) { NoGo = Eroute.Emsg("Config", -retc, "read config file", configfn); } Config.Close(); } return NoGo; } int XrdCephOss::Chmod(const char *path, mode_t mode, XrdOucEnv *envP) { return -ENOTSUP; } int XrdCephOss::Create(const char *tident, const char *path, mode_t access_mode, XrdOucEnv &env, int Opts) { return -ENOTSUP; } int XrdCephOss::Init(XrdSysLogger *logger, const char* configFn) { return 0; } //SCS - lie to posix-assuming clients about directories [fixes brittleness in GFAL2] int XrdCephOss::Mkdir(const char *path, mode_t mode, int mkpath, XrdOucEnv *envP) { return 0; } //SCS - lie to posix-assuming clients about directories [fixes brittleness in GFAL2] int XrdCephOss::Remdir(const char *path, int Opts, XrdOucEnv *eP) { return 0; } int XrdCephOss::Rename(const char *from, const char *to, XrdOucEnv *eP1, XrdOucEnv *eP2) { return -ENOTSUP; } int XrdCephOss::Stat(const char* path, struct stat* buff, int opts, XrdOucEnv* env) { try { if (!strcmp(path, "/")) { // special case of a stat made by the locate interface // we intend to then list all files memset(buff, 0, sizeof(*buff)); buff->st_mode = S_IFDIR | 0700; return 0; } else { return ceph_posix_stat(env, path, buff); } } catch (std::exception &e) { XrdCephEroute.Say("stat : invalid syntax in file parameters"); return -EINVAL; } } int XrdCephOss::StatFS(const char *path, char *buff, int &blen, XrdOucEnv *eP) { XrdOssVSInfo sP; int rc = StatVS(&sP, 0, 0); if (rc) { return rc; } int percentUsedSpace = (sP.Usage*100)/sP.Total; blen = snprintf(buff, blen, "%d %lld %d %d %lld %d", 1, sP.Free, percentUsedSpace, 0, 0LL, 0); return XrdOssOK; } int XrdCephOss::StatVS(XrdOssVSInfo *sP, const char *sname, int updt) { int rc = ceph_posix_statfs(&(sP->Total), &(sP->Free)); if (rc) { return rc; } sP->Large = sP->Total; sP->LFree = sP->Free; sP->Usage = sP->Total-sP->Free; sP->Extents = 1; return XrdOssOK; } int XrdCephOss::Truncate (const char* path, unsigned long long size, XrdOucEnv* env) { try { return ceph_posix_truncate(env, path, size); } catch (std::exception &e) { XrdCephEroute.Say("truncate : invalid syntax in file parameters"); return -EINVAL; } } int XrdCephOss::Unlink(const char *path, int Opts, XrdOucEnv *env) { try { return ceph_posix_unlink(env, path); } catch (std::exception &e) { XrdCephEroute.Say("unlink : invalid syntax in file parameters"); return -EINVAL; } } XrdOssDF* XrdCephOss::newDir(const char *tident) { return new XrdCephOssDir(this); } XrdOssDF* XrdCephOss::newFile(const char *tident) { return new XrdCephOssFile(this); } xrootd-5.6.9/src/XrdCeph/src/XrdCeph/XrdCephOss.hh000066400000000000000000000072201457266313600217030ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2014-2015 by European Organization for Nuclear Research (CERN) // Author: Sebastien Ponce //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #ifndef __CEPH_OSS_HH__ #define __CEPH_OSS_HH__ #include #include //------------------------------------------------------------------------------ //! This class implements XrdOss interface for usage with a CEPH storage. //! It should be loaded via the ofs.osslib directive. //! //! This plugin is able to use any pool of ceph with any userId. //! There are several ways to provide the pool and userId to be used for a given //! operation. Here is the ordered list of possibilities. //! First one defined wins : //! - the path can be prepended with userId and pool. Syntax is : //! [[userId@]pool:] //! - the XrdOucEnv parameter, when existing, can have 'cephUserId' and/or //! 'cephPool' entries //! - the ofs.osslib directive can provide an argument with format : //! [userID@]pool //! - default are 'admin' and 'default' for userId and pool respectively //! //! Note that the definition of a default via the ofs.osslib directive may //! clash with one used in a ofs.xattrlib directive. In case both directives //! have a default and they are different, the behavior is not defined. //! In case one of the two only has a default, it will be applied for both plugins. //------------------------------------------------------------------------------ class XrdCephOss : public XrdOss { public: XrdCephOss(const char *, XrdSysError &); virtual ~XrdCephOss(); int Configure(const char *, XrdSysError &); virtual int Chmod(const char *, mode_t mode, XrdOucEnv *eP=0); virtual int Create(const char *, const char *, mode_t, XrdOucEnv &, int opts=0); virtual int Init(XrdSysLogger *, const char*); virtual int Mkdir(const char *, mode_t mode, int mkpath=0, XrdOucEnv *eP=0); virtual int Remdir(const char *, int Opts=0, XrdOucEnv *eP=0); virtual int Rename(const char *, const char *, XrdOucEnv *eP1=0, XrdOucEnv *eP2=0); virtual int Stat(const char *, struct stat *, int opts=0, XrdOucEnv *eP=0); virtual int StatFS(const char *path, char *buff, int &blen, XrdOucEnv *eP=0); virtual int StatVS(XrdOssVSInfo *sP, const char *sname=0, int updt=0); virtual int Truncate(const char *, unsigned long long, XrdOucEnv *eP=0); virtual int Unlink(const char *path, int Opts=0, XrdOucEnv *eP=0); virtual XrdOssDF *newDir(const char *tident); virtual XrdOssDF *newFile(const char *tident); }; #endif /* __CEPH_OSS_HH__ */ xrootd-5.6.9/src/XrdCeph/src/XrdCeph/XrdCephOssDir.cc000066400000000000000000000040011457266313600223220ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2014-2015 by European Organization for Nuclear Research (CERN) // Author: Sebastien Ponce //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #include "XrdCeph/XrdCephPosix.hh" #include "XrdCeph/XrdCephOssDir.hh" #include "XrdSys/XrdSysError.hh" #include "XrdOuc/XrdOucTrace.hh" extern XrdSysError XrdCephEroute; XrdCephOssDir::XrdCephOssDir(XrdCephOss *cephOss) : m_dirp(0), m_cephOss(cephOss) {} int XrdCephOssDir::Opendir(const char *path, XrdOucEnv &env) { try { m_dirp = ceph_posix_opendir(&env, path); if (0 == m_dirp) { return -errno; } return XrdOssOK; } catch (std::exception &e) { XrdCephEroute.Say("opendir : invalid syntax in file parameters"); return -EINVAL; } } int XrdCephOssDir::Close(long long *retsz) { ceph_posix_closedir(m_dirp); return XrdOssOK; } int XrdCephOssDir::Readdir(char *buff, int blen) { return ceph_posix_readdir(m_dirp, buff, blen); } xrootd-5.6.9/src/XrdCeph/src/XrdCeph/XrdCephOssDir.hh000066400000000000000000000060611457266313600223440ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2014-2015 by European Organization for Nuclear Research (CERN) // Author: Sebastien Ponce //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #ifndef __XRD_CEPH_OSS_DIR_HH__ #define __XRD_CEPH_OSS_DIR_HH__ #include "XrdOss/XrdOss.hh" #include "XrdCeph/XrdCephOss.hh" //------------------------------------------------------------------------------ //! This class implements XrdOssDF interface for usage with a CEPH storage. //! It has a very restricted usage as the only valid path for opendir is '/'. //! The reason is that ceph is an object store where you can only list all //! objects, and that has no notion of hierarchy //! //! This plugin is able to use any pool of ceph with any userId. //! There are several ways to provide the pool and userId to be used for a given //! operation. Here is the ordered list of possibilities. //! First one defined wins : //! - the path can be prepended with userId and pool. Syntax is : //! [[userId@]pool:] //! - the XrdOucEnv parameter, when existing, can have 'cephUserId' and/or //! 'cephPool' entries //! - the ofs.osslib directive can provide an argument with format : //! [userID@]pool //! - default are 'admin' and 'default' for userId and pool respectively //! //! Note that the definition of a default via the ofs.osslib directive may //! clash with one used in a ofs.xattrlib directive. In case both directives //! have a default and they are different, the behavior is not defined. //! In case one of the two only has a default, it will be applied for both plugins. //------------------------------------------------------------------------------ class XrdCephOssDir : public XrdOssDF { public: XrdCephOssDir(XrdCephOss *cephoss); virtual ~XrdCephOssDir() {}; virtual int Opendir(const char *, XrdOucEnv &); virtual int Readdir(char *buff, int blen); virtual int Close(long long *retsz=0); private: DIR *m_dirp; XrdCephOss *m_cephOss; }; #endif /* __XRD_CEPH_OSS_DIR_HH__ */ xrootd-5.6.9/src/XrdCeph/src/XrdCeph/XrdCephOssFile.cc000066400000000000000000000062111457266313600224700ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2014-2015 by European Organization for Nuclear Research (CERN) // Author: Sebastien Ponce //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #include #include #include "XrdCeph/XrdCephPosix.hh" #include "XrdOuc/XrdOucEnv.hh" #include "XrdSys/XrdSysError.hh" #include "XrdOuc/XrdOucTrace.hh" #include "XrdSfs/XrdSfsAio.hh" #include "XrdCeph/XrdCephOssFile.hh" #include "XrdCeph/XrdCephOss.hh" extern XrdSysError XrdCephEroute; XrdCephOssFile::XrdCephOssFile(XrdCephOss *cephOss) : m_fd(-1), m_cephOss(cephOss) {} int XrdCephOssFile::Open(const char *path, int flags, mode_t mode, XrdOucEnv &env) { try { int rc = ceph_posix_open(&env, path, flags, mode); if (rc < 0) return rc; m_fd = rc; return XrdOssOK; } catch (std::exception &e) { XrdCephEroute.Say("open : invalid syntax in file parameters"); return -EINVAL; } } int XrdCephOssFile::Close(long long *retsz) { return ceph_posix_close(m_fd); } ssize_t XrdCephOssFile::Read(off_t offset, size_t blen) { return XrdOssOK; } ssize_t XrdCephOssFile::Read(void *buff, off_t offset, size_t blen) { return ceph_posix_pread(m_fd, buff, blen, offset); } static void aioReadCallback(XrdSfsAio *aiop, size_t rc) { aiop->Result = rc; aiop->doneRead(); } int XrdCephOssFile::Read(XrdSfsAio *aiop) { return ceph_aio_read(m_fd, aiop, aioReadCallback); } ssize_t XrdCephOssFile::ReadRaw(void *buff, off_t offset, size_t blen) { return Read(buff, offset, blen); } int XrdCephOssFile::Fstat(struct stat *buff) { return ceph_posix_fstat(m_fd, buff); } ssize_t XrdCephOssFile::Write(const void *buff, off_t offset, size_t blen) { return ceph_posix_pwrite(m_fd, buff, blen, offset); } static void aioWriteCallback(XrdSfsAio *aiop, size_t rc) { aiop->Result = rc; aiop->doneWrite(); } int XrdCephOssFile::Write(XrdSfsAio *aiop) { return ceph_aio_write(m_fd, aiop, aioWriteCallback); } int XrdCephOssFile::Fsync() { return ceph_posix_fsync(m_fd); } int XrdCephOssFile::Ftruncate(unsigned long long len) { return ceph_posix_ftruncate(m_fd, len); } xrootd-5.6.9/src/XrdCeph/src/XrdCeph/XrdCephOssFile.hh000066400000000000000000000064021457266313600225040ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2014-2015 by European Organization for Nuclear Research (CERN) // Author: Sebastien Ponce //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #ifndef __XRD_CEPH_OSS_FILE_HH__ #define __XRD_CEPH_OSS_FILE_HH__ #include "XrdOss/XrdOss.hh" #include "XrdCeph/XrdCephOss.hh" //------------------------------------------------------------------------------ //! This class implements XrdOssDF interface for usage with a CEPH storage. //! //! This plugin is able to use any pool of ceph with any userId. //! There are several ways to provide the pool and userId to be used for a given //! operation. Here is the ordered list of possibilities. //! First one defined wins : //! - the path can be prepended with userId and pool. Syntax is : //! [[userId@]pool:] //! - the XrdOucEnv parameter, when existing, can have 'cephUserId' and/or //! 'cephPool' entries //! - the ofs.osslib directive can provide an argument with format : //! [userID@]pool //! - default are 'admin' and 'default' for userId and pool respectively //! //! Note that the definition of a default via the ofs.osslib directive may //! clash with one used in a ofs.xattrlib directive. In case both directives //! have a default and they are different, the behavior is not defined. //! In case one of the two only has a default, it will be applied for both plugins. //------------------------------------------------------------------------------ class XrdCephOssFile : public XrdOssDF { public: XrdCephOssFile(XrdCephOss *cephoss); virtual ~XrdCephOssFile() {}; virtual int Open(const char *path, int flags, mode_t mode, XrdOucEnv &env); virtual int Close(long long *retsz=0); virtual ssize_t Read(off_t offset, size_t blen); virtual ssize_t Read(void *buff, off_t offset, size_t blen); virtual int Read(XrdSfsAio *aiop); virtual ssize_t ReadRaw(void *, off_t, size_t); virtual int Fstat(struct stat *buff); virtual ssize_t Write(const void *buff, off_t offset, size_t blen); virtual int Write(XrdSfsAio *aiop); virtual int Fsync(void); virtual int Ftruncate(unsigned long long); private: int m_fd; XrdCephOss *m_cephOss; }; #endif /* __XRD_CEPH_OSS_FILE_HH__ */ xrootd-5.6.9/src/XrdCeph/src/XrdCeph/XrdCephPosix.cc000066400000000000000000001257101457266313600222340ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2014-2015 by European Organization for Nuclear Research (CERN) // Author: Sebastien Ponce //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ /* * This interface provides wrapper methods for using ceph through a POSIX API. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "XrdSfs/XrdSfsAio.hh" #include "XrdSys/XrdSysPthread.hh" #include "XrdOuc/XrdOucName2Name.hh" #include "XrdSys/XrdSysPlatform.hh" #include "XrdCeph/XrdCephPosix.hh" /// small structs to store file metadata struct CephFile { std::string name; std::string pool; std::string userId; unsigned int nbStripes; unsigned long long stripeUnit; unsigned long long objectSize; }; struct CephFileRef : CephFile { int flags; mode_t mode; uint64_t offset; // This mutex protects against parallel updates of the stats. XrdSysMutex statsMutex; uint64_t maxOffsetWritten; uint64_t bytesAsyncWritePending; uint64_t bytesWritten; unsigned rdcount; unsigned wrcount; unsigned asyncRdStartCount; unsigned asyncRdCompletionCount; unsigned asyncWrStartCount; unsigned asyncWrCompletionCount; ::timeval lastAsyncSubmission; double longestAsyncWriteTime; double longestCallbackInvocation; }; /// small struct for directory listing struct DirIterator { librados::NObjectIterator m_iterator; librados::IoCtx *m_ioctx; }; /// small struct for aio API callbacks struct AioArgs { AioArgs(XrdSfsAio* a, AioCB *b, size_t n, int _fd, ceph::bufferlist *_bl=0) : aiop(a), callback(b), nbBytes(n), fd(_fd), bl(_bl) { ::gettimeofday(&startTime, nullptr); } XrdSfsAio* aiop; AioCB *callback; size_t nbBytes; int fd; ::timeval startTime; ceph::bufferlist *bl; }; /// global variables holding stripers/ioCtxs/cluster objects /// Note that we have a pool of them to circumvent the limitation /// of having a single objecter/messenger per IoCtx typedef std::map StriperDict; std::vector g_radosStripers; typedef std::map IOCtxDict; std::vector g_ioCtx; std::vector g_cluster; /// mutex protecting the striper and ioctx maps XrdSysMutex g_striper_mutex; /// index of current Striper/IoCtx to be used unsigned int g_cephPoolIdx = 0; /// size of the Striper/IoCtx pool, defaults to 1 /// may be overwritten in the configuration file /// (See XrdCephOss::configure) unsigned int g_maxCephPoolIdx = 1; /// pointer to library providing Name2Name interface. 0 be default /// populated in case of ceph.namelib entry in the config file in XrdCephOss XrdOucName2Name *g_namelib = 0; /// global variable holding a list of files currently opened for write std::multiset g_filesOpenForWrite; /// global variable holding a map of file descriptor to file reference std::map g_fds; /// global variable remembering the next available file descriptor unsigned int g_nextCephFd = 0; /// mutex protecting the map of file descriptors and the openForWrite multiset XrdSysMutex g_fd_mutex; /// mutex protecting initialization of ceph clusters XrdSysMutex g_init_mutex; /// Accessor to next ceph pool index /// Note that this is not thread safe, but we do not care /// as we only want a rough load balancing unsigned int getCephPoolIdxAndIncrease() { if (g_radosStripers.size() == 0) { // make sure we do not have a race condition here XrdSysMutexHelper lock(g_init_mutex); // double check now that we have the lock if (g_radosStripers.size() == 0) { // initialization phase : allocate corresponding places in the vectors for (unsigned int i = 0; i < g_maxCephPoolIdx; i++) { g_radosStripers.push_back(StriperDict()); g_ioCtx.push_back(IOCtxDict()); g_cluster.push_back(0); } } } unsigned int res = g_cephPoolIdx; unsigned nextValue = g_cephPoolIdx+1; if (nextValue >= g_maxCephPoolIdx) { nextValue = 0; } g_cephPoolIdx = nextValue; return res; } /// check whether a file is open for write bool isOpenForWrite(std::string& name) { XrdSysMutexHelper lock(g_fd_mutex); return g_filesOpenForWrite.find(name) != g_filesOpenForWrite.end(); } /// look for a FileRef from its file descriptor CephFileRef* getFileRef(int fd) { XrdSysMutexHelper lock(g_fd_mutex); std::map::iterator it = g_fds.find(fd); if (it != g_fds.end()) { // We will release the lock upon exiting this function. // The structure here is not protected from deletion, but we trust xrootd to // ensure close (which does the deletion) will not be called before all previous // calls are complete (including the async ones). return &(it->second); } else { return 0; } } /// deletes a FileRef from the global table of file descriptors void deleteFileRef(int fd, const CephFileRef &fr) { XrdSysMutexHelper lock(g_fd_mutex); if (fr.flags & (O_WRONLY|O_RDWR)) { g_filesOpenForWrite.erase(g_filesOpenForWrite.find(fr.name)); } std::map::iterator it = g_fds.find(fd); if (it != g_fds.end()) { g_fds.erase(it); } } /** * inserts a new FileRef into the global table of file descriptors * and return the associated file descriptor */ int insertFileRef(CephFileRef &fr) { XrdSysMutexHelper lock(g_fd_mutex); g_fds[g_nextCephFd] = fr; g_nextCephFd++; if (fr.flags & (O_WRONLY|O_RDWR)) { g_filesOpenForWrite.insert(fr.name); } return g_nextCephFd-1; } /// global variable containing defaults for CephFiles CephFile g_defaultParams = { "", "default", // default pool "admin", // default user 1, // default nbStripes 4 * 1024 * 1024, // default stripeUnit : 4 MB 4 * 1024 * 1024}; // default objectSize : 4 MB std::string g_defaultUserId = "admin"; std::string g_defaultPool = "default"; /// global variable for the log function static void (*g_logfunc) (char *, va_list argp) = 0; static void logwrapper(char* format, ...) { if (0 == g_logfunc) return; va_list arg; va_start(arg, format); (*g_logfunc)(format, arg); va_end(arg); } /// simple integer parsing, to be replaced by std::stoll when C++11 can be used static unsigned long long int stoull(const std::string &s) { char* end; errno = 0; unsigned long long int res = strtoull(s.c_str(), &end, 10); if (0 != *end) { throw std::invalid_argument(s); } if (ERANGE == errno) { throw std::out_of_range(s); } return res; } /// simple integer parsing, to be replaced by std::stoi when C++11 can be used static unsigned int stoui(const std::string &s) { char* end; errno = 0; unsigned long int res = strtoul(s.c_str(), &end, 10); if (0 != *end) { throw std::invalid_argument(s); } if (ERANGE == errno || res > std::numeric_limits::max()) { throw std::out_of_range(s); } return (unsigned int)res; } /// fills the userId of a ceph file struct from a string and an environment /// returns position of first character after the userId static int fillCephUserId(const std::string ¶ms, XrdOucEnv *env, CephFile &file) { // default file.userId = g_defaultParams.userId; // parsing size_t atPos = params.find('@'); if (std::string::npos != atPos) { file.userId = params.substr(0, atPos); return atPos+1; } else { if (0 != env) { char* cuser = env->Get("cephUserId"); if (0 != cuser) { file.userId = cuser; } } return 0; } } /// fills the pool of a ceph file struct from a string and an environment /// returns position of first character after the pool static int fillCephPool(const std::string ¶ms, unsigned int offset, XrdOucEnv *env, CephFile &file) { // default file.pool = g_defaultParams.pool; // parsing size_t comPos = params.find(',', offset); if (std::string::npos == comPos) { if (params.size() == offset) { if (NULL != env) { char* cpool = env->Get("cephPool"); if (0 != cpool) { file.pool = cpool; } } } else { file.pool = params.substr(offset); } return params.size(); } else { file.pool = params.substr(offset, comPos-offset); return comPos+1; } } /// fills the nbStriped of a ceph file struct from a string and an environment /// returns position of first character after the nbStripes // this may raise std::invalid_argument and std::out_of_range static int fillCephNbStripes(const std::string ¶ms, unsigned int offset, XrdOucEnv *env, CephFile &file) { // default file.nbStripes = g_defaultParams.nbStripes; // parsing size_t comPos = params.find(',', offset); if (std::string::npos == comPos) { if (params.size() == offset) { if (NULL != env) { char* cNbStripes = env->Get("cephNbStripes"); if (0 != cNbStripes) { file.nbStripes = stoui(cNbStripes); } } } else { file.nbStripes = stoui(params.substr(offset)); } return params.size(); } else { file.nbStripes = stoui(params.substr(offset, comPos-offset)); return comPos+1; } } /// fills the stripeUnit of a ceph file struct from a string and an environment /// returns position of first character after the stripeUnit // this may raise std::invalid_argument and std::out_of_range static int fillCephStripeUnit(const std::string ¶ms, unsigned int offset, XrdOucEnv *env, CephFile &file) { // default file.stripeUnit = g_defaultParams.stripeUnit; // parsing size_t comPos = params.find(',', offset); if (std::string::npos == comPos) { if (params.size() == offset) { if (NULL != env) { char* cStripeUnit = env->Get("cephStripeUnit"); if (0 != cStripeUnit) { file.stripeUnit = ::stoull(cStripeUnit); } } } else { file.stripeUnit = ::stoull(params.substr(offset)); } return params.size(); } else { file.stripeUnit = ::stoull(params.substr(offset, comPos-offset)); return comPos+1; } } /// fills the objectSize of a ceph file struct from a string and an environment /// returns position of first character after the objectSize // this may raise std::invalid_argument and std::out_of_range static void fillCephObjectSize(const std::string ¶ms, unsigned int offset, XrdOucEnv *env, CephFile &file) { // default file.objectSize = g_defaultParams.objectSize; // parsing if (params.size() == offset) { if (NULL != env) { char* cObjectSize = env->Get("cephObjectSize"); if (0 != cObjectSize) { file.objectSize = ::stoull(cObjectSize); } } } else { file.objectSize = ::stoull(params.substr(offset)); } } /// fill the parameters of a ceph file struct (all but name) from a string and an environment /// see fillCephFile for the detailed syntax void fillCephFileParams(const std::string ¶ms, XrdOucEnv *env, CephFile &file) { // parse the params one by one unsigned int afterUser = fillCephUserId(params, env, file); unsigned int afterPool = fillCephPool(params, afterUser, env, file); unsigned int afterNbStripes = fillCephNbStripes(params, afterPool, env, file); unsigned int afterStripeUnit = fillCephStripeUnit(params, afterNbStripes, env, file); fillCephObjectSize(params, afterStripeUnit, env, file); } /// sets the default userId, pool and file layout /// syntax is [user@]pool[,nbStripes[,stripeUnit[,objectSize]]] /// may throw std::invalid_argument or std::out_of_range in case of error void ceph_posix_set_defaults(const char* value) { if (value) { CephFile newdefault; fillCephFileParams(value, NULL, newdefault); g_defaultParams = newdefault; } } /// converts a logical filename to physical one if needed void translateFileName(std::string &physName, std::string logName){ if (0 != g_namelib) { char physCName[MAXPATHLEN+1]; int retc = g_namelib->lfn2pfn(logName.c_str(), physCName, sizeof(physCName)); if (retc) { logwrapper((char*)"ceph_namelib : failed to translate %s using namelib plugin, using it as is", logName.c_str()); physName = logName; } else { physName = physCName; } } else { physName = logName; } } /// fill a ceph file struct from a path and an environment void fillCephFile(const char *path, XrdOucEnv *env, CephFile &file) { // Syntax of the given path is : // [[userId@]pool[,nbStripes[,stripeUnit[,objectSize]]]:] // for the missing parts, if env is not null the entries // cephUserId, cephPool, cephNbStripes, cephStripeUnit, and cephObjectSize // of env will be used. // If env is null or no entry is found for what is missing, defaults are // applied. These defaults are initially set to 'admin', 'default', 1, 4MB and 4MB // but can be changed via a call to ceph_posix_set_defaults std::string spath = path; size_t colonPos = spath.find(':'); if (std::string::npos == colonPos) { // deal with name translation translateFileName(file.name, spath); fillCephFileParams("", env, file); } else { translateFileName(file.name, spath.substr(colonPos+1)); fillCephFileParams(spath.substr(0, colonPos), env, file); } } static CephFile getCephFile(const char *path, XrdOucEnv *env) { CephFile file; fillCephFile(path, env, file); return file; } static CephFileRef getCephFileRef(const char *path, XrdOucEnv *env, int flags, mode_t mode, unsigned long long offset) { CephFileRef fr; fillCephFile(path, env, fr); fr.flags = flags; fr.mode = mode; fr.offset = 0; fr.maxOffsetWritten = 0; fr.bytesAsyncWritePending = 0; fr.bytesWritten = 0; fr.rdcount = 0; fr.wrcount = 0; fr.asyncRdStartCount = 0; fr.asyncRdCompletionCount = 0; fr.asyncWrStartCount = 0; fr.asyncWrCompletionCount = 0; fr.lastAsyncSubmission.tv_sec = 0; fr.lastAsyncSubmission.tv_usec = 0; fr.longestAsyncWriteTime = 0.0l; fr.longestCallbackInvocation = 0.0l; return fr; } inline librados::Rados* checkAndCreateCluster(unsigned int cephPoolIdx, std::string userId = g_defaultParams.userId) { if (0 == g_cluster[cephPoolIdx]) { // create connection to cluster librados::Rados *cluster = new librados::Rados; if (0 == cluster) { return 0; } int rc = cluster->init(userId.c_str()); if (rc) { logwrapper((char*)"checkAndCreateCluster : cluster init failed"); delete cluster; return 0; } rc = cluster->conf_read_file(NULL); if (rc) { logwrapper((char*)"checkAndCreateCluster : cluster read config failed, rc = %d", rc); cluster->shutdown(); delete cluster; return 0; } cluster->conf_parse_env(NULL); rc = cluster->connect(); if (rc) { logwrapper((char*)"checkAndCreateCluster : cluster connect failed, rc = %d", rc); cluster->shutdown(); delete cluster; return 0; } g_cluster[cephPoolIdx] = cluster; } return g_cluster[cephPoolIdx]; } int checkAndCreateStriper(unsigned int cephPoolIdx, std::string &userAtPool, const CephFile& file) { StriperDict &sDict = g_radosStripers[cephPoolIdx]; StriperDict::iterator it = sDict.find(userAtPool); if (it == sDict.end()) { // we need to create a new radosStriper // Get a cluster librados::Rados* cluster = checkAndCreateCluster(cephPoolIdx, file.userId); if (0 == cluster) { logwrapper((char*)"checkAndCreateStriper : checkAndCreateCluster failed"); return 0; } // create IoCtx for our pool librados::IoCtx *ioctx = new librados::IoCtx; if (0 == ioctx) { logwrapper((char*)"checkAndCreateStriper : IoCtx instantiation failed"); cluster->shutdown(); delete cluster; g_cluster[cephPoolIdx] = 0; return 0; } int rc = g_cluster[cephPoolIdx]->ioctx_create(file.pool.c_str(), *ioctx); if (rc != 0) { logwrapper((char*)"checkAndCreateStriper : ioctx_create failed, rc = %d", rc); cluster->shutdown(); delete cluster; g_cluster[cephPoolIdx] = 0; delete ioctx; return 0; } // create RadosStriper connection libradosstriper::RadosStriper *striper = new libradosstriper::RadosStriper; if (0 == striper) { logwrapper((char*)"checkAndCreateStriper : RadosStriper instantiation failed"); delete ioctx; cluster->shutdown(); delete cluster; g_cluster[cephPoolIdx] = 0; return 0; } rc = libradosstriper::RadosStriper::striper_create(*ioctx, striper); if (rc != 0) { logwrapper((char*)"checkAndCreateStriper : striper_create failed, rc = %d", rc); delete striper; delete ioctx; cluster->shutdown(); delete cluster; g_cluster[cephPoolIdx] = 0; return 0; } // setup layout rc = striper->set_object_layout_stripe_count(file.nbStripes); if (rc != 0) { logwrapper((char*)"checkAndCreateStriper : invalid nbStripes %d", file.nbStripes); delete striper; delete ioctx; cluster->shutdown(); delete cluster; g_cluster[cephPoolIdx] = 0; return 0; } rc = striper->set_object_layout_stripe_unit(file.stripeUnit); if (rc != 0) { logwrapper((char*)"checkAndCreateStriper : invalid stripeUnit %d (must be non 0, multiple of 64K)", file.stripeUnit); delete striper; delete ioctx; cluster->shutdown(); delete cluster; g_cluster[cephPoolIdx] = 0; return 0; } rc = striper->set_object_layout_object_size(file.objectSize); if (rc != 0) { logwrapper((char*)"checkAndCreateStriper : invalid objectSize %d (must be non 0, multiple of stripe_unit)", file.objectSize); delete striper; delete ioctx; cluster->shutdown(); delete cluster; g_cluster[cephPoolIdx] = 0; return 0; } IOCtxDict & ioDict = g_ioCtx[cephPoolIdx]; ioDict.insert(std::pair(userAtPool, ioctx)); sDict.insert(std::pair (userAtPool, striper)).first; } return 1; } static libradosstriper::RadosStriper* getRadosStriper(const CephFile& file) { XrdSysMutexHelper lock(g_striper_mutex); std::stringstream ss; ss << file.userId << '@' << file.pool << ',' << file.nbStripes << ',' << file.stripeUnit << ',' << file.objectSize; std::string userAtPool = ss.str(); unsigned int cephPoolIdx = getCephPoolIdxAndIncrease(); if (checkAndCreateStriper(cephPoolIdx, userAtPool, file) == 0) { logwrapper((char*)"getRadosStriper : checkAndCreateStriper failed"); return 0; } return g_radosStripers[cephPoolIdx][userAtPool]; } static librados::IoCtx* getIoCtx(const CephFile& file) { XrdSysMutexHelper lock(g_striper_mutex); std::stringstream ss; ss << file.userId << '@' << file.pool << ',' << file.nbStripes << ',' << file.stripeUnit << ',' << file.objectSize; std::string userAtPool = ss.str(); unsigned int cephPoolIdx = getCephPoolIdxAndIncrease(); if (checkAndCreateStriper(cephPoolIdx, userAtPool, file) == 0) { return 0; } return g_ioCtx[cephPoolIdx][userAtPool]; } void ceph_posix_disconnect_all() { XrdSysMutexHelper lock(g_striper_mutex); for (unsigned int i= 0; i < g_maxCephPoolIdx; i++) { for (StriperDict::iterator it2 = g_radosStripers[i].begin(); it2 != g_radosStripers[i].end(); it2++) { delete it2->second; } for (IOCtxDict::iterator it2 = g_ioCtx[i].begin(); it2 != g_ioCtx[i].end(); it2++) { delete it2->second; } delete g_cluster[i]; } g_radosStripers.clear(); g_ioCtx.clear(); g_cluster.clear(); } void ceph_posix_set_logfunc(void (*logfunc) (char *, va_list argp)) { g_logfunc = logfunc; }; static int ceph_posix_internal_truncate(const CephFile &file, unsigned long long size); /** * * brief ceph_posix_open function opens a file for read or write * * details This function either: * * Opens a file for reading. If the file doesn't exist, this is an error. * * Opens a file for writing. If the file already exists, check whether overwrite has been requested. If overwrite * * hasn't been requested for an existing file, this is an error. * * param env XrdOucEnv* Unused * * param pathname const char* Specify the file to open. * * param flags int Indicates whether reading or writing, and whether to overwrite an existing file. * * param mode mode_t Unused * * return int This is a file descriptor (non-negative) if the operation is successful, * * or an error code (negative value) if the operation fails * */ int ceph_posix_open(XrdOucEnv* env, const char *pathname, int flags, mode_t mode){ CephFileRef fr = getCephFileRef(pathname, env, flags, mode, 0); struct stat buf; libradosstriper::RadosStriper *striper = getRadosStriper(fr); //Get a handle to the RADOS striper API if (NULL == striper) { logwrapper((char*)"Cannot create striper"); return -EINVAL; } int rc = striper->stat(fr.name, (uint64_t*)&(buf.st_size), &(buf.st_atime)); //Get details about a file bool fileExists = (rc != -ENOENT); //Make clear what condition we are testing if ((flags&O_ACCMODE) == O_RDONLY) { // Access mode is READ if (fileExists) { int fd = insertFileRef(fr); logwrapper((char*)"File descriptor %d associated to file %s opened in read mode", fd, pathname); return fd; } else { return -ENOENT; } } else { // Access mode is WRITE if (fileExists) { if (flags & O_TRUNC) { int rc = ceph_posix_unlink(env, pathname); if (rc < 0 && rc != -ENOENT) { return rc; } } else { return -EEXIST; } } // At this point, we know either the target file didn't exist, or the ceph_posix_unlink above removed it int fd = insertFileRef(fr); logwrapper((char*)"File descriptor %d associated to file %s opened in write mode", fd, pathname); return fd; } } int ceph_posix_close(int fd) { CephFileRef* fr = getFileRef(fd); if (fr) { ::timeval now; ::gettimeofday(&now, nullptr); XrdSysMutexHelper lock(fr->statsMutex); double lastAsyncAge = 0.0; // Only compute an age if the starting point was set. if (fr->lastAsyncSubmission.tv_sec && fr->lastAsyncSubmission.tv_usec) { lastAsyncAge = 1.0 * (now.tv_sec - fr->lastAsyncSubmission.tv_sec) + 0.000001 * (now.tv_usec - fr->lastAsyncSubmission.tv_usec); } logwrapper((char*)"ceph_close: closed fd %d for file %s, read ops count %d, write ops count %d, " "async write ops %d/%d, async pending write bytes %ld, " "async read ops %d/%d, bytes written/max offset %ld/%ld, " "longest async write %f, longest callback invocation %f, last async op age %f", fd, fr->name.c_str(), fr->rdcount, fr->wrcount, fr->asyncWrCompletionCount, fr->asyncWrStartCount, fr->bytesAsyncWritePending, fr->asyncRdCompletionCount, fr->asyncRdStartCount, fr->bytesWritten, fr->maxOffsetWritten, fr->longestAsyncWriteTime, fr->longestCallbackInvocation, (lastAsyncAge)); deleteFileRef(fd, *fr); return 0; } else { return -EBADF; } } static off64_t lseek_compute_offset(CephFileRef &fr, off64_t offset, int whence) { switch (whence) { case SEEK_SET: fr.offset = offset; break; case SEEK_CUR: fr.offset += offset; break; default: return -EINVAL; } return fr.offset; } off_t ceph_posix_lseek(int fd, off_t offset, int whence) { CephFileRef* fr = getFileRef(fd); if (fr) { logwrapper((char*)"ceph_lseek: for fd %d, offset=%lld, whence=%d", fd, offset, whence); return (off_t)lseek_compute_offset(*fr, offset, whence); } else { return -EBADF; } } off64_t ceph_posix_lseek64(int fd, off64_t offset, int whence) { CephFileRef* fr = getFileRef(fd); if (fr) { logwrapper((char*)"ceph_lseek64: for fd %d, offset=%lld, whence=%d", fd, offset, whence); return lseek_compute_offset(*fr, offset, whence); } else { return -EBADF; } } ssize_t ceph_posix_write(int fd, const void *buf, size_t count) { CephFileRef* fr = getFileRef(fd); if (fr) { logwrapper((char*)"ceph_write: for fd %d, count=%d", fd, count); if ((fr->flags & (O_WRONLY|O_RDWR)) == 0) { return -EBADF; } libradosstriper::RadosStriper *striper = getRadosStriper(*fr); if (0 == striper) { return -EINVAL; } ceph::bufferlist bl; bl.append((const char*)buf, count); int rc = striper->write(fr->name, bl, count, fr->offset); if (rc) return rc; fr->offset += count; XrdSysMutexHelper lock(fr->statsMutex); fr->wrcount++; fr->bytesWritten+=count; if (fr->offset) fr->maxOffsetWritten = std::max(fr->offset - 1, fr->maxOffsetWritten); return count; } else { return -EBADF; } } ssize_t ceph_posix_pwrite(int fd, const void *buf, size_t count, off64_t offset) { CephFileRef* fr = getFileRef(fd); if (fr) { // TODO implement proper logging level for this plugin - this should be only debug //logwrapper((char*)"ceph_write: for fd %d, count=%d", fd, count); if ((fr->flags & (O_WRONLY|O_RDWR)) == 0) { return -EBADF; } libradosstriper::RadosStriper *striper = getRadosStriper(*fr); if (0 == striper) { return -EINVAL; } ceph::bufferlist bl; bl.append((const char*)buf, count); int rc = striper->write(fr->name, bl, count, offset); if (rc) return rc; XrdSysMutexHelper lock(fr->statsMutex); fr->wrcount++; fr->bytesWritten+=count; if (offset + count) fr->maxOffsetWritten = std::max(uint64_t(offset + count - 1), fr->maxOffsetWritten); return count; } else { return -EBADF; } } static void ceph_aio_write_complete(rados_completion_t c, void *arg) { AioArgs *awa = reinterpret_cast(arg); size_t rc = rados_aio_get_return_value(c); // Compute statistics before reportng to xrootd, so that a close cannot happen // in the meantime. CephFileRef* fr = getFileRef(awa->fd); if (fr) { XrdSysMutexHelper lock(fr->statsMutex); fr->asyncWrCompletionCount++; fr->bytesAsyncWritePending -= awa->nbBytes; fr->bytesWritten += awa->nbBytes; if (awa->aiop->sfsAio.aio_nbytes) fr->maxOffsetWritten = std::max(fr->maxOffsetWritten, uint64_t(awa->aiop->sfsAio.aio_offset + awa->aiop->sfsAio.aio_nbytes - 1)); ::timeval now; ::gettimeofday(&now, nullptr); double writeTime = 0.000001 * (now.tv_usec - awa->startTime.tv_usec) + 1.0 * (now.tv_sec - awa->startTime.tv_sec); fr->longestAsyncWriteTime = std::max(fr->longestAsyncWriteTime, writeTime); } ::timeval before, after; if (fr) ::gettimeofday(&before, nullptr); awa->callback(awa->aiop, rc == 0 ? awa->nbBytes : rc); if (fr) { ::gettimeofday(&after, nullptr); double callbackInvocationTime = 0.000001 * (after.tv_usec - before.tv_usec) + 1.0 * (after.tv_sec - before.tv_sec); XrdSysMutexHelper lock(fr->statsMutex); fr->longestCallbackInvocation = std::max(fr->longestCallbackInvocation, callbackInvocationTime); } delete(awa); } ssize_t ceph_aio_write(int fd, XrdSfsAio *aiop, AioCB *cb) { CephFileRef* fr = getFileRef(fd); if (fr) { // get the parameters from the Xroot aio object size_t count = aiop->sfsAio.aio_nbytes; const char *buf = (const char*)aiop->sfsAio.aio_buf; size_t offset = aiop->sfsAio.aio_offset; // TODO implement proper logging level for this plugin - this should be only debug //logwrapper((char*)"ceph_aio_write: for fd %d, count=%d", fd, count); if ((fr->flags & (O_WRONLY|O_RDWR)) == 0) { return -EBADF; } // get the striper object libradosstriper::RadosStriper *striper = getRadosStriper(*fr); if (0 == striper) { return -EINVAL; } // prepare a bufferlist around the given buffer ceph::bufferlist bl; bl.append(buf, count); // get the poolIdx to use int cephPoolIdx = getCephPoolIdxAndIncrease(); // Get the cluster to use librados::Rados* cluster = checkAndCreateCluster(cephPoolIdx); if (0 == cluster) { return -EINVAL; } // prepare a ceph AioCompletion object and do async call AioArgs *args = new AioArgs(aiop, cb, count, fd); librados::AioCompletion *completion = cluster->aio_create_completion(args, ceph_aio_write_complete, NULL); // do the write int rc = striper->aio_write(fr->name, completion, bl, count, offset); completion->release(); XrdSysMutexHelper lock(fr->statsMutex); fr->asyncWrStartCount++; ::gettimeofday(&fr->lastAsyncSubmission, nullptr); fr->bytesAsyncWritePending+=count; return rc; } else { return -EBADF; } } ssize_t ceph_posix_read(int fd, void *buf, size_t count) { CephFileRef* fr = getFileRef(fd); if (fr) { // TODO implement proper logging level for this plugin - this should be only debug //logwrapper((char*)"ceph_read: for fd %d, count=%d", fd, count); if ((fr->flags & O_WRONLY) != 0) { return -EBADF; } libradosstriper::RadosStriper *striper = getRadosStriper(*fr); if (0 == striper) { return -EINVAL; } ceph::bufferlist bl; int rc = striper->read(fr->name, &bl, count, fr->offset); if (rc < 0) return rc; bl.begin().copy(rc, (char*)buf); XrdSysMutexHelper lock(fr->statsMutex); fr->offset += rc; fr->rdcount++; return rc; } else { return -EBADF; } } ssize_t ceph_posix_pread(int fd, void *buf, size_t count, off64_t offset) { CephFileRef* fr = getFileRef(fd); if (fr) { // TODO implement proper logging level for this plugin - this should be only debug //logwrapper((char*)"ceph_read: for fd %d, count=%d", fd, count); if ((fr->flags & O_WRONLY) != 0) { return -EBADF; } libradosstriper::RadosStriper *striper = getRadosStriper(*fr); if (0 == striper) { return -EINVAL; } ceph::bufferlist bl; int rc = striper->read(fr->name, &bl, count, offset); if (rc < 0) return rc; bl.begin().copy(rc, (char*)buf); XrdSysMutexHelper lock(fr->statsMutex); fr->rdcount++; return rc; } else { return -EBADF; } } static void ceph_aio_read_complete(rados_completion_t c, void *arg) { AioArgs *awa = reinterpret_cast(arg); size_t rc = rados_aio_get_return_value(c); if (awa->bl) { if (rc > 0) { awa->bl->begin().copy(rc, (char*)awa->aiop->sfsAio.aio_buf); } delete awa->bl; awa->bl = 0; } // Compute statistics before reportng to xrootd, so that a close cannot happen // in the meantime. CephFileRef* fr = getFileRef(awa->fd); if (fr) { XrdSysMutexHelper lock(fr->statsMutex); fr->asyncRdCompletionCount++; } awa->callback(awa->aiop, rc ); delete(awa); } ssize_t ceph_aio_read(int fd, XrdSfsAio *aiop, AioCB *cb) { CephFileRef* fr = getFileRef(fd); if (fr) { // get the parameters from the Xroot aio object size_t count = aiop->sfsAio.aio_nbytes; size_t offset = aiop->sfsAio.aio_offset; // TODO implement proper logging level for this plugin - this should be only debug //logwrapper((char*)"ceph_aio_read: for fd %d, count=%d", fd, count); if ((fr->flags & O_WRONLY) != 0) { return -EBADF; } // get the striper object libradosstriper::RadosStriper *striper = getRadosStriper(*fr); if (0 == striper) { return -EINVAL; } // prepare a bufferlist to receive data ceph::bufferlist *bl = new ceph::bufferlist(); // get the poolIdx to use int cephPoolIdx = getCephPoolIdxAndIncrease(); // Get the cluster to use librados::Rados* cluster = checkAndCreateCluster(cephPoolIdx); if (0 == cluster) { return -EINVAL; } // prepare a ceph AioCompletion object and do async call AioArgs *args = new AioArgs(aiop, cb, count, fd, bl); librados::AioCompletion *completion = cluster->aio_create_completion(args, ceph_aio_read_complete, NULL); // do the read int rc = striper->aio_read(fr->name, completion, bl, count, offset); completion->release(); XrdSysMutexHelper lock(fr->statsMutex); fr->asyncRdStartCount++; return rc; } else { return -EBADF; } } int ceph_posix_fstat(int fd, struct stat *buf) { CephFileRef* fr = getFileRef(fd); if (fr) { logwrapper((char*)"ceph_stat: fd %d", fd); // minimal stat : only size and times are filled // atime, mtime and ctime are set all to the same value // mode is set arbitrarily to 0666 | S_IFREG libradosstriper::RadosStriper *striper = getRadosStriper(*fr); if (0 == striper) { logwrapper((char*)"ceph_stat: getRadosStriper failed"); return -EINVAL; } memset(buf, 0, sizeof(*buf)); int rc = striper->stat(fr->name, (uint64_t*)&(buf->st_size), &(buf->st_atime)); if (rc != 0) { return -rc; } buf->st_mtime = buf->st_atime; buf->st_ctime = buf->st_atime; buf->st_mode = 0666 | S_IFREG; return 0; } else { return -EBADF; } } int ceph_posix_stat(XrdOucEnv* env, const char *pathname, struct stat *buf) { logwrapper((char*)"ceph_stat: %s", pathname); // minimal stat : only size and times are filled // atime, mtime and ctime are set all to the same value // mode is set arbitrarily to 0666 | S_IFREG CephFile file = getCephFile(pathname, env); libradosstriper::RadosStriper *striper = getRadosStriper(file); if (0 == striper) { return -EINVAL; } memset(buf, 0, sizeof(*buf)); int rc = striper->stat(file.name, (uint64_t*)&(buf->st_size), &(buf->st_atime)); if (rc != 0) { // for non existing file. Check that we did not open it for write recently // in that case, we return 0 size and current time if (-ENOENT == rc && isOpenForWrite(file.name)) { buf->st_size = 0; buf->st_atime = time(NULL); } else { return -rc; } } buf->st_mtime = buf->st_atime; buf->st_ctime = buf->st_atime; buf->st_mode = 0666 | S_IFREG; return 0; } int ceph_posix_fsync(int fd) { CephFileRef* fr = getFileRef(fd); if (fr) { // no locking of fr as it is not used. logwrapper((char*)"ceph_sync: fd %d", fd); return 0; } else { return -EBADF; } } int ceph_posix_fcntl(int fd, int cmd, ... /* arg */ ) { CephFileRef* fr = getFileRef(fd); if (fr) { logwrapper((char*)"ceph_fcntl: fd %d cmd=%d", fd, cmd); // minimal implementation switch (cmd) { case F_GETFL: return fr->mode; default: return -EINVAL; } } else { return -EBADF; } } static ssize_t ceph_posix_internal_getxattr(const CephFile &file, const char* name, void* value, size_t size) { libradosstriper::RadosStriper *striper = getRadosStriper(file); if (0 == striper) { return -EINVAL; } ceph::bufferlist bl; int rc = striper->getxattr(file.name, name, bl); if (rc < 0) return rc; size_t returned_size = (size_t)rcsetxattr(file.name, name, bl); if (rc) { return -rc; } return 0; } ssize_t ceph_posix_setxattr(XrdOucEnv* env, const char* path, const char* name, const void* value, size_t size, int flags) { logwrapper((char*)"ceph_setxattr: path %s name=%s value=%s", path, name, value); return ceph_posix_internal_setxattr(getCephFile(path, env), name, value, size, flags); } int ceph_posix_fsetxattr(int fd, const char* name, const void* value, size_t size, int flags) { CephFileRef* fr = getFileRef(fd); if (fr) { logwrapper((char*)"ceph_fsetxattr: fd %d name=%s value=%s", fd, name, value); return ceph_posix_internal_setxattr(*fr, name, value, size, flags); } else { return -EBADF; } } static int ceph_posix_internal_removexattr(const CephFile &file, const char* name) { libradosstriper::RadosStriper *striper = getRadosStriper(file); if (0 == striper) { return -EINVAL; } int rc = striper->rmxattr(file.name, name); if (rc) { return -rc; } return 0; } int ceph_posix_removexattr(XrdOucEnv* env, const char* path, const char* name) { logwrapper((char*)"ceph_removexattr: path %s name=%s", path, name); return ceph_posix_internal_removexattr(getCephFile(path, env), name); } int ceph_posix_fremovexattr(int fd, const char* name) { CephFileRef* fr = getFileRef(fd); if (fr) { logwrapper((char*)"ceph_fremovexattr: fd %d name=%s", fd, name); return ceph_posix_internal_removexattr(*fr, name); } else { return -EBADF; } } static int ceph_posix_internal_listxattrs(const CephFile &file, XrdSysXAttr::AList **aPL, int getSz) { libradosstriper::RadosStriper *striper = getRadosStriper(file); if (0 == striper) { return -EINVAL; } // call ceph std::map attrset; int rc = striper->getxattrs(file.name, attrset); if (rc) { return -rc; } // build result *aPL = 0; int maxSize = 0; for (std::map::const_iterator it = attrset.begin(); it != attrset.end(); it++) { XrdSysXAttr::AList* newItem = (XrdSysXAttr::AList*)malloc(sizeof(XrdSysXAttr::AList)+it->first.size()); newItem->Next = *aPL; newItem->Vlen = it->second.length(); if (newItem->Vlen > maxSize) { maxSize = newItem->Vlen; } newItem->Nlen = it->first.size(); strncpy(newItem->Name, it->first.c_str(), newItem->Vlen+1); *aPL = newItem; } if (getSz) { return 0; } else { return maxSize; } } int ceph_posix_listxattrs(XrdOucEnv* env, const char* path, XrdSysXAttr::AList **aPL, int getSz) { logwrapper((char*)"ceph_listxattrs: path %s", path); return ceph_posix_internal_listxattrs(getCephFile(path, env), aPL, getSz); } int ceph_posix_flistxattrs(int fd, XrdSysXAttr::AList **aPL, int getSz) { CephFileRef* fr = getFileRef(fd); if (fr) { logwrapper((char*)"ceph_flistxattrs: fd %d", fd); return ceph_posix_internal_listxattrs(*fr, aPL, getSz); } else { return -EBADF; } } void ceph_posix_freexattrlist(XrdSysXAttr::AList *aPL) { while (aPL) { free(aPL->Name); XrdSysXAttr::AList *cur = aPL; aPL = aPL->Next; free(cur); } } int ceph_posix_statfs(long long *totalSpace, long long *freeSpace) { logwrapper((char*)"ceph_posix_statfs"); // get the poolIdx to use int cephPoolIdx = getCephPoolIdxAndIncrease(); // Get the cluster to use librados::Rados* cluster = checkAndCreateCluster(cephPoolIdx); if (0 == cluster) { return -EINVAL; } // call ceph stat librados::cluster_stat_t result; int rc = cluster->cluster_stat(result); if (0 == rc) { *totalSpace = result.kb * 1024; *freeSpace = result.kb_avail * 1024; } return rc; } static int ceph_posix_internal_truncate(const CephFile &file, unsigned long long size) { libradosstriper::RadosStriper *striper = getRadosStriper(file); if (0 == striper) { return -EINVAL; } return striper->trunc(file.name, size); } int ceph_posix_ftruncate(int fd, unsigned long long size) { CephFileRef* fr = getFileRef(fd); if (fr) { logwrapper((char*)"ceph_posix_ftruncate: fd %d, size %d", fd, size); return ceph_posix_internal_truncate(*fr, size); } else { return -EBADF; } } int ceph_posix_truncate(XrdOucEnv* env, const char *pathname, unsigned long long size) { logwrapper((char*)"ceph_posix_truncate : %s", pathname); // minimal stat : only size and times are filled CephFile file = getCephFile(pathname, env); return ceph_posix_internal_truncate(file, size); } int ceph_posix_unlink(XrdOucEnv* env, const char *pathname) { logwrapper((char*)"ceph_posix_unlink : %s", pathname); // minimal stat : only size and times are filled CephFile file = getCephFile(pathname, env); libradosstriper::RadosStriper *striper = getRadosStriper(file); if (0 == striper) { return -EINVAL; } return striper->remove(file.name); } DIR* ceph_posix_opendir(XrdOucEnv* env, const char *pathname) { logwrapper((char*)"ceph_posix_opendir : %s", pathname); // only accept root dir, as there is no concept of dirs in object stores CephFile file = getCephFile(pathname, env); if (file.name.size() != 1 || file.name[0] != '/') { errno = -ENOENT; return 0; } librados::IoCtx *ioctx = getIoCtx(file); if (0 == ioctx) { errno = EINVAL; return 0; } DirIterator* res = new DirIterator(); res->m_iterator = ioctx->nobjects_begin(); res->m_ioctx = ioctx; return (DIR*)res; } int ceph_posix_readdir(DIR *dirp, char *buff, int blen) { librados::NObjectIterator &iterator = ((DirIterator*)dirp)->m_iterator; librados::IoCtx *ioctx = ((DirIterator*)dirp)->m_ioctx; while (iterator->get_oid().compare(iterator->get_oid().size()-17, 17, ".0000000000000000") && iterator != ioctx->nobjects_end()) { iterator++; } if (iterator == ioctx->nobjects_end()) { buff[0] = 0; } else { int l = iterator->get_oid().size()-17; if (l < blen) blen = l; strncpy(buff, iterator->get_oid().c_str(), blen-1); buff[blen-1] = 0; iterator++; } return 0; } int ceph_posix_closedir(DIR *dirp) { delete ((DirIterator*)dirp); return 0; } xrootd-5.6.9/src/XrdCeph/src/XrdCeph/XrdCephPosix.hh000066400000000000000000000074551457266313600222530ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2014-2015 by European Organization for Nuclear Research (CERN) // Author: Sebastien Ponce //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ /* * This interface provides wrapper methods for using ceph through a POSIX API. */ #ifndef _XRD_CEPH_POSIX_H #define _XRD_CEPH_POSIX_H #include #include #include #include #include class XrdSfsAio; typedef void(AioCB)(XrdSfsAio*, size_t); void ceph_posix_set_defaults(const char* value); void ceph_posix_disconnect_all(); void ceph_posix_set_logfunc(void (*logfunc) (char *, va_list argp)); int ceph_posix_open(XrdOucEnv* env, const char *pathname, int flags, mode_t mode); int ceph_posix_close(int fd); off_t ceph_posix_lseek(int fd, off_t offset, int whence); off64_t ceph_posix_lseek64(int fd, off64_t offset, int whence); ssize_t ceph_posix_write(int fd, const void *buf, size_t count); ssize_t ceph_posix_pwrite(int fd, const void *buf, size_t count, off64_t offset); ssize_t ceph_aio_write(int fd, XrdSfsAio *aiop, AioCB *cb); ssize_t ceph_posix_read(int fd, void *buf, size_t count); ssize_t ceph_posix_pread(int fd, void *buf, size_t count, off64_t offset); ssize_t ceph_aio_read(int fd, XrdSfsAio *aiop, AioCB *cb); int ceph_posix_fstat(int fd, struct stat *buf); int ceph_posix_stat(XrdOucEnv* env, const char *pathname, struct stat *buf); int ceph_posix_fsync(int fd); int ceph_posix_fcntl(int fd, int cmd, ... /* arg */ ); ssize_t ceph_posix_getxattr(XrdOucEnv* env, const char* path, const char* name, void* value, size_t size); ssize_t ceph_posix_fgetxattr(int fd, const char* name, void* value, size_t size); ssize_t ceph_posix_setxattr(XrdOucEnv* env, const char* path, const char* name, const void* value, size_t size, int flags); int ceph_posix_fsetxattr(int fd, const char* name, const void* value, size_t size, int flags); int ceph_posix_removexattr(XrdOucEnv* env, const char* path, const char* name); int ceph_posix_fremovexattr(int fd, const char* name); int ceph_posix_listxattrs(XrdOucEnv* env, const char* path, XrdSysXAttr::AList **aPL, int getSz); int ceph_posix_flistxattrs(int fd, XrdSysXAttr::AList **aPL, int getSz); void ceph_posix_freexattrlist(XrdSysXAttr::AList *aPL); int ceph_posix_statfs(long long *totalSpace, long long *freeSpace); int ceph_posix_truncate(XrdOucEnv* env, const char *pathname, unsigned long long size); int ceph_posix_ftruncate(int fd, unsigned long long size); int ceph_posix_unlink(XrdOucEnv* env, const char *pathname); DIR* ceph_posix_opendir(XrdOucEnv* env, const char *pathname); int ceph_posix_readdir(DIR* dirp, char *buff, int blen); int ceph_posix_closedir(DIR *dirp); #endif // __XRD_CEPH_POSIX__ xrootd-5.6.9/src/XrdCeph/src/XrdCeph/XrdCephXAttr.cc000066400000000000000000000073761457266313600222030ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2014-2015 by European Organization for Nuclear Research (CERN) // Author: Sebastien Ponce //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #include "XrdVersion.hh" #include "XrdCeph/XrdCephPosix.hh" #include "XrdSys/XrdSysError.hh" #include "XrdOuc/XrdOucTrace.hh" #include "XrdCeph/XrdCephXAttr.hh" XrdSysError XrdCephXattrEroute(0); XrdOucTrace XrdCephXattrTrace(&XrdCephXattrEroute); extern "C" { XrdSysXAttr* XrdSysGetXAttrObject(XrdSysError *errP, const char *config_fn, const char *parms) { // Do the herald thing XrdCephXattrEroute.SetPrefix("cephxattr_"); XrdCephXattrEroute.logger(errP->logger()); XrdCephXattrEroute.Say("++++++ CERN/IT-DSS XrdCephXattr"); // set parameters try { ceph_posix_set_defaults(parms); } catch (std::exception &e) { XrdCephXattrEroute.Say("CephXattr loading failed with exception. Check the syntax of parameters : ", parms); return 0; } return new XrdCephXAttr(); } } XrdCephXAttr::XrdCephXAttr() {} XrdCephXAttr::~XrdCephXAttr() {} int XrdCephXAttr::Del(const char *Aname, const char *Path, int fd) { try { return ceph_posix_removexattr(0, Path, Aname); } catch (std::exception &e) { XrdCephXattrEroute.Say("Del : invalid syntax in file parameters", Path); return -EINVAL; } } void XrdCephXAttr::Free(AList *aPL) { ceph_posix_freexattrlist(aPL); } int XrdCephXAttr::Get(const char *Aname, void *Aval, int Avsz, const char *Path, int fd) { if (fd >= 0) { return ceph_posix_fgetxattr(fd, Aname, Aval, Avsz); } else { try { return ceph_posix_getxattr(0, Path, Aname, Aval, Avsz); } catch (std::exception &e) { XrdCephXattrEroute.Say("Get : invalid syntax in file parameters", Path); return -EINVAL; } } } int XrdCephXAttr::List(AList **aPL, const char *Path, int fd, int getSz) { if (fd > 0) { return ceph_posix_flistxattrs(fd, aPL, getSz); } else { try { return ceph_posix_listxattrs(0, Path, aPL, getSz); } catch (std::exception &e) { XrdCephXattrEroute.Say("List : invalid syntax in file parameters", Path); return -EINVAL; } } } int XrdCephXAttr::Set(const char *Aname, const void *Aval, int Avsz, const char *Path, int fd, int isNew) { if (fd >= 0) { return ceph_posix_fsetxattr(fd, Aname, Aval, Avsz, 0); } else { try { return ceph_posix_setxattr(0, Path, Aname, Aval, Avsz, 0); } catch (std::exception &e) { XrdCephXattrEroute.Say("Set : invalid syntax in file parameters", Path); return -EINVAL; } } } XrdVERSIONINFO(XrdSysGetXAttrObject, XrdCephXAttr); xrootd-5.6.9/src/XrdCeph/src/XrdCeph/XrdCephXAttr.hh000066400000000000000000000173051457266313600222060ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2014-2015 by European Organization for Nuclear Research (CERN) // Author: Sebastien Ponce //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #ifndef __XRD_CEPH_XATTR_HH__ #define __XRD_CEPH_XATTR_HH__ #include //------------------------------------------------------------------------------ //! This class implements XrdSysXAttr interface for usage with a CEPH storage. //! It should be loaded via the ofs.xattrlib directive. //! //! This plugin is able to use any pool of ceph with any userId. //! There are several ways to provide the pool and userId to be used for a given //! operation. Here is the ordered list of possibilities. //! First one defined wins : //! - the path can be prepended with userId and pool. Syntax is : //! [[userId@]pool:] //! - the XrdOucEnv parameter, when existing, can have 'cephUserId' and/or //! 'cephPool' entries //! - the ofs.xattrlib directive can provide an argument with format : //! [userID@]pool //! - default are 'admin' and 'default' for userId and pool respectively //! //! Note that the definition of a default via the ofs.xattrlib directive may //! clash with one used in a ofs.osslib directive. In case both directives //! have a default and they are different, the behavior is not defined. //! In case one of the two only has a default, it will be applied for both plugins. //------------------------------------------------------------------------------ class XrdCephXAttr : public XrdSysXAttr { public: //------------------------------------------------------------------------------ //! Constructor //------------------------------------------------------------------------------ XrdCephXAttr(); //------------------------------------------------------------------------------ //! Destructor //------------------------------------------------------------------------------ virtual ~XrdCephXAttr(); //------------------------------------------------------------------------------ //! Remove an extended attribute. //! //! @param Aname -> The attribute name. //! @param Path -> Path of the file whose attribute is to be removed. //! @param fd If >=0 is the file descriptor of the opened subject file. //! //! @return =0 Attribute was successfully removed or did not exist. //! @return <0 Attribute was not removed, the return value is -errno that //! describes the reason for the failure. //------------------------------------------------------------------------------ virtual int Del(const char *Aname, const char *Path, int fd=-1); //------------------------------------------------------------------------------ //! Release storage occupied by the Alist structure returned by List(). //! //! @param aPL -> The first element of the AList structure. //------------------------------------------------------------------------------ virtual void Free(AList *aPL); //------------------------------------------------------------------------------ //! Get an attribute value and its size. //! //! @param Aname -> The attribute name. //! @param Aval -> Buffer to receive the attribute value. //! @param Avsz Length of the buffer in bytes. Only up to this number of //! bytes should be returned. However, should Avsz be zero //! the the size of the attribute value should be returned //! and the Aval argument should be ignored. //! @param Path -> Path of the file whose attribute is to be fetched. //! @param fd -> If >=0 is the file descriptor of the opened subject file. //! //! @return >0 The number of bytes placed in Aval. However, if avsz is zero //! then the value is the actual size of the attribute value. //! @return =0 The attribute does not exist. //! @return <0 The attribute value could not be returned. The returned //! value is -errno describing the reason. //------------------------------------------------------------------------------ virtual int Get(const char *Aname, void *Aval, int Avsz, const char *Path, int fd=-1); //------------------------------------------------------------------------------ //! Get all of the attributes associated with a file. //! //! @param aPL -> the pointer to hold the first element of AList. The //! storage occupied by the returned AList must be released //! by calling Free(). //! @param Path -> Path of the file whose attributes are t be returned. //! @param fd -> If >=0 is the file descriptor of the opened subject file. //! @param getSz When != 0 then the size of the maximum attribute value //! should be returned. Otherwise, upon success 0 is returned. //! //! @return >0 Attributes were returned and aPL points to the first //! attribute value. The returned value is the largest size //! of an attribute value encountered (getSz != 0). //! @return =0 Attributes were returned and aPL points to the first //! attribute value (getSz == 0). //! @return <0 The attribute values could not be returned. The returned //! value is -errno describing the reason. //------------------------------------------------------------------------------ virtual int List(AList **aPL, const char *Path, int fd=-1, int getSz=0); //------------------------------------------------------------------------------ //! Set an attribute. //! //! @param Aname -> The attribute name. //! @param Aval -> Buffer holding the attribute value. //! @param Avsz Length of the buffer in bytes. This is the length of the //! attribute value which may contain binary data. //! @param Path -> Path of the file whose attribute is to be set. //! @param fd -> If >=0 is the file descriptor of the opened subject file. //! @param isNew When !0 then the attribute must not exist (i.e. new). //! Otherwise, if it does exist, the value is replaced. In //! either case, if it does not exist it should be created. //! //! @return =0 The attribute was successfully set. //! @return <0 The attribute values could not be set. The returned //! value is -errno describing the reason. //------------------------------------------------------------------------------ virtual int Set(const char *Aname, const void *Aval, int Avsz, const char *Path, int fd=-1, int isNew=0); }; #endif /* __XRD_CEPH_XATTR_HH__ */ xrootd-5.6.9/src/XrdCeph/src/XrdVersion.hh.in000066400000000000000000000120661457266313600210400ustar00rootroot00000000000000/******************************************************************************/ /* */ /* X r d V e r s i o n . h h . i n */ /* */ /* (c) 2012 by the Board of Trustees of the Leland Stanford, Jr., University */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ // this file is automatically updated by the genversion.sh script // if you touch anything make sure that it still works #ifndef __XRD_VERSION_H__ #define __XRD_VERSION_H__ #define XrdVERSION "unknown" // Numeric representation of the version tag // The format for the released code is: xyyzz, where: x is the major version, // y is the minor version and zz is the bugfix revision number // For the non-released code the value is 1000000 #define XrdVNUMUNK 1000000 #define XrdVNUMBER 1000000 #if XrdDEBUG #define XrdVSTRING XrdVERSION "_dbg" #else #define XrdVSTRING XrdVERSION #endif // The following defines the shared library version number of any plug-in. // Generally, all plug-ins have a uniform version number releative to a // specific compilation. This version is appended to the so-name and for // dylibs becomes part of he actual filename (MacOS format). // #ifndef XRDPLUGIN_SOVERSION #define XRDPLUGIN_SOVERSION "4" #endif #define XrdDEFAULTPORT 1094; // The following macros extract version digits from a numeric version number #define XrdMajorVNUM(x) x/10000 #define XrdMinorVNUM(x) x/100%100 #define XrdPatchVNUM(x) x%100 // The following structure defines the standard way to record a version. You can // determine component version numbers within an object file by simply executing // "strings | grep '@V:'". // struct XrdVersionInfo {int vNum; const char vOpt; const char vPfx[3];\ const char vStr[40];}; // Macro to define the suffix to use when generating the extern version symbol. // This is used by SysPlugin. We cannot use it here as cpp does not expand the // macro when catenating tokens togther and we want to avoid yet another macro. // #define XrdVERSIONINFOSFX "_" // The following macro defines a local copy of version information. Parameters: // x -> The variable name of the version information structure // y -> An unquoted 1- to 15-character component name (e.g. cmsd, seckrb5) // vn -> The integer version number to be used // vs -> The string version number to be used // #define XrdVERSIONINFODEF(x,y,vn,vs) \ XrdVersionInfo x = \ {vn, (sizeof(#y)-1) & 0x0f,{'@','V',':'}, #y " " vs} // The following macro defines an externally referencable structure that records // the version used to compile code. It is used by the plugin loader. Parms: // x -> The variable name of the version information structure // y -> An unquoted 1- to 15-character component name (e.g. cmsd, seckrb5, etc). // #define XrdVERSIONINFO(x,y) \ extern "C" {XrdVERSIONINFODEF(x##_,y,XrdVNUMBER,XrdVERSION);} // The following macro is an easy way to declare externally defined version // information. This macro must be used at file level. // #define XrdVERSIONINFOREF(x) extern "C" XrdVersionInfo x##_ // The following macro can be used to reference externally defined version // information. As the composition of the symbolic name may change you should // use this macro to refer to the version information declaration. // #define XrdVERSIONINFOVAR(x) x##_ #endif xrootd-5.6.9/src/XrdCeph/tests/000077500000000000000000000000001457266313600163555ustar00rootroot00000000000000xrootd-5.6.9/src/XrdCeph/tests/CMakeLists.txt000066400000000000000000000000411457266313600211100ustar00rootroot00000000000000add_subdirectory( XrdCephTests ) xrootd-5.6.9/src/XrdCeph/tests/XrdCephTests/000077500000000000000000000000001457266313600207355ustar00rootroot00000000000000xrootd-5.6.9/src/XrdCeph/tests/XrdCephTests/CMakeLists.txt000066400000000000000000000012401457266313600234720ustar00rootroot00000000000000message( "XROOTD_INCLUDE_DIR : ${XROOTD_INCLUDE_DIR}" ) add_library( XrdCephTests MODULE CephParsingTest.cc ) target_link_libraries( XrdCephTests pthread ${CPPUNIT_LIBRARIES} ${ZLIB_LIBRARY} XrdCephPosix ) target_include_directories(XrdCephTests PRIVATE ${CPPUNIT_INCLUDE_DIRS} ${RADOS_INCLUDE_DIR} ${XROOTD_INCLUDE_DIR} ${PROJECT_SOURCE_DIR}/src) #------------------------------------------------------------------------------- # Install #------------------------------------------------------------------------------- install( TARGETS XrdCephTests RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} ) xrootd-5.6.9/src/XrdCeph/tests/XrdCephTests/CephParsingTest.cc000066400000000000000000000150051457266313600243100ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2012 by European Organization for Nuclear Research (CERN) // Author: Sebastien Ponce //------------------------------------------------------------------------------ // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . //------------------------------------------------------------------------------ #include #include #include #define MB 1024*1024 struct CephFile { std::string name; std::string pool; std::string userId; unsigned int nbStripes; unsigned long long stripeUnit; unsigned long long objectSize; }; void fillCephFileParams(const std::string ¶ms, XrdOucEnv *env, CephFile &file); void fillCephFile(const char *path, XrdOucEnv *env, CephFile &file); //------------------------------------------------------------------------------ // Declaration //------------------------------------------------------------------------------ class CephParsingTest: public CppUnit::TestCase { public: CPPUNIT_TEST_SUITE( CephParsingTest ); CPPUNIT_TEST( ParamTest ); CPPUNIT_TEST( FileTest ); CPPUNIT_TEST_SUITE_END(); void ParamTest(); void FileTest(); }; CPPUNIT_TEST_SUITE_REGISTRATION( CephParsingTest ); //------------------------------------------------------------------------------ // Helper functions //------------------------------------------------------------------------------ static CephFile parseParam(std::string param, XrdOucEnv *env= NULL) { CephFile cf; fillCephFileParams(param, env, cf); return cf; } static CephFile parseFile(std::string param, XrdOucEnv *env= NULL) { CephFile cf; fillCephFile(param.c_str(), env, cf); return cf; } void checkResult(CephFile a, CephFile b) { std::cout << a.name << " " << a.pool << " " << a.userId << " " << a.nbStripes << " " << a.stripeUnit << " " << a.objectSize << " / " << b.name << " " << b.pool << " " << b.userId << " " << b.nbStripes << " " << b.stripeUnit << " " << b.objectSize << std::endl; CPPUNIT_ASSERT(a.name == b.name); CPPUNIT_ASSERT(a.pool == b.pool); CPPUNIT_ASSERT(a.userId == b.userId); CPPUNIT_ASSERT(a.nbStripes == b.nbStripes); CPPUNIT_ASSERT(a.stripeUnit == b.stripeUnit); CPPUNIT_ASSERT(a.objectSize == b.objectSize); } //------------------------------------------------------------------------------ // Param test //------------------------------------------------------------------------------ void CephParsingTest::ParamTest() { std::map inputs; inputs[""] = (CephFile){"", "default", "admin", 1, 4*MB, 4*MB}; inputs["pool"] = (CephFile){"", "pool", "admin", 1, 4*MB, 4*MB}; inputs["@"] = (CephFile){"", "default", "", 1, 4*MB, 4*MB}; inputs["@pool"] = (CephFile){"", "pool", "", 1, 4*MB, 4*MB}; inputs["user@"] = (CephFile){"", "default", "user", 1, 4*MB, 4*MB}; inputs["user@pool"] = (CephFile){"", "pool", "user", 1, 4*MB, 4*MB}; inputs["pool,1"] = (CephFile){"", "pool", "admin", 1, 4*MB, 4*MB}; inputs["user@pool,1"] = (CephFile){"", "pool", "user", 1, 4*MB, 4*MB}; inputs["pool,5"] = (CephFile){"", "pool", "admin", 5, 4*MB, 4*MB}; inputs["user@pool,5"] = (CephFile){"", "pool", "user", 5, 4*MB, 4*MB}; inputs["pool,5,200"] = (CephFile){"", "pool", "admin", 5, 200, 4*MB}; inputs["user@pool,5,200"] = (CephFile){"", "pool", "user", 5, 200, 4*MB}; inputs["pool,5,200,800"] = (CephFile){"", "pool", "admin", 5, 200, 800}; inputs["user@pool,5,200,800"] = (CephFile){"", "pool", "user", 5, 200, 800}; for (std::map::const_iterator it = inputs.begin(); it != inputs.end(); it++) { std::cout << it->first << std::endl; checkResult(parseParam(it->first), it->second); } } //------------------------------------------------------------------------------ // File test //------------------------------------------------------------------------------ void CephParsingTest::FileTest() { std::map inputs; std::vector filenames; filenames.push_back(""); filenames.push_back("foo"); filenames.push_back("/foo/bar"); filenames.push_back("foo@bar"); filenames.push_back("foo@bar,1"); filenames.push_back("foo@bar,1,2"); filenames.push_back("foo@bar,1,2,3"); filenames.push_back("foo:bar"); filenames.push_back(":foo"); for (std::vector::const_iterator it = filenames.begin(); it != filenames.end(); it++) { if (std::string::npos == it->find(':')) { inputs[*it] = (CephFile){*it, "default", "admin", 1, 4*MB, 4*MB}; } inputs[":" + *it] = (CephFile){*it, "default", "admin", 1, 4*MB, 4*MB}; inputs["pool:" + *it] = (CephFile){*it, "pool", "admin", 1, 4*MB, 4*MB}; inputs["@:" + *it] = (CephFile){*it, "default", "", 1, 4*MB, 4*MB}; inputs["@pool:" + *it] = (CephFile){*it, "pool", "", 1, 4*MB, 4*MB}; inputs["user@:" + *it] = (CephFile){*it, "default", "user", 1, 4*MB, 4*MB}; inputs["user@pool:" + *it] = (CephFile){*it, "pool", "user", 1, 4*MB, 4*MB}; inputs["pool,1:" + *it] = (CephFile){*it, "pool", "admin", 1, 4*MB, 4*MB}; inputs["user@pool,1:" + *it] = (CephFile){*it, "pool", "user", 1, 4*MB, 4*MB}; inputs["pool,5:" + *it] = (CephFile){*it, "pool", "admin", 5, 4*MB, 4*MB}; inputs["user@pool,5:" + *it] = (CephFile){*it, "pool", "user", 5, 4*MB, 4*MB}; inputs["pool,5,200:" + *it] = (CephFile){*it, "pool", "admin", 5, 200, 4*MB}; inputs["user@pool,5,200:" + *it] = (CephFile){*it, "pool", "user", 5, 200, 4*MB}; inputs["pool,5,200,800:" + *it] = (CephFile){*it, "pool", "admin", 5, 200, 800}; inputs["user@pool,5,200,800:" + *it] = (CephFile){*it, "pool", "user", 5, 200, 800}; } for (std::map::const_iterator it = inputs.begin(); it != inputs.end(); it++) { std::cout << it->first << std::endl; checkResult(parseFile(it->first), it->second); } } xrootd-5.6.9/src/XrdCks/000077500000000000000000000000001457266313600150545ustar00rootroot00000000000000xrootd-5.6.9/src/XrdCks/XrdCks.hh000066400000000000000000000400741457266313600166000ustar00rootroot00000000000000#ifndef __XRDCKS_HH__ #define __XRDCKS_HH__ /******************************************************************************/ /* */ /* X r d C k s . h h */ /* */ /* (c) 2011 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include "XrdCks/XrdCksData.hh" class XrdCks; class XrdCksCalc; class XrdOucStream; class XrdSysError; class XrdSysPlugin; /*! This class defines the checksum management interface. It should be used as the base class for a plugin. When used that way, the shared library holding the plugin must define a "C" entry point named XrdCksInit() as described at the end of this include file. Note that you can also base you plugin on the native implementation, XrdCks, and replace only selected methods. */ /******************************************************************************/ /* X r d C k s P C B */ /******************************************************************************/ /*! The XrdCksPCB object defines a callback hat allows he caller to monitor the progress of a checksum calculation (calc or verify). */ class XrdCksPCB { public: //------------------------------------------------------------------------------ //! Report checksum progress. //! @param fsize The file size in bytes. //! @param csbytes Bytes checksummed so far. //------------------------------------------------------------------------------ virtual void Info(long long fsize, long long csbytes); long long rsvd; XrdCksPCB() : rsvd(0) {} virtual ~XrdCksPCB() {} }; /******************************************************************************/ /* X r d C k s */ /******************************************************************************/ //------------------------------------------------------------------------------ //! @note Filenames passed to any of the manager's methods may be either logical //! (lfn) or physical (pfn). By default, these are always physical file names. //! However, it is possible to configure the default manager to use the Oss //! plugin for all I/O related functions. In this case, a logical filename may //! be passed if this is required by the Oss plugin as it will translate the //! logical name to the physical one. See the "ofs.osslib" directive for //! details. Additionally, when using the default Oss or Pss plugins as the I/O //! provider, logical file names are always provided to the manager. Note that //! default manager is automatically configured to accept the correct type of //! file name. There is no mechanism to do this for a specialized manager. So, //! it's possible to create a configuration file that requires logical name to //! be supplied when the manager override only accepts physical ones. //------------------------------------------------------------------------------ class XrdCks { public: //------------------------------------------------------------------------------ //! Calculate a new checksum for a physical file using the checksum algorithm //! named in the Cks parameter. //! //! @param Xfn The logical or physical name of the file to be checksumed. //! @param Cks For input, it specifies the checksum algorithm to be used. //! For output, the checksum value is returned upon success. //! @param doSet When true, the new value must replace any existing value //! in the Xfn's extended file attributes. //! @param pcbP In the second form, the pointer to the callback object. //! A nil pointer does not invoke any callback. //! //! @return Success: zero with Cks structure holding the checksum value. //! Failure: -errno (see significant error numbers below). //------------------------------------------------------------------------------ virtual int Calc( const char *Xfn, XrdCksData &Cks, int doSet=1) = 0; virtual int Calc( const char *Xfn, XrdCksData &Cks, XrdCksPCB *pcbP, int doSet=1) {(void)pcbP; return Calc(Xfn, Cks, doSet);} //------------------------------------------------------------------------------ //! Delete the checksum from the Xfn's xattrs. //! //! @param Xfn The logical or physical name of the file to be checksumed. //! @param Cks Specifies the checksum type to delete. //! //! @return Success: 0 //! Failure: -errno (see significant error numbers below). //------------------------------------------------------------------------------ virtual int Del( const char *Xfn, XrdCksData &Cks) = 0; //------------------------------------------------------------------------------ //! Retreive the checksum from the Xfn's xattrs and return it and indicate //! whether or not it is stale (i.e. the file modification has changed or the //! name and length are not the expected values). //! //! @param Xfn The logical or physical name of the file to be checksumed. //! @param Cks For input, it specifies the checksum type to return. //! For output, the checksum value is returned upon success. //! //! @return Success: The length of the binary checksum in the Cks structure. //! Failure: -errno (see significant error numbers below). //------------------------------------------------------------------------------ virtual int Get( const char *Xfn, XrdCksData &Cks) = 0; //------------------------------------------------------------------------------ //! Parse a configuration directives specific to the checksum manager. //! //! @param Token Points to the directive that triggered the call. //! @param Line All the characters after the directive. //! //! @return Success: 1 //! Failure: 0 //------------------------------------------------------------------------------ virtual int Config(const char *Token, char *Line) = 0; //------------------------------------------------------------------------------ //! Fully initialize the manager which includes loading any plugins. //! //! @param ConfigFN Points to the configuration file path. //! @param DfltCalc Is the default checksum and should be defaulted if NULL. //! The default implementation defaults this to adler32. A //! default is only needed should the checksum name in the //! XrdCksData object be omitted. //! //!@return Success: 1 //! Failure: 0 //------------------------------------------------------------------------------ virtual int Init(const char *ConfigFN, const char *DfltCalc=0) = 0; //------------------------------------------------------------------------------ //! List names of the checksums associated with a Xfn or all supported ones. //! //! @param Xfn The logical or physical file name whose checksum names are //! to be returned. When Xfn is null, return all supported //! checksum algorithm names. //! @param Buff Points to a buffer, at least 64 bytes in length, to hold //! a "Sep" separated list of checksum names. //! @param Blen The length of the buffer. //! @param Sep The separation character to be used between adjacent names. //! //! @return Success: Pointer to Buff holding at least one checksum name. //! Failure: A nil pointer is returned. //------------------------------------------------------------------------------ virtual char *List(const char *Xfn, char *Buff, int Blen, char Sep=' ') = 0; //------------------------------------------------------------------------------ //! Get the name of the checksums associated with a sequence number. Note that //! Name() may be called prior to final config to see if there are any chksums //! to configure and avoid unintended errors. //! //! @param seqNum The sequence number. Zero signifies the default name. //! Higher numbers are alternates. //! @return Success: Pointer to the name. //! Failure: A nil pointer is returned (no more alternates exist). //------------------------------------------------------------------------------ virtual const char *Name(int seqNum=0) = 0; //------------------------------------------------------------------------------ //! Get a new XrdCksCalc object that can calculate the checksum corresponding to //! the specified name or the default object if name is a null pointer. The //! object can be used to compute checksums on the fly. The object's Recycle() //! method must be used to delete it. //! //! @param name The name of the checksum algorithm. If null, use the //! default one. //! //! @return Success: A pointer to the object is returned. //! Failure: Zero if no corresponding object exists. //------------------------------------------------------------------------------ virtual XrdCksCalc *Object(const char *name) { (void)name; return 0; } //------------------------------------------------------------------------------ //! Get the binary length of the checksum with the corresponding name. //! //! @param Name The checksum algorithm name. If null, use the default name. //! //! @return Success: checksum length. //! Failure: Zero if the checksum name does not exist. //------------------------------------------------------------------------------ virtual int Size( const char *Name=0) = 0; //------------------------------------------------------------------------------ //! Set a file's checksum in the extended attributes along with the file's mtime //! and the time of setting. //! //! @param Xfn The logical or physical name of the file to be set. //! @param Cks Specifies the checksum name and value. //! @param myTime When true then the fmTime and gmTime in the Cks structure //! are to be used; as opposed to the current time. //! //! @return Success: zero is returned. //! Failure: -errno (see significant error numbers below). //------------------------------------------------------------------------------ virtual int Set( const char *Xfn, XrdCksData &Cks, int myTime=0) = 0; //------------------------------------------------------------------------------ //! Retreive the checksum from the Xfn's xattrs and compare it to the supplied //! checksum. If the checksum is not available or is stale, a new checksum is //! calculated and written to the extended attributes. //! //! @param Xfn The logical or physical name of the file to be verified. //! @param Cks Specifies the checksum name and value. //! @param pcbP In the second form, the pointer to the callback object. //! A nil pointer does not invoke any callback. //! //! @return Success: True //! Failure: False (the checksums do not match) or -errno indicating //! that verification could not be performed (see significant //! error numbers below). //------------------------------------------------------------------------------ virtual int Ver( const char *Xfn, XrdCksData &Cks) = 0; virtual int Ver( const char *Xfn, XrdCksData &Cks, XrdCksPCB *pcbP) {(void)pcbP; return Ver(Xfn, Cks);} //------------------------------------------------------------------------------ //! Constructor //------------------------------------------------------------------------------ XrdCks(XrdSysError *erP) : eDest(erP) {} //------------------------------------------------------------------------------ //! Destructor //------------------------------------------------------------------------------ virtual ~XrdCks() {} /*! Significant errno values: -EDOM The supplied checksum length is invalid for the checksum name. -ENOTSUP The supplied or default checksum name is not supported. -ESRCH Checksum does not exist for file. -ESTALE The file's checksum is no longer valid. */ protected: XrdSysError *eDest; }; /******************************************************************************/ /* X r d C k s I n i t */ /******************************************************************************/ #define XRDCKSINITPARMS XrdSysError *, const char *, const char * //------------------------------------------------------------------------------ //! Obtain an instance of the checksum manager. //! //! XrdCksInit() is an extern "C" function that is called to obtain an instance //! of a checksum manager object that will be used for all subsequent checksums. //! This is useful when checksums come from an alternate source (e.g. database). //! The proxy server uses this mechanism to obtain checksums from the underlying //! data server. You can also defined plugins for specific checksum calculations //! (see XrdCksCalc.hh). The function must be defined in the plug-in shared //! library. All the following extern symbols must be defined at file level! //! //! @param eDest-> The XrdSysError object for messages. //! @param cfn -> The name of the configuration file //! @param parm -> Parameters specified on the ckslib directive. If none it is //! zero. //! //! @return Success: A pointer to the checksum manager object. //! Failure: Null pointer which causes initialization to fail. //------------------------------------------------------------------------------ /*! extern "C" XrdCks *XrdCksInit(XrdSysError *eDest, const char *cFN, const char *Parms ); */ //------------------------------------------------------------------------------ //! Declare the compilation version number. //! //! Additionally, you *should* declare the xrootd version you used to compile //! your plug-in. While not currently required, it is highly recommended to //! avoid execution issues should the class definition change. Declare it as: //------------------------------------------------------------------------------ /*! #include "XrdVersion.hh" XrdVERSIONINFO(XrdCksInit,); where is a 1- to 15-character unquoted name identifying your plugin. */ #endif xrootd-5.6.9/src/XrdCks/XrdCksAssist.cc000066400000000000000000000171251457266313600177560ustar00rootroot00000000000000/******************************************************************************/ /* */ /* X r d C k s A s s i s t . c c */ /* */ /* (c) 2017 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include #include #include #include #include #include #include #include "XrdCks/XrdCksData.hh" /******************************************************************************/ /* S t a t i c s */ /******************************************************************************/ namespace { struct csTable {const char *csName; int csLenC; int csLenB;} csTab[] = {{"adler32", 8, 4}, {"crc32", 8, 4}, {"crc64", 16, 8}, {"md5", 32, 16}, {"sha1", 40, 20}, {"sha2", 64, 32}, {"sha256", 64, 32}, {"sha512", 128, 64} }; static const int csNum = sizeof(csTab)/sizeof(csTable); bool LowerCase(const char *src, char *dst, int dstlen) { int n = strlen(src); // Verify that the result will fit in the supplied buffer with a null // if (n >= dstlen) return false; // Convert to ower case with trailing nulls // memset(dst+n, 0, dstlen-n); for (int i = 0; i < n; i++) dst[i] = tolower(src[i]); return true; } } /******************************************************************************/ /* X r d C k s A t t r D a t a */ /******************************************************************************/ // Return the data portion of a checksum attribute so that it can be use // to set an attribute value. std::vector XrdCksAttrData(const char *cstype, const char *csval, time_t mtime) { std::vector cksError; // Null vector XrdCksData cksData; char csName[XrdCksData::NameSize]; int n = strlen(cstype); // Convert the checksum type to lower case // if (!LowerCase(cstype, csName, sizeof(csName))) {errno = ENAMETOOLONG; return cksError;} // For cheksums we know, verify that the legnth of the input string // corresponds to the checksum type. // n = strlen(csval); for (int i = 0; i < csNum; i++) {if (!strcmp(csTab[i].csName, csName) && csTab[i].csLenC != n) {errno = EINVAL; return cksError;} } // we simply fill out the cksdata structure with the provided information // if (!cksData.Set(csName)) {errno = ENAMETOOLONG; return cksError;} if (!cksData.Set(csval, n)) {errno = EOVERFLOW; return cksError;} cksData.fmTime = mtime; cksData.csTime = time(0) - mtime; // Convert the checksum data to a string of bytes and return the vector // return std::vector( (char *)&cksData, ((char *)&cksData) + sizeof(cksData)); } /******************************************************************************/ /* X r d C k s A t t r N a m e */ /******************************************************************************/ // Return the extended attribute variable name for a particular checksum type. std::string XrdCksAttrName(const char *cstype, const char *nspfx) { std::string xaName; char csName[XrdCksData::NameSize]; int pfxlen = strlen(nspfx); // Do some easy checks for this we know are common // if (!pfxlen) {if (!strcmp(cstype, "adler32")) return std::string("XrdCks.adler32"); if (!strcmp(cstype, "md5" )) return std::string("XrdCks.md5"); if (!strcmp(cstype, "crc32" )) return std::string("XrdCks.crc32"); } // Convert the checksum type to lower case // if (!LowerCase(cstype, csName, sizeof(csName))) {errno = ENAMETOOLONG; return xaName;} // Reserve storage for the string and construct the variable name // xaName.reserve(strlen(nspfx) + strlen(cstype) + 8); if (pfxlen) {xaName = nspfx; if (nspfx[pfxlen-1] != '.') xaName += '.'; } xaName += "XrdCks."; xaName += csName; // Return the variable name // return xaName; } /******************************************************************************/ /* X r d C k s A t t r V a l u e */ /******************************************************************************/ std::string XrdCksAttrValue(const char *cstype, const char *csbuff, int csblen) { XrdCksData cksData; std::string csError; char csBuff[XrdCksData::ValuSize*2+1]; // Verify that the length matches our object length // if (csblen != (int)sizeof(cksData)) {errno = EMSGSIZE; return csError;} // Copy the buffer into the object // memcpy(&cksData, csbuff, sizeof(cksData)); // Now verify that all the fields are consistent // if (strncasecmp(cksData.Name, cstype, XrdCksData::NameSize)) {errno = ENOENT; return csError;} if (cksData.Length <= 0 || cksData.Length > XrdCksData::ValuSize) {errno = EINVAL; return csError;} // For known checksum values make sure the length matches // for (int i = 0; i < csNum; i++) {if (!strcmp(csTab[i].csName, cstype) && csTab[i].csLenB != int(cksData.Length)) {errno = EINVAL; return csError;} } // Convert value to a hex string // if (!cksData.Get(csBuff, sizeof(csBuff))) {errno = EOVERFLOW; return csError;} // Return string version of the hex string // return std::string(csBuff); } xrootd-5.6.9/src/XrdCks/XrdCksAssist.hh000066400000000000000000000151211457266313600177620ustar00rootroot00000000000000#ifndef __XRDCKSASSIST_HH__ #define __XRDCKSASSIST_HH__ /******************************************************************************/ /* */ /* X r d C k s A s s i s t . h h */ /* */ /* (c) 2017 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include #include //------------------------------------------------------------------------------ //! This header file defines linkages to various XRootD checksum assistants. //! The functions described here are located in libXrdUtils.so. //------------------------------------------------------------------------------ //------------------------------------------------------------------------------ //! Generate the extended attribute data for a particular checksum that can //! be used to set the corresponding checksum attribute variable. //! //! @param cstype A null terminated string holding the checksum type //! (e.g. "adler32", "md5", "sha2", etc). //! @param csval A null terminated string holding the corresonding //! checksum value. This must be an ASCII hex representation //! of the value and must be of appropriate length. //! @param mtime The subject file's modification time. //! //! @return A vector of bytes that should be usedto set the attribute variable. //! If the size of the vector is zero, then the supplied parameters //! were incorrect and valid data cannot be generated; errno is: //! EINVAL - csval length is incorrect for checksum type or //! contains a non-hex digit. //! ENAMETOOLONG - checksum type is too long. //! EOVERFLOW - csval could not be represented in the data. //------------------------------------------------------------------------------ extern std::vector XrdCksAttrData(const char *cstype, const char *csval, time_t mtime); //------------------------------------------------------------------------------ //! Generate the extended attribute variable name for a particular checksum. //! //! @param cstype A null terminated string holding the checksum type //! (e.g. "adler32", "md5", "sha2", etc). //! @param nspfx Is the namespace prefix to add to the variable name. //! By default no prefix os used. Certain platforms and/or //! filesystems require that user attributes start with a //! particular prefix (e.g. Linux requires 'user.') others //! do not. If your are going to use the variable name to get //! or set an attribute you should specify any required //! prefix. If specified and it does not end with a dot, a //! dot is automatically added to the nspfx. //! //! @return A string holding the variable name that should be used to get or //! set the extended attribute holding the correspnding checksum. If //! a null string is returned, the variable could not be generated; //! errno is set to: //! ENAMETOOLONG - checksum type is too long. //------------------------------------------------------------------------------ extern std::string XrdCksAttrName(const char *cstype, const char *nspfx=""); //------------------------------------------------------------------------------ //! Extract th checksum value from checksum extended attribute data. //! //! @param cstype A null terminated string holding the checksum type //! (e.g. "adler32", "md5", "sha2", etc). //! @param csbuff A pointer to a buffer hlding the checksum data. //! @param csblen The length of the checksum data (i.e. the length of the //! retrieved extended attribute). //! //! @return A string holding the ASCII hexstring correspoding to the checksum //! value. If a null string is returned then the checksum data was //! invalid or did not correspond to the specified checksum type, the //! errno is set to: //! EINVAL - the checksum length in csbuff is incorrect. //! EMSGSIZE - csblen was not the expected value. //! ENOENT - the specified cstype did not match the one in csbuff. //! EOVERFLOW - checksum value could not be generated from csbuff. //------------------------------------------------------------------------------ extern std::string XrdCksAttrValue(const char *cstype, const char *csbuff, int csblen); #endif xrootd-5.6.9/src/XrdCks/XrdCksCalc.hh000066400000000000000000000205151457266313600173610ustar00rootroot00000000000000#ifndef __XRDCKSCALC_HH__ #define __XRDCKSCALC_HH__ /******************************************************************************/ /* */ /* X r d C k s C a l c . h h */ /* */ /* (c) 2011 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ /*! This class defines the interface to a checksum computation. When this class is used to define a plugin computation, the initial XrdCksCalc computation object is created by the XrdCksCalcInit() function defined at the end of this file. */ class XrdCksCalc { public: //------------------------------------------------------------------------------ //! Calculate a one-time checksum. The obvious default implementation is //! provided and assumes that Init() may be called more than once. //! //! @param Buff -> Data to be checksummed. //! @param BLen -> Length of the data in Buff. //! //! @return the checksum value in binary format. The pointer to the value //! becomes invalid once the associated object is deleted. //------------------------------------------------------------------------------ virtual char *Calc(const char *Buff, int BLen) {Init(); Update(Buff, BLen); return Final();} //------------------------------------------------------------------------------ //! Get the current binary checksum value (defaults to final). However, the //! final checksum result is not affected. //! //! @return the checksum value in binary format. The pointer to the value //! becomes invalid once the associated object is deleted. //------------------------------------------------------------------------------ virtual char *Current() {return Final();} //------------------------------------------------------------------------------ //! Get the actual checksum in binary format. //! //! @return the checksum value in binary format. The pointer to the value //! becomes invalid once the associated object is deleted. //------------------------------------------------------------------------------ virtual char *Final() = 0; //------------------------------------------------------------------------------ //! Initializes data structures (must be called by constructor). This is always //! called to reuse the object for a new checksum. //------------------------------------------------------------------------------ virtual void Init() = 0; //------------------------------------------------------------------------------ //! Get a new instance of the underlying checksum calculation object. //! //! @return the checksum calculation object. //------------------------------------------------------------------------------ virtual XrdCksCalc *New() = 0; //------------------------------------------------------------------------------ //! Recycle the checksum object as it is no longer needed. A default is given. //------------------------------------------------------------------------------ virtual void Recycle() {delete this;} //------------------------------------------------------------------------------ //! Get the checksum object algorithm name and the number bytes (i.e. size) //! required for the checksum value. //! //! @param csSize -> Parameter to hold the size of the checksum value. //! //! @return the checksum algorithm's name. The name persists event after the //! checksum object is deleted. //------------------------------------------------------------------------------ virtual const char *Type(int &csSize) = 0; //------------------------------------------------------------------------------ //! Compute a running checksum. This method may be called repeatedly for data //! segments; with Final() returning the full checksum. //! //! @param Buff -> Data to be checksummed. //! @param BLen -> Length of the data in Buff. //------------------------------------------------------------------------------ virtual void Update(const char *Buff, int BLen) = 0; //------------------------------------------------------------------------------ //! Constructor //------------------------------------------------------------------------------ XrdCksCalc() {} //------------------------------------------------------------------------------ //! Destructor //------------------------------------------------------------------------------ virtual ~XrdCksCalc() {} }; /******************************************************************************/ /* C h e c k s u m O b j e c t C r e a t o r */ /******************************************************************************/ //------------------------------------------------------------------------------ //! Obtain an instance of the checksum calculation object. //! //! XrdCksCalcInit() is an extern "C" function that is called to obtain an //! initial instance of a checksum calculation object. You may create custom //! checksum calculation and use them as plug-ins to the checksum manager //! (see XrdCks.hh). The function must be defined in the plug-in shared library. //! All the following extern symbols must be defined at file level! //! //! @param eDest -> The XrdSysError object for messages. //! @param csName -> The name of the checksum algorithm. //! @param cFN -> The name of the configuration file //! @param Parms -> Parameters specified on the ckslib directive. If none it is //! zero. //------------------------------------------------------------------------------ /*! extern "C" XrdCksCalc *XrdCksCalcInit(XrdSysError *eDest, const char *csName, const char *cFN, const char *Parms); */ //------------------------------------------------------------------------------ //! Declare the compilation version number. //! //! Additionally, you *should* declare the xrootd version you used to compile //! your plug-in. While not currently required, it is highly recommended to //! avoid execution issues should the class definition change. Declare it as: //------------------------------------------------------------------------------ /*! #include "XrdVersion.hh" XrdVERSIONINFO(XrdCksCalcInit,); where is a 1- to 15-character unquoted name identifying your plugin. */ #endif xrootd-5.6.9/src/XrdCks/XrdCksCalcadler32.hh000066400000000000000000000130671457266313600205420ustar00rootroot00000000000000#ifndef __XRDCKSCALCADLER32_HH__ #define __XRDCKSCALCADLER32_HH__ /******************************************************************************/ /* */ /* X r d C k s C a l c a d l e r 3 2 . h h */ /* */ /* (c) 2011 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include #include #include "XrdCks/XrdCksCalc.hh" #include "XrdSys/XrdSysPlatform.hh" /* The following implementation of adler32 was derived from zlib and is * Copyright (C) 1995-1998 Mark Adler Below are the zlib license terms for this implementation. */ /* zlib.h -- interface of the 'zlib' general purpose compression library version 1.1.4, March 11th, 2002 Copyright (C) 1995-2002 Jean-loup Gailly and Mark Adler This software is provided 'as-is', without any express or implied warranty. In no event will the authors be held liable for any damages arising from the use of this software. Permission is granted to anyone to use this software for any purpose, including commercial applications, and to alter it and redistribute it freely, subject to the following restrictions: 1. The origin of this software must not be misrepresented; you must not claim that you wrote the original software. If you use this software in a product, an acknowledgment in the product documentation would be appreciated but is not required. 2. Altered source versions must be plainly marked as such, and must not be misrepresented as being the original software. 3. This notice may not be removed or altered from any source distribution. Jean-loup Gailly Mark Adler jloup@gzip.org madler@alumni.caltech.edu The data format used by the zlib library is described by RFCs (Request for Comments) 1950 to 1952 in the files ftp://ds.internic.net/rfc/rfc1950.txt (zlib format), rfc1951.txt (deflate format) and rfc1952.txt (gzip format). */ #define DO1(buf) {unSum1 += *buf++; unSum2 += unSum1;} #define DO2(buf) DO1(buf); DO1(buf); #define DO4(buf) DO2(buf); DO2(buf); #define DO8(buf) DO4(buf); DO4(buf); #define DO16(buf) DO8(buf); DO8(buf); class XrdCksCalcadler32 : public XrdCksCalc { public: char *Final() {AdlerValue = (unSum2 << 16) | unSum1; #ifndef Xrd_Big_Endian AdlerValue = htonl(AdlerValue); #endif return (char *)&AdlerValue; } void Init() {unSum1 = AdlerStart; unSum2 = 0;} XrdCksCalc *New() {return (XrdCksCalc *)new XrdCksCalcadler32;} void Update(const char *Buff, int BLen) {int k; unsigned char *buff = (unsigned char *)Buff; while(BLen > 0) {k = (BLen < AdlerNMax ? BLen : AdlerNMax); BLen -= k; while(k >= 16) {DO16(buff); k -= 16;} if (k != 0) do {DO1(buff);} while (--k); unSum1 %= AdlerBase; unSum2 %= AdlerBase; } } const char *Type(int &csSize) {csSize = sizeof(AdlerValue); return "adler32";} XrdCksCalcadler32() {Init();} virtual ~XrdCksCalcadler32() {} private: static const unsigned int AdlerBase = 0xFFF1; static const unsigned int AdlerStart = 0x0001; static const int AdlerNMax = 5552; /* NMAX is the largest n such that 255n(n+1)/2 + (n+1)(BASE-1) <= 2^32-1 */ unsigned int AdlerValue; unsigned int unSum1; unsigned int unSum2; }; #endif xrootd-5.6.9/src/XrdCks/XrdCksCalccrc32.cc000066400000000000000000000206771457266313600202150ustar00rootroot00000000000000/******************************************************************************/ /* */ /* X r d C k s C a l c c r c 3 2 . c c */ /* */ /* (c) 2011 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include "XrdCks/XrdCksCalccrc32.hh" /* C++ implementation of CRC-32 checksums. Code is based upon and utilizes algorithm published by Ross Williams as initially implemented by Eric Durbin. This file contains: CRC lookup table function CalcCRC32 for calculating CRC-32 checksum Provided by: Eric Durbin Kentucky Cancer Registry University of Kentucky October 14, 1998 Status: Public Domain */ /*****************************************************************/ /* */ /* CRC LOOKUP TABLE */ /* ================ */ /* The following CRC lookup table was generated automagically */ /* by the Rocksoft^tm Model CRC Algorithm Table Generation */ /* Program V1.0 using the following model parameters: */ /* */ /* Width : 4 bytes. */ /* Poly : 0x04C11DB7L */ /* Reverse : FALSE */ /* */ /* For more information on the Rocksoft^tm Model CRC Algorithm, */ /* see the document titled "A Painless Guide to CRC Error */ /* Detection Algorithms" by Ross Williams */ /* (ross@guest.adelaide.edu.au.). This document is likely to be */ /* in the FTP archive "ftp.adelaide.edu.au/pub/rocksoft". */ /* */ /*****************************************************************/ unsigned int XrdCksCalccrc32::crctable[256] = { 0x00000000L, 0x04C11DB7L, 0x09823B6EL, 0x0D4326D9L, 0x130476DCL, 0x17C56B6BL, 0x1A864DB2L, 0x1E475005L, 0x2608EDB8L, 0x22C9F00FL, 0x2F8AD6D6L, 0x2B4BCB61L, 0x350C9B64L, 0x31CD86D3L, 0x3C8EA00AL, 0x384FBDBDL, 0x4C11DB70L, 0x48D0C6C7L, 0x4593E01EL, 0x4152FDA9L, 0x5F15ADACL, 0x5BD4B01BL, 0x569796C2L, 0x52568B75L, 0x6A1936C8L, 0x6ED82B7FL, 0x639B0DA6L, 0x675A1011L, 0x791D4014L, 0x7DDC5DA3L, 0x709F7B7AL, 0x745E66CDL, 0x9823B6E0L, 0x9CE2AB57L, 0x91A18D8EL, 0x95609039L, 0x8B27C03CL, 0x8FE6DD8BL, 0x82A5FB52L, 0x8664E6E5L, 0xBE2B5B58L, 0xBAEA46EFL, 0xB7A96036L, 0xB3687D81L, 0xAD2F2D84L, 0xA9EE3033L, 0xA4AD16EAL, 0xA06C0B5DL, 0xD4326D90L, 0xD0F37027L, 0xDDB056FEL, 0xD9714B49L, 0xC7361B4CL, 0xC3F706FBL, 0xCEB42022L, 0xCA753D95L, 0xF23A8028L, 0xF6FB9D9FL, 0xFBB8BB46L, 0xFF79A6F1L, 0xE13EF6F4L, 0xE5FFEB43L, 0xE8BCCD9AL, 0xEC7DD02DL, 0x34867077L, 0x30476DC0L, 0x3D044B19L, 0x39C556AEL, 0x278206ABL, 0x23431B1CL, 0x2E003DC5L, 0x2AC12072L, 0x128E9DCFL, 0x164F8078L, 0x1B0CA6A1L, 0x1FCDBB16L, 0x018AEB13L, 0x054BF6A4L, 0x0808D07DL, 0x0CC9CDCAL, 0x7897AB07L, 0x7C56B6B0L, 0x71159069L, 0x75D48DDEL, 0x6B93DDDBL, 0x6F52C06CL, 0x6211E6B5L, 0x66D0FB02L, 0x5E9F46BFL, 0x5A5E5B08L, 0x571D7DD1L, 0x53DC6066L, 0x4D9B3063L, 0x495A2DD4L, 0x44190B0DL, 0x40D816BAL, 0xACA5C697L, 0xA864DB20L, 0xA527FDF9L, 0xA1E6E04EL, 0xBFA1B04BL, 0xBB60ADFCL, 0xB6238B25L, 0xB2E29692L, 0x8AAD2B2FL, 0x8E6C3698L, 0x832F1041L, 0x87EE0DF6L, 0x99A95DF3L, 0x9D684044L, 0x902B669DL, 0x94EA7B2AL, 0xE0B41DE7L, 0xE4750050L, 0xE9362689L, 0xEDF73B3EL, 0xF3B06B3BL, 0xF771768CL, 0xFA325055L, 0xFEF34DE2L, 0xC6BCF05FL, 0xC27DEDE8L, 0xCF3ECB31L, 0xCBFFD686L, 0xD5B88683L, 0xD1799B34L, 0xDC3ABDEDL, 0xD8FBA05AL, 0x690CE0EEL, 0x6DCDFD59L, 0x608EDB80L, 0x644FC637L, 0x7A089632L, 0x7EC98B85L, 0x738AAD5CL, 0x774BB0EBL, 0x4F040D56L, 0x4BC510E1L, 0x46863638L, 0x42472B8FL, 0x5C007B8AL, 0x58C1663DL, 0x558240E4L, 0x51435D53L, 0x251D3B9EL, 0x21DC2629L, 0x2C9F00F0L, 0x285E1D47L, 0x36194D42L, 0x32D850F5L, 0x3F9B762CL, 0x3B5A6B9BL, 0x0315D626L, 0x07D4CB91L, 0x0A97ED48L, 0x0E56F0FFL, 0x1011A0FAL, 0x14D0BD4DL, 0x19939B94L, 0x1D528623L, 0xF12F560EL, 0xF5EE4BB9L, 0xF8AD6D60L, 0xFC6C70D7L, 0xE22B20D2L, 0xE6EA3D65L, 0xEBA91BBCL, 0xEF68060BL, 0xD727BBB6L, 0xD3E6A601L, 0xDEA580D8L, 0xDA649D6FL, 0xC423CD6AL, 0xC0E2D0DDL, 0xCDA1F604L, 0xC960EBB3L, 0xBD3E8D7EL, 0xB9FF90C9L, 0xB4BCB610L, 0xB07DABA7L, 0xAE3AFBA2L, 0xAAFBE615L, 0xA7B8C0CCL, 0xA379DD7BL, 0x9B3660C6L, 0x9FF77D71L, 0x92B45BA8L, 0x9675461FL, 0x8832161AL, 0x8CF30BADL, 0x81B02D74L, 0x857130C3L, 0x5D8A9099L, 0x594B8D2EL, 0x5408ABF7L, 0x50C9B640L, 0x4E8EE645L, 0x4A4FFBF2L, 0x470CDD2BL, 0x43CDC09CL, 0x7B827D21L, 0x7F436096L, 0x7200464FL, 0x76C15BF8L, 0x68860BFDL, 0x6C47164AL, 0x61043093L, 0x65C52D24L, 0x119B4BE9L, 0x155A565EL, 0x18197087L, 0x1CD86D30L, 0x029F3D35L, 0x065E2082L, 0x0B1D065BL, 0x0FDC1BECL, 0x3793A651L, 0x3352BBE6L, 0x3E119D3FL, 0x3AD08088L, 0x2497D08DL, 0x2056CD3AL, 0x2D15EBE3L, 0x29D4F654L, 0xC5A92679L, 0xC1683BCEL, 0xCC2B1D17L, 0xC8EA00A0L, 0xD6AD50A5L, 0xD26C4D12L, 0xDF2F6BCBL, 0xDBEE767CL, 0xE3A1CBC1L, 0xE760D676L, 0xEA23F0AFL, 0xEEE2ED18L, 0xF0A5BD1DL, 0xF464A0AAL, 0xF9278673L, 0xFDE69BC4L, 0x89B8FD09L, 0x8D79E0BEL, 0x803AC667L, 0x84FBDBD0L, 0x9ABC8BD5L, 0x9E7D9662L, 0x933EB0BBL, 0x97FFAD0CL, 0xAFB010B1L, 0xAB710D06L, 0xA6322BDFL, 0xA2F33668L, 0xBCB4666DL, 0xB8757BDAL, 0xB5365D03L, 0xB1F740B4L }; /*****************************************************************/ /* End of CRC Lookup Table */ /*****************************************************************/ /* Calculate CRC-32 Checksum for NAACCR Record, skipping area of record containing checksum field. Uses non-reflected table driven method documented by Ross Williams. PARAMETERS: unsigned char *p Record Buffer unsigned long reclen Record Length RETURNS: checksum value Author: Eric Durbin 1998-10-14 Simplified by Andrew Hanushevsky 2007-07-20 Status: Public Domain Changes: Compute CRC for complete buffer Use unsigned int instead of long to insure 32 bit values. Include length bits at the end to correspond to the Posix 1003.2 spec. Make this a C++ class. */ void XrdCksCalccrc32::Update(const char *p, int reclen) { // Process each byte // TotLen += reclen; while(reclen-- > 0) C32Result = (C32Result<<8) ^ crctable[(unsigned char)((C32Result>>24)^*p++)]; } xrootd-5.6.9/src/XrdCks/XrdCksCalccrc32.hh000066400000000000000000000071061457266313600202170ustar00rootroot00000000000000#ifndef __XRDCKSCALCCRC32_HH__ #define __XRDCKSCALCCRC32_HH__ /******************************************************************************/ /* */ /* X r d C k s C a l c c r c 3 2 . h h */ /* */ /* (c) 2011 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include #include #include #include "XrdCks/XrdCksCalc.hh" #include "XrdSys/XrdSysPlatform.hh" class XrdCksCalccrc32 : public XrdCksCalc { public: char *Final() {char buff[sizeof(long long)]; long long tLcs = TotLen; int i = 0; if (tLcs) {while(tLcs) {buff[i++] = tLcs & 0xff ; tLcs >>= 8;} Update(buff, i); } TheResult = C32Result ^ CRC32_XOROT; #ifndef Xrd_Big_Endian TheResult = htonl(TheResult); #endif return (char *)&TheResult; } void Init() {C32Result = CRC32_XINIT; TotLen = 0;} XrdCksCalc *New() {return (XrdCksCalc *)new XrdCksCalccrc32;} void Update(const char *Buff, int BLen); const char *Type(int &csSz) {csSz = sizeof(TheResult); return "crc32";} XrdCksCalccrc32() {Init();} virtual ~XrdCksCalccrc32() {} private: static const unsigned int CRC32_XINIT = 0; static const unsigned int CRC32_XOROT = 0xffffffff; static unsigned int crctable[256]; unsigned int C32Result; unsigned int TheResult; long long TotLen; }; #endif xrootd-5.6.9/src/XrdCks/XrdCksCalccrc32C.cc000066400000000000000000000017531457266313600203120ustar00rootroot00000000000000#include "XrdCks/XrdCksCalccrc32C.hh" /* C++ implementation of CRC-32C checksums based upon unattributed library functions. This file contains: functions implementing the methods of the XrdCksCalc class Provided by: Anton Schwarz University of Heidelberg July 26, 2021 Status: Public Domain */ void XrdCksCalccrc32C::Update(const char *Buff, int BLen) { C32CResult = (unsigned int)XrdOucCRC::Calc32C(Buff, BLen, C32CResult); } const char *XrdCksCalccrc32C::Type(int &csSz) { csSz = sizeof(TheResult); return "crc32c"; } XrdCksCalc *XrdCksCalccrc32C::New() { return (XrdCksCalc *)new XrdCksCalccrc32C; } void XrdCksCalccrc32C::Init() { C32CResult = C32C_XINIT; } char *XrdCksCalccrc32C::Final() { TheResult = C32CResult; #ifndef Xrd_Big_Endian TheResult = htonl(TheResult); #endif return (char *)&TheResult; } XrdCksCalccrc32C::XrdCksCalccrc32C() { Init(); } XrdCksCalccrc32C::~XrdCksCalccrc32C() {};xrootd-5.6.9/src/XrdCks/XrdCksCalccrc32C.hh000066400000000000000000000056521457266313600203260ustar00rootroot00000000000000#ifndef __XRDCKSCALCCRC32C_HH__ #define __XRDCKSCALCCRC32C_HH__ /******************************************************************************/ /* */ /* X r d C k s C a l c c r c 3 2 C . h h */ /* */ /* (c) 2021 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include #include #include #include "XrdCks/XrdCksCalc.hh" #include "XrdSys/XrdSysPlatform.hh" #include "XrdOuc/XrdOucCRC.hh" class XrdCksCalccrc32C : public XrdCksCalc { public: char *Final(); void Init(); XrdCksCalc *New(); void Update(const char *Buff, int BLen); const char *Type(int &csSz); XrdCksCalccrc32C(); virtual ~XrdCksCalccrc32C(); private: static const unsigned int C32C_XINIT = 0; unsigned int C32CResult; unsigned int TheResult; }; #endifxrootd-5.6.9/src/XrdCks/XrdCksCalcmd5.cc000066400000000000000000000277441457266313600177700ustar00rootroot00000000000000/******************************************************************************/ /* */ /* X r d C k s C a l c m d 5 . c c */ /* */ /* (c) 2011 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include "XrdCks/XrdCksCalcmd5.hh" #include "XrdSys/XrdSysPlatform.hh" /* * This code implements the MD5 message-digest algorithm. * The algorithm is due to Ron Rivest. This code was * written by Colin Plumb in 1993, no copyright is claimed. * This code is in the public domain; do with it what you wish. * * Equivalent code is available from RSA Data Security, Inc. * This code has been tested against that, and is equivalent, * except that you don't need to include two pages of legalese * with every copy. * * To compute the message digest of a chunk of bytes, declare an * MD5Context structure, pass it to MD5Init, call MD5Update as * needed on buffers full of bytes, and then call MD5Final, which * will fill a supplied 16-byte array with the digest. */ /******************************************************************************/ /* B y t e R e v e r s e */ /******************************************************************************/ #ifndef Xrd_Big_Endian void XrdCksCalcmd5::byteReverse(unsigned char *buf, unsigned longs) {} /* Nothing */ #else #ifndef ASM_MD5 void XrdCksCalcmd5::byteReverse(unsigned char *buf, unsigned longs) { unsigned int t; do {t = (unsigned int) ((unsigned) buf[3] << 8 | buf[2]) << 16 | ((unsigned) buf[1] << 8 | buf[0]); *(unsigned int *) buf = t; buf += 4; } while (--longs); } #endif #endif /******************************************************************************/ /* I n i t */ /******************************************************************************/ /******************************************************************************/ /* M D 5 I n i t */ /******************************************************************************/ /* Start MD5 accumulation. Set bit count to 0 and buffer to mysterious initialization constants. */ void XrdCksCalcmd5::Init() { myContext.buf[0] = 0x67452301; myContext.buf[1] = 0xefcdab89; myContext.buf[2] = 0x98badcfe; myContext.buf[3] = 0x10325476; myContext.bits[0] = 0; myContext.bits[1] = 0; } /******************************************************************************/ /* M D 5 U p d a t e */ /******************************************************************************/ /* Update context to reflect the concatenation of another buffer full of bytes. */ void XrdCksCalcmd5::MD5Update(unsigned char const *buf, unsigned int len) { unsigned int t; // Update bitcount // t = myContext.bits[0]; if ((myContext.bits[0] = t + ((unsigned int) len << 3)) < t) // Carry from low to high // myContext.bits[1]++; myContext.bits[1] += len >> 29; // Bytes already in shsInfo->data // t = (t >> 3) & 0x3f; // Handle any leading odd-sized chunks // if (t) {unsigned char *p = (unsigned char *) myContext.in + t; t = 64 - t; if (len < t) {memcpy(p, buf, len); return;} memcpy(p, buf, t); byteReverse(myContext.in, 16); MD5Transform(myContext.buf, (unsigned int *) myContext.in); buf += t; len -= t; } // Process data in 64-byte chunks // while(len >= 64) {memcpy(myContext.in, buf, 64); byteReverse(myContext.in, 16); MD5Transform(myContext.buf, (unsigned int *) myContext.in); buf += 64; len -= 64; } // Handle any remaining bytes of data. memcpy(myContext.in, buf, len); } /******************************************************************************/ /* F i n a l */ /******************************************************************************/ /* Final wrapup - pad to 64-byte boundary with the bit pattern 1 0* (64-bit count of bits processed, MSB-first) */ char *XrdCksCalcmd5::Final() { unsigned count; unsigned char *p; // Compute number of bytes mod 64 // count = (myContext.bits[0] >> 3) & 0x3F; // Set the first char of padding to 0x80. This is safe since there is // always at least one byte free. // p = myContext.in + count; *p++ = 0x80; // Bytes of padding needed to make 64 bytes // count = 64 - 1 - count; // Pad out to 56 mod 64 // if (count < 8) // Two lots of padding: Pad the first block to 64 bytes {memset(p, 0, count); byteReverse(myContext.in, 16); MD5Transform(myContext.buf, (unsigned int *) myContext.in); memset(myContext.in, 0, 56); // Now fill the next block with 56 bytes } else memset(p, 0, count - 8); // Else pad block to 56 bytes byteReverse(myContext.in, 14); // Append length in bits and transform (original code in comments) // // ((unsigned int *) myContext.in)[14] = myContext.bits[0]; // ((unsigned int *) myContext.in)[15] = myContext.bits[1]; myContext.i64[7] = myContext.b64; MD5Transform(myContext.buf, (unsigned int *) myContext.in); byteReverse((unsigned char *) myContext.buf, 4); // Copy to a separate buffer and return ASCII value if so wanted // memcpy(myDigest, myContext.buf, 16); return (char *)myDigest; } /******************************************************************************/ /* M D 5 T r a n s f o r m */ /******************************************************************************/ #ifndef ASM_MD5 /* The four core functions - F1 is optimized somewhat */ // #define F1(x, y, z) (x & y | ~x & z) // #define F1(x, y, z) (z ^ (x & (y ^ z))) #define F2(x, y, z) F1(z, x, y) #define F3(x, y, z) (x ^ y ^ z) #define F4(x, y, z) (y ^ (x | ~z)) // This is the central step in the MD5 algorithm. // #define MD5STEP(f, w, x, y, z, data, s) \ ( w += f(x, y, z) + data, w = w<>(32-s), w += x ) /* The core of the MD5 algorithm, this alters an existing MD5 hash to reflect the addition of 16 longwords of new data. MD5Update blocks the data and converts bytes into longwords for this routine. */ void XrdCksCalcmd5::MD5Transform(unsigned int buf[4], unsigned int const in[16]) { unsigned int a, b, c, d; a = buf[0]; b = buf[1]; c = buf[2]; d = buf[3]; MD5STEP(F1, a, b, c, d, in[0] + 0xd76aa478, 7); MD5STEP(F1, d, a, b, c, in[1] + 0xe8c7b756, 12); MD5STEP(F1, c, d, a, b, in[2] + 0x242070db, 17); MD5STEP(F1, b, c, d, a, in[3] + 0xc1bdceee, 22); MD5STEP(F1, a, b, c, d, in[4] + 0xf57c0faf, 7); MD5STEP(F1, d, a, b, c, in[5] + 0x4787c62a, 12); MD5STEP(F1, c, d, a, b, in[6] + 0xa8304613, 17); MD5STEP(F1, b, c, d, a, in[7] + 0xfd469501, 22); MD5STEP(F1, a, b, c, d, in[8] + 0x698098d8, 7); MD5STEP(F1, d, a, b, c, in[9] + 0x8b44f7af, 12); MD5STEP(F1, c, d, a, b, in[10] + 0xffff5bb1, 17); MD5STEP(F1, b, c, d, a, in[11] + 0x895cd7be, 22); MD5STEP(F1, a, b, c, d, in[12] + 0x6b901122, 7); MD5STEP(F1, d, a, b, c, in[13] + 0xfd987193, 12); MD5STEP(F1, c, d, a, b, in[14] + 0xa679438e, 17); MD5STEP(F1, b, c, d, a, in[15] + 0x49b40821, 22); MD5STEP(F2, a, b, c, d, in[1] + 0xf61e2562, 5); MD5STEP(F2, d, a, b, c, in[6] + 0xc040b340, 9); MD5STEP(F2, c, d, a, b, in[11] + 0x265e5a51, 14); MD5STEP(F2, b, c, d, a, in[0] + 0xe9b6c7aa, 20); MD5STEP(F2, a, b, c, d, in[5] + 0xd62f105d, 5); MD5STEP(F2, d, a, b, c, in[10] + 0x02441453, 9); MD5STEP(F2, c, d, a, b, in[15] + 0xd8a1e681, 14); MD5STEP(F2, b, c, d, a, in[4] + 0xe7d3fbc8, 20); MD5STEP(F2, a, b, c, d, in[9] + 0x21e1cde6, 5); MD5STEP(F2, d, a, b, c, in[14] + 0xc33707d6, 9); MD5STEP(F2, c, d, a, b, in[3] + 0xf4d50d87, 14); MD5STEP(F2, b, c, d, a, in[8] + 0x455a14ed, 20); MD5STEP(F2, a, b, c, d, in[13] + 0xa9e3e905, 5); MD5STEP(F2, d, a, b, c, in[2] + 0xfcefa3f8, 9); MD5STEP(F2, c, d, a, b, in[7] + 0x676f02d9, 14); MD5STEP(F2, b, c, d, a, in[12] + 0x8d2a4c8a, 20); MD5STEP(F3, a, b, c, d, in[5] + 0xfffa3942, 4); MD5STEP(F3, d, a, b, c, in[8] + 0x8771f681, 11); MD5STEP(F3, c, d, a, b, in[11] + 0x6d9d6122, 16); MD5STEP(F3, b, c, d, a, in[14] + 0xfde5380c, 23); MD5STEP(F3, a, b, c, d, in[1] + 0xa4beea44, 4); MD5STEP(F3, d, a, b, c, in[4] + 0x4bdecfa9, 11); MD5STEP(F3, c, d, a, b, in[7] + 0xf6bb4b60, 16); MD5STEP(F3, b, c, d, a, in[10] + 0xbebfbc70, 23); MD5STEP(F3, a, b, c, d, in[13] + 0x289b7ec6, 4); MD5STEP(F3, d, a, b, c, in[0] + 0xeaa127fa, 11); MD5STEP(F3, c, d, a, b, in[3] + 0xd4ef3085, 16); MD5STEP(F3, b, c, d, a, in[6] + 0x04881d05, 23); MD5STEP(F3, a, b, c, d, in[9] + 0xd9d4d039, 4); MD5STEP(F3, d, a, b, c, in[12] + 0xe6db99e5, 11); MD5STEP(F3, c, d, a, b, in[15] + 0x1fa27cf8, 16); MD5STEP(F3, b, c, d, a, in[2] + 0xc4ac5665, 23); MD5STEP(F4, a, b, c, d, in[0] + 0xf4292244, 6); MD5STEP(F4, d, a, b, c, in[7] + 0x432aff97, 10); MD5STEP(F4, c, d, a, b, in[14] + 0xab9423a7, 15); MD5STEP(F4, b, c, d, a, in[5] + 0xfc93a039, 21); MD5STEP(F4, a, b, c, d, in[12] + 0x655b59c3, 6); MD5STEP(F4, d, a, b, c, in[3] + 0x8f0ccc92, 10); MD5STEP(F4, c, d, a, b, in[10] + 0xffeff47d, 15); MD5STEP(F4, b, c, d, a, in[1] + 0x85845dd1, 21); MD5STEP(F4, a, b, c, d, in[8] + 0x6fa87e4f, 6); MD5STEP(F4, d, a, b, c, in[15] + 0xfe2ce6e0, 10); MD5STEP(F4, c, d, a, b, in[6] + 0xa3014314, 15); MD5STEP(F4, b, c, d, a, in[13] + 0x4e0811a1, 21); MD5STEP(F4, a, b, c, d, in[4] + 0xf7537e82, 6); MD5STEP(F4, d, a, b, c, in[11] + 0xbd3af235, 10); MD5STEP(F4, c, d, a, b, in[2] + 0x2ad7d2bb, 15); MD5STEP(F4, b, c, d, a, in[9] + 0xeb86d391, 21); buf[0] += a; buf[1] += b; buf[2] += c; buf[3] += d; } #endif xrootd-5.6.9/src/XrdCks/XrdCksCalcmd5.hh000066400000000000000000000067541457266313600200000ustar00rootroot00000000000000#ifndef __XRDCKSCALCMD5_HH__ #define __XRDCKSCALCMD5_HH__ /******************************************************************************/ /* */ /* X r d C k s C a l c m d 5 . h h */ /* */ /* (c) 2011 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include "XrdCks/XrdCksCalc.hh" class XrdCksCalcmd5 : public XrdCksCalc { public: char *Current() {MD5Context saveCTX = myContext; char *md5P = Final(); myContext = saveCTX; return (char *)md5P; } void Init(); XrdCksCalc *New() {return (XrdCksCalc *)new XrdCksCalcmd5;} char *Final(); void Update(const char *Buff, int BLen) {MD5Update((unsigned char *)Buff,(unsigned)BLen);} const char *Type(int &csSz) {csSz = sizeof(myDigest); return "md5";} XrdCksCalcmd5() {Init();} ~XrdCksCalcmd5() {} private: struct MD5Context {unsigned int buf[4]; union {long long b64; unsigned int bits[2]; }; union {long long i64[8]; unsigned char in[64]; }; }; MD5Context myContext; unsigned char myDigest[16]; void byteReverse(unsigned char *buf, unsigned longs); void MD5Update(unsigned char const *buf, unsigned int len); #ifndef ASM_MD5 void MD5Transform(unsigned int buf[4], unsigned int const in[16]); #endif }; #endif xrootd-5.6.9/src/XrdCks/XrdCksCalczcrc32.cc000066400000000000000000000116541457266313600204020ustar00rootroot00000000000000/******************************************************************************/ /* */ /* X r d C k s C a l c z c r c 3 2 . h h */ /* */ /* Copyright (c) 2012 by European Organization of Nuclear Research (CERN) */ /* Produced by Lukasz Janyst */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #ifndef __XRDCKSCALCZCRC32_HH__ #define __XRDCKSCALCZCRC32_HH__ #include "XrdCks/XrdCksCalc.hh" #include "XrdSys/XrdSysError.hh" #include "XrdVersion.hh" #include #include //------------------------------------------------------------------------------ // CRC32 checkum according to the algorithm implemented in zlib //------------------------------------------------------------------------------ class XrdCksCalczcrc32: public XrdCksCalc { public: //-------------------------------------------------------------------------- //! Constructor //-------------------------------------------------------------------------- XrdCksCalczcrc32() { Init(); } //-------------------------------------------------------------------------- //! Destructor //-------------------------------------------------------------------------- virtual ~XrdCksCalczcrc32() { } //-------------------------------------------------------------------------- //! Final checksum //-------------------------------------------------------------------------- char *Final() { return (char *)&pCheckSum; } //-------------------------------------------------------------------------- //! Initialize //-------------------------------------------------------------------------- void Init() { pCheckSum = crc32( 0L, Z_NULL, 0 ); } //-------------------------------------------------------------------------- //! Virtual constructor //-------------------------------------------------------------------------- XrdCksCalc *New() { return new XrdCksCalczcrc32(); } //-------------------------------------------------------------------------- //! Update current checksum //-------------------------------------------------------------------------- void Update( const char *Buff, int BLen ) { pCheckSum = crc32( pCheckSum, (const Bytef*)Buff, BLen ); } //-------------------------------------------------------------------------- //! Checksum algorithm name //-------------------------------------------------------------------------- const char *Type(int &csSz) { csSz = 4; return "zcrc32"; } private: uint32_t pCheckSum; }; //------------------------------------------------------------------------------ // Plugin callback //------------------------------------------------------------------------------ extern "C" XrdCksCalc *XrdCksCalcInit(XrdSysError *eDest, const char *csName, const char *cFN, const char *Parms) { return new XrdCksCalczcrc32(); } XrdVERSIONINFO(XrdCksCalcInit, zcrc32); #endif // __XRDCKSCALCZCRC32_HH__ xrootd-5.6.9/src/XrdCks/XrdCksConfig.cc000066400000000000000000000237561457266313600177240ustar00rootroot00000000000000/******************************************************************************/ /* */ /* X r d C k s C o n f i g . c c */ /* */ /* (c) 2011 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include #include #include #include #include #include "XrdVersion.hh" #include "XrdCks/XrdCks.hh" #include "XrdCks/XrdCksData.hh" #include "XrdCks/XrdCksConfig.hh" #include "XrdCks/XrdCksManager.hh" #include "XrdCks/XrdCksManOss.hh" #include "XrdCks/XrdCksWrapper.hh" #include "XrdOuc/XrdOucPinLoader.hh" #include "XrdOuc/XrdOucStream.hh" #include "XrdOuc/XrdOucUtils.hh" #include "XrdSys/XrdSysError.hh" #include "XrdSys/XrdSysPlatform.hh" #include "XrdSys/XrdSysPlugin.hh" /******************************************************************************/ /* C o n s t r u c t o r */ /******************************************************************************/ XrdCksConfig::XrdCksConfig(const char *cFN, XrdSysError *Eroute, int &aOK, XrdVersionInfo &vInfo) : eDest(Eroute), cfgFN(cFN), CksLib(0), CksParm(0), CksList(0), CksLast(0), LibList(0), LibLast(0), myVersion(vInfo) { static XrdVERSIONINFODEF(myVer, XrdCks, XrdVNUMBER, XrdVERSION); // Verify caller's version against ours // if (vInfo.vNum <= 0 || vInfo.vNum == myVer.vNum || XrdSysPlugin::VerCmp(vInfo, myVer)) aOK = 1; else aOK = 0; } /******************************************************************************/ /* a d d C k s */ /******************************************************************************/ XrdCks *XrdCksConfig::addCks(XrdCks *pCks, XrdOucEnv *envP) { XrdOucPinLoader *myLib; XrdCks *(*ep)(XRDCKSADD2PARMS); const char *theParm; XrdOucTList *tP = LibList; // Create a plugin object (we will throw this away without deletion because // the library must stay open but we never want to reference it again). // while(tP) {if (!(myLib = new XrdOucPinLoader(eDest,&myVersion,"ckslib",tP->text))) return 0; // Now get the entry point of the object creator // ep = (XrdCks *(*)(XRDCKSADD2PARMS))(myLib->Resolve("XrdCksAdd2")); if (!ep) {myLib->Unload(true); return 0;} // Get the Object now // delete myLib; theParm = (tP->val ? (tP->text) + tP->val : 0); pCks = ep(*pCks, eDest, cfgFN, theParm, envP); if (!pCks) return 0; // Move on to the next stacked plugin // tP = tP->next; } // All done // return pCks; } /******************************************************************************/ /* C o n f i g u r e */ /******************************************************************************/ XrdCks *XrdCksConfig::Configure(const char *dfltCalc, int rdsz, XrdOss *ossP, XrdOucEnv *envP) { XrdCks *myCks = getCks(ossP, rdsz); XrdOucTList *tP = CksList; int NoGo = 0; // Check if we have a cks object // if (!myCks) return 0; // Stack all pugins // myCks = addCks(myCks, envP); if (!myCks) return 0; // Configure the object // while(tP) {NoGo |= myCks->Config("ckslib", tP->text); tP = tP->next;} // Configure if all went well // if (!NoGo) NoGo = !myCks->Init(cfgFN, dfltCalc); // All done // if (NoGo) {delete myCks; myCks = 0;} return myCks; } /******************************************************************************/ /* Private: g e t C k s */ /******************************************************************************/ XrdCks *XrdCksConfig::getCks(XrdOss *ossP, int rdsz) { XrdOucPinLoader *myLib; XrdCks *(*ep)(XRDCKSINITPARMS); // Cks manager comes from the library or we use the default // if (!CksLib) {if (ossP) return (XrdCks *)new XrdCksManOss (ossP,eDest,rdsz,myVersion); else return (XrdCks *)new XrdCksManager( eDest,rdsz,myVersion); } // Create a plugin object (we will throw this away without deletion because // the library must stay open but we never want to reference it again). // if (!(myLib = new XrdOucPinLoader(eDest, &myVersion, "ckslib", CksLib))) return 0; // Now get the entry point of the object creator // ep = (XrdCks *(*)(XRDCKSINITPARMS))(myLib->Resolve("XrdCksInit")); if (!ep) {myLib->Unload(true); return 0;} // Get the Object now // delete myLib; return ep(eDest, cfgFN, CksParm); } /******************************************************************************/ /* M a n a g e r */ /******************************************************************************/ /* Function: Manager Purpose: Reset the manager plugin library path and parameters. path to the library. optional parms to be passed Output: 0 upon success or !0 upon failure. */ int XrdCksConfig::Manager(const char *Path, const char *Parms) { // Replace the library path and parameters // if (CksLib) free(CksLib); CksLib = strdup(Path); if (CksParm) free(CksParm); CksParm = (Parms && *Parms ? strdup(Parms) : 0); return 0; } /******************************************************************************/ /* P a r s e L i b */ /******************************************************************************/ /* Function: ParseLib Purpose: To parse the directive: ckslib [] the name of the checksum. The special name "*" is used load the checksum manager library. the path of the checksum library to be used. optional parms to be passed Output: 0 upon success or !0 upon failure. */ int XrdCksConfig::ParseLib(XrdOucStream &Config, int &libType) { static const int nameSize = XrdCksData::NameSize; static const int pathSize = MAXPATHLEN; static const int parmSize = 1024; XrdOucTList *tP; char *val, buff[nameSize + pathSize + parmSize + 8], parms[parmSize], *bP; int n; // Get the digest // if (!(val = Config.GetWord()) || !val[0]) {eDest->Emsg("Config", "ckslib digest not specified"); return 1;} n = strlen(val); if (n >= nameSize) {eDest->Emsg("Config", "ckslib digest name too long -", val); return 1;} strcpy(buff, val); XrdOucUtils::toLower(buff); bP = buff+n; *bP++ = ' '; // Get the path // if (!(val = Config.GetWord()) || !val[0]) {eDest->Emsg("Config", "ckslib path not specified for", buff); return 1;} n = strlen(val); if (n > pathSize) {eDest->Emsg("Config", "ckslib path name too long -", val); return 1;} strcpy(bP, val); bP += n; // Record any parms // *parms = 0; if (!Config.GetRest(parms, parmSize)) {eDest->Emsg("Config", "ckslib parameters too long for", buff); return 1;} // Check if this is for the manager // if ((*buff == '*' || *buff == '=') && *(buff+1) == ' ') {libType = (*buff == '*' ? -1 : 1); return Manager(buff+2, parms); } else libType = 0; // Create a new TList object either for a digest or stackable library // n = (strncmp(buff, "++ ", 3) ? 0 : 3); *bP = ' '; strcpy(bP+1, parms); tP = new XrdOucTList(buff + n); // Add this digest to the list of digests or stackable library list // if (n) {n = (bP - buff) - n; tP->text[n] = 0; tP->val = (*parms ? n+1 : 0); if (LibLast) LibLast->next = tP; else LibList = tP; LibLast = tP; } else { if (CksLast) CksLast->next = tP; else CksList = tP; CksLast = tP; } return 0; } xrootd-5.6.9/src/XrdCks/XrdCksConfig.hh000066400000000000000000000071171457266313600177270ustar00rootroot00000000000000#ifndef __XRDCKSCONFIG_HH__ #define __XRDCKSCONFIG_HH__ /******************************************************************************/ /* */ /* X r d C k s C o n f i g . h h */ /* */ /* (c) 2011 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include "XrdOuc/XrdOucTList.hh" class XrdCks; class XrdOss; class XrdOucEnv; class XrdOucStream; class XrdSysError; struct XrdVersionInfo; class XrdCksConfig { public: XrdCks *Configure(const char *dfltCalc=0, int rdsz=0, XrdOss *ossP=0, XrdOucEnv *envP=0); int Manager() {return CksLib != 0;} int Manager(const char *Path, const char *Parms); const char *ManLib() {return CksLib;} int ParseLib(XrdOucStream &Config, int &libType); XrdCksConfig(const char *cFN, XrdSysError *Eroute, int &aOK, XrdVersionInfo &vInfo); ~XrdCksConfig() {XrdOucTList *tP; if (CksLib) free(CksLib); if (CksParm) free(CksParm); while((tP = CksList)) {CksList = tP->next; delete tP;} while((tP = LibList)) {LibList = tP->next; delete tP;} } private: XrdCks *addCks(XrdCks *pCks, XrdOucEnv *envP); XrdCks *getCks(XrdOss *ossP, int rdsz); XrdSysError *eDest; const char *cfgFN; char *CksLib; char *CksParm; XrdOucTList *CksList; XrdOucTList *CksLast; XrdOucTList *LibList; XrdOucTList *LibLast; XrdVersionInfo &myVersion; }; #endif xrootd-5.6.9/src/XrdCks/XrdCksData.hh000066400000000000000000000130471457266313600173720ustar00rootroot00000000000000#ifndef __XRDCKSDATA_HH__ #define __XRDCKSDATA_HH__ /******************************************************************************/ /* */ /* X r d C k s D a t a . h h */ /* */ /* (c) 2011 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include class XrdOucEnv; class XrdCksData { public: static const int NameSize = 16; // Max name length is NameSize - 1 static const int ValuSize = 64; // Max value length is 512 bits char Name[NameSize]; // Checksum algorithm name union { long long fmTime; // Out: File's mtime when checksum was computed. XrdOucEnv*envP; // In: Set for get & calc, only! }; int csTime; // Delta from fmTime when checksum was computed. short Rsvd1; // Reserved field char Rsvd2; // Reserved field char Length; // Length, in bytes, of the checksum value char Value[ValuSize]; // The binary checksum value inline int operator==(const XrdCksData &oth) {return (!strncmp(Name, oth.Name, NameSize) && Length == oth.Length && !memcmp(Value, oth.Value, Length)); } inline int operator!=(const XrdCksData &oth) {return (strncmp(Name, oth.Name, NameSize) || Length != oth.Length || memcmp(Value, oth.Value, Length)); } int Get(char *Buff, int Blen) {const char *hv = "0123456789abcdef"; int i, j = 0; if (Blen < Length*2+1) return 0; for (i = 0; i < Length; i++) {Buff[j++] = hv[(Value[i] >> 4) & 0x0f]; Buff[j++] = hv[ Value[i] & 0x0f]; } Buff[j] = '\0'; return Length*2; } int Set(const char *csName) {size_t len = strlen(csName); if (len >= sizeof(Name)) return 0; memcpy(Name, csName, len); Name[len]=0; return 1; } int Set(const void *csVal, int csLen) {if (csLen > ValuSize || csLen < 1) return 0; memcpy(Value, csVal, csLen); Length = csLen; return 1; } int Set(const char *csVal, int csLen) {int n, i = 0, Odd = 0; if (csLen > (int)sizeof(Value)*2 || (csLen & 1)) return 0; Length = csLen/2; while(csLen--) { if (*csVal >= '0' && *csVal <= '9') n = *csVal-48; else if (*csVal >= 'a' && *csVal <= 'f') n = *csVal-87; else if (*csVal >= 'A' && *csVal <= 'F') n = *csVal-55; else return 0; if (Odd) Value[i++] |= n; else Value[i ] = n << 4; csVal++; Odd = ~Odd; } return 1; } void Reset() {memset(Name, 0, sizeof(Name)); memset(Value,0, sizeof(Value)); fmTime = 0; csTime = 0; Rsvd1 = 0; Rsvd2 = 0; Length = 0; } XrdCksData() {Reset();} bool HasValue() { return *Value; } }; #endif xrootd-5.6.9/src/XrdCks/XrdCksLoader.cc000066400000000000000000000173701457266313600177200ustar00rootroot00000000000000/******************************************************************************/ /* */ /* X r d C k s L o a d e r . c c */ /* */ /* (c) 2012 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include #include #include #include #include #include "XrdCks/XrdCksCalc.hh" #include "XrdCks/XrdCksCalcadler32.hh" #include "XrdCks/XrdCksCalccrc32.hh" #include "XrdCks/XrdCksCalcmd5.hh" #include "XrdCks/XrdCksLoader.hh" #include "XrdOuc/XrdOucPinLoader.hh" #include "XrdSys/XrdSysPlugin.hh" #include "XrdSys/XrdSysPthread.hh" #include "XrdVersion.hh" /******************************************************************************/ /* C o n s t r u c t o r */ /******************************************************************************/ XrdCksLoader::XrdCksLoader(XrdVersionInfo &vInfo, const char *libPath) { static XrdVERSIONINFODEF(myVersion, XrdCks, XrdVNUMBER, XrdVERSION); static const char libSfx[] = "/libXrdCksCalc%s.so"; int k, n; // Verify that versions are compatible. // if (vInfo.vNum != myVersion.vNum && !XrdSysPlugin::VerCmp(vInfo, myVersion, true)) {char buff[1024]; snprintf(buff, sizeof(buff), "Version %s is incompatible with %s.", vInfo.vStr, myVersion.vStr); verMsg = strdup(buff); urVersion = 0; return; } urVersion = &vInfo; verMsg = 0; // Prefill the native digests we support // csTab[0].Name = strdup("adler32"); csTab[1].Name = strdup("crc32"); csTab[2].Name = strdup("md5"); csLast = 2; // Record the over-ride loader path // if (libPath) {n = strlen(libPath); ldPath = (char *)malloc(n+sizeof(libSfx)+1); k = (libPath[n-1] == '/'); strcpy(ldPath, libPath); strcpy(ldPath+n, libSfx+k); } else ldPath = strdup(libSfx+1); } /******************************************************************************/ /* D e s t r u c t o r */ /******************************************************************************/ XrdCksLoader::~XrdCksLoader() { int i; for (i = 0; i <= csLast; i++) {if (csTab[i].Name) free( csTab[i].Name); if (csTab[i].Obj) csTab[i].Obj->Recycle(); if (csTab[i].Plugin) delete csTab[i].Plugin; } if (ldPath) free(ldPath); if (verMsg) free(verMsg); } /******************************************************************************/ /* L o a d */ /******************************************************************************/ #define XRDOSSCKSLIBARGS XrdSysError *, const char *, const char *, const char * XrdCksCalc *XrdCksLoader::Load(const char *csName, const char *csParms, char *eBuff, int eBlen, bool orig) { static XrdSysMutex myMutex; XrdSysMutexHelper ldMutex(myMutex); XrdCksCalc *(*ep)(XRDOSSCKSLIBARGS); XrdCksCalc *csObj; XrdOucPinLoader *myPin; csInfo *csIP; char ldBuff[2048]; int n; // Verify that version checking succeeded // if (verMsg) {if (eBuff) strncpy(eBuff, verMsg, eBlen); return 0;} // First check if we have loaded this before // if ((csIP = Find(csName))) {if (!(csIP->Obj)) { if (!strcmp("adler32", csIP->Name)) csIP->Obj = new XrdCksCalcadler32; else if (!strcmp("crc32", csIP->Name)) csIP->Obj = new XrdCksCalccrc32; else if (!strcmp("md5", csIP->Name)) csIP->Obj = new XrdCksCalcmd5; else {if (eBuff) snprintf(eBuff, eBlen, "Logic error configuring %s " "checksum.", csName); return 0; } } return (orig ? csIP->Obj : csIP->Obj->New()); } // Check if we can add a new entry // if (csLast+1 >= csMax) {if (eBuff) strncpy(eBuff, "Maximum number of checksums loaded.", eBlen); return 0; } // Get the path where this object lives // snprintf(ldBuff, sizeof(ldBuff), ldPath, csName); // Get the plugin loader // if (!(myPin = new XrdOucPinLoader(eBuff,eBlen,urVersion,"ckslib",ldBuff))) return 0; // Find the entry point // if (!(ep = (XrdCksCalc *(*)(XRDOSSCKSLIBARGS)) (myPin->Resolve("XrdCksCalcInit")))) {myPin->Unload(true); return 0;} // Get the initial object // if (!(csObj = ep(0, 0, csName, csParms))) {if (eBuff) snprintf(eBuff, eBlen, "%s checksum initialization failed.", csName); myPin->Unload(true); return 0; } // Verify the object // if (strcmp(csName, csObj->Type(n))) {if (eBuff) snprintf(eBuff, eBlen, "%s cksum plugin returned wrong name - %s", csName, csObj->Type(n)); delete csObj; myPin->Unload(true); return 0; } // Allocate a new entry in the table and initialize it // csLast++; csTab[csLast].Name = strdup(csName); csTab[csLast].Obj = csObj; csTab[csLast].Plugin = myPin->Export(); // Return new instance of this object // return (orig ? csObj : csObj->New()); } /******************************************************************************/ /* F i n d */ /******************************************************************************/ XrdCksLoader::csInfo *XrdCksLoader::Find(const char *Name) { int i; for (i = 0; i <= csLast; i++) if (!strcmp(Name, csTab[i].Name)) return &csTab[i]; return 0; } xrootd-5.6.9/src/XrdCks/XrdCksLoader.hh000066400000000000000000000137311457266313600177270ustar00rootroot00000000000000#ifndef __XRDCKSLOADER_HH__ #define __XRDCKSLOADER_HH__ /******************************************************************************/ /* */ /* X r d C k s L o a d e r . h h */ /* */ /* (c) 2012 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ class XrdCksCalc; class XrdSysPlugin; struct XrdVersionInfo; /*! This class defines the checksum loader interface. It is intended to be used by xrootd clients to obtain an instance of a checksum calculation object. This object may be builtin or may come from a shared library. */ class XrdCksLoader { public: //------------------------------------------------------------------------------ //! Get a new XrdCksCalc object that can calculate the checksum corresponding to //! the specified name. The object can be used to compute checksums on the fly. //! The object's Recycle() method must be used to delete it. The adler32, crc32, //! and md5 checksums are natively supported. Up to five more checksum //! algorithms can be loaded from shared libraries. //! //! @param csName The name of the checksum algorithm (e.g. md5). //! @param csParms Any parameters that might be needed by the checksum //! algorithm should it be loaded from a shared library. //! @param eBuff Optional pointer to a buffer to receive the reason for a //! load failure as a null terminated string. //! @param eBlen The length of the buffer. //! @param orig Returns the original object not a new instance of it. //! This is usually used by CksManager during an autoload. //! //! @return Success: A pointer to a new checksum calculation object. //! Failure: Zero if the corresponding checksum object could not be //! loaded. If eBuff was supplied, it holds the reason. //------------------------------------------------------------------------------ XrdCksCalc *Load(const char *csName, const char *csParms=0, char *eBuff=0, int eBlen=0, bool orig=false); //------------------------------------------------------------------------------ //! Constructor //! //! @param vInfo Is the reference to the version information corresponding //! to the xrootd version you compiled with. You define this //! information using the XrdVERSIONINFODEF macro defined in //! XrdVersion.hh. You must supply your version information //! and it must be compatible with the loader and any shared //! libraries that it might load on your behalf. //! //! @param libPath The path where dynamic checksum calculators are to be //! found and dynamically loaded, if need be. If libPath is //! nil then the default loader search order is used. //! The name of the shared library must follow the naming //! convention "libXrdCksCalc.so" where is the //! checksum name. So, an sha256 checksum would try to load //! libXrdCksCalcsha256.so shared library. //------------------------------------------------------------------------------ XrdCksLoader(XrdVersionInfo &vInfo, const char *libPath=0); //------------------------------------------------------------------------------ //! Destructor //------------------------------------------------------------------------------ ~XrdCksLoader(); private: struct csInfo {char *Name; XrdCksCalc *Obj; XrdSysPlugin *Plugin; csInfo() : Name(0), Obj(0), Plugin(0) {} ~csInfo() {} }; csInfo *Find(const char *Name); char *verMsg; // This member must be the 1st member XrdVersionInfo *urVersion; // This member must be the 2nd member char *ldPath; static const int csMax = 8; csInfo csTab[csMax]; int csLast; }; #endif xrootd-5.6.9/src/XrdCks/XrdCksManOss.cc000066400000000000000000000221751457266313600177110ustar00rootroot00000000000000/******************************************************************************/ /* */ /* X r d C k s M a n O s s . c c */ /* */ /* (c) 2014 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include #include #include #include #include #include #include #include #include "XrdCks/XrdCksCalc.hh" #include "XrdCks/XrdCksManOss.hh" #include "XrdOss/XrdOss.hh" #include "XrdOuc/XrdOucEnv.hh" #include "XrdSys/XrdSysError.hh" #include "XrdSys/XrdSysPlatform.hh" /******************************************************************************/ /* L o c a l S t a t i c s */ /******************************************************************************/ namespace { XrdOss *ossP = 0; int rdSz = 67108864; } /******************************************************************************/ /* L o c a l C l a s s e s */ /******************************************************************************/ class LfnPfn {public: const char *Lfn; char Pfn[MAXPATHLEN+8]; LfnPfn(const char *lfn, int &rc) : Lfn(lfn) {rc = ossP->Lfn2Pfn(lfn, Pfn, MAXPATHLEN); if (rc > 0) rc = -rc; } ~LfnPfn() {} }; namespace { const char *Pfn2Lfn(const char *Lfn) {LfnPfn *Xfn = (LfnPfn *)(Lfn-sizeof(const char *)); return Xfn->Lfn; } } /******************************************************************************/ /* C o n s t r u c t o r */ /******************************************************************************/ XrdCksManOss::XrdCksManOss(XrdOss *ossX, XrdSysError *erP, int iosz, XrdVersionInfo &vInfo, bool autoload) : XrdCksManager(erP, rdSz, vInfo, autoload) {if (rdSz <= 65536) rdSz = 67108864; else rdSz = ((rdSz/65536) + (rdSz%65536 != 0)) * 65536; eDest = erP; ossP = ossX; } /******************************************************************************/ /* C a l c */ /******************************************************************************/ int XrdCksManOss::Calc(const char *Lfn, XrdCksData &Cks, int doSet) { int rc; LfnPfn Xfn(Lfn, rc); // If lfn conversion failed, bail out // if (rc) return rc; // Return the result // return XrdCksManager::Calc(Xfn.Pfn, Cks, doSet); } /******************************************************************************/ int XrdCksManOss::Calc(const char *Pfn, time_t &MTime, XrdCksCalc *csP) { class inFile {public: XrdOssDF *fP; inFile() {fP = ossP->newFile("ckscalc");} ~inFile() {if (fP) delete fP;} } In; XrdOucEnv openEnv; const char *Lfn = Pfn2Lfn(Pfn); struct stat Stat; char *buffP; off_t Offset=0, fileSize; size_t ioSize, calcSize; int rc; // Open the input file // if ((rc = In.fP->Open(Lfn,O_RDONLY,0,openEnv))) return (rc > 0 ? -rc : rc); // Get the file characteristics // if ((rc = In.fP->Fstat(&Stat))) return (rc > 0 ? -rc : rc); if (!(Stat.st_mode & S_IFREG)) return -EPERM; calcSize = fileSize = Stat.st_size; MTime = Stat.st_mtime; // Compute read size and allocate a buffer // ioSize = (fileSize < (off_t)rdSz ? fileSize : rdSz); rc = 0; buffP = (char *)malloc(ioSize); if (!buffP) return -ENOMEM; // We now compute checksum 64MB at a time using mmap I/O // while(calcSize) {if ((rc= In.fP->Read(buffP, Offset, ioSize)) < 0) break; csP->Update(buffP, ioSize); calcSize -= ioSize; Offset += ioSize; if (calcSize < (size_t)ioSize) ioSize = calcSize; } free(buffP); // Issue error message if we have an error // if (rc < 0) eDest->Emsg("Cks", rc, "read", Pfn); // Return // return (rc < 0 ? rc : 0); } /******************************************************************************/ /* D e l */ /******************************************************************************/ int XrdCksManOss::Del(const char *Lfn, XrdCksData &Cks) { int rc; LfnPfn Xfn(Lfn, rc); // If lfn conversion failed, bail out // if (rc) return rc; // Delete the attribute and return the result // return XrdCksManager::Del(Xfn.Pfn, Cks); } /******************************************************************************/ /* G e t */ /******************************************************************************/ int XrdCksManOss::Get(const char *Lfn, XrdCksData &Cks) { int rc; LfnPfn Xfn(Lfn, rc); // If lfn conversion failed, bail out // if (rc) return rc; // Return result // return XrdCksManager::Get(Xfn.Pfn, Cks); } /******************************************************************************/ /* L i s t */ /******************************************************************************/ char *XrdCksManOss::List(const char *Lfn, char *Buff, int Blen, char Sep) { int rc; LfnPfn Xfn(Lfn, rc); // If lfn conversion failed, bail out // if (rc) return 0; // Simply invoke the base class list // return XrdCksManager::List(Xfn.Pfn, Buff, Blen,Sep); } /******************************************************************************/ /* M o d T i m e */ /******************************************************************************/ int XrdCksManOss::ModTime(const char *Pfn, time_t &MTime) { const char *Lfn = Pfn2Lfn(Pfn); struct stat Stat; int rc; if (!(rc = ossP->Stat(Lfn, &Stat))) MTime = Stat.st_mtime; return (rc > 0 ? -rc : 0); } /******************************************************************************/ /* S e t */ /******************************************************************************/ int XrdCksManOss::Set(const char *Lfn, XrdCksData &Cks, int myTime) { int rc; LfnPfn Xfn(Lfn, rc); // If lfn conversion failed, bail out // if (rc) return rc; // Now set the checksum information in the extended attribute object // return XrdCksManager::Set(Xfn.Pfn, Cks, myTime); } /******************************************************************************/ /* V e r */ /******************************************************************************/ int XrdCksManOss::Ver(const char *Lfn, XrdCksData &Cks) { int rc; LfnPfn Xfn(Lfn, rc); // If lfn conversion failed, bail out // if (rc) return rc; // Return result invoking the base class // return XrdCksManager::Ver(Lfn, Cks); } xrootd-5.6.9/src/XrdCks/XrdCksManOss.hh000066400000000000000000000066301457266313600177210ustar00rootroot00000000000000#ifndef __XRDCKSMANOSS_HH__ #define __XRDCKSMANOSS_HH__ /******************************************************************************/ /* */ /* X r d k s M a n O s s . h h */ /* */ /* (c) 2014 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include "sys/types.h" #include "XrdCks/XrdCksManager.hh" /* This class defines the checksum management interface using the oss plugin. It is used internally to provide checksums for oss-based storage systems. */ class XrdOss; class XrdCksManOss : public XrdCksManager { public: virtual int Calc(const char *Lfn, XrdCksData &Cks, int doSet=1); virtual int Del( const char *Lfn, XrdCksData &Cks); virtual int Get( const char *Lfn, XrdCksData &Cks); virtual char *List(const char *Lfn, char *Buff, int Blen, char Sep=' '); virtual int Set( const char *Lfn, XrdCksData &Cks, int myTime=0); virtual int Ver( const char *Lfn, XrdCksData &Cks); XrdCksManOss(XrdOss *ossX, XrdSysError *erP, int iosz, XrdVersionInfo &vInfo, bool autoload=false); virtual ~XrdCksManOss() {} protected: virtual int Calc(const char *Lfn, time_t &MTime, XrdCksCalc *CksObj); virtual int ModTime(const char *Pfn, time_t &MTime); private: int buffSZ; }; #endif xrootd-5.6.9/src/XrdCks/XrdCksManager.cc000066400000000000000000000512511457266313600200600ustar00rootroot00000000000000/******************************************************************************/ /* */ /* X r d O s s C k s M a n a g e r . c c */ /* */ /* (c) 2011 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include #include #include #include #include #include #include #include #include "XrdCks/XrdCksCalc.hh" #include "XrdCks/XrdCksCalcadler32.hh" #include "XrdCks/XrdCksCalccrc32.hh" #include "XrdCks/XrdCksCalccrc32C.hh" #include "XrdCks/XrdCksCalcmd5.hh" #include "XrdCks/XrdCksLoader.hh" #include "XrdCks/XrdCksManager.hh" #include "XrdCks/XrdCksXAttr.hh" #include "XrdOuc/XrdOucPinLoader.hh" #include "XrdOuc/XrdOucTokenizer.hh" #include "XrdOuc/XrdOucUtils.hh" #include "XrdOuc/XrdOucXAttr.hh" #include "XrdSys/XrdSysError.hh" #include "XrdSys/XrdSysFAttr.hh" #include "XrdSys/XrdSysPlugin.hh" #include "XrdSys/XrdSysPthread.hh" #ifndef ENOATTR #define ENOATTR ENODATA #endif /******************************************************************************/ /* C o n s t r u c t o r */ /******************************************************************************/ XrdCksManager::XrdCksManager(XrdSysError *erP, int rdsz, XrdVersionInfo &vInfo, bool autoload) : XrdCks(erP), myVersion(vInfo) { // Get a dynamic loader if so wanted // if (autoload) cksLoader = new XrdCksLoader(vInfo); else cksLoader = 0; // Prefill the native digests we support // strcpy(csTab[0].Name, "adler32"); strcpy(csTab[1].Name, "crc32"); strcpy(csTab[2].Name, "crc32c"); strcpy(csTab[3].Name, "md5"); csLast = 3; // Compute the i/o size // if (rdsz <= 65536) segSize = 67108864; else segSize = ((rdsz/65536) + (rdsz%65536 != 0)) * 65536; } /******************************************************************************/ /* D e s t r u c t o r */ /******************************************************************************/ XrdCksManager::~XrdCksManager() { int i; for (i = 0; i <= csLast; i++) {if (csTab[i].Obj && csTab[i].doDel) csTab[i].Obj->Recycle(); if (csTab[i].Path) free( csTab[i].Path); if (csTab[i].Parms) free( csTab[i].Parms); if (csTab[i].Plugin) delete csTab[i].Plugin; } if (cksLoader) delete cksLoader; } /******************************************************************************/ /* C a l c */ /******************************************************************************/ int XrdCksManager::Calc(const char *Pfn, XrdCksData &Cks, int doSet) { XrdCksCalc *csP; csInfo *csIP = &csTab[0]; time_t MTime; int rc; // Determine which checksum to get // if (csLast < 0) return -ENOTSUP; if (!(*Cks.Name)) Cks.Set(csIP->Name); else if (!(csIP = Find(Cks.Name))) return -ENOTSUP; // If we need not set the checksum then see if we can get it from the // extended attributes. // Obtain a new checksum object // if (!(csP = csIP->Obj->New())) return -ENOMEM; // Use the calculator to get and possibly set the checksum // if (!(rc = Calc(Pfn, MTime, csP))) {memcpy(Cks.Value, csP->Final(), csIP->Len); Cks.fmTime = static_cast(MTime); Cks.csTime = static_cast(time(0) - MTime); Cks.Length = csIP->Len; csP->Recycle(); if (doSet) {XrdOucXAttr xCS; memcpy(&xCS.Attr.Cks, &Cks, sizeof(xCS.Attr.Cks)); if ((rc = xCS.Set(Pfn))) return -rc; } } // All done // return rc; } /******************************************************************************/ int XrdCksManager::Calc(const char *Pfn, time_t &MTime, XrdCksCalc *csP) { class ioFD {public: int FD; ioFD() : FD(-1) {} ~ioFD() {if (FD >= 0) close(FD);} } In; struct stat Stat; char *inBuff; off_t Offset=0, fileSize; size_t ioSize, calcSize; int rc; // Open the input file // if ((In.FD = open(Pfn, O_RDONLY)) < 0) return -errno; // Get the file characteristics // if (fstat(In.FD, &Stat)) return -errno; if (!(Stat.st_mode & S_IFREG)) return -EPERM; calcSize = fileSize = Stat.st_size; MTime = Stat.st_mtime; // We now compute checksum 64MB at a time using mmap I/O // ioSize = (fileSize < (off_t)segSize ? fileSize : segSize); rc = 0; while(calcSize) {if ((inBuff = (char *)mmap(0, ioSize, PROT_READ, #if defined(__FreeBSD__) MAP_RESERVED0040|MAP_PRIVATE, In.FD, Offset)) == MAP_FAILED) #elif defined(__GNU__) MAP_PRIVATE, In.FD, Offset)) == MAP_FAILED) #else MAP_NORESERVE|MAP_PRIVATE, In.FD, Offset)) == MAP_FAILED) #endif {rc = errno; eDest->Emsg("Cks", rc, "memory map", Pfn); break;} madvise(inBuff, ioSize, MADV_SEQUENTIAL); csP->Update(inBuff, ioSize); calcSize -= ioSize; Offset += ioSize; if (munmap(inBuff, ioSize) < 0) {rc = errno; eDest->Emsg("Cks",rc,"unmap memory for",Pfn); break;} if (calcSize < (size_t)segSize) ioSize = calcSize; } // Return if we failed // if (calcSize) return (rc ? -rc : -EIO); return 0; } /******************************************************************************/ /* C o n f i g */ /******************************************************************************/ /* Purpose: To parse the directive: ckslib [] the name of the checksum. the path of the checksum library to be used. optional parms to be passed Output: 0 upon success or !0 upon failure. */ int XrdCksManager::Config(const char *Token, char *Line) { XrdOucTokenizer Cfg(Line); char *val, *path = 0, name[XrdCksData::NameSize], *parms; int i; // Get the the checksum name // Cfg.GetLine(); if (!(val = Cfg.GetToken()) || !val[0]) {eDest->Emsg("Config", "checksum name not specified"); return 1;} if (int(strlen(val)) >= XrdCksData::NameSize) {eDest->Emsg("Config", "checksum name too long"); return 1;} strcpy(name, val); XrdOucUtils::toLower(name); // Get the path and optional parameters // val = Cfg.GetToken(&parms); if (val && val[0]) path = strdup(val); else {eDest->Emsg("Config","library path missing for ckslib digest",name); return 1; } // Check if this replaces an existing checksum // for (i = 0; i < csMax; i++) if (!(*csTab[i].Name) || !strcmp(csTab[i].Name, name)) break; // See if we can insert a new checksum (or replace one) // if (i >= csMax) {eDest->Emsg("Config", "too many checksums specified"); if (path) free(path); return 1; } else if (!(*csTab[i].Name)) csLast = i; // Insert the new checksum // strcpy(csTab[i].Name, name); if (csTab[i].Path) free(csTab[i].Path); csTab[i].Path = path; if (csTab[i].Parms) free(csTab[i].Parms); csTab[i].Parms = (parms && *parms ? strdup(parms) : 0); // All done // return 0; } /******************************************************************************/ /* I n i t */ /******************************************************************************/ int XrdCksManager::Init(const char *ConfigFN, const char *DfltCalc) { int i; // See if we need to set the default calculation // if (DfltCalc) {for (i = 0; i < csLast; i++) if (!strcmp(csTab[i].Name, DfltCalc)) break; if (i >= csMax) {eDest->Emsg("Config", DfltCalc, "cannot be made the default; " "not supported."); return 0; } if (i) {csInfo Temp = csTab[i]; csTab[i] = csTab[0]; csTab[0] = Temp;} } // See if there are any checksums to configure // if (csLast < 0) {eDest->Emsg("Config", "No checksums defined; cannot configure!"); return 0; } // Complete the checksum table // for (i = 0; i <= csLast; i++) {if (csTab[i].Path) {if (!(Config(ConfigFN, csTab[i]))) return 0;} else { if (!strcmp("adler32", csTab[i].Name)) csTab[i].Obj = new XrdCksCalcadler32; else if (!strcmp("crc32", csTab[i].Name)) csTab[i].Obj = new XrdCksCalccrc32; else if (!strcmp("crc32c", csTab[i].Name)) csTab[i].Obj = new XrdCksCalccrc32C; else if (!strcmp("md5", csTab[i].Name)) csTab[i].Obj = new XrdCksCalcmd5; else {eDest->Emsg("Config", "Invalid native checksum -", csTab[i].Name); return 0; } csTab[i].Obj->Type(csTab[i].Len); } } // All done // return 1; } /******************************************************************************/ #define XRDOSSCKSLIBARGS XrdSysError *, const char *, const char *, const char * int XrdCksManager::Config(const char *cFN, csInfo &Info) { XrdOucPinLoader myPin(eDest, &myVersion, "ckslib", Info.Path); XrdCksCalc *(*ep)(XRDOSSCKSLIBARGS); int n; // Find the entry point // Info.Plugin = 0; if (!(ep = (XrdCksCalc *(*)(XRDOSSCKSLIBARGS)) (myPin.Resolve("XrdCksCalcInit")))) {eDest->Emsg("Config", "Unable to configure cksum", Info.Name); myPin.Unload(); return 0; } // Get the initial object // if (!(Info.Obj = ep(eDest,cFN,Info.Name,(Info.Parms ? Info.Parms : "")))) {eDest->Emsg("Config", Info.Name, "checksum initialization failed"); myPin.Unload(); return 0; } // Verify the object // if (strcmp(Info.Name, Info.Obj->Type(n))) {eDest->Emsg("Config",Info.Name,"cksum plugin returned wrong name -", Info.Obj->Type(n)); myPin.Unload(); return 0; } if (n > XrdCksData::ValuSize || n <= 0) {eDest->Emsg("Config",Info.Name,"cksum plugin has an unsupported " "checksum length"); myPin.Unload(); return 0; } // All is well // Info.Plugin = myPin.Export(); Info.Len = n; return 1; } /******************************************************************************/ /* F i n d */ /******************************************************************************/ XrdCksManager::csInfo *XrdCksManager::Find(const char *Name) { static XrdSysMutex myMutex; XrdCksCalc *myCalc; int i; // Find the pre-loaded checksum // for (i = 0; i <= csLast; i++) if (!strcmp(Name, csTab[i].Name)) return &csTab[i]; // If we have loader see if we can auto-load this object // if (!cksLoader) return 0; myMutex.Lock(); // An entry could have been added as we were running unlocked // for (i = 0; i <= csLast; i++) if (!strcmp(Name, csTab[i].Name)) {myMutex.UnLock(); return &csTab[i]; } // Check if we have room in the table // if (csLast >= csMax) {myMutex.UnLock(); eDest->Emsg("CksMan","Unable to load",Name,"; checksum limit reached."); return 0; } // Attempt to dynamically load this object // { char buff[2048]; *buff = 0; if (!(myCalc = cksLoader->Load(Name, 0, buff, sizeof(buff), true))) {myMutex.UnLock(); eDest->Emsg("CksMan", "Unable to load", Name); if (*buff) eDest->Emsg("CksMan", buff); return 0; } } // Fill out the table // i = csLast + 1; strncpy(csTab[i].Name, Name, XrdCksData::NameSize); csTab[i].Obj = myCalc; csTab[i].Path = 0; csTab[i].Parms = 0; csTab[i].Plugin = 0; csTab[i].doDel = false; myCalc->Type(csTab[i].Len); // Return the result // csLast = i; myMutex.UnLock(); return &csTab[i]; } /******************************************************************************/ /* D e l */ /******************************************************************************/ int XrdCksManager::Del(const char *Pfn, XrdCksData &Cks) { XrdOucXAttr xCS; // Set the checksum name // xCS.Attr.Cks.Set(Cks.Name); // Delete the attribute and return the result // return xCS.Del(Pfn); } /******************************************************************************/ /* G e t */ /******************************************************************************/ int XrdCksManager::Get(const char *Pfn, XrdCksData &Cks) { XrdOucXAttr xCS; time_t MTime; int rc, nFault; // Determine which checksum to get (we will accept unsupported ones as well) // if (csLast < 0) return -ENOTSUP; if (!*Cks.Name) Cks.Set(csTab[0].Name); if (!xCS.Attr.Cks.Set(Cks.Name)) return -ENOTSUP; // Retreive the attribute // if ((rc = xCS.Get(Pfn)) <= 0) return (rc && rc != -ENOATTR ? rc : -ESRCH); // Mark state of the name and copy the attribute over // nFault = strcmp(xCS.Attr.Cks.Name, Cks.Name); Cks = xCS.Attr.Cks; // Verify the file // if ((rc = ModTime(Pfn, MTime))) return rc; // Return result // return (Cks.fmTime != MTime || nFault || Cks.Length > XrdCksData::ValuSize || Cks.Length <= 0 ? -ESTALE : int(Cks.Length)); } /******************************************************************************/ /* L i s t */ /******************************************************************************/ char *XrdCksManager::List(const char *Pfn, char *Buff, int Blen, char Sep) { static const char *vPfx = "XrdCks."; static const int vPln = strlen(vPfx); XrdSysFAttr::AList *vP, *axP = 0; char *bP = Buff; int i, n; // Verify that the buffer is large enough // if (Blen < 2) return 0; // Check if the default list is wanted // if (!Pfn) {if (csLast < 0) return 0; i = 0; while(i <= csLast && Blen > 1) {n = strlen(csTab[i].Name); if (n >= Blen) break; if (bP != Buff) *bP++ = Sep; strcpy(bP, csTab[i].Name); bP += n; *bP = 0; } return (bP == Buff ? 0 : Buff); } // Get a list of attributes for this file // if (XrdSysFAttr::Xat->List(&axP, Pfn) < 0 || !axP) return 0; // Go through the list extracting what we are looking for // vP = axP; while(vP) {if (vP->Nlen > vPln && !strncmp(vP->Name, vPfx, vPln)) {n = vP->Nlen - vPln; if (n >= Blen) break; if (bP != Buff) *bP++ = Sep; strcpy(bP, vP->Name + vPln); bP += n; *bP = 0; } vP = vP->Next; } // All done // XrdSysFAttr::Xat->Free(axP); return (bP == Buff ? 0 : Buff); } /******************************************************************************/ /* M o d T i m e */ /******************************************************************************/ int XrdCksManager::ModTime(const char *Pfn, time_t &MTime) { struct stat Stat; if (stat(Pfn, &Stat)) return -errno; MTime = Stat.st_mtime; return 0; } /******************************************************************************/ /* N a m e */ /******************************************************************************/ const char *XrdCksManager::Name(int seqNum) { return (seqNum < 0 || seqNum > csLast ? 0 : csTab[seqNum].Name); } /******************************************************************************/ /* O b j e c t */ /******************************************************************************/ XrdCksCalc *XrdCksManager::Object(const char *name) { csInfo *csIP = &csTab[0]; // Return an object it at all possible // if (name && !(csIP = Find(name))) return 0; return csIP->Obj->New(); } /******************************************************************************/ /* S i z e */ /******************************************************************************/ int XrdCksManager::Size(const char *Name) { csInfo *iP = (Name != 0 ? Find(Name) : &csTab[0]); return (iP != 0 ? iP->Len : 0); } /******************************************************************************/ /* S e t */ /******************************************************************************/ int XrdCksManager::Set(const char *Pfn, XrdCksData &Cks, int myTime) { XrdOucXAttr xCS; csInfo *csIP = &csTab[0]; // Verify the incoming checksum for correctness // if (csLast < 0 || (*Cks.Name && !(csIP = Find(Cks.Name)))) return -ENOTSUP; if (Cks.Length != csIP->Len) return -EDOM; memcpy(&xCS.Attr.Cks, &Cks, sizeof(xCS.Attr.Cks)); // Set correct times if need be // if (!myTime) {time_t MTime; int rc = ModTime(Pfn, MTime); if (rc) return rc; xCS.Attr.Cks.fmTime = static_cast(MTime); xCS.Attr.Cks.csTime = static_cast(time(0) - MTime); } // Now set the checksum information in the extended attribute object // return xCS.Set(Pfn); } /******************************************************************************/ /* V e r */ /******************************************************************************/ int XrdCksManager::Ver(const char *Pfn, XrdCksData &Cks) { XrdOucXAttr xCS; time_t MTime; csInfo *csIP = &csTab[0]; int rc; // Determine which checksum to get // if (csLast < 0 || (*Cks.Name && !(csIP = Find(Cks.Name)))) return -ENOTSUP; xCS.Attr.Cks.Set(csIP->Name); // Verify the file // if ((rc = ModTime(Pfn, MTime))) return rc; // Retreive the attribute. Return upon fatal error. // if ((rc = xCS.Get(Pfn)) < 0) return rc; // Verify the checksum and see if we need to recalculate it // if (!rc || xCS.Attr.Cks.fmTime != MTime || strcmp(xCS.Attr.Cks.Name, csIP->Name) || xCS.Attr.Cks.Length != csIP->Len) {strcpy(xCS.Attr.Cks.Name, Cks.Name); if ((rc = Calc(Pfn, xCS.Attr.Cks, 1)) < 0) return rc; } // Compare the checksums // return (xCS.Attr.Cks.Length == Cks.Length && !memcmp(xCS.Attr.Cks.Value, Cks.Value, csIP->Len)); } xrootd-5.6.9/src/XrdCks/XrdCksManager.hh000066400000000000000000000116611457266313600200730ustar00rootroot00000000000000#ifndef __XRDCKSMANAGER_HH__ #define __XRDCKSMANAGER_HH__ /******************************************************************************/ /* */ /* X r d C k s M a n a g e r . h h */ /* */ /* (c) 2011 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include "sys/types.h" #include "XrdCks/XrdCks.hh" #include "XrdCks/XrdCksData.hh" /* This class defines the checksum management interface. It may also be used as the base class for a plugin. This allows you to replace selected methods which may be needed for handling certain filesystems (see protected ones). */ class XrdCksCalc; class XrdCksLoader; class XrdSysError; struct XrdVersionInfo; class XrdCksManager : public XrdCks { public: virtual int Calc( const char *Pfn, XrdCksData &Cks, int doSet=1); virtual int Config(const char *Token, char *Line); virtual int Del( const char *Pfn, XrdCksData &Cks); virtual int Get( const char *Pfn, XrdCksData &Cks); virtual int Init(const char *ConfigFN, const char *AddCalc=0); virtual char *List(const char *Pfn, char *Buff, int Blen, char Sep=' '); virtual const char *Name(int seqNum=0); virtual XrdCksCalc *Object(const char *name); virtual int Size( const char *Name=0); virtual int Set( const char *Pfn, XrdCksData &Cks, int myTime=0); virtual int Ver( const char *Pfn, XrdCksData &Cks); XrdCksManager(XrdSysError *erP, int iosz, XrdVersionInfo &vInfo, bool autoload=false); virtual ~XrdCksManager(); protected: /* Calc() returns 0 if the checksum was successfully calculated using the supplied CksObj and places the file's modification time in MTime. Otherwise, it returns -errno. The default implementation uses open(), fstat(), mmap(), and unmap() to calculate the results. */ virtual int Calc(const char *Pfn, time_t &MTime, XrdCksCalc *CksObj); /* ModTime() returns 0 and places file's modification time in MTime. Otherwise, it return -errno. The default implementation uses stat(). */ virtual int ModTime(const char *Pfn, time_t &MTime); private: struct csInfo {char Name[XrdCksData::NameSize]; XrdCksCalc *Obj; char *Path; char *Parms; XrdSysPlugin *Plugin; int Len; bool doDel; csInfo() : Obj(0), Path(0), Parms(0), Plugin(0), Len(0), doDel(true) {memset(Name, 0, sizeof(Name));} }; int Config(const char *cFN, csInfo &Info); csInfo *Find(const char *Name); static const int csMax = 8; csInfo csTab[csMax]; int csLast; int segSize; XrdCksLoader *cksLoader; XrdVersionInfo &myVersion; }; #endif xrootd-5.6.9/src/XrdCks/XrdCksWrapper.hh000066400000000000000000000351501457266313600201400ustar00rootroot00000000000000#ifndef __XRDCKSWRAPPER_HH__ #define __XRDCKSWRAPPER_HH__ /******************************************************************************/ /* */ /* X r d C k s W r a p p e r . h h */ /* */ /* (c) 2021 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include "XrdCks/XrdCks.hh" class XrdCksCalc; class XrdCksData; class XrdOucEnv; class XrdOucStream; class XrdSysError; /*! This class defines the wrapper for the checksum management interface. It should be used as the base class for a stacked plugin. When used that way, the shared library holding the plugin must define a "C" entry point named XrdCksAdd2() as described at the end of this include file. Note you pass a reference to the previous plugin-in in the plug-in chain as a constructor argument as supplied to the XrdCksAdd2() function. Override the methods you wish to wrap. Methods that are not overridden are forwarded to the previous plug-in. */ class XrdCksWrapper : public XrdCks { public: //------------------------------------------------------------------------------ //! Calculate a new checksum for a physical file using the checksum algorithm //! named in the Cks parameter. //! //! @param Xfn The logical or physical name of the file to be checksumed. //! @param Cks For input, it specifies the checksum algorithm to be used. //! For output, the checksum value is returned upon success. //! @param doSet When true, the new value must replace any existing value //! in the Xfn's extended file attributes. //! @param pcbP In the second form, the pointer to the callback object. //! A nil pointer does not invoke any callback. //! //! @return Success: zero with Cks structure holding the checksum value. //! Failure: -errno (see significant error numbers below). //------------------------------------------------------------------------------ virtual int Calc( const char *Xfn, XrdCksData &Cks, int doSet=1) {return cksPI.Calc(Xfn, Cks, doSet);} virtual int Calc( const char *Xfn, XrdCksData &Cks, XrdCksPCB *pcbP, int doSet=1) {(void)pcbP; return Calc(Xfn, Cks, doSet);} //------------------------------------------------------------------------------ //! Delete the checksum from the Xfn's xattrs. //! //! @param Xfn The logical or physical name of the file to be checksumed. //! @param Cks Specifies the checksum type to delete. //! //! @return Success: 0 //! Failure: -errno (see significant error numbers below). //------------------------------------------------------------------------------ virtual int Del( const char *Xfn, XrdCksData &Cks) {return cksPI.Del(Xfn, Cks);} //------------------------------------------------------------------------------ //! Retreive the checksum from the Xfn's xattrs and return it and indicate //! whether or not it is stale (i.e. the file modification has changed or the //! name and length are not the expected values). //! //! @param Xfn The logical or physical name of the file to be checksumed. //! @param Cks For input, it specifies the checksum type to return. //! For output, the checksum value is returned upon success. //! //! @return Success: The length of the binary checksum in the Cks structure. //! Failure: -errno (see significant error numbers below). //------------------------------------------------------------------------------ virtual int Get( const char *Xfn, XrdCksData &Cks) {return cksPI.Get(Xfn, Cks);} //------------------------------------------------------------------------------ //! Parse a configuration directives specific to the checksum manager. //! //! @param Token Points to the directive that triggered the call. //! @param Line All the characters after the directive. //! //! @return Success: 1 //! Failure: 0 //------------------------------------------------------------------------------ virtual int Config(const char *Token, char *Line) {return cksPI.Config(Token, Line);} //------------------------------------------------------------------------------ //! Fully initialize the manager which includes loading any plugins. //! //! @param ConfigFN Points to the configuration file path. //! @param DfltCalc Is the default checksum and should be defaulted if NULL. //! The default implementation defaults this to adler32. A //! default is only needed should the checksum name in the //! XrdCksData object be omitted. //! //!@return Success: 1 //! Failure: 0 //------------------------------------------------------------------------------ virtual int Init(const char *ConfigFN, const char *DfltCalc=0) {return cksPI.Init(ConfigFN, DfltCalc);} //------------------------------------------------------------------------------ //! List names of the checksums associated with a Xfn or all supported ones. //! //! @param Xfn The logical or physical file name whose checksum names are //! to be returned. When Xfn is null, return all supported //! checksum algorithm names. //! @param Buff Points to a buffer, at least 64 bytes in length, to hold //! a "Sep" separated list of checksum names. //! @param Blen The length of the buffer. //! @param Sep The separation character to be used between adjacent names. //! //! @return Success: Pointer to Buff holding at least one checksum name. //! Failure: A nil pointer is returned. //------------------------------------------------------------------------------ virtual char *List(const char *Xfn, char *Buff, int Blen, char Sep=' ') {return cksPI.List(Xfn, Buff, Blen, Sep);} //------------------------------------------------------------------------------ //! Get the name of the checksums associated with a sequence number. Note that //! Name() may be called prior to final config to see if there are any chksums //! to configure and avoid unintended errors. //! //! @param seqNum The sequence number. Zero signifies the default name. //! Higher numbers are alternates. //! @return Success: Pointer to the name. //! Failure: A nil pointer is returned (no more alternates exist). //------------------------------------------------------------------------------ virtual const char *Name(int seqNum=0) {return cksPI.Name(seqNum);} //------------------------------------------------------------------------------ //! Get a new XrdCksCalc object that can calculate the checksum corresponding to //! the specified name or the default object if name is a null pointer. The //! object can be used to compute checksums on the fly. The object's Recycle() //! method must be used to delete it. //! //! @param name The name of the checksum algorithm. If null, use the //! default one. //! //! @return Success: A pointer to the object is returned. //! Failure: Zero if no corresponding object exists. //------------------------------------------------------------------------------ virtual XrdCksCalc *Object(const char *name) {return cksPI.Object(name);} //------------------------------------------------------------------------------ //! Get the binary length of the checksum with the corresponding name. //! //! @param Name The checksum algorithm name. If null, use the default name. //! //! @return Success: checksum length. //! Failure: Zero if the checksum name does not exist. //------------------------------------------------------------------------------ virtual int Size( const char *Name=0) {return cksPI.Size(Name);} //------------------------------------------------------------------------------ //! Set a file's checksum in the extended attributes along with the file's mtime //! and the time of setting. //! //! @param Xfn The logical or physical name of the file to be set. //! @param Cks Specifies the checksum name and value. //! @param myTime When true then the fmTime and gmTime in the Cks structure //! are to be used; as opposed to the current time. //! //! @return Success: zero is returned. //! Failure: -errno (see significant error numbers below). //------------------------------------------------------------------------------ virtual int Set( const char *Xfn, XrdCksData &Cks, int myTime=0) {return cksPI.Set(Xfn, Cks, myTime);} //------------------------------------------------------------------------------ //! Retreive the checksum from the Xfn's xattrs and compare it to the supplied //! checksum. If the checksum is not available or is stale, a new checksum is //! calculated and written to the extended attributes. //! //! @param Xfn The logical or physical name of the file to be verified. //! @param Cks Specifies the checksum name and value. //! @param pcbP In the second form, the pointer to the callback object. //! A nil pointer does not invoke any callback. //! //! @return Success: True //! Failure: False (the checksums do not match) or -errno indicating //! that verification could not be performed (see significant //! error numbers below). //------------------------------------------------------------------------------ virtual int Ver( const char *Xfn, XrdCksData &Cks) {return cksPI.Ver(Xfn, Cks);} virtual int Ver( const char *Xfn, XrdCksData &Cks, XrdCksPCB *pcbP) {(void)pcbP; return Ver(Xfn, Cks);} //------------------------------------------------------------------------------ //! Constructor //! //! @param prevPI Reference to the antecedent plugin. //! @param errP Pointer to error message object //------------------------------------------------------------------------------ XrdCksWrapper(XrdCks &prevPI, XrdSysError *errP) : XrdCks(errP), cksPI(prevPI) {} //------------------------------------------------------------------------------ //! Destructor //------------------------------------------------------------------------------ virtual ~XrdCksWrapper() {} protected: XrdCks &cksPI; }; /******************************************************************************/ /* X r d C k s A d d 2 */ /******************************************************************************/ #define XRDCKSADD2PARMS XrdCks &, XrdSysError *, const char *, \ const char *, XrdOucEnv * //------------------------------------------------------------------------------ //! Obtain an instance of a stacked checksum manager. //! //! XrdCksAdd2() is an extern "C" function that is called to obtain an //! instance of a stacked checksum manager object that will be used for all //! subsequent checksums. This function is passed a refernce to the previous //! checksum manager. Overridden methods should, if appropriate, invoke the //! previous plug-ins corresponding method to complete the required task. //! All the following extern symbols must be defined at file level! //! //! @param pPI -> Reference to the previous checksum manager plug-in. //! @param eDest-> The XrdSysError object for messages. //! @param cFN -> The name of the configuration file //! @param Parm -> Parameters specified on the ckslib directive. If none it is //! zero. //! @param pEnv -> Pointer to environmental information or nil. //! //! @return Success: A pointer to the checksum manager object. //! Failure: Null pointer which causes initialization to fail. //------------------------------------------------------------------------------ /*! extern "C" XrdCks *XrdCksAdd2(XrdCks &pPI, XrdSysError *eDest, const char *cFN, const char *Parm, XrdOucEnv *envP ); */ //------------------------------------------------------------------------------ //! Declare the compilation version number. //! //! Additionally, you *should* declare the xrootd version you used to compile //! your plug-in. While not currently required, it is highly recommended to //! avoid execution issues should the class definition change. Declare it as: //------------------------------------------------------------------------------ /*! #include "XrdVersion.hh" XrdVERSIONINFO(XrdCksAdd2,); where is a 1- to 15-character unquoted name identifying your plugin. */ #endif xrootd-5.6.9/src/XrdCks/XrdCksXAttr.hh000066400000000000000000000102731457266313600175610ustar00rootroot00000000000000#ifndef __XRDCKSXATTR_HH__ #define __XRDCKSXATTR_HH__ /******************************************************************************/ /* */ /* X r d C k s X A t t r . h h */ /* */ /* (c) 2011 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include #include #include "XrdCks/XrdCksData.hh" #include "XrdSys/XrdSysPlatform.hh" /* XrdCksXAttr encapsulates the extended attributes needed to save a checksum. */ class XrdCksXAttr { public: XrdCksData Cks; // Check sum information /* postGet() will put fmTime and csTime in host byte order (see preSet()). */ int postGet(int Result) {if (Result > 0) {Cks.fmTime = ntohll(Cks.fmTime); Cks.csTime = ntohl (Cks.csTime); } return Result; } /* preSet() will put fmTime and csTime in network byte order to allow the attribute to be copied to different architectures and still work. */ XrdCksXAttr *preSet(XrdCksXAttr &tmp) {memcpy(&tmp.Cks, &Cks, sizeof(Cks)); tmp.Cks.fmTime = htonll(Cks.fmTime); tmp.Cks.csTime = htonl (Cks.csTime); return &tmp; } /* Name() returns the extended attribute name for this object. */ const char *Name() {if (!(*VarName)) //01234567 {strcpy(VarName, "XrdCks."); strcpy(VarName+7, Cks.Name); } return VarName; } /* sizeGet() and sizeSet() return the actual size of the object is used. */ int sizeGet() {return sizeof(Cks);} int sizeSet() {return sizeof(Cks);} XrdCksXAttr() {*VarName = 0;} ~XrdCksXAttr() {} private: char VarName[XrdCksData::NameSize+8]; }; #endif xrootd-5.6.9/src/XrdCl/000077500000000000000000000000001457266313600146725ustar00rootroot00000000000000xrootd-5.6.9/src/XrdCl/CMakeLists.txt000066400000000000000000000167731457266313600174500ustar00rootroot00000000000000 #------------------------------------------------------------------------------- # Shared library version #------------------------------------------------------------------------------- set( XRD_CL_VERSION 3.0.0 ) set( XRD_CL_SOVERSION 3 ) #------------------------------------------------------------------------------- # The XrdCl Pipelining API # (we build this API only if the compiler has a good support for c++11) #------------------------------------------------------------------------------- if( XrdClPipelines ) set( XrdClPipelineSources XrdClOperations.cc XrdClOperations.hh XrdClOperationHandlers.hh XrdClArg.hh XrdClFwd.hh XrdClParallelOperation.hh XrdClFileOperations.hh XrdClFileSystemOperations.hh XrdClZipOperations.hh ) endif() #------------------------------------------------------------------------------- # XrdEc sources #------------------------------------------------------------------------------- if( BUILD_XRDEC ) set( XrdEcSources ${CMAKE_SOURCE_DIR}/src/XrdEc/XrdEcRedundancyProvider.cc ${CMAKE_SOURCE_DIR}/src/XrdEc/XrdEcUtilities.cc ${CMAKE_SOURCE_DIR}/src/XrdEc/XrdEcStrmWriter.cc ${CMAKE_SOURCE_DIR}/src/XrdEc/XrdEcReader.cc XrdClEcHandler.cc ) add_compile_definitions( WITH_XRDEC ) endif() #------------------------------------------------------------------------------- # The XrdCl lib #------------------------------------------------------------------------------- add_library( XrdCl SHARED XrdClLog.cc XrdClLog.hh XrdClUtils.cc XrdClUtils.hh XrdClOptimizers.hh XrdClConstants.hh XrdClEnv.cc XrdClEnv.hh XrdClDefaultEnv.cc XrdClDefaultEnv.hh XrdClURL.cc XrdClURL.hh XrdClStatus.cc XrdClStatus.hh XrdClSocket.cc XrdClSocket.hh XrdClTls.cc XrdClTls.hh XrdClPoller.hh XrdClPollerFactory.cc XrdClPollerFactory.hh XrdClPollerBuiltIn.cc XrdClPollerBuiltIn.hh XrdClPostMaster.cc XrdClPostMaster.hh XrdClPostMasterInterfaces.hh XrdClChannel.cc XrdClChannel.hh XrdClStream.cc XrdClStream.hh XrdClXRootDTransport.cc XrdClXRootDTransport.hh XrdClInQueue.cc XrdClInQueue.hh XrdClOutQueue.cc XrdClOutQueue.hh XrdClTaskManager.cc XrdClTaskManager.hh XrdClSIDManager.cc XrdClSIDManager.hh XrdClFileSystem.cc XrdClFileSystem.hh XrdClXRootDMsgHandler.cc XrdClXRootDMsgHandler.hh XrdClBuffer.hh XrdClMessage.hh XrdClMessageUtils.cc XrdClMessageUtils.hh XrdClXRootDResponses.cc XrdClXRootDResponses.hh XrdClRequestSync.hh XrdClFile.cc XrdClFile.hh XrdClFileStateHandler.cc XrdClFileStateHandler.hh XrdClCopyProcess.cc XrdClCopyProcess.hh XrdClClassicCopyJob.cc XrdClClassicCopyJob.hh XrdClThirdPartyCopyJob.cc XrdClThirdPartyCopyJob.hh XrdClAsyncSocketHandler.cc XrdClAsyncSocketHandler.hh XrdClChannelHandlerList.cc XrdClChannelHandlerList.hh XrdClForkHandler.cc XrdClForkHandler.hh XrdClCheckSumManager.cc XrdClCheckSumManager.hh XrdClTransportManager.cc XrdClTransportManager.hh XrdClSyncQueue.hh XrdClJobManager.cc XrdClJobManager.hh XrdClResponseJob.hh XrdClFileTimer.cc XrdClFileTimer.hh XrdClPlugInInterface.hh XrdClPlugInManager.cc XrdClPlugInManager.hh XrdClPropertyList.hh XrdClCopyJob.hh XrdClFileSystemUtils.cc XrdClFileSystemUtils.hh XrdClTPFallBackCopyJob.cc XrdClTPFallBackCopyJob.hh XrdClMetalinkRedirector.cc XrdClMetalinkRedirector.hh XrdClRedirectorRegistry.cc XrdClRedirectorRegistry.hh XrdClXCpCtx.cc XrdClXCpCtx.hh XrdClXCpSrc.cc XrdClXCpSrc.hh XrdClLocalFileHandler.cc XrdClLocalFileHandler.hh XrdClLocalFileTask.cc XrdClLocalFileTask.hh XrdClZipListHandler.cc XrdClZipListHandler.hh XrdClZipArchive.cc XrdClZipArchive.hh ${XrdClPipelineSources} ${XrdEcSources} ) target_link_libraries( XrdCl PRIVATE XrdXml XrdUtils uuid::uuid ZLIB::ZLIB OpenSSL::SSL ${CMAKE_THREAD_LIBS_INIT} ${EXTRA_LIBS} ${CMAKE_DL_LIBS}) set_target_properties( XrdCl PROPERTIES VERSION ${XRD_CL_VERSION} SOVERSION ${XRD_CL_SOVERSION} ) if( BUILD_XRDEC ) target_include_directories(XrdCl PUBLIC ${ISAL_INCLUDE_DIRS}) target_link_libraries(XrdCl PRIVATE ${ISAL_LIBRARIES}) endif() #------------------------------------------------------------------------------- # xrdfs #------------------------------------------------------------------------------- if( NOT XRDCL_LIB_ONLY ) add_executable( xrdfs XrdClFS.cc XrdClFSExecutor.cc XrdClFSExecutor.hh ) target_link_libraries( xrdfs ${CMAKE_THREAD_LIBS_INIT} XrdCl XrdUtils ${READLINE_LIBRARY} ${NCURSES_LIBRARY} ) if( READLINE_FOUND ) target_include_directories(xrdfs PRIVATE ${READLINE_INCLUDE_DIR}) endif() endif() #------------------------------------------------------------------------------- # xrdcopy #------------------------------------------------------------------------------- if( NOT XRDCL_LIB_ONLY ) add_executable( xrdcp XrdClCopy.cc ) target_link_libraries( xrdcp XrdCl XrdUtils XrdAppUtils ) endif() #------------------------------------------------------------------------------- # Install #------------------------------------------------------------------------------- install( TARGETS XrdCl RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} ) if( NOT XRDCL_LIB_ONLY ) install( TARGETS xrdfs xrdcp RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} ) endif() install( FILES XrdClAnyObject.hh XrdClBuffer.hh XrdClConstants.hh XrdClCopyProcess.hh XrdClDefaultEnv.hh XrdClEnv.hh XrdClFile.hh XrdClFileSystem.hh XrdClFileSystemUtils.hh XrdClMonitor.hh XrdClStatus.hh XrdClURL.hh XrdClXRootDResponses.hh XrdClOptional.hh XrdClPlugInInterface.hh XrdClPropertyList.hh XrdClLog.hh DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/xrootd/XrdCl ) install( FILES # Additional client headers XrdClJobManager.hh XrdClMessage.hh XrdClPlugInManager.hh XrdClPostMaster.hh XrdClPostMasterInterfaces.hh XrdClTransportManager.hh XrdClResponseJob.hh XrdClSyncQueue.hh XrdClZipArchive.hh XrdClZipCache.hh # Declarative operations XrdClOperations.hh XrdClOperationHandlers.hh XrdClOperationTimeout.hh XrdClArg.hh XrdClCtx.hh XrdClFwd.hh XrdClParallelOperation.hh XrdClFileOperations.hh XrdClFileSystemOperations.hh XrdClFinalOperation.hh XrdClUtils.hh XrdClXRootDTransport.hh XrdClZipOperations.hh DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/xrootd/private/XrdCl ) if( NOT XRDCL_LIB_ONLY ) install( CODE " EXECUTE_PROCESS( COMMAND ln -sf xrdcp xrdcopy WORKING_DIRECTORY \$ENV{DESTDIR}/${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_BINDIR} )" ) endif() xrootd-5.6.9/src/XrdCl/XrdClAnyObject.hh000066400000000000000000000127511457266313600200340ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2012 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . //------------------------------------------------------------------------------ #ifndef __XRD_CL_ANY_OBJECT_HH__ #define __XRD_CL_ANY_OBJECT_HH__ #include #include namespace XrdCl { //---------------------------------------------------------------------------- //! Simple implementation of a type safe holder for any object pointer //! It would have been a better idea to use boost::any here but we don't //! want to depend on boost //---------------------------------------------------------------------------- class AnyObject { public: //------------------------------------------------------------------------ //! Constructor //------------------------------------------------------------------------ AnyObject(): pHolder(0), pTypeInfo(0), pOwn( true ) {}; //------------------------------------------------------------------------ //! Destructor //------------------------------------------------------------------------ ~AnyObject() { if( pHolder && pOwn ) pHolder->Delete(); delete pHolder; } //------------------------------------------------------------------------ //! Grab an object //! By default the ownership of the object is taken as well, ie. //! the object will be deleted when the AnyObject holding it is deleted. //! To release an object grab a zero pointer, ie. (int *)0 //! //! @param object object pointer //! @param own take the ownership or not //------------------------------------------------------------------------ template void Set( Type object, bool own = true ) { if( !object ) { delete pHolder; pHolder = 0; pTypeInfo = 0; return; } delete pHolder; pHolder = new ConcreteHolder( object ); pOwn = own; pTypeInfo = &typeid( Type ); } //------------------------------------------------------------------------ //! Retrieve the object being held //------------------------------------------------------------------------ template void Get( Type &object ) { if( !pHolder || (strcmp( pTypeInfo->name(), typeid( Type ).name() )) ) { object = 0; return; } object = static_cast( pHolder->Get() ); } //------------------------------------------------------------------------ //! @return true is AnyObject holds an instance of given type //------------------------------------------------------------------------ template inline bool Has() { if( !pHolder ) return false; return strcmp( pTypeInfo->name(), typeid( Type* ).name() ) == 0; } //------------------------------------------------------------------------ //! Check if we own the object being stored //------------------------------------------------------------------------ bool HasOwnership() const { return pOwn; } private: //------------------------------------------------------------------------ // Abstract holder object //------------------------------------------------------------------------ class Holder { public: virtual ~Holder() {} virtual void Delete() = 0; virtual void *Get() = 0; }; //------------------------------------------------------------------------ // Concrete holder //------------------------------------------------------------------------ template class ConcreteHolder: public Holder { public: ConcreteHolder( Type object ): pObject( object ) {} virtual void Delete() { delete pObject; } virtual void *Get() { return (void *)pObject; } private: Type pObject; }; Holder *pHolder; const std::type_info *pTypeInfo; bool pOwn; }; //---------------------------------------------------------------------------- //! Helper function for extracting an object from AnyObject //! @param any : an instance of AnyObject //! @return : the underlying value of type T //---------------------------------------------------------------------------- template inline T& To( AnyObject &any ) { T* object; any.Get( object ); return *object; } } #endif // __XRD_CL_ANY_OBJECT_HH__ xrootd-5.6.9/src/XrdCl/XrdClApply.hh000066400000000000000000000064441457266313600172450ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2012 by European Organization for Nuclear Research (CERN) // Author: Michal Simon //------------------------------------------------------------------------------ // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . //------------------------------------------------------------------------------ #ifndef SRC_XRDCL_XRDCLAPPLY_HH_ #define SRC_XRDCL_XRDCLAPPLY_HH_ #include #include namespace XrdCl { // This is the type which holds sequences template struct sequence {}; // First define the template signature template struct seq_gen; // Recursion case template struct seq_gen { using type = typename seq_gen::type; }; // Recursion abort template struct seq_gen<0, Ns...> { using type = sequence; }; template inline static auto tuple_call_impl( FUNC &func, TUPL &args, sequence ) -> decltype( func( std::move( std::get( args ) )... ) ) { return func( std::move( std::get( args ) )... ); } //--------------------------------------------------------------------------- // Applies tuple members as arguments to the function // // @param func : function to be called // @param tup : tuple with the packed argumnets // // Note: Once we can use c++17 we can move to std::apply //--------------------------------------------------------------------------- template inline static auto Apply( FUNC &&func, std::tuple &tup ) -> decltype( tuple_call_impl( func, tup, typename seq_gen::type{} ) ) { return tuple_call_impl( func, tup, typename seq_gen::type{} ); } //--------------------------------------------------------------------------- // Applies tuple members as arguments to the function // // @param method : method to be called // @param obj : the object to call the method on // @param tup : tuple with the packed argumnets // // Note: Once we can use c++17 we can move to std::apply //--------------------------------------------------------------------------- template inline static auto Apply( METH &&method, OBJ &obj, std::tuple &tup ) -> decltype( Apply( std::bind( method, &obj, std::placeholders::_1, std::placeholders::_2 ), tup ) ) { return Apply( std::bind( method, &obj, std::placeholders::_1, std::placeholders::_2 ), tup ); } } #endif /* SRC_XRDCL_XRDCLAPPLY_HH_ */ xrootd-5.6.9/src/XrdCl/XrdClArg.hh000066400000000000000000000315671457266313600166750ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2017 by European Organization for Nuclear Research (CERN) // Author: Krzysztof Jamrog , // Michal Simon //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #ifndef __XRD_CL_OPERATION_PARAMS_HH__ #define __XRD_CL_OPERATION_PARAMS_HH__ #include "XrdCl/XrdClFwd.hh" #include "XrdCl/XrdClOptional.hh" #include #include #include #include namespace XrdCl { //---------------------------------------------------------------------------- //! Base class for Arg. //---------------------------------------------------------------------------- template class ArgBase { public: //------------------------------------------------------------------------ //! Default Constructor. //------------------------------------------------------------------------ ArgBase() { } //------------------------------------------------------------------------ //! Destructor. //------------------------------------------------------------------------ virtual ~ArgBase() { } //------------------------------------------------------------------------ //! Constructor //! //! @param value : the value of the argument //------------------------------------------------------------------------ ArgBase( T value ) : holder( new PlainValue( std::move( value ) ) ) { } //------------------------------------------------------------------------ //! Constructor. //! //! @param ftr : future value of the argument //------------------------------------------------------------------------ ArgBase( std::future &&ftr ) : holder( new FutureValue( std::move( ftr ) ) ) { } //------------------------------------------------------------------------ //! Constructor. //! //! @param fwd : forwarded value of the argument //------------------------------------------------------------------------ ArgBase( const Fwd &fwd ) : holder( new FwdValue( fwd ) ) { } //------------------------------------------------------------------------ //! Get Constructor. //------------------------------------------------------------------------ ArgBase( ArgBase &&arg ) : holder( std::move( arg.holder ) ) { } //------------------------------------------------------------------------ //! @return : value of the argument //------------------------------------------------------------------------ inline T& Get() const { if( !holder ) throw std::logic_error( "XrdCl::ArgBase::Get(): value not set." ); return holder->Get(); } operator T() const { return Get(); } protected: //------------------------------------------------------------------------ //! Abstract class for holding a value //------------------------------------------------------------------------ struct ValueHolder { //---------------------------------------------------------------------- //! Virtual Destructor (important ;-). //---------------------------------------------------------------------- virtual ~ValueHolder() { } //---------------------------------------------------------------------- //! @return : the value //---------------------------------------------------------------------- virtual T& Get() = 0; }; //------------------------------------------------------------------------ //! A helper class for holding plain value //------------------------------------------------------------------------ struct PlainValue : public ValueHolder { //-------------------------------------------------------------------- //! Constructor //! //! @param value : the value to be hold by us //-------------------------------------------------------------------- PlainValue( T &&value ) : value( std::move( value ) ) { } //-------------------------------------------------------------------- //! @return : the value //-------------------------------------------------------------------- T& Get() { return value; } private: //-------------------------------------------------------------------- //! the value //-------------------------------------------------------------------- T value; }; //------------------------------------------------------------------------ //! A helper class for holding future value //------------------------------------------------------------------------ struct FutureValue : public ValueHolder { //-------------------------------------------------------------------- //! Constructor //! //! @param ftr : the future value to be hold by us //-------------------------------------------------------------------- FutureValue( std::future &&ftr ) : ftr( std::move( ftr ) ) { } //-------------------------------------------------------------------- //! @return : the value //-------------------------------------------------------------------- T& Get() { if( val ) return *val; val = ftr.get(); return *val; } private: //-------------------------------------------------------------------- //! the future value //-------------------------------------------------------------------- std::future ftr; Optional val; }; //------------------------------------------------------------------------ //! A helper class for holding forwarded value //------------------------------------------------------------------------ struct FwdValue : public ValueHolder { //-------------------------------------------------------------------- //! Constructor //! //! @param fwd : the forwarded value to be hold by us //-------------------------------------------------------------------- FwdValue( const Fwd &fwd ) : fwd( fwd ) { } //-------------------------------------------------------------------- //! @return : the value //-------------------------------------------------------------------- T& Get() { return *fwd; } private: //-------------------------------------------------------------------- //! the forwarded value //-------------------------------------------------------------------- Fwd fwd; }; //------------------------------------------------------------------------ //! Holds the value of the argument //------------------------------------------------------------------------ std::unique_ptr holder; }; //---------------------------------------------------------------------------- //! Operation argument. //! The argument is optional, user may initialize it with 'notdef' //! //! @arg T : real type of the argument //---------------------------------------------------------------------------- template class Arg : public ArgBase { public: //------------------------------------------------------------------------ //! Default Constructor. //------------------------------------------------------------------------ Arg() { } //------------------------------------------------------------------------ //! Constructor. //! //! @param value : value of the argument (will be std::moved) //------------------------------------------------------------------------ Arg( T value ) : ArgBase( std::move( value ) ) { } //------------------------------------------------------------------------ //! Constructor. //! //! @param ftr : future value of the argument (will be std::moved) //------------------------------------------------------------------------ Arg( std::future &&ftr ) : ArgBase( std::move( ftr ) ) { } //------------------------------------------------------------------------ //! Constructor. //! //! @param fwd : forwarded value of the argument (will be std::moved) //------------------------------------------------------------------------ Arg( const Fwd &fwd ) : ArgBase( fwd ) { } //------------------------------------------------------------------------ //! Get Constructor. //------------------------------------------------------------------------ Arg( Arg &&arg ) : ArgBase( std::move( arg ) ) { } //------------------------------------------------------------------------ //! Get-Assignment. //------------------------------------------------------------------------ Arg& operator=( Arg &&arg ) { if( &arg == this ) return *this; this->holder = std::move( arg.holder ); return *this; } }; //---------------------------------------------------------------------------- //! Operation argument. //! Specialized for 'std::string', might be constructed in addition from c-like //! string (const char*) //---------------------------------------------------------------------------- template<> class Arg : public ArgBase { public: //------------------------------------------------------------------------ //! Default Constructor. //------------------------------------------------------------------------ Arg() { } //------------------------------------------------------------------------ //! Constructor. //! //! @param str : value of the argument //------------------------------------------------------------------------ Arg( std::string str ) : ArgBase( str ) { } //------------------------------------------------------------------------ //! Constructor. //! //! @param cstr : value of the argument //------------------------------------------------------------------------ Arg( const char *cstr ) : ArgBase( cstr ) { } //------------------------------------------------------------------------ //! Constructor. //------------------------------------------------------------------------ Arg( std::future &&ftr ) : ArgBase( std::move( ftr ) ) { } //------------------------------------------------------------------------ //! Constructor. //------------------------------------------------------------------------ Arg( const Fwd &fwd ) : ArgBase( fwd ) { } //------------------------------------------------------------------------ //! Get Constructor. //----------------------------------------------------------------------- Arg( Arg &&arg ) : ArgBase( std::move( arg ) ) { } //------------------------------------------------------------------------ //! Get-Assignment. //------------------------------------------------------------------------ Arg& operator=( Arg &&arg ) { if( &arg == this ) return *this; this->holder = std::move( arg.holder ); return *this; } }; } #endif // __XRD_CL_OPERATION_PARAMS_HH__ xrootd-5.6.9/src/XrdCl/XrdClAsyncDiscardReader.hh000066400000000000000000000065571457266313600216570ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2012 by European Organization for Nuclear Research (CERN) // Author: Michal Simon //------------------------------------------------------------------------------ // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . //------------------------------------------------------------------------------ #ifndef SRC_XRDCL_XRDCLASYNCDISCARDREADER_HH_ #define SRC_XRDCL_XRDCLASYNCDISCARDREADER_HH_ #include "XrdCl/XrdClXRootDResponses.hh" #include "XrdCl/XrdClSocket.hh" #include "XrdCl/XrdClDefaultEnv.hh" #include "XrdCl/XrdClLog.hh" #include "XrdCl/XrdClConstants.hh" #include "XrdClAsyncRawReaderIntfc.hh" namespace XrdCl { //---------------------------------------------------------------------------- //! Object for discarding data //---------------------------------------------------------------------------- class AsyncDiscardReader : public AsyncRawReaderIntfc { public: //------------------------------------------------------------------------ //! Constructor //! //! @param url : channel URL //! @param request : client request //------------------------------------------------------------------------ AsyncDiscardReader( const URL &url, const Message &request ) : AsyncRawReaderIntfc( url, request ) { } //------------------------------------------------------------------------ //! Readout raw data from socket //! //! @param socket : the socket //! @param btsret : number of bytes read //! @return : operation status //------------------------------------------------------------------------ XRootDStatus Read( Socket &socket, uint32_t &btsret ) { Log *log = DefaultEnv::GetLog(); log->Error( XRootDMsg, "[%s] Handling response to %s: " "DiscardReader: we were not expecting " "raw data.", url.GetHostId().c_str(), request.GetDescription().c_str() ); // Just drop the connection, we don't know if the stream is sane anymore. // Recover with a reconnect. return XRootDStatus( stError, errCorruptedHeader ); } //------------------------------------------------------------------------ //! Get the response, since we received some unexpected data we always //! return an error to the end user. //------------------------------------------------------------------------ XRootDStatus GetResponse( AnyObject *&response ) { response = nullptr; return XRootDStatus( stError, errInvalidResponse ); } }; } /* namespace XrdCl */ #endif /* SRC_XRDCL_XRDCLASYNCVECTORREADER_HH_ */ xrootd-5.6.9/src/XrdCl/XrdClAsyncHSReader.hh000066400000000000000000000167551457266313600206210ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2012 by European Organization for Nuclear Research (CERN) // Author: Michal Simon //------------------------------------------------------------------------------ // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . //------------------------------------------------------------------------------ #ifndef SRC_XRDCL_XRDCLASYNCHSREADER_HH_ #define SRC_XRDCL_XRDCLASYNCHSREADER_HH_ #include "XrdCl/XrdClMessage.hh" #include "XrdCl/XrdClPostMasterInterfaces.hh" #include "XrdCl/XrdClXRootDResponses.hh" #include "XrdCl/XrdClSocket.hh" #include "XrdCl/XrdClConstants.hh" #include "XrdCl/XrdClStream.hh" #include namespace XrdCl { //---------------------------------------------------------------------------- //! Utility class encapsulating reading hand-shake response logic //---------------------------------------------------------------------------- class AsyncHSReader { public: //------------------------------------------------------------------------ //! Constructor //! //! @param xrdTransport : the (xrootd) transport layer //! @param socket : the socket with the message to be read out //! @param strmname : stream name //! @param strm : the stream encapsulating the connection //! @param substrmnb : the substream number //------------------------------------------------------------------------ AsyncHSReader( TransportHandler &xrdTransport, Socket &socket, const std::string &strmname, Stream &strm, uint16_t substrmnb) : readstage( ReadStart ), xrdTransport( xrdTransport ), socket( socket ), strmname( strmname ), strm( strm ), substrmnb( substrmnb ) { } //------------------------------------------------------------------------ //! Read out the response from the socket //------------------------------------------------------------------------ XRootDStatus Read() { Log *log = DefaultEnv::GetLog(); while( true ) { switch( readstage ) { //------------------------------------------------------------------ // There is no incoming message currently being processed so we // create a new one //------------------------------------------------------------------ case ReadStart: { inmsg.reset( new Message() ); //---------------------------------------------------------------- // The next step is to read the header //---------------------------------------------------------------- readstage = ReadHeader; continue; } //------------------------------------------------------------------ // We need to read the header //------------------------------------------------------------------ case ReadHeader: { XRootDStatus st = xrdTransport.GetHeader( *inmsg, &socket ); if( !st.IsOK() || st.code == suRetry ) return st; log->Dump( AsyncSockMsg, "[%s] Received message header, size: %d", strmname.c_str(), inmsg->GetCursor() ); //---------------------------------------------------------------- // The next step is to read the message body //---------------------------------------------------------------- readstage = ReadMsgBody; continue; } //------------------------------------------------------------------ // We read the message to the buffer //------------------------------------------------------------------ case ReadMsgBody: { XRootDStatus st = xrdTransport.GetBody( *inmsg, &socket ); if( !st.IsOK() || st.code == suRetry ) return st; log->Dump( AsyncSockMsg, "[%s] Received a message of %d bytes", strmname.c_str(), inmsg->GetSize() ); readstage = ReadDone; return st; } case ReadDone: return XRootDStatus(); } // just in case ... break; } //---------------------------------------------------------------------- // We are done //---------------------------------------------------------------------- return XRootDStatus(); } //------------------------------------------------------------------------ //! Transfer the received message ownership //------------------------------------------------------------------------ std::unique_ptr ReleaseMsg() { readstage = ReadStart; return std::move( inmsg ); } //------------------------------------------------------------------------ //! Reset the state of the object (makes it ready to read out next msg) //------------------------------------------------------------------------ inline void Reset() { readstage = ReadStart; inmsg.reset(); } private: //------------------------------------------------------------------------ //! Stages of reading out a response from the socket //------------------------------------------------------------------------ enum Stage { ReadStart, //< the next step is to initialize the read ReadHeader, //< the next step is to read the header ReadMsgBody, //< the next step is to read the body ReadDone //< we are done }; //------------------------------------------------------------------------ // Current read stage //------------------------------------------------------------------------ Stage readstage; //------------------------------------------------------------------------ // The context of the read operation //------------------------------------------------------------------------ TransportHandler &xrdTransport; Socket &socket; const std::string &strmname; Stream &strm; uint16_t substrmnb; //------------------------------------------------------------------------ // The internal state of the the reader //------------------------------------------------------------------------ std::unique_ptr inmsg; }; } #endif /* SRC_XRDCL_XRDCLASYNCHSREADER_HH_ */ xrootd-5.6.9/src/XrdCl/XrdClAsyncHSWriter.hh000066400000000000000000000135561457266313600206670ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2012 by European Organization for Nuclear Research (CERN) // Author: Michal Simon //------------------------------------------------------------------------------ // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . //------------------------------------------------------------------------------ #ifndef SRC_XRDCL_XRDCLASYNCHSWRITER_HH_ #define SRC_XRDCL_XRDCLASYNCHSWRITER_HH_ #include "XrdCl/XrdClMessage.hh" #include "XrdCl/XrdClPostMasterInterfaces.hh" #include "XrdCl/XrdClXRootDResponses.hh" #include "XrdCl/XrdClSocket.hh" #include "XrdCl/XrdClConstants.hh" #include "XrdCl/XrdClStream.hh" #include "XrdSys/XrdSysE2T.hh" #include namespace XrdCl { //---------------------------------------------------------------------------- //! Utility class encapsulating writing hand-shake request logic //---------------------------------------------------------------------------- class AsyncHSWriter { public: //------------------------------------------------------------------------ //! Constructor //! //! @param socket : the socket with the message to be read out //! @param strmname : stream name //------------------------------------------------------------------------ AsyncHSWriter( Socket &socket, const std::string &strmname ) : writestage( WriteRequest ), socket( socket ), strmname( strmname ), outmsg( nullptr ) { } //------------------------------------------------------------------------ //! Reset the state of the object (makes it ready to read out next msg) //------------------------------------------------------------------------ inline void Reset( Message *msg = nullptr ) { writestage = WriteRequest; outmsg.reset( msg ); } //------------------------------------------------------------------------ //! Replay the message that has been sent //------------------------------------------------------------------------ inline void Replay() { if( !outmsg ) return; writestage = WriteRequest; outmsg->SetCursor( 0 ); } //------------------------------------------------------------------------ //! Check if writer was assigned with a message //------------------------------------------------------------------------ inline bool HasMsg() { return bool( outmsg ); } //------------------------------------------------------------------------ //! Write the request into the socket //------------------------------------------------------------------------ XRootDStatus Write() { Log *log = DefaultEnv::GetLog(); while( true ) { switch( writestage ) { case WriteRequest: { XRootDStatus st = socket.Send( *outmsg, strmname ); if( !st.IsOK() || st.code == suRetry ) return st; //---------------------------------------------------------------- // The next step is to write the signature //---------------------------------------------------------------- writestage = WriteDone; continue; } case WriteDone: { XRootDStatus st = socket.Flash(); if( !st.IsOK() ) { log->Error( AsyncSockMsg, "[%s] Unable to flash the socket: %s", strmname.c_str(), XrdSysE2T( st.errNo ) ); } return st; } } // just in case ... break; } //---------------------------------------------------------------------- // We are done //---------------------------------------------------------------------- return XRootDStatus(); } private: //------------------------------------------------------------------------ //! Stages of reading out a response from the socket //------------------------------------------------------------------------ enum Stage { WriteRequest, //< the next step is to write the request WriteDone //< the next step is to finalize the write }; //------------------------------------------------------------------------ // Current read stage //------------------------------------------------------------------------ Stage writestage; //------------------------------------------------------------------------ // The context of the read operation //------------------------------------------------------------------------ Socket &socket; const std::string &strmname; //------------------------------------------------------------------------ // The internal state of the the reader //------------------------------------------------------------------------ std::unique_ptr outmsg; }; } #endif /* SRC_XRDCL_XRDCLASYNCHSWRITER_HH_ */ xrootd-5.6.9/src/XrdCl/XrdClAsyncMsgReader.hh000066400000000000000000000363461457266313600210330ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2012 by European Organization for Nuclear Research (CERN) // Author: Michal Simon //------------------------------------------------------------------------------ // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . //------------------------------------------------------------------------------ #ifndef SRC_XRDCL_XRDCLASYNCMSGREADER_HH_ #define SRC_XRDCL_XRDCLASYNCMSGREADER_HH_ #include "XrdCl/XrdClMessage.hh" #include "XrdCl/XrdClPostMasterInterfaces.hh" #include "XrdCl/XrdClXRootDResponses.hh" #include "XrdCl/XrdClSocket.hh" #include "XrdCl/XrdClConstants.hh" #include "XrdCl/XrdClStream.hh" #include namespace XrdCl { //---------------------------------------------------------------------------- //! Utility class encapsulating reading response message logic //---------------------------------------------------------------------------- class AsyncMsgReader { public: //------------------------------------------------------------------------ //! Constructor //! //! @param xrdTransport : the (xrootd) transport layer //! @param socket : the socket with the message to be read out //! @param strmname : stream name //! @param strm : the stream encapsulating the connection //! @param substrmnb : the substream number //------------------------------------------------------------------------ AsyncMsgReader( TransportHandler &xrdTransport, Socket &socket, const std::string &strmname, Stream &strm, uint16_t substrmnb) : readstage( ReadStart ), xrdTransport( xrdTransport ), socket( socket ), strmname( strmname ), strm( strm ), substrmnb( substrmnb ), inmsgsize( 0 ), inhandler( nullptr ) { } //------------------------------------------------------------------------ //! Destructor //------------------------------------------------------------------------ virtual ~AsyncMsgReader(){ } //------------------------------------------------------------------------ //! Reset the state of the object (makes it ready to read out next msg) //------------------------------------------------------------------------ inline void Reset() { readstage = ReadStart; inmsg.reset(); inmsgsize = 0; inhandler = nullptr; } //------------------------------------------------------------------------ //! Read out the response from the socket //------------------------------------------------------------------------ XRootDStatus Read() { Log *log = DefaultEnv::GetLog(); while( true ) { switch( readstage ) { //------------------------------------------------------------------ // There is no incoming message currently being processed so we // create a new one //------------------------------------------------------------------ case ReadStart: { inmsg = std::make_shared(); //---------------------------------------------------------------- // The next step is to read the header //---------------------------------------------------------------- readstage = ReadHeader; continue; } //------------------------------------------------------------------ // We need to read the header //------------------------------------------------------------------ case ReadHeader: { XRootDStatus st = xrdTransport.GetHeader( *inmsg, &socket ); if( !st.IsOK() || st.code == suRetry ) return st; log->Dump( AsyncSockMsg, "[%s] Received message header for 0x%x size: %d", strmname.c_str(), inmsg.get(), inmsg->GetCursor() ); ServerResponse *rsp = (ServerResponse*)inmsg->GetBuffer(); if( rsp->hdr.status == kXR_attn ) { log->Dump( AsyncSockMsg, "[%s] Will readout the attn action code " "of message 0x%x", strmname.c_str(), inmsg.get() ); inmsg->ReAllocate( 16 ); // header (bytes 8) + action code (8 bytes) readstage = ReadAttn; continue; } inmsgsize = inmsg->GetCursor(); inhandler = strm.InstallIncHandler( inmsg, substrmnb ); if( inhandler ) { log->Dump( AsyncSockMsg, "[%s] Will use the raw handler to read body " "of message 0x%x", strmname.c_str(), inmsg.get() ); //-------------------------------------------------------------- // The next step is to read raw data //-------------------------------------------------------------- readstage = ReadRawData; continue; } //---------------------------------------------------------------- // The next step is to read the message body //---------------------------------------------------------------- readstage = ReadMsgBody; continue; } //------------------------------------------------------------------ // Before proceeding we need to figure out the attn action code //------------------------------------------------------------------ case ReadAttn: { XRootDStatus st = ReadAttnActnum(); if( !st.IsOK() || st.code == suRetry ) return st; //---------------------------------------------------------------- // There is an embedded response, overwrite the message with that //---------------------------------------------------------------- if( HasEmbeddedRsp() ) { inmsg->Free(); readstage = ReadHeader; continue; } //---------------------------------------------------------------- // Readout the rest of the body //---------------------------------------------------------------- inmsgsize = inmsg->GetCursor(); readstage = ReadMsgBody; continue; } //------------------------------------------------------------------ // kXR_status is special as it can have both body and raw data, // handle it separately //------------------------------------------------------------------ case ReadMore: { XRootDStatus st = xrdTransport.GetMore( *inmsg, &socket ); if( !st.IsOK() || st.code == suRetry ) return st; inmsgsize = inmsg->GetCursor(); //---------------------------------------------------------------- // The next step is to finalize the read //---------------------------------------------------------------- readstage = ReadDone; continue; } //------------------------------------------------------------------ // We need to call a raw message handler to get the data from the // socket //------------------------------------------------------------------ case ReadRawData: { uint32_t bytesRead = 0; XRootDStatus st = inhandler->ReadMessageBody( inmsg.get(), &socket, bytesRead ); if( !st.IsOK() ) return st; inmsgsize += bytesRead; if( st.code == suRetry ) return st; //---------------------------------------------------------------- // The next step is to finalize the read //---------------------------------------------------------------- readstage = ReadDone; continue; } //------------------------------------------------------------------ // No raw handler, so we read the message to the buffer //------------------------------------------------------------------ case ReadMsgBody: { XRootDStatus st = xrdTransport.GetBody( *inmsg, &socket ); if( !st.IsOK() || st.code == suRetry ) return st; inmsgsize = inmsg->GetCursor(); //---------------------------------------------------------------- // kXR_status response needs special handling as it can have // either (body + raw data) or (body + additional body data) //---------------------------------------------------------------- if( IsStatusRsp() ) { uint16_t action = strm.InspectStatusRsp( substrmnb, inhandler ); if( action & MsgHandler::Corrupted ) return XRootDStatus( stError, errCorruptedHeader ); if( action & MsgHandler::Raw ) { //-------------------------------------------------------------- // The next step is to read the raw data //-------------------------------------------------------------- readstage = ReadRawData; continue; } if( action & MsgHandler::More ) { //-------------------------------------------------------------- // The next step is to read the additional data in the message // body //-------------------------------------------------------------- readstage = ReadMore; continue; } } //---------------------------------------------------------------- // The next step is to finalize the read //---------------------------------------------------------------- readstage = ReadDone; continue; } case ReadDone: { //---------------------------------------------------------------- // Report the incoming message //---------------------------------------------------------------- log->Dump( AsyncSockMsg, "[%s] Received message 0x%x of %d bytes", strmname.c_str(), inmsg.get(), inmsgsize ); strm.OnIncoming( substrmnb, std::move( inmsg ), inmsgsize ); } } // just in case break; } //---------------------------------------------------------------------- // We are done //---------------------------------------------------------------------- return XRootDStatus(); } private: XRootDStatus ReadAttnActnum() { //---------------------------------------------------------------------- // Readout the action code from the socket. We are reading out 8 bytes // into the message, the 8 byte header is already there. //---------------------------------------------------------------------- size_t btsleft = 8 - ( inmsg->GetCursor() - 8 ); while( btsleft > 0 ) { int btsrd = 0; XRootDStatus st = socket.Read( inmsg->GetBufferAtCursor(), btsleft, btsrd ); if( !st.IsOK() || st.code == suRetry ) return st; btsleft -= btsrd; inmsg->AdvanceCursor( btsrd ); } //---------------------------------------------------------------------- // Marshal the action code //---------------------------------------------------------------------- ServerResponseBody_Attn *attn = (ServerResponseBody_Attn*)inmsg->GetBuffer( 8 ); attn->actnum = ntohl( attn->actnum ); return XRootDStatus(); } inline bool IsStatusRsp() { ServerResponseHeader *hdr = (ServerResponseHeader*)inmsg->GetBuffer(); return ( hdr->status == kXR_status ); } inline bool HasEmbeddedRsp() { ServerResponseBody_Attn *attn = (ServerResponseBody_Attn*)inmsg->GetBuffer( 8 ); return ( attn->actnum == kXR_asynresp ); } //------------------------------------------------------------------------ //! Stages of reading out a response from the socket //------------------------------------------------------------------------ enum Stage { ReadStart, //< the next step is to initialize the read ReadHeader, //< the next step is to read the header ReadAttn, //< the next step is to read attn action code ReadMore, //< the next step is to read more status body ReadMsgBody, //< the next step is to read the body ReadRawData, //< the next step is to read the raw data ReadDone //< the next step is to finalize the read }; //------------------------------------------------------------------------ // Current read stage //------------------------------------------------------------------------ Stage readstage; //------------------------------------------------------------------------ // The context of the read operation //------------------------------------------------------------------------ TransportHandler &xrdTransport; Socket &socket; const std::string &strmname; Stream &strm; uint16_t substrmnb; //------------------------------------------------------------------------ // The internal state of the the reader //------------------------------------------------------------------------ std::shared_ptr inmsg; //< the ownership is shared with MsgHandler uint32_t inmsgsize; MsgHandler *inhandler; }; } /* namespace XrdCl */ #endif /* SRC_XRDCL_XRDCLASYNCMSGREADER_HH_ */ xrootd-5.6.9/src/XrdCl/XrdClAsyncMsgWriter.hh000066400000000000000000000252141457266313600210750ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2012 by European Organization for Nuclear Research (CERN) // Author: Michal Simon //------------------------------------------------------------------------------ // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . //------------------------------------------------------------------------------ #ifndef SRC_XRDCL_XRDCLASYNCMSGWRITER_HH_ #define SRC_XRDCL_XRDCLASYNCMSGWRITER_HH_ #include "XrdCl/XrdClMessage.hh" #include "XrdCl/XrdClPostMasterInterfaces.hh" #include "XrdCl/XrdClXRootDResponses.hh" #include "XrdCl/XrdClSocket.hh" #include "XrdCl/XrdClConstants.hh" #include "XrdCl/XrdClStream.hh" #include "XrdSys/XrdSysE2T.hh" #include namespace XrdCl { //---------------------------------------------------------------------------- //! Utility class encapsulating writing request logic //---------------------------------------------------------------------------- class AsyncMsgWriter { public: //------------------------------------------------------------------------ //! Constructor //! //! @param xrdTransport : the (xrootd) transport layer //! @param socket : the socket with the message to be read out //! @param strmname : stream name //! @param strm : the stream encapsulating the connection //! @param substrmnb : the substream number //------------------------------------------------------------------------ AsyncMsgWriter( TransportHandler &xrdTransport, Socket &socket, const std::string &strmname, Stream &strm, uint16_t substrmnb, AnyObject &chdata ) : writestage( WriteStart ), xrdTransport( xrdTransport ), socket( socket ), strmname( strmname ), strm( strm ), substrmnb( substrmnb ), chdata( chdata ), outmsg( nullptr ), outmsgsize( 0 ), outhandler( nullptr ) { } //------------------------------------------------------------------------ //! Reset the state of the object (makes it ready to read out next msg) //------------------------------------------------------------------------ inline void Reset() { writestage = WriteStart; outmsg = nullptr; outmsgsize = 0;; outhandler = nullptr; outsign.reset(); } //------------------------------------------------------------------------ //! Write the request into the socket //------------------------------------------------------------------------ XRootDStatus Write() { Log *log = DefaultEnv::GetLog(); while( true ) { switch( writestage ) { //------------------------------------------------------------------ // Pick up a message if we're not in process of writing something //------------------------------------------------------------------ case WriteStart: { std::pair toBeSent; toBeSent = strm.OnReadyToWrite( substrmnb ); outmsg = toBeSent.first; outhandler = toBeSent.second; if( !outmsg ) return XRootDStatus( stOK, suAlreadyDone ); outmsg->SetCursor( 0 ); outmsgsize = outmsg->GetSize(); //---------------------------------------------------------------- // Secure the message if necessary //---------------------------------------------------------------- Message *signature = nullptr; XRootDStatus st = xrdTransport.GetSignature( outmsg, signature, chdata ); if( !st.IsOK() ) return st; outsign.reset( signature ); if( outsign ) outmsgsize += outsign->GetSize(); //---------------------------------------------------------------- // The next step is to write the signature //---------------------------------------------------------------- writestage = WriteSign; continue; } //------------------------------------------------------------------ // First write the signature (if there is one) //------------------------------------------------------------------ case WriteSign: { //---------------------------------------------------------------- // If there is a signature for the request send it over the socket //---------------------------------------------------------------- if( outsign ) { XRootDStatus st = socket.Send( *outsign, strmname ); if( !st.IsOK() || st.code == suRetry ) return st; } //---------------------------------------------------------------- // The next step is to write the signature //---------------------------------------------------------------- writestage = WriteRequest; continue; } //------------------------------------------------------------------ // Then write the request itself //------------------------------------------------------------------ case WriteRequest: { XRootDStatus st = socket.Send( *outmsg, strmname ); if( !st.IsOK() || st.code == suRetry ) return st; //---------------------------------------------------------------- // The next step is to write the signature //---------------------------------------------------------------- writestage = WriteRawData; continue; } //------------------------------------------------------------------ // And then write the raw data (if any) //------------------------------------------------------------------ case WriteRawData: { if( outhandler->IsRaw() ) { uint32_t wrtcnt = 0; XRootDStatus st = outhandler->WriteMessageBody( &socket, wrtcnt ); if( !st.IsOK() || st.code == suRetry ) return st; outmsgsize += wrtcnt; log->Dump( AsyncSockMsg, "[%s] Wrote %d bytes of raw data of message" "(0x%x) body.", strmname.c_str(), wrtcnt, outmsg ); } //---------------------------------------------------------------- // The next step is to finalize the write operation //---------------------------------------------------------------- writestage = WriteDone; continue; } //------------------------------------------------------------------ // Finally, finalize the write operation //------------------------------------------------------------------ case WriteDone: { XRootDStatus st = socket.Flash(); if( !st.IsOK() ) { log->Error( AsyncSockMsg, "[%s] Unable to flash the socket: %s", strmname.c_str(), XrdSysE2T( st.errNo ) ); return st; } log->Dump( AsyncSockMsg, "[%s] Successfully sent message: %s (0x%x).", strmname.c_str(), outmsg->GetDescription().c_str(), outmsg ); strm.OnMessageSent( substrmnb, outmsg, outmsgsize ); return XRootDStatus(); } } // just in case ... break; } //---------------------------------------------------------------------- // We are done //---------------------------------------------------------------------- return XRootDStatus(); } private: //------------------------------------------------------------------------ //! Stages of reading out a response from the socket //------------------------------------------------------------------------ enum Stage { WriteStart, //< the next step is to initialize the read WriteSign, //< the next step is to write the signature WriteRequest, //< the next step is to write the request WriteRawData, //< the next step is to write the raw data WriteDone //< the next step is to finalize the write }; //------------------------------------------------------------------------ // Current read stage //------------------------------------------------------------------------ Stage writestage; //------------------------------------------------------------------------ // The context of the read operation //------------------------------------------------------------------------ TransportHandler &xrdTransport; Socket &socket; const std::string &strmname; Stream &strm; uint16_t substrmnb; AnyObject &chdata; //------------------------------------------------------------------------ // The internal state of the the reader //------------------------------------------------------------------------ Message *outmsg; //< we don't own the message uint32_t outmsgsize; MsgHandler *outhandler; std::unique_ptr outsign; }; } #endif /* SRC_XRDCL_XRDCLASYNCMSGWRITER_HH_ */ xrootd-5.6.9/src/XrdCl/XrdClAsyncPageReader.hh000066400000000000000000000307611457266313600211540ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2012 by European Organization for Nuclear Research (CERN) // Author: Michal Simon //------------------------------------------------------------------------------ // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . //------------------------------------------------------------------------------ #ifndef SRC_XRDCL_XRDCLASYNCPAGEREADER_HH_ #define SRC_XRDCL_XRDCLASYNCPAGEREADER_HH_ #include "XrdCl/XrdClXRootDResponses.hh" #include "XrdCl/XrdClSocket.hh" #include "XrdOuc/XrdOucPgrwUtils.hh" #include "XrdSys/XrdSysPageSize.hh" #include #include #include namespace XrdCl { //------------------------------------------------------------------------------ //! Object for reading out data from the PgRead response //------------------------------------------------------------------------------ class AsyncPageReader { public: //-------------------------------------------------------------------------- //! Constructor //! //! @param chunks : list of buffer for the data //! @param digests : a vector that will be filled with crc32c digest data //-------------------------------------------------------------------------- AsyncPageReader( ChunkList &chunks, std::vector &digests ) : chunks( chunks ), digests( digests ), dlen( 0 ), rspoff( 0 ), chindex( 0 ), choff( 0 ), dgindex( 0 ), dgoff( 0 ), iovcnt( 0 ), iovindex( 0 ) { uint64_t rdoff = chunks.front().offset; uint32_t rdlen = 0; for( auto &ch : chunks ) rdlen += ch.length; int fpglen, lpglen; int pgcnt = XrdOucPgrwUtils::csNum( rdoff, rdlen, fpglen, lpglen); digests.resize( pgcnt ); } //-------------------------------------------------------------------------- //! Destructor //-------------------------------------------------------------------------- virtual ~AsyncPageReader() { } //-------------------------------------------------------------------------- //! Sets message data size //-------------------------------------------------------------------------- void SetRsp( ServerResponseV2 *rsp ) { dlen = rsp->status.bdy.dlen; rspoff = rsp->info.pgread.offset; uint64_t bufoff = rspoff - chunks[0].offset; chindex = 0; for( chindex = 0; chindex < chunks.size(); ++chindex ) { if( chunks[chindex].length < bufoff ) { bufoff -= chunks[chindex].length; continue; } break; } choff = bufoff; } //-------------------------------------------------------------------------- //! Readout data from the socket //! @param socket : the socket with the data //! @param btsread : number of user data read from the socket //! @return : operation status //-------------------------------------------------------------------------- XRootDStatus Read( Socket &socket, uint32_t &btsread ) { if( dlen == 0 || chindex >= chunks.size() ) return XRootDStatus(); btsread = 0; int nbbts = 0; do { // Prepare the IO vector for receiving the data if( iov.empty() ) InitIOV(); // read the data into the buffer nbbts = 0; auto st = socket.ReadV( iov.data() + iovindex, iovcnt, nbbts ); if( !st.IsOK() ) return st; btsread += nbbts; dlen -= nbbts; ShiftIOV( nbbts ); if( st.code == suRetry ) return st; } while( nbbts > 0 && dlen > 0 && chindex < chunks.size() ); return XRootDStatus(); } private: //-------------------------------------------------------------------------- //! Helper class for retrieving the maximum size of the I/O vector //-------------------------------------------------------------------------- struct iovmax_t { iovmax_t() { #ifdef _SC_IOV_MAX value = sysconf(_SC_IOV_MAX); if (value == -1) #endif #ifdef IOV_MAX value = IOV_MAX; #else value = 1024; #endif value &= ~uint32_t( 1 ); // make sure it is an even number } int32_t value; }; //-------------------------------------------------------------------------- //! @return : maximum size of I/O vector //-------------------------------------------------------------------------- inline static int max_iovcnt() { static iovmax_t iovmax; return iovmax.value; } //-------------------------------------------------------------------------- //! Add I/O buffer to the vector //-------------------------------------------------------------------------- inline void addiov( char *&buf, size_t len ) { iov.emplace_back(); iov.back().iov_base = buf; iov.back().iov_len = len; buf += len; ++iovcnt; } //-------------------------------------------------------------------------- //! Add I/O buffer to the vector and update number of bytes left to be read //-------------------------------------------------------------------------- inline void addiov( char *&buf, uint32_t len, uint32_t &dleft ) { if( len > dleft ) len = dleft; addiov( buf, len ); dleft -= len; } //-------------------------------------------------------------------------- //! Calculate the size of the I/O vector //! @param dleft : data to be accomodated by the I/O vector //-------------------------------------------------------------------------- inline static uint32_t CalcIOVSize( uint32_t dleft ) { uint32_t ret = ( dleft / PageWithDigest + 2 ) * 2; return ( ret > uint32_t( max_iovcnt() ) ? max_iovcnt() : ret ); } //-------------------------------------------------------------------------- //! Calculate the size of the data to be read //-------------------------------------------------------------------------- uint32_t CalcRdSize() { // data size in the server response (including digests) uint32_t dleft = dlen; // space in our page buffer uint32_t pgspace = chunks[chindex].length - choff; // space in our digest buffer uint32_t dgspace = sizeof( uint32_t ) * (digests.size() - dgindex ) - dgoff; if( dleft > pgspace + dgspace ) dleft = pgspace + dgspace; return dleft; } //-------------------------------------------------------------------------- //! Initialize the I/O vector //-------------------------------------------------------------------------- void InitIOV() { iovindex = 0; // figure out the number of data we can read in one go uint32_t dleft = CalcRdSize(); // and reset the I/O vector iov.clear(); iovcnt = 0; iov.reserve( CalcIOVSize( dleft ) ); // now prepare the page and digest buffers ChunkInfo ch = chunks[chindex]; char* pgbuf = static_cast( ch.buffer ) + choff; uint64_t rdoff = ch.offset + choff; char* dgbuf = reinterpret_cast( digests.data() + dgindex ) + dgoff; // handle the first digest uint32_t fdglen = sizeof( uint32_t ) - dgoff; addiov( dgbuf, fdglen, dleft ); if( dleft == 0 || iovcnt >= max_iovcnt() ) return; // handle the first page uint32_t fpglen = XrdSys::PageSize - rdoff % XrdSys::PageSize; addiov( pgbuf, fpglen, dleft ); if( dleft == 0 || iovcnt >= max_iovcnt() ) return; // handle all the subsequent aligned pages size_t fullpgs = dleft / PageWithDigest; for( size_t i = 0; i < fullpgs; ++i ) { addiov( dgbuf, sizeof( uint32_t ), dleft ); if( dleft == 0 || iovcnt >= max_iovcnt() ) return; addiov( pgbuf, XrdSys::PageSize, dleft ); if( dleft == 0 || iovcnt >= max_iovcnt() ) return; } // handle the last digest uint32_t ldglen = sizeof( uint32_t ); addiov( dgbuf, ldglen, dleft ); if( dleft == 0 || iovcnt >= max_iovcnt() ) return; // handle the last page addiov( pgbuf, dleft, dleft ); } //-------------------------------------------------------------------------- //! Shift buffer by a number of bytes //-------------------------------------------------------------------------- inline void shift( void *&buffer, size_t nbbts ) { char *buf = static_cast( buffer ); buf += nbbts; buffer = buf; } //-------------------------------------------------------------------------- //! shift digest buffer by `btsread` //! @param btsread : total number of bytes read (will be decremented by bytes //! shifted in buffer) //-------------------------------------------------------------------------- inline void shiftdgbuf( uint32_t &btsread ) { if( iov[iovindex].iov_len > btsread ) { iov[iovindex].iov_len -= btsread; shift( iov[iovindex].iov_base, btsread ); dgoff += btsread; btsread = 0; return; } btsread -= iov[iovindex].iov_len; iov[iovindex].iov_len = 0; dgoff = 0; digests[dgindex] = ntohl( digests[dgindex] ); ++dgindex; ++iovindex; --iovcnt; } //-------------------------------------------------------------------------- //! shift page buffer by `btsread` //! @param btsread : total number of bytes read (will be decremented by bytes //! shifted in buffer) //-------------------------------------------------------------------------- inline void shiftpgbuf( uint32_t &btsread ) { if( iov[iovindex].iov_len > btsread ) { iov[iovindex].iov_len -= btsread; shift( iov[iovindex].iov_base, btsread ); choff += btsread; btsread = 0; return; } btsread -= iov[iovindex].iov_len; choff += iov[iovindex].iov_len; iov[iovindex].iov_len = 0; ++iovindex; --iovcnt; } //-------------------------------------------------------------------------- //! shift the I/O vector by the number of bytes read //-------------------------------------------------------------------------- void ShiftIOV( uint32_t btsread ) { // if iovindex is even it point to digest, otherwise it points to a page if( iovindex % 2 == 0 ) shiftdgbuf( btsread ); // adjust as many I/O buffers as necessary while( btsread > 0 ) { // handle page shiftpgbuf( btsread ); if( btsread == 0 ) break; // handle digest shiftdgbuf( btsread ); } // if we filled the buffer, move to the next one if( iovcnt == 0 ) iov.clear(); // do we need to move to the next chunk? if( choff >= chunks[chindex].length ) { ++chindex; choff = 0; } } ChunkList &chunks; //< list of data chunks to be filled with user data std::vector &digests; //< list of crc32c digests for every 4KB page of data uint32_t dlen; //< size of the data in the message uint64_t rspoff; //< response offset size_t chindex; //< index of the current data buffer size_t choff; //< offset within the current buffer size_t dgindex; //< index of the current digest buffer size_t dgoff; //< offset within the current digest buffer std::vector iov; //< I/O vector int iovcnt; //< size of the I/O vector size_t iovindex; //< index of the first valid element in the I/O vector static const int PageWithDigest = XrdSys::PageSize + sizeof( uint32_t ); }; } /* namespace XrdEc */ #endif /* SRC_XRDCL_XRDCLASYNCPAGEREADER_HH_ */ xrootd-5.6.9/src/XrdCl/XrdClAsyncRawReader.hh000066400000000000000000000206561457266313600210330ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2012 by European Organization for Nuclear Research (CERN) // Author: Michal Simon //------------------------------------------------------------------------------ // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . //------------------------------------------------------------------------------ #ifndef SRC_XRDCL_XRDCLASYNCRAWREADER_HH_ #define SRC_XRDCL_XRDCLASYNCRAWREADER_HH_ #include "XrdCl/XrdClXRootDResponses.hh" #include "XrdCl/XrdClSocket.hh" #include "XrdCl/XrdClStream.hh" #include "XrdClAsyncRawReaderIntfc.hh" namespace XrdCl { //---------------------------------------------------------------------------- //! Object for reading out data from the kXR_read response //---------------------------------------------------------------------------- class AsyncRawReader : public AsyncRawReaderIntfc { public: //------------------------------------------------------------------------ //! Constructor //! //! @param url : channel URL //! @param request : client request //------------------------------------------------------------------------ AsyncRawReader( const URL &url, const Message &request ) : AsyncRawReaderIntfc( url, request ) { } //------------------------------------------------------------------------ //! Readout raw data from socket //! //! @param socket : the socket //! @param btsret : number of bytes read //! @return : operation status //------------------------------------------------------------------------ XRootDStatus Read( Socket &socket, uint32_t &btsret ) { while( true ) { switch( readstage ) { //------------------------------------------------------------------ // Prepare to readout a new response //------------------------------------------------------------------ case ReadStart: { msgbtsrd = 0; chlen = ( *chunks )[0].length; readstage = ReadRaw; continue; } //------------------------------------------------------------------ // Readout the raw data //------------------------------------------------------------------ case ReadRaw: { //---------------------------------------------------------------- // Make sure we are not reading past the end of the read response //---------------------------------------------------------------- if( msgbtsrd + chlen > dlen ) chlen = dlen - msgbtsrd; //---------------------------------------------------------------- // Readout the raw data from the socket //---------------------------------------------------------------- uint32_t btsrd = 0; char *buff = static_cast( ( *chunks )[chidx].buffer ); Status st = ReadBytesAsync( socket, buff + choff, chlen, btsrd ); choff += btsrd; chlen -= btsrd; msgbtsrd += btsrd; rawbtsrd += btsrd; btsret += btsrd; if( !st.IsOK() || st.code == suRetry ) return st; //---------------------------------------------------------------- // If the chunk is full, move to the next buffer //---------------------------------------------------------------- if( choff == ( *chunks )[chidx].length ) { ++chidx; choff = 0; chlen = ( chidx < chunks->size() ? ( *chunks )[chidx].length : 0 ); } //---------------------------------------------------------------- // Check if there are some data left in the response to be readout // from the socket. //---------------------------------------------------------------- if( msgbtsrd < dlen ) { //-------------------------------------------------------------- // We run out of space, the server has send too much data //-------------------------------------------------------------- if( chidx >= chunks->size() ) { readstage = ReadDiscard; continue; } readstage = ReadRaw; continue; } //---------------------------------------------------------------- // We are done //---------------------------------------------------------------- readstage = ReadDone; continue; } //------------------------------------------------------------------ // We've had an error and we are in the discarding mode //------------------------------------------------------------------ case ReadDiscard: { DefaultEnv::GetLog()->Error( XRootDMsg, "[%s] RawReader: Handling " "response to %s: user supplied buffer is " "too small for the received data.", url.GetHostId().c_str(), request.GetDescription().c_str() ); // Just drop the connection, we don't know if the stream is sane // anymore. Recover with a reconnect. return XRootDStatus( stError, errCorruptedHeader ); } //------------------------------------------------------------------ // Finalize the read //------------------------------------------------------------------ case ReadDone: { break; } //------------------------------------------------------------------ // Others should not happen //------------------------------------------------------------------ default : return XRootDStatus( stError, errInternal ); } // just in case break; } //---------------------------------------------------------------------- // We are done //---------------------------------------------------------------------- return XRootDStatus(); } //------------------------------------------------------------------------ //! Get the response //------------------------------------------------------------------------ XRootDStatus GetResponse( AnyObject *&response ) { if( dataerr ) return XRootDStatus( stError, errInvalidResponse ); std::unique_ptr rsp( new AnyObject() ); if( request.GetVirtReqID() == kXR_virtReadv ) rsp->Set( GetVectorReadInfo() ); else rsp->Set( GetChunkInfo() ); response = rsp.release(); return XRootDStatus(); } private: inline ChunkInfo* GetChunkInfo() { ChunkInfo *info = new ChunkInfo( chunks->front() ); info->length = rawbtsrd; return info; } inline VectorReadInfo* GetVectorReadInfo() { VectorReadInfo *info = new VectorReadInfo(); info->SetSize( rawbtsrd ); int btsleft = rawbtsrd; for( auto &chunk : *chunks ) { int length = uint32_t( btsleft ) >= chunk.length ? chunk.length : btsleft; info->GetChunks().emplace_back( chunk.offset, length, chunk.buffer ); btsleft -= length; } return info; } }; } /* namespace XrdCl */ #endif /* SRC_XRDCL_XRDCLASYNCVECTORREADER_HH_ */ xrootd-5.6.9/src/XrdCl/XrdClAsyncRawReaderIntfc.hh000066400000000000000000000156711457266313600220200ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2012 by European Organization for Nuclear Research (CERN) // Author: Michal Simon //------------------------------------------------------------------------------ // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . //------------------------------------------------------------------------------ #ifndef SRC_XRDCL_XRDCLASYNCRAWREADERINTFC_HH_ #define SRC_XRDCL_XRDCLASYNCRAWREADERINTFC_HH_ #include "XrdCl/XrdClXRootDResponses.hh" #include "XrdCl/XrdClSocket.hh" #include "XrdCl/XrdClDefaultEnv.hh" #include "XrdCl/XrdClLog.hh" #include "XrdCl/XrdClConstants.hh" namespace XrdCl { //---------------------------------------------------------------------------- //! Base class for any message's body reader //---------------------------------------------------------------------------- class AsyncRawReaderIntfc { public: AsyncRawReaderIntfc( const URL &url, const Message &request ) : readstage( ReadStart ), url( url ), request( request ), chunks( nullptr ), dlen( 0 ), msgbtsrd( 0 ), rawbtsrd( 0 ), chidx( 0 ), choff( 0 ), chlen( 0 ), dataerr( false ) { } //------------------------------------------------------------------------ //! Destructor //------------------------------------------------------------------------ virtual ~AsyncRawReaderIntfc() { } //------------------------------------------------------------------------ //! Sets response data length //------------------------------------------------------------------------ void SetDataLength( int dlen ) { this->dlen = dlen; this->readstage = ReadStart; } //------------------------------------------------------------------------ //! Sets the chunk list with user buffers //------------------------------------------------------------------------ void SetChunkList( ChunkList *chunks ) { this->chunks = chunks; if( chunks ) this->chstatus.resize( chunks->size() ); } //------------------------------------------------------------------------ //! Readout raw data from socket //! //! @param socket : the socket //! @param btsret : number of bytes read //! @return : operation status //------------------------------------------------------------------------ virtual XRootDStatus Read( Socket &socket, uint32_t &btsret ) = 0; //------------------------------------------------------------------------ //! Get the response //------------------------------------------------------------------------ virtual XRootDStatus GetResponse( AnyObject *&response ) = 0; protected: //-------------------------------------------------------------------------- // Read a buffer asynchronously //-------------------------------------------------------------------------- XRootDStatus ReadBytesAsync( Socket &socket, char *buffer, uint32_t toBeRead, uint32_t &bytesRead ) { size_t shift = 0; while( toBeRead > 0 ) { int btsRead = 0; Status status = socket.Read( buffer + shift, toBeRead, btsRead ); if( !status.IsOK() || status.code == suRetry ) return status; bytesRead += btsRead; toBeRead -= btsRead; shift += btsRead; } return XRootDStatus( stOK, suDone ); } //------------------------------------------------------------------------ // Helper struct for async reading of chunks //------------------------------------------------------------------------ struct ChunkStatus { ChunkStatus(): sizeerr( false ), done( false ) {} bool sizeerr; bool done; }; //------------------------------------------------------------------------ // internal buffer type //------------------------------------------------------------------------ using buffer_t = std::vector; //------------------------------------------------------------------------ //! Stages of reading out a response from the socket //------------------------------------------------------------------------ enum Stage { ReadStart, //< the next step is to initialize the read ReadRdLst, //< the next step is to read the read_list ReadRaw, //< the next step is to read the raw data ReadDiscard, //< there was an error, we are in discard mode ReadDone //< the next step is to finalize the read }; //------------------------------------------------------------------------ // Current read stage //------------------------------------------------------------------------ Stage readstage; //------------------------------------------------------------------------ // The context of the read operation //------------------------------------------------------------------------ const URL &url; //< for logging purposes const Message &request; //< client request ChunkList *chunks; //< list of data chunks to be filled with user data std::vector chstatus; //< status per chunk uint32_t dlen; //< size of the data in the message uint32_t msgbtsrd; //< number of bytes read out from the socket for the current message uint32_t rawbtsrd; //< total number of bytes read out from the socket (raw data only) size_t chidx; //< index of the current data buffer size_t choff; //< offset within the current buffer size_t chlen; //< bytes left to be readout into the current chunk buffer_t discardbuff; //< buffer for discarding data in case of an error bool dataerr; //< true if the server send us too much data, false otherwise }; } #endif /* SRC_XRDCL_XRDCLASYNCRAWREADERINTFC_HH_ */ xrootd-5.6.9/src/XrdCl/XrdClAsyncSocketHandler.cc000066400000000000000000001044321457266313600216660ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2012 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . //------------------------------------------------------------------------------ #include "XrdCl/XrdClStream.hh" #include "XrdCl/XrdClConstants.hh" #include "XrdCl/XrdClLog.hh" #include "XrdCl/XrdClMessage.hh" #include "XrdCl/XrdClAsyncSocketHandler.hh" #include "XrdCl/XrdClXRootDTransport.hh" #include "XrdCl/XrdClXRootDMsgHandler.hh" #include "XrdCl/XrdClOptimizers.hh" #include "XrdSys/XrdSysE2T.hh" #include namespace XrdCl { //---------------------------------------------------------------------------- // Constructor //---------------------------------------------------------------------------- AsyncSocketHandler::AsyncSocketHandler( const URL &url, Poller *poller, TransportHandler *transport, AnyObject *channelData, uint16_t subStreamNum, Stream *strm ): pPoller( poller ), pTransport( transport ), pChannelData( channelData ), pSubStreamNum( subStreamNum ), pStream( strm ), pStreamName( ToStreamName( url, subStreamNum ) ), pSocket( new Socket() ), pHandShakeDone( false ), pConnectionStarted( 0 ), pConnectionTimeout( 0 ), pHSWaitStarted( 0 ), pHSWaitSeconds( 0 ), pUrl( url ), pTlsHandShakeOngoing( false ) { Env *env = DefaultEnv::GetEnv(); int timeoutResolution = DefaultTimeoutResolution; env->GetInt( "TimeoutResolution", timeoutResolution ); pTimeoutResolution = timeoutResolution; pSocket->SetChannelID( pChannelData ); pLastActivity = time(0); } //---------------------------------------------------------------------------- // Destructor //---------------------------------------------------------------------------- AsyncSocketHandler::~AsyncSocketHandler() { Close(); delete pSocket; } //---------------------------------------------------------------------------- // Connect to given address //---------------------------------------------------------------------------- XRootDStatus AsyncSocketHandler::Connect( time_t timeout ) { Log *log = DefaultEnv::GetLog(); pLastActivity = pConnectionStarted = ::time(0); pConnectionTimeout = timeout; //-------------------------------------------------------------------------- // Initialize the socket //-------------------------------------------------------------------------- XRootDStatus st = pSocket->Initialize( pSockAddr.Family() ); if( !st.IsOK() ) { log->Error( AsyncSockMsg, "[%s] Unable to initialize socket: %s", pStreamName.c_str(), st.ToString().c_str() ); st.status = stFatal; return st; } //-------------------------------------------------------------------------- // Set the keep-alive up //-------------------------------------------------------------------------- Env *env = DefaultEnv::GetEnv(); int keepAlive = DefaultTCPKeepAlive; env->GetInt( "TCPKeepAlive", keepAlive ); if( keepAlive ) { int param = 1; XRootDStatus st = pSocket->SetSockOpt( SOL_SOCKET, SO_KEEPALIVE, ¶m, sizeof(param) ); if( !st.IsOK() ) log->Error( AsyncSockMsg, "[%s] Unable to turn on keepalive: %s", st.ToString().c_str() ); #if ( defined(__linux__) || defined(__GNU__) ) && defined( TCP_KEEPIDLE ) && \ defined( TCP_KEEPINTVL ) && defined( TCP_KEEPCNT ) param = DefaultTCPKeepAliveTime; env->GetInt( "TCPKeepAliveTime", param ); st = pSocket->SetSockOpt(SOL_TCP, TCP_KEEPIDLE, ¶m, sizeof(param)); if( !st.IsOK() ) log->Error( AsyncSockMsg, "[%s] Unable to set keepalive time: %s", st.ToString().c_str() ); param = DefaultTCPKeepAliveInterval; env->GetInt( "TCPKeepAliveInterval", param ); st = pSocket->SetSockOpt(SOL_TCP, TCP_KEEPINTVL, ¶m, sizeof(param)); if( !st.IsOK() ) log->Error( AsyncSockMsg, "[%s] Unable to set keepalive interval: %s", st.ToString().c_str() ); param = DefaultTCPKeepAliveProbes; env->GetInt( "TCPKeepAliveProbes", param ); st = pSocket->SetSockOpt(SOL_TCP, TCP_KEEPCNT, ¶m, sizeof(param)); if( !st.IsOK() ) log->Error( AsyncSockMsg, "[%s] Unable to set keepalive probes: %s", st.ToString().c_str() ); #endif } pHandShakeDone = false; pTlsHandShakeOngoing = false; pHSWaitStarted = 0; pHSWaitSeconds = 0; //-------------------------------------------------------------------------- // Initiate async connection to the address //-------------------------------------------------------------------------- char nameBuff[256]; pSockAddr.Format( nameBuff, sizeof(nameBuff), XrdNetAddrInfo::fmtAdv6 ); log->Debug( AsyncSockMsg, "[%s] Attempting connection to %s", pStreamName.c_str(), nameBuff ); st = pSocket->ConnectToAddress( pSockAddr, 0 ); if( !st.IsOK() ) { log->Error( AsyncSockMsg, "[%s] Unable to initiate the connection: %s", pStreamName.c_str(), st.ToString().c_str() ); return st; } pSocket->SetStatus( Socket::Connecting ); //-------------------------------------------------------------------------- // We should get the ready to write event once we're really connected // so we need to listen to it //-------------------------------------------------------------------------- if( !pPoller->AddSocket( pSocket, this ) ) { XRootDStatus st( stFatal, errPollerError ); pSocket->Close(); return st; } if( !pPoller->EnableWriteNotification( pSocket, true, pTimeoutResolution ) ) { XRootDStatus st( stFatal, errPollerError ); pPoller->RemoveSocket( pSocket ); pSocket->Close(); return st; } return XRootDStatus(); } //---------------------------------------------------------------------------- // Close the connection //---------------------------------------------------------------------------- XRootDStatus AsyncSocketHandler::Close() { Log *log = DefaultEnv::GetLog(); log->Debug( AsyncSockMsg, "[%s] Closing the socket", pStreamName.c_str() ); pTransport->Disconnect( *pChannelData, pSubStreamNum ); pPoller->RemoveSocket( pSocket ); pSocket->Close(); return XRootDStatus(); } std::string AsyncSocketHandler::ToStreamName( const URL &url, uint16_t strmnb ) { std::ostringstream o; o << url.GetHostId(); o << "." << strmnb; return o.str(); } //---------------------------------------------------------------------------- // Handler a socket event //---------------------------------------------------------------------------- void AsyncSocketHandler::Event( uint8_t type, XrdCl::Socket */*socket*/ ) { //-------------------------------------------------------------------------- // First check if the socket itself wants to apply some mapping on the // event. E.g. in case of TLS socket it might want to map read events to // write events and vice-versa. //-------------------------------------------------------------------------- type = pSocket->MapEvent( type ); //-------------------------------------------------------------------------- // Handle any read or write events. If any of the handlers indicate an error // we will have been disconnected. A disconnection may cause the current // object to be asynchronously reused or deleted, so we return immediately. //-------------------------------------------------------------------------- if( !EventRead( type ) ) return; if( !EventWrite( type ) ) return; } //---------------------------------------------------------------------------- // Handler for read related socket events //---------------------------------------------------------------------------- bool AsyncSocketHandler::EventRead( uint8_t type ) { //-------------------------------------------------------------------------- // Read event //-------------------------------------------------------------------------- if( type & ReadyToRead ) { pLastActivity = time(0); if( unlikely( pTlsHandShakeOngoing ) ) return OnTLSHandShake(); if( likely( pHandShakeDone ) ) return OnRead(); return OnReadWhileHandshaking(); } //-------------------------------------------------------------------------- // Read timeout //-------------------------------------------------------------------------- else if( type & ReadTimeOut ) { if( pHSWaitSeconds ) { if( !CheckHSWait() ) return false; } if( likely( pHandShakeDone ) ) return OnReadTimeout(); return OnTimeoutWhileHandshaking(); } return true; } //---------------------------------------------------------------------------- // Handler for write related socket events //---------------------------------------------------------------------------- bool AsyncSocketHandler::EventWrite( uint8_t type ) { //-------------------------------------------------------------------------- // Write event //-------------------------------------------------------------------------- if( type & ReadyToWrite ) { pLastActivity = time(0); if( unlikely( pSocket->GetStatus() == Socket::Connecting ) ) return OnConnectionReturn(); //------------------------------------------------------------------------ // Make sure we are not writing anything if we have been told to wait. //------------------------------------------------------------------------ if( pHSWaitSeconds != 0 ) return true; if( unlikely( pTlsHandShakeOngoing ) ) return OnTLSHandShake(); if( likely( pHandShakeDone ) ) return OnWrite(); return OnWriteWhileHandshaking(); } //-------------------------------------------------------------------------- // Write timeout //-------------------------------------------------------------------------- else if( type & WriteTimeOut ) { if( likely( pHandShakeDone ) ) return OnWriteTimeout(); return OnTimeoutWhileHandshaking(); } return true; } //---------------------------------------------------------------------------- // Connect returned //---------------------------------------------------------------------------- bool AsyncSocketHandler::OnConnectionReturn() { //-------------------------------------------------------------------------- // Check whether we were able to connect //-------------------------------------------------------------------------- Log *log = DefaultEnv::GetLog(); log->Debug( AsyncSockMsg, "[%s] Async connection call returned", pStreamName.c_str() ); int errorCode = 0; socklen_t optSize = sizeof( errorCode ); XRootDStatus st = pSocket->GetSockOpt( SOL_SOCKET, SO_ERROR, &errorCode, &optSize ); //-------------------------------------------------------------------------- // This is an internal error really (either logic or system fault), // so we call it a day and don't retry //-------------------------------------------------------------------------- if( !st.IsOK() ) { log->Error( AsyncSockMsg, "[%s] Unable to get the status of the " "connect operation: %s", pStreamName.c_str(), XrdSysE2T( errno ) ); pStream->OnConnectError( pSubStreamNum, XRootDStatus( stFatal, errSocketOptError, errno ) ); return false; } //-------------------------------------------------------------------------- // We were unable to connect //-------------------------------------------------------------------------- if( errorCode ) { log->Error( AsyncSockMsg, "[%s] Unable to connect: %s", pStreamName.c_str(), XrdSysE2T( errorCode ) ); pStream->OnConnectError( pSubStreamNum, XRootDStatus( stError, errConnectionError ) ); return false; } pSocket->SetStatus( Socket::Connected ); //-------------------------------------------------------------------------- // Cork the socket //-------------------------------------------------------------------------- st = pSocket->Cork(); if( !st.IsOK() ) { pStream->OnConnectError( pSubStreamNum, st ); return false; } //-------------------------------------------------------------------------- // Initialize the handshake //-------------------------------------------------------------------------- pHandShakeData.reset( new HandShakeData( pStream->GetURL(), pSubStreamNum ) ); pHandShakeData->serverAddr = pSocket->GetServerAddress(); pHandShakeData->clientName = pSocket->GetSockName(); pHandShakeData->streamName = pStreamName; st = pTransport->HandShake( pHandShakeData.get(), *pChannelData ); if( !st.IsOK() ) { log->Error( AsyncSockMsg, "[%s] Connection negotiation failed", pStreamName.c_str() ); pStream->OnConnectError( pSubStreamNum, st ); return false; } if( st.code != suRetry ) ++pHandShakeData->step; //-------------------------------------------------------------------------- // Initialize the hand-shake reader and writer //-------------------------------------------------------------------------- hswriter.reset( new AsyncHSWriter( *pSocket, pStreamName ) ); hsreader.reset( new AsyncHSReader( *pTransport, *pSocket, pStreamName, *pStream, pSubStreamNum ) ); //-------------------------------------------------------------------------- // Transport has given us something to send //-------------------------------------------------------------------------- if( pHandShakeData->out ) { hswriter->Reset( pHandShakeData->out ); pHandShakeData->out = nullptr; } //-------------------------------------------------------------------------- // Listen to what the server has to say //-------------------------------------------------------------------------- if( !pPoller->EnableReadNotification( pSocket, true, pTimeoutResolution ) ) { pStream->OnConnectError( pSubStreamNum, XRootDStatus( stFatal, errPollerError ) ); return false; } return true; } //---------------------------------------------------------------------------- // Got a write readiness event //---------------------------------------------------------------------------- bool AsyncSocketHandler::OnWrite() { if( !reqwriter ) { OnFault( XRootDStatus( stError, errInternal, 0, "Request writer is null." ) ); return false; } //-------------------------------------------------------------------------- // Let's do the writing ... //-------------------------------------------------------------------------- XRootDStatus st = reqwriter->Write(); if( !st.IsOK() ) { //------------------------------------------------------------------------ // We failed //------------------------------------------------------------------------ OnFault( st ); return false; } //-------------------------------------------------------------------------- // We are not done yet //-------------------------------------------------------------------------- if( st.code == suRetry) return true; //-------------------------------------------------------------------------- // Disable the respective substream if empty //-------------------------------------------------------------------------- reqwriter->Reset(); pStream->DisableIfEmpty( pSubStreamNum ); return true; } //---------------------------------------------------------------------------- // Got a write readiness event while handshaking //---------------------------------------------------------------------------- bool AsyncSocketHandler::OnWriteWhileHandshaking() { XRootDStatus st; if( !hswriter || !hswriter->HasMsg() ) { if( !(st = DisableUplink()).IsOK() ) { OnFaultWhileHandshaking( st ); return false; } return true; } //-------------------------------------------------------------------------- // Let's do the writing ... //-------------------------------------------------------------------------- st = hswriter->Write(); if( !st.IsOK() ) { //------------------------------------------------------------------------ // We failed //------------------------------------------------------------------------ OnFaultWhileHandshaking( st ); return false; } //-------------------------------------------------------------------------- // We are not done yet //-------------------------------------------------------------------------- if( st.code == suRetry ) return true; //-------------------------------------------------------------------------- // Disable the uplink // Note: at this point we don't deallocate the HS message as we might need // to re-send it in case of a kXR_wait response //-------------------------------------------------------------------------- if( !(st = DisableUplink()).IsOK() ) { OnFaultWhileHandshaking( st ); return false; } return true; } //---------------------------------------------------------------------------- // Got a read readiness event //---------------------------------------------------------------------------- bool AsyncSocketHandler::OnRead() { //-------------------------------------------------------------------------- // Make sure the response reader object exists //-------------------------------------------------------------------------- if( !rspreader ) { OnFault( XRootDStatus( stError, errInternal, 0, "Response reader is null." ) ); return false; } //-------------------------------------------------------------------------- // Readout the data from the socket //-------------------------------------------------------------------------- XRootDStatus st = rspreader->Read(); //-------------------------------------------------------------------------- // Handler header corruption //-------------------------------------------------------------------------- if( !st.IsOK() && st.code == errCorruptedHeader ) { OnHeaderCorruption(); return false; } //-------------------------------------------------------------------------- // Handler other errors //-------------------------------------------------------------------------- if( !st.IsOK() ) { OnFault( st ); return false; } //-------------------------------------------------------------------------- // We are not done yet //-------------------------------------------------------------------------- if( st.code == suRetry ) return true; //-------------------------------------------------------------------------- // We are done, reset the response reader so we can read out next message //-------------------------------------------------------------------------- rspreader->Reset(); return true; } //---------------------------------------------------------------------------- // Got a read readiness event while handshaking //---------------------------------------------------------------------------- bool AsyncSocketHandler::OnReadWhileHandshaking() { //-------------------------------------------------------------------------- // Make sure the response reader object exists //-------------------------------------------------------------------------- if( !hsreader ) { OnFault( XRootDStatus( stError, errInternal, 0, "Hand-shake reader is null." ) ); return false; } //-------------------------------------------------------------------------- // Read the message and let the transport handler look at it when // reading has finished //-------------------------------------------------------------------------- XRootDStatus st = hsreader->Read(); if( !st.IsOK() ) { OnFaultWhileHandshaking( st ); return false; } if( st.code != suDone ) return true; return HandleHandShake( hsreader->ReleaseMsg() ); } //------------------------------------------------------------------------ // Handle the handshake message //------------------------------------------------------------------------ bool AsyncSocketHandler::HandleHandShake( std::unique_ptr msg ) { //-------------------------------------------------------------------------- // OK, we have a new message, let's deal with it; //-------------------------------------------------------------------------- pHandShakeData->in = msg.release(); XRootDStatus st = pTransport->HandShake( pHandShakeData.get(), *pChannelData ); //-------------------------------------------------------------------------- // Deal with wait responses //-------------------------------------------------------------------------- kXR_int32 waitSeconds = HandleWaitRsp( pHandShakeData->in ); delete pHandShakeData->in; pHandShakeData->in = 0; if( !st.IsOK() ) { OnFaultWhileHandshaking( st ); return false; } if( st.code == suRetry ) { //------------------------------------------------------------------------ // We are handling a wait response and the transport handler told // as to retry the request //------------------------------------------------------------------------ if( waitSeconds >=0 ) { time_t resendTime = ::time( 0 ) + waitSeconds; if( resendTime > pConnectionStarted + pConnectionTimeout ) { Log *log = DefaultEnv::GetLog(); log->Error( AsyncSockMsg, "[%s] Won't retry kXR_endsess request because would" "reach connection timeout.", pStreamName.c_str() ); OnFaultWhileHandshaking( XRootDStatus( stError, errSocketTimeout ) ); return false; } else { //-------------------------------------------------------------------- // We need to wait before replaying the request //-------------------------------------------------------------------- Log *log = DefaultEnv::GetLog(); log->Debug( AsyncSockMsg, "[%s] Received a wait response to endsess request, " "will wait for %d seconds before replaying the endsess request", pStreamName.c_str(), waitSeconds ); pHSWaitStarted = time( 0 ); pHSWaitSeconds = waitSeconds; } return true; } //------------------------------------------------------------------------ // We are re-sending a protocol request //------------------------------------------------------------------------ else if( pHandShakeData->out ) { return SendHSMsg(); } } //-------------------------------------------------------------------------- // If now is the time to enable encryption //-------------------------------------------------------------------------- if( !pSocket->IsEncrypted() && pTransport->NeedEncryption( pHandShakeData.get(), *pChannelData ) ) { XRootDStatus st = DoTlsHandShake(); if( !st.IsOK() ) return false; if ( st.code == suRetry ) return true; } //-------------------------------------------------------------------------- // Now prepare the next step of the hand-shake procedure //-------------------------------------------------------------------------- return HandShakeNextStep( st.IsOK() && st.code == suDone ); } //------------------------------------------------------------------------ // Prepare the next step of the hand-shake procedure //------------------------------------------------------------------------ bool AsyncSocketHandler::HandShakeNextStep( bool done ) { //-------------------------------------------------------------------------- // We successfully proceeded to the next step //-------------------------------------------------------------------------- ++pHandShakeData->step; //-------------------------------------------------------------------------- // The hand shake process is done //-------------------------------------------------------------------------- if( done ) { pHandShakeData.reset(); hswriter.reset(); hsreader.reset(); //------------------------------------------------------------------------ // Initialize the request writer & reader //------------------------------------------------------------------------ reqwriter.reset( new AsyncMsgWriter( *pTransport, *pSocket, pStreamName, *pStream, pSubStreamNum, *pChannelData ) ); rspreader.reset( new AsyncMsgReader( *pTransport, *pSocket, pStreamName, *pStream, pSubStreamNum ) ); XRootDStatus st; if( !(st = EnableUplink()).IsOK() ) { OnFaultWhileHandshaking( st ); return false; } pHandShakeDone = true; pStream->OnConnect( pSubStreamNum ); } //-------------------------------------------------------------------------- // The transport handler gave us something to write //-------------------------------------------------------------------------- else if( pHandShakeData->out ) { return SendHSMsg(); } return true; } //---------------------------------------------------------------------------- // Handle fault //---------------------------------------------------------------------------- void AsyncSocketHandler::OnFault( XRootDStatus st ) { Log *log = DefaultEnv::GetLog(); log->Error( AsyncSockMsg, "[%s] Socket error encountered: %s", pStreamName.c_str(), st.ToString().c_str() ); pStream->OnError( pSubStreamNum, st ); } //---------------------------------------------------------------------------- // Handle fault while handshaking //---------------------------------------------------------------------------- void AsyncSocketHandler::OnFaultWhileHandshaking( XRootDStatus st ) { Log *log = DefaultEnv::GetLog(); log->Error( AsyncSockMsg, "[%s] Socket error while handshaking: %s", pStreamName.c_str(), st.ToString().c_str() ); pStream->OnConnectError( pSubStreamNum, st ); } //---------------------------------------------------------------------------- // Handle write timeout //---------------------------------------------------------------------------- bool AsyncSocketHandler::OnWriteTimeout() { return pStream->OnWriteTimeout( pSubStreamNum ); } //---------------------------------------------------------------------------- // Handler read timeout //---------------------------------------------------------------------------- bool AsyncSocketHandler::OnReadTimeout() { return pStream->OnReadTimeout( pSubStreamNum ); } //---------------------------------------------------------------------------- // Handle timeout while handshaking //---------------------------------------------------------------------------- bool AsyncSocketHandler::OnTimeoutWhileHandshaking() { time_t now = time(0); if( now > pConnectionStarted+pConnectionTimeout ) { OnFaultWhileHandshaking( XRootDStatus( stError, errSocketTimeout ) ); return false; } return true; } //---------------------------------------------------------------------------- // Handle header corruption in case of kXR_status response //---------------------------------------------------------------------------- void AsyncSocketHandler::OnHeaderCorruption() { //-------------------------------------------------------------------------- // We need to force a socket error so this is handled in a similar way as // a stream t/o and all requests are retried //-------------------------------------------------------------------------- pStream->ForceError( XRootDStatus( stError, errSocketError ) ); } //---------------------------------------------------------------------------- // Carry out the TLS hand-shake //---------------------------------------------------------------------------- XRootDStatus AsyncSocketHandler::DoTlsHandShake() { Log *log = DefaultEnv::GetLog(); log->Debug( AsyncSockMsg, "[%s] TLS hand-shake exchange.", pStreamName.c_str() ); XRootDStatus st; if( !( st = pSocket->TlsHandShake( this, pUrl.GetHostName() ) ).IsOK() ) { pTlsHandShakeOngoing = false; OnFaultWhileHandshaking( st ); return st; } if( st.code == suRetry ) { pTlsHandShakeOngoing = true; return st; } pTlsHandShakeOngoing = false; log->Info( AsyncSockMsg, "[%s] TLS hand-shake done.", pStreamName.c_str() ); return st; } //---------------------------------------------------------------------------- // Handle read/write event if we are in the middle of a TLS hand-shake //---------------------------------------------------------------------------- inline bool AsyncSocketHandler::OnTLSHandShake() { XRootDStatus st = DoTlsHandShake(); if( !st.IsOK() ) return false; if ( st.code == suRetry ) return true; return HandShakeNextStep( pTransport->HandShakeDone( pHandShakeData.get(), *pChannelData ) ); } //---------------------------------------------------------------------------- // Prepare a HS writer for sending and enable uplink //---------------------------------------------------------------------------- bool AsyncSocketHandler::SendHSMsg() { if( !hswriter ) { OnFaultWhileHandshaking( XRootDStatus( stError, errInternal, 0, "HS writer object missing!" ) ); return false; } //-------------------------------------------------------------------------- // We only set a new HS message if this is not a replay due to kXR_wait //-------------------------------------------------------------------------- if( !pHSWaitSeconds ) { hswriter->Reset( pHandShakeData->out ); pHandShakeData->out = nullptr; } //-------------------------------------------------------------------------- // otherwise we replay the kXR_endsess request //-------------------------------------------------------------------------- else hswriter->Replay(); //-------------------------------------------------------------------------- // Enable writing so we can replay the HS message //-------------------------------------------------------------------------- XRootDStatus st; if( !(st = EnableUplink()).IsOK() ) { OnFaultWhileHandshaking( st ); return false; } return true; } kXR_int32 AsyncSocketHandler::HandleWaitRsp( Message *msg ) { // It would be more coherent if this could be done in the // transport layer, unfortunately the API does not allow it. kXR_int32 waitSeconds = -1; ServerResponse *rsp = (ServerResponse*)msg->GetBuffer(); if( rsp->hdr.status == kXR_wait ) waitSeconds = rsp->body.wait.seconds; return waitSeconds; } //---------------------------------------------------------------------------- // Check if HS wait time elapsed //---------------------------------------------------------------------------- bool AsyncSocketHandler::CheckHSWait() { time_t now = time( 0 ); if( now - pHSWaitStarted >= pHSWaitSeconds ) { Log *log = DefaultEnv::GetLog(); log->Debug( AsyncSockMsg, "[%s] The hand-shake wait time elapsed, will " "replay the endsess request.", pStreamName.c_str() ); if( !SendHSMsg() ) return false; //------------------------------------------------------------------------ // Make sure the wait state is reset //------------------------------------------------------------------------ pHSWaitSeconds = 0; pHSWaitStarted = 0; } return true; } //------------------------------------------------------------------------ // Get the IP stack //------------------------------------------------------------------------ std::string AsyncSocketHandler::GetIpStack() const { std::string ipstack( ( pSockAddr.isIPType( XrdNetAddr::IPType::IPv6 ) && !pSockAddr.isMapped() ) ? "IPv6" : "IPv4" ); return ipstack; } //------------------------------------------------------------------------ // Get IP address //------------------------------------------------------------------------ std::string AsyncSocketHandler::GetIpAddr() { char nameBuff[256]; pSockAddr.Format( nameBuff, sizeof(nameBuff), XrdNetAddrInfo::fmtAddr, XrdNetAddrInfo::noPort ); return nameBuff; } //------------------------------------------------------------------------ //! Get hostname //------------------------------------------------------------------------ std::string AsyncSocketHandler::GetHostName() { const char *cstr = pSockAddr.Name(); if( !cstr ) return std::string(); return cstr; } } xrootd-5.6.9/src/XrdCl/XrdClAsyncSocketHandler.hh000066400000000000000000000325741457266313600217070ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2012 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . //------------------------------------------------------------------------------ #ifndef __XRD_CL_ASYNC_SOCKET_HANDLER_HH__ #define __XRD_CL_ASYNC_SOCKET_HANDLER_HH__ #include "XrdCl/XrdClSocket.hh" #include "XrdCl/XrdClDefaultEnv.hh" #include "XrdCl/XrdClPoller.hh" #include "XrdCl/XrdClPostMasterInterfaces.hh" #include "XrdCl/XrdClTaskManager.hh" #include "XrdCl/XrdClXRootDResponses.hh" #include "XrdCl/XrdClURL.hh" #include "XrdCl/XrdClAsyncMsgReader.hh" #include "XrdCl/XrdClAsyncHSReader.hh" #include "XrdCl/XrdClAsyncMsgWriter.hh" #include "XrdCl/XrdClAsyncHSWriter.hh" #include "XrdOuc/XrdOucCompiler.hh" namespace XrdCl { class Stream; //---------------------------------------------------------------------------- //! Utility class handling asynchronous socket interactions and forwarding //! events to the parent stream. //---------------------------------------------------------------------------- class AsyncSocketHandler: public SocketHandler { public: //------------------------------------------------------------------------ //! Constructor //------------------------------------------------------------------------ AsyncSocketHandler( const URL &url, Poller *poller, TransportHandler *transport, AnyObject *channelData, uint16_t subStreamNum, Stream *strm ); //------------------------------------------------------------------------ //! Destructor //------------------------------------------------------------------------ ~AsyncSocketHandler(); //------------------------------------------------------------------------ //! Set address //------------------------------------------------------------------------ void SetAddress( const XrdNetAddr &address ) { pSockAddr = address; } //------------------------------------------------------------------------ //! Get the address that the socket is connected to //------------------------------------------------------------------------ const XrdNetAddr &GetAddress() const { return pSockAddr; } //------------------------------------------------------------------------ //! Connect to the currently set address //------------------------------------------------------------------------ XRootDStatus Connect( time_t timeout ); //------------------------------------------------------------------------ //! Close the connection //------------------------------------------------------------------------ XRootDStatus Close(); //------------------------------------------------------------------------ //! Handle a socket event //------------------------------------------------------------------------ virtual void Event( uint8_t type, XrdCl::Socket */*socket*/ ); //------------------------------------------------------------------------ //! Enable uplink //------------------------------------------------------------------------ XRootDStatus EnableUplink() { if( !pPoller->EnableWriteNotification( pSocket, true, pTimeoutResolution ) ) return XRootDStatus( stFatal, errPollerError ); return XRootDStatus(); } //------------------------------------------------------------------------ //! Disable uplink //------------------------------------------------------------------------ XRootDStatus DisableUplink() { if( !pPoller->EnableWriteNotification( pSocket, false ) ) return XRootDStatus( stFatal, errPollerError ); return XRootDStatus(); } //------------------------------------------------------------------------ //! Get stream name //------------------------------------------------------------------------ const std::string &GetStreamName() { return pStreamName; } //------------------------------------------------------------------------ //! Get timestamp of last registered socket activity //------------------------------------------------------------------------ time_t GetLastActivity() { return pLastActivity; } //------------------------------------------------------------------------ //! Get the IP stack //------------------------------------------------------------------------ std::string GetIpStack() const; //------------------------------------------------------------------------ //! Get IP address //------------------------------------------------------------------------ std::string GetIpAddr(); //------------------------------------------------------------------------ //! Get hostname //------------------------------------------------------------------------ std::string GetHostName(); protected: //------------------------------------------------------------------------ //! Convert Stream object and sub-stream number to stream name //------------------------------------------------------------------------ static std::string ToStreamName( const URL &url, uint16_t strmnb ); //------------------------------------------------------------------------ // Connect returned //------------------------------------------------------------------------ virtual bool OnConnectionReturn() XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ // Got a write readiness event //------------------------------------------------------------------------ bool OnWrite() XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ // Got a write readiness event while handshaking //------------------------------------------------------------------------ bool OnWriteWhileHandshaking() XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ // Got a read readiness event //------------------------------------------------------------------------ bool OnRead() XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ // Got a read readiness event while handshaking //------------------------------------------------------------------------ bool OnReadWhileHandshaking() XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ // Handle the handshake message //------------------------------------------------------------------------ bool HandleHandShake( std::unique_ptr msg ) XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ // Prepare the next step of the hand-shake procedure //------------------------------------------------------------------------ bool HandShakeNextStep( bool done ) XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ // Handle fault //------------------------------------------------------------------------ void OnFault( XRootDStatus st ); //------------------------------------------------------------------------ // Handle fault while handshaking //------------------------------------------------------------------------ void OnFaultWhileHandshaking( XRootDStatus st ); //------------------------------------------------------------------------ // Handle write timeout event //------------------------------------------------------------------------ bool OnWriteTimeout() XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ // Handle read timeout event //------------------------------------------------------------------------ bool OnReadTimeout() XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ // Handle timeout event while handshaking //------------------------------------------------------------------------ bool OnTimeoutWhileHandshaking() XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ // Handle header corruption in case of kXR_status response //------------------------------------------------------------------------ void OnHeaderCorruption(); //------------------------------------------------------------------------ // Carry out the TLS hand-shake // // The TLS hand-shake is being initiated in HandleHandShake() by calling // Socket::TlsHandShake(), however it returns suRetry the TLS hand-shake // needs to be followed up by OnTlsHandShake(). // // However, once the TLS connection has been established the server may // decide to redo the TLS hand-shake at any time, this operation is handled // under the hood by read and write requests and facilitated by // Socket::MapEvent() //------------------------------------------------------------------------ XRootDStatus DoTlsHandShake(); //------------------------------------------------------------------------ // Handle read/write event if we are in the middle of a TLS hand-shake //------------------------------------------------------------------------ // Handle read/write event if we are in the middle of a TLS hand-shake bool OnTLSHandShake() XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ // Prepare a HS writer for sending and enable uplink //------------------------------------------------------------------------ bool SendHSMsg() XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ // Extract the value of a wait response // // @param rsp : the server response // @return : if rsp is a wait response then its value // otherwise -1 //------------------------------------------------------------------------ inline kXR_int32 HandleWaitRsp( Message *rsp ); //------------------------------------------------------------------------ // Check if HS wait time elapsed //------------------------------------------------------------------------ bool CheckHSWait() XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ // Handler for read related socket events //------------------------------------------------------------------------ inline bool EventRead( uint8_t type ) XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ // Handler for write related socket events //------------------------------------------------------------------------ inline bool EventWrite( uint8_t type ) XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ // Data members //------------------------------------------------------------------------ Poller *pPoller; TransportHandler *pTransport; AnyObject *pChannelData; uint16_t pSubStreamNum; Stream *pStream; std::string pStreamName; Socket *pSocket; XrdNetAddr pSockAddr; std::unique_ptr pHandShakeData; bool pHandShakeDone; uint16_t pTimeoutResolution; time_t pConnectionStarted; time_t pConnectionTimeout; time_t pLastActivity; time_t pHSWaitStarted; time_t pHSWaitSeconds; URL pUrl; bool pTlsHandShakeOngoing; std::unique_ptr hswriter; std::unique_ptr rspreader; std::unique_ptr hsreader; std::unique_ptr reqwriter; }; } #endif // __XRD_CL_ASYNC_SOCKET_HANDLER_HH__ xrootd-5.6.9/src/XrdCl/XrdClAsyncVectorReader.hh000066400000000000000000000270051457266313600215370ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2012 by European Organization for Nuclear Research (CERN) // Author: Michal Simon //------------------------------------------------------------------------------ // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . //------------------------------------------------------------------------------ #ifndef SRC_XRDCL_XRDCLASYNCVECTORREADER_HH_ #define SRC_XRDCL_XRDCLASYNCVECTORREADER_HH_ #include "XrdCl/XrdClXRootDResponses.hh" #include "XrdCl/XrdClSocket.hh" #include "XrdClAsyncRawReaderIntfc.hh" namespace XrdCl { //---------------------------------------------------------------------------- //! Object for reading out data from the VectorRead response //---------------------------------------------------------------------------- class AsyncVectorReader : public AsyncRawReaderIntfc { public: //------------------------------------------------------------------------ //! Constructor //! //! @param url : channel URL //------------------------------------------------------------------------ AsyncVectorReader( const URL &url, const Message &request ) : AsyncRawReaderIntfc( url, request ), rdlstoff( 0 ), rdlstlen( 0 ) { memset( &rdlst, 0, sizeof( readahead_list ) ); } //------------------------------------------------------------------------ //! Readout raw data from socket //! //! @param socket : the socket //! @param btsret : number of bytes read //! @return : operation status //------------------------------------------------------------------------ XRootDStatus Read( Socket &socket, uint32_t &btsret ) { Log *log = DefaultEnv::GetLog(); while( true ) { switch( readstage ) { //------------------------------------------------------------------ // Prepare to readout a new response //------------------------------------------------------------------ case ReadStart: { msgbtsrd = 0; rdlstoff = 0; rdlstlen = sizeof( readahead_list ); readstage = ReadRdLst; continue; } //------------------------------------------------------------------ // Readout the read_list //------------------------------------------------------------------ case ReadRdLst: { //---------------------------------------------------------------- // We cannot afford to read the next header from the stream // because we will cross the message boundary //---------------------------------------------------------------- if( msgbtsrd + rdlstlen > dlen ) { uint32_t btsleft = dlen - msgbtsrd; log->Error( XRootDMsg, "[%s] VectorReader: No enough data to read " "another chunk header. Discarding %d bytes.", url.GetHostId().c_str(), btsleft ); readstage = ReadDiscard; continue; } //---------------------------------------------------------------- // Let's readout the read list record from the socket //---------------------------------------------------------------- uint32_t btsrd = 0; char *buff = reinterpret_cast( &rdlst ); Status st = ReadBytesAsync( socket, buff + rdlstoff, rdlstlen, btsrd ); rdlstoff += btsrd; rdlstlen -= btsrd; msgbtsrd += btsrd; btsret += btsrd; if( !st.IsOK() || st.code == suRetry ) return st; //---------------------------------------------------------------- // We have a complete read list record, now we need to marshal it //---------------------------------------------------------------- rdlst.rlen = ntohl( rdlst.rlen ); rdlst.offset = ntohll( rdlst.offset ); choff = 0; chlen = rdlst.rlen; //---------------------------------------------------------------- // Find the buffer corresponding to the chunk //---------------------------------------------------------------- bool chfound = false; for( size_t i = 0; i < chunks->size(); ++i ) { if( ( *chunks )[i].offset == uint64_t( rdlst.offset ) && ( *chunks )[i].length == uint32_t( rdlst.rlen ) ) { chfound = true; chidx = i; break; } } //---------------------------------------------------------------- // If the chunk was not found this is a bogus response, switch // to discard mode //---------------------------------------------------------------- if( !chfound ) { log->Error( XRootDMsg, "[%s] VectorReader: Impossible to find chunk " "buffer corresponding to %d bytes at %ld", url.GetHostId().c_str(), rdlst.rlen, rdlst.offset ); readstage = ReadDiscard; continue; } readstage = ReadRaw; continue; } //------------------------------------------------------------------ // Readout the raw data //------------------------------------------------------------------ case ReadRaw: { //---------------------------------------------------------------- // The chunk was found, but reading all the data will cross the // message boundary //---------------------------------------------------------------- if( msgbtsrd + chlen > dlen ) { uint32_t btsleft = dlen - msgbtsrd; log->Error( XRootDMsg, "[%s] VectorReader: Malformed chunk header: " "reading %d bytes from message would cross the message " "boundary, discarding %d bytes.", url.GetHostId().c_str(), rdlst.rlen, btsleft ); chstatus[chidx].sizeerr = true; readstage = ReadDiscard; continue; } //---------------------------------------------------------------- // Readout the raw data from the socket //---------------------------------------------------------------- uint32_t btsrd = 0; char *buff = static_cast( ( *chunks )[chidx].buffer ); Status st = ReadBytesAsync( socket, buff + choff, chlen, btsrd ); choff += btsrd; chlen -= btsrd; msgbtsrd += btsrd; rawbtsrd += btsrd; btsret += btsrd; if( !st.IsOK() || st.code == suRetry ) return st; log->Dump( XRootDMsg, "[%s] VectorReader: read buffer for chunk %d@%ld", url.GetHostId().c_str(), rdlst.rlen, rdlst.offset ); //---------------------------------------------------------------- // Mark chunk as done //---------------------------------------------------------------- chstatus[chidx].done = true; //---------------------------------------------------------------- // There is still data to be read, we need to readout the next // read list record. //---------------------------------------------------------------- if( msgbtsrd < dlen ) { rdlstoff = 0; rdlstlen = sizeof( readahead_list ); readstage = ReadRdLst; continue; } readstage = ReadDone; continue; } //------------------------------------------------------------------ // We've had an error and we are in the discarding mode //------------------------------------------------------------------ case ReadDiscard: { // Just drop the connection, we don't know if the stream is sane // anymore. Recover with a reconnect. return XRootDStatus( stError, errCorruptedHeader ); } //------------------------------------------------------------------ // Finalize the read //------------------------------------------------------------------ case ReadDone: { chidx = 0; choff = 0; chlen = 0; rdlstoff = 0; rdlstlen = 0; break; } //------------------------------------------------------------------ // Others should not happen //------------------------------------------------------------------ default : return XRootDStatus( stError, errInternal ); } // just in case break; } //---------------------------------------------------------------------- // We are done //---------------------------------------------------------------------- return XRootDStatus(); } //------------------------------------------------------------------------ //! Get the response //------------------------------------------------------------------------ XRootDStatus GetResponse( AnyObject *&response ) { //-------------------------------------------------------------------------- // See if all the chunks are OK and put them in the response //-------------------------------------------------------------------------- std::unique_ptr ptr( new VectorReadInfo() ); for( uint32_t i = 0; i < chunks->size(); ++i ) { if( !chstatus[i].done ) return Status( stFatal, errInvalidResponse ); ptr->GetChunks().emplace_back( ( *chunks )[i].offset, ( *chunks )[i].length, ( *chunks )[i].buffer ); } ptr->SetSize( rawbtsrd ); response = new AnyObject(); response->Set( ptr.release() ); return XRootDStatus(); } private: size_t rdlstoff; //< offset within the current read_list readahead_list rdlst; //< the readahead list for the current chunk size_t rdlstlen; //< bytes left to be readout into read list }; } /* namespace XrdCl */ #endif /* SRC_XRDCL_XRDCLASYNCVECTORREADER_HH_ */ xrootd-5.6.9/src/XrdCl/XrdClBuffer.hh000066400000000000000000000216331457266313600173660ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2012 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . //------------------------------------------------------------------------------ #ifndef __XRD_CL_BUFFER_HH__ #define __XRD_CL_BUFFER_HH__ #include #include #include #include #include namespace XrdCl { //---------------------------------------------------------------------------- //! Binary blob representation //---------------------------------------------------------------------------- class Buffer { public: //------------------------------------------------------------------------ //! Constructor //------------------------------------------------------------------------ Buffer( uint32_t size = 0 ): pBuffer(0), pSize(0), pCursor(0) { if( size ) { Allocate( size ); } } //------------------------------------------------------------------------ //! Move Constructor //------------------------------------------------------------------------ Buffer( Buffer &&buffer ) { Steal( std::move( buffer ) ); } //------------------------------------------------------------------------ //! Move assignment operator //------------------------------------------------------------------------ Buffer& operator=( Buffer && buffer ) { Steal( std::move( buffer ) ); return *this; } //------------------------------------------------------------------------ //! Destructor //------------------------------------------------------------------------ virtual ~Buffer() { Free(); } //------------------------------------------------------------------------ //! Get the message buffer //------------------------------------------------------------------------ const char *GetBuffer( uint32_t offset = 0 ) const { return pBuffer+offset; } //------------------------------------------------------------------------ //! Get the message buffer //------------------------------------------------------------------------ char *GetBuffer( uint32_t offset = 0 ) { return pBuffer+offset; } //------------------------------------------------------------------------ //! Reallocate the buffer to a new location of a given size //------------------------------------------------------------------------ void ReAllocate( uint32_t size ) { pBuffer = (char *)realloc( pBuffer, size ); if( !pBuffer ) throw std::bad_alloc(); pSize = size; } //------------------------------------------------------------------------ //! Free the buffer //------------------------------------------------------------------------ void Free() { free( pBuffer ); pBuffer = 0; pSize = 0; pCursor = 0; } //------------------------------------------------------------------------ //! Allocate the buffer //------------------------------------------------------------------------ void Allocate( uint32_t size ) { if( !size ) return; pBuffer = (char *)malloc( size ); if( !pBuffer ) throw std::bad_alloc(); pSize = size; } //------------------------------------------------------------------------ //! Zero //------------------------------------------------------------------------ void Zero() { memset( pBuffer, 0, pSize ); } //------------------------------------------------------------------------ //! Get the size of the message //------------------------------------------------------------------------ uint32_t GetSize() const { return pSize; } //------------------------------------------------------------------------ //! Get append cursor //------------------------------------------------------------------------ uint32_t GetCursor() const { return pCursor; } //------------------------------------------------------------------------ //! Set the cursor //------------------------------------------------------------------------ void SetCursor( uint32_t cursor ) { pCursor = cursor; } //------------------------------------------------------------------------ //! Advance the cursor //------------------------------------------------------------------------ void AdvanceCursor( uint32_t delta ) { pCursor += delta; } //------------------------------------------------------------------------ //! Append data at the position pointed to by the append cursor //------------------------------------------------------------------------ void Append( const char *buffer, uint32_t size ) { uint32_t remaining = pSize-pCursor; if( remaining < size ) ReAllocate( pCursor+size ); memcpy( pBuffer+pCursor, buffer, size ); pCursor += size; } //------------------------------------------------------------------------ //! Append data at the given offset //------------------------------------------------------------------------ void Append( const char *buffer, uint32_t size, uint32_t offset ) { uint32_t remaining = pSize-offset; if( remaining < size ) ReAllocate( offset+size ); memcpy( pBuffer+offset, buffer, size ); } //------------------------------------------------------------------------ //! Get the buffer pointer at the append cursor //------------------------------------------------------------------------ char *GetBufferAtCursor() { return GetBuffer( pCursor ); } //------------------------------------------------------------------------ //! Get the buffer pointer at the append cursor //------------------------------------------------------------------------ const char *GetBufferAtCursor() const { return GetBuffer( pCursor ); } //------------------------------------------------------------------------ //! Fill the buffer from a string //------------------------------------------------------------------------ void FromString( const std::string str ) { ReAllocate( str.length()+1 ); memcpy( pBuffer, str.c_str(), str.length() ); pBuffer[str.length()] = 0; } //------------------------------------------------------------------------ //! Convert the buffer to a string //------------------------------------------------------------------------ std::string ToString() const { char *bf = new char[pSize+1]; bf[pSize] = 0; memcpy( bf, pBuffer, pSize ); std::string tmp = bf; delete [] bf; return tmp; } //------------------------------------------------------------------------ //! Grab a buffer allocated outside //------------------------------------------------------------------------ void Grab( char *buffer, uint32_t size ) { Free(); pBuffer = buffer; pSize = size; } //------------------------------------------------------------------------ //! Release the buffer //------------------------------------------------------------------------ char *Release() { char *buffer = pBuffer; pBuffer = 0; pSize = 0; pCursor = 0; return buffer; } protected: void Steal( Buffer &&buffer ) { pBuffer = buffer.pBuffer; buffer.pBuffer = 0; pSize = buffer.pSize; buffer.pSize = 0; pCursor = buffer.pCursor; buffer.pCursor = 0; } private: Buffer( const Buffer& ); Buffer& operator=( const Buffer& ); char *pBuffer; uint32_t pSize; uint32_t pCursor; }; } #endif // __XRD_CL_BUFFER_HH__ xrootd-5.6.9/src/XrdCl/XrdClChannel.cc000066400000000000000000000214501457266313600175100ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2014 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #include "XrdCl/XrdClChannel.hh" #include "XrdCl/XrdClDefaultEnv.hh" #include "XrdCl/XrdClStream.hh" #include "XrdCl/XrdClSocket.hh" #include "XrdCl/XrdClConstants.hh" #include "XrdCl/XrdClLog.hh" #include "XrdCl/XrdClRedirectorRegistry.hh" #include "XrdCl/XrdClXRootDTransport.hh" #include "XrdSys/XrdSysPthread.hh" #include namespace XrdCl { class TickGeneratorTask: public XrdCl::Task { public: //------------------------------------------------------------------------ // Constructor //------------------------------------------------------------------------ TickGeneratorTask( XrdCl::Channel *channel, const std::string &hostId ): pChannel( channel ) { std::string name = "TickGeneratorTask for: "; name += hostId; SetName( name ); } //------------------------------------------------------------------------ // Run the task //------------------------------------------------------------------------ time_t Run( time_t now ) { XrdSysMutexHelper lck( pMtx ); if( !pChannel ) return 0; using namespace XrdCl; pChannel->Tick( now ); Env *env = DefaultEnv::GetEnv(); int timeoutResolution = DefaultTimeoutResolution; env->GetInt( "TimeoutResolution", timeoutResolution ); return now+timeoutResolution; } void Invalidate() { XrdSysMutexHelper lck( pMtx ); pChannel = 0; } private: XrdCl::Channel *pChannel; XrdSysMutex pMtx; }; //---------------------------------------------------------------------------- // Constructor //---------------------------------------------------------------------------- Channel::Channel( const URL &url, Poller *poller, TransportHandler *transport, TaskManager *taskManager, JobManager *jobManager, const URL &prefurl ): pUrl( url.GetHostId() ), pPoller( poller ), pTransport( transport ), pTaskManager( taskManager ), pTickGenerator( 0 ), pJobManager( jobManager ) { Env *env = DefaultEnv::GetEnv(); Log *log = DefaultEnv::GetLog(); int timeoutResolution = DefaultTimeoutResolution; env->GetInt( "TimeoutResolution", timeoutResolution ); pTransport->InitializeChannel( url, pChannelData ); log->Debug( PostMasterMsg, "Creating new channel to: %s", url.GetChannelId().c_str() ); pUrl.SetParams( url.GetParams() ); pUrl.SetProtocol( url.GetProtocol() ); //-------------------------------------------------------------------------- // Create the stream //-------------------------------------------------------------------------- pStream = new Stream( &pUrl, prefurl ); pStream->SetTransport( transport ); pStream->SetPoller( poller ); pStream->SetIncomingQueue( &pIncoming ); pStream->SetTaskManager( taskManager ); pStream->SetJobManager( jobManager ); pStream->SetChannelData( &pChannelData ); pStream->Initialize(); //-------------------------------------------------------------------------- // Register the task generating timeout events //-------------------------------------------------------------------------- pTickGenerator = new TickGeneratorTask( this, pUrl.GetChannelId() ); pTaskManager->RegisterTask( pTickGenerator, ::time(0)+timeoutResolution ); } //---------------------------------------------------------------------------- // Destructor //---------------------------------------------------------------------------- Channel::~Channel() { pTickGenerator->Invalidate(); delete pStream; pTransport->FinalizeChannel( pChannelData ); } //---------------------------------------------------------------------------- // Send the message asynchronously //---------------------------------------------------------------------------- XRootDStatus Channel::Send( Message *msg, MsgHandler *handler, bool stateful, time_t expires ) { return pStream->Send( msg, handler, stateful, expires ); } //---------------------------------------------------------------------------- // Handle a time event //---------------------------------------------------------------------------- void Channel::Tick( time_t now ) { pStream->Tick( now ); } //---------------------------------------------------------------------------- // Force disconnect of all streams //---------------------------------------------------------------------------- Status Channel::ForceDisconnect() { //-------------------------------------------------------------------------- // Disconnect and recreate the streams //-------------------------------------------------------------------------- pStream->ForceError( Status( stError, errOperationInterrupted ) ); return Status(); } //---------------------------------------------------------------------------- // Force reconnect //---------------------------------------------------------------------------- Status Channel::ForceReconnect() { pStream->ForceConnect(); return Status(); } //------------------------------------------------------------------------ // Get the number of connected data streams //------------------------------------------------------------------------ uint16_t Channel::NbConnectedStrm() { return XRootDTransport::NbConnectedStrm( pChannelData ); } //------------------------------------------------------------------------ // Set the on-connect handler for data streams //------------------------------------------------------------------------ void Channel::SetOnDataConnectHandler( std::shared_ptr &onConnJob ) { pStream->SetOnDataConnectHandler( onConnJob ); } //------------------------------------------------------------------------ // Check if channel can be collapsed using given URL //------------------------------------------------------------------------ bool Channel::CanCollapse( const URL &url ) { return pStream->CanCollapse( url ); } //------------------------------------------------------------------------ // Decrement file object instance count bound to this channel //------------------------------------------------------------------------ void Channel::DecFileInstCnt() { pTransport->DecFileInstCnt( pChannelData ); } //---------------------------------------------------------------------------- // Query the transport handler //---------------------------------------------------------------------------- Status Channel::QueryTransport( uint16_t query, AnyObject &result ) { if( query < 2000 ) return pTransport->Query( query, result, pChannelData ); return pStream->Query( query, result ); } //---------------------------------------------------------------------------- // Register channel event handler //---------------------------------------------------------------------------- void Channel::RegisterEventHandler( ChannelEventHandler *handler ) { pStream->RegisterEventHandler( handler ); } //------------------------------------------------------------------------ // Remove a channel event handler //------------------------------------------------------------------------ void Channel::RemoveEventHandler( ChannelEventHandler *handler ) { pStream->RemoveEventHandler( handler ); } } xrootd-5.6.9/src/XrdCl/XrdClChannel.hh000066400000000000000000000164371457266313600175330ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2012 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . //------------------------------------------------------------------------------ #ifndef __XRD_CL_POST_CHANNEL_HH__ #define __XRD_CL_POST_CHANNEL_HH__ #include #include #include #include #include "XrdCl/XrdClStatus.hh" #include "XrdCl/XrdClURL.hh" #include "XrdCl/XrdClPoller.hh" #include "XrdCl/XrdClInQueue.hh" #include "XrdCl/XrdClPostMasterInterfaces.hh" #include "XrdCl/XrdClAnyObject.hh" #include "XrdCl/XrdClTaskManager.hh" #include "XrdSys/XrdSysPthread.hh" namespace XrdCl { class Stream; class JobManager; class VirtualRedirector; class TickGeneratorTask; class Job; //---------------------------------------------------------------------------- //! A communication channel between the client and the server //---------------------------------------------------------------------------- class Channel { public: //------------------------------------------------------------------------ //! Constructor //! //! @param url address of the server to connect to //! @param poller poller object to be used for non-blocking IO //! @param transport protocol specific transport handler //! @param taskManager async task handler to be used by the channel //! @param jobManager worker thread handler to be used by the channel //------------------------------------------------------------------------ Channel( const URL &url, Poller *poller, TransportHandler *transport, TaskManager *taskManager, JobManager *jobManager, const URL &prefurl = URL() ); //------------------------------------------------------------------------ //! Destructor //------------------------------------------------------------------------ ~Channel(); //------------------------------------------------------------------------ //! Get the URL //------------------------------------------------------------------------ const URL &GetURL() const { return pUrl; } //------------------------------------------------------------------------ //! Send the message asynchronously - the message is inserted into the //! send queue and a listener is called when the message is successfully //! pushed through the wire or when the timeout elapses //! //! @param msg message to be sent //! @param handler handler to be notified about the status //! @param stateful physical stream disconnection causes an error //! @param expires unix timestamp after which a failure is reported //! to the listener //! @return success if the message was successfully inserted //! into the send queues, failure otherwise //------------------------------------------------------------------------ XRootDStatus Send( Message *msg, MsgHandler *handler, bool stateful, time_t expires ); //------------------------------------------------------------------------ //! Query the transport handler //! //! @param query the query as defined in the TransportQuery struct or //! others that may be recognized by the protocol transport //! @param result the result of the query //! @return status of the query //------------------------------------------------------------------------ Status QueryTransport( uint16_t query, AnyObject &result ); //------------------------------------------------------------------------ //! Register channel event handler //------------------------------------------------------------------------ void RegisterEventHandler( ChannelEventHandler *handler ); //------------------------------------------------------------------------ //! Remove a channel event handler //------------------------------------------------------------------------ void RemoveEventHandler( ChannelEventHandler *handler ); //------------------------------------------------------------------------ //! Handle a time event //------------------------------------------------------------------------ void Tick( time_t now ); //------------------------------------------------------------------------ //! Force disconnect of all streams //------------------------------------------------------------------------ Status ForceDisconnect(); //------------------------------------------------------------------------ //! Force reconnect //------------------------------------------------------------------------ Status ForceReconnect(); //------------------------------------------------------------------------ //! Get the number of connected data streams //------------------------------------------------------------------------ uint16_t NbConnectedStrm(); //------------------------------------------------------------------------ //! Set the on-connect handler for data streams //------------------------------------------------------------------------ void SetOnDataConnectHandler( std::shared_ptr &onConnJob ); //------------------------------------------------------------------------ //! @return : true if this channel can be collapsed using this URL, false //! otherwise //------------------------------------------------------------------------ bool CanCollapse( const URL &url ); //------------------------------------------------------------------------ //! Decrement file object instance count bound to this channel //------------------------------------------------------------------------ void DecFileInstCnt(); private: URL pUrl; Poller *pPoller; TransportHandler *pTransport; TaskManager *pTaskManager; Stream *pStream; XrdSysMutex pMutex; AnyObject pChannelData; InQueue pIncoming; TickGeneratorTask *pTickGenerator; JobManager *pJobManager; }; } #endif // __XRD_CL_POST_CHANNEL_HH__ xrootd-5.6.9/src/XrdCl/XrdClChannelHandlerList.cc000066400000000000000000000052151457266313600216430ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2012 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . //------------------------------------------------------------------------------ #include "XrdCl/XrdClChannelHandlerList.hh" #include "XrdCl/XrdClPostMasterInterfaces.hh" namespace XrdCl { //---------------------------------------------------------------------------- // Add a channel event handler //---------------------------------------------------------------------------- void ChannelHandlerList::AddHandler( ChannelEventHandler *handler ) { XrdSysMutexHelper scopedLock( pMutex ); pHandlers.push_back( handler ); } //---------------------------------------------------------------------------- // Remove the channel event handler //---------------------------------------------------------------------------- void ChannelHandlerList::RemoveHandler( ChannelEventHandler *handler ) { XrdSysMutexHelper scopedLock( pMutex ); std::list::iterator it; for( it = pHandlers.begin(); it != pHandlers.end(); ++it ) { if( *it == handler ) { pHandlers.erase( it ); return; } } } //---------------------------------------------------------------------------- // Report an event to the channel event handlers //---------------------------------------------------------------------------- void ChannelHandlerList::ReportEvent( ChannelEventHandler::ChannelEvent event, Status status ) { XrdSysMutexHelper scopedLock( pMutex ); std::list::iterator it; for( it = pHandlers.begin(); it != pHandlers.end(); ) { bool st = (*it)->OnChannelEvent( event, status ); if( !st ) it = pHandlers.erase( it ); else ++it; } } } xrootd-5.6.9/src/XrdCl/XrdClChannelHandlerList.hh000066400000000000000000000047631457266313600216640ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2012 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . //------------------------------------------------------------------------------ #ifndef __XRD_CL_CHANNEL_HANDLER_LIST_HH__ #define __XRD_CL_CHANNEL_HANDLER_LIST_HH__ #include #include #include "XrdCl/XrdClPostMasterInterfaces.hh" #include "XrdCl/XrdClStatus.hh" #include namespace XrdCl { //---------------------------------------------------------------------------- //! A helper for handling channel event handlers //---------------------------------------------------------------------------- class ChannelHandlerList { public: //------------------------------------------------------------------------ //! Add a channel event handler //------------------------------------------------------------------------ void AddHandler( ChannelEventHandler *handler ); //------------------------------------------------------------------------ //! Remove the channel event handler //------------------------------------------------------------------------ void RemoveHandler( ChannelEventHandler *handler ); //------------------------------------------------------------------------ //! Report an event to the channel event handlers //------------------------------------------------------------------------ void ReportEvent( ChannelEventHandler::ChannelEvent event, Status status ); private: std::list pHandlers; XrdSysMutex pMutex; }; } #endif // __XRD_CL_CHANNEL_HANDLER_LIST_HH__ xrootd-5.6.9/src/XrdCl/XrdClCheckSumHelper.hh000066400000000000000000000131061457266313600210130ustar00rootroot00000000000000/* * CheckSumHelper.hh * * Created on: Sep 20, 2021 * Author: simonm */ #ifndef SRC_XRDCL_XRDCLCHECKSUMHELPER_HH_ #define SRC_XRDCL_XRDCLCHECKSUMHELPER_HH_ #include "XrdCl/XrdClXRootDResponses.hh" #include "XrdCl/XrdClConstants.hh" #include "XrdCks/XrdCksCalc.hh" #include "XrdCl/XrdClCheckSumManager.hh" #include "XrdCl/XrdClLog.hh" #include "XrdCl/XrdClUtils.hh" #include namespace XrdCl { //---------------------------------------------------------------------------- //! Check sum helper for stdio //---------------------------------------------------------------------------- class CheckSumHelper { public: //------------------------------------------------------------------------ //! Constructor //------------------------------------------------------------------------ CheckSumHelper( const std::string &name, const std::string &ckSumType ): pName( name ), pCkSumType( ckSumType ), pCksCalcObj( 0 ) {}; //------------------------------------------------------------------------ //! Destructor //------------------------------------------------------------------------ virtual ~CheckSumHelper() { delete pCksCalcObj; } //------------------------------------------------------------------------ //! Initialize //------------------------------------------------------------------------ XRootDStatus Initialize() { if( pCkSumType.empty() ) return XRootDStatus(); Log *log = DefaultEnv::GetLog(); CheckSumManager *cksMan = DefaultEnv::GetCheckSumManager(); if( !cksMan ) { log->Error( UtilityMsg, "Unable to get the checksum manager" ); return XRootDStatus( stError, errInternal ); } pCksCalcObj = cksMan->GetCalculator( pCkSumType ); if( !pCksCalcObj ) { log->Error( UtilityMsg, "Unable to get a calculator for %s", pCkSumType.c_str() ); return XRootDStatus( stError, errCheckSumError ); } return XRootDStatus(); } //------------------------------------------------------------------------ // Update the checksum //------------------------------------------------------------------------ void Update( const void *buffer, uint32_t size ) { if( pCksCalcObj ) pCksCalcObj->Update( (const char *)buffer, size ); } //------------------------------------------------------------------------ // Get checksum //------------------------------------------------------------------------ XRootDStatus GetCheckSum( std::string &checkSum, std::string &checkSumType ) { using namespace XrdCl; Log *log = DefaultEnv::GetLog(); int calcSize = 0; auto st = GetCheckSumImpl( checkSumType, calcSize ); if( !st.IsOK() ) return st; //---------------------------------------------------------------------- // Response //---------------------------------------------------------------------- XrdCksData ckSum; ckSum.Set( checkSumType.c_str() ); ckSum.Set( (void*)pCksCalcObj->Final(), calcSize ); char *cksBuffer = new char[265]; ckSum.Get( cksBuffer, 256 ); checkSum = checkSumType + ":"; checkSum += Utils::NormalizeChecksum( checkSumType, cksBuffer ); delete [] cksBuffer; log->Dump( UtilityMsg, "Checksum for %s is: %s", pName.c_str(), checkSum.c_str() ); return XRootDStatus(); } template XRootDStatus GetRawCheckSum( const std::string &checkSumType, T &value ) { int calcSize = 0; auto st = GetCheckSumImpl( checkSumType, calcSize ); if( !st.IsOK() ) return st; if( sizeof( T ) != calcSize ) return XRootDStatus( stError, errInvalidArgs, 0, "checksum size mismatch" ); value = *reinterpret_cast( pCksCalcObj->Final() ); return XRootDStatus(); } const std::string& GetType() { return pCkSumType; } private: //------------------------------------------------------------------------ // Get checksum //------------------------------------------------------------------------ inline XRootDStatus GetCheckSumImpl( const std::string &checkSumType, int &calcSize ) { using namespace XrdCl; Log *log = DefaultEnv::GetLog(); //---------------------------------------------------------------------- // Sanity check //---------------------------------------------------------------------- if( !pCksCalcObj ) { log->Error( UtilityMsg, "Calculator for %s was not initialized", pCkSumType.c_str() ); return XRootDStatus( stError, errCheckSumError ); } std::string calcType = pCksCalcObj->Type( calcSize ); if( calcType != checkSumType ) { log->Error( UtilityMsg, "Calculated checksum: %s, requested " "checksum: %s", pCkSumType.c_str(), checkSumType.c_str() ); return XRootDStatus( stError, errCheckSumError ); } return XRootDStatus(); } std::string pName; std::string pCkSumType; XrdCksCalc *pCksCalcObj; }; } #endif /* SRC_XRDCL_XRDCLCHECKSUMHELPER_HH_ */ xrootd-5.6.9/src/XrdCl/XrdClCheckSumManager.cc000066400000000000000000000133231457266313600211350ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2012 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . //------------------------------------------------------------------------------ #include "XrdCl/XrdClCheckSumManager.hh" #include "XrdCl/XrdClLog.hh" #include "XrdCl/XrdClUtils.hh" #include "XrdCl/XrdClDefaultEnv.hh" #include "XrdCl/XrdClConstants.hh" #include "XrdCks/XrdCksCalc.hh" #include "XrdCks/XrdCksLoader.hh" #include "XrdCks/XrdCksCalc.hh" #include "XrdCks/XrdCksCalcmd5.hh" #include "XrdCks/XrdCksCalccrc32.hh" #include "XrdCks/XrdCksCalccrc32C.hh" #include "XrdCks/XrdCksCalcadler32.hh" #include "XrdSys/XrdSysE2T.hh" #include "XrdSys/XrdSysPthread.hh" #include "XrdVersion.hh" #include #include #include #include XrdVERSIONINFOREF( XrdCl ); namespace XrdCl { //---------------------------------------------------------------------------- // Constructor //---------------------------------------------------------------------------- CheckSumManager::CheckSumManager() { pLoader = new XrdCksLoader( XrdVERSIONINFOVAR( XrdCl ) ); pCalculators["md5"] = new XrdCksCalcmd5(); pCalculators["crc32"] = new XrdCksCalccrc32; pCalculators["crc32c"] = new XrdCksCalccrc32C; pCalculators["adler32"] = new XrdCksCalcadler32; } //---------------------------------------------------------------------------- // Destructor //---------------------------------------------------------------------------- CheckSumManager::~CheckSumManager() { CalcMap::iterator it; for( it = pCalculators.begin(); it != pCalculators.end(); ++it ) delete it->second; delete pLoader; } //---------------------------------------------------------------------------- // Get a calculator //---------------------------------------------------------------------------- XrdCksCalc *CheckSumManager::GetCalculator( const std::string &algName ) { Log *log = DefaultEnv::GetLog(); XrdSysMutexHelper scopedLock( pMutex ); CalcMap::iterator it = pCalculators.find( algName ); if( it == pCalculators.end() ) { char *errBuff = new char[1024]; log->Dump( UtilityMsg, "Attempting to load a calculator for: %s", algName.c_str() ); XrdCksCalc *c = pLoader->Load( algName.c_str(), "", errBuff, 1024 ); if( !c ) { log->Error( UtilityMsg, "Unable to load %s calculator: %s", algName.c_str(), errBuff ); delete [] errBuff; return 0; } delete [] errBuff; pCalculators[algName] = c; return c->New(); } return it->second->New();; } //---------------------------------------------------------------------------- // Stop the manager //---------------------------------------------------------------------------- bool CheckSumManager::Calculate( XrdCksData &result, const std::string &algName, const std::string &filePath ) { //-------------------------------------------------------------------------- // Get a calculator //-------------------------------------------------------------------------- Log *log = DefaultEnv::GetLog(); XrdCksCalc *calc = GetCalculator( algName ); if( !calc ) { log->Error( UtilityMsg, "Unable to get a calculator for %s", algName.c_str() ); return false; } std::unique_ptr calcPtr( calc ); //-------------------------------------------------------------------------- // Open the file //-------------------------------------------------------------------------- log->Debug( UtilityMsg, "Opening %s for reading (checksum calc)", filePath.c_str() ); int fd = open( filePath.c_str(), O_RDONLY ); if( fd == -1 ) { log->Error( UtilityMsg, "Unable to open %s: %s", filePath.c_str(), XrdSysE2T( errno ) ); return false; } //-------------------------------------------------------------------------- // Calculate the checksum //-------------------------------------------------------------------------- const uint32_t buffSize = 2*1024*1024; char *buffer = new char[buffSize]; int64_t bytesRead = 0; while( (bytesRead = read( fd, buffer, buffSize )) ) { if( bytesRead == -1 ) { log->Error( UtilityMsg, "Unable read from %s: %s", filePath.c_str(), XrdSysE2T( errno ) ); close( fd ); delete [] buffer; return false; } calc->Update( buffer, bytesRead ); } int size; calc->Type( size ); result.Set( (void*)calc->Final(), size ); //-------------------------------------------------------------------------- // Clean up //-------------------------------------------------------------------------- delete [] buffer; close( fd ); return true; } } xrootd-5.6.9/src/XrdCl/XrdClCheckSumManager.hh000066400000000000000000000064471457266313600211600ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2012-2014 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #ifndef __XRD_CL_CHECK_SUM_MANAGER_HH__ #define __XRD_CL_CHECK_SUM_MANAGER_HH__ #include #include #include "XrdSys/XrdSysPthread.hh" #include "XrdCks/XrdCksData.hh" class XrdCksLoader; class XrdCksCalc; namespace XrdCl { //---------------------------------------------------------------------------- //! Manage the checksum calc objects //---------------------------------------------------------------------------- class CheckSumManager { public: //------------------------------------------------------------------------ //! Constructor //------------------------------------------------------------------------ CheckSumManager(); //------------------------------------------------------------------------ // Destructor //------------------------------------------------------------------------ virtual ~CheckSumManager(); //------------------------------------------------------------------------ //! Get the check sum calc object for a given checksum type //! //! @param algName name of the checksumming algorithm //! @return the appropriate calc object (must be deleted by the user) //! or 0 if a calculator cannot be obtained //------------------------------------------------------------------------ XrdCksCalc *GetCalculator( const std::string &algName ); //------------------------------------------------------------------------ //! Calculate a checksum of for a given file //------------------------------------------------------------------------ bool Calculate( XrdCksData &result, const std::string &algName, const std::string &filePath ); private: CheckSumManager(const CheckSumManager &other); CheckSumManager &operator = (const CheckSumManager &other); typedef std::map CalcMap; CalcMap pCalculators; XrdCksLoader *pLoader; XrdSysMutex pMutex; }; } #endif // __XRD_CL_CHECK_SUM_MANAGER_HH__ xrootd-5.6.9/src/XrdCl/XrdClCheckpointOperation.hh000066400000000000000000000212721457266313600221240ustar00rootroot00000000000000/* * XrdClCheckpointOperation.hh * * Created on: 31 May 2021 * Author: simonm */ #ifndef SRC_XRDCL_XRDCLCHECKPOINTOPERATION_HH_ #define SRC_XRDCL_XRDCLCHECKPOINTOPERATION_HH_ #include "XrdCl/XrdClFileOperations.hh" namespace XrdCl { //---------------------------------------------------------------------------- //! Checkpoint operation code //---------------------------------------------------------------------------- enum ChkPtCode { BEGIN = kXR_ckpBegin, COMMIT = kXR_ckpCommit, ROLLBACK = kXR_ckpRollback }; //---------------------------------------------------------------------------- //! Checkpoint operation (@see FileOperation) //---------------------------------------------------------------------------- template class CheckpointImpl: public FileOperation, Arg> { public: //------------------------------------------------------------------------ //! Inherit constructors from FileOperation (@see FileOperation) //------------------------------------------------------------------------ using FileOperation, Arg>::FileOperation; //------------------------------------------------------------------------ //! Argument indexes in the args tuple //------------------------------------------------------------------------ enum { CodeArg }; //------------------------------------------------------------------------ //! @return : name of the operation (@see Operation) //------------------------------------------------------------------------ std::string ToString() { return "Checkpoint"; } protected: //------------------------------------------------------------------------ //! RunImpl operation (@see Operation) //! //! @param params : container with parameters forwarded from //! previous operation //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus RunImpl( PipelineHandler *handler, uint16_t pipelineTimeout ) { ChkPtCode code = std::get( this->args ).Get(); uint16_t timeout = pipelineTimeout < this->timeout ? pipelineTimeout : this->timeout; return this->file->Checkpoint( code, handler, timeout ); } }; //---------------------------------------------------------------------------- //! Factory for creating ReadImpl objects //---------------------------------------------------------------------------- inline CheckpointImpl Checkpoint( Ctx file, Arg code, uint16_t timeout = 0 ) { return CheckpointImpl( std::move( file ), std::move( code ) ).Timeout( timeout ); } //---------------------------------------------------------------------------- //! Checkpointed write operation (@see FileOperation) //---------------------------------------------------------------------------- template class ChkptWrtImpl: public FileOperation, Arg, Arg, Arg> { public: //------------------------------------------------------------------------ //! Inherit constructors from FileOperation (@see FileOperation) //------------------------------------------------------------------------ using FileOperation, Arg, Arg, Arg>::FileOperation; //------------------------------------------------------------------------ //! Argument indexes in the args tuple //------------------------------------------------------------------------ enum { OffArg, LenArg, BufArg }; //------------------------------------------------------------------------ //! @return : name of the operation (@see Operation) //------------------------------------------------------------------------ std::string ToString() { return "ChkptWrt"; } protected: //------------------------------------------------------------------------ //! RunImpl operation (@see Operation) //! //! @param params : container with parameters forwarded from //! previous operation //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus RunImpl( PipelineHandler *handler, uint16_t pipelineTimeout ) { uint64_t off = std::get( this->args ).Get(); uint32_t len = std::get( this->args ).Get(); const void* buf = std::get( this->args ).Get(); uint16_t timeout = pipelineTimeout < this->timeout ? pipelineTimeout : this->timeout; return this->file->ChkptWrt( off, len, buf, handler, timeout ); } }; //---------------------------------------------------------------------------- //! Factory for creating ReadImpl objects //---------------------------------------------------------------------------- inline ChkptWrtImpl ChkptWrt( Ctx file, Arg offset, Arg size, Arg buffer, uint16_t timeout = 0 ) { return ChkptWrtImpl( std::move( file ), std::move( offset ), std::move( size ), std::move( buffer ) ).Timeout( timeout ); } //---------------------------------------------------------------------------- //! Checkpointed WriteV operation (@see FileOperation) //---------------------------------------------------------------------------- template class ChkptWrtVImpl: public FileOperation, Arg, Arg>> { public: //------------------------------------------------------------------------ //! Inherit constructors from FileOperation (@see FileOperation) //------------------------------------------------------------------------ using FileOperation, Arg, Arg>>::FileOperation; //------------------------------------------------------------------------ //! Argument indexes in the args tuple //------------------------------------------------------------------------ enum { OffArg, IovecArg, }; //------------------------------------------------------------------------ //! @return : name of the operation (@see Operation) //------------------------------------------------------------------------ std::string ToString() { return "ChkptWrtV"; } protected: //------------------------------------------------------------------------ //! RunImpl operation (@see Operation) //! //! @param params : container with parameters forwarded from //! previous operation //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus RunImpl( PipelineHandler *handler, uint16_t pipelineTimeout ) { uint64_t off = std::get( this->args ).Get(); std::vector &stdiov = std::get( this->args ).Get(); uint16_t timeout = pipelineTimeout < this->timeout ? pipelineTimeout : this->timeout; int iovcnt = stdiov.size(); iovec iov[iovcnt]; for( size_t i = 0; i < stdiov.size(); ++i ) { iov[i].iov_base = stdiov[i].iov_base; iov[i].iov_len = stdiov[i].iov_len; } return this->file->ChkptWrtV( off, iov, iovcnt, handler, timeout ); } }; //---------------------------------------------------------------------------- //! Factory for creating ChkptWrtVImpl objects //---------------------------------------------------------------------------- inline ChkptWrtVImpl ChkptWrtV( Ctx file, Arg offset, Arg> iov, uint16_t timeout = 0 ) { return ChkptWrtVImpl( std::move( file ), std::move( offset ), std::move( iov ) ).Timeout( timeout ); } } #endif /* SRC_XRDCL_XRDCLCHECKPOINTOPERATION_HH_ */ xrootd-5.6.9/src/XrdCl/XrdClClassicCopyJob.cc000066400000000000000000003257331457266313600210220ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2014 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #include "XrdCl/XrdClClassicCopyJob.hh" #include "XrdCl/XrdClConstants.hh" #include "XrdCl/XrdClLog.hh" #include "XrdCl/XrdClDefaultEnv.hh" #include "XrdCl/XrdClFile.hh" #include "XrdCl/XrdClMonitor.hh" #include "XrdCl/XrdClUtils.hh" #include "XrdCl/XrdClCheckSumManager.hh" #include "XrdCks/XrdCksCalc.hh" #include "XrdCl/XrdClRedirectorRegistry.hh" #include "XrdCl/XrdClZipArchive.hh" #include "XrdCl/XrdClZipOperations.hh" #include "XrdCl/XrdClPostMaster.hh" #include "XrdCl/XrdClJobManager.hh" #include "XrdCl/XrdClXRootDTransport.hh" #include "XrdClXCpCtx.hh" #include "XrdCl/XrdClCheckSumHelper.hh" #include "XrdSys/XrdSysE2T.hh" #include "XrdSys/XrdSysPthread.hh" #include #include #include #include #include #include #include #include #include #include #include #include #if __cplusplus < 201103L #include #endif namespace { //---------------------------------------------------------------------------- //! Helper timer class //---------------------------------------------------------------------------- template> class mytimer_t { public: mytimer_t() : start( clock_t::now() ){ } void reset(){ start = clock_t::now(); } uint64_t elapsed() const { return std::chrono::duration_cast( clock_t::now() - start ).count(); } private: typedef std::chrono::high_resolution_clock clock_t; typedef std::chrono::duration unit_t; std::chrono::time_point start; }; using timer_sec_t = mytimer_t<>; using timer_nsec_t = mytimer_t; inline XrdCl::XRootDStatus Translate( std::vector &in, std::vector &out ) { std::vector ret; ret.reserve( in.size() ); std::vector::iterator itr = in.begin(); for( ; itr != in.end() ; ++itr ) { if( !itr->status.IsOK() ) return itr->status; XrdCl::xattr_t xa( itr->name, itr->value ); ret.push_back( std::move( xa ) ); } out.swap( ret ); return XrdCl::XRootDStatus(); } //---------------------------------------------------------------------------- //! Helper function for retrieving extended-attributes //---------------------------------------------------------------------------- inline XrdCl::XRootDStatus GetXAttr( XrdCl::File &file, std::vector &xattrs ) { std::vector rsp; XrdCl::XRootDStatus st = file.ListXAttr( rsp ); if( !st.IsOK() ) return st; return Translate( rsp, xattrs ); } //---------------------------------------------------------------------------- //! Helper function for retrieving extended-attributes //---------------------------------------------------------------------------- inline XrdCl::XRootDStatus GetXAttr( const std::string &url, std::vector &xattrs ) { XrdCl::URL u( url ); XrdCl::FileSystem fs( u ); std::vector rsp; XrdCl::XRootDStatus st = fs.ListXAttr( u.GetPath(), rsp ); if( !st.IsOK() ) return st; return Translate( rsp, xattrs ); } inline XrdCl::XRootDStatus SetXAttr( XrdCl::File &file, const std::vector &xattrs ) { std::vector rsp; file.SetXAttr( xattrs, rsp ); std::vector::iterator itr = rsp.begin(); for( ; itr != rsp.end() ; ++itr ) if( !itr->status.IsOK() ) return itr->status; return XrdCl::XRootDStatus(); } //---------------------------------------------------------------------------- //! Abstract chunk source //---------------------------------------------------------------------------- class Source { public: //------------------------------------------------------------------------ // Destructor //------------------------------------------------------------------------ Source( const std::string &checkSumType = "", const std::vector &addcks = std::vector() ) : pCkSumHelper( 0 ), pContinue( false ) { if( !checkSumType.empty() ) pCkSumHelper = new XrdCl::CheckSumHelper( "source", checkSumType ); for( auto &type : addcks ) pAddCksHelpers.push_back( new XrdCl::CheckSumHelper( "source", type ) ); }; virtual ~Source() { delete pCkSumHelper; for( auto ptr : pAddCksHelpers ) delete ptr; } //------------------------------------------------------------------------ //! Initialize the source //------------------------------------------------------------------------ virtual XrdCl::XRootDStatus Initialize() = 0; //------------------------------------------------------------------------ //! Get size //------------------------------------------------------------------------ virtual int64_t GetSize() = 0; //------------------------------------------------------------------------ //! Start reading from the source at given offset //------------------------------------------------------------------------ virtual XrdCl::XRootDStatus StartAt( uint64_t offset ) = 0; //------------------------------------------------------------------------ //! Get a data chunk from the source //! //! @param ci chunk information //! @return status of the operation //! suContinue - there are some chunks left //! suDone - no chunks left //------------------------------------------------------------------------ virtual XrdCl::XRootDStatus GetChunk( XrdCl::PageInfo &ci ) = 0; //------------------------------------------------------------------------ //! Get check sum //------------------------------------------------------------------------ virtual XrdCl::XRootDStatus GetCheckSum( std::string &checkSum, std::string &checkSumType ) = 0; //------------------------------------------------------------------------ //! Get additional checksums //------------------------------------------------------------------------ virtual std::vector GetAddCks() = 0; //------------------------------------------------------------------------ //! Get extended attributes //------------------------------------------------------------------------ virtual XrdCl::XRootDStatus GetXAttr( std::vector &xattrs ) = 0; //------------------------------------------------------------------------ //! Try different server //------------------------------------------------------------------------ virtual XrdCl::XRootDStatus TryOtherServer() { return XrdCl::XRootDStatus( XrdCl::stError, XrdCl::errNotImplemented ); } protected: XrdCl::CheckSumHelper *pCkSumHelper; std::vector pAddCksHelpers; bool pContinue; }; //---------------------------------------------------------------------------- //! Abstract chunk destination //---------------------------------------------------------------------------- class Destination { public: //------------------------------------------------------------------------ //! Constructor //------------------------------------------------------------------------ Destination( const std::string &checkSumType = "" ): pPosc( false ), pForce( false ), pCoerce( false ), pMakeDir( false ), pContinue( false ), pCkSumHelper( 0 ) { if( !checkSumType.empty() ) pCkSumHelper = new XrdCl::CheckSumHelper( "destination", checkSumType ); } //------------------------------------------------------------------------ //! Destructor //------------------------------------------------------------------------ virtual ~Destination() { delete pCkSumHelper; } //------------------------------------------------------------------------ //! Initialize the destination //------------------------------------------------------------------------ virtual XrdCl::XRootDStatus Initialize() = 0; //------------------------------------------------------------------------ //! Finalize the destination //------------------------------------------------------------------------ virtual XrdCl::XRootDStatus Finalize() = 0; //------------------------------------------------------------------------ //! Put a data chunk at a destination //! //! @param ci chunk information //! @return status of the operation //------------------------------------------------------------------------ virtual XrdCl::XRootDStatus PutChunk( XrdCl::PageInfo &&ci ) = 0; //------------------------------------------------------------------------ //! Flush chunks that might have been queues //------------------------------------------------------------------------ virtual XrdCl::XRootDStatus Flush() = 0; //------------------------------------------------------------------------ //! Get check sum //------------------------------------------------------------------------ virtual XrdCl::XRootDStatus GetCheckSum( std::string &checkSum, std::string &checkSumType ) = 0; //------------------------------------------------------------------------ //! Set extended attributes //------------------------------------------------------------------------ virtual XrdCl::XRootDStatus SetXAttr( const std::vector &xattrs ) = 0; //------------------------------------------------------------------------ //! Get size //------------------------------------------------------------------------ virtual int64_t GetSize() = 0; //------------------------------------------------------------------------ //! Set POSC //------------------------------------------------------------------------ void SetPOSC( bool posc ) { pPosc = posc; } //------------------------------------------------------------------------ //! Set force //------------------------------------------------------------------------ void SetForce( bool force ) { pForce = force; } //------------------------------------------------------------------------ //! Set continue //------------------------------------------------------------------------ void SetContinue( bool continue_ ) { pContinue = continue_; } //------------------------------------------------------------------------ //! Set coerce //------------------------------------------------------------------------ void SetCoerce( bool coerce ) { pCoerce = coerce; } //------------------------------------------------------------------------ //! Set makedir //------------------------------------------------------------------------ void SetMakeDir( bool makedir ) { pMakeDir = makedir; } //------------------------------------------------------------------------ //! Get last URL //------------------------------------------------------------------------ virtual const std::string& GetLastURL() const { static const std::string empty; return empty; } //------------------------------------------------------------------------ //! Get write-recovery redirector //------------------------------------------------------------------------ virtual const std::string& GetWrtRecoveryRedir() const { static const std::string empty; return empty; } protected: bool pPosc; bool pForce; bool pCoerce; bool pMakeDir; bool pContinue; XrdCl::CheckSumHelper *pCkSumHelper; }; //---------------------------------------------------------------------------- //! StdIn source //---------------------------------------------------------------------------- class StdInSource: public Source { public: //------------------------------------------------------------------------ //! Constructor //------------------------------------------------------------------------ StdInSource( const std::string &ckSumType, uint32_t chunkSize, const std::vector &addcks ): Source( ckSumType, addcks ), pCurrentOffset(0), pChunkSize( chunkSize ) { } //------------------------------------------------------------------------ //! Destructor //------------------------------------------------------------------------ virtual ~StdInSource() { } //------------------------------------------------------------------------ //! Initialize the source //------------------------------------------------------------------------ virtual XrdCl::XRootDStatus Initialize() { if( pCkSumHelper ) { auto st = pCkSumHelper->Initialize(); if( !st.IsOK() ) return st; for( auto cksHelper : pAddCksHelpers ) { st = cksHelper->Initialize(); if( !st.IsOK() ) return st; } } return XrdCl::XRootDStatus(); } //------------------------------------------------------------------------ //! Get size //------------------------------------------------------------------------ virtual int64_t GetSize() { return -1; } //------------------------------------------------------------------------ //! Start reading from the source at given offset //------------------------------------------------------------------------ virtual XrdCl::XRootDStatus StartAt( uint64_t ) { return XrdCl::XRootDStatus( XrdCl::stError, XrdCl::errNotSupported, ENOTSUP, "Cannot continue from stdin!" ); } //------------------------------------------------------------------------ //! Get a data chunk from the source //------------------------------------------------------------------------ virtual XrdCl::XRootDStatus GetChunk( XrdCl::PageInfo &ci ) { using namespace XrdCl; Log *log = DefaultEnv::GetLog(); uint32_t toRead = pChunkSize; char *buffer = new char[toRead]; int64_t bytesRead = 0; uint32_t offset = 0; while( toRead ) { int64_t bRead = read( 0, buffer+offset, toRead ); if( bRead == -1 ) { log->Debug( UtilityMsg, "Unable to read from stdin: %s", XrdSysE2T( errno ) ); delete [] buffer; return XRootDStatus( stError, errOSError, errno ); } if( bRead == 0 ) break; bytesRead += bRead; offset += bRead; toRead -= bRead; } if( bytesRead == 0 ) { delete [] buffer; return XRootDStatus( stOK, suDone ); } if( pCkSumHelper ) pCkSumHelper->Update( buffer, bytesRead ); for( auto cksHelper : pAddCksHelpers ) cksHelper->Update( buffer, bytesRead ); ci = XrdCl::PageInfo( pCurrentOffset, bytesRead, buffer ); pCurrentOffset += bytesRead; return XRootDStatus( stOK, suContinue ); } //------------------------------------------------------------------------ //! Get check sum //------------------------------------------------------------------------ virtual XrdCl::XRootDStatus GetCheckSumImpl( XrdCl::CheckSumHelper *cksHelper, std::string &checkSum, std::string &checkSumType ) { using namespace XrdCl; if( cksHelper ) return cksHelper->GetCheckSum( checkSum, checkSumType ); return XRootDStatus( stError, errCheckSumError ); } //------------------------------------------------------------------------ //! Get check sum //------------------------------------------------------------------------ virtual XrdCl::XRootDStatus GetCheckSum( std::string &checkSum, std::string &checkSumType ) { return GetCheckSumImpl( pCkSumHelper, checkSum, checkSumType ); } //------------------------------------------------------------------------ //! Get additional checksums //------------------------------------------------------------------------ std::vector GetAddCks() { std::vector ret; for( auto cksHelper : pAddCksHelpers ) { std::string type = cksHelper->GetType(); std::string cks; GetCheckSumImpl( cksHelper, cks, type ); ret.push_back( type + ":" + cks ); } return ret; } //------------------------------------------------------------------------ //! Get extended attributes //------------------------------------------------------------------------ virtual XrdCl::XRootDStatus GetXAttr( std::vector &xattrs ) { return XrdCl::XRootDStatus(); } private: StdInSource(const StdInSource &other); StdInSource &operator = (const StdInSource &other); uint64_t pCurrentOffset; uint32_t pChunkSize; }; //---------------------------------------------------------------------------- //! XRootDSource //---------------------------------------------------------------------------- class XRootDSource: public Source { struct CancellableJob : public XrdCl::Job { virtual void Cancel() = 0; std::mutex mtx; }; //---------------------------------------------------------------------------- // On-connect callback job, a lambda would be more elegant, but we still have // to support SLC6 //---------------------------------------------------------------------------- template struct OnConnJob : public CancellableJob { OnConnJob( XRootDSource *self, READER *reader ) : self( self ), reader( reader ) { } void Run( void* ) { std::unique_lock lck( mtx ); if( !self || !reader ) return; // add new chunks to the queue if( self->pNbConn < self->pMaxNbConn ) self->FillQueue( reader ); } void Cancel() { std::unique_lock lck( mtx ); self = 0; reader = 0; } private: XRootDSource *self; READER *reader; }; public: //------------------------------------------------------------------------ //! Try different server //------------------------------------------------------------------------ XrdCl::XRootDStatus TryOtherServer() { return pFile->TryOtherServer(); } //------------------------------------------------------------------------ //! Constructor //------------------------------------------------------------------------ XRootDSource( const XrdCl::URL *url, uint32_t chunkSize, uint8_t parallelChunks, const std::string &ckSumType, const std::vector &addcks, bool doserver ): Source( ckSumType, addcks ), pUrl( url ), pFile( new XrdCl::File() ), pSize( -1 ), pCurrentOffset( 0 ), pChunkSize( chunkSize ), pParallel( parallelChunks ), pNbConn( 0 ), pUsePgRead( false ), pDoServer( doserver ) { int val = XrdCl::DefaultSubStreamsPerChannel; XrdCl::DefaultEnv::GetEnv()->GetInt( "SubStreamsPerChannel", val ); pMaxNbConn = val - 1; // account for the control stream } //------------------------------------------------------------------------ //! Destructor //------------------------------------------------------------------------ virtual ~XRootDSource() { if( pDataConnCB ) pDataConnCB->Cancel(); CleanUpChunks(); if( pFile->IsOpen() ) XrdCl::XRootDStatus status = pFile->Close(); delete pFile; } //------------------------------------------------------------------------ //! Initialize the source //------------------------------------------------------------------------ virtual XrdCl::XRootDStatus Initialize() { using namespace XrdCl; Log *log = DefaultEnv::GetLog(); log->Debug( UtilityMsg, "Opening %s for reading", pUrl->GetURL().c_str() ); std::string value; DefaultEnv::GetEnv()->GetString( "ReadRecovery", value ); pFile->SetProperty( "ReadRecovery", value ); XRootDStatus st = pFile->Open( pUrl->GetURL(), OpenFlags::Read ); if( !st.IsOK() ) return st; StatInfo *statInfo; st = pFile->Stat( false, statInfo ); if( !st.IsOK() ) return st; pSize = statInfo->GetSize(); delete statInfo; if( pUrl->IsLocalFile() && !pUrl->IsMetalink() && pCkSumHelper && !pContinue ) { st = pCkSumHelper->Initialize(); if( !st.IsOK() ) return st; for( auto cksHelper : pAddCksHelpers ) { st = cksHelper->Initialize(); if( !st.IsOK() ) return st; } } //---------------------------------------------------------------------- // Figere out the actual data server we are talking to //---------------------------------------------------------------------- if( !pUrl->IsLocalFile() || ( pUrl->IsLocalFile() && pUrl->IsMetalink() ) ) { pFile->GetProperty( "LastURL", pDataServer ); } if( ( !pUrl->IsLocalFile() && !pFile->IsSecure() ) || ( pUrl->IsLocalFile() && pUrl->IsMetalink() ) ) { //-------------------------------------------------------------------- // Decide whether we can use PgRead //-------------------------------------------------------------------- int val = XrdCl::DefaultCpUsePgWrtRd; XrdCl::DefaultEnv::GetEnv()->GetInt( "CpUsePgWrtRd", val ); pUsePgRead = XrdCl::Utils::HasPgRW( pDataServer ) && ( val == 1 ); } //---------------------------------------------------------------------- // Print the IPv4/IPv6 stack to the stderr if we are running in server // mode //---------------------------------------------------------------------- if( pDoServer && !pUrl->IsLocalFile() ) { AnyObject obj; DefaultEnv::GetPostMaster()->QueryTransport( pDataServer, StreamQuery::IpStack, obj ); std::string *ipstack = nullptr; obj.Get( ipstack ); std::cerr << "!-!" << *ipstack << std::endl; delete ipstack; } SetOnDataConnectHandler( pFile ); return XRootDStatus(); } //------------------------------------------------------------------------ //! Get size //------------------------------------------------------------------------ virtual int64_t GetSize() { return pSize; } //------------------------------------------------------------------------ //! Start reading from the source at given offset //------------------------------------------------------------------------ virtual XrdCl::XRootDStatus StartAt( uint64_t offset ) { pCurrentOffset = offset; pContinue = true; return XrdCl::XRootDStatus(); } //------------------------------------------------------------------------ //! Get a data chunk from the source //! //! @param ci chunk information //! @return status of the operation //! suContinue - there are some chunks left //! suDone - no chunks left //------------------------------------------------------------------------ virtual XrdCl::XRootDStatus GetChunk( XrdCl::PageInfo &ci ) { return GetChunkImpl( pFile, ci ); } //------------------------------------------------------------------------ //! Get extended attributes //------------------------------------------------------------------------ virtual XrdCl::XRootDStatus GetXAttr( std::vector &xattrs ) { return ::GetXAttr( *pFile, xattrs ); } //------------------------------------------------------------------------ // Clean up the chunks that are flying //------------------------------------------------------------------------ void CleanUpChunks() { while( !pChunks.empty() ) { ChunkHandler *ch = pChunks.front(); pChunks.pop(); ch->sem->Wait(); delete [] (char *)ch->chunk.GetBuffer(); delete ch; } } //------------------------------------------------------------------------ // Get check sum //------------------------------------------------------------------------ virtual XrdCl::XRootDStatus GetCheckSum( std::string &checkSum, std::string &checkSumType ) { return GetCheckSumImpl( pCkSumHelper, checkSum, checkSumType ); } XrdCl::XRootDStatus GetCheckSumImpl( XrdCl::CheckSumHelper *cksHelper, std::string &checkSum, std::string &checkSumType ) { if( pUrl->IsMetalink() ) { XrdCl::RedirectorRegistry ®istry = XrdCl::RedirectorRegistry::Instance(); XrdCl::VirtualRedirector *redirector = registry.Get( *pUrl ); checkSum = redirector->GetCheckSum( checkSumType ); if( !checkSum.empty() ) return XrdCl::XRootDStatus(); } if( pUrl->IsLocalFile() ) { if( pContinue ) // in case of --continue option we have to calculate the checksum from scratch return XrdCl::Utils::GetLocalCheckSum( checkSum, checkSumType, pUrl->GetPath() ); if( cksHelper ) return cksHelper->GetCheckSum( checkSum, checkSumType ); return XrdCl::XRootDStatus( XrdCl::stError, XrdCl::errCheckSumError ); } std::string dataServer; pFile->GetProperty( "DataServer", dataServer ); std::string lastUrl; pFile->GetProperty( "LastURL", lastUrl ); return XrdCl::Utils::GetRemoteCheckSum( checkSum, checkSumType, XrdCl::URL( lastUrl ) ); } //------------------------------------------------------------------------ //! Get additional checksums //------------------------------------------------------------------------ std::vector GetAddCks() { std::vector ret; for( auto cksHelper : pAddCksHelpers ) { std::string type = cksHelper->GetType(); std::string cks; GetCheckSumImpl( cksHelper, cks, type ); ret.push_back( cks ); } return ret; } private: XRootDSource(const XRootDSource &other); XRootDSource &operator = (const XRootDSource &other); protected: //------------------------------------------------------------------------ // Fill the queue with in-the-fly read requests //------------------------------------------------------------------------ template inline void FillQueue( READER *reader ) { //---------------------------------------------------------------------- // Get the number of connected streams //---------------------------------------------------------------------- uint16_t parallel = pParallel; if( pNbConn < pMaxNbConn ) { pNbConn = XrdCl::DefaultEnv::GetPostMaster()-> NbConnectedStrm( pDataServer ); } if( pNbConn ) parallel *= pNbConn; while( pChunks.size() < parallel && pCurrentOffset < pSize ) { uint64_t chunkSize = pChunkSize; if( pCurrentOffset + chunkSize > (uint64_t)pSize ) chunkSize = pSize - pCurrentOffset; char *buffer = new char[chunkSize]; ChunkHandler *ch = new ChunkHandler(); ch->status = pUsePgRead ? reader->PgRead( pCurrentOffset, chunkSize, buffer, ch ) : reader->Read( pCurrentOffset, chunkSize, buffer, ch ); pChunks.push( ch ); pCurrentOffset += chunkSize; if( !ch->status.IsOK() ) { ch->sem->Post(); break; } } } //------------------------------------------------------------------------ // Set the on-connect handler for data streams //------------------------------------------------------------------------ template void SetOnDataConnectHandler( READER *reader ) { // we need to create the object anyway as it contains our mutex now pDataConnCB.reset( new OnConnJob( this, reader ) ); // check if it is a local file if( pDataServer.empty() ) return; XrdCl::DefaultEnv::GetPostMaster()->SetOnDataConnectHandler( pDataServer, pDataConnCB ); } //------------------------------------------------------------------------ //! Get a data chunk from the source //! //! @param reader : reader to read data from //! @param ci : chunk information //! @return : status of the operation //! suContinue - there are some chunks left //! suDone - no chunks left //------------------------------------------------------------------------ template XrdCl::XRootDStatus GetChunkImpl( READER *reader, XrdCl::PageInfo &ci ) { //---------------------------------------------------------------------- // Sanity check //---------------------------------------------------------------------- using namespace XrdCl; Log *log = DefaultEnv::GetLog(); //---------------------------------------------------------------------- // Fill the queue //---------------------------------------------------------------------- std::unique_lock lck( pDataConnCB->mtx ); FillQueue( reader ); //---------------------------------------------------------------------- // Pick up a chunk from the front and wait for status //---------------------------------------------------------------------- if( pChunks.empty() ) return XRootDStatus( stOK, suDone ); std::unique_ptr ch( pChunks.front() ); pChunks.pop(); lck.unlock(); ch->sem->Wait(); if( !ch->status.IsOK() ) { log->Debug( UtilityMsg, "Unable read %d bytes at %ld from %s: %s", ch->chunk.GetLength(), ch->chunk.GetOffset(), pUrl->GetURL().c_str(), ch->status.ToStr().c_str() ); delete [] (char *)ch->chunk.GetBuffer(); CleanUpChunks(); return ch->status; } ci = std::move( ch->chunk ); // if it is a local file update the checksum if( pUrl->IsLocalFile() && !pUrl->IsMetalink() && !pContinue ) { if( pCkSumHelper ) pCkSumHelper->Update( ci.GetBuffer(), ci.GetLength() ); for( auto cksHelper : pAddCksHelpers ) cksHelper->Update( ci.GetBuffer(), ci.GetLength() ); } return XRootDStatus( stOK, suContinue ); } //------------------------------------------------------------------------ // Asynchronous chunk handler //------------------------------------------------------------------------ class ChunkHandler: public XrdCl::ResponseHandler { public: ChunkHandler(): sem( new XrdSysSemaphore(0) ) {} virtual ~ChunkHandler() { delete sem; } virtual void HandleResponse( XrdCl::XRootDStatus *statusval, XrdCl::AnyObject *response ) { this->status = *statusval; delete statusval; if( response ) { chunk = ToChunk( response ); delete response; } sem->Post(); } XrdCl::PageInfo ToChunk( XrdCl::AnyObject *response ) { if( response->Has() ) { XrdCl::PageInfo *resp = nullptr; response->Get( resp ); return std::move( *resp ); } else { XrdCl::ChunkInfo *resp = nullptr; response->Get( resp ); return XrdCl::PageInfo( resp->GetOffset(), resp->GetLength(), resp->GetBuffer() ); } } XrdSysSemaphore *sem; XrdCl::PageInfo chunk; XrdCl::XRootDStatus status; }; const XrdCl::URL *pUrl; XrdCl::File *pFile; int64_t pSize; int64_t pCurrentOffset; uint32_t pChunkSize; uint16_t pParallel; std::queue pChunks; std::string pDataServer; uint16_t pNbConn; uint16_t pMaxNbConn; bool pUsePgRead; bool pDoServer; std::shared_ptr pDataConnCB; }; //---------------------------------------------------------------------------- //! XRootDSourceZip //---------------------------------------------------------------------------- class XRootDSourceZip: public XRootDSource { public: //------------------------------------------------------------------------ //! Constructor //------------------------------------------------------------------------ XRootDSourceZip( const std::string &filename, const XrdCl::URL *archive, uint32_t chunkSize, uint8_t parallelChunks, const std::string &ckSumType, const std::vector &addcks, bool doserver ): XRootDSource( archive, chunkSize, parallelChunks, ckSumType, addcks, doserver ), pFilename( filename ), pZipArchive( new XrdCl::ZipArchive() ) { } //------------------------------------------------------------------------ //! Destructor //------------------------------------------------------------------------ virtual ~XRootDSourceZip() { CleanUpChunks(); XrdCl::WaitFor( XrdCl::CloseArchive( pZipArchive ) ); delete pZipArchive; } //------------------------------------------------------------------------ //! Initialize the source //------------------------------------------------------------------------ virtual XrdCl::XRootDStatus Initialize() { using namespace XrdCl; Log *log = DefaultEnv::GetLog(); log->Debug( UtilityMsg, "Opening %s for reading", pUrl->GetURL().c_str() ); std::string value; DefaultEnv::GetEnv()->GetString( "ReadRecovery", value ); pZipArchive->SetProperty( "ReadRecovery", value ); XRootDStatus st = XrdCl::WaitFor( XrdCl::OpenArchive( pZipArchive, pUrl->GetURL(), XrdCl::OpenFlags::Read ) ); if( !st.IsOK() ) return st; st = pZipArchive->OpenFile( pFilename ); if( !st.IsOK() ) return st; XrdCl::StatInfo *info = 0; st = pZipArchive->Stat( info ); if( st.IsOK() ) { pSize = info->GetSize(); delete info; } else return st; if( pUrl->IsLocalFile() && !pUrl->IsMetalink() && pCkSumHelper ) { auto st = pCkSumHelper->Initialize(); if( !st.IsOK() ) return st; for( auto cksHelper : pAddCksHelpers ) { st = cksHelper->Initialize(); if( !st.IsOK() ) return st; } } if( ( !pUrl->IsLocalFile() && !pZipArchive->IsSecure() ) || ( pUrl->IsLocalFile() && pUrl->IsMetalink() ) ) { pZipArchive->GetProperty( "DataServer", pDataServer ); //-------------------------------------------------------------------- // Decide whether we can use PgRead //-------------------------------------------------------------------- int val = XrdCl::DefaultCpUsePgWrtRd; XrdCl::DefaultEnv::GetEnv()->GetInt( "CpUsePgWrtRd", val ); pUsePgRead = XrdCl::Utils::HasPgRW( pDataServer ) && ( val == 1 ); } SetOnDataConnectHandler( pZipArchive ); return XrdCl::XRootDStatus(); } //------------------------------------------------------------------------ //! Get a data chunk from the source //! //! @param buffer buffer for the data //! @param ci chunk information //! @return status of the operation //! suContinue - there are some chunks left //! suDone - no chunks left //------------------------------------------------------------------------ virtual XrdCl::XRootDStatus GetChunk( XrdCl::PageInfo &ci ) { return GetChunkImpl( pZipArchive, ci ); } //------------------------------------------------------------------------ // Get check sum //------------------------------------------------------------------------ virtual XrdCl::XRootDStatus GetCheckSum( std::string &checkSum, std::string &checkSumType ) { return GetCheckSumImpl( checkSum, checkSumType, pCkSumHelper ); } //------------------------------------------------------------------------ // Get check sum implementation //------------------------------------------------------------------------ virtual XrdCl::XRootDStatus GetCheckSumImpl( std::string &checkSum, std::string &checkSumType, XrdCl::CheckSumHelper *cksHelper ) { // The ZIP archive by default contains a ZCRC32 checksum if( checkSumType == "zcrc32" ) { uint32_t cksum = 0; auto st = pZipArchive->GetCRC32( pFilename, cksum ); if( !st.IsOK() ) return st; XrdCksData ckSum; ckSum.Set( "zcrc32" ); ckSum.Set( reinterpret_cast( &cksum ), sizeof( uint32_t ) ); char cksBuffer[265]; ckSum.Get( cksBuffer, 256 ); checkSum = "zcrc32:"; checkSum += XrdCl::Utils::NormalizeChecksum( "zcrc32", cksBuffer ); return st; } int useMtlnCksum = XrdCl::DefaultZipMtlnCksum; XrdCl::Env *env = XrdCl::DefaultEnv::GetEnv(); env->GetInt( "ZipMtlnCksum", useMtlnCksum ); if( useMtlnCksum && pUrl->IsMetalink() ) { XrdCl::RedirectorRegistry ®istry = XrdCl::RedirectorRegistry::Instance(); XrdCl::VirtualRedirector *redirector = registry.Get( *pUrl ); checkSum = redirector->GetCheckSum( checkSumType ); if( !checkSum.empty() ) return XrdCl::XRootDStatus(); } // if it is a local file we can calculate the checksum ourself if( pUrl->IsLocalFile() && !pUrl->IsMetalink() && cksHelper && !pContinue ) return cksHelper->GetCheckSum( checkSum, checkSumType ); // if it is a remote file other types of checksum are not supported return XrdCl::XRootDStatus( XrdCl::stError, XrdCl::errNotSupported ); } //------------------------------------------------------------------------ //! Get additional checksums //------------------------------------------------------------------------ std::vector GetAddCks() { std::vector ret; for( auto cksHelper : pAddCksHelpers ) { std::string type = cksHelper->GetType(); std::string cks; GetCheckSumImpl( cks, type, cksHelper ); ret.push_back( cks ); } return ret; } //------------------------------------------------------------------------ //! Get extended attributes //------------------------------------------------------------------------ virtual XrdCl::XRootDStatus GetXAttr( std::vector &xattrs ) { return XrdCl::XRootDStatus(); } private: XRootDSourceZip(const XRootDSourceZip &other); XRootDSourceZip &operator = (const XRootDSourceZip &other); const std::string pFilename; XrdCl::ZipArchive *pZipArchive; }; //---------------------------------------------------------------------------- //! XRootDSourceDynamic //---------------------------------------------------------------------------- class XRootDSourceDynamic: public Source { public: //------------------------------------------------------------------------ //! Try different server //------------------------------------------------------------------------ XrdCl::XRootDStatus TryOtherServer() { return pFile->TryOtherServer(); } //------------------------------------------------------------------------ //! Constructor //------------------------------------------------------------------------ XRootDSourceDynamic( const XrdCl::URL *url, uint32_t chunkSize, const std::string &ckSumType, const std::vector &addcks ): Source( ckSumType, addcks ), pUrl( url ), pFile( new XrdCl::File() ), pCurrentOffset( 0 ), pChunkSize( chunkSize ), pDone( false ), pUsePgRead( false ) { } //------------------------------------------------------------------------ //! Destructor //------------------------------------------------------------------------ virtual ~XRootDSourceDynamic() { XrdCl::XRootDStatus status = pFile->Close(); delete pFile; } //------------------------------------------------------------------------ //! Initialize the source //------------------------------------------------------------------------ virtual XrdCl::XRootDStatus Initialize() { using namespace XrdCl; Log *log = DefaultEnv::GetLog(); log->Debug( UtilityMsg, "Opening %s for reading", pUrl->GetURL().c_str() ); std::string value; DefaultEnv::GetEnv()->GetString( "ReadRecovery", value ); pFile->SetProperty( "ReadRecovery", value ); XRootDStatus st = pFile->Open( pUrl->GetURL(), OpenFlags::Read ); if( !st.IsOK() ) return st; if( pUrl->IsLocalFile() && !pUrl->IsMetalink() && pCkSumHelper && !pContinue ) { auto st = pCkSumHelper->Initialize(); if( !st.IsOK() ) return st; for( auto cksHelper : pAddCksHelpers ) { st = cksHelper->Initialize(); if( !st.IsOK() ) return st; } } if( ( !pUrl->IsLocalFile() && !pFile->IsSecure() ) || ( pUrl->IsLocalFile() && pUrl->IsMetalink() ) ) { std::string datasrv; pFile->GetProperty( "DataServer", datasrv ); //-------------------------------------------------------------------- // Decide whether we can use PgRead //-------------------------------------------------------------------- int val = XrdCl::DefaultCpUsePgWrtRd; XrdCl::DefaultEnv::GetEnv()->GetInt( "CpUsePgWrtRd", val ); pUsePgRead = XrdCl::Utils::HasPgRW( datasrv ) && ( val == 1 ); } return XRootDStatus(); } //------------------------------------------------------------------------ //! Get size //------------------------------------------------------------------------ virtual int64_t GetSize() { return -1; } //------------------------------------------------------------------------ //! Start reading from the source at given offset //------------------------------------------------------------------------ virtual XrdCl::XRootDStatus StartAt( uint64_t offset ) { pCurrentOffset = offset; pContinue = true; return XrdCl::XRootDStatus(); } //------------------------------------------------------------------------ //! Get a data chunk from the source //! //! @param buffer buffer for the data //! @param ci chunk information //! @return status of the operation //! suContinue - there are some chunks left //! suDone - no chunks left //------------------------------------------------------------------------ virtual XrdCl::XRootDStatus GetChunk( XrdCl::PageInfo &ci ) { //---------------------------------------------------------------------- // Sanity check //---------------------------------------------------------------------- using namespace XrdCl; if( pDone ) return XRootDStatus( stOK, suDone ); //---------------------------------------------------------------------- // Fill the queue //---------------------------------------------------------------------- char *buffer = new char[pChunkSize]; uint32_t bytesRead = 0; std::vector cksums; XRootDStatus st = pUsePgRead ? pFile->PgRead( pCurrentOffset, pChunkSize, buffer, cksums, bytesRead ) : pFile->Read( pCurrentOffset, pChunkSize, buffer, bytesRead ); if( !st.IsOK() ) { delete [] buffer; return st; } if( !bytesRead ) { delete [] buffer; return XRootDStatus( stOK, suDone ); } if( bytesRead < pChunkSize ) pDone = true; // if it is a local file update the checksum if( pUrl->IsLocalFile() && !pUrl->IsMetalink() && !pContinue ) { if( pCkSumHelper ) pCkSumHelper->Update( buffer, bytesRead ); for( auto cksHelper : pAddCksHelpers ) cksHelper->Update( buffer, bytesRead ); } ci = XrdCl::PageInfo( pCurrentOffset, bytesRead, buffer ); pCurrentOffset += bytesRead; return XRootDStatus( stOK, suContinue ); } //------------------------------------------------------------------------ // Get check sum //------------------------------------------------------------------------ virtual XrdCl::XRootDStatus GetCheckSum( std::string &checkSum, std::string &checkSumType ) { return GetCheckSumImpl( pCkSumHelper, checkSum, checkSumType ); } XrdCl::XRootDStatus GetCheckSumImpl( XrdCl::CheckSumHelper *cksHelper, std::string &checkSum, std::string &checkSumType ) { if( pUrl->IsMetalink() ) { XrdCl::RedirectorRegistry ®istry = XrdCl::RedirectorRegistry::Instance(); XrdCl::VirtualRedirector *redirector = registry.Get( *pUrl ); checkSum = redirector->GetCheckSum( checkSumType ); if( !checkSum.empty() ) return XrdCl::XRootDStatus(); } if( pUrl->IsLocalFile() ) { if( pContinue) // in case of --continue option we have to calculate the checksum from scratch return XrdCl::Utils::GetLocalCheckSum( checkSum, checkSumType, pUrl->GetPath() ); if( cksHelper ) return cksHelper->GetCheckSum( checkSum, checkSumType ); return XrdCl::XRootDStatus( XrdCl::stError, XrdCl::errCheckSumError ); } std::string dataServer; pFile->GetProperty( "DataServer", dataServer ); std::string lastUrl; pFile->GetProperty( "LastURL", lastUrl ); return XrdCl::Utils::GetRemoteCheckSum( checkSum, checkSumType, XrdCl::URL( lastUrl ) ); } //------------------------------------------------------------------------ //! Get additional checksums //------------------------------------------------------------------------ std::vector GetAddCks() { std::vector ret; for( auto cksHelper : pAddCksHelpers ) { std::string type = cksHelper->GetType(); std::string cks; GetCheckSumImpl( cksHelper, cks, type ); ret.push_back( cks ); } return ret; } //------------------------------------------------------------------------ //! Get extended attributes //------------------------------------------------------------------------ virtual XrdCl::XRootDStatus GetXAttr( std::vector &xattrs ) { return ::GetXAttr( *pFile, xattrs ); } private: XRootDSourceDynamic(const XRootDSourceDynamic &other); XRootDSourceDynamic &operator = (const XRootDSourceDynamic &other); const XrdCl::URL *pUrl; XrdCl::File *pFile; int64_t pCurrentOffset; uint32_t pChunkSize; bool pDone; bool pUsePgRead; }; //---------------------------------------------------------------------------- //! XRootDSourceDynamic //---------------------------------------------------------------------------- class XRootDSourceXCp: public Source { public: //------------------------------------------------------------------------ //! Constructor //------------------------------------------------------------------------ XRootDSourceXCp( const XrdCl::URL* url, uint32_t chunkSize, uint16_t parallelChunks, int32_t nbSrc, uint64_t blockSize ): pXCpCtx( 0 ), pUrl( url ), pChunkSize( chunkSize ), pParallelChunks( parallelChunks ), pNbSrc( nbSrc ), pBlockSize( blockSize ) { } ~XRootDSourceXCp() { if( pXCpCtx ) pXCpCtx->Delete(); } //------------------------------------------------------------------------ //! Initialize the source //------------------------------------------------------------------------ virtual XrdCl::XRootDStatus Initialize() { XrdCl::Log *log = XrdCl::DefaultEnv::GetLog(); int64_t fileSize = -1; if( pUrl->IsMetalink() ) { XrdCl::RedirectorRegistry ®istry = XrdCl::RedirectorRegistry::Instance(); XrdCl::VirtualRedirector *redirector = registry.Get( *pUrl ); fileSize = redirector->GetSize(); pReplicas = redirector->GetReplicas(); } else { XrdCl::LocationInfo *li = 0; XrdCl::FileSystem fs( *pUrl ); XrdCl::XRootDStatus st = fs.DeepLocate( pUrl->GetPath(), XrdCl::OpenFlags::Compress | XrdCl::OpenFlags::PrefName, li ); if( !st.IsOK() ) return st; XrdCl::LocationInfo::Iterator itr; for( itr = li->Begin(); itr != li->End(); ++itr) { std::string url = "root://" + itr->GetAddress() + "/" + pUrl->GetPath(); pReplicas.push_back( url ); } delete li; } std::stringstream ss; ss << "XCp sources: "; std::vector::iterator itr; for( itr = pReplicas.begin() ; itr != pReplicas.end() ; ++itr ) { ss << *itr << ", "; } log->Debug( XrdCl::UtilityMsg, ss.str().c_str() ); pXCpCtx = new XrdCl::XCpCtx( pReplicas, pBlockSize, pNbSrc, pChunkSize, pParallelChunks, fileSize ); return pXCpCtx->Initialize(); } //------------------------------------------------------------------------ //! Get size //------------------------------------------------------------------------ virtual int64_t GetSize() { return pXCpCtx->GetSize(); } //------------------------------------------------------------------------ //! Start reading from the source at given offset //------------------------------------------------------------------------ virtual XrdCl::XRootDStatus StartAt( uint64_t offset ) { return XrdCl::XRootDStatus( XrdCl::stError, XrdCl::errNotImplemented ); } //------------------------------------------------------------------------ //! Get a data chunk from the source //! //! @param buffer buffer for the data //! @param ci chunk information //! @return status of the operation //! suContinue - there are some chunks left //! suDone - no chunks left //------------------------------------------------------------------------ virtual XrdCl::XRootDStatus GetChunk( XrdCl::PageInfo &ci ) { XrdCl::XRootDStatus st; do { st = pXCpCtx->GetChunk( ci ); } while( st.IsOK() && st.code == XrdCl::suRetry ); return st; } //------------------------------------------------------------------------ // Get check sum //------------------------------------------------------------------------ virtual XrdCl::XRootDStatus GetCheckSum( std::string &checkSum, std::string &checkSumType ) { if( pUrl->IsMetalink() ) { XrdCl::RedirectorRegistry ®istry = XrdCl::RedirectorRegistry::Instance(); XrdCl::VirtualRedirector *redirector = registry.Get( *pUrl ); checkSum = redirector->GetCheckSum( checkSumType ); if( !checkSum.empty() ) return XrdCl::XRootDStatus(); } std::vector::iterator itr; for( itr = pReplicas.begin() ; itr != pReplicas.end() ; ++itr ) { XrdCl::URL url( *itr ); XrdCl::XRootDStatus st = XrdCl::Utils::GetRemoteCheckSum( checkSum, checkSumType, url ); if( st.IsOK() ) return st; } return XrdCl::XRootDStatus( XrdCl::stError, XrdCl::errNoMoreReplicas ); } //------------------------------------------------------------------------ //! Get additional checksums //------------------------------------------------------------------------ std::vector GetAddCks() { return std::vector(); } //------------------------------------------------------------------------ //! Get extended attributes //------------------------------------------------------------------------ virtual XrdCl::XRootDStatus GetXAttr( std::vector &xattrs ) { XrdCl::XRootDStatus st; std::vector::iterator itr = pReplicas.begin(); for( ; itr < pReplicas.end() ; ++itr ) { st = ::GetXAttr( *itr, xattrs ); if( st.IsOK() ) return st; } return st; } private: XrdCl::XCpCtx *pXCpCtx; const XrdCl::URL *pUrl; std::vector pReplicas; uint32_t pChunkSize; uint16_t pParallelChunks; int32_t pNbSrc; uint64_t pBlockSize; }; //---------------------------------------------------------------------------- //! SrdOut destination //---------------------------------------------------------------------------- class StdOutDestination: public Destination { public: //------------------------------------------------------------------------ //! Constructor //------------------------------------------------------------------------ StdOutDestination( const std::string &ckSumType ): Destination( ckSumType ), pCurrentOffset(0) { } //------------------------------------------------------------------------ //! Destructor //------------------------------------------------------------------------ virtual ~StdOutDestination() { } //------------------------------------------------------------------------ //! Initialize the destination //------------------------------------------------------------------------ virtual XrdCl::XRootDStatus Initialize() { if( pContinue ) return XrdCl::XRootDStatus( XrdCl::stError, XrdCl::errNotSupported, ENOTSUP, "Cannot continue to stdout." ); if( pCkSumHelper ) return pCkSumHelper->Initialize(); return XrdCl::XRootDStatus(); } //------------------------------------------------------------------------ //! Finalize the destination //------------------------------------------------------------------------ virtual XrdCl::XRootDStatus Finalize() { return XrdCl::XRootDStatus(); } //------------------------------------------------------------------------ //! Put a data chunk at a destination //! //! @param ci chunk information //! @return status of the operation //------------------------------------------------------------------------ virtual XrdCl::XRootDStatus PutChunk( XrdCl::PageInfo &&ci ) { using namespace XrdCl; Log *log = DefaultEnv::GetLog(); if( pCurrentOffset != ci.GetOffset() ) { log->Error( UtilityMsg, "Got out-of-bounds chunk, expected offset:" " %ld, got %ld", pCurrentOffset, ci.GetOffset() ); return XRootDStatus( stError, errInternal ); } int64_t wr = 0; uint32_t length = ci.GetLength(); char *cursor = (char*)ci.GetBuffer(); do { wr = write( 1, cursor, length ); if( wr == -1 ) { log->Debug( UtilityMsg, "Unable to write to stdout: %s", XrdSysE2T( errno ) ); delete [] (char*)ci.GetBuffer(); return XRootDStatus( stError, errOSError, errno ); } pCurrentOffset += wr; cursor += wr; length -= wr; } while( length ); if( pCkSumHelper ) pCkSumHelper->Update( ci.GetBuffer(), ci.GetLength() ); delete [] (char*)ci.GetBuffer(); return XRootDStatus(); } //------------------------------------------------------------------------ //! Flush chunks that might have been queues //------------------------------------------------------------------------ virtual XrdCl::XRootDStatus Flush() { return XrdCl::XRootDStatus(); } //------------------------------------------------------------------------ //! Get check sum //------------------------------------------------------------------------ virtual XrdCl::XRootDStatus GetCheckSum( std::string &checkSum, std::string &checkSumType ) { if( pCkSumHelper ) return pCkSumHelper->GetCheckSum( checkSum, checkSumType ); return XrdCl::XRootDStatus( XrdCl::stError, XrdCl::errCheckSumError ); } //------------------------------------------------------------------------ //! Set extended attributes //------------------------------------------------------------------------ virtual XrdCl::XRootDStatus SetXAttr( const std::vector &xattrs ) { return XrdCl::XRootDStatus(); } //------------------------------------------------------------------------ //! Get size //------------------------------------------------------------------------ virtual int64_t GetSize() { return -1; } private: StdOutDestination(const StdOutDestination &other); StdOutDestination &operator = (const StdOutDestination &other); uint64_t pCurrentOffset; }; //---------------------------------------------------------------------------- //! XRootD destination //---------------------------------------------------------------------------- class XRootDDestination: public Destination { public: //------------------------------------------------------------------------ //! Constructor //------------------------------------------------------------------------ XRootDDestination( const XrdCl::URL &url, uint8_t parallelChunks, const std::string &ckSumType, const XrdCl::ClassicCopyJob &cpjob ): Destination( ckSumType ), pUrl( url ), pFile( new XrdCl::File( XrdCl::File::DisableVirtRedirect ) ), pParallel( parallelChunks ), pSize( -1 ), pUsePgWrt( false ), cpjob( cpjob ) { } //------------------------------------------------------------------------ //! Destructor //------------------------------------------------------------------------ virtual ~XRootDDestination() { CleanUpChunks(); delete pFile; XrdCl::Log *log = XrdCl::DefaultEnv::GetLog(); //---------------------------------------------------------------------- // Make sure we clean up the cp-target symlink //---------------------------------------------------------------------- std::string cptarget = XrdCl::DefaultCpTarget; XrdCl::DefaultEnv::GetEnv()->GetString( "CpTarget", cptarget ); if( !cptarget.empty() ) { XrdCl::FileSystem fs( "file://localhost" ); XrdCl::XRootDStatus st = fs.Rm( cptarget ); if( !st.IsOK() ) log->Warning( XrdCl::UtilityMsg, "Could not delete cp-target symlink: %s", st.ToString().c_str() ); } //---------------------------------------------------------------------- // If the copy failed and user requested posc and we are dealing with // a local destination, remove the file //---------------------------------------------------------------------- if( pUrl.IsLocalFile() && pPosc && !cpjob.GetResult().IsOK() ) { XrdCl::FileSystem fs( pUrl ); XrdCl::XRootDStatus st = fs.Rm( pUrl.GetPath() ); if( !st.IsOK() ) log->Error( XrdCl::UtilityMsg, "Failed to remove local destination" " on failure: %s", st.ToString().c_str() ); } } //------------------------------------------------------------------------ //! Initialize the destination //------------------------------------------------------------------------ virtual XrdCl::XRootDStatus Initialize() { using namespace XrdCl; Log *log = DefaultEnv::GetLog(); log->Debug( UtilityMsg, "Opening %s for writing", pUrl.GetURL().c_str() ); std::string value; DefaultEnv::GetEnv()->GetString( "WriteRecovery", value ); pFile->SetProperty( "WriteRecovery", value ); OpenFlags::Flags flags = OpenFlags::Update; if( pForce ) flags |= OpenFlags::Delete; else if( !pContinue ) flags |= OpenFlags::New; if( pPosc ) flags |= OpenFlags::POSC; if( pCoerce ) flags |= OpenFlags::Force; if( pMakeDir) flags |= OpenFlags::MakePath; Access::Mode mode = Access::UR|Access::UW|Access::GR|Access::OR; XrdCl::XRootDStatus st = pFile->Open( pUrl.GetURL(), flags, mode ); if( !st.IsOK() ) return st; if( ( !pUrl.IsLocalFile() && !pFile->IsSecure() ) || ( pUrl.IsLocalFile() && pUrl.IsMetalink() ) ) { std::string datasrv; pFile->GetProperty( "DataServer", datasrv ); //-------------------------------------------------------------------- // Decide whether we can use PgRead //-------------------------------------------------------------------- int val = XrdCl::DefaultCpUsePgWrtRd; XrdCl::DefaultEnv::GetEnv()->GetInt( "CpUsePgWrtRd", val ); pUsePgWrt = XrdCl::Utils::HasPgRW( datasrv ) && ( val == 1 ); } std::string cptarget = XrdCl::DefaultCpTarget; XrdCl::DefaultEnv::GetEnv()->GetString( "CpTarget", cptarget ); if( !cptarget.empty() ) { std::string targeturl; pFile->GetProperty( "LastURL", targeturl ); targeturl = URL( targeturl ).GetLocation(); if( symlink( targeturl.c_str(), cptarget.c_str() ) == -1 ) log->Warning( UtilityMsg, "Could not create cp-target symlink: %s", XrdSysE2T( errno ) ); else log->Info( UtilityMsg, "Created cp-target symlink: %s -> %s", cptarget.c_str(), targeturl.c_str() ); } StatInfo *info = 0; st = pFile->Stat( false, info ); if( !st.IsOK() ) return st; pSize = info->GetSize(); delete info; if( pUrl.IsLocalFile() && pCkSumHelper && !pContinue ) return pCkSumHelper->Initialize(); return XRootDStatus(); } //------------------------------------------------------------------------ //! Finalize the destination //------------------------------------------------------------------------ virtual XrdCl::XRootDStatus Finalize() { return pFile->Close(); } //------------------------------------------------------------------------ //! Put a data chunk at a destination //! //! @param ci chunk information //! @return status of the operation //------------------------------------------------------------------------ virtual XrdCl::XRootDStatus PutChunk( XrdCl::PageInfo &&ci ) { using namespace XrdCl; if( !pFile->IsOpen() ) { delete[] (char*)ci.GetBuffer(); // we took the ownership of the buffer return XRootDStatus( stError, errUninitialized ); } //---------------------------------------------------------------------- // If there is still place for this chunk to be sent send it //---------------------------------------------------------------------- if( pChunks.size() < pParallel ) return QueueChunk( std::move( ci ) ); //---------------------------------------------------------------------- // We wait for a chunk to be sent so that we have space for the current // one //---------------------------------------------------------------------- std::unique_ptr ch( pChunks.front() ); pChunks.pop(); ch->sem->Wait(); delete [] (char*)ch->chunk.GetBuffer(); if( !ch->status.IsOK() ) { Log *log = DefaultEnv::GetLog(); log->Debug( UtilityMsg, "Unable write %d bytes at %ld from %s: %s", ch->chunk.GetLength(), ch->chunk.GetOffset(), pUrl.GetURL().c_str(), ch->status.ToStr().c_str() ); delete[] (char*)ci.GetBuffer(); // we took the ownership of the buffer CleanUpChunks(); //-------------------------------------------------------------------- // Check if we should re-try the transfer from scratch at a different // data server //-------------------------------------------------------------------- return CheckIfRetriable( ch->status ); } return QueueChunk( std::move( ci ) ); } //------------------------------------------------------------------------ //! Get size //------------------------------------------------------------------------ virtual int64_t GetSize() { return pSize; } //------------------------------------------------------------------------ //! Clean up the chunks that are flying //------------------------------------------------------------------------ void CleanUpChunks() { while( !pChunks.empty() ) { ChunkHandler *ch = pChunks.front(); pChunks.pop(); ch->sem->Wait(); delete [] (char *)ch->chunk.GetBuffer(); delete ch; } } //------------------------------------------------------------------------ //! Queue a chunk //------------------------------------------------------------------------ XrdCl::XRootDStatus QueueChunk( XrdCl::PageInfo &&ci ) { // we are writing chunks in order so we can calc the checksum // in case of local files if( pUrl.IsLocalFile() && pCkSumHelper && !pContinue ) pCkSumHelper->Update( ci.GetBuffer(), ci.GetLength() ); ChunkHandler *ch = new ChunkHandler( std::move( ci ) ); XrdCl::XRootDStatus st; st = pUsePgWrt ? pFile->PgWrite(ch->chunk.GetOffset(), ch->chunk.GetLength(), ch->chunk.GetBuffer(), ch->chunk.GetCksums(), ch) : pFile->Write( ch->chunk.GetOffset(), ch->chunk.GetLength(), ch->chunk.GetBuffer(), ch ); if( !st.IsOK() ) { CleanUpChunks(); delete [] (char*)ch->chunk.GetBuffer(); delete ch; return st; } pChunks.push( ch ); return XrdCl::XRootDStatus(); } //------------------------------------------------------------------------ //! Flush chunks that might have been queues //------------------------------------------------------------------------ virtual XrdCl::XRootDStatus Flush() { XrdCl::XRootDStatus st; while( !pChunks.empty() ) { ChunkHandler *ch = pChunks.front(); pChunks.pop(); ch->sem->Wait(); if( !ch->status.IsOK() ) { //-------------------------------------------------------------------- // Check if we should re-try the transfer from scratch at a different // data server //-------------------------------------------------------------------- st = CheckIfRetriable( ch->status ); } delete [] (char *)ch->chunk.GetBuffer(); delete ch; } return st; } //------------------------------------------------------------------------ //! Get check sum //------------------------------------------------------------------------ virtual XrdCl::XRootDStatus GetCheckSum( std::string &checkSum, std::string &checkSumType ) { if( pUrl.IsLocalFile() ) { if( pContinue ) // in case of --continue option we have to calculate the checksum from scratch return XrdCl::Utils::GetLocalCheckSum( checkSum, checkSumType, pUrl.GetPath() ); if( pCkSumHelper ) return pCkSumHelper->GetCheckSum( checkSum, checkSumType ); return XrdCl::XRootDStatus( XrdCl::stError, XrdCl::errCheckSumError ); } std::string lastUrl; pFile->GetProperty( "LastURL", lastUrl ); return XrdCl::Utils::GetRemoteCheckSum( checkSum, checkSumType, XrdCl::URL( lastUrl ) ); } //------------------------------------------------------------------------ //! Set extended attributes //------------------------------------------------------------------------ virtual XrdCl::XRootDStatus SetXAttr( const std::vector &xattrs ) { return ::SetXAttr( *pFile, xattrs ); } //------------------------------------------------------------------------ //! Get last URL //------------------------------------------------------------------------ const std::string& GetLastURL() const { return pLastURL; } //------------------------------------------------------------------------ //! Get write-recovery redirector //------------------------------------------------------------------------ const std::string& GetWrtRecoveryRedir() const { return pWrtRecoveryRedir; } private: XRootDDestination(const XRootDDestination &other); XRootDDestination &operator = (const XRootDDestination &other); //------------------------------------------------------------------------ // Asynchronous chunk handler //------------------------------------------------------------------------ class ChunkHandler: public XrdCl::ResponseHandler { public: ChunkHandler( XrdCl::PageInfo &&ci ): sem( new XrdSysSemaphore(0) ), chunk(std::move( ci ) ) {} virtual ~ChunkHandler() { delete sem; } virtual void HandleResponse( XrdCl::XRootDStatus *statusval, XrdCl::AnyObject */*response*/ ) { this->status = *statusval; delete statusval; sem->Post(); } XrdSysSemaphore *sem; XrdCl::PageInfo chunk; XrdCl::XRootDStatus status; }; inline XrdCl::XRootDStatus CheckIfRetriable( XrdCl::XRootDStatus &status ) { if( status.IsOK() ) return status; //-------------------------------------------------------------------- // Check if we should re-try the transfer from scratch at a different // data server //-------------------------------------------------------------------- std::string value; if( pFile->GetProperty( "WrtRecoveryRedir", value ) ) { pWrtRecoveryRedir = value; if( pFile->GetProperty( "LastURL", value ) ) pLastURL = value; return XrdCl::XRootDStatus( XrdCl::stError, XrdCl::errRetry ); } return status; } const XrdCl::URL pUrl; XrdCl::File *pFile; uint8_t pParallel; std::queue pChunks; int64_t pSize; std::string pWrtRecoveryRedir; std::string pLastURL; bool pUsePgWrt; const XrdCl::ClassicCopyJob &cpjob; }; //---------------------------------------------------------------------------- //! XRootD destination //---------------------------------------------------------------------------- class XRootDZipDestination: public Destination { public: //------------------------------------------------------------------------ //! Constructor //------------------------------------------------------------------------ XRootDZipDestination( const XrdCl::URL &url, const std::string &fn, int64_t size, uint8_t parallelChunks, XrdCl::ClassicCopyJob &cpjob ): Destination( "zcrc32" ), pUrl( url ), pFilename( fn ), pZip( new XrdCl::ZipArchive() ), pParallel( parallelChunks ), pSize( size ), cpjob( cpjob ) { } //------------------------------------------------------------------------ //! Destructor //------------------------------------------------------------------------ virtual ~XRootDZipDestination() { CleanUpChunks(); delete pZip; //---------------------------------------------------------------------- // If the copy failed and user requested posc and we are dealing with // a local destination, remove the file //---------------------------------------------------------------------- if( pUrl.IsLocalFile() && pPosc && !cpjob.GetResult().IsOK() ) { XrdCl::FileSystem fs( pUrl ); XrdCl::XRootDStatus st = fs.Rm( pUrl.GetPath() ); if( !st.IsOK() ) { XrdCl::Log *log = XrdCl::DefaultEnv::GetLog(); log->Error( XrdCl::UtilityMsg, "Failed to remove local destination" " on failure: %s", st.ToString().c_str() ); } } } //------------------------------------------------------------------------ //! Initialize the destination //------------------------------------------------------------------------ virtual XrdCl::XRootDStatus Initialize() { using namespace XrdCl; Log *log = DefaultEnv::GetLog(); log->Debug( UtilityMsg, "Opening %s for writing", pUrl.GetURL().c_str() ); std::string value; DefaultEnv::GetEnv()->GetString( "WriteRecovery", value ); pZip->SetProperty( "WriteRecovery", value ); OpenFlags::Flags flags = OpenFlags::Update; FileSystem fs( pUrl ); StatInfo *info = nullptr; auto st = fs.Stat( pUrl.GetPath(), info ); if( !st.IsOK() && st.code == errErrorResponse && st.errNo == kXR_NotFound ) flags |= OpenFlags::New; if( pPosc ) flags |= OpenFlags::POSC; if( pCoerce ) flags |= OpenFlags::Force; if( pMakeDir) flags |= OpenFlags::MakePath; st = XrdCl::WaitFor( XrdCl::OpenArchive( pZip, pUrl.GetURL(), flags ) ); if( !st.IsOK() ) return st; std::string cptarget = XrdCl::DefaultCpTarget; XrdCl::DefaultEnv::GetEnv()->GetString( "CpTarget", cptarget ); if( !cptarget.empty() ) { std::string targeturl; pZip->GetProperty( "LastURL", targeturl ); if( symlink( targeturl.c_str(), cptarget.c_str() ) == -1 ) log->Warning( UtilityMsg, "Could not create cp-target symlink: %s", XrdSysE2T( errno ) ); } st = pZip->OpenFile( pFilename, XrdCl::OpenFlags::New | XrdCl::OpenFlags::Write, pSize ); if( !st.IsOK() ) return st; return pCkSumHelper->Initialize(); } //------------------------------------------------------------------------ //! Finalize the destination //------------------------------------------------------------------------ virtual XrdCl::XRootDStatus Finalize() { uint32_t crc32 = 0; auto st = pCkSumHelper->GetRawCheckSum( "zcrc32", crc32 ); if( !st.IsOK() ) return st; pZip->UpdateMetadata( crc32 ); pZip->CloseFile(); return XrdCl::WaitFor( XrdCl::CloseArchive( pZip ) ); } //------------------------------------------------------------------------ //! Put a data chunk at a destination //! //! @param ci chunk information //! @return status of the operation //------------------------------------------------------------------------ virtual XrdCl::XRootDStatus PutChunk( XrdCl::PageInfo &&ci ) { using namespace XrdCl; //---------------------------------------------------------------------- // If there is still place for this chunk to be sent send it //---------------------------------------------------------------------- if( pChunks.size() < pParallel ) return QueueChunk( std::move( ci ) ); //---------------------------------------------------------------------- // We wait for a chunk to be sent so that we have space for the current // one //---------------------------------------------------------------------- std::unique_ptr ch( pChunks.front() ); pChunks.pop(); ch->sem->Wait(); delete [] (char*)ch->chunk.GetBuffer(); if( !ch->status.IsOK() ) { Log *log = DefaultEnv::GetLog(); log->Debug( UtilityMsg, "Unable write %d bytes at %ld from %s: %s", ch->chunk.GetLength(), ch->chunk.GetOffset(), pUrl.GetURL().c_str(), ch->status.ToStr().c_str() ); CleanUpChunks(); //-------------------------------------------------------------------- // Check if we should re-try the transfer from scratch at a different // data server //-------------------------------------------------------------------- return CheckIfRetriable( ch->status ); } return QueueChunk( std::move( ci ) ); } //------------------------------------------------------------------------ //! Get size //------------------------------------------------------------------------ virtual int64_t GetSize() { return -1; } //------------------------------------------------------------------------ //! Clean up the chunks that are flying //------------------------------------------------------------------------ void CleanUpChunks() { while( !pChunks.empty() ) { ChunkHandler *ch = pChunks.front(); pChunks.pop(); ch->sem->Wait(); delete [] (char *)ch->chunk.GetBuffer(); delete ch; } } //------------------------------------------------------------------------ // Queue a chunk //------------------------------------------------------------------------ XrdCl::XRootDStatus QueueChunk( XrdCl::PageInfo &&ci ) { // we are writing chunks in order so we can calc the checksum // in case of local files if( pCkSumHelper ) pCkSumHelper->Update( ci.GetBuffer(), ci.GetLength() ); ChunkHandler *ch = new ChunkHandler( std::move( ci ) ); XrdCl::XRootDStatus st; //---------------------------------------------------------------------- // TODO // In order to use PgWrite with ZIP append we need first to implement // PgWriteV!!! //---------------------------------------------------------------------- st = pZip->Write( ch->chunk.GetLength(), ch->chunk.GetBuffer(), ch ); if( !st.IsOK() ) { CleanUpChunks(); delete [] (char*)ch->chunk.GetBuffer(); delete ch; return st; } pChunks.push( ch ); return XrdCl::XRootDStatus(); } //------------------------------------------------------------------------ //! Flush chunks that might have been queues //------------------------------------------------------------------------ virtual XrdCl::XRootDStatus Flush() { XrdCl::XRootDStatus st; while( !pChunks.empty() ) { ChunkHandler *ch = pChunks.front(); pChunks.pop(); ch->sem->Wait(); if( !ch->status.IsOK() ) { //-------------------------------------------------------------------- // Check if we should re-try the transfer from scratch at a different // data server //-------------------------------------------------------------------- st = CheckIfRetriable( ch->status ); } delete [] (char *)ch->chunk.GetBuffer(); delete ch; } return st; } //------------------------------------------------------------------------ //! Get check sum //------------------------------------------------------------------------ virtual XrdCl::XRootDStatus GetCheckSum( std::string &checkSum, std::string &checkSumType ) { return XrdCl::XRootDStatus( XrdCl::stError, XrdCl::errNotSupported ); } //------------------------------------------------------------------------ //! Set extended attributes //------------------------------------------------------------------------ virtual XrdCl::XRootDStatus SetXAttr( const std::vector &xattrs ) { return XrdCl::XRootDStatus( XrdCl::stError, XrdCl::errNotSupported ); } //------------------------------------------------------------------------ //! Get last URL //------------------------------------------------------------------------ const std::string& GetLastURL() const { return pLastURL; } //------------------------------------------------------------------------ //! Get write-recovery redirector //------------------------------------------------------------------------ const std::string& GetWrtRecoveryRedir() const { return pWrtRecoveryRedir; } private: XRootDZipDestination(const XRootDDestination &other); XRootDZipDestination &operator = (const XRootDDestination &other); //------------------------------------------------------------------------ // Asynchronous chunk handler //------------------------------------------------------------------------ class ChunkHandler: public XrdCl::ResponseHandler { public: ChunkHandler( XrdCl::PageInfo &&ci ): sem( new XrdSysSemaphore(0) ), chunk( std::move( ci ) ) {} virtual ~ChunkHandler() { delete sem; } virtual void HandleResponse( XrdCl::XRootDStatus *statusval, XrdCl::AnyObject */*response*/ ) { this->status = *statusval; delete statusval; sem->Post(); } XrdSysSemaphore *sem; XrdCl::PageInfo chunk; XrdCl::XRootDStatus status; }; inline XrdCl::XRootDStatus CheckIfRetriable( XrdCl::XRootDStatus &status ) { if( status.IsOK() ) return status; //-------------------------------------------------------------------- // Check if we should re-try the transfer from scratch at a different // data server //-------------------------------------------------------------------- std::string value; if( pZip->GetProperty( "WrtRecoveryRedir", value ) ) { pWrtRecoveryRedir = value; if( pZip->GetProperty( "LastURL", value ) ) pLastURL = value; return XrdCl::XRootDStatus( XrdCl::stError, XrdCl::errRetry ); } return status; } const XrdCl::URL pUrl; std::string pFilename; XrdCl::ZipArchive *pZip; uint8_t pParallel; std::queue pChunks; int64_t pSize; std::string pWrtRecoveryRedir; std::string pLastURL; XrdCl::ClassicCopyJob &cpjob; }; } //------------------------------------------------------------------------------ // Get current time in nanoseconds //------------------------------------------------------------------------------ inline std::chrono::nanoseconds time_nsec() { using namespace std::chrono; auto since_epoch = high_resolution_clock::now().time_since_epoch(); return duration_cast( since_epoch ); } //------------------------------------------------------------------------------ // Convert seconds to nanoseconds //------------------------------------------------------------------------------ inline long long to_nsec( long long sec ) { return sec * 1000000000; } //------------------------------------------------------------------------------ // Sleep for # nanoseconds //------------------------------------------------------------------------------ inline void sleep_nsec( long long nsec ) { #if __cplusplus >= 201103L using namespace std::chrono; std::this_thread::sleep_for( nanoseconds( nsec ) ); #else timespec req; req.tv_sec = nsec / to_nsec( 1 ); req.tv_nsec = nsec % to_nsec( 1 ); nanosleep( &req, 0 ); #endif } namespace XrdCl { //---------------------------------------------------------------------------- // Constructor //---------------------------------------------------------------------------- ClassicCopyJob::ClassicCopyJob( uint16_t jobId, PropertyList *jobProperties, PropertyList *jobResults ): CopyJob( jobId, jobProperties, jobResults ) { Log *log = DefaultEnv::GetLog(); log->Debug( UtilityMsg, "Creating a classic copy job, from %s to %s", GetSource().GetURL().c_str(), GetTarget().GetURL().c_str() ); } //---------------------------------------------------------------------------- // Run the copy job //---------------------------------------------------------------------------- XRootDStatus ClassicCopyJob::Run( CopyProgressHandler *progress ) { Log *log = DefaultEnv::GetLog(); std::string checkSumMode; std::string checkSumType; std::string checkSumPreset; std::string zipSource; uint16_t parallelChunks; uint32_t chunkSize; uint64_t blockSize; bool posc, force, coerce, makeDir, dynamicSource, zip, xcp, preserveXAttr, rmOnBadCksum, continue_, zipappend, doserver; int32_t nbXcpSources; long long xRate; long long xRateThreshold; uint16_t cpTimeout; std::vector addcksums; pProperties->Get( "checkSumMode", checkSumMode ); pProperties->Get( "checkSumType", checkSumType ); pProperties->Get( "checkSumPreset", checkSumPreset ); pProperties->Get( "parallelChunks", parallelChunks ); pProperties->Get( "chunkSize", chunkSize ); pProperties->Get( "posc", posc ); pProperties->Get( "force", force ); pProperties->Get( "coerce", coerce ); pProperties->Get( "makeDir", makeDir ); pProperties->Get( "dynamicSource", dynamicSource ); pProperties->Get( "zipArchive", zip ); pProperties->Get( "xcp", xcp ); pProperties->Get( "xcpBlockSize", blockSize ); pProperties->Get( "preserveXAttr", preserveXAttr ); pProperties->Get( "xrate", xRate ); pProperties->Get( "xrateThreshold", xRateThreshold ); pProperties->Get( "rmOnBadCksum", rmOnBadCksum ); pProperties->Get( "continue", continue_ ); pProperties->Get( "cpTimeout", cpTimeout ); pProperties->Get( "zipAppend", zipappend ); pProperties->Get( "addcksums", addcksums ); pProperties->Get( "doServer", doserver ); if( zip ) pProperties->Get( "zipSource", zipSource ); if( xcp ) pProperties->Get( "nbXcpSources", nbXcpSources ); if( force && continue_ ) return SetResult( stError, errInvalidArgs, EINVAL, "Invalid argument combination: continue + force." ); if( zipappend && ( continue_ || force ) ) return SetResult( stError, errInvalidArgs, EINVAL, "Invalid argument combination: ( continue | force ) + zip-append." ); //-------------------------------------------------------------------------- // Start the cp t/o timer if necessary //-------------------------------------------------------------------------- std::unique_ptr cptimer; if( cpTimeout ) cptimer.reset( new timer_sec_t() ); //-------------------------------------------------------------------------- // Remove on bad checksum implies that POSC semantics has to be enabled //-------------------------------------------------------------------------- if( rmOnBadCksum ) posc = true; //-------------------------------------------------------------------------- // Resolve the 'auto' checksum type. //-------------------------------------------------------------------------- if( checkSumType == "auto" ) { checkSumType = Utils::InferChecksumType( GetSource(), GetTarget(), zip ); if( checkSumType.empty() ) return SetResult( stError, errCheckSumError, ENOTSUP, "Could not infer checksum type." ); else log->Info( UtilityMsg, "Using inferred checksum type: %s.", checkSumType.c_str() ); } if( cptimer && cptimer->elapsed() > cpTimeout ) // check the CP timeout return SetResult( stError, errOperationExpired, 0, "CPTimeout exceeded." ); //-------------------------------------------------------------------------- // Initialize the source and the destination //-------------------------------------------------------------------------- std::unique_ptr src; if( xcp ) src.reset( new XRootDSourceXCp( &GetSource(), chunkSize, parallelChunks, nbXcpSources, blockSize ) ); else if( zip ) // TODO make zip work for xcp src.reset( new XRootDSourceZip( zipSource, &GetSource(), chunkSize, parallelChunks, checkSumType, addcksums , doserver) ); else if( GetSource().GetProtocol() == "stdio" ) src.reset( new StdInSource( checkSumType, chunkSize, addcksums ) ); else { if( dynamicSource ) src.reset( new XRootDSourceDynamic( &GetSource(), chunkSize, checkSumType, addcksums ) ); else src.reset( new XRootDSource( &GetSource(), chunkSize, parallelChunks, checkSumType, addcksums, doserver ) ); } XRootDStatus st = src->Initialize(); if( !st.IsOK() ) return SourceError( st ); uint64_t size = src->GetSize() >= 0 ? src->GetSize() : 0; if( cptimer && cptimer->elapsed() > cpTimeout ) // check the CP timeout return SetResult( stError, errOperationExpired, 0, "CPTimeout exceeded." ); std::unique_ptr dest; URL newDestUrl( GetTarget() ); if( GetTarget().GetProtocol() == "stdio" ) dest.reset( new StdOutDestination( checkSumType ) ); else if( zipappend ) { std::string fn = GetSource().GetPath(); size_t pos = fn.rfind( '/' ); if( pos != std::string::npos ) fn = fn.substr( pos + 1 ); int64_t size = src->GetSize(); dest.reset( new XRootDZipDestination( newDestUrl, fn, size, parallelChunks, *this ) ); } //-------------------------------------------------------------------------- // For xrootd destination build the oss.asize hint //-------------------------------------------------------------------------- else { if( src->GetSize() >= 0 ) { URL::ParamsMap params = newDestUrl.GetParams(); std::ostringstream o; o << src->GetSize(); params["oss.asize"] = o.str(); newDestUrl.SetParams( params ); // makeDir = true; // Backward compatibility for xroot destinations!!! } dest.reset( new XRootDDestination( newDestUrl, parallelChunks, checkSumType, *this ) ); } dest->SetForce( force ); dest->SetPOSC( posc ); dest->SetCoerce( coerce ); dest->SetMakeDir( makeDir ); dest->SetContinue( continue_ ); st = dest->Initialize(); if( !st.IsOK() ) return DestinationError( st ); if( cptimer && cptimer->elapsed() > cpTimeout ) // check the CP timeout return SetResult( stError, errOperationExpired, 0, "CPTimeout exceeded." ); //-------------------------------------------------------------------------- // Copy the chunks //-------------------------------------------------------------------------- if( continue_ ) { size -= dest->GetSize(); XrdCl::XRootDStatus st = src->StartAt( dest->GetSize() ); if( !st.IsOK() ) return SetResult( st ); } PageInfo pageInfo; uint64_t total_processed = 0; uint64_t processed = 0; auto start = time_nsec(); uint16_t threshold_interval = parallelChunks; bool threshold_draining = false; timer_nsec_t threshold_timer; while( 1 ) { st = src->GetChunk( pageInfo ); if( !st.IsOK() ) return SourceError( st); if( st.IsOK() && st.code == suDone ) break; if( cptimer && cptimer->elapsed() > cpTimeout ) // check the CP timeout return SetResult( stError, errOperationExpired, 0, "CPTimeout exceeded." ); if( xRate ) { auto elapsed = ( time_nsec() - start ).count(); double transferred = total_processed + pageInfo.GetLength(); double expected = double( xRate ) / to_nsec( 1 ) * elapsed; //---------------------------------------------------------------------- // check if our transfer rate didn't exceeded the limit // (we are too fast) //---------------------------------------------------------------------- if( elapsed && // make sure elapsed time is greater than 0 transferred > expected ) { auto nsec = ( transferred / xRate * to_nsec( 1 ) ) - elapsed; sleep_nsec( nsec ); } } if( xRateThreshold ) { auto elapsed = threshold_timer.elapsed(); double transferred = processed + pageInfo.GetLength(); double expected = double( xRateThreshold ) / to_nsec( 1 ) * elapsed; //---------------------------------------------------------------------- // check if our transfer rate dropped below the threshold // (we are too slow) //---------------------------------------------------------------------- if( elapsed && // make sure elapsed time is greater than 0 transferred < expected && threshold_interval == 0 ) // we check every # parallelChunks { if( !threshold_draining ) { log->Warning( UtilityMsg, "Transfer rate dropped below requested ehreshold," " trying different source!" ); XRootDStatus st = src->TryOtherServer(); if( !st.IsOK() ) return SetResult( stError, errThresholdExceeded, 0, "The transfer rate dropped below " "requested threshold!" ); threshold_draining = true; // before the next measurement we need to drain // all the chunks that will come from the old server } else // now that all the chunks from the old server have { // been received we can start another measurement processed = 0; threshold_timer.reset(); threshold_interval = parallelChunks; threshold_draining = false; } } threshold_interval = threshold_interval > 0 ? threshold_interval - 1 : parallelChunks; } total_processed += pageInfo.GetLength(); processed += pageInfo.GetLength(); st = dest->PutChunk( std::move( pageInfo ) ); if( !st.IsOK() ) { if( st.code == errRetry ) { pResults->Set( "LastURL", dest->GetLastURL() ); pResults->Set( "WrtRecoveryRedir", dest->GetWrtRecoveryRedir() ); return SetResult( st ); } return DestinationError( st ); } if( progress ) { progress->JobProgress( pJobId, total_processed, size ); if( progress->ShouldCancel( pJobId ) ) return SetResult( stError, errOperationInterrupted, kXR_Cancelled, "The copy-job has been cancelled!" ); } } st = dest->Flush(); if( !st.IsOK() ) return DestinationError( st ); //-------------------------------------------------------------------------- // Copy extended attributes //-------------------------------------------------------------------------- if( preserveXAttr && Utils::HasXAttr( GetSource() ) && Utils::HasXAttr( GetTarget() ) ) { std::vector xattrs; st = src->GetXAttr( xattrs ); if( !st.IsOK() ) return SourceError( st ); st = dest->SetXAttr( xattrs ); if( !st.IsOK() ) return DestinationError( st ); } //-------------------------------------------------------------------------- // The size of the source is known and not enough data has been transferred // to the destination //-------------------------------------------------------------------------- if( src->GetSize() >= 0 && size != total_processed ) { log->Error( UtilityMsg, "The declared source size is %ld bytes, but " "received %ld bytes.", size, total_processed ); return SetResult( stError, errDataError ); } pResults->Set( "size", total_processed ); //-------------------------------------------------------------------------- // Finalize the destination //-------------------------------------------------------------------------- st = dest->Finalize(); if( !st.IsOK() ) return DestinationError( st ); //-------------------------------------------------------------------------- // Verify the checksums if needed //-------------------------------------------------------------------------- if( checkSumMode != "none" ) { log->Debug( UtilityMsg, "Attempting checksum calculation, mode: %s.", checkSumMode.c_str() ); std::string sourceCheckSum; std::string targetCheckSum; if( cptimer && cptimer->elapsed() > cpTimeout ) // check the CP timeout return SetResult( stError, errOperationExpired, 0, "CPTimeout exceeded." ); //------------------------------------------------------------------------ // Get the check sum at source //------------------------------------------------------------------------ timeval oStart, oEnd; XRootDStatus st; if( checkSumMode == "end2end" || checkSumMode == "source" || !checkSumPreset.empty() ) { gettimeofday( &oStart, 0 ); if( !checkSumPreset.empty() ) { sourceCheckSum = checkSumType + ":"; sourceCheckSum += Utils::NormalizeChecksum( checkSumType, checkSumPreset ); } else { st = src->GetCheckSum( sourceCheckSum, checkSumType ); } gettimeofday( &oEnd, 0 ); if( !st.IsOK() ) return SourceError( st ); pResults->Set( "sourceCheckSum", sourceCheckSum ); } if( !addcksums.empty() ) pResults->Set( "additionalCkeckSum", src->GetAddCks() ); if( cptimer && cptimer->elapsed() > cpTimeout ) // check the CP timeout return SetResult( stError, errOperationExpired, 0, "CPTimeout exceeded." ); //------------------------------------------------------------------------ // Get the check sum at destination //------------------------------------------------------------------------ timeval tStart, tEnd; if( checkSumMode == "end2end" || checkSumMode == "target" ) { gettimeofday( &tStart, 0 ); st = dest->GetCheckSum( targetCheckSum, checkSumType ); if( !st.IsOK() ) return DestinationError( st ); gettimeofday( &tEnd, 0 ); pResults->Set( "targetCheckSum", targetCheckSum ); } if( cptimer && cptimer->elapsed() > cpTimeout ) // check the CP timeout return SetResult( stError, errOperationExpired, 0, "CPTimeout exceeded." ); //------------------------------------------------------------------------ // Make sure the checksums are both lower case //------------------------------------------------------------------------ auto sanitize_cksum = []( char c ) { std::locale loc; if( std::isalpha( c ) ) return std::tolower( c, loc ); return c; }; std::transform( sourceCheckSum.begin(), sourceCheckSum.end(), sourceCheckSum.begin(), sanitize_cksum ); std::transform( targetCheckSum.begin(), targetCheckSum.end(), targetCheckSum.begin(), sanitize_cksum ); //------------------------------------------------------------------------ // Compare and inform monitoring //------------------------------------------------------------------------ if( !sourceCheckSum.empty() && !targetCheckSum.empty() ) { bool match = false; if( sourceCheckSum == targetCheckSum ) match = true; Monitor *mon = DefaultEnv::GetMonitor(); if( mon ) { Monitor::CheckSumInfo i; i.transfer.origin = &GetSource(); i.transfer.target = &GetTarget(); i.cksum = sourceCheckSum; i.oTime = Utils::GetElapsedMicroSecs( oStart, oEnd ); i.tTime = Utils::GetElapsedMicroSecs( tStart, tEnd ); i.isOK = match; mon->Event( Monitor::EvCheckSum, &i ); } if( !match ) { if( rmOnBadCksum ) { FileSystem fs( newDestUrl ); st = fs.Rm( newDestUrl.GetPath() ); if( !st.IsOK() ) log->Error( UtilityMsg, "Invalid checksum: failed to remove the target file: %s", st.ToString().c_str() ); else log->Info( UtilityMsg, "Target file removed due to bad checksum!" ); } st = dest->Finalize(); if( !st.IsOK() ) log->Error( UtilityMsg, "Failed to finalize the destination: %s", st.ToString().c_str() ); return SetResult( stError, errCheckSumError, 0 ); } log->Info( UtilityMsg, "Checksum verification: succeeded." ); } } return SetResult(); } } xrootd-5.6.9/src/XrdCl/XrdClClassicCopyJob.hh000066400000000000000000000075211457266313600210240ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2012 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . //------------------------------------------------------------------------------ #ifndef __XRD_CL_CLASSIC_COPY_JOB_HH__ #define __XRD_CL_CLASSIC_COPY_JOB_HH__ #include "XrdCl/XrdClCopyProcess.hh" #include "XrdCl/XrdClCopyJob.hh" namespace XrdCl { class ClassicCopyJob: public CopyJob { public: //------------------------------------------------------------------------ // Constructor //------------------------------------------------------------------------ ClassicCopyJob( uint16_t jobId, PropertyList *jobProperties, PropertyList *jobResults ); //------------------------------------------------------------------------ //! Run the copy job //! //! @param progress the handler to be notified about the copy progress //! @return status of the copy operation //------------------------------------------------------------------------ virtual XRootDStatus Run( CopyProgressHandler *progress = 0 ); //------------------------------------------------------------------------ // Get the final result //------------------------------------------------------------------------ inline const XRootDStatus& GetResult() const { return result; } private: //------------------------------------------------------------------------ // Update the final result so it is clear that it is a source error //------------------------------------------------------------------------ inline XrdCl::XRootDStatus& SourceError( XrdCl::XRootDStatus &status ) { std::string msg = status.GetErrorMessage(); msg += " (source)"; status.SetErrorMessage( msg ); result = status; return status; } //------------------------------------------------------------------------ // Update the final result do it is clear that it is a destination error //------------------------------------------------------------------------ inline XrdCl::XRootDStatus& DestinationError( XrdCl::XRootDStatus &status ) { std::string msg = status.GetErrorMessage(); msg += " (destination)"; status.SetErrorMessage( msg ); result = status; return status; } //------------------------------------------------------------------------ // Set the final result //------------------------------------------------------------------------ template inline XRootDStatus& SetResult( Args&&... args ) { result = XrdCl::XRootDStatus( std::forward(args)... ); return result; } //------------------------------------------------------------------------ // The final result //------------------------------------------------------------------------ XRootDStatus result; }; } #endif // __XRD_CL_CLASSIC_COPY_JOB_HH__ xrootd-5.6.9/src/XrdCl/XrdClConstants.hh000066400000000000000000000231601457266313600201260ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2012 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . //------------------------------------------------------------------------------ #ifndef __XRD_CL_CONSTANTS_HH__ #define __XRD_CL_CONSTANTS_HH__ #include #include #include #include namespace XrdCl { //---------------------------------------------------------------------------- // Log message types //---------------------------------------------------------------------------- const uint64_t AppMsg = 0x0000000000000001ULL; const uint64_t UtilityMsg = 0x0000000000000002ULL; const uint64_t FileMsg = 0x0000000000000004ULL; const uint64_t PollerMsg = 0x0000000000000008ULL; const uint64_t PostMasterMsg = 0x0000000000000010ULL; const uint64_t XRootDTransportMsg = 0x0000000000000020ULL; const uint64_t TaskMgrMsg = 0x0000000000000040ULL; const uint64_t XRootDMsg = 0x0000000000000080ULL; const uint64_t FileSystemMsg = 0x0000000000000100ULL; const uint64_t AsyncSockMsg = 0x0000000000000200ULL; const uint64_t JobMgrMsg = 0x0000000000000400ULL; const uint64_t PlugInMgrMsg = 0x0000000000000800ULL; const uint64_t ExDbgMsg = 0x0000000000001000ULL; //special type debugging extra-hard problems const uint64_t TlsMsg = 0x0000000000002000ULL; const uint64_t ZipMsg = 0x0000000000004000ULL; //---------------------------------------------------------------------------- // Environment settings //---------------------------------------------------------------------------- const int DefaultSubStreamsPerChannel = 1; const int DefaultConnectionWindow = 120; const int DefaultConnectionRetry = 5; const int DefaultRequestTimeout = 1800; const int DefaultStreamTimeout = 60; const int DefaultTimeoutResolution = 15; const int DefaultStreamErrorWindow = 1800; const int DefaultRunForkHandler = 1; const int DefaultRedirectLimit = 16; const int DefaultWorkerThreads = 3; const int DefaultCPChunkSize = 8388608; const int DefaultCPParallelChunks = 4; const int DefaultDataServerTTL = 300; const int DefaultLoadBalancerTTL = 1200; const int DefaultCPInitTimeout = 600; const int DefaultCPTPCTimeout = 1800; const int DefaultCPTimeout = 0; const int DefaultTCPKeepAlive = 0; const int DefaultTCPKeepAliveTime = 7200; const int DefaultTCPKeepAliveInterval = 75; const int DefaultTCPKeepAliveProbes = 9; const int DefaultMultiProtocol = 0; const int DefaultParallelEvtLoop = 10; const int DefaultMetalinkProcessing = 1; const int DefaultLocalMetalinkFile = 0; const int DefaultXRateThreshold = 0; const int DefaultXCpBlockSize = 134217728; // DefaultCPChunkSize * DefaultCPParallelChunks * 2 #ifdef __APPLE__ // we don't have corking on osx so we cannot turn of nagle const int DefaultNoDelay = 0; #else const int DefaultNoDelay = 1; #endif const int DefaultAioSignal = 0; const int DefaultPreferIPv4 = 0; const int DefaultMaxMetalinkWait = 60; const int DefaultPreserveLocateTried = 1; const int DefaultNotAuthorizedRetryLimit = 3; const int DefaultPreserveXAttrs = 0; const int DefaultNoTlsOK = 0; const int DefaultTlsNoData = 0; const int DefaultTlsMetalink = 0; const int DefaultZipMtlnCksum = 0; const int DefaultIPNoShuffle = 0; const int DefaultWantTlsOnNoPgrw = 0; const int DefaultRetryWrtAtLBLimit = 3; const int DefaultCpRetry = 0; const int DefaultCpUsePgWrtRd = 1; const char * const DefaultPollerPreference = "built-in"; const char * const DefaultNetworkStack = "IPAuto"; const char * const DefaultClientMonitor = ""; const char * const DefaultClientMonitorParam = ""; const char * const DefaultPlugInConfDir = ""; const char * const DefaultPlugIn = ""; const char * const DefaultReadRecovery = "true"; const char * const DefaultWriteRecovery = "true"; const char * const DefaultOpenRecovery = "true"; const char * const DefaultGlfnRedirector = ""; const char * const DefaultTlsDbgLvl = "OFF"; const char * const DefaultClConfDir = ""; const char * const DefaultClConfFile = ""; const char * const DefaultCpTarget = ""; const char * const DefaultCpRetryPolicy = "force"; inline static std::string to_lower( std::string str ) { std::transform( str.begin(), str.end(), str.begin(), ::tolower ); return str; } static std::unordered_map theDefaultInts { { to_lower( "SubStreamsPerChannel" ), DefaultSubStreamsPerChannel }, { to_lower( "ConnectionWindow" ), DefaultConnectionWindow }, { to_lower( "ConnectionRetry" ), DefaultConnectionRetry }, { to_lower( "RequestTimeout" ), DefaultRequestTimeout }, { to_lower( "StreamTimeout" ), DefaultStreamTimeout }, { to_lower( "TimeoutResolution" ), DefaultTimeoutResolution }, { to_lower( "StreamErrorWindow" ), DefaultStreamErrorWindow }, { to_lower( "RunForkHandler" ), DefaultRunForkHandler }, { to_lower( "RedirectLimit" ), DefaultRedirectLimit }, { to_lower( "WorkerThreads" ), DefaultWorkerThreads }, { to_lower( "CPChunkSize" ), DefaultCPChunkSize }, { to_lower( "CPParallelChunks" ), DefaultCPParallelChunks }, { to_lower( "DataServerTTL" ), DefaultDataServerTTL }, { to_lower( "LoadBalancerTTL" ), DefaultLoadBalancerTTL }, { to_lower( "CPInitTimeout" ), DefaultCPInitTimeout }, { to_lower( "CPTPCTimeout" ), DefaultCPTPCTimeout }, { to_lower( "CPTimeout" ), DefaultCPTimeout }, { to_lower( "TCPKeepAlive" ), DefaultTCPKeepAlive }, { to_lower( "TCPKeepAliveTime" ), DefaultTCPKeepAliveTime }, { to_lower( "TCPKeepAliveInterval" ), DefaultTCPKeepAliveInterval }, { to_lower( "TCPKeepAliveProbes" ), DefaultTCPKeepAliveProbes }, { to_lower( "MultiProtocol" ), DefaultMultiProtocol }, { to_lower( "ParallelEvtLoop" ), DefaultParallelEvtLoop }, { to_lower( "MetalinkProcessing" ), DefaultMetalinkProcessing }, { to_lower( "LocalMetalinkFile" ), DefaultLocalMetalinkFile }, { to_lower( "XRateThreshold" ), DefaultXRateThreshold }, { to_lower( "XCpBlockSize" ), DefaultXCpBlockSize }, { to_lower( "NoDelay" ), DefaultNoDelay }, { to_lower( "AioSignal" ), DefaultAioSignal }, { to_lower( "PreferIPv4" ), DefaultPreferIPv4 }, { to_lower( "MaxMetalinkWait" ), DefaultMaxMetalinkWait }, { to_lower( "PreserveLocateTried" ), DefaultPreserveLocateTried }, { to_lower( "NotAuthorizedRetryLimit" ), DefaultNotAuthorizedRetryLimit }, { to_lower( "PreserveXAttrs" ), DefaultPreserveXAttrs }, { to_lower( "NoTlsOK" ), DefaultNoTlsOK }, { to_lower( "TlsNoData" ), DefaultTlsNoData }, { to_lower( "TlsMetalink" ), DefaultTlsMetalink }, { to_lower( "ZipMtlnCksum" ), DefaultZipMtlnCksum }, { to_lower( "IPNoShuffle" ), DefaultIPNoShuffle }, { to_lower( "WantTlsOnNoPgrw" ), DefaultWantTlsOnNoPgrw }, { to_lower( "RetryWrtAtLBLimit" ), DefaultRetryWrtAtLBLimit } }; static std::unordered_map theDefaultStrs { { to_lower( "PollerPreference" ), DefaultPollerPreference }, { to_lower( "NetworkStack" ), DefaultNetworkStack }, { to_lower( "ClientMonitor" ), DefaultClientMonitor }, { to_lower( "ClientMonitorParam" ), DefaultClientMonitorParam }, { to_lower( "PlugInConfDir" ), DefaultPlugInConfDir }, { to_lower( "PlugIn" ), DefaultPlugIn }, { to_lower( "ReadRecovery" ), DefaultReadRecovery }, { to_lower( "WriteRecovery" ), DefaultWriteRecovery }, { to_lower( "OpenRecovery" ), DefaultOpenRecovery }, { to_lower( "GlfnRedirector" ), DefaultGlfnRedirector }, { to_lower( "TlsDbgLvl" ), DefaultTlsDbgLvl }, { to_lower( "ClConfDir" ), DefaultClConfDir }, { to_lower( "DefaultClConfFile" ), DefaultClConfFile }, { to_lower( "CpTarget" ), DefaultCpTarget } }; } #endif // __XRD_CL_CONSTANTS_HH__ xrootd-5.6.9/src/XrdCl/XrdClCopy.cc000066400000000000000000001005141457266313600170510ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2014 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #include "XrdApps/XrdCpConfig.hh" #include "XrdApps/XrdCpFile.hh" #include "XrdCl/XrdClConstants.hh" #include "XrdCl/XrdClCopyProcess.hh" #include "XrdCl/XrdClDefaultEnv.hh" #include "XrdCl/XrdClLog.hh" #include "XrdCl/XrdClFileSystem.hh" #include "XrdCl/XrdClUtils.hh" #include "XrdCl/XrdClDlgEnv.hh" #include "XrdSys/XrdSysE2T.hh" #include "XrdSys/XrdSysPthread.hh" #include #include #include #include //------------------------------------------------------------------------------ // Progress notifier //------------------------------------------------------------------------------ class ProgressDisplay: public XrdCl::CopyProgressHandler { public: //-------------------------------------------------------------------------- //! Constructor //-------------------------------------------------------------------------- ProgressDisplay(): pPrevious(0), pPrintProgressBar(true), pPrintSourceCheckSum(false), pPrintTargetCheckSum(false), pPrintAdditionalCheckSum(false) {} //-------------------------------------------------------------------------- //! Begin job //-------------------------------------------------------------------------- virtual void BeginJob( uint16_t jobNum, uint16_t jobTotal, const XrdCl::URL *source, const XrdCl::URL *destination ) { XrdSysMutexHelper scopedLock( pMutex ); if( pPrintProgressBar ) { if( jobTotal > 1 ) { std::cerr << "Job: " << jobNum << "/" << jobTotal << std::endl; std::cerr << "Source: " << source->GetURL() << std::endl; std::cerr << "Target: " << destination->GetURL() << std::endl; } } pPrevious = 0; JobData d; d.started = time(0); d.source = source; d.target = destination; pOngoingJobs[jobNum] = d; } //-------------------------------------------------------------------------- //! End job //-------------------------------------------------------------------------- virtual void EndJob( uint16_t jobNum, const XrdCl::PropertyList *results ) { XrdSysMutexHelper scopedLock( pMutex ); std::map::iterator it = pOngoingJobs.find( jobNum ); if( it == pOngoingJobs.end() ) return; JobData &d = it->second; // make sure the last available status was printed, which may not be // the case when processing stdio since we throttle printing and don't // know the total size JobProgress( jobNum, d.bytesProcessed, d.bytesTotal ); if( pPrintProgressBar ) { if( pOngoingJobs.size() > 1 ) std::cerr << "\r" << std::string(70, ' ') << "\r"; else std::cerr << std::endl; } XrdCl::XRootDStatus st; results->Get( "status", st ); if( !st.IsOK() ) { pOngoingJobs.erase(it); return; } std::string checkSum; uint64_t size; results->Get( "size", size ); if( pPrintSourceCheckSum ) { results->Get( "sourceCheckSum", checkSum ); PrintCheckSum( d.source, checkSum, size ); } if( pPrintTargetCheckSum ) { results->Get( "targetCheckSum", checkSum ); PrintCheckSum( d.target, checkSum, size ); } if( pPrintAdditionalCheckSum ) { std::vector addcksums; results->Get( "additionalCkeckSum", addcksums ); for( auto &cks : addcksums ) PrintCheckSum( d.source, cks, size ); } pOngoingJobs.erase(it); } //-------------------------------------------------------------------------- //! Get progress bar //-------------------------------------------------------------------------- std::string GetProgressBar( time_t now ) { JobData &d = pOngoingJobs.begin()->second; uint64_t speed = 0; if( now-d.started ) speed = d.bytesProcessed/(now-d.started); else speed = d.bytesProcessed; std::string bar; int prog = 0; int proc = 0; if( d.bytesTotal ) { prog = (int)((double)d.bytesProcessed/d.bytesTotal*50); proc = (int)((double)d.bytesProcessed/d.bytesTotal*100); } else { prog = 50; proc = 100; } bar.append( prog, '=' ); if( prog < 50 ) bar += ">"; std::ostringstream o; o << "[" << XrdCl::Utils::BytesToString(d.bytesProcessed) << "B/"; o << XrdCl::Utils::BytesToString(d.bytesTotal) << "B]"; o << "[" << std::setw(3) << std::right << proc << "%]"; o << "[" << std::setw(50) << std::left; o << bar; o << "]"; o << "[" << XrdCl::Utils::BytesToString(speed) << "B/s] "; return o.str(); } //-------------------------------------------------------------------------- //! Get sumary bar //-------------------------------------------------------------------------- std::string GetSummaryBar( time_t now ) { std::map::iterator it; std::ostringstream o; for( it = pOngoingJobs.begin(); it != pOngoingJobs.end(); ++it ) { JobData &d = it->second; uint16_t jobNum = it->first; uint64_t speed = 0; if( now-d.started ) speed = d.bytesProcessed/(now-d.started); int proc = 0; if( d.bytesTotal ) proc = (int)((double)d.bytesProcessed/d.bytesTotal*100); else proc = 100; o << "[#" << jobNum << ": "; o << proc << "% "; o << XrdCl::Utils::BytesToString(speed) << "B/s] "; } o << " "; return o.str(); } //-------------------------------------------------------------------------- //! Job progress //-------------------------------------------------------------------------- virtual void JobProgress( uint16_t jobNum, uint64_t bytesProcessed, uint64_t bytesTotal ) { XrdSysMutexHelper scopedLock( pMutex ); if( pPrintProgressBar ) { time_t now = time(0); if( (now - pPrevious < 1) && (bytesProcessed != bytesTotal) ) return; pPrevious = now; std::map::iterator it = pOngoingJobs.find( jobNum ); if( it == pOngoingJobs.end() ) return; JobData &d = it->second; d.bytesProcessed = bytesProcessed; d.bytesTotal = bytesTotal; std::string progress; if( pOngoingJobs.size() == 1 ) progress = GetProgressBar( now ); else progress = GetSummaryBar( now ); std::cerr << "\r" << progress << std::flush; } } //-------------------------------------------------------------------------- //! Print the checksum //-------------------------------------------------------------------------- void PrintCheckSum( const XrdCl::URL *url, const std::string &checkSum, uint64_t size ) { if( checkSum.empty() ) return; std::string::size_type i = checkSum.find( ':' ); std::cerr << checkSum.substr( 0, i+1 ) << " "; std::cerr << checkSum.substr( i+1, checkSum.length()-i ) << " "; if( url->IsLocalFile() ) std::cerr << url->GetPath() << " "; else { std::cerr << url->GetProtocol() << "://" << url->GetHostId(); std::cerr << url->GetPath() << " "; } std::cerr << size; std::cerr << std::endl; } //-------------------------------------------------------------------------- // Printing flags //-------------------------------------------------------------------------- void PrintProgressBar( bool print ) { pPrintProgressBar = print; } void PrintSourceCheckSum( bool print ) { pPrintSourceCheckSum = print; } void PrintTargetCheckSum( bool print ) { pPrintTargetCheckSum = print; } void PrintAdditionalCheckSum( bool print ) { pPrintAdditionalCheckSum = print; } private: struct JobData { JobData(): bytesProcessed(0), bytesTotal(0), started(0), source(0), target(0) {} uint64_t bytesProcessed; uint64_t bytesTotal; time_t started; const XrdCl::URL *source; const XrdCl::URL *target; }; time_t pPrevious; bool pPrintProgressBar; bool pPrintSourceCheckSum; bool pPrintTargetCheckSum; bool pPrintAdditionalCheckSum; std::map pOngoingJobs; XrdSysRecMutex pMutex; }; //------------------------------------------------------------------------------ // Check if we support all the specified user options //------------------------------------------------------------------------------ bool AllOptionsSupported( XrdCpConfig *config ) { if( config->pHost ) { std::cerr << "SOCKS Proxies are not yet supported" << std::endl; return false; } return true; } //------------------------------------------------------------------------------ // Append extra cgi info to existing URL //------------------------------------------------------------------------------ void AppendCGI( std::string &url, const char *newCGI ) { if( !newCGI || !(*newCGI) ) return; if( *newCGI == '&' ) ++newCGI; if( url.find( '?' ) == std::string::npos ) url += "?"; if( url.find( '&' ) == std::string::npos ) url += "&"; url += newCGI; } //------------------------------------------------------------------------------ // Process commandline environment settings //------------------------------------------------------------------------------ void ProcessCommandLineEnv( XrdCpConfig *config ) { XrdCl::Env *env = XrdCl::DefaultEnv::GetEnv(); XrdCpConfig::defVar *cursor = config->intDefs; while( cursor ) { env->PutInt( cursor->vName, cursor->intVal ); cursor = cursor->Next; } cursor = config->strDefs; while( cursor ) { env->PutString( cursor->vName, cursor->strVal ); cursor = cursor->Next; } } //------------------------------------------------------------------------------ // Translate file type to a string for diagnostics purposes //------------------------------------------------------------------------------ const char *FileType2String( XrdCpFile::PType type ) { switch( type ) { case XrdCpFile::isDir: return "directory"; case XrdCpFile::isFile: return "local file"; case XrdCpFile::isXroot: return "xroot"; case XrdCpFile::isHttp: return "http"; case XrdCpFile::isHttps: return "https"; case XrdCpFile::isStdIO: return "stdio"; default: return "other"; }; } //------------------------------------------------------------------------------ // Count the sources //------------------------------------------------------------------------------ uint32_t CountSources( XrdCpFile *file ) { uint32_t count; for( count = 0; file; file = file->Next, ++count ) {}; return count; } //------------------------------------------------------------------------------ // Adjust file information for the cases when XrdCpConfig cannot do this //------------------------------------------------------------------------------ void AdjustFileInfo( XrdCpFile *file ) { //---------------------------------------------------------------------------- // If the file is url and the directory offset is not set we set it // to the last slash //---------------------------------------------------------------------------- if( file->Doff == 0 ) { char *slash = file->Path; for( ; *slash; ++slash ) {}; for( ; *slash != '/' && slash > file->Path; --slash ) {}; file->Doff = slash - file->Path; } }; //------------------------------------------------------------------------------ // Recursively index all files and directories inside a remote directory //------------------------------------------------------------------------------ XrdCpFile *IndexRemote( XrdCl::FileSystem *fs, std::string basePath, uint16_t dirOffset ) { using namespace XrdCl; Log *log = DefaultEnv::GetLog(); log->Debug( AppMsg, "Indexing %s", basePath.c_str() ); DirectoryList *dirList = 0; XRootDStatus st = fs->DirList( URL( basePath ).GetPath(), DirListFlags::Recursive | DirListFlags::Locate | DirListFlags::Merge, dirList ); if( !st.IsOK() ) { log->Info( AppMsg, "Failed to get directory listing for %s: %s", basePath.c_str(), st.GetErrorMessage().c_str() ); return 0; } XrdCpFile start, *current = 0; XrdCpFile *end = &start; int badUrl = 0; for( auto itr = dirList->Begin(); itr != dirList->End(); ++itr ) { DirectoryList::ListEntry *e = *itr; if( e->GetStatInfo()->TestFlags( StatInfo::IsDir ) ) continue; std::string path = basePath + '/' + e->GetName(); current = new XrdCpFile( path.c_str(), badUrl ); if( badUrl ) { log->Error( AppMsg, "Bad URL: %s", current->Path ); delete current; return 0; } current->Doff = dirOffset; end->Next = current; end = current; } delete dirList; return start.Next; } //------------------------------------------------------------------------------ // Clean up the copy job descriptors //------------------------------------------------------------------------------ void CleanUpResults( std::vector &results ) { std::vector::iterator it; for( it = results.begin(); it != results.end(); ++it ) delete *it; } //-------------------------------------------------------------------------- // Let the show begin //------------------------------------------------------------------------------ int main( int argc, char **argv ) { using namespace XrdCl; //---------------------------------------------------------------------------- // Configure the copy command, if it returns then everything went well, ugly //---------------------------------------------------------------------------- XrdCpConfig config( argv[0] ); config.Config( argc, argv, XrdCpConfig::optRmtRec ); if( !AllOptionsSupported( &config ) ) return 50; // generic error ProcessCommandLineEnv( &config ); //---------------------------------------------------------------------------- // Set options //---------------------------------------------------------------------------- CopyProcess process; Log *log = DefaultEnv::GetLog(); if( config.Dlvl ) { if( config.Dlvl == 1 ) log->SetLevel( Log::InfoMsg ); else if( config.Dlvl == 2 ) log->SetLevel( Log::DebugMsg ); else if( config.Dlvl == 3 ) log->SetLevel( Log::DumpMsg ); } ProgressDisplay progress; if( config.Want(XrdCpConfig::DoNoPbar) || !isatty( fileno( stdout ) ) ) progress.PrintProgressBar( false ); bool posc = false; bool force = false; bool coerce = false; bool makedir = false; bool dynSrc = false; bool delegate = false; bool preserveXAttr = false; bool rmOnBadCksum = false; bool continue_ = false; bool recurse = false; bool zipappend = false; bool doserver = false; std::string thirdParty = "none"; if( config.Want( XrdCpConfig::DoPosc ) ) posc = true; if( config.Want( XrdCpConfig::DoForce ) ) force = true; if( config.Want( XrdCpConfig::DoCoerce ) ) coerce = true; if( config.Want( XrdCpConfig::DoTpc ) ) thirdParty = "first"; if( config.Want( XrdCpConfig::DoTpcOnly ) ) thirdParty = "only"; if( config.Want( XrdCpConfig::DoZipAppend ) ) zipappend = true; if( config.Want( XrdCpConfig::DoServer ) ) doserver = true; if( config.Want( XrdCpConfig::DoTpcDlgt ) ) { // the env var is being set already here (we are issuing a stat // inhere and we need the env var when we are establishing the // connection and authenticating), but we are also setting a delegate // parameter for CopyJob so it can be used on its own. DlgEnv::Instance().Enable(); delegate = true; } else DlgEnv::Instance().Disable(); if( config.Want( XrdCpConfig::DoRecurse ) ) { makedir = true; recurse = true; } if( config.Want( XrdCpConfig::DoPath ) ) makedir = true; if( config.Want( XrdCpConfig::DoDynaSrc ) ) dynSrc = true; if( config.Want( XrdCpConfig::DoXAttr ) ) preserveXAttr = true; if( config.Want( XrdCpConfig::DoRmOnBadCksum ) ) rmOnBadCksum = true; if( config.Want( XrdCpConfig::DoContinue ) ) continue_ = true; if( force && continue_ ) { std::cerr << "Invalid argument combination: continue + force." << std::endl; return 50; } //---------------------------------------------------------------------------- // Checksums //---------------------------------------------------------------------------- std::string checkSumType; std::string checkSumPreset; std::string checkSumMode = "none"; if( config.Want( XrdCpConfig::DoCksum ) ) { checkSumMode = "end2end"; std::vector ckSumParams; Utils::splitString( ckSumParams, config.CksVal, ":" ); if( ckSumParams.size() > 1 ) { if( ckSumParams[1] == "print" ) { checkSumMode = "target"; progress.PrintTargetCheckSum( true ); } else checkSumPreset = ckSumParams[1]; } checkSumType = ckSumParams[0]; } if( config.Want( XrdCpConfig::DoCksrc ) ) { checkSumMode = "source"; std::vector ckSumParams; Utils::splitString( ckSumParams, config.CksVal, ":" ); if( ckSumParams.size() == 2 ) { checkSumMode = "source"; checkSumType = ckSumParams[0]; progress.PrintSourceCheckSum( true ); } else { std::cerr << "Invalid parameter: " << config.CksVal << std::endl; return 50; // generic error } } if( !config.AddCksVal.empty() ) progress.PrintAdditionalCheckSum( true ); //---------------------------------------------------------------------------- // ZIP archive //---------------------------------------------------------------------------- std::string zipFile; bool zip = false; if( config.Want( XrdCpConfig::DoZip ) ) { zipFile = config.zipFile; zip = true; } //---------------------------------------------------------------------------- // Extreme Copy //---------------------------------------------------------------------------- int nbSources = 0; bool xcp = false; if( config.Want( XrdCpConfig::DoSources ) ) { nbSources = config.nSrcs; xcp = true; } //---------------------------------------------------------------------------- // Environment settings //---------------------------------------------------------------------------- XrdCl::Env *env = XrdCl::DefaultEnv::GetEnv(); if( config.nStrm != 0 ) env->PutInt( "SubStreamsPerChannel", config.nStrm + 1 /*stands for the control stream*/ ); if( config.Retry != -1 ) { env->PutInt( "CpRetry", config.Retry ); env->PutString( "CpRetryPolicy", config.RetryPolicy ); } if( config.Want( XrdCpConfig::DoNoTlsOK ) ) env->PutInt( "NoTlsOK", 1 ); if( config.Want( XrdCpConfig::DoTlsNoData ) ) env->PutInt( "TlsNoData", 1 ); if( config.Want( XrdCpConfig::DoTlsMLF ) ) env->PutInt( "TlsMetalink", 1 ); if( config.Want( XrdCpConfig::DoZipMtlnCksum ) ) env->PutInt( "ZipMtlnCksum", 1 ); int chunkSize = DefaultCPChunkSize; env->GetInt( "CPChunkSize", chunkSize ); int blockSize = DefaultXCpBlockSize; env->GetInt( "XCpBlockSize", blockSize ); int parallelChunks = DefaultCPParallelChunks; env->GetInt( "CPParallelChunks", parallelChunks ); if( parallelChunks < 1 || parallelChunks > std::numeric_limits::max() ) { std::cerr << "Can only handle between 1 and "; std::cerr << (int)std::numeric_limits::max(); std::cerr << " chunks in parallel. You asked for " << parallelChunks; std::cerr << "." << std::endl; return 50; // generic error } if( !preserveXAttr ) { int val = DefaultPreserveXAttrs; env->GetInt( "PreserveXAttrs", val ); if( val ) preserveXAttr = true; } log->Dump( AppMsg, "Chunk size: %d, parallel chunks %d, streams: %d", chunkSize, parallelChunks, config.nStrm + 1 ); //---------------------------------------------------------------------------- // Build the URLs //---------------------------------------------------------------------------- std::vector resultVect; std::string dest; if( config.dstFile->Protocol == XrdCpFile::isDir || config.dstFile->Protocol == XrdCpFile::isFile ) { dest = "file://"; // if it is not an absolute path append cwd if( config.dstFile->Path[0] != '/' ) { char buf[FILENAME_MAX]; char *cwd = getcwd( buf, FILENAME_MAX ); if( !cwd ) { XRootDStatus st( stError, XProtocol::mapError( errno ), errno ); std::cerr << st.GetErrorMessage() << std::endl; return st.GetShellCode(); } dest += cwd; dest += '/'; } } dest += config.dstFile->Path; //---------------------------------------------------------------------------- // We need to check whether our target is a file or a directory: // 1) it's a file, so we can accept only one source // 2) it's a directory, so: // * we can accept multiple sources // * we need to append the source name //---------------------------------------------------------------------------- bool targetIsDir = false; bool targetExists = false; if( config.dstFile->Protocol == XrdCpFile::isDir ) targetIsDir = true; else if( config.dstFile->Protocol == XrdCpFile::isXroot || config.dstFile->Protocol == XrdCpFile::isXroots ) { URL target( dest ); FileSystem fs( target ); StatInfo *statInfo = 0; XRootDStatus st = fs.Stat( target.GetPathWithParams(), statInfo ); if( st.IsOK() ) { if( statInfo->TestFlags( StatInfo::IsDir ) ) targetIsDir = true; targetExists = true; } else if( st.errNo == kXR_NotFound && config.Want( XrdCpConfig::DoPath ) ) { int n = strlen(config.dstFile->Path); if( config.dstFile->Path[n-1] == '/' ) targetIsDir = true; } else if( st.errNo == kXR_NotAuthorized ) { log->Error( AppMsg, "%s (destination)", st.ToString().c_str() ); std::cerr << st.ToStr() << std::endl; return st.GetShellCode(); } delete statInfo; } if( !targetIsDir && targetExists && !force && !recurse && !zipappend ) { XRootDStatus st( stError, errInvalidOp, EEXIST ); // Unable to create /tmp/test.txt; file exists log->Error( AppMsg, "%s (destination)", st.ToString().c_str() ); std::cerr << "Run: " << st.ToStr() << std::endl; return st.GetShellCode(); } //---------------------------------------------------------------------------- // If we have multiple sources and target is neither a directory nor stdout // then we cannot proceed //---------------------------------------------------------------------------- if( CountSources(config.srcFile) > 1 && !targetIsDir && config.dstFile->Protocol != XrdCpFile::isStdIO ) { std::cerr << "Multiple sources were given but target is not a directory."; std::cerr << std::endl; return 50; // generic error } //---------------------------------------------------------------------------- // If we're doing remote recursive copy, chain all the files (if it's a // directory) //---------------------------------------------------------------------------- bool remoteSrcIsDir = false; if( config.Want( XrdCpConfig::DoRecurse ) && config.srcFile->Protocol == XrdCpFile::isXroot ) { URL source( config.srcFile->Path ); FileSystem *fs = new FileSystem( source ); StatInfo *statInfo = 0; XRootDStatus st = fs->Stat( source.GetPath(), statInfo ); if( st.IsOK() && statInfo->TestFlags( StatInfo::IsDir ) ) { remoteSrcIsDir = true; //------------------------------------------------------------------------ // Recursively index the remote directory //------------------------------------------------------------------------ delete config.srcFile; std::string url = source.GetURL(); config.srcFile = IndexRemote( fs, url, url.size() ); if ( !config.srcFile ) { std::cerr << "Error indexing remote directory."; return 50; // generic error } } delete fs; delete statInfo; } XrdCpFile *sourceFile = config.srcFile; //---------------------------------------------------------------------------- // Process the sources //---------------------------------------------------------------------------- while( sourceFile ) { AdjustFileInfo( sourceFile ); //-------------------------------------------------------------------------- // Create a job for every source //-------------------------------------------------------------------------- PropertyList properties; PropertyList *results = new PropertyList; std::string source = sourceFile->Path; if( sourceFile->Protocol == XrdCpFile::isFile ) { // make sure it is an absolute path if( source[0] == '/' ) source = "file://" + source; else { char buf[FILENAME_MAX]; char *cwd = getcwd( buf, FILENAME_MAX ); if( !cwd ) { XRootDStatus st( stError, XProtocol::mapError( errno ), errno ); std::cerr << st.GetErrorMessage() << std::endl; return st.GetShellCode(); } source = "file://" + std::string( cwd ) + '/' + source; } } AppendCGI( source, config.srcOpq ); log->Dump( AppMsg, "Processing source entry: %s, type %s, target file: %s", sourceFile->Path, FileType2String( sourceFile->Protocol ), dest.c_str() ); //-------------------------------------------------------------------------- // Set up the job //-------------------------------------------------------------------------- std::string target = dest; bool srcIsDir = false; // if this is local file, for a directory Dlen + Doff will overlap with path size if( strncmp( sourceFile->ProtName, "file", 4 ) == 0 ) srcIsDir = std::string( sourceFile->Path ).size() == size_t( sourceFile->Doff + sourceFile->Dlen ); // otherwise we are handling a remote file else srcIsDir = remoteSrcIsDir; // if this is a recursive copy make sure we preserve the directory structure if( config.Want( XrdCpConfig::DoRecurse ) && srcIsDir ) { // get the source directory std::string srcDir( sourceFile->Path, sourceFile->Doff ); // remove the trailing slash if( srcDir[srcDir.size() - 1] == '/' ) srcDir = srcDir.substr( 0, srcDir.size() - 1 ); size_t diroff = srcDir.rfind( '/' ); // if there is no '/' it means a directory name has been given as relative path if( diroff == std::string::npos ) diroff = 0; target += '/'; target += sourceFile->Path + diroff; // remove the filename from destination path as it will be appended later anyway target = target.substr( 0 , target.rfind('/') ); } AppendCGI( target, config.dstOpq ); properties.Set( "source", source ); properties.Set( "target", target ); properties.Set( "force", force ); properties.Set( "posc", posc ); properties.Set( "coerce", coerce ); properties.Set( "makeDir", makedir ); properties.Set( "dynamicSource", dynSrc ); properties.Set( "thirdParty", thirdParty ); properties.Set( "checkSumMode", checkSumMode ); properties.Set( "checkSumType", checkSumType ); properties.Set( "checkSumPreset", checkSumPreset ); properties.Set( "chunkSize", chunkSize ); properties.Set( "parallelChunks", parallelChunks ); properties.Set( "zipArchive", zip ); properties.Set( "xcp", xcp ); properties.Set( "xcpBlockSize", blockSize ); properties.Set( "delegate", delegate ); properties.Set( "targetIsDir", targetIsDir ); properties.Set( "preserveXAttr", preserveXAttr ); properties.Set( "xrate", config.xRate ); properties.Set( "xrateThreshold", config.xRateThreshold ); properties.Set( "rmOnBadCksum", rmOnBadCksum ); properties.Set( "continue", continue_ ); properties.Set( "zipAppend", zipappend ); properties.Set( "addcksums", config.AddCksVal ); properties.Set( "doServer", doserver ); if( zip ) properties.Set( "zipSource", zipFile ); if( xcp ) properties.Set( "nbXcpSources", nbSources ); XRootDStatus st = process.AddJob( properties, results ); if( !st.IsOK() ) { std::cerr << "AddJob " << source << " -> " << target << ": "; std::cerr << st.ToStr() << std::endl; } resultVect.push_back( results ); sourceFile = sourceFile->Next; } //---------------------------------------------------------------------------- // Configure the copy process //---------------------------------------------------------------------------- PropertyList processConfig; processConfig.Set( "jobType", "configuration" ); processConfig.Set( "parallel", config.Parallel ); process.AddJob( processConfig, 0 ); //---------------------------------------------------------------------------- // Prepare and run the copy process //---------------------------------------------------------------------------- XRootDStatus st = process.Prepare(); if( !st.IsOK() ) { CleanUpResults( resultVect ); std::cerr << "Prepare: " << st.ToStr() << std::endl; return st.GetShellCode(); } st = process.Run( &progress ); if( !st.IsOK() ) { if( resultVect.size() == 1 ) std::cerr << "Run: " << st.ToStr() << std::endl; else { std::vector::iterator it; uint16_t i = 1; uint16_t jobsRun = 0; uint16_t errors = 0; for( it = resultVect.begin(); it != resultVect.end(); ++it, ++i ) { if( !(*it)->HasProperty( "status" ) ) continue; XRootDStatus st = (*it)->Get("status"); if( !st.IsOK() ) { std::cerr << "Job #" << i << ": " << st.ToStr(); ++errors; } ++jobsRun; } std::cerr << "Jobs total: " << resultVect.size(); std::cerr << ", run: " << jobsRun; std::cerr << ", errors: " << errors << std::endl; } CleanUpResults( resultVect ); return st.GetShellCode(); } CleanUpResults( resultVect ); XrdCl::DefaultEnv::GetPostMaster()->Stop(); return 0; } xrootd-5.6.9/src/XrdCl/XrdClCopyJob.hh000066400000000000000000000102401457266313600175120ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2012 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //----------------------------------------------------------------------------- // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #ifndef __XRD_CL_COPY_JOB_HH__ #define __XRD_CL_COPY_JOB_HH__ #include "XrdCl/XrdClPropertyList.hh" namespace XrdCl { //---------------------------------------------------------------------------- //! Copy job //---------------------------------------------------------------------------- class CopyJob { public: //------------------------------------------------------------------------ //! Constructor //------------------------------------------------------------------------ CopyJob( uint16_t jobId, PropertyList *jobProperties, PropertyList *jobResults ): pProperties( jobProperties ), pResults( jobResults ), pJobId( jobId ) { Init(); } //------------------------------------------------------------------------ //! Virtual destructor //------------------------------------------------------------------------ virtual ~CopyJob() { } //------------------------------------------------------------------------ // Initialize members //------------------------------------------------------------------------ inline void Init() { pProperties->Get( "source", pSource ); pProperties->Get( "target", pTarget ); } //------------------------------------------------------------------------ //! Run the copy job //! //! @param progress the handler to be notified about the copy progress //! @return status of the copy operation //------------------------------------------------------------------------ virtual XRootDStatus Run( CopyProgressHandler *progress = 0 ) = 0; //------------------------------------------------------------------------ //! Get the job properties //------------------------------------------------------------------------ PropertyList *GetProperties() { return pProperties; } //------------------------------------------------------------------------ //! Get the job results //------------------------------------------------------------------------ PropertyList *GetResults() { return pResults; } //------------------------------------------------------------------------ //! Get source //------------------------------------------------------------------------ const URL &GetSource() const { return pSource; } //------------------------------------------------------------------------ //! Get target //------------------------------------------------------------------------ const URL &GetTarget() const { return pTarget; } protected: PropertyList *pProperties; PropertyList *pResults; URL pSource; URL pTarget; uint16_t pJobId; }; } #endif // __XRD_CL_COPY_JOB_HH__ xrootd-5.6.9/src/XrdCl/XrdClCopyProcess.cc000066400000000000000000000516331457266313600204170ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2014 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #include "XrdCl/XrdClCopyProcess.hh" #include "XrdCl/XrdClConstants.hh" #include "XrdCl/XrdClLog.hh" #include "XrdCl/XrdClDefaultEnv.hh" #include "XrdCl/XrdClClassicCopyJob.hh" #include "XrdCl/XrdClTPFallBackCopyJob.hh" #include "XrdCl/XrdClFileSystem.hh" #include "XrdCl/XrdClMonitor.hh" #include "XrdCl/XrdClCopyJob.hh" #include "XrdCl/XrdClUtils.hh" #include "XrdCl/XrdClJobManager.hh" #include "XrdCl/XrdClRedirectorRegistry.hh" #include "XrdCl/XrdClConstants.hh" #include "XrdSys/XrdSysPthread.hh" #include #include namespace { class QueuedCopyJob: public XrdCl::Job { public: QueuedCopyJob( XrdCl::CopyJob *job, XrdCl::CopyProgressHandler *progress, uint16_t currentJob, uint16_t totalJobs, XrdSysSemaphore *sem = 0 ): pJob(job), pProgress(progress), pCurrentJob(currentJob), pTotalJobs(totalJobs), pSem(sem), pWrtRetryCnt( XrdCl::DefaultRetryWrtAtLBLimit ), pRetryCnt( XrdCl::DefaultCpRetry ), pRetryPolicy( XrdCl::DefaultCpRetryPolicy ) { XrdCl::DefaultEnv::GetEnv()->GetInt( "RetryWrtAtLBLimit", pWrtRetryCnt ); XrdCl::DefaultEnv::GetEnv()->GetInt( "CpRetry", pRetryCnt ); XrdCl::DefaultEnv::GetEnv()->GetString( "CpRetryPolicy", pRetryPolicy ); } //------------------------------------------------------------------------ //! Run the job //------------------------------------------------------------------------ virtual void Run( void * ) { XrdCl::Monitor *mon = XrdCl::DefaultEnv::GetMonitor(); timeval bTOD; //---------------------------------------------------------------------- // Report beginning of the copy //---------------------------------------------------------------------- if( pProgress ) pProgress->BeginJob( pCurrentJob, pTotalJobs, &pJob->GetSource(), &pJob->GetTarget() ); if( mon ) { XrdCl::Monitor::CopyBInfo i; i.transfer.origin = &pJob->GetSource(); i.transfer.target = &pJob->GetTarget(); mon->Event( XrdCl::Monitor::EvCopyBeg, &i ); } gettimeofday( &bTOD, 0 ); //---------------------------------------------------------------------- // Do the copy //---------------------------------------------------------------------- XrdCl::XRootDStatus st; while( true ) { st = pJob->Run( pProgress ); //-------------------------------------------------------------------- // Retry due to write-recovery //-------------------------------------------------------------------- if( !st.IsOK() && st.code == XrdCl::errRetry && pWrtRetryCnt > 0 ) { std::string url; pJob->GetResults()->Get( "LastURL", url ); XrdCl::URL lastURL( url ); XrdCl::URL::ParamsMap cgi = lastURL.GetParams(); auto itr = cgi.find( "tried" ); if( itr != cgi.end() ) { std::string tried = itr->second; if( tried[tried.size() - 1] != ',' ) tried += ','; tried += lastURL.GetHostName(); cgi["tried"] = tried; } else cgi["tried"] = lastURL.GetHostName(); std::string recoveryRedir; pJob->GetResults()->Get( "WrtRecoveryRedir", recoveryRedir ); XrdCl::URL recRedirURL( recoveryRedir ); std::string target; pJob->GetProperties()->Get( "target", target ); XrdCl::URL trgURL( target ); trgURL.SetHostName( recRedirURL.GetHostName() ); trgURL.SetPort( recRedirURL.GetPort() ); trgURL.SetProtocol( recRedirURL.GetProtocol() ); trgURL.SetParams( cgi ); pJob->GetProperties()->Set( "target", trgURL.GetURL() ); pJob->Init(); // we have a new job, let's try again --pWrtRetryCnt; continue; } //-------------------------------------------------------------------- // Copy job retry //-------------------------------------------------------------------- if( !st.IsOK() && pRetryCnt > 0 && ( XrdCl::Status::IsSocketError( st.code ) || st.code == XrdCl::errOperationExpired || st.code == XrdCl::errThresholdExceeded ) ) { if( pRetryPolicy == "continue" ) { pJob->GetProperties()->Set( "force", false ); pJob->GetProperties()->Set( "continue", true ); } else { pJob->GetProperties()->Set( "force", true ); pJob->GetProperties()->Set( "continue", false ); } --pRetryCnt; continue; } // we only loop in case of retry error break; } pJob->GetResults()->Set( "status", st ); //---------------------------------------------------------------------- // Report end of the copy //---------------------------------------------------------------------- if( mon ) { std::vector sources; pJob->GetResults()->Get( "sources", sources ); XrdCl::Monitor::CopyEInfo i; i.transfer.origin = &pJob->GetSource(); i.transfer.target = &pJob->GetTarget(); i.sources = sources.size(); i.bTOD = bTOD; gettimeofday( &i.eTOD, 0 ); i.status = &st; mon->Event( XrdCl::Monitor::EvCopyEnd, &i ); } if( pProgress ) pProgress->EndJob( pCurrentJob, pJob->GetResults() ); if( pSem ) pSem->Post(); } private: XrdCl::CopyJob *pJob; XrdCl::CopyProgressHandler *pProgress; uint16_t pCurrentJob; uint16_t pTotalJobs; XrdSysSemaphore *pSem; int pWrtRetryCnt; int pRetryCnt; std::string pRetryPolicy; }; }; namespace XrdCl { struct CopyProcessImpl { std::vector pJobProperties; std::vector pJobResults; std::vector pJobs; }; //---------------------------------------------------------------------------- // Destructor //---------------------------------------------------------------------------- CopyProcess::CopyProcess() : pImpl( new CopyProcessImpl() ) { } //---------------------------------------------------------------------------- // Destructor //---------------------------------------------------------------------------- CopyProcess::~CopyProcess() { CleanUpJobs(); delete pImpl; } //---------------------------------------------------------------------------- // Add job //---------------------------------------------------------------------------- XRootDStatus CopyProcess::AddJob( const PropertyList &properties, PropertyList *results ) { Env *env = DefaultEnv::GetEnv(); //-------------------------------------------------------------------------- // Process a configuraion job //-------------------------------------------------------------------------- if( properties.HasProperty( "jobType" ) && properties.Get( "jobType" ) == "configuration" ) { if( pImpl->pJobProperties.size() > 0 && pImpl->pJobProperties.rbegin()->HasProperty( "jobType" ) && pImpl->pJobProperties.rbegin()->Get( "jobType" ) == "configuration" ) { PropertyList &config = *pImpl->pJobProperties.rbegin(); PropertyList::PropertyMap::const_iterator it; for( it = properties.begin(); it != properties.end(); ++it ) config.Set( it->first, it->second ); } else pImpl->pJobProperties.push_back( properties ); return XRootDStatus(); } //-------------------------------------------------------------------------- // Validate properties //-------------------------------------------------------------------------- if( !properties.HasProperty( "source" ) ) return XRootDStatus( stError, errInvalidArgs, 0, "source not specified" ); if( !properties.HasProperty( "target" ) ) return XRootDStatus( stError, errInvalidArgs, 0, "target not specified" ); pImpl->pJobProperties.push_back( properties ); PropertyList &p = pImpl->pJobProperties.back(); const char *bools[] = {"target", "force", "posc", "coerce", "makeDir", "zipArchive", "xcp", "preserveXAttr", "rmOnBadCksum", "continue", "zipAppend", "doServer", 0}; for( int i = 0; bools[i]; ++i ) if( !p.HasProperty( bools[i] ) ) p.Set( bools[i], false ); if( !p.HasProperty( "thirdParty" ) ) p.Set( "thirdParty", "none" ); if( !p.HasProperty( "checkSumMode" ) ) p.Set( "checkSumMode", "none" ); else { if( !p.HasProperty( "checkSumType" ) ) { pImpl->pJobProperties.pop_back(); return XRootDStatus( stError, errInvalidArgs, 0, "checkSumType not specified" ); } else { //---------------------------------------------------------------------- // Checksum type has to be case insensitive //---------------------------------------------------------------------- std::string checkSumType; p.Get( "checkSumType", checkSumType ); std::transform(checkSumType.begin(), checkSumType.end(), checkSumType.begin(), ::tolower); p.Set( "checkSumType", checkSumType ); } } if( !p.HasProperty( "parallelChunks" ) ) { int val = DefaultCPParallelChunks; env->GetInt( "CPParallelChunks", val ); p.Set( "parallelChunks", val ); } if( !p.HasProperty( "chunkSize" ) ) { int val = DefaultCPChunkSize; env->GetInt( "CPChunkSize", val ); p.Set( "chunkSize", val ); } if( !p.HasProperty( "xcpBlockSize" ) ) { int val = DefaultXCpBlockSize; env->GetInt( "XCpBlockSize", val ); p.Set( "xcpBlockSize", val ); } if( !p.HasProperty( "initTimeout" ) ) { int val = DefaultCPInitTimeout; env->GetInt( "CPInitTimeout", val ); p.Set( "initTimeout", val ); } if( !p.HasProperty( "tpcTimeout" ) ) { int val = DefaultCPTPCTimeout; env->GetInt( "CPTPCTimeout", val ); p.Set( "tpcTimeout", val ); } if( !p.HasProperty( "cpTimeout" ) ) { int val = DefaultCPTimeout; env->GetInt( "CPTimeout", val ); p.Set( "cpTimeout", val ); } if( !p.HasProperty( "dynamicSource" ) ) p.Set( "dynamicSource", false ); if( !p.HasProperty( "xrate" ) ) p.Set( "xrate", 0 ); if( !p.HasProperty( "xrateThreshold" ) || p.Get( "xrateThreshold" ) == 0 ) { int val = DefaultXRateThreshold; env->GetInt( "XRateThreshold", val ); p.Set( "xrateThreshold", val ); } //-------------------------------------------------------------------------- // Insert the properties //-------------------------------------------------------------------------- Log *log = DefaultEnv::GetLog(); Utils::LogPropertyList( log, UtilityMsg, "Adding job with properties: %s", p ); pImpl->pJobResults.push_back( results ); return XRootDStatus(); } //---------------------------------------------------------------------------- // Prepare the copy jobs //---------------------------------------------------------------------------- XRootDStatus CopyProcess::Prepare() { Log *log = DefaultEnv::GetLog(); std::vector::iterator it; log->Debug( UtilityMsg, "CopyProcess: %d jobs to prepare", pImpl->pJobProperties.size() ); std::map targetFlags; int i = 0; for( it = pImpl->pJobProperties.begin(); it != pImpl->pJobProperties.end(); ++it, ++i ) { PropertyList &props = *it; if( props.HasProperty( "jobType" ) && props.Get( "jobType" ) == "configuration" ) continue; PropertyList *res = pImpl->pJobResults[i]; std::string tmp; props.Get( "source", tmp ); URL source = tmp; if( !source.IsValid() ) return XRootDStatus( stError, errInvalidArgs, 0, "invalid source" ); //-------------------------------------------------------------------------- // Create a virtual redirector if it is a Metalink file //-------------------------------------------------------------------------- if( source.IsMetalink() ) { RedirectorRegistry ®istry = RedirectorRegistry::Instance(); XRootDStatus st = registry.RegisterAndWait( source ); if( !st.IsOK() ) return st; } // handle UNZIP CGI const URL::ParamsMap &cgi = source.GetParams(); URL::ParamsMap::const_iterator itr = cgi.find( "xrdcl.unzip" ); if( itr != cgi.end() ) { props.Set( "zipArchive", true ); props.Set( "zipSource", itr->second ); } props.Get( "target", tmp ); URL target = tmp; if( !target.IsValid() ) return XRootDStatus( stError, errInvalidArgs, 0, "invalid target" ); if( target.GetProtocol() != "stdio" ) { // handle directories bool targetIsDir = false; props.Get( "targetIsDir", targetIsDir ); if( targetIsDir ) { std::string path = target.GetPath() + '/'; std::string fn; bool isZip = false; props.Get( "zipArchive", isZip ); if( isZip ) { props.Get( "zipSource", fn ); } else if( source.IsMetalink() ) { RedirectorRegistry ®istry = XrdCl::RedirectorRegistry::Instance(); VirtualRedirector *redirector = registry.Get( source ); fn = redirector->GetTargetName(); } else { fn = source.GetPath(); } size_t pos = fn.rfind( '/' ); if( pos != std::string::npos ) fn = fn.substr( pos + 1 ); path += fn; target.SetPath( path ); props.Set( "target", target.GetURL() ); } } bool tpc = false; props.Get( "thirdParty", tmp ); if( tmp != "none" ) tpc = true; //------------------------------------------------------------------------ // Check if we have all we need //------------------------------------------------------------------------ if( source.GetProtocol() != "stdio" && source.GetPath().empty() ) { log->Debug( UtilityMsg, "CopyProcess (job #%d): no source specified.", i ); CleanUpJobs(); XRootDStatus st = XRootDStatus( stError, errInvalidArgs ); res->Set( "status", st ); return st; } if( target.GetProtocol() != "stdio" && target.GetPath().empty() ) { log->Debug( UtilityMsg, "CopyProcess (job #%d): no target specified.", i ); CleanUpJobs(); XRootDStatus st = XRootDStatus( stError, errInvalidArgs ); res->Set( "status", st ); return st; } //------------------------------------------------------------------------ // Check what kind of job we should do //------------------------------------------------------------------------ CopyJob *job = 0; if( tpc == true ) { MarkTPC( props ); job = new TPFallBackCopyJob( i+1, &props, res ); } else job = new ClassicCopyJob( i+1, &props, res ); pImpl->pJobs.push_back( job ); } return XRootDStatus(); } //---------------------------------------------------------------------------- // Run the copy jobs //---------------------------------------------------------------------------- XRootDStatus CopyProcess::Run( CopyProgressHandler *progress ) { //-------------------------------------------------------------------------- // Get the configuration //-------------------------------------------------------------------------- uint8_t parallelThreads = 1; if( pImpl->pJobProperties.size() > 0 && pImpl->pJobProperties.rbegin()->HasProperty( "jobType" ) && pImpl->pJobProperties.rbegin()->Get( "jobType" ) == "configuration" ) { PropertyList &config = *pImpl->pJobProperties.rbegin(); if( config.HasProperty( "parallel" ) ) parallelThreads = (uint8_t)config.Get( "parallel" ); } //-------------------------------------------------------------------------- // Run the show //-------------------------------------------------------------------------- std::vector::iterator it; uint16_t currentJob = 1; uint16_t totalJobs = pImpl->pJobs.size(); //-------------------------------------------------------------------------- // Single thread //-------------------------------------------------------------------------- if( parallelThreads == 1 ) { XRootDStatus err; for( it = pImpl->pJobs.begin(); it != pImpl->pJobs.end(); ++it ) { QueuedCopyJob j( *it, progress, currentJob, totalJobs ); j.Run(0); XRootDStatus st = (*it)->GetResults()->Get( "status" ); if( err.IsOK() && !st.IsOK() ) { err = st; } ++currentJob; } if( !err.IsOK() ) return err; } //-------------------------------------------------------------------------- // Multiple threads //-------------------------------------------------------------------------- else { uint16_t workers = std::min( (uint16_t)parallelThreads, (uint16_t)pImpl->pJobs.size() ); JobManager jm( workers ); jm.Initialize(); if( !jm.Start() ) return XRootDStatus( stError, errOSError, 0, "Unable to start job manager" ); XrdSysSemaphore *sem = new XrdSysSemaphore(0); std::vector queued; for( it = pImpl->pJobs.begin(); it != pImpl->pJobs.end(); ++it ) { QueuedCopyJob *j = new QueuedCopyJob( *it, progress, currentJob, totalJobs, sem ); queued.push_back( j ); jm.QueueJob(j, 0); ++currentJob; } std::vector::iterator itQ; for( itQ = queued.begin(); itQ != queued.end(); ++itQ ) sem->Wait(); delete sem; if( !jm.Stop() ) return XRootDStatus( stError, errOSError, 0, "Unable to stop job manager" ); jm.Finalize(); for( itQ = queued.begin(); itQ != queued.end(); ++itQ ) delete *itQ; for( it = pImpl->pJobs.begin(); it != pImpl->pJobs.end(); ++it ) { XRootDStatus st = (*it)->GetResults()->Get( "status" ); if( !st.IsOK() ) return st; } }; return XRootDStatus(); } void CopyProcess::CleanUpJobs() { std::vector::iterator itJ; for( itJ = pImpl->pJobs.begin(); itJ != pImpl->pJobs.end(); ++itJ ) { CopyJob *job = *itJ; URL src = job->GetSource(); if( src.IsMetalink() ) { RedirectorRegistry ®istry = RedirectorRegistry::Instance(); registry.Release( src ); } delete job; } pImpl->pJobs.clear(); } } xrootd-5.6.9/src/XrdCl/XrdClCopyProcess.hh000066400000000000000000000223671457266313600204330ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2014 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #ifndef __XRD_CL_COPY_PROCESS_HH__ #define __XRD_CL_COPY_PROCESS_HH__ #include "XrdCl/XrdClURL.hh" #include "XrdCl/XrdClXRootDResponses.hh" #include "XrdCl/XrdClPropertyList.hh" #include #include namespace XrdCl { class CopyJob; //---------------------------------------------------------------------------- //! Interface for copy progress notification //---------------------------------------------------------------------------- class CopyProgressHandler { public: virtual ~CopyProgressHandler() {} //------------------------------------------------------------------------ //! Notify when a new job is about to start //! //! @param jobNum the job number of the copy job concerned //! @param jobTotal total number of jobs being processed //! @param source the source url of the current job //! @param destination the destination url of the current job //------------------------------------------------------------------------ virtual void BeginJob( uint16_t jobNum, uint16_t jobTotal, const URL *source, const URL *destination ) { (void)jobNum; (void)jobTotal; (void)source; (void)destination; }; //------------------------------------------------------------------------ //! Notify when the previous job has finished //! //! @param jobNum job number //! @param result result of the job //------------------------------------------------------------------------ virtual void EndJob( uint16_t jobNum, const PropertyList *result ) { (void)jobNum; (void)result; }; //------------------------------------------------------------------------ //! Notify about the progress of the current job //! //! @param jobNum job number //! @param bytesProcessed bytes processed by the current job //! @param bytesTotal total number of bytes to be processed by the //! current job //------------------------------------------------------------------------ virtual void JobProgress( uint16_t jobNum, uint64_t bytesProcessed, uint64_t bytesTotal ) { (void)jobNum; (void)bytesProcessed; (void)bytesTotal; }; //------------------------------------------------------------------------ //! Determine whether the job should be canceled //------------------------------------------------------------------------ virtual bool ShouldCancel( uint16_t jobNum ) { (void)jobNum; return false; } }; //---------------------------------------------------------------------------- // Forward declaration of implementation holding CopyProcess' data members //---------------------------------------------------------------------------- struct CopyProcessImpl; //---------------------------------------------------------------------------- //! Copy the data from one point to another //---------------------------------------------------------------------------- class CopyProcess { public: //------------------------------------------------------------------------ //! Constructor //------------------------------------------------------------------------ CopyProcess(); //------------------------------------------------------------------------ //! Destructor //------------------------------------------------------------------------ virtual ~CopyProcess(); //------------------------------------------------------------------------ //! Add job //! //! @param properties job configuration parameters //! @param results placeholder for the results //! //! Configuration properties: //! source [string] - original source URL //! target [string] - target directory or file //! sourceLimit [uint16_t] - maximum number sources //! force [bool] - overwrite target if exists //! posc [bool] - persistify only on successful close //! coerce [bool] - ignore locking semantics on destination //! makeDir [bool] - create path to the file if it doesn't //! exist //! thirdParty [string] - "first" try third party copy, if it fails //! try normal copy; "only" only try third //! party copy //! checkSumMode [string] - "none" - no checksumming //! "end2end" - end to end checksumming //! "source" - calculate checksum at source //! "target" - calculate checksum at target //! checkSumType [string] - type of the checksum to be used //! checkSumPreset [string] - checksum preset //! chunkSize [uint32_t] - size of a copy chunks in bytes //! parallelChunks [uint8_t] - number of chunks that should be requested //! in parallel //! initTimeout [uint16_t] - time limit for successfull initialization //! of the copy job //! tpcTimeout [uint16_t] - time limit for the actual copy to finish //! dynamicSource [bool] - support for the case where the size source //! file may change during reading process //! //! Configuration job - this is a job that that is supposed to configure //! the copy process as a whole instead of adding a copy job: //! //! jobType [string] - "configuration" - for configuraion //! parallel [uint8_t] - nomber of copy jobs to be run in parallel //! //! Results: //! sourceCheckSum [string] - checksum at source, if requested //! targetCheckSum [string] - checksum at target, if requested //! size [uint64_t] - file size //! status [XRootDStatus] - status of the copy operation //! sources [vector] - all sources used //! realTarget [string] - the actual disk server target //------------------------------------------------------------------------ XRootDStatus AddJob( const PropertyList &properties, PropertyList *results ); //------------------------------------------------------------------------ // Prepare the copy jobs //------------------------------------------------------------------------ XRootDStatus Prepare(); //------------------------------------------------------------------------ //! Run the copy jobs //------------------------------------------------------------------------ XRootDStatus Run( CopyProgressHandler *handler ); private: void CleanUpJobs(); //------------------------------------------------------------------------ //! Mark the URLs in the property list as ment for TPC //------------------------------------------------------------------------ inline static void MarkTPC( PropertyList &properties ) { std::string keys[] = { "source", "target" }; size_t size = sizeof( keys ) / sizeof( std::string ); for( size_t i = 0; i < size; ++i ) { URL url; properties.Get( keys[i], url ); URL::ParamsMap params = url.GetParams(); params["xrdcl.intent"] = "tpc"; url.SetParams( params ); properties.Set( keys[i], url.GetURL() ); } } //------------------------------------------------------------------------ //! Pointer to implementation //------------------------------------------------------------------------ CopyProcessImpl *pImpl; }; } #endif // __XRD_CL_COPY_PROCESS_HH__ xrootd-5.6.9/src/XrdCl/XrdClCtx.hh000066400000000000000000000112101457266313600167010ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2017 by European Organization for Nuclear Research (CERN) // Author: Krzysztof Jamrog , // Michal Simon //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #ifndef SRC_XRDCL_XRDCLCTX_HH_ #define SRC_XRDCL_XRDCLCTX_HH_ #include #include namespace XrdCl { //--------------------------------------------------------------------------- //! Utility class for storing a pointer to operation context //--------------------------------------------------------------------------- template struct Ctx : protected std::shared_ptr { //------------------------------------------------------------------------- //! Default constructor //------------------------------------------------------------------------- Ctx() : std::shared_ptr( std::make_shared() ) { } //------------------------------------------------------------------------- //! Constructor (from pointer) //------------------------------------------------------------------------- Ctx( T *ctx ) : std::shared_ptr( std::make_shared( ctx ) ) { } //------------------------------------------------------------------------- //! Constructor (from reference) //------------------------------------------------------------------------- Ctx( T &ctx ) : std::shared_ptr( std::make_shared( &ctx ) ) { } //------------------------------------------------------------------------- //! Copy constructor //------------------------------------------------------------------------- Ctx( const Ctx &ctx ) : std::shared_ptr( ctx ) { } //------------------------------------------------------------------------- //! Move constructor //------------------------------------------------------------------------- Ctx( Ctx &&ctx ) : std::shared_ptr( std::move( ctx ) ) { } //------------------------------------------------------------------------- //! Assignment operator (from pointer) //------------------------------------------------------------------------- Ctx& operator=( T *ctx ) { *this->get() = ctx; return *this; } //------------------------------------------------------------------------- //! Assignment operator (from reference) //------------------------------------------------------------------------- Ctx& operator=( T &ctx ) { *this->get() = &ctx; return *this; } //------------------------------------------------------------------------ //! Dereferencing operator. Note if Ctx is a null-reference this will //! trigger an exception //! //! @return : reference to the underlying value //! @throws : std::logic_error //------------------------------------------------------------------------ T& operator*() const { if( !bool( *this->get() ) ) throw std::logic_error( "XrdCl::Ctx contains no value!" ); return **this->get(); } //------------------------------------------------------------------------ //! Dereferencing member operator. Note if Ctx is a null-reference //! this will trigger an exception //! //! @return : pointer to the underlying value //! @throws : std::logic_error //------------------------------------------------------------------------ T* operator->() const { if( !bool( *this->get() ) ) throw std::logic_error( "XrdCl::Ctx contains no value!" ); return *this->get(); } }; } #endif /* SRC_XRDCL_XRDCLCTX_HH_ */ xrootd-5.6.9/src/XrdCl/XrdClDefaultEnv.cc000066400000000000000000001036631457266313600202040ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2012 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . //------------------------------------------------------------------------------ #include "XrdCl/XrdClDefaultEnv.hh" #include "XrdCl/XrdClConstants.hh" #include "XrdCl/XrdClPostMaster.hh" #include "XrdCl/XrdClLog.hh" #include "XrdCl/XrdClForkHandler.hh" #include "XrdCl/XrdClFileTimer.hh" #include "XrdCl/XrdClUtils.hh" #include "XrdCl/XrdClMonitor.hh" #include "XrdCl/XrdClCheckSumManager.hh" #include "XrdCl/XrdClTransportManager.hh" #include "XrdCl/XrdClPlugInManager.hh" #include "XrdCl/XrdClOptimizers.hh" #include "XrdOuc/XrdOucPreload.hh" #include "XrdSys/XrdSysAtomics.hh" #include "XrdSys/XrdSysUtils.hh" #include "XrdSys/XrdSysPwd.hh" #include "XrdVersion.hh" #include #include #include #include #include #include #include #include #include #include XrdVERSIONINFO( XrdCl, client ); //------------------------------------------------------------------------------ // Forking functions //------------------------------------------------------------------------------ extern "C" { //---------------------------------------------------------------------------- // Prepare for the forking //---------------------------------------------------------------------------- static void prepare() { using namespace XrdCl; Log *log = DefaultEnv::GetLog(); Env *env = DefaultEnv::GetEnv(); ForkHandler *forkHandler = DefaultEnv::GetForkHandler(); log->Debug( UtilityMsg, "In the prepare fork handler for process %d", getpid() ); //-------------------------------------------------------------------------- // Run the fork handler if it's enabled //-------------------------------------------------------------------------- int runForkHandler = DefaultRunForkHandler; env->GetInt( "RunForkHandler", runForkHandler ); if( runForkHandler ) forkHandler->Prepare(); } //---------------------------------------------------------------------------- // Parent handler //---------------------------------------------------------------------------- static void parent() { using namespace XrdCl; Log *log = DefaultEnv::GetLog(); Env *env = DefaultEnv::GetEnv(); ForkHandler *forkHandler = DefaultEnv::GetForkHandler(); pid_t pid = getpid(); log->Debug( UtilityMsg, "In the parent fork handler for process %d", pid ); //-------------------------------------------------------------------------- // Run the fork handler if it's enabled //-------------------------------------------------------------------------- int runForkHandler = DefaultRunForkHandler; env->GetInt( "RunForkHandler", runForkHandler ); if( runForkHandler ) { log->SetPid(pid); forkHandler->Parent(); } } //---------------------------------------------------------------------------- // Child handler //---------------------------------------------------------------------------- static void child() { using namespace XrdCl; DefaultEnv::ReInitializeLogging(); Log *log = DefaultEnv::GetLog(); Env *env = DefaultEnv::GetEnv(); ForkHandler *forkHandler = DefaultEnv::GetForkHandler(); env->RecreateLock(); pid_t pid = getpid(); log->Debug( UtilityMsg, "In the child fork handler for process %d", pid ); //-------------------------------------------------------------------------- // Run the fork handler if it's enabled //-------------------------------------------------------------------------- int runForkHandler = DefaultRunForkHandler; env->GetInt( "RunForkHandler", runForkHandler ); if( runForkHandler ) { log->SetPid(pid); forkHandler->Child(); } } } namespace { //---------------------------------------------------------------------------- // Translate a string into a topic mask //---------------------------------------------------------------------------- struct MaskTranslator { //-------------------------------------------------------------------------- // Initialize the translation array //-------------------------------------------------------------------------- MaskTranslator() { masks["AppMsg"] = XrdCl::AppMsg; masks["UtilityMsg"] = XrdCl::UtilityMsg; masks["FileMsg"] = XrdCl::FileMsg; masks["PollerMsg"] = XrdCl::PollerMsg; masks["PostMasterMsg"] = XrdCl::PostMasterMsg; masks["XRootDTransportMsg"] = XrdCl::XRootDTransportMsg; masks["TaskMgrMsg"] = XrdCl::TaskMgrMsg; masks["XRootDMsg"] = XrdCl::XRootDMsg; masks["FileSystemMsg"] = XrdCl::FileSystemMsg; masks["AsyncSockMsg"] = XrdCl::AsyncSockMsg; masks["JobMgrMsg"] = XrdCl::JobMgrMsg; masks["PlugInMgrMsg"] = XrdCl::PlugInMgrMsg; masks["ExDbgMsg"] = XrdCl::ExDbgMsg; masks["TlsMsg"] = XrdCl::TlsMsg; } //-------------------------------------------------------------------------- // Translate the mask //-------------------------------------------------------------------------- uint64_t translateMask( const std::string mask ) { if( mask == "" ) return 0xffffffffffffffffULL; std::vector topics; std::vector::iterator it; XrdCl::Utils::splitString( topics, mask, "|" ); uint64_t resultMask = 0; std::map::iterator maskIt; for( it = topics.begin(); it != topics.end(); ++it ) { //---------------------------------------------------------------------- // Check for resetting pseudo topics //---------------------------------------------------------------------- if( *it == "All" ) { resultMask = 0xffffffffffffffffULL; continue; } if( *it == "None" ) { resultMask = 0ULL; continue; } //---------------------------------------------------------------------- // Check whether given topic should be disabled or enabled //---------------------------------------------------------------------- std::string topic = *it; bool disable = false; if( !topic.empty() && topic[0] == '^' ) { disable = true; topic = topic.substr( 1, topic.length()-1 ); } maskIt = masks.find( topic ); if( maskIt == masks.end() ) continue; if( disable ) resultMask &= (0xffffffffffffffffULL ^ maskIt->second); else resultMask |= maskIt->second; } return resultMask; } std::map masks; }; //---------------------------------------------------------------------------- // Helper for handling environment variables //---------------------------------------------------------------------------- template struct EnvVarHolder { EnvVarHolder( const std::string &name_, const Item &def_ ): name( name_ ), def( def_ ) {} std::string name; Item def; }; } #define REGISTER_VAR_INT( array, name, def ) \ array.push_back( EnvVarHolder( name, def ) ) #define REGISTER_VAR_STR( array, name, def ) \ array.push_back( EnvVarHolder( name, def ) ) namespace XrdCl { //---------------------------------------------------------------------------- // Statics //---------------------------------------------------------------------------- XrdSysMutex DefaultEnv::sInitMutex; Env *DefaultEnv::sEnv = 0; PostMaster *DefaultEnv::sPostMaster = 0; Log *DefaultEnv::sLog = 0; ForkHandler *DefaultEnv::sForkHandler = 0; FileTimer *DefaultEnv::sFileTimer = 0; Monitor *DefaultEnv::sMonitor = 0; XrdOucPinLoader *DefaultEnv::sMonitorLibHandle = 0; bool DefaultEnv::sMonitorInitialized = false; CheckSumManager *DefaultEnv::sCheckSumManager = 0; TransportManager *DefaultEnv::sTransportManager = 0; PlugInManager *DefaultEnv::sPlugInManager = 0; //---------------------------------------------------------------------------- // Constructor //---------------------------------------------------------------------------- DefaultEnv::DefaultEnv() { Log *log = GetLog(); log->Debug( UtilityMsg, "Initializing xrootd client version: %s", XrdVERSION ); //-------------------------------------------------------------------------- // Declate the variables to be processed //-------------------------------------------------------------------------- std::vector > varsInt; std::vector > varsStr; REGISTER_VAR_INT( varsInt, "ConnectionWindow", DefaultConnectionWindow ); REGISTER_VAR_INT( varsInt, "ConnectionRetry", DefaultConnectionRetry ); REGISTER_VAR_INT( varsInt, "RequestTimeout", DefaultRequestTimeout ); REGISTER_VAR_INT( varsInt, "StreamTimeout", DefaultStreamTimeout ); REGISTER_VAR_INT( varsInt, "SubStreamsPerChannel", DefaultSubStreamsPerChannel ); REGISTER_VAR_INT( varsInt, "TimeoutResolution", DefaultTimeoutResolution ); REGISTER_VAR_INT( varsInt, "StreamErrorWindow", DefaultStreamErrorWindow ); REGISTER_VAR_INT( varsInt, "RunForkHandler", DefaultRunForkHandler ); REGISTER_VAR_INT( varsInt, "RedirectLimit", DefaultRedirectLimit ); REGISTER_VAR_INT( varsInt, "WorkerThreads", DefaultWorkerThreads ); REGISTER_VAR_INT( varsInt, "CPChunkSize", DefaultCPChunkSize ); REGISTER_VAR_INT( varsInt, "CPParallelChunks", DefaultCPParallelChunks ); REGISTER_VAR_INT( varsInt, "DataServerTTL", DefaultDataServerTTL ); REGISTER_VAR_INT( varsInt, "LoadBalancerTTL", DefaultLoadBalancerTTL ); REGISTER_VAR_INT( varsInt, "CPInitTimeout", DefaultCPInitTimeout ); REGISTER_VAR_INT( varsInt, "CPTPCTimeout", DefaultCPTPCTimeout ); REGISTER_VAR_INT( varsInt, "CPTimeout", DefaultCPTimeout ); REGISTER_VAR_INT( varsInt, "TCPKeepAlive", DefaultTCPKeepAlive ); REGISTER_VAR_INT( varsInt, "TCPKeepAliveTime", DefaultTCPKeepAliveTime ); REGISTER_VAR_INT( varsInt, "TCPKeepAliveInterval", DefaultTCPKeepAliveInterval ); REGISTER_VAR_INT( varsInt, "TCPKeepProbes", DefaultTCPKeepAliveProbes ); REGISTER_VAR_INT( varsInt, "MultiProtocol", DefaultMultiProtocol ); REGISTER_VAR_INT( varsInt, "ParallelEvtLoop", DefaultParallelEvtLoop ); REGISTER_VAR_INT( varsInt, "MetalinkProcessing", DefaultMetalinkProcessing ); REGISTER_VAR_INT( varsInt, "LocalMetalinkFile", DefaultLocalMetalinkFile ); REGISTER_VAR_INT( varsInt, "XCpBlockSize", DefaultXCpBlockSize ); REGISTER_VAR_INT( varsInt, "NoDelay", DefaultNoDelay ); REGISTER_VAR_INT( varsInt, "AioSignal", DefaultAioSignal ); REGISTER_VAR_INT( varsInt, "PreferIPv4", DefaultPreferIPv4 ); REGISTER_VAR_INT( varsInt, "MaxMetalinkWait", DefaultMaxMetalinkWait ); REGISTER_VAR_INT( varsInt, "PreserveLocateTried", DefaultPreserveLocateTried ); REGISTER_VAR_INT( varsInt, "NotAuthorizedRetryLimit", DefaultNotAuthorizedRetryLimit ); REGISTER_VAR_INT( varsInt, "PreserveXAttrs", DefaultPreserveXAttrs ); REGISTER_VAR_INT( varsInt, "NoTlsOK", DefaultNoTlsOK ); REGISTER_VAR_INT( varsInt, "TlsNoData", DefaultTlsNoData ); REGISTER_VAR_INT( varsInt, "TlsMetalink", DefaultTlsMetalink ); REGISTER_VAR_INT( varsInt, "ZipMtlnCksum", DefaultZipMtlnCksum ); REGISTER_VAR_INT( varsInt, "IPNoShuffle", DefaultIPNoShuffle ); REGISTER_VAR_INT( varsInt, "WantTlsOnNoPgrw", DefaultWantTlsOnNoPgrw ); REGISTER_VAR_INT( varsInt, "RetryWrtAtLBLimit", DefaultRetryWrtAtLBLimit ); REGISTER_VAR_INT( varsInt, "XRateThreshold", DefaultXRateThreshold ); REGISTER_VAR_INT( varsInt, "CpRetry", DefaultCpRetry ); REGISTER_VAR_INT( varsInt, "CpUsePgWrtRd", DefaultCpUsePgWrtRd ); REGISTER_VAR_STR( varsStr, "ClientMonitor", DefaultClientMonitor ); REGISTER_VAR_STR( varsStr, "ClientMonitorParam", DefaultClientMonitorParam ); REGISTER_VAR_STR( varsStr, "NetworkStack", DefaultNetworkStack ); REGISTER_VAR_STR( varsStr, "PlugIn", DefaultPlugIn ); REGISTER_VAR_STR( varsStr, "PlugInConfDir", DefaultPlugInConfDir ); REGISTER_VAR_STR( varsStr, "ReadRecovery", DefaultReadRecovery ); REGISTER_VAR_STR( varsStr, "WriteRecovery", DefaultWriteRecovery ); REGISTER_VAR_STR( varsStr, "OpenRecovery", DefaultOpenRecovery ); REGISTER_VAR_STR( varsStr, "GlfnRedirector", DefaultGlfnRedirector ); REGISTER_VAR_STR( varsStr, "TlsDbgLvl", DefaultTlsDbgLvl ); REGISTER_VAR_STR( varsStr, "CpTarget", DefaultCpTarget ); REGISTER_VAR_STR( varsStr, "CpRetryPolicy", DefaultCpRetryPolicy ); //-------------------------------------------------------------------------- // Process the configuration files //-------------------------------------------------------------------------- std::map config, userConfig; Status st = Utils::ProcessConfig( config, "/etc/xrootd/client.conf" ); if( !st.IsOK() ) log->Warning( UtilityMsg, "Unable to process global config file: %s", st.ToString().c_str() ); XrdSysPwd pwdHandler; passwd *pwd = pwdHandler.Get( getuid() ); if( pwd ) { std::string userConfigFile = pwd->pw_dir; userConfigFile += "/.xrootd/client.conf"; st = Utils::ProcessConfig( userConfig, userConfigFile ); if( !st.IsOK() ) log->Debug( UtilityMsg, "Unable to process user config file: %s", st.ToString().c_str() ); } else log->Debug( UtilityMsg, "Unable to find user home directory." ); char *conffile = getenv( "XRD_CLCONFFILE" ); if( conffile ) { st = Utils::ProcessConfig( userConfig, conffile ); if( !st.IsOK() ) log->Debug( UtilityMsg, "Unable to process %s file: %s", conffile, st.ToString().c_str() ); } char *confdir = getenv( "XRD_CLCONFDIR" ); if( confdir ) { st = Utils::ProcessConfigDir( userConfig, confdir ); if( !st.IsOK() ) log->Debug( UtilityMsg, "Unable to process %s file: %s", confdir, st.ToString().c_str() ); } std::map::iterator it; for( it = config.begin(); it != config.end(); ++it ) log->Dump( UtilityMsg, "[Global config] \"%s\" = \"%s\"", it->first.c_str(), it->second.c_str() ); for( it = userConfig.begin(); it != userConfig.end(); ++it ) { config[it->first] = it->second; log->Dump( UtilityMsg, "[User config] \"%s\" = \"%s\"", it->first.c_str(), it->second.c_str() ); } for( it = config.begin(); it != config.end(); ++it ) log->Debug( UtilityMsg, "[Effective config] \"%s\" = \"%s\"", it->first.c_str(), it->second.c_str() ); //-------------------------------------------------------------------------- // Monitoring settings //-------------------------------------------------------------------------- char *tmp = strdup( XrdSysUtils::ExecName() ); char *appName = basename( tmp ); PutString( "AppName", appName ); free( tmp ); ImportString( "AppName", "XRD_APPNAME" ); PutString( "MonInfo", "" ); ImportString( "MonInfo", "XRD_MONINFO" ); //-------------------------------------------------------------------------- // Process ints //-------------------------------------------------------------------------- for( size_t i = 0; i < varsInt.size(); ++i ) { PutInt( varsInt[i].name, varsInt[i].def ); it = config.find( varsInt[i].name ); if( it != config.end() ) { char *endPtr = 0; int value = (int)strtol( it->second.c_str(), &endPtr, 0 ); if( *endPtr ) log->Warning( UtilityMsg, "Unable to set %s to %s: not a proper " "integer", varsInt[i].name.c_str(), it->second.c_str() ); else PutInt( varsInt[i].name, value ); } std::string name = "XRD_" + varsInt[i].name; std::transform( name.begin(), name.end(), name.begin(), ::toupper ); ImportInt( varsInt[i].name, name ); } //-------------------------------------------------------------------------- // Process strings //-------------------------------------------------------------------------- for( size_t i = 0; i < varsStr.size(); ++i ) { PutString( varsStr[i].name, varsStr[i].def ); it = config.find( varsStr[i].name ); if( it != config.end() ) PutString( varsStr[i].name, it->second ); std::string name = "XRD_" + varsStr[i].name; std::transform( name.begin(), name.end(), name.begin(), ::toupper ); ImportString( varsStr[i].name, name ); } //-------------------------------------------------------------------------- // Register fork handlers //-------------------------------------------------------------------------- pthread_atfork( prepare, parent, child ); } //---------------------------------------------------------------------------- // Get default client environment //---------------------------------------------------------------------------- Env *DefaultEnv::GetEnv() { return sEnv; } //---------------------------------------------------------------------------- // Get default post master //---------------------------------------------------------------------------- PostMaster *DefaultEnv::GetPostMaster() { PostMaster* postMaster = AtomicGet(sPostMaster); if( unlikely( !postMaster ) ) { XrdSysMutexHelper scopedLock( sInitMutex ); postMaster = AtomicGet(sPostMaster); if( postMaster ) return postMaster; postMaster = new PostMaster(); if( !postMaster->Initialize() ) { delete postMaster; postMaster = 0; return 0; } if( !postMaster->Start() ) { postMaster->Finalize(); delete postMaster; postMaster = 0; return 0; } sForkHandler->RegisterPostMaster( postMaster ); postMaster->GetTaskManager()->RegisterTask( sFileTimer, time(0), false ); AtomicCAS(sPostMaster, sPostMaster, postMaster); } return postMaster; } //---------------------------------------------------------------------------- // Get log //---------------------------------------------------------------------------- Log *DefaultEnv::GetLog() { return sLog; } //---------------------------------------------------------------------------- // Set log level //---------------------------------------------------------------------------- void DefaultEnv::SetLogLevel( const std::string &level ) { Log *log = GetLog(); log->SetLevel( level ); } //---------------------------------------------------------------------------- // Set log file //---------------------------------------------------------------------------- bool DefaultEnv::SetLogFile( const std::string &filepath ) { Log *log = GetLog(); LogOutFile *out = new LogOutFile(); if( out->Open( filepath ) ) { log->SetOutput( out ); return true; } delete out; return false; } //---------------------------------------------------------------------------- //! Set log mask. //------------------------------------------------------------------------ void DefaultEnv::SetLogMask( const std::string &level, const std::string &mask ) { Log *log = GetLog(); MaskTranslator translator; uint64_t topicMask = translator.translateMask( mask ); if( level == "All" ) { log->SetMask( Log::ErrorMsg, topicMask ); log->SetMask( Log::WarningMsg, topicMask ); log->SetMask( Log::InfoMsg, topicMask ); log->SetMask( Log::DebugMsg, topicMask ); log->SetMask( Log::DumpMsg, topicMask ); return; } log->SetMask( level, topicMask ); } //---------------------------------------------------------------------------- // Get fork handler //---------------------------------------------------------------------------- ForkHandler *DefaultEnv::GetForkHandler() { return sForkHandler; } //---------------------------------------------------------------------------- // Get fork handler //---------------------------------------------------------------------------- FileTimer *DefaultEnv::GetFileTimer() { return sFileTimer; } //---------------------------------------------------------------------------- // Get the monitor object //---------------------------------------------------------------------------- Monitor *DefaultEnv::GetMonitor() { if( unlikely( !sMonitorInitialized ) ) { XrdSysMutexHelper scopedLock( sInitMutex ); if( !sMonitorInitialized ) { //---------------------------------------------------------------------- // Check the environment settings //---------------------------------------------------------------------- Env *env = GetEnv(); Log *log = GetLog(); sMonitorInitialized = true; std::string monitorLib = DefaultClientMonitor; env->GetString( "ClientMonitor", monitorLib ); if( monitorLib.empty() ) { log->Debug( UtilityMsg, "Monitor library name not set. No " "monitoring" ); return 0; } std::string monitorParam = DefaultClientMonitorParam; env->GetString( "ClientMonitorParam", monitorParam ); log->Debug( UtilityMsg, "Initializing monitoring, lib: %s, param: %s", monitorLib.c_str(), monitorParam.c_str() ); //---------------------------------------------------------------------- // Loading the plugin //---------------------------------------------------------------------- char *errBuffer = new char[4000]; sMonitorLibHandle = new XrdOucPinLoader( errBuffer, 4000, &XrdVERSIONINFOVAR( XrdCl ), "monitor", monitorLib.c_str() ); typedef XrdCl::Monitor *(*MonLoader)(const char *, const char *); MonLoader loader; loader = (MonLoader)sMonitorLibHandle->Resolve( "XrdClGetMonitor", -1 ); if( !loader ) { log->Error( UtilityMsg, "Unable to initialize user monitoring: %s", errBuffer ); delete [] errBuffer; sMonitorLibHandle->Unload(); delete sMonitorLibHandle; sMonitorLibHandle = 0; return 0; } //---------------------------------------------------------------------- // Instantiating the monitor object //---------------------------------------------------------------------- const char *param = monitorParam.empty() ? 0 : monitorParam.c_str(); sMonitor = (*loader)( XrdSysUtils::ExecName(), param ); if( !sMonitor ) { log->Error( UtilityMsg, "Unable to initialize user monitoring: %s", errBuffer ); delete [] errBuffer; sMonitorLibHandle->Unload(); delete sMonitorLibHandle; sMonitorLibHandle = 0; return 0; } log->Debug( UtilityMsg, "Successfully initialized monitoring from: %s", monitorLib.c_str() ); delete [] errBuffer; } } return sMonitor; } //---------------------------------------------------------------------------- // Get checksum manager //---------------------------------------------------------------------------- CheckSumManager *DefaultEnv::GetCheckSumManager() { if( unlikely( !sCheckSumManager ) ) { XrdSysMutexHelper scopedLock( sInitMutex ); if( !sCheckSumManager ) sCheckSumManager = new CheckSumManager(); } return sCheckSumManager; } //---------------------------------------------------------------------------- // Get transport manager //---------------------------------------------------------------------------- TransportManager *DefaultEnv::GetTransportManager() { if( unlikely( !sTransportManager ) ) { XrdSysMutexHelper scopedLock( sInitMutex ); if( !sTransportManager ) sTransportManager = new TransportManager(); } return sTransportManager; } //---------------------------------------------------------------------------- // Get plug-in manager //---------------------------------------------------------------------------- PlugInManager *DefaultEnv::GetPlugInManager() { return sPlugInManager; } //---------------------------------------------------------------------------- // Retrieve the plug-in factory for the given URL //---------------------------------------------------------------------------- PlugInFactory *DefaultEnv::GetPlugInFactory( const std::string url ) { return sPlugInManager->GetFactory( url ); } //---------------------------------------------------------------------------- // Initialize the environment //---------------------------------------------------------------------------- void DefaultEnv::Initialize() { sLog = new Log(); SetUpLog(); sEnv = new DefaultEnv(); sForkHandler = new ForkHandler(); sFileTimer = new FileTimer(); sPlugInManager = new PlugInManager(); sPlugInManager->ProcessEnvironmentSettings(); sForkHandler->RegisterFileTimer( sFileTimer ); //-------------------------------------------------------------------------- // MacOSX library loading is completely moronic. We cannot dlopen a library // from a thread other than a main thread, so we-pre dlopen all the // libraries that we may potentially want. //-------------------------------------------------------------------------- #ifdef __APPLE__ char *errBuff = new char[1024]; const char *libs[] = { "libXrdSeckrb5.so", "libXrdSecgsi.so", "libXrdSecgsiAuthzVO.so", "libXrdSecgsiGMAPDN.so", "libXrdSecpwd.so", "libXrdSecsss.so", "libXrdSecunix.so", 0 }; for( int i = 0; libs[i]; ++i ) { sLog->Debug( UtilityMsg, "Attempting to pre-load: %s", libs[i] ); bool ok = XrdOucPreload( libs[i], errBuff, 1024 ); if( !ok ) sLog->Error( UtilityMsg, "Unable to pre-load %s: %s", libs[i], errBuff ); } delete [] errBuff; #endif } //---------------------------------------------------------------------------- // Finalize the environment //---------------------------------------------------------------------------- void DefaultEnv::Finalize() { if( sPostMaster ) { sPostMaster->Stop(); sPostMaster->Finalize(); delete sPostMaster; sPostMaster = 0; } delete sTransportManager; sTransportManager = 0; delete sCheckSumManager; sCheckSumManager = 0; delete sMonitor; sMonitor = 0; if( sMonitorLibHandle ) sMonitorLibHandle->Unload(); delete sMonitorLibHandle; sMonitorLibHandle = 0; delete sForkHandler; sForkHandler = 0; delete sFileTimer; sFileTimer = 0; delete sPlugInManager; sPlugInManager = 0; delete sEnv; sEnv = 0; delete sLog; sLog = 0; } //---------------------------------------------------------------------------- // Re-initialize the logging //---------------------------------------------------------------------------- void DefaultEnv::ReInitializeLogging() { delete sLog; sLog = new Log(); SetUpLog(); } //---------------------------------------------------------------------------- // Set up the log //---------------------------------------------------------------------------- void DefaultEnv::SetUpLog() { Log *log = GetLog(); //-------------------------------------------------------------------------- // Check if the log level has been defined in the environment //-------------------------------------------------------------------------- char *level = getenv( "XRD_LOGLEVEL" ); if( level ) log->SetLevel( level ); //-------------------------------------------------------------------------- // Check if we need to log to a file //-------------------------------------------------------------------------- char *file = getenv( "XRD_LOGFILE" ); if( file ) { LogOutFile *out = new LogOutFile(); if( out->Open( file ) ) log->SetOutput( out ); else delete out; } //-------------------------------------------------------------------------- // Log mask defaults //-------------------------------------------------------------------------- MaskTranslator translator; log->SetMask( Log::DumpMsg, translator.translateMask( "All|^PollerMsg" ) ); //-------------------------------------------------------------------------- // Initialize the topic mask //-------------------------------------------------------------------------- char *logMask = getenv( "XRD_LOGMASK" ); if( logMask ) { uint64_t mask = translator.translateMask( logMask ); log->SetMask( Log::ErrorMsg, mask ); log->SetMask( Log::WarningMsg, mask ); log->SetMask( Log::InfoMsg, mask ); log->SetMask( Log::DebugMsg, mask ); log->SetMask( Log::DumpMsg, mask ); } logMask = getenv( "XRD_LOGMASK_ERROR" ); if( logMask ) log->SetMask( Log::ErrorMsg, translator.translateMask( logMask ) ); logMask = getenv( "XRD_LOGMASK_WARNING" ); if( logMask ) log->SetMask( Log::WarningMsg, translator.translateMask( logMask ) ); logMask = getenv( "XRD_LOGMASK_INFO" ); if( logMask ) log->SetMask( Log::InfoMsg, translator.translateMask( logMask ) ); logMask = getenv( "XRD_LOGMASK_DEBUG" ); if( logMask ) log->SetMask( Log::DebugMsg, translator.translateMask( logMask ) ); logMask = getenv( "XRD_LOGMASK_DUMP" ); if( logMask ) log->SetMask( Log::DumpMsg, translator.translateMask( logMask ) ); //-------------------------------------------------------------------------- // Set up the topic strings //-------------------------------------------------------------------------- log->SetTopicName( AppMsg, "App" ); log->SetTopicName( UtilityMsg, "Utility" ); log->SetTopicName( FileMsg, "File" ); log->SetTopicName( PollerMsg, "Poller" ); log->SetTopicName( PostMasterMsg, "PostMaster" ); log->SetTopicName( XRootDTransportMsg, "XRootDTransport" ); log->SetTopicName( TaskMgrMsg, "TaskMgr" ); log->SetTopicName( XRootDMsg, "XRootD" ); log->SetTopicName( FileSystemMsg, "FileSystem" ); log->SetTopicName( AsyncSockMsg, "AsyncSock" ); log->SetTopicName( JobMgrMsg, "JobMgr" ); log->SetTopicName( PlugInMgrMsg, "PlugInMgr" ); log->SetTopicName( ExDbgMsg, "ExDbgMsg" ); log->SetTopicName( TlsMsg, "TlsMsg" ); log->SetTopicName( ZipMsg, "ZipMsg" ); } } //------------------------------------------------------------------------------ // Static initialization and finalization //------------------------------------------------------------------------------ int EnvInitializer::counter = 0; //------------------------------------------------------------------------------ // The constructor will be invoked in every translation unit // that includes XrdClDefaultEnv.hh, but the DefaultEnv will // be initialized only in the first one //------------------------------------------------------------------------------ EnvInitializer::EnvInitializer () { if( counter++ == 0 ) XrdCl::DefaultEnv::Initialize(); } //------------------------------------------------------------------------------ // The destructor will be invoked in every translation unit // that includes XrdClDefaultEnv.hh, but the DefaultEnv will // be finalized only once in the last one //------------------------------------------------------------------------------ EnvInitializer::~EnvInitializer () { if( --counter == 0 ) XrdCl::DefaultEnv::Finalize(); } xrootd-5.6.9/src/XrdCl/XrdClDefaultEnv.hh000066400000000000000000000200701457266313600202040ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2012 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . //------------------------------------------------------------------------------ #ifndef __XRD_CL_DEFAULT_ENV_HH__ #define __XRD_CL_DEFAULT_ENV_HH__ #include "XrdSys/XrdSysPthread.hh" #include "XrdCl/XrdClEnv.hh" #include "XrdVersion.hh" class XrdOucPinLoader; struct EnvInitializer; namespace XrdCl { class PostMaster; class Log; class ForkHandler; class Monitor; class CheckSumManager; class TransportManager; class FileTimer; class PlugInManager; class PlugInFactory; //---------------------------------------------------------------------------- //! Default environment for the client. Responsible for setting/importing //! defaults for the variables used by the client. And holding other //! global stuff. //---------------------------------------------------------------------------- class DefaultEnv: public Env { friend struct ::EnvInitializer; //------------------------------------------------------------------------ //! Constructor //------------------------------------------------------------------------ DefaultEnv(); public: //------------------------------------------------------------------------ //! Get client version //------------------------------------------------------------------------ inline static std::string GetVersion() { return XrdVERSION; } //------------------------------------------------------------------------ //! Get default client environment //------------------------------------------------------------------------ static Env *GetEnv(); //------------------------------------------------------------------------ //! Get default post master //------------------------------------------------------------------------ static PostMaster *GetPostMaster(); //------------------------------------------------------------------------ //! Get default log //------------------------------------------------------------------------ static Log *GetLog(); //------------------------------------------------------------------------ //! Set log level //! //! @param level Dump, Debug, Info, Warning or Error //------------------------------------------------------------------------ static void SetLogLevel( const std::string &level ); //------------------------------------------------------------------------ //! Set log file //! //! @param filepath path to the log file //------------------------------------------------------------------------ static bool SetLogFile( const std::string &filepath ); //------------------------------------------------------------------------ //! Set log mask. //! Determines which diagnostics topics should be printed. It's a //! "|" separated list of topics. The first element may be "All" in which //! case all the topics are enabled and the subsequent elements may turn //! them off, or "None" in which case all the topics are disabled and //! the subsequent flags may turn them on. If the topic name is prefixed //! with "^", then it means that the topic should be disabled. If the //! topic name is not prefixed, then it means that the topic should be //! enabled. //! //! The default for each level is "All", except for the "Dump" level, //! where the default is "All|^PollerMsg". This means that, at the //! "Dump" level, all the topics but "PollerMsg" are enabled. //! //! Available topics: AppMsg, UtilityMsg, FileMsg, PollerMsg, //! PostMasterMsg, XRootDTransportMsg, TaskMgrMsg, XRootDMsg, //! FileSystemMsg, AsyncSockMsg //! //! @param level log level or "All" for all levels //! @param mask log mask //------------------------------------------------------------------------ static void SetLogMask( const std::string &level, const std::string &mask ); //------------------------------------------------------------------------ //! Get the fork handler //------------------------------------------------------------------------ static ForkHandler *GetForkHandler(); //------------------------------------------------------------------------ //! Get file timer task //------------------------------------------------------------------------ static FileTimer *GetFileTimer(); //------------------------------------------------------------------------ //! Get the monitor object //------------------------------------------------------------------------ static Monitor *GetMonitor(); //------------------------------------------------------------------------ //! Get checksum manager //------------------------------------------------------------------------ static CheckSumManager *GetCheckSumManager(); //------------------------------------------------------------------------ //! Get transport manager //------------------------------------------------------------------------ static TransportManager *GetTransportManager(); //------------------------------------------------------------------------ //! Get plug-in manager //------------------------------------------------------------------------ static PlugInManager *GetPlugInManager(); //------------------------------------------------------------------------ //! Retrieve the plug-in factory for the given URL //! //! @return you do not own the returned memory //------------------------------------------------------------------------ static PlugInFactory *GetPlugInFactory( const std::string url ); //------------------------------------------------------------------------ //! Re-initialize the logging //------------------------------------------------------------------------ static void ReInitializeLogging(); private: //------------------------------------------------------------------------ //! Initialize the environment //------------------------------------------------------------------------ static void Initialize(); //------------------------------------------------------------------------ //! Finalize the environment //------------------------------------------------------------------------ static void Finalize(); static void SetUpLog(); static XrdSysMutex sInitMutex; static Env *sEnv; static PostMaster *sPostMaster; static Log *sLog; static ForkHandler *sForkHandler; static FileTimer *sFileTimer; static Monitor *sMonitor; static XrdOucPinLoader *sMonitorLibHandle; static bool sMonitorInitialized; static CheckSumManager *sCheckSumManager; static TransportManager *sTransportManager; static PlugInManager *sPlugInManager; }; } static struct EnvInitializer { EnvInitializer(); ~EnvInitializer(); static int counter; } initializer; #endif // __XRD_CL_DEFAULT_ENV_HH__ xrootd-5.6.9/src/XrdCl/XrdClDlgEnv.hh000066400000000000000000000046241457266313600173350ustar00rootroot00000000000000/* * XrdClDlgEnv.hh * * Created on: Oct 17, 2018 * Author: simonm */ #ifndef SRC_XRDCL_XRDCLDLGENV_HH_ #define SRC_XRDCL_XRDCLDLGENV_HH_ #include namespace XrdCl { //---------------------------------------------------------------------------- //! Helper class for setting and unsetting the 'XrdSecGSIDELEGPROXY' //! environment variable. //---------------------------------------------------------------------------- class DlgEnv { public: //------------------------------------------------------------------------ //! @return : instance of DlgEnv //------------------------------------------------------------------------ static DlgEnv& Instance() { static DlgEnv instance; return instance; } //------------------------------------------------------------------------ //! Destructor //! //! Release the memory used to set environment //------------------------------------------------------------------------ ~DlgEnv() { unsetenv( "XrdSecGSIDELEGPROXY" ); } //------------------------------------------------------------------------ //! Enable delegation in the environment //------------------------------------------------------------------------ void Enable() { setenv( "XrdSecGSIDELEGPROXY", "1", 1 ); } //------------------------------------------------------------------------ //! Disable delegation in the environment //------------------------------------------------------------------------ void Disable() { setenv( "XrdSecGSIDELEGPROXY", "0", 1 ); } private: //------------------------------------------------------------------------ //! Default constructor //------------------------------------------------------------------------ DlgEnv() { } //------------------------------------------------------------------------ //! Copy constructor - deleted //------------------------------------------------------------------------ DlgEnv( const DlgEnv& ); //------------------------------------------------------------------------ //! Assigment operator - deleted //------------------------------------------------------------------------ DlgEnv& operator=( const DlgEnv& ); }; } #endif /* SRC_XRDCL_XRDCLDLGENV_HH_ */ xrootd-5.6.9/src/XrdCl/XrdClEcHandler.cc000066400000000000000000000156221457266313600177710ustar00rootroot00000000000000/* * XrdClEcHandler.cc * * Created on: Nov 2, 2021 * Author: simonm */ #include "XrdCl/XrdClEcHandler.hh" namespace XrdCl { ServerSpaceInfo::ServerSpaceInfo() { if (getenv("XrdCl_EC_X_RATIO")) { xRatio = atoi(getenv("XrdCl_EC_X_RATIO")); } else { xRatio = 1; } }; void ServerSpaceInfo::SelectLocations(XrdCl::LocationInfo &oldList, XrdCl::LocationInfo &newList, uint32_t n) { TryInitExportPaths(); AddServers(oldList); UpdateSpaceInfo(); lock.lock(); if (oldList.GetSize() > n && ! BlindSelect()) { for (uint32_t j=0; jToString(); int b = resp.find("oss.free=", 0); int e = resp.find("&", b); uint64_t s = 0; std::stringstream sstream0( resp.substr(b+9, e-(b+9)) ); sstream0 >> s; if (queryResp) delete queryResp; return s; } if (queryResp) delete queryResp; } return 0; }; bool ServerSpaceInfo::BlindSelect() { auto ms_since_epoch = std::chrono::system_clock::now().time_since_epoch() / std::chrono::nanoseconds(1); return (ms_since_epoch % 10 > xRatio ? true : false); }; void ServerSpaceInfo::UpdateSpaceInfo() { if (! initExportPaths) return; time_t t = time(NULL); if (t < lastUpdateT + 300) return; lock.lock(); if (t > lastUpdateT + 300) { for (uint32_t j=0; jsecond ); itr = params.find( "xrdec.nbprt" ); if( itr == params.end() ) return nullptr; uint8_t nbprt = std::stoul( itr->second ); itr = params.find( "xrdec.blksz" ); if( itr == params.end() ) return nullptr; uint64_t blksz = std::stoul( itr->second ); itr = params.find( "xrdec.plgr" ); if( itr == params.end() ) return nullptr; std::vector plgr; Utils::splitString( plgr, itr->second, "," ); if( plgr.size() < nbdta + nbprt ) return nullptr; itr = params.find( "xrdec.objid" ); if( itr == params.end() ) return nullptr; std::string objid = itr->second; itr = params.find( "xrdec.format" ); if( itr == params.end() ) return nullptr; size_t format = std::stoul( itr->second ); if( format != 1 ) return nullptr; // TODO use constant std::vector dtacgi; itr = params.find( "xrdec.dtacgi" ); if( itr != params.end() ) { Utils::splitString( dtacgi, itr->second, "," ); if( plgr.size() != dtacgi.size() ) return nullptr; } std::vector mdtacgi; itr = params.find( "xrdec.mdtacgi" ); if( itr != params.end() ) { Utils::splitString( mdtacgi, itr->second, "," ); if( plgr.size() != mdtacgi.size() ) return nullptr; } itr = params.find( "xrdec.cosc" ); if( itr == params.end() ) return nullptr; std::string cosc_str = itr->second; if( cosc_str != "true" && cosc_str != "false" ) return nullptr; bool cosc = cosc_str == "true"; std::string ckstype; itr = params.find( "xrdec.cksum" ); if( cosc && itr == params.end() ) return nullptr; if( cosc ) ckstype = itr->second; std::string chdigest; itr = params.find( "xrdec.chdigest" ); if( itr == params.end() ) chdigest = "crc32c"; else chdigest = itr->second; bool usecrc32c = ( chdigest == "crc32c" ); bool nomtfile = false; itr = params.find( "xrdec.nomtfile" ); if( itr != params.end() ) nomtfile = ( itr->second == "true" ); XrdEc::ObjCfg *objcfg = new XrdEc::ObjCfg( objid, nbdta, nbprt, blksz / nbdta, usecrc32c ); objcfg->plgr = std::move( plgr ); objcfg->dtacgi = std::move( dtacgi ); objcfg->mdtacgi = std::move( mdtacgi ); objcfg->nomtfile = nomtfile; std::unique_ptr cksHelper( cosc ? new CheckSumHelper( "", ckstype ) : nullptr ); if( cksHelper ) { auto st = cksHelper->Initialize(); if( !st.IsOK() ) return nullptr; } return new EcHandler( headnode, objcfg, std::move( cksHelper ) ); } } xrootd-5.6.9/src/XrdCl/XrdClEcHandler.hh000066400000000000000000000427341457266313600200070ustar00rootroot00000000000000/* * XrdClEcHandler.hh * * Created on: 23 Mar 2021 * Author: simonm */ #ifndef SRC_XRDCL_XRDCLECHANDLER_HH_ #define SRC_XRDCL_XRDCLECHANDLER_HH_ #include "XrdCl/XrdClPlugInInterface.hh" #include "XrdCl/XrdClUtils.hh" #include "XrdCl/XrdClCheckSumHelper.hh" #include "XrdCl/XrdClResponseJob.hh" #include "XrdEc/XrdEcReader.hh" #include "XrdEc/XrdEcStrmWriter.hh" #include "XrdOuc/XrdOucCRC.hh" #include "XrdOuc/XrdOucPgrwUtils.hh" #include #include #include #include #include namespace XrdCl { class FreeSpace { public: std::string address; uint64_t freeSpace; FreeSpace() {}; bool operator<(const FreeSpace &a) const { return ((freeSpace > a.freeSpace) ? true : false); } void Dump() const { std::cout << address << " : " << freeSpace << std::endl; } }; class ServerSpaceInfo { public: ServerSpaceInfo(); ~ServerSpaceInfo() {}; // From the old location list, select a new location list // n: select at least "n" nodes in the new location list void SelectLocations(XrdCl::LocationInfo &oldList, XrdCl::LocationInfo &newList, uint32_t n); void Dump(); private: std::vector ServerList; std::vector ExportPaths; time_t lastUpdateT = 0; int xRatio = 10; std::mutex lock; bool initExportPaths = false; void TryInitExportPaths(); uint64_t GetFreeSpace(const std::string addr); bool BlindSelect(); void UpdateSpaceInfo(); bool Exists(XrdCl::LocationInfo::Location &loc); void AddServers(XrdCl::LocationInfo &locInfo); }; class EcPgReadResponseHandler : public ResponseHandler { private: XrdCl::ResponseHandler *realHandler; public: // constructor EcPgReadResponseHandler(ResponseHandler *a) : realHandler(a) {} // Response Handler void HandleResponse(XRootDStatus *status, AnyObject *rdresp) { if( !status->IsOK() ) { realHandler->HandleResponse( status, rdresp ); delete this; return; } ChunkInfo *chunk = 0; rdresp->Get(chunk); std::vector cksums; size_t nbpages = chunk->length / XrdSys::PageSize; if( chunk->length % XrdSys::PageSize ) ++nbpages; cksums.reserve( nbpages ); size_t size = chunk->length; char *buffer = reinterpret_cast( chunk->buffer ); for( size_t pg = 0; pg < nbpages; ++pg ) { size_t pgsize = XrdSys::PageSize; if( pgsize > size ) pgsize = size; uint32_t crcval = XrdOucCRC::Calc32C( buffer, pgsize ); cksums.push_back( crcval ); buffer += pgsize; size -= pgsize; } PageInfo *pages = new PageInfo(chunk->offset, chunk->length, chunk->buffer, std::move(cksums)); delete rdresp; AnyObject *response = new AnyObject(); response->Set( pages ); realHandler->HandleResponse( status, response ); delete this; } }; class EcHandler : public FilePlugIn { public: EcHandler( const URL &redir, XrdEc::ObjCfg *objcfg, std::unique_ptr cksHelper ) : redir( redir ), fs( redir, false ), objcfg( objcfg ), curroff( 0 ), cksHelper( std::move( cksHelper ) ) { XrdEc::Config::Instance().enable_plugins = false; } virtual ~EcHandler() { } XRootDStatus Open( uint16_t flags, ResponseHandler *handler, uint16_t timeout ) { if( ( flags & OpenFlags::Write ) || ( flags & OpenFlags::Update ) ) { if( !( flags & OpenFlags::New ) || // it has to be a new file ( flags & OpenFlags::Delete ) || // truncation is not supported ( flags & OpenFlags::Read ) ) // write + read is not supported return XRootDStatus( stError, errNotSupported ); if( objcfg->plgr.empty() ) { XRootDStatus st = LoadPlacement(); if( !st.IsOK() ) return st; } writer.reset( new XrdEc::StrmWriter( *objcfg ) ); writer->Open( handler, timeout ); return XRootDStatus(); } if( flags & OpenFlags::Read ) { if( flags & OpenFlags::Write ) return XRootDStatus( stError, errNotSupported ); if( objcfg->plgr.empty() ) { XRootDStatus st = LoadPlacement( redir.GetPath() ); if( !st.IsOK() ) return st; } reader.reset( new XrdEc::Reader( *objcfg ) ); reader->Open( handler, timeout ); return XRootDStatus(); } return XRootDStatus( stError, errNotSupported ); } XRootDStatus Open( const std::string &url, OpenFlags::Flags flags, Access::Mode mode, ResponseHandler *handler, uint16_t timeout ) { (void)url; (void)mode; return Open( flags, handler, timeout ); } //------------------------------------------------------------------------ //! //------------------------------------------------------------------------ XRootDStatus Close( ResponseHandler *handler, uint16_t timeout ) { if( writer ) { writer->Close( ResponseHandler::Wrap( [this, handler]( XRootDStatus *st, AnyObject *rsp ) { writer.reset(); if( st->IsOK() && bool( cksHelper ) ) { std::string commit = redir.GetPath() + "?xrdec.objid=" + objcfg->obj + "&xrdec.close=true&xrdec.size=" + std::to_string( curroff ); if( cksHelper ) { std::string ckstype = cksHelper->GetType(); std::string cksval; auto st = cksHelper->GetCheckSum( cksval, ckstype ); if( !st.IsOK() ) { handler->HandleResponse( new XRootDStatus( st ), nullptr ); return; } commit += "&xrdec.cksum=" + cksval; } Buffer arg; arg.FromString( commit ); auto st = fs.Query( QueryCode::OpaqueFile, arg, handler ); if( !st.IsOK() ) handler->HandleResponse( new XRootDStatus( st ), nullptr ); return; } handler->HandleResponse( st, rsp ); } ), timeout ); return XRootDStatus(); } if( reader ) { reader->Close( ResponseHandler::Wrap( [this, handler]( XRootDStatus *st, AnyObject *rsp ) { reader.reset(); handler->HandleResponse( st, rsp ); } ), timeout ); return XRootDStatus(); } return XRootDStatus( stError, errNotSupported ); } //------------------------------------------------------------------------ //! //------------------------------------------------------------------------ XRootDStatus Stat( bool force, ResponseHandler *handler, uint16_t timeout ) { if( !objcfg->nomtfile ) return fs.Stat( redir.GetPath(), handler, timeout ); if( !force && statcache ) { auto rsp = StatRsp( statcache->GetSize() ); Schedule( handler, rsp ); return XRootDStatus(); } if( writer ) { statcache.reset( new StatInfo() ); statcache->SetSize( writer->GetSize() ); auto rsp = StatRsp( statcache->GetSize() ); Schedule( handler, rsp ); return XRootDStatus(); } if( reader ) { statcache.reset( new StatInfo() ); statcache->SetSize( reader->GetSize() ); auto rsp = StatRsp( statcache->GetSize() ); Schedule( handler, rsp ); return XRootDStatus(); } return XRootDStatus( stError, errInvalidOp, 0, "File not open." ); } //------------------------------------------------------------------------ //! //------------------------------------------------------------------------ XRootDStatus Read( uint64_t offset, uint32_t size, void *buffer, ResponseHandler *handler, uint16_t timeout ) { if( !reader ) return XRootDStatus( stError, errInternal ); reader->Read( offset, size, buffer, handler, timeout ); return XRootDStatus(); } //------------------------------------------------------------------------ //! @see XrdCl::File::PgRead - async //------------------------------------------------------------------------ XRootDStatus PgRead(uint64_t offset, uint32_t size, void *buffer, ResponseHandler *handler, uint16_t timeout) { ResponseHandler *substitHandler = new EcPgReadResponseHandler( handler ); XRootDStatus st = Read(offset, size, buffer, substitHandler, timeout); return st; } //------------------------------------------------------------------------ //! @see File::Write //------------------------------------------------------------------------ XRootDStatus Write( uint64_t offset, uint32_t size, const void *buffer, ResponseHandler *handler, uint16_t timeout ) { if( cksHelper ) cksHelper->Update( buffer, size ); if( !writer ) return XRootDStatus( stError, errInternal ); if( offset != curroff ) return XRootDStatus( stError, errNotSupported ); writer->Write( size, buffer, handler ); curroff += size; return XRootDStatus(); } //------------------------------------------------------------------------ //! @see XrdCl::File::PgWrite - async //------------------------------------------------------------------------ XRootDStatus PgWrite( uint64_t offset, uint32_t size, const void *buffer, std::vector &cksums, ResponseHandler *handler, uint16_t timeout = 0 ) { if(! cksums.empty() ) { const char *data = static_cast( buffer ); std::vector local_cksums; XrdOucPgrwUtils::csCalc( data, offset, size, local_cksums ); if (data) delete data; if (local_cksums != cksums) return XRootDStatus( stError, errInvalidArgs, 0, "data and crc32c digests do not match." ); } return Write(offset, size, buffer, handler, timeout); } //------------------------------------------------------------------------ //! //------------------------------------------------------------------------ bool IsOpen() const { return writer || reader; } private: inline XRootDStatus LoadPlacement() { LocationInfo *infoAll = nullptr; XRootDStatus st = fs.DeepLocate( "*", OpenFlags::None, infoAll ); std::unique_ptr ptr( infoAll ); if( !st.IsOK() ) return st; LocationInfo *info = new LocationInfo(); std::unique_ptr ptr1( info ); static ServerSpaceInfo ssi; ssi.SelectLocations(*infoAll, *info, objcfg->nbchunks); if( info->GetSize() < objcfg->nbchunks ) return XRootDStatus( stError, errInvalidOp, 0, "Too few data servers." ); unsigned seed = std::chrono::system_clock::now().time_since_epoch().count(); shuffle (info->Begin(), info->End(), std::default_random_engine(seed)); for( size_t i = 0; i < objcfg->nbchunks; ++i ) { auto &location = info->At( i ); objcfg->plgr.emplace_back( "root://" + location.GetAddress() + '/' ); } return XRootDStatus(); } inline XRootDStatus LoadPlacement( const std::string &path ) { LocationInfo *info = nullptr; XRootDStatus st = fs.DeepLocate( "*", OpenFlags::None, info ); std::unique_ptr ptr( info ); if( !st.IsOK() ) return st; // The following check become meaningless if( info->GetSize() < objcfg->nbdata ) return XRootDStatus( stError, errInvalidOp, 0, "Too few data servers." ); uint64_t verNumMax = 0; std::vector verNums; std::vector xattrkeys; std::vector xattrvals; xattrkeys.push_back("xrdec.strpver"); for( size_t i = 0; i < info->GetSize(); ++i ) { FileSystem *fs_i = new FileSystem(info->At( i ).GetAddress()); xattrvals.clear(); st = fs_i->GetXAttr(path, xattrkeys, xattrvals, 0); if (st.IsOK() && ! xattrvals[0].value.empty()) { std::stringstream sstream(xattrvals[0].value); uint64_t verNum; sstream >> verNum; verNums.push_back(verNum); if (verNum > verNumMax) verNumMax = verNum; } else verNums.push_back(0); delete fs_i; } int n = 0; for( size_t i = 0; i < info->GetSize(); ++i ) { if ( verNums.at(i) == 0 || verNums.at(i) != verNumMax ) continue; else n++; auto &location = info->At( i ); objcfg->plgr.emplace_back( "root://" + location.GetAddress() + '/' ); } if (n < objcfg->nbdata ) return XRootDStatus( stError, errInvalidOp, 0, "Too few data servers." ); return XRootDStatus(); } inline static AnyObject* StatRsp( uint64_t size ) { StatInfo *info = new StatInfo(); info->SetSize( size ); AnyObject *rsp = new AnyObject(); rsp->Set( info ); return rsp; } inline static void Schedule( ResponseHandler *handler, AnyObject *rsp ) { ResponseJob *job = new ResponseJob( handler, new XRootDStatus(), rsp, nullptr ); XrdCl::DefaultEnv::GetPostMaster()->GetJobManager()->QueueJob( job ); } URL redir; FileSystem fs; std::unique_ptr objcfg; std::unique_ptr writer; std::unique_ptr reader; uint64_t curroff; std::unique_ptr cksHelper; std::unique_ptr statcache; }; //---------------------------------------------------------------------------- //! Plugin factory //---------------------------------------------------------------------------- class EcPlugInFactory : public PlugInFactory { public: //------------------------------------------------------------------------ //! Constructor //------------------------------------------------------------------------ EcPlugInFactory( uint8_t nbdta, uint8_t nbprt, uint64_t chsz, std::vector && plgr ) : nbdta( nbdta ), nbprt( nbprt ), chsz( chsz ), plgr( std::move( plgr ) ) { } //------------------------------------------------------------------------ //! Destructor //------------------------------------------------------------------------ virtual ~EcPlugInFactory() { } //------------------------------------------------------------------------ //! Create a file plug-in for the given URL //------------------------------------------------------------------------ virtual FilePlugIn *CreateFile( const std::string &u ) { URL url( u ); XrdEc::ObjCfg *objcfg = new XrdEc::ObjCfg( url.GetPath(), nbdta, nbprt, chsz, false, true ); objcfg->plgr = std::move( plgr ); return new EcHandler( url, objcfg, nullptr ); } //------------------------------------------------------------------------ //! Create a file system plug-in for the given URL //------------------------------------------------------------------------ virtual FileSystemPlugIn *CreateFileSystem( const std::string &url ) { return nullptr; } private: uint8_t nbdta; uint8_t nbprt; uint64_t chsz; std::vector plgr; }; EcHandler* GetEcHandler( const URL &headnode, const URL &redirurl ); } /* namespace XrdCl */ #endif /* SRC_XRDCL_XRDCLECHANDLER_HH_ */ xrootd-5.6.9/src/XrdCl/XrdClEnv.cc000066400000000000000000000201101457266313600166600ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2012 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . //------------------------------------------------------------------------------ #include #include "XrdCl/XrdClEnv.hh" #include "XrdCl/XrdClDefaultEnv.hh" #include "XrdCl/XrdClLog.hh" #include "XrdCl/XrdClConstants.hh" namespace XrdCl { //---------------------------------------------------------------------------- // Get string //---------------------------------------------------------------------------- bool Env::GetString( const std::string &k, std::string &value ) { std::string key = UnifyKey( k ); XrdSysRWLockHelper scopedLock( pLock ); StringMap::iterator it; it = pStringMap.find( key ); if( it == pStringMap.end() ) { Log *log = DefaultEnv::GetLog(); log->Debug( UtilityMsg, "Env: trying to get a non-existent string entry: %s", key.c_str() ); return false; } value = it->second.first; return true; } //---------------------------------------------------------------------------- // Put string //---------------------------------------------------------------------------- bool Env::PutString( const std::string &k, const std::string &value ) { std::string key = UnifyKey( k ); XrdSysRWLockHelper scopedLock( pLock, false ); //-------------------------------------------------------------------------- // Insert the string if it's not there yet //-------------------------------------------------------------------------- StringMap::iterator it; it = pStringMap.find( key ); if( it == pStringMap.end() ) { pStringMap[key] = std::make_pair( value, false ); return true; } //-------------------------------------------------------------------------- // The entry exists and it has been imported from the shell //-------------------------------------------------------------------------- Log *log = DefaultEnv::GetLog(); if( it->second.second ) { log->Debug( UtilityMsg, "Env: trying to override a shell-imported string entry: %s", key.c_str() ); return false; } log->Debug( UtilityMsg, "Env: overriding entry: %s=\"%s\" with \"%s\"", key.c_str(), it->second.first.c_str(), value.c_str() ); pStringMap[key] = std::make_pair( value, false ); return true; } //---------------------------------------------------------------------------- // Get int //---------------------------------------------------------------------------- bool Env::GetInt( const std::string &k, int &value ) { std::string key = UnifyKey( k ); XrdSysRWLockHelper scopedLock( pLock ); IntMap::iterator it; it = pIntMap.find( key ); if( it == pIntMap.end() ) { Log *log = DefaultEnv::GetLog(); log->Debug( UtilityMsg, "Env: trying to get a non-existent integer entry: %s", key.c_str() ); return false; } value = it->second.first; return true; } //---------------------------------------------------------------------------- // Put int //---------------------------------------------------------------------------- bool Env::PutInt( const std::string &k, int value ) { std::string key = UnifyKey( k ); XrdSysRWLockHelper scopedLock( pLock, false ); //-------------------------------------------------------------------------- // Insert the string if it's not there yet //-------------------------------------------------------------------------- IntMap::iterator it; it = pIntMap.find( key ); if( it == pIntMap.end() ) { pIntMap[key] = std::make_pair( value, false ); return true; } //-------------------------------------------------------------------------- // The entry exists and it has been imported from the shell //-------------------------------------------------------------------------- Log *log = DefaultEnv::GetLog(); if( it->second.second ) { log->Debug( UtilityMsg, "Env: trying to override a shell-imported integer entry: %s", key.c_str() ); return false; } log->Debug( UtilityMsg, "Env: overriding entry: %s=%d with %d", key.c_str(), it->second.first, value ); pIntMap[key] = std::make_pair( value, false ); return true; } //---------------------------------------------------------------------------- // Import int //---------------------------------------------------------------------------- bool Env::ImportInt( const std::string &k, const std::string &shellKey ) { std::string key = UnifyKey( k ); XrdSysRWLockHelper scopedLock( pLock, false ); std::string strValue = GetEnv( shellKey ); if( strValue == "" ) return false; Log *log = DefaultEnv::GetLog(); char *endPtr; int value = (int)strtol( strValue.c_str(), &endPtr, 0 ); if( *endPtr ) { log->Error( UtilityMsg, "Env: Unable to import %s as %s: %s is not a proper integer", shellKey.c_str(), key.c_str(), strValue.c_str() ); return false; } log->Info( UtilityMsg, "Env: Importing from shell %s=%d as %s", shellKey.c_str(), value, key.c_str() ); pIntMap[key] = std::make_pair( value, true ); return true; } //---------------------------------------------------------------------------- // Import string //---------------------------------------------------------------------------- bool Env::ImportString( const std::string &k, const std::string &shellKey ) { std::string key = UnifyKey( k ); XrdSysRWLockHelper scopedLock( pLock, false ); std::string value = GetEnv( shellKey ); if( value == "" ) return false; Log *log = DefaultEnv::GetLog(); log->Info( UtilityMsg, "Env: Importing from shell %s=%s as %s", shellKey.c_str(), value.c_str(), key.c_str() ); pStringMap[key] = std::make_pair( value, true ); return true; } //------------------------------------------------------------------------ // Get default integer value for the given key //------------------------------------------------------------------------ bool Env::GetDefaultIntValue( const std::string &k, int &value ) { std::string key = UnifyKey( k ); auto itr = theDefaultInts.find( key ); if( itr == theDefaultInts.end() ) return false; value = itr->second; return true; } //------------------------------------------------------------------------ // Get default string value for the given key //------------------------------------------------------------------------ bool Env::GetDefaultStringValue( const std::string &k, std::string &value ) { std::string key = UnifyKey( k ); auto itr = theDefaultStrs.find( key ); if( itr == theDefaultStrs.end() ) return false; value = itr->second; return true; } //---------------------------------------------------------------------------- // Get a string from the environment //---------------------------------------------------------------------------- std::string Env::GetEnv( const std::string &key ) { char *var = getenv( key.c_str() ); if( !var ) return ""; return var; } } xrootd-5.6.9/src/XrdCl/XrdClEnv.hh000066400000000000000000000173721457266313600167120ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2012 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . //------------------------------------------------------------------------------ #ifndef __XRD_CL_ENV_HH__ #define __XRD_CL_ENV_HH__ #include #include #include #include #include "XrdSys/XrdSysPthread.hh" namespace XrdCl { //---------------------------------------------------------------------------- //! A simple key value store intended to hold global configuration. //! It is able to import the settings from the shell environment, the //! variables imported this way supersede these provided from the C++ //! code. //---------------------------------------------------------------------------- class Env { public: //------------------------------------------------------------------------ //! Destructor //------------------------------------------------------------------------ virtual ~Env() {} //------------------------------------------------------------------------ //! Get a string associated to the given key //! //! @return true if the value was found, false otherwise //------------------------------------------------------------------------ bool GetString( const std::string &key, std::string &value ); //------------------------------------------------------------------------ //! Associate a string with the given key //! //! @return false if there is already a shell-imported setting for this //! key, true otherwise //------------------------------------------------------------------------ bool PutString( const std::string &key, const std::string &value ); //------------------------------------------------------------------------ //! Get an int associated to the given key //! //! @return true if the value was found, false otherwise //------------------------------------------------------------------------ bool GetInt( const std::string &key, int &value ); //------------------------------------------------------------------------ //! Associate an int with the given key //! //! @return false if there is already a shell-imported setting for this //! key, true otherwise //------------------------------------------------------------------------ bool PutInt( const std::string &key, int value ); //------------------------------------------------------------------------ //! Import an int from the shell environment. Any imported setting //! takes precedence over the one set by other means. //! //! @return true if the setting exists in the shell, false otherwise //------------------------------------------------------------------------ bool ImportInt( const std::string &key, const std::string &shellKey ); //------------------------------------------------------------------------ //! Import a string from the shell environment. Any imported setting //! takes precedence over the one set by ther means. //! //! @return true if the setting exists in the shell, false otherwise //------------------------------------------------------------------------ bool ImportString( const std::string &key, const std::string &shellKey ); //------------------------------------------------------------------------ //! Get default integer value for the given key //! @param key : the key //! @param value : output parameter, default value corresponding to //! the key //! @return : true if a default integer value for the given key //! exists, false otherwise //------------------------------------------------------------------------ bool GetDefaultIntValue( const std::string &key, int &value ); //------------------------------------------------------------------------ //! Get default string value for the given key //! @param key : the key //! @param value : output parameter, default value corresponding to //! the key //! @return : true if a default string value for the given key //! exists, false otherwise //------------------------------------------------------------------------ bool GetDefaultStringValue( const std::string &key, std::string &value ); //------------------------------------------------------------------------ // Lock the environment for writing //------------------------------------------------------------------------ void WriteLock() { pLock.WriteLock(); } //------------------------------------------------------------------------ // Unlock the environment //------------------------------------------------------------------------ void UnLock() { pLock.UnLock(); } //------------------------------------------------------------------------ // Re-initialize the lock //------------------------------------------------------------------------ void ReInitializeLock() { // this is really shaky, but seems to work on linux and fork safety // is probably not required anywhere else pLock.UnLock(); pLock.ReInitialize(); } //------------------------------------------------------------------------ // Re-create the lock in the same memory //------------------------------------------------------------------------ void RecreateLock() { new( &pLock )XrdSysRWLock(); } private: //------------------------------------------------------------------------ // Unify the key, make sure it is not case sensitive and strip it of // the XRD_ prefix if necessary //------------------------------------------------------------------------ inline std::string UnifyKey( std::string key ) { //---------------------------------------------------------------------- // Make the key lower case //---------------------------------------------------------------------- std::transform( key.begin(), key.end(), key.begin(), ::tolower ); //---------------------------------------------------------------------- // Strip the `xrd_` prefix if necessary //---------------------------------------------------------------------- static const char prefix[] = "xrd_"; if( key.compare( 0, sizeof( prefix ) - 1, prefix ) == 0 ) key = key.substr( sizeof( prefix ) - 1 ); return key; } std::string GetEnv( const std::string &key ); typedef std::map > StringMap; typedef std::map > IntMap; XrdSysRWLock pLock; StringMap pStringMap; IntMap pIntMap; }; } #endif // __XRD_CL_ENV_HH__ xrootd-5.6.9/src/XrdCl/XrdClFS.cc000066400000000000000000002325401457266313600164540ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2014 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #include "XrdCl/XrdClFileSystem.hh" #include "XrdCl/XrdClFileSystemUtils.hh" #include "XrdCl/XrdClFSExecutor.hh" #include "XrdCl/XrdClURL.hh" #include "XrdCl/XrdClLog.hh" #include "XrdCl/XrdClDefaultEnv.hh" #include "XrdCl/XrdClConstants.hh" #include "XrdCl/XrdClUtils.hh" #include "XrdCl/XrdClCopyProcess.hh" #include "XrdCl/XrdClFile.hh" #include "XrdCl/XrdClFileSystemOperations.hh" #include "XrdCl/XrdClParallelOperation.hh" #include "XrdSys/XrdSysE2T.hh" #include #include #include #include #include #ifdef HAVE_READLINE #include #include #endif using namespace XrdCl; //------------------------------------------------------------------------------ // Build a path //------------------------------------------------------------------------------ XRootDStatus BuildPath( std::string &newPath, Env *env, const std::string &path ) { if( path.empty() ) return XRootDStatus( stError, errInvalidArgs ); int noCwd = 0; env->GetInt( "NoCWD", noCwd ); if( path[0] == '/' || noCwd ) { newPath = path; return XRootDStatus(); } std::string cwd = "/"; env->GetString( "CWD", cwd ); newPath = cwd; newPath += "/"; newPath += path; //---------------------------------------------------------------------------- // Collapse the dots //---------------------------------------------------------------------------- std::list pathComponents; std::list::iterator it; XrdCl::Utils::splitString( pathComponents, newPath, "/" ); newPath = "/"; for( it = pathComponents.begin(); it != pathComponents.end(); ) { if( *it == "." ) { it = pathComponents.erase( it ); continue; } if( *it == ".." ) { if( it == pathComponents.begin() ) return XRootDStatus( stError, errInvalidArgs ); std::list::iterator it1 = it; --it1; it = pathComponents.erase( it1 ); it = pathComponents.erase( it ); continue; } ++it; } newPath = "/"; for( it = pathComponents.begin(); it != pathComponents.end(); ++it ) { newPath += *it; newPath += "/"; } if( newPath.length() > 1 ) newPath.erase( newPath.length()-1, 1 ); return XRootDStatus(); } //------------------------------------------------------------------------------ // Convert mode string to uint16_t //------------------------------------------------------------------------------ XRootDStatus ConvertMode( Access::Mode &mode, const std::string &modeStr ) { if( modeStr.length() != 9 ) return XRootDStatus( stError, errInvalidArgs ); mode = Access::None; for( int i = 0; i < 3; ++i ) { if( modeStr[i] == 'r' ) mode |= Access::UR; else if( modeStr[i] == 'w' ) mode |= Access::UW; else if( modeStr[i] == 'x' ) mode |= Access::UX; else if( modeStr[i] != '-' ) return XRootDStatus( stError, errInvalidArgs ); } for( int i = 3; i < 6; ++i ) { if( modeStr[i] == 'r' ) mode |= Access::GR; else if( modeStr[i] == 'w' ) mode |= Access::GW; else if( modeStr[i] == 'x' ) mode |= Access::GX; else if( modeStr[i] != '-' ) return XRootDStatus( stError, errInvalidArgs ); } for( int i = 6; i < 9; ++i ) { if( modeStr[i] == 'r' ) mode |= Access::OR; else if( modeStr[i] == 'w' ) mode |= Access::OW; else if( modeStr[i] == 'x' ) mode |= Access::OX; else if( modeStr[i] != '-' ) return XRootDStatus( stError, errInvalidArgs ); } return XRootDStatus(); } //------------------------------------------------------------------------------ // Perform a cache operation //------------------------------------------------------------------------------ XRootDStatus DoCache( FileSystem *fs, Env *env, const FSExecutor::CommandParams &args ) { //---------------------------------------------------------------------------- // Check up the args //---------------------------------------------------------------------------- Log *log = DefaultEnv::GetLog(); uint32_t argc = args.size(); if( argc != 3 ) { log->Error( AppMsg, "Wrong number of arguments." ); return XRootDStatus( stError, errInvalidArgs, 0, "Wrong number of arguments." ); } if( args[1] != "evict" && args[1] != "fevict") { log->Error( AppMsg, "Invalid cache operation." ); return XRootDStatus( stError, errInvalidArgs, 0, "Invalid cache operation." ); } std::string fullPath; if( !BuildPath( fullPath, env, args[2] ).IsOK() ) { log->Error( AppMsg, "Invalid cache path." ); return XRootDStatus( stError, errInvalidArgs, 0, "Invalid cache path." ); } //---------------------------------------------------------------------------- // Create the command //---------------------------------------------------------------------------- std::string cmd = args[1]; cmd.append(" "); cmd.append(fullPath); //---------------------------------------------------------------------------- // Run the operation //---------------------------------------------------------------------------- Buffer *response = 0; XRootDStatus st = fs->SendCache( cmd, response ); if( !st.IsOK() ) { log->Error( AppMsg, "Unable set cache %s: %s", fullPath.c_str(), st.ToStr().c_str() ); return st; } if( response ) { std::cout << response->ToString() << '\n'; } delete response; return XRootDStatus(); } //------------------------------------------------------------------------------ // Change current working directory //------------------------------------------------------------------------------ XRootDStatus DoCD( FileSystem *fs, Env *env, const FSExecutor::CommandParams &args ) { //---------------------------------------------------------------------------- // Check up the args //---------------------------------------------------------------------------- Log *log = DefaultEnv::GetLog(); if( args.size() != 2 ) { log->Error( AppMsg, "Invalid arguments. Expected a path." ); return XRootDStatus( stError, errInvalidArgs ); } //---------------------------------------------------------------------------- // cd excludes NoCWD //---------------------------------------------------------------------------- env->PutInt( "NoCWD", 0 ); std::string newPath; if( !BuildPath( newPath, env, args[1] ).IsOK() ) { log->Error( AppMsg, "Invalid path." ); return XRootDStatus( stError, errInvalidArgs ); } //---------------------------------------------------------------------------- // Check if the path exist and is not a directory //---------------------------------------------------------------------------- StatInfo *info; XRootDStatus st = fs->Stat( newPath, info ); if( !st.IsOK() ) { log->Error( AppMsg, "Unable to stat the path: %s", st.ToStr().c_str() ); return st; } if( !info->TestFlags( StatInfo::IsDir ) ) { log->Error( AppMsg, "%s is not a directory.", newPath.c_str() ); return XRootDStatus( stError, errInvalidArgs ); } env->PutString( "CWD", newPath ); delete info; return XRootDStatus(); } //------------------------------------------------------------------------------ // Helper function to calculate number of digits in a number //------------------------------------------------------------------------------ uint32_t nbDigits( uint64_t nb ) { if( nb == 0 ) return 1; return uint32_t( log10( double(nb) ) + 1); } void PrintDirListStatInfo( StatInfo *info, bool hascks = false, uint32_t ownerwidth = 0, uint32_t groupwidth = 0, uint32_t sizewidth = 0 ) { if( info->ExtendedFormat() ) { if( info->TestFlags( StatInfo::IsDir ) ) std::cout << "d"; else std::cout << "-"; std::cout << info->GetModeAsOctString(); std::cout << " " << std::setw( ownerwidth ) << info->GetOwner(); std::cout << " " << std::setw( groupwidth ) << info->GetGroup(); std::cout << " " << std::setw( sizewidth ) << info->GetSize(); if( hascks && info->HasChecksum() ) std::cout << " " << std::setw( sizewidth ) << info->GetChecksum(); std::cout << " " << info->GetModTimeAsString() << " "; } else { if( info->TestFlags( StatInfo::IsDir ) ) std::cout << "d"; else std::cout << "-"; if( info->TestFlags( StatInfo::IsReadable ) ) std::cout << "r"; else std::cout << "-"; if( info->TestFlags( StatInfo::IsWritable ) ) std::cout << "w"; else std::cout << "-"; if( info->TestFlags( StatInfo::XBitSet ) ) std::cout << "x"; else std::cout << "-"; std::cout << " " << info->GetModTimeAsString(); uint64_t size = info->GetSize(); int width = nbDigits( size ) + 2; if( width < 12 ) width = 12; std::cout << std::setw( width ) << info->GetSize() << " "; } } //------------------------------------------------------------------------------ // List a directory //------------------------------------------------------------------------------ XRootDStatus DoLS( FileSystem *fs, Env *env, const FSExecutor::CommandParams &args ) { //---------------------------------------------------------------------------- // Check up the args //---------------------------------------------------------------------------- Log *log = DefaultEnv::GetLog(); uint32_t argc = args.size(); bool stats = false; bool showUrls = false; bool hascks = false; std::string path; DirListFlags::Flags flags = DirListFlags::Locate | DirListFlags::Merge; if( argc > 6 ) { log->Error( AppMsg, "Too many arguments." ); return XRootDStatus( stError, errInvalidArgs ); } for( uint32_t i = 1; i < args.size(); ++i ) { if( args[i] == "-l" ) { stats = true; flags |= DirListFlags::Stat; } else if( args[i] == "-u" ) showUrls = true; else if( args[i] == "-R" ) { flags |= DirListFlags::Recursive; } else if( args[i] == "-D" ) { // show duplicates flags &= ~DirListFlags::Merge; } else if( args[i] == "-Z" ) { // check if file is a ZIP archive if yes list content flags |= DirListFlags::Zip; } else if( args[i] == "-C" ) { // query checksum for each entry in the directory hascks = true; stats = true; flags |= DirListFlags::Cksm; } else path = args[i]; } if( showUrls ) // we don't merge the duplicate entries // in case we print the full URL flags &= ~DirListFlags::Merge; std::string newPath = "/"; if( path.empty() ) env->GetString( "CWD", newPath ); else { if( !BuildPath( newPath, env, path ).IsOK() ) { log->Error( AppMsg, "Invalid arguments. Invalid path." ); return XRootDStatus( stError, errInvalidArgs ); } } //---------------------------------------------------------------------------- // Stat the entry so we know if it is a file or a directory //---------------------------------------------------------------------------- log->Debug( AppMsg, "Attempting to stat: %s", newPath.c_str() ); StatInfo *info = 0; XRootDStatus st = fs->Stat( newPath, info ); std::unique_ptr ptr( info ); if( !st.IsOK() ) { log->Error( AppMsg, "Unable to stat the path: %s", st.ToStr().c_str() ); return st; } if( !info->TestFlags( StatInfo::IsDir ) && !( flags & DirListFlags::Zip ) ) { if( stats ) PrintDirListStatInfo( info ); if( showUrls ) { std::string url; fs->GetProperty( "LastURL", url ); std::cout << url; } std::cout << newPath << std::endl; return XRootDStatus(); } //---------------------------------------------------------------------------- // Ask for the list //---------------------------------------------------------------------------- log->Debug( AppMsg, "Attempting to list: %s", newPath.c_str() ); DirectoryList *list; st = fs->DirList( newPath, flags, list ); if( !st.IsOK() ) { log->Error( AppMsg, "Unable to list the path: %s", st.ToStr().c_str() ); return st; } if( st.code == suPartial ) { std::cerr << "[!] Some of the requests failed. The result may be "; std::cerr << "incomplete." << std::endl; } uint32_t ownerwidth = 0, groupwidth = 0, sizewidth = 0, ckswidth = 0; DirectoryList::Iterator it; for( it = list->Begin(); it != list->End() && stats; ++it ) { StatInfo *info = (*it)->GetStatInfo(); if( ownerwidth < info->GetOwner().size() ) ownerwidth = info->GetOwner().size(); if( groupwidth < info->GetGroup().size() ) groupwidth = info->GetGroup().size(); if( sizewidth < nbDigits( info->GetSize() ) ) sizewidth = nbDigits( info->GetSize() ); if( ckswidth < info->GetChecksum().size() ) ckswidth = info->GetChecksum().size(); } //---------------------------------------------------------------------------- // Print the results //---------------------------------------------------------------------------- for( it = list->Begin(); it != list->End(); ++it ) { if( stats ) { StatInfo *info = (*it)->GetStatInfo(); if( !info ) std::cout << "---- 0000-00-00 00:00:00 ? "; else PrintDirListStatInfo( info, hascks, ownerwidth, groupwidth, sizewidth ); } if( showUrls ) std::cout << "root://" << (*it)->GetHostAddress() << "/"; std::cout << list->GetParentName() << (*it)->GetName() << std::endl; } delete list; return XRootDStatus(); } //------------------------------------------------------------------------------ // Create a directory //------------------------------------------------------------------------------ XRootDStatus DoMkDir( FileSystem *fs, Env *env, const FSExecutor::CommandParams &args ) { //---------------------------------------------------------------------------- // Check up the args //---------------------------------------------------------------------------- Log *log = DefaultEnv::GetLog(); uint32_t argc = args.size(); if( argc < 2 || argc > 4 ) { log->Error( AppMsg, "Too few arguments." ); return XRootDStatus( stError, errInvalidArgs ); } MkDirFlags::Flags flags = MkDirFlags::None; Access::Mode mode = Access::None; std::string modeStr = "rwxr-x---"; std::string path = ""; for( uint32_t i = 1; i < args.size(); ++i ) { if( args[i] == "-p" ) flags |= MkDirFlags::MakePath; else if( !args[i].compare( 0, 2, "-m" ) ) modeStr = args[i].substr( 2, 9 ); else path = args[i]; } XRootDStatus st = ConvertMode( mode, modeStr ); if( !st.IsOK() ) { log->Error( AppMsg, "Invalid mode string." ); return st; } std::string newPath; if( !BuildPath( newPath, env, path ).IsOK() ) { log->Error( AppMsg, "Invalid path." ); return XRootDStatus( stError, errInvalidArgs ); } //---------------------------------------------------------------------------- // Run the query //---------------------------------------------------------------------------- st = fs->MkDir( newPath, flags, mode ); if( !st.IsOK() ) { log->Error( AppMsg, "Unable create directory %s: %s", newPath.c_str(), st.ToStr().c_str() ); return st; } return XRootDStatus(); } //------------------------------------------------------------------------------ // Remove a directory //------------------------------------------------------------------------------ XRootDStatus DoRmDir( FileSystem *query, Env *env, const FSExecutor::CommandParams &args ) { //---------------------------------------------------------------------------- // Check up the args //---------------------------------------------------------------------------- Log *log = DefaultEnv::GetLog(); uint32_t argc = args.size(); if( argc != 2 ) { log->Error( AppMsg, "Wrong number of arguments." ); return XRootDStatus( stError, errInvalidArgs ); } std::string fullPath; if( !BuildPath( fullPath, env, args[1] ).IsOK() ) { log->Error( AppMsg, "Invalid path." ); return XRootDStatus( stError, errInvalidArgs ); } //---------------------------------------------------------------------------- // Run the query //---------------------------------------------------------------------------- XRootDStatus st = query->RmDir( fullPath ); if( !st.IsOK() ) { log->Error( AppMsg, "Unable remove directory %s: %s", fullPath.c_str(), st.ToStr().c_str() ); return st; } return XRootDStatus(); } //------------------------------------------------------------------------------ // Move a file or directory //------------------------------------------------------------------------------ XRootDStatus DoMv( FileSystem *fs, Env *env, const FSExecutor::CommandParams &args ) { //---------------------------------------------------------------------------- // Check up the args //---------------------------------------------------------------------------- Log *log = DefaultEnv::GetLog(); uint32_t argc = args.size(); if( argc != 3 ) { log->Error( AppMsg, "Wrong number of arguments." ); return XRootDStatus( stError, errInvalidArgs ); } std::string fullPath1; if( !BuildPath( fullPath1, env, args[1] ).IsOK() ) { log->Error( AppMsg, "Invalid source path." ); return XRootDStatus( stError, errInvalidArgs ); } std::string fullPath2; if( !BuildPath( fullPath2, env, args[2] ).IsOK() ) { log->Error( AppMsg, "Invalid destination path." ); return XRootDStatus( stError, errInvalidArgs ); } //---------------------------------------------------------------------------- // Run the query //---------------------------------------------------------------------------- XRootDStatus st = fs->Mv( fullPath1, fullPath2 ); if( !st.IsOK() ) { log->Error( AppMsg, "Unable move %s to %s: %s", fullPath1.c_str(), fullPath2.c_str(), st.ToStr().c_str() ); return st; } return XRootDStatus(); } //------------------------------------------------------------------------------ // Remove a file //------------------------------------------------------------------------------ XRootDStatus DoRm( FileSystem *fs, Env *env, const FSExecutor::CommandParams &args ) { //---------------------------------------------------------------------------- // Check up the args //---------------------------------------------------------------------------- Log *log = DefaultEnv::GetLog(); uint32_t argc = args.size(); if( argc < 2 ) { log->Error( AppMsg, "Wrong number of arguments." ); return XRootDStatus( stError, errInvalidArgs ); } struct print_t { void print( const std::string &msg ) { std::unique_lock lck( mtx ); std::cout << msg << '\n'; } std::mutex mtx; }; std::shared_ptr print; if( argc - 1 > 0 ) print = std::make_shared(); std::vector rms; rms.reserve( argc - 1 ); for( size_t i = 1; i < argc; ++i ) { std::string fullPath; if( !BuildPath( fullPath, env, args[i] ).IsOK() ) { log->Error( AppMsg, "Invalid path: %s", fullPath.c_str() ); return XRootDStatus( stError, errInvalidArgs ); } rms.emplace_back( Rm( fs, fullPath ) >> [log, fullPath, print]( XRootDStatus &st ) { if( !st.IsOK() ) { log->Error( AppMsg, "Unable remove %s: %s", fullPath.c_str(), st.ToStr().c_str() ); } if( print ) { print->print( "rm " + fullPath + " : " + st.ToString() ); } } ); } //---------------------------------------------------------------------------- // Run the query: // Parallel() will take the vector of Pipeline by reference and empty the // vector, so rms.size() will change after the call. //---------------------------------------------------------------------------- const size_t rs = rms.size(); XRootDStatus st = WaitFor( Parallel( rms ).AtLeast( rs ) ); if( !st.IsOK() ) return st; return XRootDStatus(); } //------------------------------------------------------------------------------ // Truncate a file //------------------------------------------------------------------------------ XRootDStatus DoTruncate( FileSystem *fs, Env *env, const FSExecutor::CommandParams &args ) { //---------------------------------------------------------------------------- // Check up the args //---------------------------------------------------------------------------- Log *log = DefaultEnv::GetLog(); uint32_t argc = args.size(); if( argc != 3 ) { log->Error( AppMsg, "Wrong number of arguments." ); return XRootDStatus( stError, errInvalidArgs ); } std::string fullPath; if( !BuildPath( fullPath, env, args[1] ).IsOK() ) { log->Error( AppMsg, "Invalid path." ); return XRootDStatus( stError, errInvalidArgs ); } char *result; uint64_t size = ::strtoll( args[2].c_str(), &result, 0 ); if( *result != 0 ) { log->Error( AppMsg, "Size parameter needs to be an integer" ); return XRootDStatus( stError, errInvalidArgs ); } //---------------------------------------------------------------------------- // Run the query //---------------------------------------------------------------------------- XRootDStatus st = fs->Truncate( fullPath, size ); if( !st.IsOK() ) { log->Error( AppMsg, "Unable truncate %s: %s", fullPath.c_str(), st.ToStr().c_str() ); return st; } return XRootDStatus(); } //------------------------------------------------------------------------------ // Change the access rights to a file //------------------------------------------------------------------------------ XRootDStatus DoChMod( FileSystem *fs, Env *env, const FSExecutor::CommandParams &args ) { //---------------------------------------------------------------------------- // Check up the args //---------------------------------------------------------------------------- Log *log = DefaultEnv::GetLog(); uint32_t argc = args.size(); if( argc != 3 ) { log->Error( AppMsg, "Wrong number of arguments." ); return XRootDStatus( stError, errInvalidArgs ); } std::string fullPath; if( !BuildPath( fullPath, env, args[1] ).IsOK() ) { log->Error( AppMsg, "Invalid path." ); return XRootDStatus( stError, errInvalidArgs ); } Access::Mode mode = Access::None; XRootDStatus st = ConvertMode( mode, args[2] ); if( !st.IsOK() ) { log->Error( AppMsg, "Invalid mode string." ); return st; } //---------------------------------------------------------------------------- // Run the query //---------------------------------------------------------------------------- st = fs->ChMod( fullPath, mode ); if( !st.IsOK() ) { log->Error( AppMsg, "Unable change mode of %s: %s", fullPath.c_str(), st.ToStr().c_str() ); return st; } return XRootDStatus(); } //------------------------------------------------------------------------------ // Locate a path //------------------------------------------------------------------------------ XRootDStatus DoLocate( FileSystem *fs, Env *env, const FSExecutor::CommandParams &args ) { //---------------------------------------------------------------------------- // Check up the args //---------------------------------------------------------------------------- Log *log = DefaultEnv::GetLog(); uint32_t argc = args.size(); if( argc > 4 ) { log->Error( AppMsg, "Wrong number of arguments." ); return XRootDStatus( stError, errInvalidArgs ); } OpenFlags::Flags flags = OpenFlags::None; std::string path; bool hasPath = false; bool doDeepLocate = false; for( uint32_t i = 1; i < argc; ++i ) { if( args[i] == "-n" ) flags |= OpenFlags::NoWait; else if( args[i] == "-r" ) flags |= OpenFlags::Refresh; else if( args[i] == "-m" || args[i] == "-h" ) flags |= OpenFlags::PrefName; else if( args[i] == "-i" ) flags |= OpenFlags::Force; else if( args[i] == "-d" ) doDeepLocate = true; else if( args[i] == "-p" ) { Env *env = DefaultEnv::GetEnv(); env->PutInt( "PreserveLocateTried", 0 ); } else if( !hasPath ) { path = args[i]; hasPath = true; } else { log->Error( AppMsg, "Invalid argument: %s.", args[i].c_str() ); return XRootDStatus( stError, errInvalidArgs ); } } std::string fullPath; if( path[0] == '*' ) fullPath = path; else { if( !BuildPath( fullPath, env, path ).IsOK() ) { log->Error( AppMsg, "Invalid path." ); return XRootDStatus( stError, errInvalidArgs ); } } //---------------------------------------------------------------------------- // Run the query //---------------------------------------------------------------------------- LocationInfo *info = 0; XRootDStatus st; if( doDeepLocate ) st = fs->DeepLocate( fullPath, flags, info ); else st = fs->Locate( fullPath, flags, info ); if( !st.IsOK() ) { log->Error( AppMsg, "Unable locate %s: %s", fullPath.c_str(), st.ToStr().c_str() ); return st; } //---------------------------------------------------------------------------- // Print the result //---------------------------------------------------------------------------- if( st.code == suPartial ) { std::cerr << "[!] Some of the requests failed. The result may be "; std::cerr << "incomplete." << std::endl; } LocationInfo::Iterator it; for( it = info->Begin(); it != info->End(); ++it ) { std::cout << it->GetAddress() << " "; switch( it->GetType() ) { case LocationInfo::ManagerOnline: std::cout << "Manager "; break; case LocationInfo::ManagerPending: std::cout << "ManagerPending "; break; case LocationInfo::ServerOnline: std::cout << "Server "; break; case LocationInfo::ServerPending: std::cout << "ServerPending "; break; default: std::cout << "Unknown "; }; switch( it->GetAccessType() ) { case LocationInfo::Read: std::cout << "Read"; break; case LocationInfo::ReadWrite: std::cout << "ReadWrite "; break; default: std::cout << "Unknown "; }; std::cout << std::endl; } delete info; return XRootDStatus(); } //------------------------------------------------------------------------------ // Process stat query //------------------------------------------------------------------------------ XRootDStatus ProcessStatQuery( StatInfo &info, const std::string &query ) { Log *log = DefaultEnv::GetLog(); //---------------------------------------------------------------------------- // Process the query //---------------------------------------------------------------------------- bool isOrQuery = false; bool status = true; if( query.find( '|' ) != std::string::npos ) { isOrQuery = true; status = false; } std::vector queryFlags; if( isOrQuery ) Utils::splitString( queryFlags, query, "|" ); else Utils::splitString( queryFlags, query, "&" ); //---------------------------------------------------------------------------- // Initialize flag translation map and check the input flags //---------------------------------------------------------------------------- std::map flagMap; flagMap["XBitSet"] = StatInfo::XBitSet; flagMap["IsDir"] = StatInfo::IsDir; flagMap["Other"] = StatInfo::Other; flagMap["Offline"] = StatInfo::Offline; flagMap["POSCPending"] = StatInfo::POSCPending; flagMap["IsReadable"] = StatInfo::IsReadable; flagMap["IsWritable"] = StatInfo::IsWritable; flagMap["BackUpExists"] = StatInfo::BackUpExists; std::vector::iterator it; for( it = queryFlags.begin(); it != queryFlags.end(); ++it ) if( flagMap.find( *it ) == flagMap.end() ) { log->Error( AppMsg, "Flag '%s' is not recognized.", it->c_str() ); return XRootDStatus( stError, errInvalidArgs ); } //---------------------------------------------------------------------------- // Process the query //---------------------------------------------------------------------------- if( isOrQuery ) { for( it = queryFlags.begin(); it != queryFlags.end(); ++it ) if( info.TestFlags( flagMap[*it] ) ) return XRootDStatus(); } else { for( it = queryFlags.begin(); it != queryFlags.end(); ++it ) if( !info.TestFlags( flagMap[*it] ) ) return XRootDStatus( stError, errResponseNegative ); } if( status ) return XRootDStatus(); return XRootDStatus( stError, errResponseNegative ); } //------------------------------------------------------------------------------ // Stat a path //------------------------------------------------------------------------------ XRootDStatus DoStat( FileSystem *fs, Env *env, const FSExecutor::CommandParams &args ) { //---------------------------------------------------------------------------- // Check up the args //---------------------------------------------------------------------------- Log *log = DefaultEnv::GetLog(); uint32_t argc = args.size(); if( argc < 2 ) { log->Error( AppMsg, "Wrong number of arguments." ); return XRootDStatus( stError, errInvalidArgs ); } std::vector paths; std::string query; for( uint32_t i = 1; i < args.size(); ++i ) { if( args[i] == "-q" ) { if( i < args.size()-1 ) { query = args[i+1]; ++i; } else { log->Error( AppMsg, "Parameter '-q' requires an argument." ); return XRootDStatus( stError, errInvalidArgs ); } } else paths.emplace_back( args[i] ); } std::vector stats; std::vector, std::string>> results; for( auto &path : paths ) { std::string fullPath; if( !BuildPath( fullPath, env, path ).IsOK() ) { log->Error( AppMsg, "Invalid path." ); return XRootDStatus( stError, errInvalidArgs ); } std::future ftr; stats.emplace_back( XrdCl::Stat( fs, fullPath ) >> ftr ); results.emplace_back( std::move( ftr ), std::move( fullPath ) ); } //---------------------------------------------------------------------------- // Run the query //---------------------------------------------------------------------------- XrdCl::Async( XrdCl::Parallel( stats ) ); //---------------------------------------------------------------------------- // Print the result //---------------------------------------------------------------------------- XrdCl::XRootDStatus st; for( auto &tpl : results ) { auto &ftr = std::get<0>( tpl ); auto &fullPath = std::get<1>( tpl ); std::cout << std::endl; try { XrdCl::StatInfo info( ftr.get() ); std::string flags; if( info.TestFlags( StatInfo::XBitSet ) ) flags += "XBitSet|"; if( info.TestFlags( StatInfo::IsDir ) ) flags += "IsDir|"; if( info.TestFlags( StatInfo::Other ) ) flags += "Other|"; if( info.TestFlags( StatInfo::Offline ) ) flags += "Offline|"; if( info.TestFlags( StatInfo::POSCPending ) ) flags += "POSCPending|"; if( info.TestFlags( StatInfo::IsReadable ) ) flags += "IsReadable|"; if( info.TestFlags( StatInfo::IsWritable ) ) flags += "IsWritable|"; if( info.TestFlags( StatInfo::BackUpExists ) ) flags += "BackUpExists|"; if( !flags.empty() ) flags.erase( flags.length()-1, 1 ); std::cout << "Path: " << fullPath << std::endl; std::cout << "Id: " << info.GetId() << std::endl; std::cout << "Size: " << info.GetSize() << std::endl; std::cout << "MTime: " << info.GetModTimeAsString() << std::endl; // if extended stat information is available we can print also // change time and access time if( info.ExtendedFormat() ) { std::cout << "CTime: " << info.GetChangeTimeAsString() << std::endl; std::cout << "ATime: " << info.GetAccessTimeAsString() << std::endl; } std::cout << "Flags: " << info.GetFlags() << " (" << flags << ")"; // check if extended stat information is available if( info.ExtendedFormat() ) { std::cout << "\nMode: " << info.GetModeAsString() << std::endl; std::cout << "Owner: " << info.GetOwner() << std::endl; std::cout << "Group: " << info.GetGroup(); } std::cout << std::endl; if( query.length() != 0 ) { XRootDStatus s = ProcessStatQuery( info, query ); if( !s.IsOK() ) st = s; std::cout << "Query: " << query << " " << std::endl; } } catch( XrdCl::PipelineException &ex ) { st = ex.GetError(); log->Error( AppMsg, "Unable stat %s: %s", fullPath.c_str(), st.ToStr().c_str() ); } } return st; } //------------------------------------------------------------------------------ // Stat a VFS //------------------------------------------------------------------------------ XRootDStatus DoStatVFS( FileSystem *fs, Env *env, const FSExecutor::CommandParams &args ) { //---------------------------------------------------------------------------- // Check up the args //---------------------------------------------------------------------------- Log *log = DefaultEnv::GetLog(); uint32_t argc = args.size(); if( argc != 2 ) { log->Error( AppMsg, "Wrong number of arguments." ); return XRootDStatus( stError, errInvalidArgs ); } std::string fullPath; if( !BuildPath( fullPath, env, args[1] ).IsOK() ) { log->Error( AppMsg, "Invalid path." ); return XRootDStatus( stError, errInvalidArgs ); } //---------------------------------------------------------------------------- // Run the query //---------------------------------------------------------------------------- StatInfoVFS *info = 0; XRootDStatus st = fs->StatVFS( fullPath, info ); if( !st.IsOK() ) { log->Error( AppMsg, "Unable stat VFS at %s: %s", fullPath.c_str(), st.ToStr().c_str() ); return st; } //---------------------------------------------------------------------------- // Print the result //---------------------------------------------------------------------------- std::cout << "Path: "; std::cout << fullPath << std::endl; std::cout << "Nodes with RW space: "; std::cout << info->GetNodesRW() << std::endl; std::cout << "Size of largest RW space (MB): "; std::cout << info->GetFreeRW() << std::endl; std::cout << "Utilization of RW space (%): "; std::cout << (uint16_t)info->GetUtilizationRW() << std::endl; std::cout << "Nodes with staging space: "; std::cout << info->GetNodesStaging() << std::endl; std::cout << "Size of largest staging space (MB): "; std::cout << info->GetFreeStaging() << std::endl; std::cout << "Utilization of staging space (%): "; std::cout << (uint16_t)info->GetUtilizationStaging() << std::endl; delete info; return XRootDStatus(); } //------------------------------------------------------------------------------ // Query the server //------------------------------------------------------------------------------ XRootDStatus DoQuery( FileSystem *fs, Env *env, const FSExecutor::CommandParams &args ) { //---------------------------------------------------------------------------- // Check up the args //---------------------------------------------------------------------------- Log *log = DefaultEnv::GetLog(); uint32_t argc = args.size(); if( !( argc >= 3 ) ) { log->Error( AppMsg, "Wrong number of arguments." ); return XRootDStatus( stError, errInvalidArgs ); } QueryCode::Code qCode; if( args[1] == "config" ) qCode = QueryCode::Config; else if( args[1] == "checksumcancel" ) qCode = QueryCode::ChecksumCancel; else if( args[1] == "checksum" ) qCode = QueryCode::Checksum; else if( args[1] == "opaque" ) qCode = QueryCode::Opaque; else if( args[1] == "opaquefile" ) qCode = QueryCode::OpaqueFile; else if( args[1] == "prepare" ) qCode = QueryCode::Prepare; else if( args[1] == "space" ) qCode = QueryCode::Space; else if( args[1] == "stats" ) qCode = QueryCode::Stats; else if( args[1] == "xattr" ) qCode = QueryCode::XAttr; else { log->Error( AppMsg, "Invalid query code." ); return XRootDStatus( stError, errInvalidArgs ); } if( !( qCode & QueryCode::Prepare ) && argc != 3 ) { log->Error( AppMsg, "Wrong number of arguments." ); return XRootDStatus( stError, errInvalidArgs ); } std::string strArg = args[2]; if( qCode & QueryCode::Prepare ) { // strArg is supposed to contain already the request ID for( size_t i = 3; i < args.size(); ++i ) { std::string path = args[i]; if( !BuildPath( path, env, path ).IsOK() ) { log->Error( AppMsg, "Invalid path." ); return XRootDStatus( stError, errInvalidArgs ); } // we use new line character as delimiter strArg += '\n'; strArg += path; } } else { std::string strArg = args[2]; if( qCode == QueryCode::ChecksumCancel || qCode == QueryCode::Checksum || qCode == QueryCode::XAttr ) { if( !BuildPath( strArg, env, args[2] ).IsOK() ) { log->Error( AppMsg, "Invalid path." ); return XRootDStatus( stError, errInvalidArgs ); } } } //---------------------------------------------------------------------------- // Run the query //---------------------------------------------------------------------------- Buffer arg( strArg.size() ); arg.FromString( strArg ); Buffer *response = 0; XRootDStatus st = fs->Query( qCode, arg, response ); if( !st.IsOK() ) { log->Error( AppMsg, "Unable run query %s: %s", args[1].c_str(), st.ToStr().c_str() ); return st; } //---------------------------------------------------------------------------- // Print the result //---------------------------------------------------------------------------- std::cout << response->ToString() << std::endl; delete response; return XRootDStatus(); } //------------------------------------------------------------------------------ // Query the server //------------------------------------------------------------------------------ XRootDStatus DoPrepare( FileSystem *fs, Env *env, const FSExecutor::CommandParams &args ) { //---------------------------------------------------------------------------- // Check up the args //---------------------------------------------------------------------------- Log *log = DefaultEnv::GetLog(); uint32_t argc = args.size(); if( argc < 2 ) { log->Error( AppMsg, "Wrong number of arguments." ); return XRootDStatus( stError, errInvalidArgs ); } PrepareFlags::Flags flags = PrepareFlags::None; std::vector files; uint8_t priority = 0; std::string reqid; for( uint32_t i = 1; i < args.size(); ++i ) { if( args[i] == "-p" ) { if( i < args.size()-1 ) { char *result; int32_t param = ::strtol( args[i+1].c_str(), &result, 0 ); if( *result != 0 || param > 3 || param < 0 ) { log->Error( AppMsg, "Size priotiry needs to be an integer between 0 " "and 3" ); return XRootDStatus( stError, errInvalidArgs ); } priority = (uint8_t)param; ++i; } else { log->Error( AppMsg, "Parameter '-p' requires an argument." ); return XRootDStatus( stError, errInvalidArgs ); } } else if( args[i] == "-c" ) flags |= PrepareFlags::Colocate; else if( args[i] == "-f" ) flags |= PrepareFlags::Fresh; else if( args[i] == "-s" ) flags |= PrepareFlags::Stage; else if( args[i] == "-w" ) flags |= PrepareFlags::WriteMode; else if( args[i] == "-e" ) flags |= PrepareFlags::Evict; else if( args[i] == "-a" ) { flags |= PrepareFlags::Cancel; if( i < args.size()-1 ) { // by convention the request ID appears as the the first token // in the list of files files.push_back( args[i+1] ); ++i; } else { log->Error( AppMsg, "Parameter '-a' requires an argument." ); return XRootDStatus( stError, errInvalidArgs ); } } else files.push_back( args[i] ); } if( files.empty() ) { log->Error( AppMsg, "Filename missing." ); return XRootDStatus( stError, errInvalidArgs ); } //---------------------------------------------------------------------------- // Run the command //---------------------------------------------------------------------------- Buffer *response = 0; XRootDStatus st = fs->Prepare( files, flags, priority, response ); if( !st.IsOK() ) { log->Error( AppMsg, "Prepare request failed: %s", st.ToStr().c_str() ); return st; } if( ( flags & PrepareFlags::Stage ) && response ) { std::cout << response->ToString() << '\n'; } delete response; return XRootDStatus(); } //------------------------------------------------------------------------------ // Copy progress handler //------------------------------------------------------------------------------ class ProgressDisplay: public XrdCl::CopyProgressHandler { public: //-------------------------------------------------------------------------- // Constructor //-------------------------------------------------------------------------- ProgressDisplay(): pBytesProcessed(0), pBytesTotal(0), pPrevious(0) {} //-------------------------------------------------------------------------- // End job //-------------------------------------------------------------------------- virtual void EndJob( uint16_t jobNum, const XrdCl::PropertyList *results ) { JobProgress( jobNum, pBytesProcessed, pBytesTotal ); std::cerr << std::endl; } //-------------------------------------------------------------------------- // Job progress //-------------------------------------------------------------------------- virtual void JobProgress( uint16_t jobNum, uint64_t bytesProcessed, uint64_t bytesTotal ) { pBytesProcessed = bytesProcessed; pBytesTotal = bytesTotal; time_t now = time(0); if( (now - pPrevious < 1) && (bytesProcessed != bytesTotal) ) return; pPrevious = now; std::cerr << "\r"; std::cerr << "Progress: "; std::cerr << XrdCl::Utils::BytesToString(bytesProcessed) << "B "; if( bytesTotal ) std::cerr << "(" << bytesProcessed*100/bytesTotal << "%)"; std::cerr << std::flush; } private: uint64_t pBytesProcessed; uint64_t pBytesTotal; time_t pPrevious; }; //------------------------------------------------------------------------------ // Cat a file //------------------------------------------------------------------------------ XRootDStatus DoCat( FileSystem *fs, Env *env, const FSExecutor::CommandParams &args ) { //---------------------------------------------------------------------------- // Check up the args //---------------------------------------------------------------------------- Log *log = DefaultEnv::GetLog(); uint32_t argc = args.size(); if( argc < 2 ) { log->Error( AppMsg, "Wrong number of arguments." ); return XRootDStatus( stError, errInvalidArgs ); } std::string server; env->GetString( "ServerURL", server ); if( server.empty() ) { log->Error( AppMsg, "Invalid address: \"%s\".", server.c_str() ); return XRootDStatus( stError, errInvalidAddr ); } std::vector remotes; std::string local; for( uint32_t i = 1; i < args.size(); ++i ) { if( args[i] == "-o" ) { if( i < args.size()-1 ) { local = args[i+1]; ++i; } else { log->Error( AppMsg, "Parameter '-o' requires an argument." ); return XRootDStatus( stError, errInvalidArgs ); } } else remotes.emplace_back( args[i] ); } if( !local.empty() && remotes.size() > 1 ) { log->Error( AppMsg, "If '-o' is used only can be used with only one remote file." ); return XRootDStatus( stError, errInvalidArgs ); } std::vector remoteUrls; remoteUrls.reserve( remotes.size() ); for( auto &remote : remotes ) { std::string remoteFile; if( !BuildPath( remoteFile, env, remote ).IsOK() ) { log->Error( AppMsg, "Invalid path." ); return XRootDStatus( stError, errInvalidArgs ); } remoteUrls.emplace_back( server ); remoteUrls.back().SetPath( remoteFile ); } //---------------------------------------------------------------------------- // Fetch the data //---------------------------------------------------------------------------- CopyProgressHandler *handler = 0; ProgressDisplay d; CopyProcess process; std::vector props( remoteUrls.size() ), results( remoteUrls.size() ); for( size_t i = 0; i < remoteUrls.size(); ++i ) { props[i].Set( "source", remoteUrls[i].GetURL() ); if( !local.empty() ) { props[i].Set( "target", std::string( "file://" ) + local ); handler = &d; } else props[i].Set( "target", "stdio://-" ); props[i].Set( "dynamicSource", true ); XRootDStatus st = process.AddJob( props[i], &results[i] ); if( !st.IsOK() ) { log->Error( AppMsg, "Job adding failed: %s.", st.ToStr().c_str() ); return st; } } XRootDStatus st = process.Prepare(); if( !st.IsOK() ) { log->Error( AppMsg, "Copy preparation failed: %s.", st.ToStr().c_str() ); return st; } st = process.Run(handler); if( !st.IsOK() ) { log->Error( AppMsg, "Cope process failed: %s.", st.ToStr().c_str() ); return st; } return XRootDStatus(); } //------------------------------------------------------------------------------ // Tail a file //------------------------------------------------------------------------------ XRootDStatus DoTail( FileSystem *fs, Env *env, const FSExecutor::CommandParams &args ) { //---------------------------------------------------------------------------- // Check up the args //---------------------------------------------------------------------------- Log *log = DefaultEnv::GetLog(); uint32_t argc = args.size(); if( argc < 2 || argc > 5 ) { log->Error( AppMsg, "Wrong number of arguments." ); return XRootDStatus( stError, errInvalidArgs ); } std::string server; env->GetString( "ServerURL", server ); if( server.empty() ) { log->Error( AppMsg, "Invalid address: \"%s\".", server.c_str() ); return XRootDStatus( stError, errInvalidAddr ); } std::string remote; bool followMode = false; uint32_t offset = 512; for( uint32_t i = 1; i < args.size(); ++i ) { if( args[i] == "-f" ) followMode = true; else if( args[i] == "-c" ) { if( i < args.size()-1 ) { char *result; offset = ::strtol( args[i+1].c_str(), &result, 0 ); if( *result != 0 ) { log->Error( AppMsg, "Offset from the end needs to be a number: %s", args[i+1].c_str() ); return XRootDStatus( stError, errInvalidArgs ); } ++i; } else { log->Error( AppMsg, "Parameter '-n' requires an argument." ); return XRootDStatus( stError, errInvalidArgs ); } } else remote = args[i]; } std::string remoteFile; if( !BuildPath( remoteFile, env, remote ).IsOK() ) { log->Error( AppMsg, "Invalid path." ); return XRootDStatus( stError, errInvalidArgs ); } URL remoteUrl( server ); remoteUrl.SetPath( remoteFile ); //---------------------------------------------------------------------------- // Fetch the data //---------------------------------------------------------------------------- File file; XRootDStatus st = file.Open( remoteUrl.GetURL(), OpenFlags::Read ); if( !st.IsOK() ) { log->Error( AppMsg, "Unable to open file %s: %s", remoteUrl.GetURL().c_str(), st.ToStr().c_str() ); return st; } StatInfo *info = 0; uint64_t size = 0; st = file.Stat( false, info ); if (st.IsOK()) size = info->GetSize(); if( size < offset ) offset = 0; else offset = size - offset; uint32_t chunkSize = 1*1024*1024; char *buffer = new char[chunkSize]; uint32_t bytesRead = 0; while(1) { st = file.Read( offset, chunkSize, buffer, bytesRead ); if( !st.IsOK() ) { log->Error( AppMsg, "Unable to read from %s: %s", remoteUrl.GetURL().c_str(), st.ToStr().c_str() ); delete [] buffer; return st; } offset += bytesRead; int ret = write( 1, buffer, bytesRead ); if( ret == -1 ) { log->Error( AppMsg, "Unable to write to stdout: %s", XrdSysE2T(errno) ); delete [] buffer; return st; } if( bytesRead < chunkSize ) { if( !followMode ) break; sleep(1); } } delete [] buffer; XRootDStatus stC = file.Close(); return XRootDStatus(); } //------------------------------------------------------------------------------ // Print statistics concerning given space //------------------------------------------------------------------------------ XRootDStatus DoSpaceInfo( FileSystem *fs, Env *env, const FSExecutor::CommandParams &args ) { using namespace XrdCl; //---------------------------------------------------------------------------- // Check up the args //---------------------------------------------------------------------------- Log *log = DefaultEnv::GetLog(); uint32_t argc = args.size(); if( argc != 2 ) { log->Error( AppMsg, "Wrong number of arguments." ); return XRootDStatus( stError, errInvalidArgs ); } FileSystemUtils::SpaceInfo *i = 0; XRootDStatus st = FileSystemUtils::GetSpaceInfo( i, fs, args[1] ); if( !st.IsOK() ) return st; if( st.code == suPartial ) { std::cerr << "[!] Some of the requests failed. The result may be "; std::cerr << "incomplete." << std::endl; } std::cout << "Path: " << args[1] << std::endl; std::cout << "Total: " << i->GetTotal() << std::endl; std::cout << "Free: " << i->GetFree() << std::endl; std::cout << "Used: " << i->GetUsed() << std::endl; std::cout << "Largest free chunk: " << i->GetLargestFreeChunk() << std::endl; delete i; return XRootDStatus(); } //------------------------------------------------------------------------------ // Carry out xattr operation //------------------------------------------------------------------------------ XRootDStatus DoXAttr( FileSystem *fs, Env *env, const FSExecutor::CommandParams &args ) { //---------------------------------------------------------------------------- // Check up the args //---------------------------------------------------------------------------- Log *log = DefaultEnv::GetLog(); uint32_t argc = args.size(); if( argc < 3 ) { log->Error( AppMsg, "Wrong number of arguments." ); return XRootDStatus( stError, errInvalidArgs ); } kXR_char code = 0; if( args[2] == "set") code = kXR_fattrSet; else if( args[2] == "get" ) code = kXR_fattrGet; else if( args[2] == "del" ) code = kXR_fattrDel; else if( args[2] == "list" ) code = kXR_fattrList; else { log->Error( AppMsg, "Invalid xattr code." ); return XRootDStatus( stError, errInvalidArgs ); } std::string path; if( !BuildPath( path, env, args[1] ).IsOK() ) { log->Error( AppMsg, "Invalid path." ); return XRootDStatus( stError, errInvalidArgs ); } //---------------------------------------------------------------------------- // Issue the xattr operation //---------------------------------------------------------------------------- XRootDStatus status; switch( code ) { case kXR_fattrSet: { if( argc != 4 ) { log->Error( AppMsg, "Wrong number of arguments." ); return XRootDStatus( stError, errInvalidArgs ); } std::string key_value = args[3]; size_t pos = key_value.find( '=' ); std::string key = key_value.substr( 0, pos ); std::string value = key_value.substr( pos + 1 ); std::vector attrs; attrs.push_back( std::make_tuple( key, value ) ); std::vector result; XRootDStatus status = fs->SetXAttr( path, attrs, result ); XAttrStatus xst = status.IsOK() ? result.front() : XAttrStatus( key, status ); if( !xst.status.IsOK() ) status = xst.status; if( !status.IsOK() ) log->Error( AppMsg, "Unable to xattr set %s %s: %s", key.c_str(), value.c_str(), status.ToStr().c_str() ); return status; } case kXR_fattrGet: { if( argc != 4 ) { log->Error( AppMsg, "Wrong number of arguments." ); return XRootDStatus( stError, errInvalidArgs ); } std::string key = args[3]; std::vector attrs; attrs.push_back( key ); std::vector result; XRootDStatus status = fs->GetXAttr( path, attrs, result ); XAttr xattr = status.IsOK() ? result.front() : XAttr( key, status ); if( !xattr.status.IsOK() ) status = xattr.status; if( !status.IsOK() ) log->Error( AppMsg, "Unable to xattr get %s : %s", key.c_str(), status.ToStr().c_str() ); else { std::cout << "# file: " << path << '\n'; std::cout << xattr.name << "=\"" << xattr.value << "\"\n"; } return status; } case kXR_fattrDel: { if( argc != 4 ) { log->Error( AppMsg, "Wrong number of arguments." ); return XRootDStatus( stError, errInvalidArgs ); } std::string key = args[3]; std::vector attrs; attrs.push_back( key ); std::vector result ; XRootDStatus status = fs->DelXAttr( path, attrs, result ); XAttrStatus xst = status.IsOK() ? result.front() : XAttrStatus( key, status ); if( !xst.status.IsOK() ) status = xst.status; if( !status.IsOK() ) log->Error( AppMsg, "Unable to xattr del %s : %s", key.c_str(), status.ToStr().c_str() ); return status; } case kXR_fattrList: { if( argc != 3 ) { log->Error( AppMsg, "Wrong number of arguments." ); return XRootDStatus( stError, errInvalidArgs ); } std::vector result; XRootDStatus status = fs->ListXAttr( path, result ); if( !status.IsOK() ) log->Error( AppMsg, "Unable to xattr list : %s", status.ToStr().c_str() ); else { std::cout << "# file: " << path << '\n'; auto itr = result.begin(); for( ; itr != result.end(); ++itr ) std::cout << itr->name << "=\"" << itr->value << "\"\n"; } return status; } default: return XRootDStatus( stError, errInvalidAddr ); } } //------------------------------------------------------------------------------ // Print help //------------------------------------------------------------------------------ XRootDStatus PrintHelp( FileSystem *, Env *, const FSExecutor::CommandParams & ) { printf( "Usage:\n" ); printf( " xrdfs [--no-cwd] host[:port] - interactive mode\n" ); printf( " xrdfs host[:port] command args - batch mode\n\n" ); printf( "Available options:\n\n" ); printf( " --no-cwd no CWD is being preset\n\n" ); printf( "Available commands:\n\n" ); printf( " exit\n" ); printf( " Exits from the program.\n\n" ); printf( " help\n" ); printf( " This help screen.\n\n" ); printf( " cache {evict | fevict} \n" ); printf( " Evict a file from a cache if not in use; while fevict\n" ); printf( " focibly evicts the file causing any current uses of the\n" ); printf( " file to get read failures on a subsequent read\n\n" ); printf( " cd \n" ); printf( " Change the current working directory\n\n" ); printf( " chmod \n" ); printf( " Modify permissions. Permission string example:\n" ); printf( " rwxr-x--x\n\n" ); printf( " ls [-l] [-u] [-R] [-D] [-Z] [-C] [dirname]\n" ); printf( " Get directory listing.\n" ); printf( " -l stat every entry and pring long listing\n" ); printf( " -u print paths as URLs\n" ); printf( " -R list subdirectories recursively\n" ); printf( " -D show duplicate entries" ); printf( " -Z if a ZIP archive list its content\n" ); printf( " -C checksum every entry\n\n" ); printf( " locate [-n] [-r] [-d] [-m] [-i] [-p] \n" ); printf( " Get the locations of the path.\n" ); printf( " -r refresh, don't use cached locations\n" ); printf( " -n make the server return the response immediately even\n" ); printf( " though it may be incomplete\n" ); printf( " -d do a recursive (deep) locate\n" ); printf( " -m|-h prefer host names to IP addresses\n" ); printf( " -i ignore network dependencies\n" ); printf( " -p be passive: ignore tried/triedrc cgi opaque info\n\n" ); printf( " mkdir [-p] [-m] \n" ); printf( " Creates a directory/tree of directories.\n\n" ); printf( " mv \n" ); printf( " Move path1 to path2 locally on the same server.\n\n" ); printf( " stat [-q query] \n" ); printf( " Get info about the file or directory.\n" ); printf( " -q query optional flag query parameter that makes\n" ); printf( " xrdfs return error code to the shell if the\n" ); printf( " requested flag combination is not present;\n" ); printf( " flags may be combined together using '|' or '&'\n" ); printf( " Available flags:\n" ); printf( " XBitSet, IsDir, Other, Offline, POSCPending,\n" ); printf( " IsReadable, IsWriteable\n\n" ); printf( " statvfs \n" ); printf( " Get info about a virtual file system.\n\n" ); printf( " query \n" ); printf( " Obtain server information. Query codes:\n\n" ); printf( " config Server configuration; is\n" ); printf( " one of the following:\n" ); printf( " bind_max - the maximum number of parallel streams\n" ); printf( " chksum - the supported checksum\n" ); printf( " pio_max - maximum number of parallel I/O requests\n" ); printf( " readv_ior_max - maximum size of a readv element\n" ); printf( " readv_iov_max - maximum number of readv entries\n" ); printf( " tpc - support for third party copies\n" ); printf( " wan_port - the port to use for wan copies\n" ); printf( " wan_window - the wan_port window size\n" ); printf( " window - the tcp window size\n" ); printf( " cms - the status of the cmsd\n" ); printf( " role - the role in a cluster\n" ); printf( " sitename - the site name\n" ); printf( " version - the version of the server\n" ); printf( " checksumcancel File checksum cancellation\n" ); printf( " checksum File checksum\n" ); printf( " opaque Implementation dependent\n" ); printf( " opaquefile Implementation dependent\n" ); printf( " space Logical space stats\n" ); printf( " stats Server stats; is a list\n" ); printf( " of letters indicating information\n"); printf( " to be returned:\n" ); printf( " a - all statistics\n" ); printf( " p - protocol statistics\n" ); printf( " b - buffer usage statistics\n" ); printf( " s - scheduling statistics\n" ); printf( " d - device polling statistics\n" ); printf( " u - usage statistics\n" ); printf( " i - server identification\n" ); printf( " z - synchronized statistics\n" ); printf( " l - connection statistics\n" ); printf( " xattr Extended attributes\n" ); printf( " prepare [filenames] Prepare request status\n\n" ); printf( " rm \n" ); printf( " Remove a file.\n\n" ); printf( " rmdir \n" ); printf( " Remove a directory.\n\n" ); printf( " truncate \n" ); printf( " Truncate a file.\n\n" ); printf( " prepare [-c] [-f] [-s] [-w] [-e] [-p priority] [-a requestid] filenames\n" ); printf( " Prepare one or more files for access.\n" ); printf( " -c co-locate staged files if possible\n" ); printf( " -f refresh file access time even if the location is known\n" ); printf( " -s stage the files to disk if they are not online\n" ); printf( " -w the files will be accessed for modification\n" ); printf( " -p priority of the request, 0 (lowest) - 3 (highest)\n" ); printf( " -a abort stage request\n" ); printf( " -e evict the file from disk cache\n\n" ); printf( " cat [-o local file] files\n" ); printf( " Print contents of one or more files to stdout.\n" ); printf( " -o print to the specified local file\n\n" ); printf( " tail [-c bytes] [-f] file\n" ); printf( " Output last part of files to stdout.\n" ); printf( " -c num_bytes out last num_bytes\n" ); printf( " -f output appended data as file grows\n\n" ); printf( " spaceinfo path\n" ); printf( " Get space statistics for given path.\n\n" ); printf( " xattr \n" ); printf( " Operation on extended attributes. Codes:\n\n" ); printf( " set Set extended attribute; is\n" ); printf( " string of form name=value\n" ); printf( " get Get extended attribute\n" ); printf( " del Delete extended attribute\n" ); printf( " list List extended attributes\n\n" ); return XRootDStatus(); } //------------------------------------------------------------------------------ // Create the executor object //------------------------------------------------------------------------------ FSExecutor *CreateExecutor( const URL &url ) { Env *env = new Env(); env->PutString( "CWD", "/" ); FSExecutor *executor = new FSExecutor( url, env ); executor->AddCommand( "cache", DoCache ); executor->AddCommand( "cd", DoCD ); executor->AddCommand( "chmod", DoChMod ); executor->AddCommand( "ls", DoLS ); executor->AddCommand( "help", PrintHelp ); executor->AddCommand( "stat", DoStat ); executor->AddCommand( "statvfs", DoStatVFS ); executor->AddCommand( "locate", DoLocate ); executor->AddCommand( "mv", DoMv ); executor->AddCommand( "mkdir", DoMkDir ); executor->AddCommand( "rm", DoRm ); executor->AddCommand( "rmdir", DoRmDir ); executor->AddCommand( "query", DoQuery ); executor->AddCommand( "truncate", DoTruncate ); executor->AddCommand( "prepare", DoPrepare ); executor->AddCommand( "cat", DoCat ); executor->AddCommand( "tail", DoTail ); executor->AddCommand( "spaceinfo", DoSpaceInfo ); executor->AddCommand( "xattr", DoXAttr ); return executor; } //------------------------------------------------------------------------------ // Execute command //------------------------------------------------------------------------------ int ExecuteCommand( FSExecutor *ex, int argc, char **argv ) { // std::vector args (argv, argv + argc); std::vector args; args.reserve(argc); for (int i = 0; i < argc; ++i) { args.push_back(argv[i]); } XRootDStatus st = ex->Execute( args ); if( !st.IsOK() ) std::cerr << st.ToStr() << std::endl; return st.GetShellCode(); } //------------------------------------------------------------------------------ // Define some functions required to function when build without readline //------------------------------------------------------------------------------ #ifndef HAVE_READLINE char *readline(const char *prompt) { std::cout << prompt << std::flush; std::string input; std::getline( std::cin, input ); if( !std::cin.good() ) return 0; char *linebuf = (char *)malloc( input.size()+1 ); strncpy( linebuf, input.c_str(), input.size()+1 ); return linebuf; } void add_history( const char * ) { } void rl_bind_key( char, uint16_t ) { } uint16_t rl_insert = 0; int read_history( const char * ) { return 0; } int write_history( const char * ) { return 0; } #endif //------------------------------------------------------------------------------ // Build the command prompt //------------------------------------------------------------------------------ std::string BuildPrompt( Env *env, const URL &url ) { std::ostringstream prompt; std::string cwd = "/"; env->GetString( "CWD", cwd ); prompt << "[" << url.GetHostId() << "] " << cwd << " > "; return prompt.str(); } //------------------------------------------------------------------------ //! parse command line //! //! @ result : command parameters //! @ input : string containing the command line //! @ return : true if the command has been completed, false otherwise //------------------------------------------------------------------------ bool getArguments (std::vector & result, const std::string &input) { // the delimiter (space in the case of command line) static const char delimiter = ' '; // two types of quotes: single and double quotes const char singleQuote = '\'', doubleQuote = '\"'; // if the current character of the command has been // quoted 'currentQuote' holds the type of quote, // otherwise it holds the null character char currentQuote = '\0'; std::string tmp; for (std::string::const_iterator it = input.begin (); it != input.end (); ++it) { // if we encountered a quote character ... if (*it == singleQuote || *it == doubleQuote) { // if we are not within quoted text ... if (!currentQuote) { currentQuote = *it; // set the type of quote continue; // and continue, the quote character itself is not a part of the parameter } // otherwise if it is the closing quote character ... else if (currentQuote == *it) { currentQuote = '\0'; // reset the current quote type continue; // and continue, the quote character itself is not a part of the parameter } } // if we are within quoted text or the character is not a delimiter ... if (currentQuote || *it != delimiter) { // concatenate it tmp += *it; } else { // otherwise add a parameter and erase the tmp string if (!tmp.empty ()) { result.push_back(tmp); tmp.erase (); } } } // if the there are some remainders of the command add them if (!tmp.empty()) { result.push_back(tmp); } // return true if the quotation has been closed return currentQuote == '\0'; } //------------------------------------------------------------------------------ // Execute interactive //------------------------------------------------------------------------------ int ExecuteInteractive( const URL &url, bool noCwd = false ) { //---------------------------------------------------------------------------- // Set up the environment //---------------------------------------------------------------------------- std::string historyFile = getenv( "HOME" ); historyFile += "/.xrdquery.history"; rl_bind_key( '\t', rl_insert ); read_history( historyFile.c_str() ); FSExecutor *ex = CreateExecutor( url ); if( noCwd ) ex->GetEnv()->PutInt( "NoCWD", 1 ); //---------------------------------------------------------------------------- // Execute the commands //---------------------------------------------------------------------------- std::string cmdline; while(1) { char *linebuf = 0; // print new prompt only if the previous line was complete // (a line is considered not to be complete if a quote has // been opened but it has not been closed) linebuf = readline( cmdline.empty() ? BuildPrompt( ex->GetEnv(), url ).c_str() : "> " ); if( !linebuf || !strncmp( linebuf, "exit", 4 ) || !strncmp( linebuf, "quit", 4 ) ) { std::cout << "Goodbye." << std::endl << std::endl; break; } if( !*linebuf) { free( linebuf ); continue; } std::vector args; cmdline += linebuf; free( linebuf ); if (getArguments( args, cmdline )) { XRootDStatus st = ex->Execute( args ); add_history( cmdline.c_str() ); cmdline.erase(); if( !st.IsOK() ) std::cerr << st.ToStr() << std::endl; } } //---------------------------------------------------------------------------- // Cleanup //---------------------------------------------------------------------------- delete ex; write_history( historyFile.c_str() ); return 0; } //------------------------------------------------------------------------------ // Execute command //------------------------------------------------------------------------------ int ExecuteCommand( const URL &url, int argc, char **argv ) { //---------------------------------------------------------------------------- // Build the command to be executed //---------------------------------------------------------------------------- std::string commandline; for( int i = 0; i < argc; ++i ) { commandline += argv[i]; commandline += " "; } FSExecutor *ex = CreateExecutor( url ); ex->GetEnv()->PutInt( "NoCWD", 1 ); int st = ExecuteCommand( ex, argc, argv ); delete ex; return st; } //------------------------------------------------------------------------------ // Start the show //------------------------------------------------------------------------------ int main( int argc, char **argv ) { //---------------------------------------------------------------------------- // Check the commandline parameters //---------------------------------------------------------------------------- XrdCl::FSExecutor::CommandParams params; if( argc == 1 ) { PrintHelp( 0, 0, params ); return 1; } if( !strcmp( argv[1], "--help" ) || !strcmp( argv[1], "-h" ) ) { PrintHelp( 0, 0, params ); return 0; } bool noCwd = false; int urlIndex = 1; if( !strcmp( argv[1], "--no-cwd") ) { ++urlIndex; noCwd = true; } URL url( argv[urlIndex] ); if( !url.IsValid() ) { PrintHelp( 0, 0, params ); return 1; } if( argc == urlIndex + 1 ) return ExecuteInteractive( url, noCwd ); int shift = urlIndex + 1; return ExecuteCommand( url, argc-shift, argv+shift ); } xrootd-5.6.9/src/XrdCl/XrdClFSExecutor.cc000066400000000000000000000070161457266313600201710ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2012 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . //------------------------------------------------------------------------------ #include "XrdCl/XrdClFSExecutor.hh" #include "XrdCl/XrdClLog.hh" #include "XrdCl/XrdClDefaultEnv.hh" #include "XrdCl/XrdClConstants.hh" #include namespace XrdCl { //---------------------------------------------------------------------------- // Constructor //---------------------------------------------------------------------------- FSExecutor::FSExecutor( const URL &url, Env *env ): pFS( 0 ) { pFS = new FileSystem( url ); if( env ) pEnv = env; else pEnv = new Env(); pEnv->PutString( "ServerURL", url.GetURL() ); } //---------------------------------------------------------------------------- // Destructor //---------------------------------------------------------------------------- FSExecutor::~FSExecutor() { delete pFS; delete pEnv; } //--------------------------------------------------------------------------- // Add a command to the set of known commands //--------------------------------------------------------------------------- bool FSExecutor::AddCommand( const std::string &name, Command command ) { Log *log = DefaultEnv::GetLog(); CommandMap::iterator it = pCommands.find( name ); if( it != pCommands.end() ) { log->Error( AppMsg, "Unable to register command %s. Already exists.", name.c_str() ); return false; } pCommands.insert( std::make_pair( name, command ) ); return true; } XRootDStatus FSExecutor::Execute( const CommandParams & args) { std::stringstream cmdline; std::ostream_iterator oit(cmdline, " "); std::copy(args.begin(), args.end(), oit); Log *log = DefaultEnv::GetLog(); log->Debug( AppMsg, "Executing: %s", cmdline.str().c_str() ); if( args.empty() ) { log->Dump( AppMsg, "Empty commandline." ); return 1; } CommandParams::const_iterator parIt; int i = 0; for( parIt = args.begin(); parIt != args.end(); ++parIt, ++i ) log->Dump( AppMsg, " Param #%02d: '%s'", i, parIt->c_str() ); //-------------------------------------------------------------------------- // Extract the command name //-------------------------------------------------------------------------- std::string commandName = args.front(); CommandMap::iterator it = pCommands.find( commandName ); if( it == pCommands.end() ) { log->Error( AppMsg, "Unknown command: %s", commandName.c_str() ); return XRootDStatus( stError, errUnknownCommand ); } return it->second( pFS, pEnv, args ); } } xrootd-5.6.9/src/XrdCl/XrdClFSExecutor.hh000066400000000000000000000077241457266313600202110ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2012 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . //------------------------------------------------------------------------------ #ifndef __XRD_CL_FS_EXECUTOR_HH__ #define __XRD_CL_FS_EXECUTOR_HH__ #include "XrdCl/XrdClFileSystem.hh" #include "XrdCl/XrdClEnv.hh" #include "XrdCl/XrdClUtils.hh" #include #include #include namespace XrdCl { //---------------------------------------------------------------------------- //! Execute queries given as a commandline //---------------------------------------------------------------------------- class FSExecutor { public: //------------------------------------------------------------------------ //! Definition of command argument list //------------------------------------------------------------------------ typedef std::vector CommandParams; //------------------------------------------------------------------------ //! Definition of a command //------------------------------------------------------------------------ typedef XRootDStatus (*Command)( FileSystem *fs, Env *env, const CommandParams &args ); //------------------------------------------------------------------------ //! Constructor //! //! @param url the server that the executor should contact //! @param env execution environment, the executor takes ownership over it //------------------------------------------------------------------------ FSExecutor( const URL &url, Env *env = 0 ); //------------------------------------------------------------------------ //! Destructor //------------------------------------------------------------------------ ~FSExecutor(); //------------------------------------------------------------------------ //! Add a command to the set of known commands //! //! @param name name of the command //! @param command function pointer //! @return status //------------------------------------------------------------------------ bool AddCommand( const std::string &name, Command command ); //------------------------------------------------------------------------ //! Execute the given commandline //! //! @param args : arguments for the commandline to be executed, //! first of which is the command name //! @return status of the execution //------------------------------------------------------------------------ XRootDStatus Execute( const CommandParams & args); //------------------------------------------------------------------------ //! Get the environment //------------------------------------------------------------------------ Env *GetEnv() { return pEnv; } private: typedef std::map CommandMap; FileSystem *pFS; Env *pEnv; CommandMap pCommands; }; } #endif // __XRD_CL_FS_EXECUTOR_HH__ xrootd-5.6.9/src/XrdCl/XrdClFile.cc000066400000000000000000001050561457266313600170240ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2014 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #include "XrdCl/XrdClLog.hh" #include "XrdCl/XrdClUtils.hh" #include "XrdCl/XrdClConstants.hh" #include "XrdCl/XrdClFile.hh" #include "XrdCl/XrdClFileStateHandler.hh" #include "XrdCl/XrdClMessageUtils.hh" #include "XrdCl/XrdClDefaultEnv.hh" #include "XrdCl/XrdClPlugInInterface.hh" #include "XrdCl/XrdClPlugInManager.hh" #include "XrdCl/XrdClDefaultEnv.hh" namespace XrdCl { //---------------------------------------------------------------------------- // The implementation //---------------------------------------------------------------------------- struct FileImpl { FileImpl( FilePlugIn *plugin ) : pStateHandler( std::make_shared( plugin ) ) { } FileImpl( bool useVirtRedirector, FilePlugIn *plugin ) : pStateHandler( std::make_shared( useVirtRedirector, plugin ) ) { } std::shared_ptr pStateHandler; }; //---------------------------------------------------------------------------- // Constructor //---------------------------------------------------------------------------- File::File( bool enablePlugIns ): pPlugIn(0), pEnablePlugIns( enablePlugIns ) { pImpl = new FileImpl( pPlugIn ); } //---------------------------------------------------------------------------- // Constructor //---------------------------------------------------------------------------- File::File( VirtRedirect virtRedirect, bool enablePlugIns ): pPlugIn(0), pEnablePlugIns( enablePlugIns ) { pImpl = new FileImpl( virtRedirect == EnableVirtRedirect, pPlugIn ); } //---------------------------------------------------------------------------- // Destructor //---------------------------------------------------------------------------- File::~File() { //-------------------------------------------------------------------------- // This, in principle, should never ever happen. Except for the case // when we're interfaced with ROOT that may call this desctructor from // its garbage collector, from its __cxa_finalize, ie. after the XrdCl lib // has been finalized by the linker. So, if we don't have the log object // at this point we just give up the hope. // Also, make sure the PostMaster threads are running - if not the Close // will hang forever (this could happen when Python interpreter exits). //-------------------------------------------------------------------------- if ( DefaultEnv::GetLog() && DefaultEnv::GetPostMaster()->IsRunning() && IsOpen() ) XRootDStatus status = Close( nullptr, 0 ); delete pImpl; delete pPlugIn; } //---------------------------------------------------------------------------- // Open the file pointed to by the given URL - async //---------------------------------------------------------------------------- XRootDStatus File::Open( const std::string &url, OpenFlags::Flags flags, Access::Mode mode, ResponseHandler *handler, uint16_t timeout ) { //-------------------------------------------------------------------------- // Check if we need to install and run a plug-in for this URL //-------------------------------------------------------------------------- if( pEnablePlugIns && !pPlugIn ) { Log *log = DefaultEnv::GetLog(); PlugInFactory *fact = DefaultEnv::GetPlugInManager()->GetFactory( url ); if( fact ) { pPlugIn = fact->CreateFile( url ); if( !pPlugIn ) { log->Error( FileMsg, "Plug-in factory failed to produce a plug-in " "for %s, continuing without one", url.c_str() ); } } } //-------------------------------------------------------------------------- // Open the file //-------------------------------------------------------------------------- if( pPlugIn ) return pPlugIn->Open( url, flags, mode, handler, timeout ); return FileStateHandler::Open( pImpl->pStateHandler, url, flags, mode, handler, timeout ); } //---------------------------------------------------------------------------- // Open the file pointed to by the given URL - sync //---------------------------------------------------------------------------- XRootDStatus File::Open( const std::string &url, OpenFlags::Flags flags, Access::Mode mode, uint16_t timeout ) { SyncResponseHandler handler; XRootDStatus st = Open( url, flags, mode, &handler, timeout ); if( !st.IsOK() ) return st; return MessageUtils::WaitForStatus( &handler ); } //---------------------------------------------------------------------------- // Close the file - async //---------------------------------------------------------------------------- XRootDStatus File::Close( ResponseHandler *handler, uint16_t timeout ) { if( pPlugIn ) return pPlugIn->Close( handler, timeout ); return FileStateHandler::Close( pImpl->pStateHandler, handler, timeout ); } //---------------------------------------------------------------------------- // Close the file //---------------------------------------------------------------------------- XRootDStatus File::Close( uint16_t timeout ) { SyncResponseHandler handler; XRootDStatus st = Close( &handler, timeout ); if( !st.IsOK() || st.code == suAlreadyDone ) return st; return MessageUtils::WaitForStatus( &handler ); } //---------------------------------------------------------------------------- // Obtain status information for this file - async //---------------------------------------------------------------------------- XRootDStatus File::Stat( bool force, ResponseHandler *handler, uint16_t timeout ) { if( pPlugIn ) return pPlugIn->Stat( force, handler, timeout ); return FileStateHandler::Stat( pImpl->pStateHandler, force, handler, timeout ); } //---------------------------------------------------------------------------- // Obtain status information for this file - sync //---------------------------------------------------------------------------- XRootDStatus File::Stat( bool force, StatInfo *&response, uint16_t timeout ) { SyncResponseHandler handler; XRootDStatus st = Stat( force, &handler, timeout ); if( !st.IsOK() ) return st; return MessageUtils::WaitForResponse( &handler, response ); } //---------------------------------------------------------------------------- // Read a data chunk at a given offset - sync //---------------------------------------------------------------------------- XRootDStatus File::Read( uint64_t offset, uint32_t size, void *buffer, ResponseHandler *handler, uint16_t timeout ) { if( pPlugIn ) return pPlugIn->Read( offset, size, buffer, handler, timeout ); return FileStateHandler::Read( pImpl->pStateHandler, offset, size, buffer, handler, timeout ); } //---------------------------------------------------------------------------- // Read a data chunk at a given offset - sync //---------------------------------------------------------------------------- XRootDStatus File::Read( uint64_t offset, uint32_t size, void *buffer, uint32_t &bytesRead, uint16_t timeout ) { SyncResponseHandler handler; XRootDStatus st = Read( offset, size, buffer, &handler, timeout ); if( !st.IsOK() ) return st; ChunkInfo *chunkInfo = 0; XRootDStatus status = MessageUtils::WaitForResponse( &handler, chunkInfo ); if( status.IsOK() ) { bytesRead = chunkInfo->length; delete chunkInfo; } return status; } //------------------------------------------------------------------------ // Read number of pages at a given offset - async //------------------------------------------------------------------------ XRootDStatus File::PgRead( uint64_t offset, uint32_t size, void *buffer, ResponseHandler *handler, uint16_t timeout ) { if( pPlugIn ) return pPlugIn->PgRead( offset, size, buffer, handler, timeout ); return FileStateHandler::PgRead( pImpl->pStateHandler, offset, size, buffer, handler, timeout ); } //------------------------------------------------------------------------ // Read number of pages at a given offset - async //------------------------------------------------------------------------ XRootDStatus File::PgRead( uint64_t offset, uint32_t size, void *buffer, std::vector &cksums, uint32_t &bytesRead, uint16_t timeout ) { SyncResponseHandler handler; XRootDStatus st = PgRead( offset, size, buffer, &handler, timeout ); if( !st.IsOK() ) return st; PageInfo *pageInfo = 0; XRootDStatus status = MessageUtils::WaitForResponse( &handler, pageInfo ); if( status.IsOK() ) { bytesRead = pageInfo->GetLength(); cksums = pageInfo->GetCksums(); delete pageInfo; } return status; } //---------------------------------------------------------------------------- // Write a data chunk at a given offset - async //---------------------------------------------------------------------------- XRootDStatus File::Write( uint64_t offset, uint32_t size, const void *buffer, ResponseHandler *handler, uint16_t timeout ) { if( pPlugIn ) return pPlugIn->Write( offset, size, buffer, handler, timeout ); return FileStateHandler::Write( pImpl->pStateHandler, offset, size, buffer, handler, timeout ); } //---------------------------------------------------------------------------- // Write a data chunk at a given offset - sync //---------------------------------------------------------------------------- XRootDStatus File::Write( uint64_t offset, uint32_t size, const void *buffer, uint16_t timeout ) { SyncResponseHandler handler; XRootDStatus st = Write( offset, size, buffer, &handler, timeout ); if( !st.IsOK() ) return st; XRootDStatus status = MessageUtils::WaitForStatus( &handler ); return status; } XRootDStatus File::Write( uint64_t offset, Buffer &&buffer, ResponseHandler *handler, uint16_t timeout ) { if( pPlugIn ) return pPlugIn->Write( offset, std::move( buffer ), handler, timeout ); return FileStateHandler::Write( pImpl->pStateHandler, offset, std::move( buffer ), handler, timeout ); } //---------------------------------------------------------------------------- // Write a data chunk at a given offset - async //---------------------------------------------------------------------------- XRootDStatus File::Write( uint64_t offset, Buffer &&buffer, uint16_t timeout ) { SyncResponseHandler handler; XRootDStatus st = Write( offset, std::move( buffer ), &handler, timeout ); if( !st.IsOK() ) return st; XRootDStatus status = MessageUtils::WaitForStatus( &handler ); return status; } //------------------------------------------------------------------------ // Write a data from a given file descriptor at a given offset - async //------------------------------------------------------------------------ XRootDStatus File::Write( uint64_t offset, uint32_t size, Optional fdoff, int fd, ResponseHandler *handler, uint16_t timeout ) { if( pPlugIn ) return pPlugIn->Write( offset, size, fdoff, fd, handler, timeout ); return FileStateHandler::Write( pImpl->pStateHandler, offset, size, fdoff, fd, handler, timeout ); } //------------------------------------------------------------------------ // Write a data from a given file descriptor at a given offset - sync //------------------------------------------------------------------------ XRootDStatus File::Write( uint64_t offset, uint32_t size, Optional fdoff, int fd, uint16_t timeout ) { SyncResponseHandler handler; XRootDStatus st = Write( offset, size, fdoff, fd, &handler, timeout ); if( !st.IsOK() ) return st; XRootDStatus status = MessageUtils::WaitForStatus( &handler ); return status; } //------------------------------------------------------------------------ // Write number of pages at a given offset - async //------------------------------------------------------------------------ XRootDStatus File::PgWrite( uint64_t offset, uint32_t size, const void *buffer, std::vector &cksums, ResponseHandler *handler, uint16_t timeout ) { if( pPlugIn ) return pPlugIn->PgWrite( offset, size, buffer, cksums, handler, timeout ); return FileStateHandler::PgWrite( pImpl->pStateHandler, offset, size, buffer, cksums, handler, timeout ); } //------------------------------------------------------------------------ // Write number of pages at a given offset - sync //------------------------------------------------------------------------ XRootDStatus File::PgWrite( uint64_t offset, uint32_t size, const void *buffer, std::vector &cksums, uint16_t timeout ) { SyncResponseHandler handler; XRootDStatus st = PgWrite( offset, size, buffer, cksums, &handler, timeout ); if( !st.IsOK() ) return st; XRootDStatus status = MessageUtils::WaitForStatus( &handler ); return status; } //---------------------------------------------------------------------------- // Commit all pending disk writes - async //---------------------------------------------------------------------------- XRootDStatus File::Sync( ResponseHandler *handler, uint16_t timeout ) { if( pPlugIn ) return pPlugIn->Sync( handler, timeout ); return FileStateHandler::Sync( pImpl->pStateHandler, handler, timeout ); } //---------------------------------------------------------------------------- // Commit all pending disk writes - sync //---------------------------------------------------------------------------- XRootDStatus File::Sync( uint16_t timeout ) { SyncResponseHandler handler; XRootDStatus st = Sync( &handler, timeout ); if( !st.IsOK() ) return st; XRootDStatus status = MessageUtils::WaitForStatus( &handler ); return status; } //---------------------------------------------------------------------------- // Truncate the file to a particular size - async //---------------------------------------------------------------------------- XRootDStatus File::Truncate( uint64_t size, ResponseHandler *handler, uint16_t timeout ) { if( pPlugIn ) return pPlugIn->Truncate( size, handler, timeout ); return FileStateHandler::Truncate( pImpl->pStateHandler, size, handler, timeout ); } //---------------------------------------------------------------------------- // Truncate the file to a particular size - sync //---------------------------------------------------------------------------- XRootDStatus File::Truncate( uint64_t size, uint16_t timeout ) { SyncResponseHandler handler; XRootDStatus st = Truncate( size, &handler, timeout ); if( !st.IsOK() ) return st; XRootDStatus status = MessageUtils::WaitForStatus( &handler ); return status; } //---------------------------------------------------------------------------- // Read scattered data chunks in one operation - async //---------------------------------------------------------------------------- XRootDStatus File::VectorRead( const ChunkList &chunks, void *buffer, ResponseHandler *handler, uint16_t timeout ) { if( pPlugIn ) return pPlugIn->VectorRead( chunks, buffer, handler, timeout ); return FileStateHandler::VectorRead( pImpl->pStateHandler, chunks, buffer, handler, timeout ); } //---------------------------------------------------------------------------- // Read scattered data chunks in one operation - sync //---------------------------------------------------------------------------- XRootDStatus File::VectorRead( const ChunkList &chunks, void *buffer, VectorReadInfo *&vReadInfo, uint16_t timeout ) { SyncResponseHandler handler; XRootDStatus st = VectorRead( chunks, buffer, &handler, timeout ); if( !st.IsOK() ) return st; return MessageUtils::WaitForResponse( &handler, vReadInfo ); } //------------------------------------------------------------------------ // Write scattered data chunks in one operation - async //------------------------------------------------------------------------ XRootDStatus File::VectorWrite( const ChunkList &chunks, ResponseHandler *handler, uint16_t timeout ) { if( pPlugIn ) return pPlugIn->VectorWrite( chunks, handler, timeout ); return FileStateHandler::VectorWrite( pImpl->pStateHandler, chunks, handler, timeout ); } //------------------------------------------------------------------------ // Read scattered data chunks in one operation - sync //------------------------------------------------------------------------ XRootDStatus File::VectorWrite( const ChunkList &chunks, uint16_t timeout ) { SyncResponseHandler handler; XRootDStatus st = VectorWrite( chunks, &handler, timeout ); if( !st.IsOK() ) return st; return MessageUtils::WaitForStatus( &handler ); } //------------------------------------------------------------------------ // Write scattered buffers in one operation - async //------------------------------------------------------------------------ XRootDStatus File::WriteV( uint64_t offset, const struct iovec *iov, int iovcnt, ResponseHandler *handler, uint16_t timeout ) { if( pPlugIn ) return pPlugIn->WriteV( offset, iov, iovcnt, handler, timeout ); return FileStateHandler::WriteV( pImpl->pStateHandler, offset, iov, iovcnt, handler, timeout ); } //------------------------------------------------------------------------ // Write scattered buffers in one operation - sync //------------------------------------------------------------------------ XRootDStatus File::WriteV( uint64_t offset, const struct iovec *iov, int iovcnt, uint16_t timeout ) { SyncResponseHandler handler; XRootDStatus st = WriteV( offset, iov, iovcnt, &handler, timeout ); if( !st.IsOK() ) return st; XRootDStatus status = MessageUtils::WaitForStatus( &handler ); return status; } //------------------------------------------------------------------------ //! Read data into scattered buffers in one operation - async //! //! @param offset offset from the beginning of the file //! @param iov list of the buffers to be written //! @param iovcnt number of buffers //! @param handler handler to be notified when the response arrives //! @param timeout timeout value, if 0 then the environment default //! will be used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus File::ReadV( uint64_t offset, struct iovec *iov, int iovcnt, ResponseHandler *handler, uint16_t timeout ) { return FileStateHandler::ReadV( pImpl->pStateHandler, offset, iov, iovcnt, handler, timeout ); } //------------------------------------------------------------------------ //! Read data into scattered buffers in one operation - sync //! //! @param offset offset from the beginning of the file //! @param iov list of the buffers to be written //! @param iovcnt number of buffers //! @param handler handler to be notified when the response arrives //! @param timeout timeout value, if 0 then the environment default //! will be used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus File::ReadV( uint64_t offset, struct iovec *iov, int iovcnt, uint32_t &bytesRead, uint16_t timeout ) { SyncResponseHandler handler; XRootDStatus st = ReadV( offset, iov, iovcnt, &handler, timeout ); if( !st.IsOK() ) return st; VectorReadInfo *vrInfo = 0; XRootDStatus status = MessageUtils::WaitForResponse( &handler, vrInfo ); if( status.IsOK() ) { bytesRead = vrInfo->GetSize(); delete vrInfo; } return status; } //---------------------------------------------------------------------------- // Performs a custom operation on an open file, server implementation // dependent - async //---------------------------------------------------------------------------- XRootDStatus File::Fcntl( const Buffer &arg, ResponseHandler *handler, uint16_t timeout ) { if( pPlugIn ) return pPlugIn->Fcntl( arg, handler, timeout ); return FileStateHandler::Fcntl( pImpl->pStateHandler, arg, handler, timeout ); } //---------------------------------------------------------------------------- // Performs a custom operation on an open file, server implementation // dependent - sync //---------------------------------------------------------------------------- XRootDStatus File::Fcntl( const Buffer &arg, Buffer *&response, uint16_t timeout ) { SyncResponseHandler handler; XRootDStatus st = Fcntl( arg, &handler, timeout ); if( !st.IsOK() ) return st; return MessageUtils::WaitForResponse( &handler, response ); } //------------------------------------------------------------------------ //! Get access token to a file - async //------------------------------------------------------------------------ XRootDStatus File::Visa( ResponseHandler *handler, uint16_t timeout ) { if( pPlugIn ) return pPlugIn->Visa( handler, timeout ); return FileStateHandler::Visa( pImpl->pStateHandler, handler, timeout ); } //---------------------------------------------------------------------------- // Get access token to a file - sync //---------------------------------------------------------------------------- XRootDStatus File::Visa( Buffer *&visa, uint16_t timeout ) { SyncResponseHandler handler; XRootDStatus st = Visa( &handler, timeout ); if( !st.IsOK() ) return st; return MessageUtils::WaitForResponse( &handler, visa ); } //------------------------------------------------------------------------ // Set extended attributes - async //------------------------------------------------------------------------ XRootDStatus File::SetXAttr( const std::vector &attrs, ResponseHandler *handler, uint16_t timeout ) { if( pPlugIn ) return XRootDStatus( stError, errNotSupported ); return FileStateHandler::SetXAttr( pImpl->pStateHandler, attrs, handler, timeout ); } //------------------------------------------------------------------------ // Set extended attributes - sync //------------------------------------------------------------------------ XRootDStatus File::SetXAttr( const std::vector &attrs, std::vector &result, uint16_t timeout ) { SyncResponseHandler handler; XRootDStatus st = SetXAttr( attrs, &handler, timeout ); if( !st.IsOK() ) return st; std::vector *resp = 0; st = MessageUtils::WaitForResponse( &handler, resp ); if( resp ) result.swap( *resp ); delete resp; return st; } //------------------------------------------------------------------------ // Get extended attributes - async //------------------------------------------------------------------------ XRootDStatus File::GetXAttr( const std::vector &attrs, ResponseHandler *handler, uint16_t timeout ) { if( pPlugIn ) return XRootDStatus( stError, errNotSupported ); return FileStateHandler::GetXAttr( pImpl->pStateHandler, attrs, handler, timeout ); } //------------------------------------------------------------------------ // Get extended attributes - sync //------------------------------------------------------------------------ XRootDStatus File::GetXAttr( const std::vector &attrs, std::vector &result, uint16_t timeout ) { SyncResponseHandler handler; XRootDStatus st = GetXAttr( attrs, &handler, timeout ); if( !st.IsOK() ) return st; std::vector *resp = 0; st = MessageUtils::WaitForResponse( &handler, resp ); if( resp ) result.swap( *resp ); delete resp; return st; } //------------------------------------------------------------------------ // Delete extended attributes - async //------------------------------------------------------------------------ XRootDStatus File::DelXAttr( const std::vector &attrs, ResponseHandler *handler, uint16_t timeout ) { if( pPlugIn ) return XRootDStatus( stError, errNotSupported ); return FileStateHandler::DelXAttr( pImpl->pStateHandler, attrs, handler, timeout ); } //------------------------------------------------------------------------ // Delete extended attributes - sync //------------------------------------------------------------------------ XRootDStatus File::DelXAttr( const std::vector &attrs, std::vector &result, uint16_t timeout ) { SyncResponseHandler handler; XRootDStatus st = DelXAttr( attrs, &handler, timeout ); if( !st.IsOK() ) return st; std::vector *resp = 0; st = MessageUtils::WaitForResponse( &handler, resp ); if( resp ) result.swap( *resp ); delete resp; return st; } //------------------------------------------------------------------------ // List extended attributes - async //------------------------------------------------------------------------ XRootDStatus File::ListXAttr( ResponseHandler *handler, uint16_t timeout ) { if( pPlugIn ) return XRootDStatus( stError, errNotSupported ); return FileStateHandler::ListXAttr( pImpl->pStateHandler, handler, timeout ); } //------------------------------------------------------------------------ // List extended attributes - sync //------------------------------------------------------------------------ XRootDStatus File::ListXAttr( std::vector &result, uint16_t timeout ) { SyncResponseHandler handler; XRootDStatus st = ListXAttr( &handler, timeout ); if( !st.IsOK() ) return st; std::vector *resp = 0; st = MessageUtils::WaitForResponse( &handler, resp ); if( resp ) result.swap( *resp ); delete resp; return st; } //------------------------------------------------------------------------ // Create a checkpoint //------------------------------------------------------------------------ XRootDStatus File::Checkpoint( kXR_char code, ResponseHandler *handler, uint16_t timeout ) { if( pPlugIn ) return XRootDStatus( stError, errNotSupported ); return FileStateHandler::Checkpoint( pImpl->pStateHandler, code, handler, timeout ); } //------------------------------------------------------------------------ //! Checkpointed write - async //------------------------------------------------------------------------ XRootDStatus File::ChkptWrt( uint64_t offset, uint32_t size, const void *buffer, ResponseHandler *handler, uint16_t timeout ) { if( pPlugIn ) return XRootDStatus( stError, errNotSupported ); return FileStateHandler::ChkptWrt( pImpl->pStateHandler, offset, size, buffer, handler, timeout ); } //------------------------------------------------------------------------ //! Checkpointed WriteV - async //------------------------------------------------------------------------ XRootDStatus File::ChkptWrtV( uint64_t offset, const struct iovec *iov, int iovcnt, ResponseHandler *handler, uint16_t timeout ) { if( pPlugIn ) return XRootDStatus( stError, errNotSupported ); return FileStateHandler::ChkptWrtV( pImpl->pStateHandler, offset, iov, iovcnt, handler, timeout ); } //------------------------------------------------------------------------ // Try different data server //------------------------------------------------------------------------ XRootDStatus File::TryOtherServer( uint16_t timeout ) { return FileStateHandler::TryOtherServer( pImpl->pStateHandler, timeout ); } //---------------------------------------------------------------------------- // Check if the file is open //---------------------------------------------------------------------------- bool File::IsOpen() const { if( pPlugIn ) return pPlugIn->IsOpen(); return pImpl->pStateHandler->IsOpen(); } //------------------------------------------------------------------------ //! Check if the file is using an encrypted connection //------------------------------------------------------------------------ bool File::IsSecure() const { if( pPlugIn ) return false; return pImpl->pStateHandler->IsSecure(); } //---------------------------------------------------------------------------- // Set file property //---------------------------------------------------------------------------- bool File::SetProperty( const std::string &name, const std::string &value ) { if( pPlugIn ) return pPlugIn->SetProperty( name, value ); return pImpl->pStateHandler->SetProperty( name, value ); } //---------------------------------------------------------------------------- // Get file property //---------------------------------------------------------------------------- bool File::GetProperty( const std::string &name, std::string &value ) const { if( pPlugIn ) return pPlugIn->GetProperty( name, value ); return pImpl->pStateHandler->GetProperty( name, value ); } } xrootd-5.6.9/src/XrdCl/XrdClFile.hh000066400000000000000000001250431457266313600170340ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2014 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #ifndef __XRD_CL_FILE_HH__ #define __XRD_CL_FILE_HH__ #include "XrdCl/XrdClFileSystem.hh" #include "XrdCl/XrdClXRootDResponses.hh" #include "XrdCl/XrdClOptional.hh" #include "XrdOuc/XrdOucCompiler.hh" #include #include #include #include namespace XrdCl { struct FileImpl; class FilePlugIn; //---------------------------------------------------------------------------- //! A file //---------------------------------------------------------------------------- class File { public: enum VirtRedirect { EnableVirtRedirect, DisableVirtRedirect }; //------------------------------------------------------------------------ //! Constructor //------------------------------------------------------------------------ File( bool enablePlugIns = true ); //------------------------------------------------------------------------ //! Constructor //------------------------------------------------------------------------ File( VirtRedirect virtRedirect, bool enablePlugIns = true ); //------------------------------------------------------------------------ //! Destructor //------------------------------------------------------------------------ virtual ~File(); //------------------------------------------------------------------------ //! Open the file pointed to by the given URL - async //! //! @param url url of the file to be opened //! @param flags OpenFlags::Flags //! @param mode Access::Mode for new files, 0 otherwise //! @param handler handler to be notified about the status of the operation //! @param timeout timeout value, if 0 the environment default will be //! used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus Open( const std::string &url, OpenFlags::Flags flags, Access::Mode mode, ResponseHandler *handler, uint16_t timeout = 0 ) XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ //! Open the file pointed to by the given URL - sync //! //! @param url url of the file to be opened //! @param flags OpenFlags::Flags //! @param mode Access::Mode for new files, 0 otherwise //! @param timeout timeout value, if 0 the environment default will be //! used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus Open( const std::string &url, OpenFlags::Flags flags, Access::Mode mode = Access::None, uint16_t timeout = 0 ) XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ //! Close the file - async //! //! @param handler handler to be notified about the status of the operation //! @param timeout timeout value, if 0 the environment default will be //! used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus Close( ResponseHandler *handler, uint16_t timeout = 0 ) XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ //! Close the file - sync //! //! @param timeout timeout value, if 0 the environment default will be //! used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus Close( uint16_t timeout = 0 ) XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ //! Obtain status information for this file - async //! //! @param force do not use the cached information, force re-stating //! @param handler handler to be notified when the response arrives, //! the response parameter will hold a StatInfo object //! if the procedure is successful //! @param timeout timeout value, if 0 the environment default will //! be used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus Stat( bool force, ResponseHandler *handler, uint16_t timeout = 0 ) XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ //! Obtain status information for this file - sync //! //! @param force do not use the cached information, force re-stating //! @param response the response (to be deleted by the user) //! @param timeout timeout value, if 0 the environment default will //! be used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus Stat( bool force, StatInfo *&response, uint16_t timeout = 0 ) XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ //! Read a data chunk at a given offset - async //! //! @param offset offset from the beginning of the file //! @param size number of bytes to be read //! @param buffer a pointer to a buffer big enough to hold the data //! or 0 if the buffer should be allocated by the system //! @param handler handler to be notified when the response arrives, //! the response parameter will hold a ChunkInfo object if //! the procedure was successful //! @param timeout timeout value, if 0 the environment default will be //! used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus Read( uint64_t offset, uint32_t size, void *buffer, ResponseHandler *handler, uint16_t timeout = 0 ) XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ //! Read a data chunk at a given offset - sync //! //! @param offset offset from the beginning of the file //! @param size number of bytes to be read //! @param buffer a pointer to a buffer big enough to hold the data //! @param bytesRead number of bytes actually read //! @param timeout timeout value, if 0 the environment default will be //! used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus Read( uint64_t offset, uint32_t size, void *buffer, uint32_t &bytesRead, uint16_t timeout = 0 ) XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ //! Read number of pages at a given offset - async //! //! @param offset offset from the beginning of the file //! @param size buffer size, at least 1 page big (4KB) //! @param buffer a pointer to a buffer big enough to hold the data //! @param handler handler to be notified when the response arrives, //! the response parameter will hold a PageInfo object if //! the procedure was successful //! @param timeout timeout value, if 0 the environment default will be //! used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus PgRead( uint64_t offset, uint32_t size, void *buffer, ResponseHandler *handler, uint16_t timeout = 0 ) XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ //! Read a data chunk at a given offset - sync //! //! @param offset offset from the beginning of the file //! @param size buffer size, at least 1 page big (4KB) //! @param buffer a pointer to a buffer big enough to hold the data //! @param cksums crc32c checksum for each read 4KB page //! @param bytesRead number of bytes actually read //! @param timeout timeout value, if 0 the environment default will be //! used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus PgRead( uint64_t offset, uint32_t size, void *buffer, std::vector &cksums, uint32_t &bytesRead, uint16_t timeout = 0 ) XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ //! Write a data chunk at a given offset - async //! The call interprets and returns the server response, which may be //! either a success or a failure, it does not contain the number //! of bytes that were actually written. //! //! @param offset offset from the beginning of the file //! @param size number of bytes to be written //! @param buffer a pointer to the buffer holding the data to be written //! @param handler handler to be notified when the response arrives //! @param timeout timeout value, if 0 the environment default will be //! used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus Write( uint64_t offset, uint32_t size, const void *buffer, ResponseHandler *handler, uint16_t timeout = 0 ) XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ //! Write a data chunk at a given offset - sync //! The call interprets and returns the server response, which may be //! either a success or a failure, it does not contain the number //! of bytes that were actually written. //! //! @param offset offset from the beginning of the file //! @param size number of bytes to be written //! @param buffer a pointer to the buffer holding the data to be //! written //! @param timeout timeout value, if 0 the environment default will //! be used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus Write( uint64_t offset, uint32_t size, const void *buffer, uint16_t timeout = 0 ) XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ //! Write a data chunk at a given offset - async //! //! @param offset offset from the beginning of the file //! @param buffer r-value reference to Buffer object, in this case XrdCl //! runtime takes ownership of the buffer //! @param handler handler to be notified when the response arrives //! @param timeout timeout value, if 0 the environment default will be //! used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus Write( uint64_t offset, Buffer &&buffer, ResponseHandler *handler, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! Write a data chunk at a given offset - sync //! //! @param offset offset from the beginning of the file //! @param buffer r-value reference to Buffer object, in this case XrdCl //! runtime takes ownership of the buffer //! @param timeout timeout value, if 0 the environment default will be //! used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus Write( uint64_t offset, Buffer &&buffer, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! Write a data from a given file descriptor at a given offset - async //! //! @param offset offset from the beginning of the file //! @param size number of bytes to be written //! @param fdoff offset of the data to be written from the file descriptor //! (optional, if not provided will copy data from the file //! descriptor at the current cursor position) //! @param fd file descriptor open for reading //! @param handler handler to be notified when the response arrives //! @param timeout timeout value, if 0 the environment default will be //! used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus Write( uint64_t offset, uint32_t size, Optional fdoff, int fd, ResponseHandler *handler, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! Write a data from a given file descriptor at a given offset - sync //! //! @param offset offset from the beginning of the file //! @param size number of bytes to be written //! @param fdoff offset of the data to be written from the file descriptor //! (optional, if not provided will copy data from the file //! descriptor at the current cursor position) //! @param fd file descriptor open for reading //! @param timeout timeout value, if 0 the environment default will be //! used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus Write( uint64_t offset, uint32_t size, Optional fdoff, int fd, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! Write number of pages at a given offset - async //! //! @param offset offset from the beginning of the file //! @param size buffer size //! @param buffer a pointer to a buffer holding data pages //! @param cksums the crc32c checksums for each 4KB page //! @param handler handler to be notified when the response arrives //! @param timeout timeout value, if 0 the environment default will be //! used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus PgWrite( uint64_t offset, uint32_t size, const void *buffer, std::vector &cksums, ResponseHandler *handler, uint16_t timeout = 0 ) XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ //! Write number of pages at a given offset - sync //! //! @param offset offset from the beginning of the file //! @param size buffer size //! @param buffer a pointer to a buffer holding data pages //! @param cksums the crc32c checksums for each 4KB page //! @param timeout timeout value, if 0 the environment default will be //! used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus PgWrite( uint64_t offset, uint32_t size, const void *buffer, std::vector &cksums, uint16_t timeout = 0 ) XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ //! Commit all pending disk writes - async //! //! @param handler handler to be notified when the response arrives //! @param timeout timeout value, if 0 the environment default will be //! used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus Sync( ResponseHandler *handler, uint16_t timeout = 0 ) XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ //! Commit all pending disk writes - sync //! //! @param timeout timeout value, if 0 the environment default will be //! used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus Sync( uint16_t timeout = 0 ) XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ //! Truncate the file to a particular size - async //! //! @param size desired size of the file //! @param handler handler to be notified when the response arrives //! @param timeout timeout value, if 0 the environment default will be //! used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus Truncate( uint64_t size, ResponseHandler *handler, uint16_t timeout = 0 ) XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ //! Truncate the file to a particular size - sync //! //! @param size desired size of the file //! @param timeout timeout value, if 0 the environment default will be //! used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus Truncate( uint64_t size, uint16_t timeout = 0 ) XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ //! Read scattered data chunks in one operation - async //! //! @param chunks list of the chunks to be read and buffers to put //! the data in. The default maximum chunk size is //! 2097136 bytes and the default maximum number //! of chunks per request is 1024. The server //! may be queried using FileSystem::Query for the //! actual settings. //! @param buffer if zero the buffer pointers in the chunk list //! will be used, otherwise it needs to point to a //! buffer big enough to hold the requested data //! @param handler handler to be notified when the response arrives //! @param timeout timeout value, if 0 then the environment default //! will be used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus VectorRead( const ChunkList &chunks, void *buffer, ResponseHandler *handler, uint16_t timeout = 0 ) XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ //! Read scattered data chunks in one operation - sync //! //! @param chunks list of the chunks to be read and buffers to put //! the data in. The default maximum chunk size is //! 2097136 bytes and the default maximum number //! of chunks per request is 1024. The server //! may be queried using FileSystem::Query for the //! actual settings. //! @param buffer if zero the buffer pointers in the chunk list //! will be used, otherwise it needs to point to a //! buffer big enough to hold the requested data //! @param vReadInfo buffer size and chunk information //! @param timeout timeout value, if 0 then the environment default //! will be used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus VectorRead( const ChunkList &chunks, void *buffer, VectorReadInfo *&vReadInfo, uint16_t timeout = 0 ) XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ //! Write scattered data chunks in one operation - async //! //! @param chunks list of the chunks to be written. //! @param handler handler to be notified when the response arrives //! @param timeout timeout value, if 0 then the environment default //! will be used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus VectorWrite( const ChunkList &chunks, ResponseHandler *handler, uint16_t timeout = 0 ) XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ //! Write scattered data chunks in one operation - sync //! //! @param chunks list of the chunks to be written. //! @param timeout timeout value, if 0 then the environment default //! will be used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus VectorWrite( const ChunkList &chunks, uint16_t timeout = 0 ) XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ //! Write scattered buffers in one operation - async //! //! @param offset offset from the beginning of the file //! @param iov list of the buffers to be written //! @param iovcnt number of buffers //! @param handler handler to be notified when the response arrives //! @param timeout timeout value, if 0 then the environment default //! will be used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus WriteV( uint64_t offset, const struct iovec *iov, int iovcnt, ResponseHandler *handler, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! Write scattered buffers in one operation - sync //! //! @param offset offset from the beginning of the file //! @param iov list of the buffers to //! @param iovcnt number of buffers //! @param timeout timeout value, if 0 then the environment default //! will be used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus WriteV( uint64_t offset, const struct iovec *iov, int iovcnt, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! Read data into scattered buffers in one operation - async //! //! @param offset offset from the beginning of the file //! @param iov list of the buffers to be written //! @param iovcnt number of buffers //! @param handler handler to be notified when the response arrives //! @param timeout timeout value, if 0 then the environment default //! will be used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus ReadV( uint64_t offset, struct iovec *iov, int iovcnt, ResponseHandler *handler, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! Read data into scattered buffers in one operation - sync //! //! @param offset offset from the beginning of the file //! @param iov list of the buffers to be written //! @param iovcnt number of buffers //! @param bytesRead number of bytes actually read //! @param timeout timeout value, if 0 then the environment default //! will be used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus ReadV( uint64_t offset, struct iovec *iov, int iovcnt, uint32_t &bytesRead, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! Performs a custom operation on an open file, server implementation //! dependent - async //! //! @param arg query argument //! @param handler handler to be notified when the response arrives, //! the response parameter will hold a Buffer object //! if the procedure is successful //! @param timeout timeout value, if 0 the environment default will //! be used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus Fcntl( const Buffer &arg, ResponseHandler *handler, uint16_t timeout = 0 ) XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ //! Performs a custom operation on an open file, server implementation //! dependent - sync //! //! @param arg query argument //! @param response the response (to be deleted by the user) //! @param timeout timeout value, if 0 the environment default will //! be used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus Fcntl( const Buffer &arg, Buffer *&response, uint16_t timeout = 0 ) XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ //! Get access token to a file - async //! //! @param handler handler to be notified when the response arrives, //! the response parameter will hold a Buffer object //! if the procedure is successful //! @param timeout timeout value, if 0 the environment default will //! be used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus Visa( ResponseHandler *handler, uint16_t timeout = 0 ) XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ //! Get access token to a file - sync //! //! @param visa the access token (to be deleted by the user) //! @param timeout timeout value, if 0 the environment default will //! be used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus Visa( Buffer *&visa, uint16_t timeout = 0 ) XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ //! Set extended attributes - async //! //! @param attrs : list of extended attributes to set //! @param handler : handler to be notified when the response arrives, //! the response parameter will hold a std::vector of //! XAttrStatus objects //! @param timeout : timeout value, if 0 the environment default will //! be used //! //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus SetXAttr( const std::vector &attrs, ResponseHandler *handler, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! Set extended attributes - sync //! //! @param attrs : list of extended attributes to set //! @param result : result of the operation //! @param timeout : timeout value, if 0 the environment default will //! be used //! //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus SetXAttr( const std::vector &attrs, std::vector &result, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! Get extended attributes - async //! //! @param attrs : list of extended attributes to get //! @param handler : handler to be notified when the response arrives, //! the response parameter will hold a std::vector of //! XAttr objects //! @param timeout : timeout value, if 0 the environment default will //! be used //! //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus GetXAttr( const std::vector &attrs, ResponseHandler *handler, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! Get extended attributes - sync //! //! @param attrs : list of extended attributes to get //! @param result : result of the operation //! @param timeout : timeout value, if 0 the environment default will //! be used //! //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus GetXAttr( const std::vector &attrs, std::vector &result, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! Delete extended attributes - async //! //! @param attrs : list of extended attributes to set //! @param handler : handler to be notified when the response arrives, //! the response parameter will hold a std::vector of //! XAttrStatus objects //! @param timeout : timeout value, if 0 the environment default will //! be used //! //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus DelXAttr( const std::vector &attrs, ResponseHandler *handler, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! Delete extended attributes - sync //! //! @param attrs : list of extended attributes to set //! @param result : result of the operation //! @param timeout : timeout value, if 0 the environment default will //! be used //! //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus DelXAttr( const std::vector &attrs, std::vector &result, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! List extended attributes - async //! //! @param handler : handler to be notified when the response arrives, //! the response parameter will hold a std::vector of //! XAttr objects //! @param timeout : timeout value, if 0 the environment default will //! be used //! //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus ListXAttr( ResponseHandler *handler, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! List extended attributes - sync //! //! @param result : result of the operation //! @param timeout : timeout value, if 0 the environment default will //! be used //! //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus ListXAttr( std::vector &result, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! Try different data server //! //! @param timeout : timeout value, if 0 the environment default will //! be used //! //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus TryOtherServer( uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! Check if the file is open //------------------------------------------------------------------------ bool IsOpen() const; //------------------------------------------------------------------------ //! Check if the file is using an encrypted connection //------------------------------------------------------------------------ bool IsSecure() const; //------------------------------------------------------------------------ //! Set file property //! //! File properties: //! ReadRecovery [true/false] - enable/disable read recovery //! WriteRecovery [true/false] - enable/disable write recovery //! FollowRedirects [true/false] - enable/disable following redirections //! BundledClose [true/false] - enable/disable bundled close //------------------------------------------------------------------------ bool SetProperty( const std::string &name, const std::string &value ); //------------------------------------------------------------------------ //! Get file property //! //! @see File::SetProperty for property list //! //! Read-only properties: //! DataServer [string] - the data server the file is accessed at //! LastURL [string] - final file URL with all the cgi information //------------------------------------------------------------------------ bool GetProperty( const std::string &name, std::string &value ) const; private: template friend class CheckpointImpl; template friend class ChkptWrtImpl; template friend class ChkptWrtVImpl; //------------------------------------------------------------------------ //! Create a checkpoint - async //! //! @param handler : handler to be notified when the response arrives, //! the response parameter will hold a std::vector of //! XAttr objects //! @param timeout : timeout value, if 0 the environment default will //! be used //! //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus Checkpoint( kXR_char code, ResponseHandler *handler, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! Checkpointed write - async //! //! @param offset offset from the beginning of the file //! @param size number of bytes to be written //! @param buffer a pointer to the buffer holding the data to be written //! @param handler handler to be notified when the response arrives //! @param timeout timeout value, if 0 the environment default will be //! used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus ChkptWrt( uint64_t offset, uint32_t size, const void *buffer, ResponseHandler *handler, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! Checkpointed WriteV - async //! //! @param offset offset from the beginning of the file //! @param iov list of the buffers to be written //! @param iovcnt number of buffers //! @param handler handler to be notified when the response arrives //! @param timeout timeout value, if 0 then the environment default //! will be used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus ChkptWrtV( uint64_t offset, const struct iovec *iov, int iovcnt, ResponseHandler *handler, uint16_t timeout = 0 ); FileImpl *pImpl; FilePlugIn *pPlugIn; bool pEnablePlugIns; }; } #endif // __XRD_CL_FILE_HH__ xrootd-5.6.9/src/XrdCl/XrdClFileOperations.hh000066400000000000000000001710471457266313600211050ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2017 by European Organization for Nuclear Research (CERN) // Author: Krzysztof Jamrog , // Michal Simon //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #ifndef __XRD_CL_FILE_OPERATIONS_HH__ #define __XRD_CL_FILE_OPERATIONS_HH__ #include "XrdCl/XrdClFile.hh" #include "XrdCl/XrdClOperations.hh" #include "XrdCl/XrdClOperationHandlers.hh" #include "XrdCl/XrdClCtx.hh" namespace XrdCl { //---------------------------------------------------------------------------- //! Base class for all file related operations //! //! @arg Derived : the class that derives from this template (CRTP) //! @arg HasHndl : true if operation has a handler, false otherwise //! @arg Args : operation arguments //---------------------------------------------------------------------------- template class Derived, bool HasHndl, typename Response, typename ... Arguments> class FileOperation: public ConcreteOperation { template class, bool, typename, typename ...> friend class FileOperation; public: //------------------------------------------------------------------------ //! Constructor //! //! @param f : file on which the operation will be performed //! @param args : file operation arguments //------------------------------------------------------------------------ FileOperation( Ctx f, Arguments... args): ConcreteOperation( std::move( args )... ), file( std::move( f ) ) { } //------------------------------------------------------------------------ //! Move constructor from other states //! //! @arg from : state from which the object is being converted //! //! @param op : the object that is being converted //------------------------------------------------------------------------ template FileOperation( FileOperation && op ) : ConcreteOperation( std::move( op ) ), file( op.file ) { } //------------------------------------------------------------------------ //! Destructor //------------------------------------------------------------------------ virtual ~FileOperation() { } protected: //------------------------------------------------------------------------ //! The file object itself //------------------------------------------------------------------------ Ctx file; }; //---------------------------------------------------------------------------- //! Open operation (@see FileOperation) //---------------------------------------------------------------------------- template class OpenImpl: public FileOperation, Arg, Arg, Arg> { //------------------------------------------------------------------------ //! Helper for extending the operator>> capabilities. //! //! In addition to standard overloads for std::function adds: //! - void( XRootDStatus&, StatInfo& ) //! - void( XRootDStatus&, StatInfo&, OperationContext& ) //------------------------------------------------------------------------ struct ExResp : public Resp { //-------------------------------------------------------------------- //! Constructor //! //! @param file : the underlying XrdCl::File object //-------------------------------------------------------------------- ExResp( const Ctx &file ): file( file ) { } //-------------------------------------------------------------------- //! A factory method //! //! @param func : the function/functor/lambda that should be wrapped //! @return : ResponseHandler instance //-------------------------------------------------------------------- inline ResponseHandler* Create( std::function func ) { return new ExOpenFuncWrapper( this->file, func ); } //-------------------------------------------------------------------- //! Make other overloads of Create visible //-------------------------------------------------------------------- using Resp::Create; //-------------------------------------------------------------------- //! The underlying XrdCl::File object //-------------------------------------------------------------------- Ctx file; }; public: //------------------------------------------------------------------------ //! Constructor (@see FileOperation) //------------------------------------------------------------------------ OpenImpl( Ctx f, Arg url, Arg flags, Arg mode = Access::None ) : FileOperation, Arg, Arg, Arg>( std::move( f ), std::move( url ), std::move( flags ), std::move( mode ) ) { } //------------------------------------------------------------------------ //! Move constructor from other states //! //! @arg from : state from which the object is being converted //! //! @param open : the object that is being converted //------------------------------------------------------------------------ template OpenImpl( OpenImpl && open ) : FileOperation, Arg, Arg, Arg>( std::move( open ) ) { } //------------------------------------------------------------------------ //! Argument indexes in the args tuple //------------------------------------------------------------------------ enum { UrlArg, FlagsArg, ModeArg }; //------------------------------------------------------------------------ //! Overload of operator>> defined in ConcreteOperation, we're adding //! additional capabilities by using ExResp factory (@see ExResp). //! //! @param hdlr : function/functor/lambda //------------------------------------------------------------------------ template OpenImpl operator>>( Hdlr &&hdlr ) { ExResp factory( *this->file ); return this->StreamImpl( factory.Create( hdlr ) ); } //------------------------------------------------------------------------ //! @return : name of the operation (@see Operation) //------------------------------------------------------------------------ std::string ToString() { return "Open"; } protected: //------------------------------------------------------------------------ //! RunImpl operation (@see Operation) //! //! @param pipelineTimeout : pipeline timeout //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus RunImpl( PipelineHandler *handler, uint16_t pipelineTimeout ) { const std::string &url = std::get( this->args ); OpenFlags::Flags flags = std::get( this->args ); Access::Mode mode = std::get( this->args ); uint16_t timeout = pipelineTimeout < this->timeout ? pipelineTimeout : this->timeout; return this->file->Open( url, flags, mode, handler, timeout ); } }; //---------------------------------------------------------------------------- //! Factory for creating ReadImpl objects //---------------------------------------------------------------------------- inline OpenImpl Open( Ctx file, Arg url, Arg flags, Arg mode = Access::None, uint16_t timeout = 0 ) { return OpenImpl( std::move( file ), std::move( url ), std::move( flags ), std::move( mode ) ).Timeout( timeout ); } //---------------------------------------------------------------------------- //! Read operation (@see FileOperation) //---------------------------------------------------------------------------- template class ReadImpl: public FileOperation, Arg, Arg, Arg> { public: //------------------------------------------------------------------------ //! Inherit constructors from FileOperation (@see FileOperation) //------------------------------------------------------------------------ using FileOperation, Arg, Arg, Arg>::FileOperation; //------------------------------------------------------------------------ //! Argument indexes in the args tuple //------------------------------------------------------------------------ enum { OffsetArg, SizeArg, BufferArg }; //------------------------------------------------------------------------ //! @return : name of the operation (@see Operation) //------------------------------------------------------------------------ std::string ToString() { return "Read"; } protected: //------------------------------------------------------------------------ //! RunImpl operation (@see Operation) //! //! @param params : container with parameters forwarded from //! previous operation //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus RunImpl( PipelineHandler *handler, uint16_t pipelineTimeout ) { uint64_t offset = std::get( this->args ).Get(); uint32_t size = std::get( this->args ).Get(); void *buffer = std::get( this->args ).Get(); uint16_t timeout = pipelineTimeout < this->timeout ? pipelineTimeout : this->timeout; return this->file->Read( offset, size, buffer, handler, timeout ); } }; //---------------------------------------------------------------------------- //! Factory for creating ReadImpl objects //---------------------------------------------------------------------------- inline ReadImpl Read( Ctx file, Arg offset, Arg size, Arg buffer, uint16_t timeout = 0 ) { return ReadImpl( std::move( file ), std::move( offset ), std::move( size ), std::move( buffer ) ).Timeout( timeout ); } //---------------------------------------------------------------------------- //! PgRead operation (@see FileOperation) //---------------------------------------------------------------------------- template class PgReadImpl: public FileOperation, Arg, Arg, Arg> { public: //------------------------------------------------------------------------ //! Inherit constructors from FileOperation (@see FileOperation) //------------------------------------------------------------------------ using FileOperation, Arg, Arg, Arg>::FileOperation; //------------------------------------------------------------------------ //! Argument indexes in the args tuple //------------------------------------------------------------------------ enum { OffsetArg, SizeArg, BufferArg }; //------------------------------------------------------------------------ //! @return : name of the operation (@see Operation) //------------------------------------------------------------------------ std::string ToString() { return "PgRead"; } protected: //------------------------------------------------------------------------ //! RunImpl operation (@see Operation) //! //! @param params : container with parameters forwarded from //! previous operation //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus RunImpl( PipelineHandler *handler, uint16_t pipelineTimeout ) { uint64_t offset = std::get( this->args ).Get(); uint32_t size = std::get( this->args ).Get(); void *buffer = std::get( this->args ).Get(); uint16_t timeout = pipelineTimeout < this->timeout ? pipelineTimeout : this->timeout; return this->file->PgRead( offset, size, buffer, handler, timeout ); } }; //---------------------------------------------------------------------------- //! Factory for creating PgReadImpl objects //---------------------------------------------------------------------------- inline PgReadImpl PgRead( Ctx file, Arg offset, Arg size, Arg buffer, uint16_t timeout = 0 ) { return PgReadImpl( std::move( file ), std::move( offset ), std::move( size ), std::move( buffer ) ).Timeout( timeout ); } //---------------------------------------------------------------------------- //! RdWithRsp: factory for creating ReadImpl/PgReadImpl objects //---------------------------------------------------------------------------- template struct ReadTrait { }; template<> struct ReadTrait { using RET = ReadImpl; }; template<> struct ReadTrait { using RET = PgReadImpl; }; template inline typename ReadTrait::RET RdWithRsp( Ctx file, Arg offset, Arg size, Arg buffer, uint16_t timeout = 0 ); template<> inline ReadImpl RdWithRsp( Ctx file, Arg offset, Arg size, Arg buffer, uint16_t timeout ) { return Read( std::move( file ), std::move( offset ), std::move( size ), std::move( buffer ), timeout ); } template<> inline PgReadImpl RdWithRsp( Ctx file, Arg offset, Arg size, Arg buffer, uint16_t timeout ) { return PgRead( std::move( file ), std::move( offset ), std::move( size ), std::move( buffer ), timeout ); } //---------------------------------------------------------------------------- //! PgWrite operation (@see FileOperation) //---------------------------------------------------------------------------- template class PgWriteImpl: public FileOperation, Arg, Arg, Arg, Arg>> { public: //------------------------------------------------------------------------ //! Inherit constructors from FileOperation (@see FileOperation) //------------------------------------------------------------------------ using FileOperation, Arg, Arg, Arg, Arg>>::FileOperation; //------------------------------------------------------------------------ //! Argument indexes in the args tuple //------------------------------------------------------------------------ enum { OffsetArg, SizeArg, BufferArg, CksumsArg }; //------------------------------------------------------------------------ //! @return : name of the operation (@see Operation) //------------------------------------------------------------------------ std::string ToString() { return "PgWrite"; } protected: //------------------------------------------------------------------------ //! RunImpl operation (@see Operation) //! //! @param params : container with parameters forwarded from //! previous operation //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus RunImpl( PipelineHandler *handler, uint16_t pipelineTimeout ) { uint64_t offset = std::get( this->args ).Get(); uint32_t size = std::get( this->args ).Get(); void *buffer = std::get( this->args ).Get(); std::vector cksums = std::get( this->args ).Get(); uint16_t timeout = pipelineTimeout < this->timeout ? pipelineTimeout : this->timeout; return this->file->PgWrite( offset, size, buffer, cksums, handler, timeout ); } }; //---------------------------------------------------------------------------- //! Factory for creating PgReadImpl objects //---------------------------------------------------------------------------- inline PgWriteImpl PgWrite( Ctx file, Arg offset, Arg size, Arg buffer, Arg> cksums, uint16_t timeout = 0 ) { return PgWriteImpl( std::move( file ), std::move( offset ), std::move( size ), std::move( buffer ), std::move( cksums ) ).Timeout( timeout ); } //---------------------------------------------------------------------------- //! Factory for creating PgReadImpl objects //---------------------------------------------------------------------------- inline PgWriteImpl PgWrite( Ctx file, Arg offset, Arg size, Arg buffer, uint16_t timeout = 0 ) { std::vector cksums; return PgWriteImpl( std::move( file ), std::move( offset ), std::move( size ), std::move( buffer ), std::move( cksums ) ).Timeout( timeout ); } //---------------------------------------------------------------------------- //! Close operation (@see FileOperation) //---------------------------------------------------------------------------- template class CloseImpl: public FileOperation> { public: //------------------------------------------------------------------------ //! Inherit constructors from FileOperation (@see FileOperation) //------------------------------------------------------------------------ using FileOperation>::FileOperation; //------------------------------------------------------------------------ //! @return : name of the operation (@see Operation) //------------------------------------------------------------------------ std::string ToString() { return "Close"; } protected: //------------------------------------------------------------------------ //! RunImpl operation (@see Operation) //! //! @param params : container with parameters forwarded from //! previous operation //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus RunImpl( PipelineHandler *handler, uint16_t pipelineTimeout ) { uint16_t timeout = pipelineTimeout < this->timeout ? pipelineTimeout : this->timeout; return this->file->Close( handler, timeout ); } }; //---------------------------------------------------------------------------- //! Factory for creating CloseImpl objects //---------------------------------------------------------------------------- inline CloseImpl Close( Ctx file, uint16_t timeout = 0 ) { return CloseImpl( std::move( file ) ).Timeout( timeout ); } //---------------------------------------------------------------------------- //! Stat operation (@see FileOperation) //---------------------------------------------------------------------------- template class StatImpl: public FileOperation, Arg> { public: //------------------------------------------------------------------------ //! Inherit constructors from FileOperation (@see FileOperation) //------------------------------------------------------------------------ using FileOperation, Arg>::FileOperation; //------------------------------------------------------------------------ //! Argument indexes in the args tuple //------------------------------------------------------------------------ enum { ForceArg }; //------------------------------------------------------------------------ //! @return : name of the operation (@see Operation) //------------------------------------------------------------------------ std::string ToString() { return "Stat"; } protected: //------------------------------------------------------------------------ //! RunImpl operation (@see Operation) //! //! @param params : container with parameters forwarded from //! previous operation //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus RunImpl( PipelineHandler *handler, uint16_t pipelineTimeout ) { bool force = std::get( this->args ).Get(); uint16_t timeout = pipelineTimeout < this->timeout ? pipelineTimeout : this->timeout; return this->file->Stat( force, handler, timeout ); } }; //---------------------------------------------------------------------------- //! Factory for creating StatImpl objects (as there is another Stat in //! FileSystem there would be a clash of typenames). //---------------------------------------------------------------------------- inline StatImpl Stat( Ctx file, Arg force, uint16_t timeout = 0 ) { return StatImpl( std::move( file ), std::move( force ) ).Timeout( timeout ); } //---------------------------------------------------------------------------- //! Write operation (@see FileOperation) //---------------------------------------------------------------------------- template class WriteImpl: public FileOperation, Arg, Arg, Arg> { public: //------------------------------------------------------------------------ //! Inherit constructors from FileOperation (@see FileOperation) //------------------------------------------------------------------------ using FileOperation, Arg, Arg, Arg>::FileOperation; //------------------------------------------------------------------------ //! Argument indexes in the args tuple //------------------------------------------------------------------------ enum { OffsetArg, SizeArg, BufferArg }; //------------------------------------------------------------------------ //! @return : name of the operation (@see Operation) //------------------------------------------------------------------------ std::string ToString() { return "Write"; } protected: //------------------------------------------------------------------------ //! RunImpl operation (@see Operation) //! //! @param params : container with parameters forwarded from //! previous operation //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus RunImpl( PipelineHandler *handler, uint16_t pipelineTimeout ) { uint64_t offset = std::get( this->args ).Get(); uint32_t size = std::get( this->args ).Get(); const void *buffer = std::get( this->args ).Get(); uint16_t timeout = pipelineTimeout < this->timeout ? pipelineTimeout : this->timeout; return this->file->Write( offset, size, buffer, handler, timeout ); } }; //---------------------------------------------------------------------------- //! Factory for creating WriteImpl objects //---------------------------------------------------------------------------- inline WriteImpl Write( Ctx file, Arg offset, Arg size, Arg buffer, uint16_t timeout = 0 ) { return WriteImpl( std::move( file ), std::move( offset ), std::move( size ), std::move( buffer ) ).Timeout( timeout ); } //---------------------------------------------------------------------------- //! Sync operation (@see FileOperation) //---------------------------------------------------------------------------- template class SyncImpl: public FileOperation> { public: //------------------------------------------------------------------------ //! Inherit constructors from FileOperation (@see FileOperation) //------------------------------------------------------------------------ using FileOperation>::FileOperation; //------------------------------------------------------------------------ //! @return : name of the operation (@see Operation) //------------------------------------------------------------------------ std::string ToString() { return "Sync"; } protected: //------------------------------------------------------------------------ //! RunImpl operation (@see Operation) //! //! @param params : container with parameters forwarded from //! previous operation //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus RunImpl( PipelineHandler *handler, uint16_t pipelineTimeout ) { uint16_t timeout = pipelineTimeout < this->timeout ? pipelineTimeout : this->timeout; return this->file->Sync( handler, timeout ); } }; //---------------------------------------------------------------------------- //! Factory for creating SyncImpl objects //---------------------------------------------------------------------------- inline SyncImpl Sync( Ctx file, uint16_t timeout = 0 ) { return SyncImpl( std::move( file ) ).Timeout( timeout ); } //---------------------------------------------------------------------------- //! Truncate operation (@see FileOperation) //---------------------------------------------------------------------------- template class TruncateImpl: public FileOperation, Arg> { public: //------------------------------------------------------------------------ //! Inherit constructors from FileOperation (@see FileOperation) //------------------------------------------------------------------------ using FileOperation, Arg>::FileOperation; //------------------------------------------------------------------------ //! Argument indexes in the args tuple //------------------------------------------------------------------------ enum { SizeArg }; //------------------------------------------------------------------------ //! @return : name of the operation (@see Operation) //------------------------------------------------------------------------ std::string ToString() { return "Truncate"; } protected: //------------------------------------------------------------------------ //! RunImpl operation (@see Operation) //! //! @param params : container with parameters forwarded from //! previous operation //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus RunImpl( PipelineHandler *handler, uint16_t pipelineTimeout ) { uint64_t size = std::get( this->args ).Get(); uint16_t timeout = pipelineTimeout < this->timeout ? pipelineTimeout : this->timeout; return this->file->Truncate( size, handler, timeout ); } }; //---------------------------------------------------------------------------- //! Factory for creating TruncateImpl objects (as there is another Stat in //! FileSystem there would be a clash of typenames). //---------------------------------------------------------------------------- inline TruncateImpl Truncate( Ctx file, Arg size, uint16_t timeout ) { return TruncateImpl( std::move( file ), std::move( size ) ).Timeout( timeout ); } //---------------------------------------------------------------------------- //! VectorRead operation (@see FileOperation) //---------------------------------------------------------------------------- template class VectorReadImpl: public FileOperation, Arg, Arg> { public: //------------------------------------------------------------------------ //! Inherit constructors from FileOperation (@see FileOperation) //------------------------------------------------------------------------ using FileOperation, Arg, Arg>::FileOperation; //------------------------------------------------------------------------ //! Argument indexes in the args tuple //------------------------------------------------------------------------ enum { ChunksArg, BufferArg }; //------------------------------------------------------------------------ //! @return : name of the operation (@see Operation) //------------------------------------------------------------------------ std::string ToString() { return "VectorRead"; } protected: //------------------------------------------------------------------------ //! RunImpl operation (@see Operation) //! //! @param params : container with parameters forwarded from //! previous operation //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus RunImpl( PipelineHandler *handler, uint16_t pipelineTimeout ) { ChunkList &chunks = std::get( this->args ).Get(); void *buffer = std::get( this->args ).Get(); uint16_t timeout = pipelineTimeout < this->timeout ? pipelineTimeout : this->timeout; return this->file->VectorRead( chunks, buffer, handler, timeout ); } }; //---------------------------------------------------------------------------- //! Factory for creating VectorReadImpl objects //---------------------------------------------------------------------------- inline VectorReadImpl VectorRead( Ctx file, Arg chunks, Arg buffer, uint16_t timeout = 0 ) { return VectorReadImpl( std::move( file ), std::move( chunks ), std::move( buffer ) ).Timeout( timeout ); } inline VectorReadImpl VectorRead( Ctx file, Arg chunks, uint16_t timeout = 0 ) { return VectorReadImpl( std::move( file ), std::move( chunks ), nullptr ).Timeout( timeout ); } //---------------------------------------------------------------------------- //! VectorWrite operation (@see FileOperation) //---------------------------------------------------------------------------- template class VectorWriteImpl: public FileOperation, Arg> { public: //------------------------------------------------------------------------ //! Inherit constructors from FileOperation (@see FileOperation) //------------------------------------------------------------------------ using FileOperation, Arg>::FileOperation; //------------------------------------------------------------------------ //! Argument indexes in the args tuple //------------------------------------------------------------------------ enum { ChunksArg }; //------------------------------------------------------------------------ //! @return : name of the operation (@see Operation) //------------------------------------------------------------------------ std::string ToString() { return "VectorWrite"; } protected: //------------------------------------------------------------------------ //! RunImpl operation (@see Operation) //! //! @param params : container with parameters forwarded from //! previous operation //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus RunImpl( PipelineHandler *handler, uint16_t pipelineTimeout ) { const ChunkList &chunks = std::get( this->args ).Get(); uint16_t timeout = pipelineTimeout < this->timeout ? pipelineTimeout : this->timeout; return this->file->VectorWrite( chunks, handler, timeout ); } }; //---------------------------------------------------------------------------- //! Factory for creating VectorWriteImpl objects //---------------------------------------------------------------------------- inline VectorWriteImpl VectorWrite( Ctx file, Arg chunks, uint16_t timeout = 0 ) { return VectorWriteImpl( std::move( file ), std::move( chunks ) ).Timeout( timeout ); } //---------------------------------------------------------------------------- //! WriteV operation (@see FileOperation) //---------------------------------------------------------------------------- template class WriteVImpl: public FileOperation, Arg, Arg>> { public: //------------------------------------------------------------------------ //! Inherit constructors from FileOperation (@see FileOperation) //------------------------------------------------------------------------ using FileOperation, Arg, Arg>>::FileOperation; //------------------------------------------------------------------------ //! Argument indexes in the args tuple //------------------------------------------------------------------------ enum { OffsetArg, IovArg }; //------------------------------------------------------------------------ //! @return : name of the operation (@see Operation) //------------------------------------------------------------------------ std::string ToString() { return "WriteV"; } protected: //------------------------------------------------------------------------ //! RunImpl operation (@see Operation) //! //! @param params : container with parameters forwarded from //! previous operation //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus RunImpl( PipelineHandler *handler, uint16_t pipelineTimeout ) { uint64_t offset = std::get( this->args ).Get(); std::vector &stdiov = std::get( this->args ).Get(); uint16_t timeout = pipelineTimeout < this->timeout ? pipelineTimeout : this->timeout; int iovcnt = stdiov.size(); iovec iov[iovcnt]; for( size_t i = 0; i < stdiov.size(); ++i ) { iov[i].iov_base = stdiov[i].iov_base; iov[i].iov_len = stdiov[i].iov_len; } return this->file->WriteV( offset, iov, iovcnt, handler, timeout ); } }; //---------------------------------------------------------------------------- //! Factory for creating WriteVImpl objects //---------------------------------------------------------------------------- inline WriteVImpl WriteV( Ctx file, Arg offset, Arg> iov, uint16_t timeout = 0 ) { return WriteVImpl( std::move( file ), std::move( offset ), std::move( iov ) ).Timeout( timeout ); } //---------------------------------------------------------------------------- //! Fcntl operation (@see FileOperation) //---------------------------------------------------------------------------- template class FcntlImpl: public FileOperation, Arg> { public: //------------------------------------------------------------------------ //! Inherit constructors from FileOperation (@see FileOperation) //------------------------------------------------------------------------ using FileOperation, Arg>::FileOperation; //------------------------------------------------------------------------ //! Argument indexes in the args tuple //------------------------------------------------------------------------ enum { BufferArg }; //------------------------------------------------------------------------ //! @return : name of the operation (@see Operation) //------------------------------------------------------------------------ std::string ToString() { return "Fcntl"; } protected: //------------------------------------------------------------------------ //! RunImpl operation (@see Operation) //! //! @param params : container with parameters forwarded from //! previous operation //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus RunImpl( PipelineHandler *handler, uint16_t pipelineTimeout ) { Buffer &arg = std::get( this->args ).Get(); uint16_t timeout = pipelineTimeout < this->timeout ? pipelineTimeout : this->timeout; return this->file->Fcntl( arg, handler, timeout ); } }; typedef FcntlImpl Fcntl; //---------------------------------------------------------------------------- //! Visa operation (@see FileOperation) //---------------------------------------------------------------------------- template class VisaImpl: public FileOperation> { public: //------------------------------------------------------------------------ //! Inherit constructors from FileOperation (@see FileOperation) //------------------------------------------------------------------------ using FileOperation>::FileOperation; //------------------------------------------------------------------------ //! @return : name of the operation (@see Operation) //------------------------------------------------------------------------ std::string ToString() { return "Visa"; } protected: //------------------------------------------------------------------------ //! RunImpl operation (@see Operation) //! //! @param params : container with parameters forwarded from //! previous operation //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus RunImpl( PipelineHandler *handler, uint16_t pipelineTimeout ) { uint16_t timeout = pipelineTimeout < this->timeout ? pipelineTimeout : this->timeout; return this->file->Visa( handler, timeout ); } }; typedef VisaImpl Visa; //---------------------------------------------------------------------------- //! SetXAttr operation (@see FileOperation) //---------------------------------------------------------------------------- template class SetXAttrImpl: public FileOperation, Arg, Arg> { public: //------------------------------------------------------------------------ //! Inherit constructors from FileOperation (@see FileOperation) //------------------------------------------------------------------------ using FileOperation, Arg, Arg>::FileOperation; //------------------------------------------------------------------------ //! Argument indexes in the args tuple //------------------------------------------------------------------------ enum { NameArg, ValueArg }; //------------------------------------------------------------------------ //! @return : name of the operation (@see Operation) //------------------------------------------------------------------------ std::string ToString() { return "SetXAttrImpl"; } protected: //------------------------------------------------------------------------ //! RunImpl operation (@see Operation) //! //! @param params : container with parameters forwarded from //! previous operation //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus RunImpl( PipelineHandler *handler, uint16_t pipelineTimeout ) { std::string &name = std::get( this->args ).Get(); std::string &value = std::get( this->args ).Get(); // wrap the arguments with a vector std::vector attrs; attrs.push_back( xattr_t( name, value ) ); // wrap the PipelineHandler so the response gets unpacked properly UnpackXAttrStatus *h = new UnpackXAttrStatus( handler ); uint16_t timeout = pipelineTimeout < this->timeout ? pipelineTimeout : this->timeout; XRootDStatus st = this->file->SetXAttr( attrs, h, timeout ); if( !st.IsOK() ) delete h; return st; } }; //---------------------------------------------------------------------------- //! Factory for creating SetXAttrImpl objects (as there is another SetXAttr in //! FileSystem there would be a clash of typenames). //---------------------------------------------------------------------------- inline SetXAttrImpl SetXAttr( Ctx file, Arg name, Arg value ) { return SetXAttrImpl( std::move( file ), std::move( name ), std::move( value ) ); } //---------------------------------------------------------------------------- //! SetXAttr bulk operation (@see FileOperation) //---------------------------------------------------------------------------- template class SetXAttrBulkImpl: public FileOperation>, Arg>> { public: //------------------------------------------------------------------------ //! Inherit constructors from FileOperation (@see FileOperation) //------------------------------------------------------------------------ using FileOperation>, Arg>>::FileOperation; //------------------------------------------------------------------------ //! Argument indexes in the args tuple //------------------------------------------------------------------------ enum { AttrsArg }; //------------------------------------------------------------------------ //! @return : name of the operation (@see Operation) //------------------------------------------------------------------------ std::string ToString() { return "SetXAttrBulkImpl"; } protected: //------------------------------------------------------------------------ //! RunImpl operation (@see Operation) //! //! @param params : container with parameters forwarded from //! previous operation //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus RunImpl( PipelineHandler *handler, uint16_t pipelineTimeout ) { std::vector &attrs = std::get( this->args ).Get(); uint16_t timeout = pipelineTimeout < this->timeout ? pipelineTimeout : this->timeout; return this->file->SetXAttr( attrs, handler, timeout ); } }; //---------------------------------------------------------------------------- //! Factory for creating SetXAttrBulkImpl objects (as there is another SetXAttr //! in FileSystem there would be a clash of typenames). //---------------------------------------------------------------------------- inline SetXAttrBulkImpl SetXAttr( Ctx file, Arg> attrs ) { return SetXAttrBulkImpl( std::move( file ), std::move( attrs ) ); } //---------------------------------------------------------------------------- //! GetXAttr operation (@see FileOperation) //---------------------------------------------------------------------------- template class GetXAttrImpl: public FileOperation, Arg> { public: //------------------------------------------------------------------------ //! Inherit constructors from FileOperation (@see FileOperation) //------------------------------------------------------------------------ using FileOperation, Arg>::FileOperation; //------------------------------------------------------------------------ //! Argument indexes in the args tuple //------------------------------------------------------------------------ enum { NameArg }; //------------------------------------------------------------------------ //! @return : name of the operation (@see Operation) //------------------------------------------------------------------------ std::string ToString() { return "GetXAttrImpl"; } protected: //------------------------------------------------------------------------ //! RunImpl operation (@see Operation) //! //! @param params : container with parameters forwarded from //! previous operation //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus RunImpl( PipelineHandler *handler, uint16_t pipelineTimeout ) { std::string &name = std::get( this->args ).Get(); // wrap the argument with a vector std::vector attrs; attrs.push_back( name ); // wrap the PipelineHandler so the response gets unpacked properly UnpackXAttr *h = new UnpackXAttr( handler ); uint16_t timeout = pipelineTimeout < this->timeout ? pipelineTimeout : this->timeout; XRootDStatus st = this->file->GetXAttr( attrs, h, timeout ); if( !st.IsOK() ) delete h; return st; } }; //---------------------------------------------------------------------------- //! Factory for creating GetXAttrImpl objects (as there is another GetXAttr in //! FileSystem there would be a clash of typenames). //---------------------------------------------------------------------------- inline GetXAttrImpl GetXAttr( Ctx file, Arg name ) { return GetXAttrImpl( std::move( file ), std::move( name ) ); } //---------------------------------------------------------------------------- //! GetXAttr bulk operation (@see FileOperation) //---------------------------------------------------------------------------- template class GetXAttrBulkImpl: public FileOperation>, Arg>> { public: //------------------------------------------------------------------------ //! Inherit constructors from FileOperation (@see FileOperation) //------------------------------------------------------------------------ using FileOperation>, Arg>>::FileOperation; //------------------------------------------------------------------------ //! Argument indexes in the args tuple //------------------------------------------------------------------------ enum { NamesArg }; //------------------------------------------------------------------------ //! @return : name of the operation (@see Operation) //------------------------------------------------------------------------ std::string ToString() { return "GetXAttrBulkImpl"; } protected: //------------------------------------------------------------------------ //! RunImpl operation (@see Operation) //! //! @param params : container with parameters forwarded from //! previous operation //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus RunImpl( PipelineHandler *handler, uint16_t pipelineTimeout ) { std::vector &attrs = std::get( this->args ).Get(); uint16_t timeout = pipelineTimeout < this->timeout ? pipelineTimeout : this->timeout; return this->file->GetXAttr( attrs, handler, timeout ); } }; //---------------------------------------------------------------------------- //! Factory for creating GetXAttrBulkImpl objects (as there is another GetXAttr in //! FileSystem there would be a clash of typenames). //---------------------------------------------------------------------------- inline GetXAttrBulkImpl GetXAttr( Ctx file, Arg> attrs ) { return GetXAttrBulkImpl( std::move( file ), std::move( attrs ) ); } //---------------------------------------------------------------------------- //! DelXAttr operation (@see FileOperation) //---------------------------------------------------------------------------- template class DelXAttrImpl: public FileOperation, Arg> { public: //------------------------------------------------------------------------ //! Inherit constructors from FileOperation (@see FileOperation) //------------------------------------------------------------------------ using FileOperation, Arg>::FileOperation; //------------------------------------------------------------------------ //! Argument indexes in the args tuple //------------------------------------------------------------------------ enum { NameArg }; //------------------------------------------------------------------------ //! @return : name of the operation (@see Operation) //------------------------------------------------------------------------ std::string ToString() { return "DelXAttrImpl"; } protected: //------------------------------------------------------------------------ //! RunImpl operation (@see Operation) //! //! @param params : container with parameters forwarded from //! previous operation //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus RunImpl( PipelineHandler *handler, uint16_t pipelineTimeout ) { std::string &name = std::get( this->args ).Get(); // wrap the argument with a vector std::vector attrs; attrs.push_back( name ); // wrap the PipelineHandler so the response gets unpacked properly UnpackXAttrStatus *h = new UnpackXAttrStatus( handler ); uint16_t timeout = pipelineTimeout < this->timeout ? pipelineTimeout : this->timeout; XRootDStatus st = this->file->DelXAttr( attrs, h, timeout ); if( !st.IsOK() ) delete h; return st; } }; //---------------------------------------------------------------------------- //! Factory for creating DelXAttrImpl objects (as there is another DelXAttr in //! FileSystem there would be a clash of typenames). //---------------------------------------------------------------------------- inline DelXAttrImpl DelXAttr( Ctx file, Arg name ) { return DelXAttrImpl( std::move( file ), std::move( name ) ); } //---------------------------------------------------------------------------- //! DelXAttr bulk operation (@see FileOperation) //---------------------------------------------------------------------------- template class DelXAttrBulkImpl: public FileOperation>, Arg>> { public: //------------------------------------------------------------------------ //! Inherit constructors from FileOperation (@see FileOperation) //------------------------------------------------------------------------ using FileOperation>, Arg>>::FileOperation; //------------------------------------------------------------------------ //! Argument indexes in the args tuple //------------------------------------------------------------------------ enum { NamesArg }; //------------------------------------------------------------------------ //! @return : name of the operation (@see Operation) //------------------------------------------------------------------------ std::string ToString() { return "DelXAttrBulkImpl"; } protected: //------------------------------------------------------------------------ //! RunImpl operation (@see Operation) //! //! @param params : container with parameters forwarded from //! previous operation //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus RunImpl( PipelineHandler *handler, uint16_t pipelineTimeout ) { std::vector &attrs = std::get( this->args ).Get(); uint16_t timeout = pipelineTimeout < this->timeout ? pipelineTimeout : this->timeout; return this->file->DelXAttr( attrs, handler, timeout ); } }; //---------------------------------------------------------------------------- //! Factory for creating DelXAttrBulkImpl objects (as there is another DelXAttr //! in FileSystem there would be a clash of typenames). //---------------------------------------------------------------------------- inline DelXAttrBulkImpl DelXAttr( Ctx file, Arg> attrs ) { return DelXAttrBulkImpl( std::move( file ), std::move( attrs ) ); } //---------------------------------------------------------------------------- //! ListXAttr bulk operation (@see FileOperation) //---------------------------------------------------------------------------- template class ListXAttrImpl: public FileOperation>> { public: //------------------------------------------------------------------------ //! Inherit constructors from FileOperation (@see FileOperation) //------------------------------------------------------------------------ using FileOperation>>::FileOperation; //------------------------------------------------------------------------ //! @return : name of the operation (@see Operation) //------------------------------------------------------------------------ std::string ToString() { return "ListXAttrImpl"; } protected: //------------------------------------------------------------------------ //! RunImpl operation (@see Operation) //! //! @param params : container with parameters forwarded from //! previous operation //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus RunImpl( PipelineHandler *handler, uint16_t pipelineTimeout ) { uint16_t timeout = pipelineTimeout < this->timeout ? pipelineTimeout : this->timeout; return this->file->ListXAttr( handler, timeout ); } }; //---------------------------------------------------------------------------- //! Factory for creating ListXAttrImpl objects (as there is another ListXAttr //! in FileSystem there would be a clash of typenames). //---------------------------------------------------------------------------- inline ListXAttrImpl ListXAttr( Ctx file ) { return ListXAttrImpl( std::move( file ) ); } } #endif // __XRD_CL_FILE_OPERATIONS_HH__ xrootd-5.6.9/src/XrdCl/XrdClFileStateHandler.cc000066400000000000000000004067071457266313600213320ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2014 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #include "XrdCl/XrdClFileStateHandler.hh" #include "XrdCl/XrdClURL.hh" #include "XrdCl/XrdClLog.hh" #include "XrdCl/XrdClStatus.hh" #include "XrdCl/XrdClDefaultEnv.hh" #include "XrdCl/XrdClForkHandler.hh" #include "XrdCl/XrdClConstants.hh" #include "XrdCl/XrdClMessageUtils.hh" #include "XrdCl/XrdClXRootDTransport.hh" #include "XrdCl/XrdClXRootDResponses.hh" #include "XrdCl/XrdClMonitor.hh" #include "XrdCl/XrdClFileTimer.hh" #include "XrdCl/XrdClResponseJob.hh" #include "XrdCl/XrdClJobManager.hh" #include "XrdCl/XrdClRedirectorRegistry.hh" #include "XrdCl/XrdClAnyObject.hh" #include "XrdCl/XrdClUtils.hh" #ifdef WITH_XRDEC #include "XrdCl/XrdClEcHandler.hh" #endif #include "XrdOuc/XrdOucCRC.hh" #include "XrdOuc/XrdOucPgrwUtils.hh" #include "XrdSys/XrdSysKernelBuffer.hh" #include "XrdSys/XrdSysPageSize.hh" #include "XrdSys/XrdSysPthread.hh" #include #include #include #include #include #include namespace { //---------------------------------------------------------------------------- // Helper callback for handling PgRead responses //---------------------------------------------------------------------------- class PgReadHandler : public XrdCl::ResponseHandler { friend class PgReadRetryHandler; public: //------------------------------------------------------------------------ // Constructor //------------------------------------------------------------------------ PgReadHandler( std::shared_ptr &stateHandler, XrdCl::ResponseHandler *userHandler, uint64_t orgOffset ) : stateHandler( stateHandler ), userHandler( userHandler ), orgOffset( orgOffset ), maincall( true ), retrycnt( 0 ), nbrepair( 0 ) { } //------------------------------------------------------------------------ // Handle the response //------------------------------------------------------------------------ void HandleResponseWithHosts( XrdCl::XRootDStatus *status, XrdCl::AnyObject *response, XrdCl::HostList *hostList ) { using namespace XrdCl; std::unique_lock lck( mtx ); if( !maincall ) { //-------------------------------------------------------------------- // We are serving PgRead retry request //-------------------------------------------------------------------- --retrycnt; if( !status->IsOK() ) st.reset( status ); else { delete status; // by convention other args are null (see PgReadRetryHandler) ++nbrepair; // update number of repaired pages } if( retrycnt == 0 ) { //------------------------------------------------------------------ // All retries came back //------------------------------------------------------------------ if( st->IsOK() ) { PageInfo &pginf = XrdCl::To( *resp ); pginf.SetNbRepair( nbrepair ); userHandler->HandleResponseWithHosts( st.release(), resp.release(), hosts.release() ); } else userHandler->HandleResponseWithHosts( st.release(), 0, 0 ); lck.unlock(); delete this; } return; } //---------------------------------------------------------------------- // We are serving main PgRead request //---------------------------------------------------------------------- if( !status->IsOK() ) { //-------------------------------------------------------------------- // The main PgRead request has failed //-------------------------------------------------------------------- userHandler->HandleResponseWithHosts( status, response, hostList ); lck.unlock(); delete this; return; } maincall = false; //---------------------------------------------------------------------- // Do the integrity check //---------------------------------------------------------------------- PageInfo *pginf = 0; response->Get( pginf ); uint64_t pgoff = pginf->GetOffset(); uint32_t bytesRead = pginf->GetLength(); std::vector &cksums = pginf->GetCksums(); char *buffer = reinterpret_cast( pginf->GetBuffer() ); size_t nbpages = XrdOucPgrwUtils::csNum( pgoff, bytesRead ); uint32_t pgsize = XrdSys::PageSize - pgoff % XrdSys::PageSize; if( pgsize > bytesRead ) pgsize = bytesRead; for( size_t pgnb = 0; pgnb < nbpages; ++pgnb ) { uint32_t crcval = XrdOucCRC::Calc32C( buffer, pgsize ); if( crcval != cksums[pgnb] ) { Log *log = DefaultEnv::GetLog(); log->Info( FileMsg, "[0x%x@%s] Received corrupted page, will retry page #%d.", this, stateHandler->pFileUrl->GetURL().c_str(), pgnb ); XRootDStatus st = XrdCl::FileStateHandler::PgReadRetry( stateHandler, pgoff, pgsize, pgnb, buffer, this, 0 ); if( !st.IsOK()) { *status = st; // the reason for this failure break; } ++retrycnt; // update the retry counter } bytesRead -= pgsize; buffer += pgsize; pgoff += pgsize; pgsize = XrdSys::PageSize; if( pgsize > bytesRead ) pgsize = bytesRead; } if( retrycnt == 0 ) { //-------------------------------------------------------------------- // All went well! //-------------------------------------------------------------------- userHandler->HandleResponseWithHosts( status, response, hostList ); lck.unlock(); delete this; return; } //---------------------------------------------------------------------- // We have to wait for retries! //---------------------------------------------------------------------- resp.reset( response ); hosts.reset( hostList ); st.reset( status ); } void UpdateCksum( size_t pgnb, uint32_t crcval ) { if( resp ) { XrdCl::PageInfo *pginf = 0; resp->Get( pginf ); pginf->GetCksums()[pgnb] = crcval; } } private: std::shared_ptr stateHandler; XrdCl::ResponseHandler *userHandler; uint64_t orgOffset; std::unique_ptr resp; std::unique_ptr hosts; std::unique_ptr st; std::mutex mtx; bool maincall; size_t retrycnt; size_t nbrepair; }; //---------------------------------------------------------------------------- // Helper callback for handling PgRead retries //---------------------------------------------------------------------------- class PgReadRetryHandler : public XrdCl::ResponseHandler { public: PgReadRetryHandler( PgReadHandler *pgReadHandler, size_t pgnb ) : pgReadHandler( pgReadHandler ), pgnb( pgnb ) { } //------------------------------------------------------------------------ // Handle the response //------------------------------------------------------------------------ void HandleResponseWithHosts( XrdCl::XRootDStatus *status, XrdCl::AnyObject *response, XrdCl::HostList *hostList ) { using namespace XrdCl; if( !status->IsOK() ) { Log *log = DefaultEnv::GetLog(); log->Info( FileMsg, "[0x%x@%s] Failed to recover page #%d.", this, pgReadHandler->stateHandler->pFileUrl->GetURL().c_str(), pgnb ); pgReadHandler->HandleResponseWithHosts( status, response, hostList ); delete this; return; } XrdCl::PageInfo *pginf = 0; response->Get( pginf ); if( pginf->GetLength() > (uint32_t)XrdSys::PageSize || pginf->GetCksums().size() != 1 ) { Log *log = DefaultEnv::GetLog(); log->Info( FileMsg, "[0x%x@%s] Failed to recover page #%d.", this, pgReadHandler->stateHandler->pFileUrl->GetURL().c_str(), pgnb ); // we retry a page at a time so the length cannot exceed 4KB DeleteArgs( status, response, hostList ); pgReadHandler->HandleResponseWithHosts( new XRootDStatus( stError, errDataError ), 0, 0 ); delete this; return; } uint32_t crcval = XrdOucCRC::Calc32C( pginf->GetBuffer(), pginf->GetLength() ); if( crcval != pginf->GetCksums().front() ) { Log *log = DefaultEnv::GetLog(); log->Info( FileMsg, "[0x%x@%s] Failed to recover page #%d.", this, pgReadHandler->stateHandler->pFileUrl->GetURL().c_str(), pgnb ); DeleteArgs( status, response, hostList ); pgReadHandler->HandleResponseWithHosts( new XRootDStatus( stError, errDataError ), 0, 0 ); delete this; return; } Log *log = DefaultEnv::GetLog(); log->Info( FileMsg, "[0x%x@%s] Successfully recovered page #%d.", this, pgReadHandler->stateHandler->pFileUrl->GetURL().c_str(), pgnb ); DeleteArgs( 0, response, hostList ); pgReadHandler->UpdateCksum( pgnb, crcval ); pgReadHandler->HandleResponseWithHosts( status, 0, 0 ); delete this; } private: inline void DeleteArgs( XrdCl::XRootDStatus *status, XrdCl::AnyObject *response, XrdCl::HostList *hostList ) { delete status; delete response; delete hostList; } PgReadHandler *pgReadHandler; size_t pgnb; }; //---------------------------------------------------------------------------- // Handle PgRead substitution with ordinary Read //---------------------------------------------------------------------------- class PgReadSubstitutionHandler : public XrdCl::ResponseHandler { public: //------------------------------------------------------------------------ // Constructor //------------------------------------------------------------------------ PgReadSubstitutionHandler( std::shared_ptr &stateHandler, XrdCl::ResponseHandler *userHandler ) : stateHandler( stateHandler ), userHandler( userHandler ) { } //------------------------------------------------------------------------ // Handle the response //------------------------------------------------------------------------ void HandleResponseWithHosts( XrdCl::XRootDStatus *status, XrdCl::AnyObject *rdresp, XrdCl::HostList *hostList ) { if( !status->IsOK() ) { userHandler->HandleResponseWithHosts( status, rdresp, hostList ); delete this; return; } using namespace XrdCl; ChunkInfo *chunk = 0; rdresp->Get( chunk ); std::vector cksums; if( stateHandler->pIsChannelEncrypted ) { size_t nbpages = chunk->length / XrdSys::PageSize; if( chunk->length % XrdSys::PageSize ) ++nbpages; cksums.reserve( nbpages ); size_t size = chunk->length; char *buffer = reinterpret_cast( chunk->buffer ); for( size_t pg = 0; pg < nbpages; ++pg ) { size_t pgsize = XrdSys::PageSize; if( pgsize > size ) pgsize = size; uint32_t crcval = XrdOucCRC::Calc32C( buffer, pgsize ); cksums.push_back( crcval ); buffer += pgsize; size -= pgsize; } } PageInfo *pages = new PageInfo( chunk->offset, chunk->length, chunk->buffer, std::move( cksums ) ); delete rdresp; AnyObject *response = new AnyObject(); response->Set( pages ); userHandler->HandleResponseWithHosts( status, response, hostList ); delete this; } private: std::shared_ptr stateHandler; XrdCl::ResponseHandler *userHandler; }; //---------------------------------------------------------------------------- // Object that does things to the FileStateHandler when kXR_open returns // and then calls the user handler //---------------------------------------------------------------------------- class OpenHandler: public XrdCl::ResponseHandler { public: //------------------------------------------------------------------------ // Constructor //------------------------------------------------------------------------ OpenHandler( std::shared_ptr &stateHandler, XrdCl::ResponseHandler *userHandler ): pStateHandler( stateHandler ), pUserHandler( userHandler ) { } //------------------------------------------------------------------------ // Handle the response //------------------------------------------------------------------------ virtual void HandleResponseWithHosts( XrdCl::XRootDStatus *status, XrdCl::AnyObject *response, XrdCl::HostList *hostList ) { using namespace XrdCl; //---------------------------------------------------------------------- // Extract the statistics info //---------------------------------------------------------------------- OpenInfo *openInfo = 0; if( status->IsOK() ) response->Get( openInfo ); #ifdef WITH_XRDEC else //-------------------------------------------------------------------- // Handle EC redirect //-------------------------------------------------------------------- if( status->code == errRedirect ) { std::string ecurl = status->GetErrorMessage(); EcHandler *ecHandler = GetEcHandler( hostList->front().url, ecurl ); if( ecHandler ) { pStateHandler->pPlugin = ecHandler; // set the plugin for the File object ecHandler->Open( pStateHandler->pOpenFlags, pUserHandler, 0/*TODO figure out right value for the timeout*/ ); return; } } #endif //---------------------------------------------------------------------- // Notify the state handler and the client and say bye bye //---------------------------------------------------------------------- pStateHandler->OnOpen( status, openInfo, hostList ); delete response; if( pUserHandler ) pUserHandler->HandleResponseWithHosts( status, 0, hostList ); else { delete status; delete hostList; } delete this; } private: std::shared_ptr pStateHandler; XrdCl::ResponseHandler *pUserHandler; }; //---------------------------------------------------------------------------- // Object that does things to the FileStateHandler when kXR_close returns // and then calls the user handler //---------------------------------------------------------------------------- class CloseHandler: public XrdCl::ResponseHandler { public: //------------------------------------------------------------------------ // Constructor //------------------------------------------------------------------------ CloseHandler( std::shared_ptr &stateHandler, XrdCl::ResponseHandler *userHandler, XrdCl::Message *message ): pStateHandler( stateHandler ), pUserHandler( userHandler ), pMessage( message ) { } //------------------------------------------------------------------------ //! Destructor //------------------------------------------------------------------------ virtual ~CloseHandler() { delete pMessage; } //------------------------------------------------------------------------ // Handle the response //------------------------------------------------------------------------ virtual void HandleResponseWithHosts( XrdCl::XRootDStatus *status, XrdCl::AnyObject *response, XrdCl::HostList *hostList ) { pStateHandler->OnClose( status ); if( pUserHandler ) pUserHandler->HandleResponseWithHosts( status, response, hostList ); else { delete response; delete status; delete hostList; } delete this; } private: std::shared_ptr pStateHandler; XrdCl::ResponseHandler *pUserHandler; XrdCl::Message *pMessage; }; //---------------------------------------------------------------------------- // Stateful message handler //---------------------------------------------------------------------------- class StatefulHandler: public XrdCl::ResponseHandler { public: //------------------------------------------------------------------------ // Constructor //------------------------------------------------------------------------ StatefulHandler( std::shared_ptr &stateHandler, XrdCl::ResponseHandler *userHandler, XrdCl::Message *message, const XrdCl::MessageSendParams &sendParams ): pStateHandler( stateHandler ), pUserHandler( userHandler ), pMessage( message ), pSendParams( sendParams ) { } //------------------------------------------------------------------------ // Destructor //------------------------------------------------------------------------ virtual ~StatefulHandler() { delete pMessage; delete pSendParams.chunkList; delete pSendParams.kbuff; } //------------------------------------------------------------------------ // Handle the response //------------------------------------------------------------------------ virtual void HandleResponseWithHosts( XrdCl::XRootDStatus *status, XrdCl::AnyObject *response, XrdCl::HostList *hostList ) { using namespace XrdCl; std::unique_ptr responsePtr( response ); pSendParams.hostList = hostList; //---------------------------------------------------------------------- // Houston we have a problem... //---------------------------------------------------------------------- if( !status->IsOK() ) { XrdCl::FileStateHandler::OnStateError( pStateHandler, status, pMessage, this, pSendParams ); return; } //---------------------------------------------------------------------- // We're clear //---------------------------------------------------------------------- responsePtr.release(); XrdCl::FileStateHandler::OnStateResponse( pStateHandler, status, pMessage, response, hostList ); if( pUserHandler ) pUserHandler->HandleResponseWithHosts( status, response, hostList ); else { delete status, delete response; delete hostList; } delete this; } //------------------------------------------------------------------------ //! Get the user handler //------------------------------------------------------------------------ XrdCl::ResponseHandler *GetUserHandler() { return pUserHandler; } private: std::shared_ptr pStateHandler; XrdCl::ResponseHandler *pUserHandler; XrdCl::Message *pMessage; XrdCl::MessageSendParams pSendParams; }; //---------------------------------------------------------------------------- // Release-buffer Handler //---------------------------------------------------------------------------- class ReleaseBufferHandler: public XrdCl::ResponseHandler { public: //------------------------------------------------------------------------ // Constructor //------------------------------------------------------------------------ ReleaseBufferHandler( XrdCl::Buffer &&buffer, XrdCl::ResponseHandler *handler ) : buffer( std::move( buffer ) ), handler( handler ) { } //------------------------------------------------------------------------ // Handle the response //------------------------------------------------------------------------ virtual void HandleResponseWithHosts( XrdCl::XRootDStatus *status, XrdCl::AnyObject *response, XrdCl::HostList *hostList ) { handler->HandleResponseWithHosts( status, response, hostList ); } //------------------------------------------------------------------------ // Get the underlying buffer //------------------------------------------------------------------------ XrdCl::Buffer& GetBuffer() { return buffer; } private: XrdCl::Buffer buffer; XrdCl::ResponseHandler *handler; }; } namespace XrdCl { //---------------------------------------------------------------------------- // Constructor //---------------------------------------------------------------------------- FileStateHandler::FileStateHandler( FilePlugIn *& plugin ): pFileState( Closed ), pStatInfo( 0 ), pFileUrl( 0 ), pDataServer( 0 ), pLoadBalancer( 0 ), pStateRedirect( 0 ), pWrtRecoveryRedir( 0 ), pFileHandle( 0 ), pOpenMode( 0 ), pOpenFlags( 0 ), pSessionId( 0 ), pDoRecoverRead( true ), pDoRecoverWrite( true ), pFollowRedirects( true ), pUseVirtRedirector( true ), pIsChannelEncrypted( false ), pAllowBundledClose( false ), pPlugin( plugin ) { pFileHandle = new uint8_t[4]; ResetMonitoringVars(); DefaultEnv::GetForkHandler()->RegisterFileObject( this ); DefaultEnv::GetFileTimer()->RegisterFileObject( this ); pLFileHandler = new LocalFileHandler(); } //------------------------------------------------------------------------ //! Constructor //! //! @param useVirtRedirector if true Metalink files will be treated //! as a VirtualRedirectors //------------------------------------------------------------------------ FileStateHandler::FileStateHandler( bool useVirtRedirector, FilePlugIn *& plugin ): pFileState( Closed ), pStatInfo( 0 ), pFileUrl( 0 ), pDataServer( 0 ), pLoadBalancer( 0 ), pStateRedirect( 0 ), pWrtRecoveryRedir( 0 ), pFileHandle( 0 ), pOpenMode( 0 ), pOpenFlags( 0 ), pSessionId( 0 ), pDoRecoverRead( true ), pDoRecoverWrite( true ), pFollowRedirects( true ), pUseVirtRedirector( useVirtRedirector ), pAllowBundledClose( false ), pPlugin( plugin ) { pFileHandle = new uint8_t[4]; ResetMonitoringVars(); DefaultEnv::GetForkHandler()->RegisterFileObject( this ); DefaultEnv::GetFileTimer()->RegisterFileObject( this ); pLFileHandler = new LocalFileHandler(); } //---------------------------------------------------------------------------- // Destructor //---------------------------------------------------------------------------- FileStateHandler::~FileStateHandler() { //-------------------------------------------------------------------------- // This, in principle, should never ever happen. Except for the case // when we're interfaced with ROOT that may call this desctructor from // its garbage collector, from its __cxa_finalize, ie. after the XrdCl lib // has been finalized by the linker. So, if we don't have the log object // at this point we just give up the hope. //-------------------------------------------------------------------------- if( DefaultEnv::GetLog() && pSessionId && !pDataServer->IsLocalFile() ) // if the file object was bound to a physical connection DefaultEnv::GetPostMaster()->DecFileInstCnt( *pDataServer ); if( DefaultEnv::GetFileTimer() ) DefaultEnv::GetFileTimer()->UnRegisterFileObject( this ); if( DefaultEnv::GetForkHandler() ) DefaultEnv::GetForkHandler()->UnRegisterFileObject( this ); if( pFileState != Closed && DefaultEnv::GetLog() ) { XRootDStatus st; MonitorClose( &st ); ResetMonitoringVars(); } // check if the logger is still there, this is only for root, as root might // have unload us already so in this case we don't want to do anything if( DefaultEnv::GetLog() && pUseVirtRedirector && pFileUrl && pFileUrl->IsMetalink() ) { RedirectorRegistry& registry = RedirectorRegistry::Instance(); registry.Release( *pFileUrl ); } delete pStatInfo; delete pFileUrl; delete pDataServer; delete pLoadBalancer; delete [] pFileHandle; delete pLFileHandler; } //---------------------------------------------------------------------------- // Open the file pointed to by the given URL //---------------------------------------------------------------------------- XRootDStatus FileStateHandler::Open( std::shared_ptr &self, const std::string &url, uint16_t flags, uint16_t mode, ResponseHandler *handler, uint16_t timeout ) { XrdSysMutexHelper scopedLock( self->pMutex ); //-------------------------------------------------------------------------- // Check if we can proceed //-------------------------------------------------------------------------- if( self->pFileState == Error ) return self->pStatus; if( self->pFileState == OpenInProgress ) return XRootDStatus( stError, errInProgress ); if( self->pFileState == CloseInProgress || self->pFileState == Opened || self->pFileState == Recovering ) return XRootDStatus( stError, errInvalidOp ); self->pFileState = OpenInProgress; //-------------------------------------------------------------------------- // Check if the parameters are valid //-------------------------------------------------------------------------- Log *log = DefaultEnv::GetLog(); if( self->pFileUrl ) { if( self->pUseVirtRedirector && self->pFileUrl->IsMetalink() ) { RedirectorRegistry& registry = RedirectorRegistry::Instance(); registry.Release( *self->pFileUrl ); } delete self->pFileUrl; self->pFileUrl = 0; } self->pFileUrl = new URL( url ); //-------------------------------------------------------------------------- // Add unique uuid to each open request so replays due to error/timeout // recovery can be correctly handled. //-------------------------------------------------------------------------- URL::ParamsMap cgi = self->pFileUrl->GetParams(); uuid_t uuid; char requuid[37]= {0}; uuid_generate( uuid ); uuid_unparse( uuid, requuid ); cgi["xrdcl.requuid"] = requuid; self->pFileUrl->SetParams( cgi ); if( !self->pFileUrl->IsValid() ) { log->Error( FileMsg, "[0x%x@%s] Trying to open invalid url: %s", self.get(), self->pFileUrl->GetPath().c_str(), url.c_str() ); self->pStatus = XRootDStatus( stError, errInvalidArgs ); self->pFileState = Closed; return self->pStatus; } //-------------------------------------------------------------------------- // Check if the recovery procedures should be enabled //-------------------------------------------------------------------------- const URL::ParamsMap &urlParams = self->pFileUrl->GetParams(); URL::ParamsMap::const_iterator it; it = urlParams.find( "xrdcl.recover-reads" ); if( (it != urlParams.end() && it->second == "false") || !self->pDoRecoverRead ) { self->pDoRecoverRead = false; log->Debug( FileMsg, "[0x%x@%s] Read recovery procedures are disabled", self.get(), self->pFileUrl->GetURL().c_str() ); } it = urlParams.find( "xrdcl.recover-writes" ); if( (it != urlParams.end() && it->second == "false") || !self->pDoRecoverWrite ) { self->pDoRecoverWrite = false; log->Debug( FileMsg, "[0x%x@%s] Write recovery procedures are disabled", self.get(), self->pFileUrl->GetURL().c_str() ); } //-------------------------------------------------------------------------- // Open the file //-------------------------------------------------------------------------- log->Debug( FileMsg, "[0x%x@%s] Sending an open command", self.get(), self->pFileUrl->GetURL().c_str() ); self->pOpenMode = mode; self->pOpenFlags = flags; OpenHandler *openHandler = new OpenHandler( self, handler ); Message *msg; ClientOpenRequest *req; std::string path = self->pFileUrl->GetPathWithFilteredParams(); MessageUtils::CreateRequest( msg, req, path.length() ); req->requestid = kXR_open; req->mode = mode; req->options = flags | kXR_async | kXR_retstat; req->dlen = path.length(); msg->Append( path.c_str(), path.length(), 24 ); XRootDTransport::SetDescription( msg ); MessageSendParams params; params.timeout = timeout; params.followRedirects = self->pFollowRedirects; MessageUtils::ProcessSendParams( params ); XRootDStatus st = self->IssueRequest( *self->pFileUrl, msg, openHandler, params ); if( !st.IsOK() ) { delete openHandler; self->pStatus = st; self->pFileState = Closed; return st; } return st; } //---------------------------------------------------------------------------- // Close the file object //---------------------------------------------------------------------------- XRootDStatus FileStateHandler::Close( std::shared_ptr &self, ResponseHandler *handler, uint16_t timeout ) { XrdSysMutexHelper scopedLock( self->pMutex ); //-------------------------------------------------------------------------- // Check if we can proceed //-------------------------------------------------------------------------- if( self->pFileState == Error ) return self->pStatus; if( self->pFileState == CloseInProgress ) return XRootDStatus( stError, errInProgress ); if( self->pFileState == Closed ) return XRootDStatus( stOK, suAlreadyDone ); if( self->pFileState == OpenInProgress || self->pFileState == Recovering ) return XRootDStatus( stError, errInvalidOp ); if( !self->pAllowBundledClose && !self->pInTheFly.empty() ) return XRootDStatus( stError, errInvalidOp ); self->pFileState = CloseInProgress; Log *log = DefaultEnv::GetLog(); log->Debug( FileMsg, "[0x%x@%s] Sending a close command for handle 0x%x to " "%s", self.get(), self->pFileUrl->GetURL().c_str(), *((uint32_t*)self->pFileHandle), self->pDataServer->GetHostId().c_str() ); //-------------------------------------------------------------------------- // Close the file //-------------------------------------------------------------------------- Message *msg; ClientCloseRequest *req; MessageUtils::CreateRequest( msg, req ); req->requestid = kXR_close; memcpy( req->fhandle, self->pFileHandle, 4 ); XRootDTransport::SetDescription( msg ); msg->SetSessionId( self->pSessionId ); CloseHandler *closeHandler = new CloseHandler( self, handler, msg ); MessageSendParams params; params.timeout = timeout; params.followRedirects = false; params.stateful = true; MessageUtils::ProcessSendParams( params ); XRootDStatus st = self->IssueRequest( *self->pDataServer, msg, closeHandler, params ); if( !st.IsOK() ) { // an invalid-session error means the connection to the server has been // closed, which in turn means that the server closed the file already if( st.code == errInvalidSession || st.code == errSocketDisconnected || st.code == errConnectionError || st.code == errSocketOptError || st.code == errPollerError || st.code == errSocketError ) { self->pFileState = Closed; ResponseJob *job = new ResponseJob( closeHandler, new XRootDStatus(), nullptr, nullptr ); DefaultEnv::GetPostMaster()->GetJobManager()->QueueJob( job ); return XRootDStatus(); } delete closeHandler; self->pStatus = st; self->pFileState = Error; return st; } return st; } //---------------------------------------------------------------------------- // Stat the file //---------------------------------------------------------------------------- XRootDStatus FileStateHandler::Stat( std::shared_ptr &self, bool force, ResponseHandler *handler, uint16_t timeout ) { XrdSysMutexHelper scopedLock( self->pMutex ); if( self->pFileState == Error ) return self->pStatus; if( self->pFileState != Opened && self->pFileState != Recovering ) return XRootDStatus( stError, errInvalidOp ); //-------------------------------------------------------------------------- // Return the cached info //-------------------------------------------------------------------------- if( !force ) { AnyObject *obj = new AnyObject(); obj->Set( new StatInfo( *self->pStatInfo ) ); handler->HandleResponseWithHosts( new XRootDStatus(), obj, new HostList() ); return XRootDStatus(); } Log *log = DefaultEnv::GetLog(); log->Debug( FileMsg, "[0x%x@%s] Sending a stat command for handle 0x%x to " "%s", self.get(), self->pFileUrl->GetURL().c_str(), *((uint32_t*)self->pFileHandle), self->pDataServer->GetHostId().c_str() ); //-------------------------------------------------------------------------- // Issue a new stat request // stating a file handle doesn't work (fixed in 3.2.0) so we need to // stat the pat //-------------------------------------------------------------------------- Message *msg; ClientStatRequest *req; std::string path = self->pFileUrl->GetPath(); MessageUtils::CreateRequest( msg, req ); req->requestid = kXR_stat; memcpy( req->fhandle, self->pFileHandle, 4 ); MessageSendParams params; params.timeout = timeout; params.followRedirects = false; params.stateful = true; MessageUtils::ProcessSendParams( params ); XRootDTransport::SetDescription( msg ); StatefulHandler *stHandler = new StatefulHandler( self, handler, msg, params ); return SendOrQueue( self, *self->pDataServer, msg, stHandler, params ); } //---------------------------------------------------------------------------- // Read a data chunk at a given offset - sync //---------------------------------------------------------------------------- XRootDStatus FileStateHandler::Read( std::shared_ptr &self, uint64_t offset, uint32_t size, void *buffer, ResponseHandler *handler, uint16_t timeout ) { XrdSysMutexHelper scopedLock( self->pMutex ); if( self->pFileState == Error ) return self->pStatus; if( self->pFileState != Opened && self->pFileState != Recovering ) return XRootDStatus( stError, errInvalidOp ); Log *log = DefaultEnv::GetLog(); log->Debug( FileMsg, "[0x%x@%s] Sending a read command for handle 0x%x to " "%s", self.get(), self->pFileUrl->GetURL().c_str(), *((uint32_t*)self->pFileHandle), self->pDataServer->GetHostId().c_str() ); Message *msg; ClientReadRequest *req; MessageUtils::CreateRequest( msg, req ); req->requestid = kXR_read; req->offset = offset; req->rlen = size; memcpy( req->fhandle, self->pFileHandle, 4 ); ChunkList *list = new ChunkList(); list->push_back( ChunkInfo( offset, size, buffer ) ); XRootDTransport::SetDescription( msg ); MessageSendParams params; params.timeout = timeout; params.followRedirects = false; params.stateful = true; params.chunkList = list; MessageUtils::ProcessSendParams( params ); StatefulHandler *stHandler = new StatefulHandler( self, handler, msg, params ); return SendOrQueue( self, *self->pDataServer, msg, stHandler, params ); } //------------------------------------------------------------------------ // Read data pages at a given offset //------------------------------------------------------------------------ XRootDStatus FileStateHandler::PgRead( std::shared_ptr &self, uint64_t offset, uint32_t size, void *buffer, ResponseHandler *handler, uint16_t timeout ) { int issupported = true; AnyObject obj; XRootDStatus st1 = DefaultEnv::GetPostMaster()->QueryTransport( *self->pDataServer, XRootDQuery::ServerFlags, obj ); int protver = 0; XRootDStatus st2 = Utils::GetProtocolVersion( *self->pDataServer, protver ); if( st1.IsOK() && st2.IsOK() ) { int *ptr = 0; obj.Get( ptr ); issupported = ( *ptr & kXR_suppgrw ) && ( protver >= kXR_PROTPGRWVERSION ); delete ptr; } else issupported = false; if( !issupported ) { DefaultEnv::GetLog()->Debug( FileMsg, "[0x%x@%s] PgRead not supported; substituting with Read.", self.get(), self->pFileUrl->GetURL().c_str() ); ResponseHandler *substitHandler = new PgReadSubstitutionHandler( self, handler ); auto st = Read( self, offset, size, buffer, substitHandler, timeout ); if( !st.IsOK() ) delete substitHandler; return st; } ResponseHandler* pgHandler = new PgReadHandler( self, handler, offset ); auto st = PgReadImpl( self, offset, size, buffer, PgReadFlags::None, pgHandler, timeout ); if( !st.IsOK() ) delete pgHandler; return st; } XRootDStatus FileStateHandler::PgReadRetry( std::shared_ptr &self, uint64_t offset, uint32_t size, size_t pgnb, void *buffer, PgReadHandler *handler, uint16_t timeout ) { if( size > (uint32_t)XrdSys::PageSize ) return XRootDStatus( stError, errInvalidArgs, EINVAL, "PgRead retry size exceeded 4KB." ); ResponseHandler *retryHandler = new PgReadRetryHandler( handler, pgnb ); XRootDStatus st = PgReadImpl( self, offset, size, buffer, PgReadFlags::Retry, retryHandler, timeout ); if( !st.IsOK() ) delete retryHandler; return st; } XRootDStatus FileStateHandler::PgReadImpl( std::shared_ptr &self, uint64_t offset, uint32_t size, void *buffer, uint16_t flags, ResponseHandler *handler, uint16_t timeout ) { XrdSysMutexHelper scopedLock( self->pMutex ); if( self->pFileState == Error ) return self->pStatus; if( self->pFileState != Opened && self->pFileState != Recovering ) return XRootDStatus( stError, errInvalidOp ); Log *log = DefaultEnv::GetLog(); log->Debug( FileMsg, "[0x%x@%s] Sending a pgread command for handle 0x%x to " "%s", self.get(), self->pFileUrl->GetURL().c_str(), *((uint32_t*)self->pFileHandle), self->pDataServer->GetHostId().c_str() ); Message *msg; ClientPgReadRequest *req; MessageUtils::CreateRequest( msg, req, sizeof( ClientPgReadReqArgs ) ); req->requestid = kXR_pgread; req->offset = offset; req->rlen = size; memcpy( req->fhandle, self->pFileHandle, 4 ); //-------------------------------------------------------------------------- // Now adjust the message size so it can hold PgRead arguments //-------------------------------------------------------------------------- req->dlen = sizeof( ClientPgReadReqArgs ); void *newBuf = msg->GetBuffer( sizeof( ClientPgReadRequest ) ); memset( newBuf, 0, sizeof( ClientPgReadReqArgs ) ); ClientPgReadReqArgs *args = reinterpret_cast( msg->GetBuffer( sizeof( ClientPgReadRequest ) ) ); args->reqflags = flags; ChunkList *list = new ChunkList(); list->push_back( ChunkInfo( offset, size, buffer ) ); XRootDTransport::SetDescription( msg ); MessageSendParams params; params.timeout = timeout; params.followRedirects = false; params.stateful = true; params.chunkList = list; MessageUtils::ProcessSendParams( params ); StatefulHandler *stHandler = new StatefulHandler( self, handler, msg, params ); return SendOrQueue( self, *self->pDataServer, msg, stHandler, params ); } //---------------------------------------------------------------------------- // Write a data chunk at a given offset - async //---------------------------------------------------------------------------- XRootDStatus FileStateHandler::Write( std::shared_ptr &self, uint64_t offset, uint32_t size, const void *buffer, ResponseHandler *handler, uint16_t timeout ) { XrdSysMutexHelper scopedLock( self->pMutex ); if( self->pFileState == Error ) return self->pStatus; if( self->pFileState != Opened && self->pFileState != Recovering ) return XRootDStatus( stError, errInvalidOp ); Log *log = DefaultEnv::GetLog(); log->Debug( FileMsg, "[0x%x@%s] Sending a write command for handle 0x%x to " "%s", self.get(), self->pFileUrl->GetURL().c_str(), *((uint32_t*)self->pFileHandle), self->pDataServer->GetHostId().c_str() ); Message *msg; ClientWriteRequest *req; MessageUtils::CreateRequest( msg, req ); req->requestid = kXR_write; req->offset = offset; req->dlen = size; memcpy( req->fhandle, self->pFileHandle, 4 ); ChunkList *list = new ChunkList(); list->push_back( ChunkInfo( 0, size, (char*)buffer ) ); MessageSendParams params; params.timeout = timeout; params.followRedirects = false; params.stateful = true; params.chunkList = list; MessageUtils::ProcessSendParams( params ); XRootDTransport::SetDescription( msg ); StatefulHandler *stHandler = new StatefulHandler( self, handler, msg, params ); return SendOrQueue( self, *self->pDataServer, msg, stHandler, params ); } //---------------------------------------------------------------------------- // Write a data chunk at a given offset //---------------------------------------------------------------------------- XRootDStatus FileStateHandler::Write( std::shared_ptr &self, uint64_t offset, Buffer &&buffer, ResponseHandler *handler, uint16_t timeout ) { //-------------------------------------------------------------------------- // If the memory is not page (4KB) aligned we cannot use the kernel buffer // so fall back to normal write //-------------------------------------------------------------------------- if( !XrdSys::KernelBuffer::IsPageAligned( buffer.GetBuffer() ) || self->pIsChannelEncrypted ) { Log *log = DefaultEnv::GetLog(); log->Info( FileMsg, "[0x%x@%s] Buffer is not page aligned (4KB), cannot " "convert it to kernel space buffer.", self.get(), self->pFileUrl->GetURL().c_str(), *((uint32_t*)self->pFileHandle) ); void *buff = buffer.GetBuffer(); uint32_t size = buffer.GetSize(); ReleaseBufferHandler *wrtHandler = new ReleaseBufferHandler( std::move( buffer ), handler ); XRootDStatus st = self->Write( self, offset, size, buff, wrtHandler, timeout ); if( !st.IsOK() ) { buffer = std::move( wrtHandler->GetBuffer() ); delete wrtHandler; } return st; } //-------------------------------------------------------------------------- // Transfer the data from user space to kernel space //-------------------------------------------------------------------------- uint32_t length = buffer.GetSize(); char *ubuff = buffer.Release(); std::unique_ptr kbuff( new XrdSys::KernelBuffer() ); ssize_t ret = XrdSys::Move( ubuff, *kbuff, length ); if( ret < 0 ) return XRootDStatus( stError, errInternal, XProtocol::mapError( errno ) ); //-------------------------------------------------------------------------- // Now create a write request and enqueue it //-------------------------------------------------------------------------- return WriteKernelBuffer( self, offset, ret, std::move( kbuff ), handler, timeout ); } //---------------------------------------------------------------------------- // Write a data from a given file descriptor at a given offset - async //---------------------------------------------------------------------------- XRootDStatus FileStateHandler::Write( std::shared_ptr &self, uint64_t offset, uint32_t size, Optional fdoff, int fd, ResponseHandler *handler, uint16_t timeout ) { //-------------------------------------------------------------------------- // Read the data from the file descriptor into a kernel buffer //-------------------------------------------------------------------------- std::unique_ptr kbuff( new XrdSys::KernelBuffer() ); ssize_t ret = fdoff ? XrdSys::Read( fd, *kbuff, size, *fdoff ) : XrdSys::Read( fd, *kbuff, size ); if( ret < 0 ) return XRootDStatus( stError, errInternal, XProtocol::mapError( errno ) ); //-------------------------------------------------------------------------- // Now create a write request and enqueue it //-------------------------------------------------------------------------- return WriteKernelBuffer( self, offset, ret, std::move( kbuff ), handler, timeout ); } //---------------------------------------------------------------------------- // Write number of pages at a given offset - async //---------------------------------------------------------------------------- XRootDStatus FileStateHandler::PgWrite( std::shared_ptr &self, uint64_t offset, uint32_t size, const void *buffer, std::vector &cksums, ResponseHandler *handler, uint16_t timeout ) { //-------------------------------------------------------------------------- // Resolve timeout value //-------------------------------------------------------------------------- if( timeout == 0 ) { int val = DefaultRequestTimeout; XrdCl::DefaultEnv::GetEnv()->GetInt( "RequestTimeout", val ); timeout = val; } //-------------------------------------------------------------------------- // Validate the digest vector size //-------------------------------------------------------------------------- if( cksums.empty() ) { const char *data = static_cast( buffer ); XrdOucPgrwUtils::csCalc( data, offset, size, cksums ); } else { size_t crc32cCnt = XrdOucPgrwUtils::csNum( offset, size ); if( crc32cCnt != cksums.size() ) return XRootDStatus( stError, errInvalidArgs, 0, "Wrong number of crc32c digests." ); } //-------------------------------------------------------------------------- // Create a context for PgWrite operation //-------------------------------------------------------------------------- struct pgwrt_t { pgwrt_t( ResponseHandler *h ) : handler( h ), status( nullptr ) { } ~pgwrt_t() { if( handler ) { // if all retries were successful no error status was set if( !status ) status = new XRootDStatus(); handler->HandleResponse( status, nullptr ); } } static size_t GetPgNb( uint64_t pgoff, uint64_t offset, uint32_t fstpglen ) { if( pgoff == offset ) return 0; // we need this if statement because we operate on unsigned integers return ( pgoff - ( offset + fstpglen ) ) / XrdSys::PageSize + 1; } inline void SetStatus( XRootDStatus* s ) { if( !status ) status = s; else delete s; } ResponseHandler *handler; XRootDStatus *status; }; auto pgwrt = std::make_shared( handler ); int fLen, lLen; XrdOucPgrwUtils::csNum( offset, size, fLen, lLen ); uint32_t fstpglen = fLen; time_t start = ::time( nullptr ); auto h = ResponseHandler::Wrap( [=]( XrdCl::XRootDStatus *s, XrdCl::AnyObject *r ) mutable { std::unique_ptr scoped( r ); // if the request failed simply pass the status to the // user handler if( !s->IsOK() ) { pgwrt->SetStatus( s ); return; // pgwrt destructor will call the handler } // also if the request was sucessful and there were no // corrupted pages pass the status to the user handler RetryInfo *inf = nullptr; r->Get( inf ); if( !inf->NeedRetry() ) { pgwrt->SetStatus( s ); return; // pgwrt destructor will call the handler } delete s; // first adjust the timeout value uint16_t elapsed = ::time( nullptr ) - start; if( elapsed >= timeout ) { pgwrt->SetStatus( new XRootDStatus( stError, errOperationExpired ) ); return; // pgwrt destructor will call the handler } else timeout -= elapsed; // retransmit the corrupted pages for( size_t i = 0; i < inf->Size(); ++i ) { auto tpl = inf->At( i ); uint64_t pgoff = std::get<0>( tpl ); uint32_t pglen = std::get<1>( tpl ); const void *pgbuf = static_cast( buffer ) + ( pgoff - offset ); uint32_t pgdigest = cksums[pgwrt_t::GetPgNb( pgoff, offset, fstpglen )]; auto h = ResponseHandler::Wrap( [=]( XrdCl::XRootDStatus *s, XrdCl::AnyObject *r ) mutable { std::unique_ptr scoped( r ); // if we failed simply set the status if( !s->IsOK() ) { pgwrt->SetStatus( s ); return; // the destructor will call the handler } delete s; // otherwise check if the data were not corrupted again RetryInfo *inf = nullptr; r->Get( inf ); if( inf->NeedRetry() ) // so we failed in the end { DefaultEnv::GetLog()->Warning( FileMsg, "[0x%x@%s] Failed retransmitting corrupted " "page: pgoff=%llu, pglen=%du, pgdigest=%du", self.get(), self->pFileUrl->GetURL().c_str(), pgoff, pglen, pgdigest ); pgwrt->SetStatus( new XRootDStatus( stError, errDataError, 0, "Failed to retransmit corrupted page" ) ); } else DefaultEnv::GetLog()->Info( FileMsg, "[0x%x@%s] Succesfuly retransmitted corrupted " "page: pgoff=%llu, pglen=%du, pgdigest=%du", self.get(), self->pFileUrl->GetURL().c_str(), pgoff, pglen, pgdigest ); } ); auto st = PgWriteRetry( self, pgoff, pglen, pgbuf, pgdigest, h, timeout ); if( !st.IsOK() ) pgwrt->SetStatus( new XRootDStatus( st ) ); DefaultEnv::GetLog()->Info( FileMsg, "[0x%x@%s] Retransmitting corrupted page: " "pgoff=%llu, pglen=%du, pgdigest=%du", self.get(), self->pFileUrl->GetURL().c_str(), pgoff, pglen, pgdigest ); } } ); auto st = PgWriteImpl( self, offset, size, buffer, cksums, 0, h, timeout ); if( !st.IsOK() ) { pgwrt->handler = nullptr; delete h; } return st; } //------------------------------------------------------------------------ // Write number of pages at a given offset - async //------------------------------------------------------------------------ XRootDStatus FileStateHandler::PgWriteRetry( std::shared_ptr &self, uint64_t offset, uint32_t size, const void *buffer, uint32_t digest, ResponseHandler *handler, uint16_t timeout ) { std::vector cksums{ digest }; return PgWriteImpl( self, offset, size, buffer, cksums, PgReadFlags::Retry, handler, timeout ); } //------------------------------------------------------------------------ // Write number of pages at a given offset - async //------------------------------------------------------------------------ XRootDStatus FileStateHandler::PgWriteImpl( std::shared_ptr &self, uint64_t offset, uint32_t size, const void *buffer, std::vector &cksums, kXR_char flags, ResponseHandler *handler, uint16_t timeout ) { XrdSysMutexHelper scopedLock( self->pMutex ); if( self->pFileState == Error ) return self->pStatus; if( self->pFileState != Opened && self->pFileState != Recovering ) return XRootDStatus( stError, errInvalidOp ); Log *log = DefaultEnv::GetLog(); log->Debug( FileMsg, "[0x%x@%s] Sending a pgwrite command for handle 0x%x to " "%s", self.get(), self->pFileUrl->GetURL().c_str(), *((uint32_t*)self->pFileHandle), self->pDataServer->GetHostId().c_str() ); //-------------------------------------------------------------------------- // Create the message //-------------------------------------------------------------------------- Message *msg; ClientPgWriteRequest *req; MessageUtils::CreateRequest( msg, req ); req->requestid = kXR_pgwrite; req->offset = offset; req->dlen = size + cksums.size() * sizeof( uint32_t ); req->reqflags = flags; memcpy( req->fhandle, self->pFileHandle, 4 ); ChunkList *list = new ChunkList(); list->push_back( ChunkInfo( offset, size, (char*)buffer ) ); MessageSendParams params; params.timeout = timeout; params.followRedirects = false; params.stateful = true; params.chunkList = list; params.crc32cDigests.swap( cksums ); MessageUtils::ProcessSendParams( params ); XRootDTransport::SetDescription( msg ); StatefulHandler *stHandler = new StatefulHandler( self, handler, msg, params ); return SendOrQueue( self, *self->pDataServer, msg, stHandler, params ); } //---------------------------------------------------------------------------- // Commit all pending disk writes - async //---------------------------------------------------------------------------- XRootDStatus FileStateHandler::Sync( std::shared_ptr &self, ResponseHandler *handler, uint16_t timeout ) { XrdSysMutexHelper scopedLock( self->pMutex ); if( self->pFileState == Error ) return self->pStatus; if( self->pFileState != Opened && self->pFileState != Recovering ) return XRootDStatus( stError, errInvalidOp ); Log *log = DefaultEnv::GetLog(); log->Debug( FileMsg, "[0x%x@%s] Sending a sync command for handle 0x%x to " "%s", self.get(), self->pFileUrl->GetURL().c_str(), *((uint32_t*)self->pFileHandle), self->pDataServer->GetHostId().c_str() ); Message *msg; ClientSyncRequest *req; MessageUtils::CreateRequest( msg, req ); req->requestid = kXR_sync; memcpy( req->fhandle, self->pFileHandle, 4 ); MessageSendParams params; params.timeout = timeout; params.followRedirects = false; params.stateful = true; MessageUtils::ProcessSendParams( params ); XRootDTransport::SetDescription( msg ); StatefulHandler *stHandler = new StatefulHandler( self, handler, msg, params ); return SendOrQueue( self, *self->pDataServer, msg, stHandler, params ); } //---------------------------------------------------------------------------- // Truncate the file to a particular size - async //---------------------------------------------------------------------------- XRootDStatus FileStateHandler::Truncate( std::shared_ptr &self, uint64_t size, ResponseHandler *handler, uint16_t timeout ) { XrdSysMutexHelper scopedLock( self->pMutex ); if( self->pFileState == Error ) return self->pStatus; if( self->pFileState != Opened && self->pFileState != Recovering ) return XRootDStatus( stError, errInvalidOp ); Log *log = DefaultEnv::GetLog(); log->Debug( FileMsg, "[0x%x@%s] Sending a truncate command for handle 0x%x to " "%s", self.get(), self->pFileUrl->GetURL().c_str(), *((uint32_t*)self->pFileHandle), self->pDataServer->GetHostId().c_str() ); Message *msg; ClientTruncateRequest *req; MessageUtils::CreateRequest( msg, req ); req->requestid = kXR_truncate; memcpy( req->fhandle, self->pFileHandle, 4 ); req->offset = size; MessageSendParams params; params.timeout = timeout; params.followRedirects = false; params.stateful = true; MessageUtils::ProcessSendParams( params ); XRootDTransport::SetDescription( msg ); StatefulHandler *stHandler = new StatefulHandler( self, handler, msg, params ); return SendOrQueue( self, *self->pDataServer, msg, stHandler, params ); } //---------------------------------------------------------------------------- // Read scattered data chunks in one operation - async //---------------------------------------------------------------------------- XRootDStatus FileStateHandler::VectorRead( std::shared_ptr &self, const ChunkList &chunks, void *buffer, ResponseHandler *handler, uint16_t timeout ) { //-------------------------------------------------------------------------- // Sanity check //-------------------------------------------------------------------------- XrdSysMutexHelper scopedLock( self->pMutex ); if( self->pFileState == Error ) return self->pStatus; if( self->pFileState != Opened && self->pFileState != Recovering ) return XRootDStatus( stError, errInvalidOp ); Log *log = DefaultEnv::GetLog(); log->Debug( FileMsg, "[0x%x@%s] Sending a vector read command for handle " "0x%x to %s", self.get(), self->pFileUrl->GetURL().c_str(), *((uint32_t*)self->pFileHandle), self->pDataServer->GetHostId().c_str() ); //-------------------------------------------------------------------------- // Build the message //-------------------------------------------------------------------------- Message *msg; ClientReadVRequest *req; MessageUtils::CreateRequest( msg, req, sizeof(readahead_list)*chunks.size() ); req->requestid = kXR_readv; req->dlen = sizeof(readahead_list)*chunks.size(); ChunkList *list = new ChunkList(); char *cursor = (char*)buffer; //-------------------------------------------------------------------------- // Copy the chunk info //-------------------------------------------------------------------------- readahead_list *dataChunk = (readahead_list*)msg->GetBuffer( 24 ); for( size_t i = 0; i < chunks.size(); ++i ) { dataChunk[i].rlen = chunks[i].length; dataChunk[i].offset = chunks[i].offset; memcpy( dataChunk[i].fhandle, self->pFileHandle, 4 ); void *chunkBuffer; if( cursor ) { chunkBuffer = cursor; cursor += chunks[i].length; } else chunkBuffer = chunks[i].buffer; list->push_back( ChunkInfo( chunks[i].offset, chunks[i].length, chunkBuffer ) ); } //-------------------------------------------------------------------------- // Send the message //-------------------------------------------------------------------------- MessageSendParams params; params.timeout = timeout; params.followRedirects = false; params.stateful = true; params.chunkList = list; MessageUtils::ProcessSendParams( params ); XRootDTransport::SetDescription( msg ); StatefulHandler *stHandler = new StatefulHandler( self, handler, msg, params ); return SendOrQueue( self, *self->pDataServer, msg, stHandler, params ); } //------------------------------------------------------------------------ // Write scattered data chunks in one operation - async //------------------------------------------------------------------------ XRootDStatus FileStateHandler::VectorWrite( std::shared_ptr &self, const ChunkList &chunks, ResponseHandler *handler, uint16_t timeout ) { //-------------------------------------------------------------------------- // Sanity check //-------------------------------------------------------------------------- XrdSysMutexHelper scopedLock( self->pMutex ); if( self->pFileState == Error ) return self->pStatus; if( self->pFileState != Opened && self->pFileState != Recovering ) return XRootDStatus( stError, errInvalidOp ); Log *log = DefaultEnv::GetLog(); log->Debug( FileMsg, "[0x%x@%s] Sending a vector write command for handle " "0x%x to %s", self.get(), self->pFileUrl->GetURL().c_str(), *((uint32_t*)self->pFileHandle), self->pDataServer->GetHostId().c_str() ); //-------------------------------------------------------------------------- // Determine the size of the payload //-------------------------------------------------------------------------- // the size of write vector uint32_t payloadSize = sizeof(XrdProto::write_list) * chunks.size(); //-------------------------------------------------------------------------- // Build the message //-------------------------------------------------------------------------- Message *msg; ClientWriteVRequest *req; MessageUtils::CreateRequest( msg, req, payloadSize ); req->requestid = kXR_writev; req->dlen = sizeof(XrdProto::write_list) * chunks.size(); ChunkList *list = new ChunkList(); //-------------------------------------------------------------------------- // Copy the chunk info //-------------------------------------------------------------------------- XrdProto::write_list *writeList = reinterpret_cast( msg->GetBuffer( 24 ) ); for( size_t i = 0; i < chunks.size(); ++i ) { writeList[i].wlen = chunks[i].length; writeList[i].offset = chunks[i].offset; memcpy( writeList[i].fhandle, self->pFileHandle, 4 ); list->push_back( ChunkInfo( chunks[i].offset, chunks[i].length, chunks[i].buffer ) ); } //-------------------------------------------------------------------------- // Send the message //-------------------------------------------------------------------------- MessageSendParams params; params.timeout = timeout; params.followRedirects = false; params.stateful = true; params.chunkList = list; MessageUtils::ProcessSendParams( params ); XRootDTransport::SetDescription( msg ); StatefulHandler *stHandler = new StatefulHandler( self, handler, msg, params ); return SendOrQueue( self, *self->pDataServer, msg, stHandler, params ); } //------------------------------------------------------------------------ // Write scattered buffers in one operation - async //------------------------------------------------------------------------ XRootDStatus FileStateHandler::WriteV( std::shared_ptr &self, uint64_t offset, const struct iovec *iov, int iovcnt, ResponseHandler *handler, uint16_t timeout ) { XrdSysMutexHelper scopedLock( self->pMutex ); if( self->pFileState == Error ) return self->pStatus; if( self->pFileState != Opened && self->pFileState != Recovering ) return XRootDStatus( stError, errInvalidOp ); Log *log = DefaultEnv::GetLog(); log->Debug( FileMsg, "[0x%x@%s] Sending a write command for handle 0x%x to " "%s", self.get(), self->pFileUrl->GetURL().c_str(), *((uint32_t*)self->pFileHandle), self->pDataServer->GetHostId().c_str() ); Message *msg; ClientWriteRequest *req; MessageUtils::CreateRequest( msg, req ); ChunkList *list = new ChunkList(); uint32_t size = 0; for( int i = 0; i < iovcnt; ++i ) { if( iov[i].iov_len == 0 ) continue; size += iov[i].iov_len; list->push_back( ChunkInfo( 0, iov[i].iov_len, (char*)iov[i].iov_base ) ); } req->requestid = kXR_write; req->offset = offset; req->dlen = size; memcpy( req->fhandle, self->pFileHandle, 4 ); MessageSendParams params; params.timeout = timeout; params.followRedirects = false; params.stateful = true; params.chunkList = list; MessageUtils::ProcessSendParams( params ); XRootDTransport::SetDescription( msg ); StatefulHandler *stHandler = new StatefulHandler( self, handler, msg, params ); return SendOrQueue( self, *self->pDataServer, msg, stHandler, params ); } //------------------------------------------------------------------------ // Read data into scattered buffers in one operation - async //------------------------------------------------------------------------ XRootDStatus FileStateHandler::ReadV( std::shared_ptr &self, uint64_t offset, struct iovec *iov, int iovcnt, ResponseHandler *handler, uint16_t timeout ) { XrdSysMutexHelper scopedLock( self->pMutex ); if( self->pFileState == Error ) return self->pStatus; if( self->pFileState != Opened && self->pFileState != Recovering ) return XRootDStatus( stError, errInvalidOp ); Log *log = DefaultEnv::GetLog(); log->Debug( FileMsg, "[0x%x@%s] Sending a read command for handle 0x%x to " "%s", self.get(), self->pFileUrl->GetURL().c_str(), *((uint32_t*)self->pFileHandle), self->pDataServer->GetHostId().c_str() ); Message *msg; ClientReadRequest *req; MessageUtils::CreateRequest( msg, req ); // calculate the total read size size_t size = std::accumulate( iov, iov + iovcnt, 0, []( size_t acc, iovec &rhs ) { return acc + rhs.iov_len; } ); req->requestid = kXR_read; req->offset = offset; req->rlen = size; msg->SetVirtReqID( kXR_virtReadv ); memcpy( req->fhandle, self->pFileHandle, 4 ); ChunkList *list = new ChunkList(); list->reserve( iovcnt ); uint64_t choff = offset; for( int i = 0; i < iovcnt; ++i ) { list->emplace_back( choff, iov[i].iov_len, iov[i].iov_base ); choff += iov[i].iov_len; } XRootDTransport::SetDescription( msg ); MessageSendParams params; params.timeout = timeout; params.followRedirects = false; params.stateful = true; params.chunkList = list; MessageUtils::ProcessSendParams( params ); StatefulHandler *stHandler = new StatefulHandler( self, handler, msg, params ); return SendOrQueue( self, *self->pDataServer, msg, stHandler, params ); } //---------------------------------------------------------------------------- // Performs a custom operation on an open file, server implementation // dependent - async //---------------------------------------------------------------------------- XRootDStatus FileStateHandler::Fcntl( std::shared_ptr &self, const Buffer &arg, ResponseHandler *handler, uint16_t timeout ) { XrdSysMutexHelper scopedLock( self->pMutex ); if( self->pFileState == Error ) return self->pStatus; if( self->pFileState != Opened && self->pFileState != Recovering ) return XRootDStatus( stError, errInvalidOp ); Log *log = DefaultEnv::GetLog(); log->Debug( FileMsg, "[0x%x@%s] Sending a fcntl command for handle 0x%x to " "%s", self.get(), self->pFileUrl->GetURL().c_str(), *((uint32_t*)self->pFileHandle), self->pDataServer->GetHostId().c_str() ); Message *msg; ClientQueryRequest *req; MessageUtils::CreateRequest( msg, req, arg.GetSize() ); req->requestid = kXR_query; req->infotype = kXR_Qopaqug; req->dlen = arg.GetSize(); memcpy( req->fhandle, self->pFileHandle, 4 ); msg->Append( arg.GetBuffer(), arg.GetSize(), 24 ); MessageSendParams params; params.timeout = timeout; params.followRedirects = false; params.stateful = true; MessageUtils::ProcessSendParams( params ); XRootDTransport::SetDescription( msg ); StatefulHandler *stHandler = new StatefulHandler( self, handler, msg, params ); return SendOrQueue( self, *self->pDataServer, msg, stHandler, params ); } //---------------------------------------------------------------------------- // Get access token to a file - async //---------------------------------------------------------------------------- XRootDStatus FileStateHandler::Visa( std::shared_ptr &self, ResponseHandler *handler, uint16_t timeout ) { XrdSysMutexHelper scopedLock( self->pMutex ); if( self->pFileState == Error ) return self->pStatus; if( self->pFileState != Opened && self->pFileState != Recovering ) return XRootDStatus( stError, errInvalidOp ); Log *log = DefaultEnv::GetLog(); log->Debug( FileMsg, "[0x%x@%s] Sending a visa command for handle 0x%x to " "%s", self.get(), self->pFileUrl->GetURL().c_str(), *((uint32_t*)self->pFileHandle), self->pDataServer->GetHostId().c_str() ); Message *msg; ClientQueryRequest *req; MessageUtils::CreateRequest( msg, req ); req->requestid = kXR_query; req->infotype = kXR_Qvisa; memcpy( req->fhandle, self->pFileHandle, 4 ); MessageSendParams params; params.timeout = timeout; params.followRedirects = false; params.stateful = true; MessageUtils::ProcessSendParams( params ); XRootDTransport::SetDescription( msg ); StatefulHandler *stHandler = new StatefulHandler( self, handler, msg, params ); return SendOrQueue( self, *self->pDataServer, msg, stHandler, params ); } //------------------------------------------------------------------------ // Set extended attributes - async //------------------------------------------------------------------------ XRootDStatus FileStateHandler::SetXAttr( std::shared_ptr &self, const std::vector &attrs, ResponseHandler *handler, uint16_t timeout ) { XrdSysMutexHelper scopedLock( self->pMutex ); if( self->pFileState == Error ) return self->pStatus; if( self->pFileState != Opened && self->pFileState != Recovering ) return XRootDStatus( stError, errInvalidOp ); Log *log = DefaultEnv::GetLog(); log->Debug( FileMsg, "[0x%x@%s] Sending a fattr set command for handle 0x%x to " "%s", self.get(), self->pFileUrl->GetURL().c_str(), *((uint32_t*)self->pFileHandle), self->pDataServer->GetHostId().c_str() ); //-------------------------------------------------------------------------- // Issue a new fattr get request //-------------------------------------------------------------------------- return XAttrOperationImpl( self, kXR_fattrSet, 0, attrs, handler, timeout ); } //------------------------------------------------------------------------ // Get extended attributes - async //------------------------------------------------------------------------ XRootDStatus FileStateHandler::GetXAttr( std::shared_ptr &self, const std::vector &attrs, ResponseHandler *handler, uint16_t timeout ) { XrdSysMutexHelper scopedLock( self->pMutex ); if( self->pFileState == Error ) return self->pStatus; if( self->pFileState != Opened && self->pFileState != Recovering ) return XRootDStatus( stError, errInvalidOp ); Log *log = DefaultEnv::GetLog(); log->Debug( FileMsg, "[0x%x@%s] Sending a fattr get command for handle 0x%x to " "%s", self.get(), self->pFileUrl->GetURL().c_str(), *((uint32_t*)self->pFileHandle), self->pDataServer->GetHostId().c_str() ); //-------------------------------------------------------------------------- // Issue a new fattr get request //-------------------------------------------------------------------------- return XAttrOperationImpl( self, kXR_fattrGet, 0, attrs, handler, timeout ); } //------------------------------------------------------------------------ // Delete extended attributes - async //------------------------------------------------------------------------ XRootDStatus FileStateHandler::DelXAttr( std::shared_ptr &self, const std::vector &attrs, ResponseHandler *handler, uint16_t timeout ) { XrdSysMutexHelper scopedLock( self->pMutex ); if( self->pFileState == Error ) return self->pStatus; if( self->pFileState != Opened && self->pFileState != Recovering ) return XRootDStatus( stError, errInvalidOp ); Log *log = DefaultEnv::GetLog(); log->Debug( FileMsg, "[0x%x@%s] Sending a fattr del command for handle 0x%x to " "%s", self.get(), self->pFileUrl->GetURL().c_str(), *((uint32_t*)self->pFileHandle), self->pDataServer->GetHostId().c_str() ); //-------------------------------------------------------------------------- // Issue a new fattr del request //-------------------------------------------------------------------------- return XAttrOperationImpl( self, kXR_fattrDel, 0, attrs, handler, timeout ); } //------------------------------------------------------------------------ // List extended attributes - async //------------------------------------------------------------------------ XRootDStatus FileStateHandler::ListXAttr( std::shared_ptr &self, ResponseHandler *handler, uint16_t timeout ) { XrdSysMutexHelper scopedLock( self->pMutex ); if( self->pFileState == Error ) return self->pStatus; if( self->pFileState != Opened && self->pFileState != Recovering ) return XRootDStatus( stError, errInvalidOp ); Log *log = DefaultEnv::GetLog(); log->Debug( FileMsg, "[0x%x@%s] Sending a fattr list command for handle 0x%x to " "%s", self.get(), self->pFileUrl->GetURL().c_str(), *((uint32_t*)self->pFileHandle), self->pDataServer->GetHostId().c_str() ); //-------------------------------------------------------------------------- // Issue a new fattr get request //-------------------------------------------------------------------------- static const std::vector nothing; return XAttrOperationImpl( self, kXR_fattrList, ClientFattrRequest::aData, nothing, handler, timeout ); } //------------------------------------------------------------------------ //! Create a checkpoint //! //! @param handler : handler to be notified when the response arrives, //! the response parameter will hold a std::vector of //! XAttr objects //! @param timeout : timeout value, if 0 the environment default will //! be used //! //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus FileStateHandler::Checkpoint( std::shared_ptr &self, kXR_char code, ResponseHandler *handler, uint16_t timeout ) { XrdSysMutexHelper scopedLock( self->pMutex ); if( self->pFileState == Error ) return self->pStatus; if( self->pFileState != Opened && self->pFileState != Recovering ) return XRootDStatus( stError, errInvalidOp ); Log *log = DefaultEnv::GetLog(); log->Debug( FileMsg, "[0x%x@%s] Sending a checkpoint command for " "handle 0x%x to %s", self.get(), self->pFileUrl->GetURL().c_str(), *((uint32_t*)self->pFileHandle), self->pDataServer->GetHostId().c_str() ); Message *msg; ClientChkPointRequest *req; MessageUtils::CreateRequest( msg, req ); req->requestid = kXR_chkpoint; req->opcode = code; memcpy( req->fhandle, self->pFileHandle, 4 ); MessageSendParams params; params.timeout = timeout; params.followRedirects = false; params.stateful = true; MessageUtils::ProcessSendParams( params ); XRootDTransport::SetDescription( msg ); StatefulHandler *stHandler = new StatefulHandler( self, handler, msg, params ); return SendOrQueue( self, *self->pDataServer, msg, stHandler, params ); } //------------------------------------------------------------------------ //! Checkpointed write - async //! //! @param offset offset from the beginning of the file //! @param size number of bytes to be written //! @param buffer a pointer to the buffer holding the data to be written //! @param handler handler to be notified when the response arrives //! @param timeout timeout value, if 0 the environment default will be //! used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus FileStateHandler::ChkptWrt( std::shared_ptr &self, uint64_t offset, uint32_t size, const void *buffer, ResponseHandler *handler, uint16_t timeout ) { XrdSysMutexHelper scopedLock( self->pMutex ); if( self->pFileState == Error ) return self->pStatus; if( self->pFileState != Opened && self->pFileState != Recovering ) return XRootDStatus( stError, errInvalidOp ); Log *log = DefaultEnv::GetLog(); log->Debug( FileMsg, "[0x%x@%s] Sending a write command for handle 0x%x to " "%s", self.get(), self->pFileUrl->GetURL().c_str(), *((uint32_t*)self->pFileHandle), self->pDataServer->GetHostId().c_str() ); Message *msg; ClientChkPointRequest *req; MessageUtils::CreateRequest( msg, req, sizeof( ClientWriteRequest ) ); req->requestid = kXR_chkpoint; req->opcode = kXR_ckpXeq; req->dlen = 24; // as specified in the protocol specification memcpy( req->fhandle, self->pFileHandle, 4 ); ClientWriteRequest *wrtreq = (ClientWriteRequest*)msg->GetBuffer( sizeof(ClientChkPointRequest) ); wrtreq->requestid = kXR_write; wrtreq->offset = offset; wrtreq->dlen = size; memcpy( wrtreq->fhandle, self->pFileHandle, 4 ); ChunkList *list = new ChunkList(); list->push_back( ChunkInfo( 0, size, (char*)buffer ) ); MessageSendParams params; params.timeout = timeout; params.followRedirects = false; params.stateful = true; params.chunkList = list; MessageUtils::ProcessSendParams( params ); XRootDTransport::SetDescription( msg ); StatefulHandler *stHandler = new StatefulHandler( self, handler, msg, params ); return SendOrQueue( self, *self->pDataServer, msg, stHandler, params ); } //------------------------------------------------------------------------ //! Write scattered buffers in one operation - async //! //! @param offset offset from the beginning of the file //! @param iov list of the buffers to be written //! @param iovcnt number of buffers //! @param handler handler to be notified when the response arrives //! @param timeout timeout value, if 0 then the environment default //! will be used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus FileStateHandler::ChkptWrtV( std::shared_ptr &self, uint64_t offset, const struct iovec *iov, int iovcnt, ResponseHandler *handler, uint16_t timeout ) { XrdSysMutexHelper scopedLock( self->pMutex ); if( self->pFileState == Error ) return self->pStatus; if( self->pFileState != Opened && self->pFileState != Recovering ) return XRootDStatus( stError, errInvalidOp ); Log *log = DefaultEnv::GetLog(); log->Debug( FileMsg, "[0x%x@%s] Sending a write command for handle 0x%x to " "%s", self.get(), self->pFileUrl->GetURL().c_str(), *((uint32_t*)self->pFileHandle), self->pDataServer->GetHostId().c_str() ); Message *msg; ClientChkPointRequest *req; MessageUtils::CreateRequest( msg, req, sizeof( ClientWriteRequest ) ); req->requestid = kXR_chkpoint; req->opcode = kXR_ckpXeq; req->dlen = 24; // as specified in the protocol specification memcpy( req->fhandle, self->pFileHandle, 4 ); ChunkList *list = new ChunkList(); uint32_t size = 0; for( int i = 0; i < iovcnt; ++i ) { if( iov[i].iov_len == 0 ) continue; size += iov[i].iov_len; list->push_back( ChunkInfo( 0, iov[i].iov_len, (char*)iov[i].iov_base ) ); } ClientWriteRequest *wrtreq = (ClientWriteRequest*)msg->GetBuffer( sizeof(ClientChkPointRequest) ); wrtreq->requestid = kXR_write; wrtreq->offset = offset; wrtreq->dlen = size; memcpy( wrtreq->fhandle, self->pFileHandle, 4 ); MessageSendParams params; params.timeout = timeout; params.followRedirects = false; params.stateful = true; params.chunkList = list; MessageUtils::ProcessSendParams( params ); XRootDTransport::SetDescription( msg ); StatefulHandler *stHandler = new StatefulHandler( self, handler, msg, params ); return SendOrQueue( self, *self->pDataServer, msg, stHandler, params ); } //---------------------------------------------------------------------------- // Check if the file is open //---------------------------------------------------------------------------- bool FileStateHandler::IsOpen() const { XrdSysMutexHelper scopedLock( pMutex ); if( pFileState == Opened || pFileState == Recovering ) return true; return false; } //---------------------------------------------------------------------------- // Set file property //---------------------------------------------------------------------------- bool FileStateHandler::SetProperty( const std::string &name, const std::string &value ) { XrdSysMutexHelper scopedLock( pMutex ); if( name == "ReadRecovery" ) { if( value == "true" ) pDoRecoverRead = true; else pDoRecoverRead = false; return true; } else if( name == "WriteRecovery" ) { if( value == "true" ) pDoRecoverWrite = true; else pDoRecoverWrite = false; return true; } else if( name == "FollowRedirects" ) { if( value == "true" ) pFollowRedirects = true; else pFollowRedirects = false; return true; } else if( name == "BundledClose" ) { if( value == "true" ) pAllowBundledClose = true; else pAllowBundledClose = false; return true; } return false; } //---------------------------------------------------------------------------- // Get file property //---------------------------------------------------------------------------- bool FileStateHandler::GetProperty( const std::string &name, std::string &value ) const { XrdSysMutexHelper scopedLock( pMutex ); if( name == "ReadRecovery" ) { if( pDoRecoverRead ) value = "true"; else value = "false"; return true; } else if( name == "WriteRecovery" ) { if( pDoRecoverWrite ) value = "true"; else value = "false"; return true; } else if( name == "FollowRedirects" ) { if( pFollowRedirects ) value = "true"; else value = "false"; return true; } else if( name == "DataServer" && pDataServer ) { value = pDataServer->GetHostId(); return true; } else if( name == "LastURL" && pDataServer ) { value = pDataServer->GetURL(); return true; } else if( name == "WrtRecoveryRedir" && pWrtRecoveryRedir ) { value = pWrtRecoveryRedir->GetHostId(); return true; } value = ""; return false; } //---------------------------------------------------------------------------- // Process the results of the opening operation //---------------------------------------------------------------------------- void FileStateHandler::OnOpen( const XRootDStatus *status, const OpenInfo *openInfo, const HostList *hostList ) { Log *log = DefaultEnv::GetLog(); XrdSysMutexHelper scopedLock( pMutex ); //-------------------------------------------------------------------------- // Assign the data server and the load balancer //-------------------------------------------------------------------------- std::string lastServer = pFileUrl->GetHostId(); if( hostList ) { delete pDataServer; delete pLoadBalancer; pLoadBalancer = 0; delete pWrtRecoveryRedir; pWrtRecoveryRedir = 0; pDataServer = new URL( hostList->back().url ); pDataServer->SetParams( pFileUrl->GetParams() ); if( !( pUseVirtRedirector && pFileUrl->IsMetalink() ) ) pDataServer->SetPath( pFileUrl->GetPath() ); lastServer = pDataServer->GetHostId(); HostList::const_iterator itC; URL::ParamsMap params = pDataServer->GetParams(); for( itC = hostList->begin(); itC != hostList->end(); ++itC ) { MessageUtils::MergeCGI( params, itC->url.GetParams(), true ); } pDataServer->SetParams( params ); HostList::const_reverse_iterator it; for( it = hostList->rbegin(); it != hostList->rend(); ++it ) if( it->loadBalancer ) { pLoadBalancer = new URL( it->url ); break; } for( it = hostList->rbegin(); it != hostList->rend(); ++it ) if( it->flags & kXR_recoverWrts ) { pWrtRecoveryRedir = new URL( it->url ); break; } } log->Debug( FileMsg, "[0x%x@%s] Open has returned with status %s", this, pFileUrl->GetURL().c_str(), status->ToStr().c_str() ); if( pDataServer && !pDataServer->IsLocalFile() ) { //------------------------------------------------------------------------ // Check if we are using a secure connection //------------------------------------------------------------------------ XrdCl::AnyObject isencobj; XrdCl::XRootDStatus st = XrdCl::DefaultEnv::GetPostMaster()-> QueryTransport( *pDataServer, XRootDQuery::IsEncrypted, isencobj ); if( st.IsOK() ) { bool *isenc; isencobj.Get( isenc ); pIsChannelEncrypted = *isenc; delete isenc; } } //-------------------------------------------------------------------------- // We have failed //-------------------------------------------------------------------------- pStatus = *status; if( !pStatus.IsOK() || !openInfo ) { log->Debug( FileMsg, "[0x%x@%s] Error while opening at %s: %s", this, pFileUrl->GetURL().c_str(), lastServer.c_str(), pStatus.ToStr().c_str() ); FailQueuedMessages( pStatus ); pFileState = Error; //------------------------------------------------------------------------ // Report to monitoring //------------------------------------------------------------------------ Monitor *mon = DefaultEnv::GetMonitor(); if( mon ) { Monitor::ErrorInfo i; i.file = pFileUrl; i.status = status; i.opCode = Monitor::ErrorInfo::ErrOpen; mon->Event( Monitor::EvErrIO, &i ); } } //-------------------------------------------------------------------------- // We have succeeded //-------------------------------------------------------------------------- else { //------------------------------------------------------------------------ // Store the response info //------------------------------------------------------------------------ openInfo->GetFileHandle( pFileHandle ); pSessionId = openInfo->GetSessionId(); if( openInfo->GetStatInfo() ) { delete pStatInfo; pStatInfo = new StatInfo( *openInfo->GetStatInfo() ); } log->Debug( FileMsg, "[0x%x@%s] successfully opened at %s, handle: 0x%x, " "session id: %ld", this, pFileUrl->GetURL().c_str(), pDataServer->GetHostId().c_str(), *((uint32_t*)pFileHandle), pSessionId ); //------------------------------------------------------------------------ // Inform the monitoring about opening success //------------------------------------------------------------------------ gettimeofday( &pOpenTime, 0 ); Monitor *mon = DefaultEnv::GetMonitor(); if( mon ) { Monitor::OpenInfo i; i.file = pFileUrl; i.dataServer = pDataServer->GetHostId(); i.oFlags = pOpenFlags; i.fSize = pStatInfo ? pStatInfo->GetSize() : 0; mon->Event( Monitor::EvOpen, &i ); } //------------------------------------------------------------------------ // Resend the queued messages if any //------------------------------------------------------------------------ ReSendQueuedMessages(); pFileState = Opened; } } //---------------------------------------------------------------------------- // Process the results of the closing operation //---------------------------------------------------------------------------- void FileStateHandler::OnClose( const XRootDStatus *status ) { Log *log = DefaultEnv::GetLog(); XrdSysMutexHelper scopedLock( pMutex ); log->Debug( FileMsg, "[0x%x@%s] Close returned from %s with: %s", this, pFileUrl->GetURL().c_str(), pDataServer->GetHostId().c_str(), status->ToStr().c_str() ); log->Dump( FileMsg, "[0x%x@%s] Items in the fly %d, queued for recovery %d", this, pFileUrl->GetURL().c_str(), pInTheFly.size(), pToBeRecovered.size() ); MonitorClose( status ); ResetMonitoringVars(); pStatus = *status; pFileState = Closed; } //---------------------------------------------------------------------------- // Handle an error while sending a stateful message //---------------------------------------------------------------------------- void FileStateHandler::OnStateError( std::shared_ptr &self, XRootDStatus *status, Message *message, ResponseHandler *userHandler, MessageSendParams &sendParams ) { //-------------------------------------------------------------------------- // It may be a redirection //-------------------------------------------------------------------------- if( !status->IsOK() && status->code == errRedirect && self->pFollowRedirects ) { static const std::string root = "root", xroot = "xroot", file = "file", roots = "roots", xroots = "xroots"; std::string msg = status->GetErrorMessage(); if( !msg.compare( 0, root.size(), root ) || !msg.compare( 0, xroot.size(), xroot ) || !msg.compare( 0, file.size(), file ) || !msg.compare( 0, roots.size(), roots ) || !msg.compare( 0, xroots.size(), xroots ) ) { FileStateHandler::OnStateRedirection( self, msg, message, userHandler, sendParams ); return; } } //-------------------------------------------------------------------------- // Handle error //-------------------------------------------------------------------------- Log *log = DefaultEnv::GetLog(); XrdSysMutexHelper scopedLock( self->pMutex ); self->pInTheFly.erase( message ); log->Dump( FileMsg, "[0x%x@%s] File state error encountered. Message %s " "returned with %s", self.get(), self->pFileUrl->GetURL().c_str(), message->GetDescription().c_str(), status->ToStr().c_str() ); //-------------------------------------------------------------------------- // Report to monitoring //-------------------------------------------------------------------------- Monitor *mon = DefaultEnv::GetMonitor(); if( mon ) { Monitor::ErrorInfo i; i.file = self->pFileUrl; i.status = status; ClientRequest *req = (ClientRequest*)message->GetBuffer(); switch( req->header.requestid ) { case kXR_read: i.opCode = Monitor::ErrorInfo::ErrRead; break; case kXR_readv: i.opCode = Monitor::ErrorInfo::ErrReadV; break; case kXR_pgread: i.opCode = Monitor::ErrorInfo::ErrRead; break; case kXR_write: i.opCode = Monitor::ErrorInfo::ErrWrite; break; case kXR_writev: i.opCode = Monitor::ErrorInfo::ErrWriteV; break; case kXR_pgwrite: i.opCode = Monitor::ErrorInfo::ErrWrite; break; default: i.opCode = Monitor::ErrorInfo::ErrUnc; } mon->Event( Monitor::EvErrIO, &i ); } //-------------------------------------------------------------------------- // The message is not recoverable // (message using a kernel buffer is not recoverable by definition) //-------------------------------------------------------------------------- if( !self->IsRecoverable( *status ) || sendParams.kbuff ) { log->Error( FileMsg, "[0x%x@%s] Fatal file state error. Message %s " "returned with %s", self.get(), self->pFileUrl->GetURL().c_str(), message->GetDescription().c_str(), status->ToStr().c_str() ); self->FailMessage( RequestData( message, userHandler, sendParams ), *status ); delete status; return; } //-------------------------------------------------------------------------- // Insert the message to the recovery queue and start the recovery // procedure if we don't have any more message in the fly //-------------------------------------------------------------------------- self->pCloseReason = *status; RecoverMessage( self, RequestData( message, userHandler, sendParams ) ); delete status; } //---------------------------------------------------------------------------- // Handle stateful redirect //---------------------------------------------------------------------------- void FileStateHandler::OnStateRedirection( std::shared_ptr &self, const std::string &redirectUrl, Message *message, ResponseHandler *userHandler, MessageSendParams &sendParams ) { XrdSysMutexHelper scopedLock( self->pMutex ); self->pInTheFly.erase( message ); //-------------------------------------------------------------------------- // Register the state redirect url and append the new cgi information to // the file URL //-------------------------------------------------------------------------- if( !self->pStateRedirect ) { std::ostringstream o; self->pStateRedirect = new URL( redirectUrl ); URL::ParamsMap params = self->pFileUrl->GetParams(); MessageUtils::MergeCGI( params, self->pStateRedirect->GetParams(), false ); self->pFileUrl->SetParams( params ); } RecoverMessage( self, RequestData( message, userHandler, sendParams ) ); } //---------------------------------------------------------------------------- // Handle stateful response //---------------------------------------------------------------------------- void FileStateHandler::OnStateResponse( std::shared_ptr &self, XRootDStatus *status, Message *message, AnyObject *response, HostList */*urlList*/ ) { Log *log = DefaultEnv::GetLog(); XrdSysMutexHelper scopedLock( self->pMutex ); log->Dump( FileMsg, "[0x%x@%s] Got state response for message %s", self.get(), self->pFileUrl->GetURL().c_str(), message->GetDescription().c_str() ); //-------------------------------------------------------------------------- // Since this message may be the last "in-the-fly" and no recovery // is done if messages are in the fly, we may need to trigger recovery //-------------------------------------------------------------------------- self->pInTheFly.erase( message ); RunRecovery( self ); //-------------------------------------------------------------------------- // Play with the actual response before returning it. This is a good // place to do caching in the future. //-------------------------------------------------------------------------- ClientRequest *req = (ClientRequest*)message->GetBuffer(); switch( req->header.requestid ) { //------------------------------------------------------------------------ // Cache the stat response //------------------------------------------------------------------------ case kXR_stat: { StatInfo *info = 0; response->Get( info ); delete self->pStatInfo; self->pStatInfo = new StatInfo( *info ); break; } //------------------------------------------------------------------------ // Handle read response //------------------------------------------------------------------------ case kXR_read: { ++self->pRCount; self->pRBytes += req->read.rlen; break; } //------------------------------------------------------------------------ // Handle read response //------------------------------------------------------------------------ case kXR_pgread: { ++self->pRCount; self->pRBytes += req->pgread.rlen; break; } //------------------------------------------------------------------------ // Handle readv response //------------------------------------------------------------------------ case kXR_readv: { ++self->pVRCount; size_t segs = req->header.dlen/sizeof(readahead_list); readahead_list *dataChunk = (readahead_list*)message->GetBuffer( 24 ); for( size_t i = 0; i < segs; ++i ) self->pVRBytes += dataChunk[i].rlen; self->pVSegs += segs; break; } //------------------------------------------------------------------------ // Handle write response //------------------------------------------------------------------------ case kXR_write: { ++self->pWCount; self->pWBytes += req->write.dlen; break; } //------------------------------------------------------------------------ // Handle write response //------------------------------------------------------------------------ case kXR_pgwrite: { ++self->pWCount; self->pWBytes += req->pgwrite.dlen; break; } //------------------------------------------------------------------------ // Handle writev response //------------------------------------------------------------------------ case kXR_writev: { ++self->pVWCount; size_t size = req->header.dlen/sizeof(readahead_list); XrdProto::write_list *wrtList = reinterpret_cast( message->GetBuffer( 24 ) ); for( size_t i = 0; i < size; ++i ) self->pVWBytes += wrtList[i].wlen; break; } }; } //------------------------------------------------------------------------ //! Tick //------------------------------------------------------------------------ void FileStateHandler::Tick( time_t now ) { if (pMutex.CondLock()) {TimeOutRequests( now ); pMutex.UnLock(); } } //---------------------------------------------------------------------------- // Declare timeout on requests being recovered //---------------------------------------------------------------------------- void FileStateHandler::TimeOutRequests( time_t now ) { if( !pToBeRecovered.empty() ) { Log *log = DefaultEnv::GetLog(); log->Dump( FileMsg, "[0x%x@%s] Got a timer event", this, pFileUrl->GetURL().c_str() ); RequestList::iterator it; JobManager *jobMan = DefaultEnv::GetPostMaster()->GetJobManager(); for( it = pToBeRecovered.begin(); it != pToBeRecovered.end(); ) { if( it->params.expires <= now ) { jobMan->QueueJob( new ResponseJob( it->handler, new XRootDStatus( stError, errOperationExpired ), 0, it->params.hostList ) ); it = pToBeRecovered.erase( it ); } else ++it; } } } //---------------------------------------------------------------------------- // Called in the child process after the fork //---------------------------------------------------------------------------- void FileStateHandler::AfterForkChild() { Log *log = DefaultEnv::GetLog(); if( pFileState == Closed || pFileState == Error ) return; if( (IsReadOnly() && pDoRecoverRead) || (!IsReadOnly() && pDoRecoverWrite) ) { log->Debug( FileMsg, "[0x%x@%s] Putting the file in recovery state in " "process %d", this, pFileUrl->GetURL().c_str(), getpid() ); pFileState = Recovering; pInTheFly.clear(); pToBeRecovered.clear(); } else pFileState = Error; } //------------------------------------------------------------------------ // Try other data server //------------------------------------------------------------------------ XRootDStatus FileStateHandler::TryOtherServer( std::shared_ptr &self, uint16_t timeout ) { XrdSysMutexHelper scopedLock( self->pMutex ); if( self->pFileState != Opened || !self->pLoadBalancer ) return XRootDStatus( stError, errInvalidOp ); self->pFileState = Recovering; Log *log = DefaultEnv::GetLog(); log->Debug( FileMsg, "[0x%x@%s] Reopen file at next data server.", self.get(), self->pFileUrl->GetURL().c_str() ); // merge CGI auto lbcgi = self->pLoadBalancer->GetParams(); auto dtcgi = self->pDataServer->GetParams(); MessageUtils::MergeCGI( lbcgi, dtcgi, false ); // update tried CGI auto itr = lbcgi.find( "tried" ); if( itr == lbcgi.end() ) lbcgi["tried"] = self->pDataServer->GetHostName(); else { std::string tried = itr->second; tried += "," + self->pDataServer->GetHostName(); lbcgi["tried"] = tried; } self->pLoadBalancer->SetParams( lbcgi ); return ReOpenFileAtServer( self, *self->pLoadBalancer, timeout ); } //------------------------------------------------------------------------ // Generic implementation of xattr operation //------------------------------------------------------------------------ template Status FileStateHandler::XAttrOperationImpl( std::shared_ptr &self, kXR_char subcode, kXR_char options, const std::vector &attrs, ResponseHandler *handler, uint16_t timeout ) { //-------------------------------------------------------------------------- // Issue a new fattr request //-------------------------------------------------------------------------- Message *msg; ClientFattrRequest *req; MessageUtils::CreateRequest( msg, req ); req->requestid = kXR_fattr; req->subcode = subcode; req->numattr = attrs.size(); req->options = options; memcpy( req->fhandle, self->pFileHandle, 4 ); XRootDStatus st = MessageUtils::CreateXAttrBody( msg, attrs ); if( !st.IsOK() ) return st; MessageSendParams params; params.timeout = timeout; params.followRedirects = false; params.stateful = true; MessageUtils::ProcessSendParams( params ); XRootDTransport::SetDescription( msg ); StatefulHandler *stHandler = new StatefulHandler( self, handler, msg, params ); return SendOrQueue( self, *self->pDataServer, msg, stHandler, params ); } //---------------------------------------------------------------------------- // Send a message to a host or put it in the recovery queue //---------------------------------------------------------------------------- Status FileStateHandler::SendOrQueue( std::shared_ptr &self, const URL &url, Message *msg, ResponseHandler *handler, MessageSendParams &sendParams ) { //-------------------------------------------------------------------------- // Recovering //-------------------------------------------------------------------------- if( self->pFileState == Recovering ) { return RecoverMessage( self, RequestData( msg, handler, sendParams ), false ); } //-------------------------------------------------------------------------- // Trying to send //-------------------------------------------------------------------------- if( self->pFileState == Opened ) { msg->SetSessionId( self->pSessionId ); XRootDStatus st = self->IssueRequest( *self->pDataServer, msg, handler, sendParams ); //------------------------------------------------------------------------ // Invalid session id means that the connection has been broken while we // were idle so we haven't been informed about this fact earlier. //------------------------------------------------------------------------ if( !st.IsOK() && st.code == errInvalidSession && self->IsRecoverable( st ) ) return RecoverMessage( self, RequestData( msg, handler, sendParams ), false ); if( st.IsOK() ) self->pInTheFly.insert(msg); else delete handler; return st; } return Status( stError, errInvalidOp ); } //---------------------------------------------------------------------------- // Check if the stateful error is recoverable //---------------------------------------------------------------------------- bool FileStateHandler::IsRecoverable( const XRootDStatus &status ) const { if( status.code == errSocketError || status.code == errInvalidSession || status.code == errTlsError || status.code == errSocketTimeout || status.code == errOperationInterrupted) { if( IsReadOnly() && !pDoRecoverRead ) return false; if( !IsReadOnly() && !pDoRecoverWrite ) return false; return true; } return false; } //---------------------------------------------------------------------------- // Check if the file is open for read only //---------------------------------------------------------------------------- bool FileStateHandler::IsReadOnly() const { if( (pOpenFlags & kXR_open_read) && !(pOpenFlags & kXR_open_updt) && !(pOpenFlags & kXR_open_apnd ) ) return true; return false; } //---------------------------------------------------------------------------- // Recover a message //---------------------------------------------------------------------------- Status FileStateHandler::RecoverMessage( std::shared_ptr &self, RequestData rd, bool callbackOnFailure ) { self->pFileState = Recovering; Log *log = DefaultEnv::GetLog(); log->Dump( FileMsg, "[0x%x@%s] Putting message %s in the recovery list", self.get(), self->pFileUrl->GetURL().c_str(), rd.request->GetDescription().c_str() ); Status st = RunRecovery( self ); if( st.IsOK() ) { self->pToBeRecovered.push_back( rd ); return st; } if( callbackOnFailure ) self->FailMessage( rd, st ); return st; } //---------------------------------------------------------------------------- // Run the recovery procedure if appropriate //---------------------------------------------------------------------------- Status FileStateHandler::RunRecovery( std::shared_ptr &self ) { if( self->pFileState != Recovering ) return Status(); if( !self->pInTheFly.empty() ) return Status(); Log *log = DefaultEnv::GetLog(); log->Debug( FileMsg, "[0x%x@%s] Running the recovery procedure", self.get(), self->pFileUrl->GetURL().c_str() ); Status st; if( self->pStateRedirect ) { SendClose( self, 0 ); st = ReOpenFileAtServer( self, *self->pStateRedirect, 0 ); delete self->pStateRedirect; self->pStateRedirect = 0; } else if( self->IsReadOnly() && self->pLoadBalancer ) st = ReOpenFileAtServer( self, *self->pLoadBalancer, 0 ); else st = ReOpenFileAtServer( self, *self->pDataServer, 0 ); if( !st.IsOK() ) { self->pFileState = Error; self->pStatus = st; self->FailQueuedMessages( st ); } return st; } //---------------------------------------------------------------------------- // Send a close and ignore the response //---------------------------------------------------------------------------- XRootDStatus FileStateHandler::SendClose( std::shared_ptr &self, uint16_t timeout ) { Message *msg; ClientCloseRequest *req; MessageUtils::CreateRequest( msg, req ); req->requestid = kXR_close; memcpy( req->fhandle, self->pFileHandle, 4 ); XRootDTransport::SetDescription( msg ); msg->SetSessionId( self->pSessionId ); ResponseHandler *handler = ResponseHandler::Wrap( [self]( XRootDStatus&, AnyObject& ) mutable { self.reset(); } ); MessageSendParams params; params.timeout = timeout; params.followRedirects = false; params.stateful = true; MessageUtils::ProcessSendParams( params ); return self->IssueRequest( *self->pDataServer, msg, handler, params ); } //---------------------------------------------------------------------------- // Re-open the current file at a given server //---------------------------------------------------------------------------- XRootDStatus FileStateHandler::ReOpenFileAtServer( std::shared_ptr &self, const URL &url, uint16_t timeout ) { Log *log = DefaultEnv::GetLog(); log->Dump( FileMsg, "[0x%x@%s] Sending a recovery open command to %s", self.get(), self->pFileUrl->GetURL().c_str(), url.GetURL().c_str() ); //-------------------------------------------------------------------------- // Remove the kXR_delete and kXR_new flags, as we don't want the recovery // procedure to delete a file that has been partially updated or fail it // because a partially uploaded file already exists. //-------------------------------------------------------------------------- if( self->pOpenFlags & kXR_delete) { self->pOpenFlags &= ~kXR_delete; self->pOpenFlags |= kXR_open_updt; } self->pOpenFlags &= ~kXR_new; Message *msg; ClientOpenRequest *req; URL u = url; if( url.GetPath().empty() ) u.SetPath( self->pFileUrl->GetPath() ); std::string path = u.GetPathWithFilteredParams(); MessageUtils::CreateRequest( msg, req, path.length() ); req->requestid = kXR_open; req->mode = self->pOpenMode; req->options = self->pOpenFlags; req->dlen = path.length(); msg->Append( path.c_str(), path.length(), 24 ); // create a new reopen handler // (it is not assigned to 'pReOpenHandler' in order not to bump the reference counter // until we know that 'SendMessage' was successful) OpenHandler *openHandler = new OpenHandler( self, 0 ); MessageSendParams params; params.timeout = timeout; MessageUtils::ProcessSendParams( params ); XRootDTransport::SetDescription( msg ); //-------------------------------------------------------------------------- // Issue the open request //-------------------------------------------------------------------------- XRootDStatus st = self->IssueRequest( url, msg, openHandler, params ); // if there was a problem destroy the open handler if( !st.IsOK() ) { delete openHandler; self->pStatus = st; self->pFileState = Closed; } return st; } //------------------------------------------------------------------------ // Fail a message //------------------------------------------------------------------------ void FileStateHandler::FailMessage( RequestData rd, XRootDStatus status ) { Log *log = DefaultEnv::GetLog(); log->Dump( FileMsg, "[0x%x@%s] Failing message %s with %s", this, pFileUrl->GetURL().c_str(), rd.request->GetDescription().c_str(), status.ToStr().c_str() ); StatefulHandler *sh = dynamic_cast(rd.handler); if( !sh ) { Log *log = DefaultEnv::GetLog(); log->Error( FileMsg, "[0x%x@%s] Internal error while recovering %s", this, pFileUrl->GetURL().c_str(), rd.request->GetDescription().c_str() ); return; } JobManager *jobMan = DefaultEnv::GetPostMaster()->GetJobManager(); ResponseHandler *userHandler = sh->GetUserHandler(); jobMan->QueueJob( new ResponseJob( userHandler, new XRootDStatus( status ), 0, rd.params.hostList ) ); delete sh; } //---------------------------------------------------------------------------- // Fail queued messages //---------------------------------------------------------------------------- void FileStateHandler::FailQueuedMessages( XRootDStatus status ) { RequestList::iterator it; for( it = pToBeRecovered.begin(); it != pToBeRecovered.end(); ++it ) FailMessage( *it, status ); pToBeRecovered.clear(); } //------------------------------------------------------------------------ // Re-send queued messages //------------------------------------------------------------------------ void FileStateHandler::ReSendQueuedMessages() { RequestList::iterator it; for( it = pToBeRecovered.begin(); it != pToBeRecovered.end(); ++it ) { it->request->SetSessionId( pSessionId ); ReWriteFileHandle( it->request ); XRootDStatus st = IssueRequest( *pDataServer, it->request, it->handler, it->params ); if( !st.IsOK() ) FailMessage( *it, st ); } pToBeRecovered.clear(); } //------------------------------------------------------------------------ // Re-write file handle //------------------------------------------------------------------------ void FileStateHandler::ReWriteFileHandle( Message *msg ) { ClientRequestHdr *hdr = (ClientRequestHdr*)msg->GetBuffer(); switch( hdr->requestid ) { case kXR_read: { ClientReadRequest *req = (ClientReadRequest*)msg->GetBuffer(); memcpy( req->fhandle, pFileHandle, 4 ); break; } case kXR_write: { ClientWriteRequest *req = (ClientWriteRequest*)msg->GetBuffer(); memcpy( req->fhandle, pFileHandle, 4 ); break; } case kXR_sync: { ClientSyncRequest *req = (ClientSyncRequest*)msg->GetBuffer(); memcpy( req->fhandle, pFileHandle, 4 ); break; } case kXR_truncate: { ClientTruncateRequest *req = (ClientTruncateRequest*)msg->GetBuffer(); memcpy( req->fhandle, pFileHandle, 4 ); break; } case kXR_readv: { ClientReadVRequest *req = (ClientReadVRequest*)msg->GetBuffer(); readahead_list *dataChunk = (readahead_list*)msg->GetBuffer( 24 ); for( size_t i = 0; i < req->dlen/sizeof(readahead_list); ++i ) memcpy( dataChunk[i].fhandle, pFileHandle, 4 ); break; } case kXR_writev: { ClientWriteVRequest *req = reinterpret_cast( msg->GetBuffer() ); XrdProto::write_list *wrtList = reinterpret_cast( msg->GetBuffer( 24 ) ); size_t size = req->dlen / sizeof(XrdProto::write_list); for( size_t i = 0; i < size; ++i ) memcpy( wrtList[i].fhandle, pFileHandle, 4 ); break; } case kXR_pgread: { ClientPgReadRequest *req = (ClientPgReadRequest*) msg->GetBuffer(); memcpy( req->fhandle, pFileHandle, 4 ); break; } case kXR_pgwrite: { ClientPgWriteRequest *req = (ClientPgWriteRequest*) msg->GetBuffer(); memcpy( req->fhandle, pFileHandle, 4 ); break; } } Log *log = DefaultEnv::GetLog(); log->Dump( FileMsg, "[0x%x@%s] Rewritten file handle for %s to 0x%x", this, pFileUrl->GetURL().c_str(), msg->GetDescription().c_str(), *((uint32_t*)pFileHandle) ); XRootDTransport::SetDescription( msg ); } //---------------------------------------------------------------------------- // Dispatch monitoring information on close //---------------------------------------------------------------------------- void FileStateHandler::MonitorClose( const XRootDStatus *status ) { Monitor *mon = DefaultEnv::GetMonitor(); if( mon ) { Monitor::CloseInfo i; i.file = pFileUrl; i.oTOD = pOpenTime; gettimeofday( &i.cTOD, 0 ); i.rBytes = pRBytes; i.vrBytes = pVRBytes; i.wBytes = pWBytes; i.vwBytes = pVWBytes; i.vSegs = pVSegs; i.rCount = pRCount; i.vCount = pVRCount; i.wCount = pWCount; i.status = status; mon->Event( Monitor::EvClose, &i ); } } XRootDStatus FileStateHandler::IssueRequest( const URL &url, Message *msg, ResponseHandler *handler, MessageSendParams &sendParams ) { // first handle Metalinks if( pUseVirtRedirector && url.IsMetalink() ) return MessageUtils::RedirectMessage( url, msg, handler, sendParams, pLFileHandler ); // than local file access if( url.IsLocalFile() ) return pLFileHandler->ExecRequest( url, msg, handler, sendParams ); // and finally ordinary XRootD requests return MessageUtils::SendMessage( url, msg, handler, sendParams, pLFileHandler ); } //------------------------------------------------------------------------ // Send a write request with payload being stored in a kernel buffer //------------------------------------------------------------------------ XRootDStatus FileStateHandler::WriteKernelBuffer( std::shared_ptr &self, uint64_t offset, uint32_t length, std::unique_ptr kbuff, ResponseHandler *handler, uint16_t timeout ) { //-------------------------------------------------------------------------- // Create the write request //-------------------------------------------------------------------------- XrdSysMutexHelper scopedLock( self->pMutex ); if( self->pFileState != Opened && self->pFileState != Recovering ) return XRootDStatus( stError, errInvalidOp ); Log *log = DefaultEnv::GetLog(); log->Debug( FileMsg, "[0x%x@%s] Sending a write command for handle 0x%x to " "%s", self.get(), self->pFileUrl->GetURL().c_str(), *((uint32_t*)self->pFileHandle), self->pDataServer->GetHostId().c_str() ); Message *msg; ClientWriteRequest *req; MessageUtils::CreateRequest( msg, req ); req->requestid = kXR_write; req->offset = offset; req->dlen = length; memcpy( req->fhandle, self->pFileHandle, 4 ); MessageSendParams params; params.timeout = timeout; params.followRedirects = false; params.stateful = true; params.kbuff = kbuff.release(); params.chunkList = new ChunkList(); MessageUtils::ProcessSendParams( params ); XRootDTransport::SetDescription( msg ); StatefulHandler *stHandler = new StatefulHandler( self, handler, msg, params ); return SendOrQueue( self, *self->pDataServer, msg, stHandler, params ); } } xrootd-5.6.9/src/XrdCl/XrdClFileStateHandler.hh000066400000000000000000001403031457266313600213270ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2014 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #ifndef __XRD_CL_FILE_STATE_HANDLER_HH__ #define __XRD_CL_FILE_STATE_HANDLER_HH__ #include "XrdCl/XrdClXRootDResponses.hh" #include "XrdCl/XrdClPostMasterInterfaces.hh" #include "XrdCl/XrdClFileSystem.hh" #include "XrdCl/XrdClMessageUtils.hh" #include "XrdCl/XrdClLocalFileHandler.hh" #include "XrdCl/XrdClOptional.hh" #include "XrdCl/XrdClPlugInInterface.hh" #include "XrdSys/XrdSysPthread.hh" #include "XrdSys/XrdSysPageSize.hh" #include #include #include #include #include #include namespace { class PgReadHandler; class PgReadRetryHandler; class PgReadSubstitutionHandler; class OpenHandler; } namespace XrdCl { class Message; class EcHandler; //---------------------------------------------------------------------------- //! PgRead flags //---------------------------------------------------------------------------- struct PgReadFlags { //------------------------------------------------------------------------ //! PgRead flags //------------------------------------------------------------------------ enum Flags { None = 0, //< Nothing Retry = XrdProto::kXR_pgRetry //< Retry reading currupted page }; }; XRDOUC_ENUM_OPERATORS( PgReadFlags::Flags ) //---------------------------------------------------------------------------- //! Handle the stateful operations //---------------------------------------------------------------------------- class FileStateHandler { friend class ::PgReadHandler; friend class ::PgReadRetryHandler; friend class ::PgReadSubstitutionHandler; friend class ::OpenHandler; public: //------------------------------------------------------------------------ //! State of the file //------------------------------------------------------------------------ enum FileStatus { Closed, //!< The file is closed Opened, //!< Opening has succeeded Error, //!< Opening has failed Recovering, //!< Recovering from an error OpenInProgress, //!< Opening is in progress CloseInProgress //!< Closing operation is in progress }; //------------------------------------------------------------------------ //! Constructor //------------------------------------------------------------------------ FileStateHandler( FilePlugIn *& plugin ); //------------------------------------------------------------------------ //! Constructor //! //! @param useVirtRedirector if true Metalink files will be treated //! as a VirtualRedirectors //------------------------------------------------------------------------ FileStateHandler( bool useVirtRedirector, FilePlugIn *& plugin ); //------------------------------------------------------------------------ //! Destructor //------------------------------------------------------------------------ ~FileStateHandler(); //------------------------------------------------------------------------ //! Open the file pointed to by the given URL //! //! @param url url of the file to be opened //! @param flags OpenFlags::Flags //! @param mode Access::Mode for new files, 0 otherwise //! @param handler handler to be notified about the status of the operation //! @param timeout timeout value, if 0 the environment default will be //! used //! @return status of the operation //------------------------------------------------------------------------ static XRootDStatus Open( std::shared_ptr &self, const std::string &url, uint16_t flags, uint16_t mode, ResponseHandler *handler, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! Close the file object //! //! @param handler handler to be notified about the status of the operation //! @param timeout timeout value, if 0 the environment default will be //! used //! @return status of the operation //------------------------------------------------------------------------ static XRootDStatus Close( std::shared_ptr &self, ResponseHandler *handler, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! Obtain status information for this file - async //! //! @param force do not use the cached information, force re-stating //! @param handler handler to be notified when the response arrives, //! the response parameter will hold a StatInfo object //! if the procedure is successful //! @param timeout timeout value, if 0 the environment default will //! be used //! @return status of the operation //------------------------------------------------------------------------ static XRootDStatus Stat( std::shared_ptr &self, bool force, ResponseHandler *handler, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! Read a data chunk at a given offset - sync //! //! @param offset offset from the beginning of the file //! @param size number of bytes to be read //! @param buffer a pointer to a buffer big enough to hold the data //! or 0 if the buffer should be allocated by the system //! @param handler handler to be notified when the response arrives, //! the response parameter will hold a buffer object if //! the procedure was successful, if a preallocated //! buffer was specified then the buffer object will //! "wrap" this buffer //! @param timeout timeout value, if 0 the environment default will be //! used //! @return status of the operation //------------------------------------------------------------------------ static XRootDStatus Read( std::shared_ptr &self, uint64_t offset, uint32_t size, void *buffer, ResponseHandler *handler, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! Read data pages at a given offset //! //! @param offset : offset from the beginning of the file (Note: has to //! 4KB aligned) //! @param size : buffer size //! @param buffer : a pointer to buffer big enough to hold the data //! @param handler : handler to be notified when the response arrives, the //! response parameter will hold a PgReadInfo object if //! the procedure was successful //! @param timeout : timeout value, if 0 environment default will be used //! //! @return : status of the operation //------------------------------------------------------------------------ static XRootDStatus PgRead( std::shared_ptr &self, uint64_t offset, uint32_t size, void *buffer, ResponseHandler *handler, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! Retry reading one page of data at a given offset //! //! @param offset : offset from the beginning of the file (Note: has to //! 4KB aligned) //! @param size : buffer size //! @param buffer : a pointer to buffer big enough to hold the data //! @param handler : handler to be notified when the response arrives //! @param timeout : timeout value, if 0 environment default will be used //! //! @return : status of the operation //------------------------------------------------------------------------ static XRootDStatus PgReadRetry( std::shared_ptr &self, uint64_t offset, uint32_t size, size_t pgnb, void *buffer, PgReadHandler *handler, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! Read data pages at a given offset (actual implementation) //! //! @param offset : offset from the beginning of the file (Note: has to //! 4KB aligned) //! @param size : buffer size //! @param buffer : a pointer to buffer big enough to hold the data //! @param flags : PgRead flags //! @param handler : handler to be notified when the response arrives, the //! response parameter will hold a PgReadInfo object if //! the procedure was successful //! @param timeout : timeout value, if 0 environment default will be used //! //! @return : status of the operation //------------------------------------------------------------------------ static XRootDStatus PgReadImpl( std::shared_ptr &self, uint64_t offset, uint32_t size, void *buffer, uint16_t flags, ResponseHandler *handler, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! Write a data chunk at a given offset - async //! //! @param offset offset from the beginning of the file //! @param size number of bytes to be written //! @param buffer a pointer to the buffer holding the data to be written //! @param handler handler to be notified when the response arrives //! @param timeout timeout value, if 0 the environment default will be //! used //! @return status of the operation //------------------------------------------------------------------------ static XRootDStatus Write( std::shared_ptr &self, uint64_t offset, uint32_t size, const void *buffer, ResponseHandler *handler, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! Write a data chunk at a given offset - async //! //! @param offset offset from the beginning of the file //! @param buffer r-value reference to Buffer object, in this case XrdCl //! runtime takes ownership of the buffer //! @param handler handler to be notified when the response arrives //! @param timeout timeout value, if 0 the environment default will be //! used //! @return status of the operation //------------------------------------------------------------------------ static XRootDStatus Write( std::shared_ptr &self, uint64_t offset, Buffer &&buffer, ResponseHandler *handler, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! Write a data from a given file descriptor at a given offset - async //! //! @param offset offset from the beginning of the file //! @param size number of bytes to be written //! @param fdoff offset of the data to be written from the file descriptor //! (optional, if not provided will copy data from the file //! descriptor at the current cursor position) //! @param fd file descriptor open for reading //! @param handler handler to be notified when the response arrives //! @param timeout timeout value, if 0 the environment default will be //! used //! @return status of the operation //------------------------------------------------------------------------ static XRootDStatus Write( std::shared_ptr &self, uint64_t offset, uint32_t size, Optional fdoff, int fd, ResponseHandler *handler, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! Write number of pages at a given offset - async //! //! @param offset offset from the beginning of the file //! @param size buffer size //! @param buffer a pointer to a buffer holding data pages //! @param cksums the crc32c checksums for each 4KB page //! @param handler handler to be notified when the response arrives //! @param timeout timeout value, if 0 the environment default will be //! used //! @return status of the operation //------------------------------------------------------------------------ static XRootDStatus PgWrite( std::shared_ptr &self, uint64_t offset, uint32_t size, const void *buffer, std::vector &cksums, ResponseHandler *handler, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! Write number of pages at a given offset - async //! //! @param offset offset from the beginning of the file //! @param size buffer size //! @param buffer a pointer to a buffer holding data pages //! @param handler handler to be notified when the response arrives //! @param timeout timeout value, if 0 the environment default will be //! used //! @return status of the operation //------------------------------------------------------------------------ static XRootDStatus PgWriteRetry( std::shared_ptr &self, uint64_t offset, uint32_t size, const void *buffer, uint32_t digest, ResponseHandler *handler, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! Write number of pages at a given offset - async //! //! @param offset offset from the beginning of the file //! @param size buffer size //! @param buffer a pointer to a buffer holding data pages //! @param cksums the crc32c checksums for each 4KB page //! @param flags PgWrite flags //! @param handler handler to be notified when the response arrives //! @param timeout timeout value, if 0 the environment default will be //! used //! @return status of the operation //------------------------------------------------------------------------ static XRootDStatus PgWriteImpl( std::shared_ptr &self, uint64_t offset, uint32_t size, const void *buffer, std::vector &cksums, kXR_char flags, ResponseHandler *handler, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! Commit all pending disk writes - async //! //! @param handler handler to be notified when the response arrives //! @param timeout timeout value, if 0 the environment default will be //! used //! @return status of the operation //------------------------------------------------------------------------ static XRootDStatus Sync( std::shared_ptr &self, ResponseHandler *handler, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! Truncate the file to a particular size - async //! //! @param size desired size of the file //! @param handler handler to be notified when the response arrives //! @param timeout timeout value, if 0 the environment default will be //! used //! @return status of the operation //------------------------------------------------------------------------ static XRootDStatus Truncate( std::shared_ptr &self, uint64_t size, ResponseHandler *handler, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! Read scattered data chunks in one operation - async //! //! @param chunks list of the chunks to be read //! @param buffer a pointer to a buffer big enough to hold the data //! @param handler handler to be notified when the response arrives //! @param timeout timeout value, if 0 then the environment default //! will be used //! @return status of the operation //------------------------------------------------------------------------ static XRootDStatus VectorRead( std::shared_ptr &self, const ChunkList &chunks, void *buffer, ResponseHandler *handler, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! Write scattered data chunks in one operation - async //! //! @param chunks list of the chunks to be read //! @param handler handler to be notified when the response arrives //! @param timeout timeout value, if 0 then the environment default //! will be used //! @return status of the operation //------------------------------------------------------------------------ static XRootDStatus VectorWrite( std::shared_ptr &self, const ChunkList &chunks, ResponseHandler *handler, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! Write scattered buffers in one operation - async //! //! @param offset offset from the beginning of the file //! @param iov list of the buffers to be written //! @param iovcnt number of buffers //! @param handler handler to be notified when the response arrives //! @param timeout timeout value, if 0 then the environment default //! will be used //! @return status of the operation //------------------------------------------------------------------------ static XRootDStatus WriteV( std::shared_ptr &self, uint64_t offset, const struct iovec *iov, int iovcnt, ResponseHandler *handler, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! Read data into scattered buffers in one operation - async //! //! @param offset offset from the beginning of the file //! @param iov list of the buffers to be written //! @param iovcnt number of buffers //! @param handler handler to be notified when the response arrives //! @param timeout timeout value, if 0 then the environment default //! will be used //! @return status of the operation //------------------------------------------------------------------------ static XRootDStatus ReadV( std::shared_ptr &self, uint64_t offset, struct iovec *iov, int iovcnt, ResponseHandler *handler, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! Performs a custom operation on an open file, server implementation //! dependent - async //! //! @param arg query argument //! @param handler handler to be notified when the response arrives, //! the response parameter will hold a Buffer object //! if the procedure is successful //! @param timeout timeout value, if 0 the environment default will //! be used //! @return status of the operation //------------------------------------------------------------------------ static XRootDStatus Fcntl( std::shared_ptr &self, const Buffer &arg, ResponseHandler *handler, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! Get access token to a file - async //! //! @param handler handler to be notified when the response arrives, //! the response parameter will hold a Buffer object //! if the procedure is successful //! @param timeout timeout value, if 0 the environment default will //! be used //! @return status of the operation //------------------------------------------------------------------------ static XRootDStatus Visa( std::shared_ptr &self, ResponseHandler *handler, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! Set extended attributes - async //! //! @param attrs : list of extended attributes to set //! @param handler : handler to be notified when the response arrives, //! the response parameter will hold a std::vector of //! XAttrStatus objects //! @param timeout : timeout value, if 0 the environment default will //! be used //! //! @return : status of the operation //------------------------------------------------------------------------ static XRootDStatus SetXAttr( std::shared_ptr &self, const std::vector &attrs, ResponseHandler *handler, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! Get extended attributes - async //! //! @param attrs : list of extended attributes to get //! @param handler : handler to be notified when the response arrives, //! the response parameter will hold a std::vector of //! XAttr objects //! @param timeout : timeout value, if 0 the environment default will //! be used //! //! @return : status of the operation //------------------------------------------------------------------------ static XRootDStatus GetXAttr( std::shared_ptr &self, const std::vector &attrs, ResponseHandler *handler, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! Delete extended attributes - async //! //! @param attrs : list of extended attributes to set //! @param handler : handler to be notified when the response arrives, //! the response parameter will hold a std::vector of //! XAttrStatus objects //! @param timeout : timeout value, if 0 the environment default will //! be used //! //! @return : status of the operation //------------------------------------------------------------------------ static XRootDStatus DelXAttr( std::shared_ptr &self, const std::vector &attrs, ResponseHandler *handler, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! List extended attributes - async //! //! @param handler : handler to be notified when the response arrives, //! the response parameter will hold a std::vector of //! XAttr objects //! @param timeout : timeout value, if 0 the environment default will //! be used //! //! @return : status of the operation //------------------------------------------------------------------------ static XRootDStatus ListXAttr( std::shared_ptr &self, ResponseHandler *handler, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! Create a checkpoint //! //! @param handler : handler to be notified when the response arrives, //! the response parameter will hold a std::vector of //! XAttr objects //! @param timeout : timeout value, if 0 the environment default will //! be used //! //! @return : status of the operation //------------------------------------------------------------------------ static XRootDStatus Checkpoint( std::shared_ptr &self, kXR_char code, ResponseHandler *handler, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! Checkpointed write - async //! //! @param offset offset from the beginning of the file //! @param size number of bytes to be written //! @param buffer a pointer to the buffer holding the data to be written //! @param handler handler to be notified when the response arrives //! @param timeout timeout value, if 0 the environment default will be //! used //! @return status of the operation //------------------------------------------------------------------------ static XRootDStatus ChkptWrt( std::shared_ptr &self, uint64_t offset, uint32_t size, const void *buffer, ResponseHandler *handler, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! Checkpointed WriteV - async //! //! @param offset offset from the beginning of the file //! @param iov list of the buffers to be written //! @param iovcnt number of buffers //! @param handler handler to be notified when the response arrives //! @param timeout timeout value, if 0 then the environment default //! will be used //! @return status of the operation //------------------------------------------------------------------------ static XRootDStatus ChkptWrtV( std::shared_ptr &self, uint64_t offset, const struct iovec *iov, int iovcnt, ResponseHandler *handler, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! Process the results of the opening operation //------------------------------------------------------------------------ void OnOpen( const XRootDStatus *status, const OpenInfo *openInfo, const HostList *hostList ); //------------------------------------------------------------------------ //! Process the results of the closing operation //------------------------------------------------------------------------ void OnClose( const XRootDStatus *status ); //------------------------------------------------------------------------ //! Handle an error while sending a stateful message //------------------------------------------------------------------------ static void OnStateError( std::shared_ptr &self, XRootDStatus *status, Message *message, ResponseHandler *userHandler, MessageSendParams &sendParams ); //------------------------------------------------------------------------ //! Handle stateful redirect //------------------------------------------------------------------------ static void OnStateRedirection( std::shared_ptr &self, const std::string &redirectUrl, Message *message, ResponseHandler *userHandler, MessageSendParams &sendParams ); //------------------------------------------------------------------------ //! Handle stateful response //------------------------------------------------------------------------ static void OnStateResponse( std::shared_ptr &self, XRootDStatus *status, Message *message, AnyObject *response, HostList *hostList ); //------------------------------------------------------------------------ //! Check if the file is open //------------------------------------------------------------------------ bool IsOpen() const; //------------------------------------------------------------------------ //! Check if the file is using an encrypted connection //------------------------------------------------------------------------ inline bool IsSecure() const { return pIsChannelEncrypted; } //------------------------------------------------------------------------ //! Set file property //! //! @see File::GetProperty for propert list //------------------------------------------------------------------------ bool SetProperty( const std::string &name, const std::string &value ); //------------------------------------------------------------------------ //! Get file property //! //! @see File::SetProperty for property list //------------------------------------------------------------------------ bool GetProperty( const std::string &name, std::string &value ) const; //------------------------------------------------------------------------ //! Lock the internal lock //------------------------------------------------------------------------ void Lock() { pMutex.Lock(); } //------------------------------------------------------------------------ //! Unlock the internal lock //------------------------------------------------------------------------ void UnLock() { pMutex.UnLock(); } //------------------------------------------------------------------------ //! Tick //------------------------------------------------------------------------ void Tick( time_t now ); //------------------------------------------------------------------------ //! Declare timeout on requests being recovered //------------------------------------------------------------------------ void TimeOutRequests( time_t now ); //------------------------------------------------------------------------ //! Called in the child process after the fork //------------------------------------------------------------------------ void AfterForkChild(); //------------------------------------------------------------------------ //! Try other data server //------------------------------------------------------------------------ static XRootDStatus TryOtherServer( std::shared_ptr &self, uint16_t timeout ); private: //------------------------------------------------------------------------ // Helper for queuing messages //------------------------------------------------------------------------ struct RequestData { RequestData(): request(0), handler(0) {} RequestData( Message *r, ResponseHandler *h, const MessageSendParams &p ): request(r), handler(h), params(p) {} Message *request; ResponseHandler *handler; MessageSendParams params; }; typedef std::list RequestList; //------------------------------------------------------------------------ //! Generic implementation of xattr operation //! //! @param subcode : xattr operation code //! @param attrs : operation argument //! @param handler : operation handler //! @param timeout : operation timeout //------------------------------------------------------------------------ template static Status XAttrOperationImpl( std::shared_ptr &self, kXR_char subcode, kXR_char options, const std::vector &attrs, ResponseHandler *handler, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! Send a message to a host or put it in the recovery queue //------------------------------------------------------------------------ static Status SendOrQueue( std::shared_ptr &self, const URL &url, Message *msg, ResponseHandler *handler, MessageSendParams &sendParams ); //------------------------------------------------------------------------ //! Check if the stateful error is recoverable //------------------------------------------------------------------------ bool IsRecoverable( const XRootDStatus &stataus ) const; //------------------------------------------------------------------------ //! Recover a message //! //! @param rd request data associated with the message //! @param callbackOnFailure should the current handler be called back //! if the recovery procedure fails //------------------------------------------------------------------------ static Status RecoverMessage( std::shared_ptr &self, RequestData rd, bool callbackOnFailure = true ); //------------------------------------------------------------------------ //! Run the recovery procedure if appropriate //------------------------------------------------------------------------ static Status RunRecovery( std::shared_ptr &self ); //------------------------------------------------------------------------ // Send a close and ignore the response //------------------------------------------------------------------------ static XRootDStatus SendClose( std::shared_ptr &self, uint16_t timeout ); //------------------------------------------------------------------------ //! Check if the file is open for read only //------------------------------------------------------------------------ bool IsReadOnly() const; //------------------------------------------------------------------------ //! Re-open the current file at a given server //------------------------------------------------------------------------ static XRootDStatus ReOpenFileAtServer( std::shared_ptr &self, const URL &url, uint16_t timeout ); //------------------------------------------------------------------------ //! Fail a message //------------------------------------------------------------------------ void FailMessage( RequestData rd, XRootDStatus status ); //------------------------------------------------------------------------ //! Fail queued messages //------------------------------------------------------------------------ void FailQueuedMessages( XRootDStatus status ); //------------------------------------------------------------------------ //! Re-send queued messages //------------------------------------------------------------------------ void ReSendQueuedMessages(); //------------------------------------------------------------------------ //! Re-write file handle //------------------------------------------------------------------------ void ReWriteFileHandle( Message *msg ); //------------------------------------------------------------------------ //! Reset monitoring vars //------------------------------------------------------------------------ void ResetMonitoringVars() { pOpenTime.tv_sec = 0; pOpenTime.tv_usec = 0; pRBytes = 0; pVRBytes = 0; pWBytes = 0; pVSegs = 0; pRCount = 0; pVRCount = 0; pWCount = 0; pCloseReason = Status(); } //------------------------------------------------------------------------ //! Dispatch monitoring information on close //------------------------------------------------------------------------ void MonitorClose( const XRootDStatus *status ); //------------------------------------------------------------------------ //! Issues request: //! - if the request is for a Metalink a redirect is generated //! - if the request is for a local file, it will be passed to //! LocalFileHandler //! - otherwise vanilla XRootD request will be sent //------------------------------------------------------------------------ XRootDStatus IssueRequest( const URL &url, Message *msg, ResponseHandler *handler, MessageSendParams &sendParams ); //------------------------------------------------------------------------ //! Send a write request with payload being stored in a kernel buffer //------------------------------------------------------------------------ static XRootDStatus WriteKernelBuffer( std::shared_ptr &self, uint64_t offset, uint32_t length, std::unique_ptr kbuff, ResponseHandler *handler, uint16_t timeout ); mutable XrdSysMutex pMutex; FileStatus pFileState; XRootDStatus pStatus; StatInfo *pStatInfo; URL *pFileUrl; URL *pDataServer; URL *pLoadBalancer; URL *pStateRedirect; URL *pWrtRecoveryRedir; uint8_t *pFileHandle; uint16_t pOpenMode; uint16_t pOpenFlags; RequestList pToBeRecovered; std::set pInTheFly; uint64_t pSessionId; bool pDoRecoverRead; bool pDoRecoverWrite; bool pFollowRedirects; bool pUseVirtRedirector; bool pIsChannelEncrypted; bool pAllowBundledClose; //------------------------------------------------------------------------ // Monitoring variables //------------------------------------------------------------------------ timeval pOpenTime; uint64_t pRBytes; uint64_t pVRBytes; uint64_t pWBytes; uint64_t pVWBytes; uint64_t pVSegs; uint64_t pRCount; uint64_t pVRCount; uint64_t pWCount; uint64_t pVWCount; XRootDStatus pCloseReason; //------------------------------------------------------------------------ // Responsible for file:// operations on the local filesystem //------------------------------------------------------------------------ LocalFileHandler *pLFileHandler; //------------------------------------------------------------------------ // Responsible for Writing/Reading erasure-coded files //------------------------------------------------------------------------ FilePlugIn *&pPlugin; }; } #endif // __XRD_CL_FILE_STATE_HANDLER_HH__ xrootd-5.6.9/src/XrdCl/XrdClFileSystem.cc000066400000000000000000002416561457266313600202400ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2014 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #include "XrdCl/XrdClMessageUtils.hh" #include "XrdCl/XrdClFileSystem.hh" #include "XrdCl/XrdClDefaultEnv.hh" #include "XrdCl/XrdClLog.hh" #include "XrdCl/XrdClConstants.hh" #include "XrdCl/XrdClMessage.hh" #include "XrdCl/XrdClXRootDResponses.hh" #include "XrdCl/XrdClRequestSync.hh" #include "XrdCl/XrdClXRootDTransport.hh" #include "XrdCl/XrdClForkHandler.hh" #include "XrdCl/XrdClPlugInInterface.hh" #include "XrdCl/XrdClPlugInManager.hh" #include "XrdCl/XrdClLocalFileTask.hh" #include "XrdCl/XrdClZipListHandler.hh" #include "XrdSys/XrdSysE2T.hh" #include "XrdSys/XrdSysPthread.hh" #include #include #include #include namespace { class LocalFS { public: XrdCl::XRootDStatus Stat( const std::string &path, XrdCl::ResponseHandler *handler, uint16_t timeout ) { using namespace XrdCl; Log *log = DefaultEnv::GetLog(); struct stat ssp; if( stat( path.c_str(), &ssp ) == -1 ) { log->Error( FileMsg, "Stat: failed: %s", XrdSysE2T( errno ) ); XRootDStatus *error = new XRootDStatus( stError, errLocalError, XProtocol::mapError( errno ) ); return QueueTask( error, 0, handler ); } // TODO support other mode options uint32_t flags = S_ISDIR( ssp.st_mode ) ? kXR_isDir : 0; std::ostringstream data; data << ssp.st_dev << " " << ssp.st_size << " " << flags << " " << ssp.st_mtime; log->Debug( FileMsg, data.str().c_str() ); StatInfo *statInfo = new StatInfo(); if( !statInfo->ParseServerResponse( data.str().c_str() ) ) { log->Error( FileMsg, "Stat: ParseServerResponse failed." ); delete statInfo; return QueueTask( new XRootDStatus( stError, errErrorResponse, kXR_FSError ), 0, handler ); } AnyObject *resp = new AnyObject(); resp->Set( statInfo ); return QueueTask( new XRootDStatus(), resp, handler ); } XrdCl::XRootDStatus Rm( const std::string &path, XrdCl::ResponseHandler *handler, uint16_t timeout ) { using namespace XrdCl; Log *log = DefaultEnv::GetLog(); if( unlink( path.c_str() ) ) { log->Error( FileMsg, "Rm: failed: %s", XrdSysE2T( errno ) ); XRootDStatus *error = new XRootDStatus( stError, errLocalError, XProtocol::mapError( errno ) ); return QueueTask( error, 0, handler ); } return QueueTask( new XRootDStatus(), 0, handler ); } static LocalFS& Instance() { static LocalFS instance; return instance; } private: //------------------------------------------------------------------------ // Private constructors //------------------------------------------------------------------------ LocalFS() : jmngr( XrdCl::DefaultEnv::GetPostMaster()->GetJobManager() ) { } //------------------------------------------------------------------------ // Private copy constructors //------------------------------------------------------------------------ LocalFS( const LocalFS& ); //------------------------------------------------------------------------ // Private assignment operator //------------------------------------------------------------------------ LocalFS& operator=( const LocalFS& ); //------------------------------------------------------------------------ // QueueTask - queues error/success tasks for all operations. // Must always return stOK. // Is always creating the same HostList containing only localhost. //------------------------------------------------------------------------ XrdCl::XRootDStatus QueueTask( XrdCl::XRootDStatus *st, XrdCl::AnyObject *resp, XrdCl::ResponseHandler *handler ) { using namespace XrdCl; // if it is simply the sync handler we can release the semaphore // and return there is no need to execute this in the thread-pool SyncResponseHandler *syncHandler = dynamic_cast( handler ); if( syncHandler ) { syncHandler->HandleResponse( st, resp ); return XRootDStatus(); } LocalFileTask *task = new LocalFileTask( st, resp, 0, handler ); jmngr->QueueJob( task ); return XRootDStatus(); } XrdCl::JobManager *jmngr; }; //---------------------------------------------------------------------------- // Get delimiter for the opaque info //---------------------------------------------------------------------------- char GetCgiDelimiter( bool &hasCgi ) { if( !hasCgi ) { hasCgi = true; return '?'; } return '&'; } //---------------------------------------------------------------------------- // Filters out client specific CGI //---------------------------------------------------------------------------- std::string FilterXrdClCgi( const std::string &path ) { // first check if there's an opaque info at all size_t pos = path.find( '?' ); if( pos == std::string::npos ) return path; std::string filteredPath = path.substr( 0 , pos ); std::string cgi = path.substr( pos + 1 ); bool hasCgi = false; pos = 0; size_t xrdcl = std::string::npos; do { xrdcl = cgi.find( "xrdcl.", pos ); if( xrdcl == std::string:: npos ) { filteredPath += GetCgiDelimiter( hasCgi ); filteredPath += cgi.substr( pos ); pos = cgi.size(); } else { if( xrdcl != pos ) { filteredPath += GetCgiDelimiter( hasCgi ); filteredPath += cgi.substr( pos, xrdcl - 1 - pos ); } pos = cgi.find( '&', xrdcl ); if( pos != std::string::npos ) ++pos; } } while( pos < cgi.size() && pos != std::string::npos ); return filteredPath; } //---------------------------------------------------------------------------- //! Wrapper class used to delete FileSystem object //---------------------------------------------------------------------------- class DeallocFSHandler: public XrdCl::ResponseHandler { public: //------------------------------------------------------------------------ // Constructor and destructor //------------------------------------------------------------------------ DeallocFSHandler( XrdCl::FileSystem *fs, ResponseHandler *userHandler ): pFS(fs), pUserHandler(userHandler) {} virtual ~DeallocFSHandler() { delete pFS; } //------------------------------------------------------------------------ // Handle the response //------------------------------------------------------------------------ virtual void HandleResponse( XrdCl::XRootDStatus *status, XrdCl::AnyObject *response ) { pUserHandler->HandleResponse(status, response); delete this; } private: XrdCl::FileSystem *pFS; ResponseHandler *pUserHandler; }; //---------------------------------------------------------------------------- // Deep locate handler //---------------------------------------------------------------------------- class DeepLocateHandler: public XrdCl::ResponseHandler { public: //------------------------------------------------------------------------ // Constructor //------------------------------------------------------------------------ DeepLocateHandler( XrdCl::ResponseHandler *handler, const std::string &path, XrdCl::OpenFlags::Flags flags, time_t expires ): pFirstTime( true ), pPartial( false ), pOutstanding( 1 ), pHandler( handler ), pPath( path ), pFlags( flags ), pExpires(expires) { pLocations = new XrdCl::LocationInfo(); } //------------------------------------------------------------------------ // Destructor //------------------------------------------------------------------------ ~DeepLocateHandler() { delete pLocations; } //------------------------------------------------------------------------ // Handle the response //------------------------------------------------------------------------ virtual void HandleResponse( XrdCl::XRootDStatus *status, XrdCl::AnyObject *response ) { XrdSysMutexHelper scopedLock( pMutex ); using namespace XrdCl; Log *log = DefaultEnv::GetLog(); --pOutstanding; //---------------------------------------------------------------------- // We've got an error, react accordingly //---------------------------------------------------------------------- if( !status->IsOK() ) { log->Dump( FileSystemMsg, "[0x%x@DeepLocate(%s)] Got error " "response: %s", this, pPath.c_str(), status->ToStr().c_str() ); //-------------------------------------------------------------------- // We have failed with the first request //-------------------------------------------------------------------- if( pFirstTime ) { log->Debug( FileSystemMsg, "[0x%x@DeepLocate(%s)] Failed to get " "the initial location list: %s", this, pPath.c_str(), status->ToStr().c_str() ); pHandler->HandleResponse( status, response ); scopedLock.UnLock(); delete this; return; } pPartial = true; //-------------------------------------------------------------------- // We have no more outstanding requests, so let give to the client // what we have //-------------------------------------------------------------------- if( !pOutstanding ) { log->Debug( FileSystemMsg, "[0x%x@DeepLocate(%s)] No outstanding " "requests, give out what we've got", this, pPath.c_str() ); scopedLock.UnLock(); HandleFinalResponse(); } delete status; return; } pFirstTime = false; //---------------------------------------------------------------------- // Extract the answer //---------------------------------------------------------------------- LocationInfo *info = 0; response->Get( info ); LocationInfo::Iterator it; log->Dump( FileSystemMsg, "[0x%x@DeepLocate(%s)] Got %d locations", this, pPath.c_str(), info->GetSize() ); for( it = info->Begin(); it != info->End(); ++it ) { //-------------------------------------------------------------------- // Add the location to the list //-------------------------------------------------------------------- if( it->IsServer() ) { pLocations->Add( *it ); continue; } //-------------------------------------------------------------------- // Ask the manager for the location of servers //-------------------------------------------------------------------- if( it->IsManager() ) { ++pOutstanding; FileSystem *fs = new FileSystem( it->GetAddress() ); if( pOutstanding == 0 || // protect against overflow, short circuiting // will make sure the other part won't be executed !fs->Locate( pPath, pFlags, new DeallocFSHandler(fs, this), pExpires-::time(0)).IsOK() ) { --pOutstanding; pPartial = true; delete fs; } } } //---------------------------------------------------------------------- // Clean up and check if we have anything else to do //---------------------------------------------------------------------- delete response; delete status; if( !pOutstanding ) { scopedLock.UnLock(); HandleFinalResponse(); } } //------------------------------------------------------------------------ // Build the response for the client //------------------------------------------------------------------------ void HandleFinalResponse() { using namespace XrdCl; //---------------------------------------------------------------------- // Nothing found //---------------------------------------------------------------------- if( !pLocations->GetSize() ) { pHandler->HandleResponse( new XRootDStatus( stError, errErrorResponse, kXR_NotFound, "No valid location found" ), 0 ); } //---------------------------------------------------------------------- // We return an answer //---------------------------------------------------------------------- else { AnyObject *obj = new AnyObject(); obj->Set( pLocations ); pLocations = 0; XRootDStatus *st = new XRootDStatus(); if( pPartial ) st->code = suPartial; pHandler->HandleResponse( st, obj ); } delete this; } private: bool pFirstTime; bool pPartial; uint16_t pOutstanding; XrdCl::ResponseHandler *pHandler; XrdCl::LocationInfo *pLocations; std::string pPath; XrdCl::OpenFlags::Flags pFlags; time_t pExpires; XrdSysMutex pMutex; }; //---------------------------------------------------------------------------- // Handle stat results for a dirlist request //---------------------------------------------------------------------------- class DirListStatHandler: public XrdCl::ResponseHandler { public: //------------------------------------------------------------------------ // Constructor //------------------------------------------------------------------------ DirListStatHandler( XrdCl::DirectoryList *list, uint32_t index, XrdCl::RequestSync *sync ): pList( list ), pIndex( index ), pSync( sync ) { } //------------------------------------------------------------------------ // Check if we were successful and if so put the StatInfo object // in the appropriate entry info //------------------------------------------------------------------------ virtual void HandleResponse( XrdCl::XRootDStatus *status, XrdCl::AnyObject *response ) { if( !status->IsOK() ) { delete status; pSync->TaskDone( false ); delete this; return; } XrdCl::StatInfo *info = 0; response->Get( info ); response->Set( (char*) 0 ); pList->At( pIndex )->SetStatInfo( info ); delete status; delete response; pSync->TaskDone(); delete this; } private: XrdCl::DirectoryList *pList; uint32_t pIndex; XrdCl::RequestSync *pSync; }; //---------------------------------------------------------------------------- // Recursive dirlist common context for all handlers //---------------------------------------------------------------------------- struct RecursiveDirListCtx { RecursiveDirListCtx( const XrdCl::URL &url, const std::string &path, XrdCl::DirListFlags::Flags flags, XrdCl::ResponseHandler *handler, time_t expires ) : finalst( 0 ), pending( 1 ), dirList( new XrdCl::DirectoryList() ), expires( expires ), handler( handler ), flags( flags ), fs( new XrdCl::FileSystem( url ) ) { dirList->SetParentName( path ); } ~RecursiveDirListCtx() { delete finalst; delete dirList; delete fs; } void UpdateStatus( const XrdCl::XRootDStatus &st ) { using namespace XrdCl; if( !finalst ) { finalst = st.IsOK() ? new XRootDStatus() : new XRootDStatus( st ); return; } // if they disagree set the status to partial if( ( finalst->IsOK() && !st.IsOK() ) || ( !finalst->IsOK() && st.IsOK() ) ) *finalst = XRootDStatus( stOK, suPartial ); } XrdCl::XRootDStatus *finalst; int pending; XrdCl::DirectoryList *dirList; time_t expires; XrdCl::ResponseHandler *handler; XrdCl::DirListFlags::Flags flags; XrdCl::FileSystem *fs; XrdSysMutex mtx; }; //---------------------------------------------------------------------------- // Handle results for a recursive dirlist request //---------------------------------------------------------------------------- class RecursiveDirListHandler: public XrdCl::ResponseHandler { public: RecursiveDirListHandler( const XrdCl::URL &url, const std::string &path, XrdCl::DirListFlags::Flags flags, XrdCl::ResponseHandler *handler, time_t timeout ) { time_t expires = 0; if( timeout ) expires = ::time( 0 ) + timeout; pCtx = new RecursiveDirListCtx( url, path, flags, handler, expires ); } RecursiveDirListHandler( RecursiveDirListCtx *ctx ) : pCtx( ctx ) { } virtual void HandleResponse( XrdCl::XRootDStatus *status, XrdCl::AnyObject *response ) { using namespace XrdCl; Log *log = DefaultEnv::GetLog(); bool finalrsp = !( status->IsOK() && status->code == XrdCl::suContinue ); XrdSysMutexHelper scoped( pCtx->mtx ); // check if we have to continue with the same handler (the response // has been chunked), if not we can decrement the number of pending // DieLists if( finalrsp ) --pCtx->pending; pCtx->UpdateStatus( *status ); if( status->IsOK() ) { // get the response DirectoryList *dirList = 0; response->Get( dirList ); std::string parent = pCtx->dirList->GetParentName(); DirectoryList::Iterator itr; for( itr = dirList->Begin(); itr != dirList->End(); ++itr ) { DirectoryList::ListEntry *entry = *itr; StatInfo *info = entry->GetStatInfo(); if( !info ) { log->Error( FileMsg, "Recursive directory list operation for %s failed: " "kXR_dirlist with stat operation not supported.", parent.c_str() ); pCtx->UpdateStatus( XRootDStatus( stError, errNotSupported ) ); continue; } std::string path = dirList->GetParentName() + entry->GetName(); // add new entry to the result path = path.substr( parent.size() ); entry->SetStatInfo( 0 ); // StatInfo is no longer owned by dirList DirectoryList::ListEntry *e = new DirectoryList::ListEntry( entry->GetHostAddress(), path, info ); pCtx->dirList->Add( e ); // if it's a directory do a recursive call if( info->TestFlags( StatInfo::IsDir ) ) { // bump the pending counter ++pCtx->pending; // switch of the recursive flag, we will // provide the respective handler ourself, // make sure that stat is on DirListFlags::Flags flags = ( pCtx->flags & (~DirListFlags::Recursive) ) | DirListFlags::Stat; // the recursive dir list handler RecursiveDirListHandler *handler = new RecursiveDirListHandler( pCtx ); // timeout time_t timeout = 0; if( pCtx->expires ) { timeout = pCtx->expires - ::time( 0 ); if( timeout <= 0 ) { log->Error( FileMsg, "Recursive directory list operation for %s expired.", parent.c_str() ); pCtx->UpdateStatus( XRootDStatus( stError, errOperationExpired ) ); break; } } // send the request std::string child = parent + path; XRootDStatus st = pCtx->fs->DirList( child, flags, handler, timeout ); if( !st.IsOK() ) { log->Error( FileMsg, "Recursive directory list operation for %s failed: ", child.c_str(), st.ToString().c_str() ); pCtx->UpdateStatus( st ); continue; } } } } // if there are no more outstanding dirlist queries we can finalize the request if( pCtx->pending == 0 ) { AnyObject *resp = new AnyObject(); resp->Set( pCtx->dirList ); pCtx->dirList = 0; // dirList is no longer owned by pCtx pCtx->handler->HandleResponse( pCtx->finalst, resp ); pCtx->finalst = 0; // status is no longer owned by pCtx // finalize the common context scoped.UnLock(); delete pCtx; } // if the user requested chunked response we give what we have to the user handler else if( status->IsOK() && ( pCtx->flags & DirListFlags::Chunked ) ) { std::string parent = pCtx->dirList->GetParentName(); AnyObject *resp = new AnyObject(); resp->Set( pCtx->dirList ); pCtx->dirList = new XrdCl::DirectoryList(); pCtx->dirList->SetParentName( parent ); pCtx->handler->HandleResponse( new XRootDStatus( stOK, suContinue ), resp ); } // clean up the arguments delete status; delete response; // if we won't be continuing with the same handler, it can be deleted if( finalrsp ) delete this; } private: RecursiveDirListCtx *pCtx; }; //---------------------------------------------------------------------------- // Exception for a merge dirlist handler //---------------------------------------------------------------------------- struct MergeDirLsErr { MergeDirLsErr( XrdCl::XRootDStatus *&status, XrdCl::AnyObject *&response ) : status( status ), response( response ) { status = 0; response = 0; } MergeDirLsErr() : status( new XrdCl::XRootDStatus( XrdCl::stError, XrdCl::errInternal ) ), response( 0 ) { } XrdCl::XRootDStatus *status; XrdCl::AnyObject *response; }; //---------------------------------------------------------------------------- // Handle results for a merge dirlist request //---------------------------------------------------------------------------- class MergeDirListHandler: public XrdCl::ResponseHandler { public: MergeDirListHandler( bool allowChunked, XrdCl::ResponseHandler *handler ) : allowChunked( allowChunked ), pHandler( handler ) { } virtual void HandleResponse( XrdCl::XRootDStatus *status, XrdCl::AnyObject *response ) { XrdSysMutexHelper lck( mtx ); bool finalrsp = !( status->IsOK() && status->code == XrdCl::suContinue ); try { if( !status->IsOK() ) throw MergeDirLsErr( status, response ); if( !response ) throw MergeDirLsErr(); XrdCl::DirectoryList *dirlist = 0; response->Get( dirlist ); if( !dirlist ) throw MergeDirLsErr(); if( allowChunked ) MergeChunked( dirlist ); else Merge( dirlist ); response->Set( dirlist ); pHandler->HandleResponse( status, response ); } catch( const MergeDirLsErr &err ) { delete status; delete response; pHandler->HandleResponse( err.status, err.response ); } if( finalrsp ) { lck.UnLock(); delete this; } } void MergeChunked( XrdCl::DirectoryList *&response ) { using namespace XrdCl; std::set unique; // set of unique list entries from the response std::set tmp( response->Begin(), response->End() ); // all the unique list entries that were not reported so far std::set_difference( tmp.begin(), tmp.end(), uniquesofar.begin(), uniquesofar.end(), std::inserter( unique, unique.end() ) ); // we update the set of unique list entries that were already // reported to the user's handler for( auto itr = unique.begin(); itr != unique.end(); ++itr ) { ListEntry *ent = *itr; if( !uniquesofar.count( ent ) ) { StatInfo *info = ent->GetStatInfo() ? new StatInfo( *ent->GetStatInfo() ) : 0; ListEntry *newent = new ListEntry( ent->GetHostAddress(), ent->GetName(), info ); uniquesofar.insert( newent ); } } DirectoryList *dirlist = new DirectoryList(); dirlist->SetParentName( response->GetParentName() ); for( auto itr = unique.begin(); itr != unique.end(); ++itr ) { ListEntry *entry = *itr; dirlist->Add( new ListEntry( entry->GetHostAddress(), entry->GetName(), entry->GetStatInfo() ) ); entry->SetStatInfo( 0 ); } delete response; response = dirlist; } static void Merge( XrdCl::DirectoryList *&response ) { std::set unique( response->Begin(), response->End() ); XrdCl::DirectoryList *dirlist = new XrdCl::DirectoryList(); dirlist->SetParentName( response->GetParentName() ); for( auto itr = unique.begin(); itr != unique.end(); ++itr ) { ListEntry *entry = *itr; dirlist->Add( new ListEntry( entry->GetHostAddress(), entry->GetName(), entry->GetStatInfo() ) ); entry->SetStatInfo( 0 ); } delete response; response = dirlist; } private: typedef XrdCl::DirectoryList::ListEntry ListEntry; struct less { bool operator() (const ListEntry *x, const ListEntry *y) const { if( x->GetName() != y->GetName() ) return x->GetName() < y->GetName(); const XrdCl::StatInfo *xStatInfo = x->GetStatInfo(); const XrdCl::StatInfo *yStatInfo = y->GetStatInfo(); if( xStatInfo == yStatInfo ) return false; if( xStatInfo == 0 ) return true; if( yStatInfo == 0 ) return false; if( xStatInfo->GetSize() != yStatInfo->GetSize() ) return xStatInfo->GetSize() < yStatInfo->GetSize(); if( xStatInfo->GetFlags() != yStatInfo->GetFlags() ) return xStatInfo->GetFlags() < yStatInfo->GetFlags(); return false; } }; bool allowChunked; XrdSysMutex mtx; std::set uniquesofar; XrdCl::ResponseHandler *pHandler; }; } namespace XrdCl { struct FileSystemData; //---------------------------------------------------------------------------- //! Wrapper class used to assign a load balancer //---------------------------------------------------------------------------- class AssignLBHandler: public ResponseHandler { public: //------------------------------------------------------------------------ // Constructor and destructor //------------------------------------------------------------------------ AssignLBHandler( std::shared_ptr &fs, ResponseHandler *userHandler ): pFS(fs), pUserHandler(userHandler) {} virtual ~AssignLBHandler() {} //------------------------------------------------------------------------ // Response callback //------------------------------------------------------------------------ virtual void HandleResponseWithHosts( XRootDStatus *status, AnyObject *response, HostList *hostList ); private: std::shared_ptr pFS; ResponseHandler *pUserHandler; }; //---------------------------------------------------------------------------- //! Wrapper class used to assign last URL //---------------------------------------------------------------------------- class AssignLastURLHandler: public ResponseHandler { public: //------------------------------------------------------------------------ // Constructor and destructor //------------------------------------------------------------------------ AssignLastURLHandler( std::shared_ptr &fs, ResponseHandler *userHandler ): pFS(fs), pUserHandler(userHandler) {} virtual ~AssignLastURLHandler() {} //------------------------------------------------------------------------ // Response callback //------------------------------------------------------------------------ virtual void HandleResponseWithHosts( XRootDStatus *status, AnyObject *response, HostList *hostList ); private: std::shared_ptr pFS; ResponseHandler *pUserHandler; }; struct FileSystemData { FileSystemData( const URL &url ) : pLoadBalancerLookupDone( false ), pFollowRedirects( true ), pUrl( new URL( url.GetURL() ) ) { } //------------------------------------------------------------------------ // Send a message in a locked environment //------------------------------------------------------------------------ static XRootDStatus Send( std::shared_ptr &fs, Message *msg, ResponseHandler *handler, MessageSendParams ¶ms ) { Log *log = DefaultEnv::GetLog(); XrdSysMutexHelper scopedLock( fs->pMutex ); log->Dump( FileSystemMsg, "[0x%x@%s] Sending %s", fs.get(), fs->pUrl->GetHostId().c_str(), msg->GetDescription().c_str() ); AssignLastURLHandler *lastUrlHandler = new AssignLastURLHandler( fs, handler ); handler = lastUrlHandler; AssignLBHandler *lbHandler = nullptr; if( !fs->pLoadBalancerLookupDone && fs->pFollowRedirects ) { lbHandler = new AssignLBHandler( fs, handler ); handler = lbHandler; } params.followRedirects = fs->pFollowRedirects; auto st = MessageUtils::SendMessage( *fs->pUrl, msg, handler, params, 0 ); if( !st.IsOK() ) { delete lastUrlHandler; delete lbHandler; } return st; } //---------------------------------------------------------------------------- // Assign a load balancer if it has not already been assigned //---------------------------------------------------------------------------- void AssignLoadBalancer( const URL &url ) { Log *log = DefaultEnv::GetLog(); XrdSysMutexHelper scopedLock( pMutex ); if( pLoadBalancerLookupDone ) return; log->Dump( FileSystemMsg, "[0x%x@%s] Assigning %s as load balancer", this, pUrl->GetHostId().c_str(), url.GetHostId().c_str() ); pUrl.reset( new URL( url ) ); pLoadBalancerLookupDone = true; } //---------------------------------------------------------------------------- // Assign last URL //---------------------------------------------------------------------------- void AssignLastURL( const URL &url ) { Log *log = DefaultEnv::GetLog(); XrdSysMutexHelper scopedLock( pMutex ); log->Dump( FileSystemMsg, "[0x%x@%s] Assigning %s as last URL", this, pUrl->GetHostId().c_str(), url.GetHostId().c_str() ); pLastUrl.reset( new URL( url ) ); } XrdSysMutex pMutex; bool pLoadBalancerLookupDone; bool pFollowRedirects; std::unique_ptr pUrl; std::unique_ptr pLastUrl; }; //---------------------------------------------------------------------------- //! Implementation holding the data members //---------------------------------------------------------------------------- struct FileSystemImpl { FileSystemImpl( const URL &url ) : fsdata( std::make_shared( url ) ) { } std::shared_ptr fsdata; }; //------------------------------------------------------------------------ // Response callback //------------------------------------------------------------------------ void AssignLBHandler::HandleResponseWithHosts( XRootDStatus *status, AnyObject *response, HostList *hostList ) { if( status->IsOK() ) { HostList::reverse_iterator it; for( it = hostList->rbegin(); it != hostList->rend(); ++it ) if( it->loadBalancer ) { pFS->AssignLoadBalancer( it->url ); break; } } bool finalrsp = !( status->IsOK() && status->code == suContinue ); SyncResponseHandler * syncHandler = dynamic_cast( pUserHandler ); if( !syncHandler ) pUserHandler->HandleResponseWithHosts( status, response, hostList ); if( finalrsp ) { if( syncHandler ) pUserHandler->HandleResponseWithHosts( status, response, hostList ); delete this; } } //------------------------------------------------------------------------ // Response callback //------------------------------------------------------------------------ void AssignLastURLHandler::HandleResponseWithHosts( XRootDStatus *status, AnyObject *response, HostList *hostList ) { if( status->IsOK() && hostList ) pFS->AssignLastURL( hostList->front().url ); bool finalrsp = !( status->IsOK() && status->code == suContinue ); SyncResponseHandler *syncHandler = dynamic_cast( pUserHandler ); if( !syncHandler ) pUserHandler->HandleResponseWithHosts( status, response, hostList ); if( finalrsp ) { if( syncHandler ) pUserHandler->HandleResponseWithHosts( status, response, hostList ); delete this; } } //---------------------------------------------------------------------------- // Constructor //---------------------------------------------------------------------------- FileSystem::FileSystem( const URL &url, bool enablePlugIns ): pImpl( new FileSystemImpl( url ) ), pPlugIn(0) { //-------------------------------------------------------------------------- // Check if we need to install a plug-in for this URL //-------------------------------------------------------------------------- if( enablePlugIns ) { Log *log = DefaultEnv::GetLog(); std::string urlStr = url.GetURL(); PlugInFactory *fact = DefaultEnv::GetPlugInManager()->GetFactory(urlStr); if( fact ) { pPlugIn = fact->CreateFileSystem( urlStr ); if( !pPlugIn ) { log->Error( FileMsg, "Plug-in factory failed to produce a plug-in " "for %s, continuing without one", urlStr.c_str() ); } } } if( !pPlugIn ) DefaultEnv::GetForkHandler()->RegisterFileSystemObject( this ); } //---------------------------------------------------------------------------- // Destructor //---------------------------------------------------------------------------- FileSystem::~FileSystem() { if( !pPlugIn ) { if( DefaultEnv::GetForkHandler() ) DefaultEnv::GetForkHandler()->UnRegisterFileSystemObject( this ); } delete pPlugIn; delete pImpl; } //---------------------------------------------------------------------------- // Locate a file - async //---------------------------------------------------------------------------- XRootDStatus FileSystem::Locate( const std::string &path, OpenFlags::Flags flags, ResponseHandler *handler, uint16_t timeout ) { if( pPlugIn ) return pPlugIn->Locate( path, flags, handler, timeout ); std::string fPath = FilterXrdClCgi( path ); Message *msg; ClientLocateRequest *req; MessageUtils::CreateRequest( msg, req, fPath.length() ); req->requestid = kXR_locate; req->options = flags; req->dlen = fPath.length(); msg->Append( fPath.c_str(), fPath.length(), 24 ); MessageSendParams params; params.timeout = timeout; MessageUtils::ProcessSendParams( params ); XRootDTransport::SetDescription( msg ); return FileSystemData::Send( pImpl->fsdata, msg, handler, params ); } //---------------------------------------------------------------------------- // Locate a file - sync //---------------------------------------------------------------------------- XRootDStatus FileSystem::Locate( const std::string &path, OpenFlags::Flags flags, LocationInfo *&response, uint16_t timeout ) { SyncResponseHandler handler; Status st = Locate( path, flags, &handler, timeout ); if( !st.IsOK() ) return st; return MessageUtils::WaitForResponse( &handler, response ); } //---------------------------------------------------------------------------- // Locate a file, recursively locate all disk servers - async //---------------------------------------------------------------------------- XRootDStatus FileSystem::DeepLocate( const std::string &path, OpenFlags::Flags flags, ResponseHandler *handler, uint16_t timeout ) { return Locate( path, flags, new DeepLocateHandler( handler, path, flags, ::time(0)+timeout ), timeout ); } //---------------------------------------------------------------------------- // Locate a file, recursively locate all disk servers - sync //---------------------------------------------------------------------------- XRootDStatus FileSystem::DeepLocate( const std::string &path, OpenFlags::Flags flags, LocationInfo *&response, uint16_t timeout ) { SyncResponseHandler handler; Status st = DeepLocate( path, flags, &handler, timeout ); if( !st.IsOK() ) return st; return MessageUtils::WaitForResponse( &handler, response ); } //---------------------------------------------------------------------------- // Move a directory or a file - async //---------------------------------------------------------------------------- XRootDStatus FileSystem::Mv( const std::string &source, const std::string &dest, ResponseHandler *handler, uint16_t timeout ) { if( pPlugIn ) return pPlugIn->Mv( source, dest, handler, timeout ); std::string fSource = FilterXrdClCgi( source ); std::string fDest = FilterXrdClCgi( dest ); Message *msg; ClientMvRequest *req; MessageUtils::CreateRequest( msg, req, fSource.length() + fDest.length()+1 ); req->requestid = kXR_mv; req->dlen = fSource.length() + fDest.length()+1; req->arg1len = fSource.length(); msg->Append( fSource.c_str(), fSource.length(), 24 ); *msg->GetBuffer(24 + fSource.length()) = ' '; msg->Append( fDest.c_str(), fDest.length(), 25 + fSource.length() ); MessageSendParams params; params.timeout = timeout; MessageUtils::ProcessSendParams( params ); XRootDTransport::SetDescription( msg ); return FileSystemData::Send( pImpl->fsdata, msg, handler, params ); } //---------------------------------------------------------------------------- // Move a directory or a file - sync //---------------------------------------------------------------------------- XRootDStatus FileSystem::Mv( const std::string &source, const std::string &dest, uint16_t timeout ) { SyncResponseHandler handler; Status st = Mv( source, dest, &handler, timeout ); if( !st.IsOK() ) return st; return MessageUtils::WaitForStatus( &handler ); } //---------------------------------------------------------------------------- // Obtain server information - async //---------------------------------------------------------------------------- XRootDStatus FileSystem::Query( QueryCode::Code queryCode, const Buffer &arg, ResponseHandler *handler, uint16_t timeout ) { if( pPlugIn ) return pPlugIn->Query( queryCode, arg, handler, timeout ); Message *msg; ClientQueryRequest *req; MessageUtils::CreateRequest( msg, req, arg.GetSize() ); req->requestid = kXR_query; req->infotype = queryCode; req->dlen = arg.GetSize(); msg->Append( arg.GetBuffer(), arg.GetSize(), 24 ); MessageSendParams params; params.timeout = timeout; MessageUtils::ProcessSendParams( params ); XRootDTransport::SetDescription( msg ); return FileSystemData::Send( pImpl->fsdata, msg, handler, params ); } //---------------------------------------------------------------------------- // Obtain server information - sync //---------------------------------------------------------------------------- XRootDStatus FileSystem::Query( QueryCode::Code queryCode, const Buffer &arg, Buffer *&response, uint16_t timeout ) { SyncResponseHandler handler; Status st = Query( queryCode, arg, &handler, timeout ); if( !st.IsOK() ) return st; return MessageUtils::WaitForResponse( &handler, response ); } //---------------------------------------------------------------------------- // Truncate a file - async //---------------------------------------------------------------------------- XRootDStatus FileSystem::Truncate( const std::string &path, uint64_t size, ResponseHandler *handler, uint16_t timeout ) { if( pPlugIn ) return pPlugIn->Truncate( path, size, handler, timeout ); std::string fPath = FilterXrdClCgi( path ); Message *msg; ClientTruncateRequest *req; MessageUtils::CreateRequest( msg, req, fPath.length() ); req->requestid = kXR_truncate; req->offset = size; req->dlen = fPath.length(); msg->Append( fPath.c_str(), fPath.length(), 24 ); MessageSendParams params; params.timeout = timeout; MessageUtils::ProcessSendParams( params ); XRootDTransport::SetDescription( msg ); return FileSystemData::Send( pImpl->fsdata, msg, handler, params ); } //---------------------------------------------------------------------------- // Truncate a file - sync //---------------------------------------------------------------------------- XRootDStatus FileSystem::Truncate( const std::string &path, uint64_t size, uint16_t timeout ) { SyncResponseHandler handler; Status st = Truncate( path, size, &handler, timeout ); if( !st.IsOK() ) return st; return MessageUtils::WaitForStatus( &handler ); } //---------------------------------------------------------------------------- // Remove a file - async //---------------------------------------------------------------------------- XRootDStatus FileSystem::Rm( const std::string &path, ResponseHandler *handler, uint16_t timeout ) { if( pPlugIn ) return pPlugIn->Rm( path, handler, timeout ); if( pImpl->fsdata->pUrl->IsLocalFile() ) return LocalFS::Instance().Rm( path, handler, timeout ); std::string fPath = FilterXrdClCgi( path ); Message *msg; ClientRmRequest *req; MessageUtils::CreateRequest( msg, req, fPath.length() ); req->requestid = kXR_rm; req->dlen = fPath.length(); msg->Append( fPath.c_str(), fPath.length(), 24 ); MessageSendParams params; params.timeout = timeout; MessageUtils::ProcessSendParams( params ); XRootDTransport::SetDescription( msg ); return FileSystemData::Send( pImpl->fsdata, msg, handler, params ); } //---------------------------------------------------------------------------- // Remove a file - sync //---------------------------------------------------------------------------- XRootDStatus FileSystem::Rm( const std::string &path, uint16_t timeout ) { SyncResponseHandler handler; Status st = Rm( path, &handler, timeout ); if( !st.IsOK() ) return st; return MessageUtils::WaitForStatus( &handler ); } //---------------------------------------------------------------------------- // Create a directory - async //---------------------------------------------------------------------------- XRootDStatus FileSystem::MkDir( const std::string &path, MkDirFlags::Flags flags, Access::Mode mode, ResponseHandler *handler, uint16_t timeout ) { if( pPlugIn ) return pPlugIn->MkDir( path, flags, mode, handler, timeout ); std::string fPath = FilterXrdClCgi( path ); Message *msg; ClientMkdirRequest *req; MessageUtils::CreateRequest( msg, req, fPath.length() ); req->requestid = kXR_mkdir; req->options[0] = flags; req->mode = mode; req->dlen = fPath.length(); msg->Append( fPath.c_str(), fPath.length(), 24 ); MessageSendParams params; params.timeout = timeout; MessageUtils::ProcessSendParams( params ); XRootDTransport::SetDescription( msg ); return FileSystemData::Send( pImpl->fsdata, msg, handler, params ); } //---------------------------------------------------------------------------- // Create a directory - sync //---------------------------------------------------------------------------- XRootDStatus FileSystem::MkDir( const std::string &path, MkDirFlags::Flags flags, Access::Mode mode, uint16_t timeout ) { SyncResponseHandler handler; Status st = MkDir( path, flags, mode, &handler, timeout ); if( !st.IsOK() ) return st; return MessageUtils::WaitForStatus( &handler ); } //---------------------------------------------------------------------------- // Remove a directory - async //---------------------------------------------------------------------------- XRootDStatus FileSystem::RmDir( const std::string &path, ResponseHandler *handler, uint16_t timeout ) { if( pPlugIn ) return pPlugIn->RmDir( path, handler, timeout ); std::string fPath = FilterXrdClCgi( path ); Message *msg; ClientRmdirRequest *req; MessageUtils::CreateRequest( msg, req, fPath.length() ); req->requestid = kXR_rmdir; req->dlen = fPath.length(); msg->Append( fPath.c_str(), fPath.length(), 24 ); MessageSendParams params; params.timeout = timeout; MessageUtils::ProcessSendParams( params ); XRootDTransport::SetDescription( msg ); return FileSystemData::Send( pImpl->fsdata, msg, handler, params ); } //---------------------------------------------------------------------------- // Remove a directory - sync //---------------------------------------------------------------------------- XRootDStatus FileSystem::RmDir( const std::string &path, uint16_t timeout ) { SyncResponseHandler handler; Status st = RmDir( path, &handler, timeout ); if( !st.IsOK() ) return st; return MessageUtils::WaitForStatus( &handler ); } //---------------------------------------------------------------------------- // Change access mode on a directory or a file - async //---------------------------------------------------------------------------- XRootDStatus FileSystem::ChMod( const std::string &path, Access::Mode mode, ResponseHandler *handler, uint16_t timeout ) { if( pPlugIn ) return pPlugIn->ChMod( path, mode, handler, timeout ); std::string fPath = FilterXrdClCgi( path ); Message *msg; ClientChmodRequest *req; MessageUtils::CreateRequest( msg, req, fPath.length() ); req->requestid = kXR_chmod; req->mode = mode; req->dlen = fPath.length(); msg->Append( fPath.c_str(), fPath.length(), 24 ); MessageSendParams params; params.timeout = timeout; MessageUtils::ProcessSendParams( params ); XRootDTransport::SetDescription( msg ); return FileSystemData::Send( pImpl->fsdata, msg, handler, params ); } //---------------------------------------------------------------------------- // Change access mode on a directory or a file - async //---------------------------------------------------------------------------- XRootDStatus FileSystem::ChMod( const std::string &path, Access::Mode mode, uint16_t timeout ) { SyncResponseHandler handler; Status st = ChMod( path, mode, &handler, timeout ); if( !st.IsOK() ) return st; return MessageUtils::WaitForStatus( &handler ); } //---------------------------------------------------------------------------- // Check if the server is alive - async //---------------------------------------------------------------------------- XRootDStatus FileSystem::Ping( ResponseHandler *handler, uint16_t timeout ) { if( pPlugIn ) return pPlugIn->Ping( handler, timeout ); Message *msg; ClientPingRequest *req; MessageUtils::CreateRequest( msg, req ); req->requestid = kXR_ping; MessageSendParams params; params.timeout = timeout; MessageUtils::ProcessSendParams( params ); XRootDTransport::SetDescription( msg ); return FileSystemData::Send( pImpl->fsdata, msg, handler, params ); } //---------------------------------------------------------------------------- // Check if the server is alive - sync //---------------------------------------------------------------------------- XRootDStatus FileSystem::Ping( uint16_t timeout ) { SyncResponseHandler handler; Status st = Ping( &handler, timeout ); if( !st.IsOK() ) return st; return MessageUtils::WaitForStatus( &handler ); } //---------------------------------------------------------------------------- // Obtain status information for a path - async //---------------------------------------------------------------------------- XRootDStatus FileSystem::Stat( const std::string &path, ResponseHandler *handler, uint16_t timeout ) { if( pPlugIn ) return pPlugIn->Stat( path, handler, timeout ); if( pImpl->fsdata->pUrl->IsLocalFile() ) return LocalFS::Instance().Stat( path, handler, timeout ); std::string fPath = FilterXrdClCgi( path ); Message *msg; ClientStatRequest *req; MessageUtils::CreateRequest( msg, req, fPath.length() ); req->requestid = kXR_stat; req->options = 0; req->dlen = fPath.length(); msg->Append( fPath.c_str(), fPath.length(), 24 ); MessageSendParams params; params.timeout = timeout; MessageUtils::ProcessSendParams( params ); XRootDTransport::SetDescription( msg ); return FileSystemData::Send( pImpl->fsdata, msg, handler, params ); } //---------------------------------------------------------------------------- // Obtain status information for a path - sync //---------------------------------------------------------------------------- XRootDStatus FileSystem::Stat( const std::string &path, StatInfo *&response, uint16_t timeout ) { SyncResponseHandler handler; Status st = Stat( path, &handler, timeout ); if( !st.IsOK() ) return st; return MessageUtils::WaitForResponse( &handler, response ); } //---------------------------------------------------------------------------- // Obtain status information for a path - async //---------------------------------------------------------------------------- XRootDStatus FileSystem::StatVFS( const std::string &path, ResponseHandler *handler, uint16_t timeout ) { if( pPlugIn ) return pPlugIn->StatVFS( path, handler, timeout ); std::string fPath = FilterXrdClCgi( path ); Message *msg; ClientStatRequest *req; MessageUtils::CreateRequest( msg, req, fPath.length() ); req->requestid = kXR_stat; req->options = kXR_vfs; req->dlen = fPath.length(); msg->Append( fPath.c_str(), fPath.length(), 24 ); MessageSendParams params; params.timeout = timeout; MessageUtils::ProcessSendParams( params ); XRootDTransport::SetDescription( msg ); return FileSystemData::Send( pImpl->fsdata, msg, handler, params ); } //---------------------------------------------------------------------------- // Obtain status information for a path - sync //---------------------------------------------------------------------------- XRootDStatus FileSystem::StatVFS( const std::string &path, StatInfoVFS *&response, uint16_t timeout ) { SyncResponseHandler handler; Status st = StatVFS( path, &handler, timeout ); if( !st.IsOK() ) return st; return MessageUtils::WaitForResponse( &handler, response ); } //---------------------------------------------------------------------------- // Obtain server protocol information - async //---------------------------------------------------------------------------- XRootDStatus FileSystem::Protocol( ResponseHandler *handler, uint16_t timeout ) { if( pPlugIn ) return pPlugIn->Protocol( handler, timeout ); Message *msg; ClientProtocolRequest *req; MessageUtils::CreateRequest( msg, req ); req->requestid = kXR_protocol; req->clientpv = kXR_PROTOCOLVERSION; MessageSendParams params; params.timeout = timeout; MessageUtils::ProcessSendParams( params ); XRootDTransport::SetDescription( msg ); return FileSystemData::Send( pImpl->fsdata, msg, handler, params ); } //---------------------------------------------------------------------------- // Obtain server protocol information - sync //---------------------------------------------------------------------------- XRootDStatus FileSystem::Protocol( ProtocolInfo *&response, uint16_t timeout ) { SyncResponseHandler handler; Status st = Protocol( &handler, timeout ); if( !st.IsOK() ) return st; return MessageUtils::WaitForResponse( &handler, response ); } //---------------------------------------------------------------------------- // List entries of a directory - async //---------------------------------------------------------------------------- XRootDStatus FileSystem::DirList( const std::string &path, DirListFlags::Flags flags, ResponseHandler *handler, uint16_t timeout ) { if( pPlugIn ) return pPlugIn->DirList( path, flags, handler, timeout ); URL url = URL( path ); std::string fPath = FilterXrdClCgi( path ); if( flags & DirListFlags::Zip ) { // stat the file to check if it is a directory or a file // the ZIP handler will take care of the rest ZipListHandler *zipHandler = new ZipListHandler( *pImpl->fsdata->pUrl, path, flags, handler, timeout ); XRootDStatus st = Stat( path, zipHandler, timeout ); if( !st.IsOK() ) delete zipHandler; return st; } Message *msg; ClientDirlistRequest *req; MessageUtils::CreateRequest( msg, req, fPath.length() ); req->requestid = kXR_dirlist; req->dlen = fPath.length(); if( ( flags & DirListFlags::Stat ) || ( flags & DirListFlags::Recursive ) ) req->options[0] = kXR_dstat; if( ( flags & DirListFlags::Cksm ) ) req->options[0] = kXR_dstat | kXR_dcksm; if( flags & DirListFlags::Recursive ) handler = new RecursiveDirListHandler( *pImpl->fsdata->pUrl, url.GetPath(), flags, handler, timeout ); if( flags & DirListFlags::Merge ) handler = new MergeDirListHandler( flags & DirListFlags::Chunked, handler ); msg->Append( fPath.c_str(), fPath.length(), 24 ); MessageSendParams params; params.timeout = timeout; if( flags & DirListFlags::Chunked ) params.chunkedResponse = true; MessageUtils::ProcessSendParams( params ); XRootDTransport::SetDescription( msg ); return FileSystemData::Send( pImpl->fsdata, msg, handler, params ); } //---------------------------------------------------------------------------- // List entries of a directory - sync //---------------------------------------------------------------------------- XRootDStatus FileSystem::DirList( const std::string &path, DirListFlags::Flags flags, DirectoryList *&response, uint16_t timeout ) { //-------------------------------------------------------------------------- // Chunked response is only possible for async DirList call //-------------------------------------------------------------------------- if( flags & DirListFlags::Chunked ) return XRootDStatus( stError, errNotSupported ); //-------------------------------------------------------------------------- // If the path ends with '.zip' extension add Zip flag //-------------------------------------------------------------------------- static const std::string zip_sufix = ".zip"; if( path.size() >= zip_sufix.size() && std::equal( zip_sufix.rbegin(), zip_sufix.rend(), path.rbegin() ) ) flags |= DirListFlags::Zip; //-------------------------------------------------------------------------- // We do the deep locate and ask all the returned servers for the list //-------------------------------------------------------------------------- if( flags & DirListFlags::Locate ) { //------------------------------------------------------------------------ // Locate all the disk servers holding the directory //------------------------------------------------------------------------ LocationInfo *locations; std::string locatePath = "*"; locatePath += path; XRootDStatus st = DeepLocate( locatePath, OpenFlags::PrefName | OpenFlags::Compress | OpenFlags::IntentDirList, locations ); if( !st.IsOK() ) return st; if( locations->GetSize() == 0 ) { delete locations; return XRootDStatus( stError, errNotFound ); } // Check if destination is a data server bool isserver = false; AnyObject obj; st = DefaultEnv::GetPostMaster()->QueryTransport( *pImpl->fsdata->pUrl, XRootDQuery::ServerFlags, obj ); if( st.IsOK() ) { int *ptr = 0; obj.Get( ptr ); isserver = ( *ptr & kXR_isServer ); delete ptr; } if( !isserver ) { //------------------------------------------------------------------------ // Ask each server for a directory list //------------------------------------------------------------------------ flags &= ~DirListFlags::Locate; FileSystem *fs; DirectoryList *currentResp = 0; uint32_t errors = 0; uint32_t numLocations = locations->GetSize(); bool partial = st.code == suPartial ? true : false; response = new DirectoryList(); response->SetParentName( path ); for( uint32_t i = 0; i < locations->GetSize(); ++i ) { URL locationURL( locations->At(i).GetAddress() ); // make sure the original protocol is preserved (root vs roots) locationURL.SetProtocol( pImpl->fsdata->pUrl->GetProtocol() ); fs = new FileSystem( locationURL ); st = fs->DirList( path, flags, currentResp, timeout ); if( !st.IsOK() ) { ++errors; delete fs; continue; } if( st.code == suPartial ) partial = true; DirectoryList::Iterator it; for( it = currentResp->Begin(); it != currentResp->End(); ++it ) { response->Add( *it ); *it = 0; } delete fs; delete currentResp; fs = 0; currentResp = 0; } delete locations; if( flags & DirListFlags::Merge ) MergeDirListHandler::Merge( response ); if( errors || partial ) { if( errors == numLocations ) return st; return XRootDStatus( stOK, suPartial ); } return XRootDStatus(); } else delete locations; } //-------------------------------------------------------------------------- // We just ask the current server //-------------------------------------------------------------------------- SyncResponseHandler handler; XRootDStatus st = DirList( path, flags, &handler, timeout ); if( !st.IsOK() ) return st; st = MessageUtils::WaitForResponse( &handler, response ); if( !st.IsOK() ) return st; //-------------------------------------------------------------------------- // Do the stats on all the entries if necessary. // If we already have the stat objects it means that the bulk stat has // succeeded. //-------------------------------------------------------------------------- if( !(flags & DirListFlags::Stat) ) return st; if( response->GetSize() && response->At(0)->GetStatInfo() ) return st; uint32_t quota = response->GetSize() <= 1024 ? response->GetSize() : 1024; RequestSync sync( response->GetSize(), quota ); for( uint32_t i = 0; i < response->GetSize(); ++i ) { std::string fullPath = response->GetParentName()+response->At(i)->GetName(); ResponseHandler *handler = new DirListStatHandler( response, i, &sync ); st = Stat( fullPath, handler, timeout ); if( !st.IsOK() ) { sync.TaskDone( false ); delete handler; } sync.WaitForQuota(); } sync.WaitForAll(); if( sync.FailureCount() ) return XRootDStatus( stOK, suPartial ); return XRootDStatus(); } //---------------------------------------------------------------------------- // Send cache info to the server - async //---------------------------------------------------------------------------- XRootDStatus FileSystem::SendCache( const std::string &info, ResponseHandler *handler, uint16_t timeout ) { // Note: adding SendCache() to the FileSystemPlugin class breaks ABI! // So, the class is missing this until we do a major release. TODO //if( pPlugIn ) // return pPlugIn->SendCache( info, handler, timeout ); return SendSet("cache ", info, handler, timeout ); } //---------------------------------------------------------------------------- //! Send cache info to the server - sync //---------------------------------------------------------------------------- XRootDStatus FileSystem::SendCache( const std::string &info, Buffer *&response, uint16_t timeout ) { SyncResponseHandler handler; Status st = SendCache( info, &handler, timeout ); if( !st.IsOK() ) return st; return MessageUtils::WaitForResponse( &handler, response ); } //---------------------------------------------------------------------------- // Send info to the server - async //---------------------------------------------------------------------------- XRootDStatus FileSystem::SendInfo( const std::string &info, ResponseHandler *handler, uint16_t timeout ) { if( pPlugIn ) return pPlugIn->SendInfo( info, handler, timeout ); return SendSet("monitor info ", info, handler, timeout ); } //---------------------------------------------------------------------------- //! Send info to the server - sync //---------------------------------------------------------------------------- XRootDStatus FileSystem::SendInfo( const std::string &info, Buffer *&response, uint16_t timeout ) { SyncResponseHandler handler; Status st = SendInfo( info, &handler, timeout ); if( !st.IsOK() ) return st; return MessageUtils::WaitForResponse( &handler, response ); } //---------------------------------------------------------------------------- // Send set request to the server - async //---------------------------------------------------------------------------- XRootDStatus FileSystem::SendSet( const char *prefix, const std::string &info, ResponseHandler *handler, uint16_t timeout ) { Message *msg; ClientSetRequest *req; size_t prefixLen = strlen( prefix ); MessageUtils::CreateRequest( msg, req, info.length()+prefixLen ); req->requestid = kXR_set; req->dlen = info.length()+prefixLen; msg->Append( prefix, prefixLen, 24 ); msg->Append( info.c_str(), info.length(), 24+prefixLen ); MessageSendParams params; params.timeout = timeout; MessageUtils::ProcessSendParams( params ); XRootDTransport::SetDescription( msg ); return FileSystemData::Send( pImpl->fsdata, msg, handler, params ); } //---------------------------------------------------------------------------- // Prepare one or more files for access - async //---------------------------------------------------------------------------- XRootDStatus FileSystem::Prepare( const std::vector &fileList, PrepareFlags::Flags flags, uint8_t priority, ResponseHandler *handler, uint16_t timeout ) { if( pPlugIn ) return pPlugIn->Prepare( fileList, flags, priority, handler, timeout ); std::vector::const_iterator it; std::string list; for( it = fileList.begin(); it != fileList.end(); ++it ) { list += *it; list += "\n"; } list.erase( list.length()-1, 1 ); Message *msg; ClientPrepareRequest *req; MessageUtils::CreateRequest( msg, req, list.length() ); req->requestid = kXR_prepare; req->options = 0xff & flags; req->optionX = 0xffff & ( flags >> 8 ); req->prty = priority; req->dlen = list.length(); msg->Append( list.c_str(), list.length(), 24 ); MessageSendParams params; params.timeout = timeout; MessageUtils::ProcessSendParams( params ); XRootDTransport::SetDescription( msg ); return FileSystemData::Send( pImpl->fsdata, msg, handler, params ); } //------------------------------------------------------------------------ // Prepare one or more files for access - sync //------------------------------------------------------------------------ XRootDStatus FileSystem::Prepare( const std::vector &fileList, PrepareFlags::Flags flags, uint8_t priority, Buffer *&response, uint16_t timeout ) { SyncResponseHandler handler; Status st = Prepare( fileList, flags, priority, &handler, timeout ); if( !st.IsOK() ) return st; return MessageUtils::WaitForResponse( &handler, response ); } //------------------------------------------------------------------------ // Set extended attributes - async //------------------------------------------------------------------------ XRootDStatus FileSystem::SetXAttr( const std::string &path, const std::vector &attrs, ResponseHandler *handler, uint16_t timeout ) { if( pPlugIn ) return XRootDStatus( stError, errNotSupported ); return XAttrOperationImpl( kXR_fattrSet, 0, path, attrs, handler, timeout ); } //------------------------------------------------------------------------ // Set extended attributes - sync //------------------------------------------------------------------------ XRootDStatus FileSystem::SetXAttr( const std::string &path, const std::vector &attrs, std::vector &result, uint16_t timeout ) { SyncResponseHandler handler; XRootDStatus st = SetXAttr( path, attrs, &handler, timeout ); if( !st.IsOK() ) return st; std::vector *resp = 0; st = MessageUtils::WaitForResponse( &handler, resp ); if( resp ) result.swap( *resp ); delete resp; return st; } //------------------------------------------------------------------------ // Get extended attributes - async //------------------------------------------------------------------------ XRootDStatus FileSystem::GetXAttr( const std::string &path, const std::vector &attrs, ResponseHandler *handler, uint16_t timeout ) { if( pPlugIn ) return XRootDStatus( stError, errNotSupported ); return XAttrOperationImpl( kXR_fattrGet, 0, path, attrs, handler, timeout ); } //------------------------------------------------------------------------ // Get extended attributes - sync //------------------------------------------------------------------------ XRootDStatus FileSystem::GetXAttr( const std::string &path, const std::vector &attrs, std::vector &result, uint16_t timeout ) { SyncResponseHandler handler; XRootDStatus st = GetXAttr( path, attrs, &handler, timeout ); if( !st.IsOK() ) return st; std::vector *resp = 0; st = MessageUtils::WaitForResponse( &handler, resp ); if( resp ) result.swap( *resp ); delete resp; return st; } //------------------------------------------------------------------------ // Delete extended attributes - async //------------------------------------------------------------------------ XRootDStatus FileSystem::DelXAttr( const std::string &path, const std::vector &attrs, ResponseHandler *handler, uint16_t timeout ) { if( pPlugIn ) return XRootDStatus( stError, errNotSupported ); return XAttrOperationImpl( kXR_fattrDel, 0, path, attrs, handler, timeout ); } //------------------------------------------------------------------------ // Delete extended attributes - sync //------------------------------------------------------------------------ XRootDStatus FileSystem::DelXAttr( const std::string &path, const std::vector &attrs, std::vector &result, uint16_t timeout ) { SyncResponseHandler handler; XRootDStatus st = DelXAttr( path, attrs, &handler, timeout ); if( !st.IsOK() ) return st; std::vector *resp = 0; st = MessageUtils::WaitForResponse( &handler, resp ); if( resp ) result.swap( *resp ); delete resp; return st; } //------------------------------------------------------------------------ // List extended attributes - async //------------------------------------------------------------------------ XRootDStatus FileSystem::ListXAttr( const std::string &path, ResponseHandler *handler, uint16_t timeout ) { if( pPlugIn ) return XRootDStatus( stError, errNotSupported ); static const std::vector nothing; return XAttrOperationImpl( kXR_fattrList, ClientFattrRequest::aData, path, nothing, handler, timeout ); } //------------------------------------------------------------------------ // List extended attributes - sync //------------------------------------------------------------------------ XRootDStatus FileSystem::ListXAttr( const std::string &path, std::vector &result, uint16_t timeout ) { SyncResponseHandler handler; XRootDStatus st = ListXAttr( path, &handler, timeout ); if( !st.IsOK() ) return st; std::vector *resp = 0; st = MessageUtils::WaitForResponse( &handler, resp ); if( resp ) result.swap( *resp ); delete resp; return st; } //---------------------------------------------------------------------------- // Set file property //---------------------------------------------------------------------------- bool FileSystem::SetProperty( const std::string &name, const std::string &value ) { if( pPlugIn ) return pPlugIn->SetProperty( name, value ); if( name == "FollowRedirects" ) { if( value == "true" ) pImpl->fsdata->pFollowRedirects = true; else pImpl->fsdata->pFollowRedirects = false; return true; } return false; } //---------------------------------------------------------------------------- // Get file property //---------------------------------------------------------------------------- bool FileSystem::GetProperty( const std::string &name, std::string &value ) const { if( pPlugIn ) return pPlugIn->GetProperty( name, value ); if( name == "FollowRedirects" ) { if( pImpl->fsdata->pFollowRedirects ) value = "true"; else value = "false"; return true; } else if( name == "LastURL" ) { if( pImpl->fsdata->pLastUrl ) { value = pImpl->fsdata->pLastUrl->GetURL(); return true; } else return false; } return false; } //------------------------------------------------------------------------ // Generic implementation of xattr operation //------------------------------------------------------------------------ template Status FileSystem::XAttrOperationImpl( kXR_char subcode, kXR_char options, const std::string &path, const std::vector &attrs, ResponseHandler *handler, uint16_t timeout ) { Message *msg; ClientFattrRequest *req; MessageUtils::CreateRequest( msg, req ); req->requestid = kXR_fattr; req->subcode = subcode; req->options = options; req->numattr = attrs.size(); memset( req->fhandle, 0, 4 ); XRootDStatus st = MessageUtils::CreateXAttrBody( msg, attrs, path ); if( !st.IsOK() ) return st; MessageSendParams params; params.timeout = timeout; MessageUtils::ProcessSendParams( params ); XRootDTransport::SetDescription( msg ); return FileSystemData::Send( pImpl->fsdata, msg, handler, params ); } //------------------------------------------------------------------------ // Lock the internal lock //------------------------------------------------------------------------ void FileSystem::Lock() { pImpl->fsdata->pMutex.Lock(); } //------------------------------------------------------------------------ // Unlock the internal lock //------------------------------------------------------------------------ void FileSystem::UnLock() { pImpl->fsdata->pMutex.UnLock(); } } xrootd-5.6.9/src/XrdCl/XrdClFileSystem.hh000066400000000000000000001337521457266313600202470ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2014 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #ifndef __XRD_CL_FILE_SYSTEM_HH__ #define __XRD_CL_FILE_SYSTEM_HH__ #include "XrdCl/XrdClURL.hh" #include "XrdCl/XrdClStatus.hh" #include "XrdOuc/XrdOucEnum.hh" #include "XrdOuc/XrdOucCompiler.hh" #include "XrdCl/XrdClXRootDResponses.hh" #include "XrdSys/XrdSysPthread.hh" #include "XProtocol/XProtocol.hh" #include #include namespace XrdCl { class PostMaster; class FileSystemPlugIn; struct MessageSendParams; //---------------------------------------------------------------------------- //! XRootD query request codes //---------------------------------------------------------------------------- struct QueryCode { //-------------------------------------------------------------------------- //! XRootD query request codes //-------------------------------------------------------------------------- enum Code { Config = kXR_Qconfig, //!< Query server configuration ChecksumCancel = kXR_Qckscan, //!< Query file checksum cancellation Checksum = kXR_Qcksum, //!< Query file checksum Opaque = kXR_Qopaque, //!< Implementation dependent OpaqueFile = kXR_Qopaquf, //!< Implementation dependent Prepare = kXR_QPrep, //!< Query prepare status Space = kXR_Qspace, //!< Query logical space stats Stats = kXR_QStats, //!< Query server stats Visa = kXR_Qvisa, //!< Query file visa attributes XAttr = kXR_Qxattr //!< Query file extended attributes }; }; //---------------------------------------------------------------------------- //! Open flags, may be or'd when appropriate //---------------------------------------------------------------------------- struct OpenFlags { //-------------------------------------------------------------------------- //! Open flags, may be or'd when appropriate //-------------------------------------------------------------------------- enum Flags { None = 0, //!< Nothing Compress = kXR_compress, //!< Read compressed data for open (ignored), //!< for kXR_locate return unique hosts Delete = kXR_delete, //!< Open a new file, deleting any existing //!< file Force = kXR_force, //!< Ignore file usage rules, for kXR_locate //!< it means ignoreing network dependencies MakePath = kXR_mkpath, //!< Create directory path if it does not //!< already exist New = kXR_new, //!< Open the file only if it does not already //!< exist NoWait = kXR_nowait, //!< Open the file only if it does not cause //!< a wait. For locate: provide a location as //!< soon as one becomes known. This means //!< that not all locations are necessarily //!< returned. If the file does not exist a //!< wait is still imposed. // Append = kXR_open_apnd, //!< Open only for appending Read = kXR_open_read, //!< Open only for reading Update = kXR_open_updt, //!< Open for reading and writing Write = kXR_open_wrto, //!< Open only for writing POSC = kXR_posc, //!< Enable Persist On Successful Close //!< processing Refresh = kXR_refresh, //!< Refresh the cached information on file's //!< location. Voids NoWait. Replica = kXR_replica, //!< The file is being opened for replica //!< creation SeqIO = kXR_seqio, //!< File will be read or written sequentially PrefName = kXR_prefname, //!< Hostname response is prefered, applies //!< only to FileSystem::Locate IntentDirList = kXR_4dirlist //!< Make sure the server knows we are doing //!< locate in context of a dir list operation }; }; XRDOUC_ENUM_OPERATORS( OpenFlags::Flags ) //---------------------------------------------------------------------------- //! Access mode //---------------------------------------------------------------------------- struct Access { //-------------------------------------------------------------------------- //! Access mode //-------------------------------------------------------------------------- enum Mode { None = 0, UR = kXR_ur, //!< owner readable UW = kXR_uw, //!< owner writable UX = kXR_ux, //!< owner executable/browsable GR = kXR_gr, //!< group readable GW = kXR_gw, //!< group writable GX = kXR_gx, //!< group executable/browsable OR = kXR_or, //!< world readable OW = kXR_ow, //!< world writeable OX = kXR_ox //!< world executable/browsable }; }; XRDOUC_ENUM_OPERATORS( Access::Mode ) //---------------------------------------------------------------------------- //! MkDir flags //---------------------------------------------------------------------------- struct MkDirFlags { enum Flags { None = 0, //!< Nothing special MakePath = 1 //!< create the entire directory tree if it doesn't exist }; }; XRDOUC_ENUM_OPERATORS( MkDirFlags::Flags ) //---------------------------------------------------------------------------- //! DirList flags //---------------------------------------------------------------------------- struct DirListFlags { enum Flags { None = 0, //!< Nothing special Stat = 1, //!< Stat each entry Locate = 2, //!< Locate all servers hosting the directory and send //!< the dirlist request to all of them Recursive = 4, //!< Do a recursive listing Merge = 8, //!< Merge duplicates Chunked = 16, //!< Serve chunked results for better performance Zip = 32, //!< List content of ZIP files Cksm = 64 //!< Get checksum for every entry }; }; XRDOUC_ENUM_OPERATORS( DirListFlags::Flags ) //---------------------------------------------------------------------------- //! Prepare flags //---------------------------------------------------------------------------- struct PrepareFlags { enum Flags { None = 0, //!< no flags Colocate = kXR_coloc, //!< co-locate staged files, if possible Fresh = kXR_fresh, //!< refresh file access time even if //!< the location is known Stage = kXR_stage, //!< stage the file to disk if it is not //!< online WriteMode = kXR_wmode, //!< the file will be accessed for //!< modification Cancel = kXR_cancel, //!< cancel staging request Evict = kXR_evict << 8 //!< evict the file from disk cache //!< we have to shift kXR_evict as its value //!< is the same as cancel's because this //!< flag goes to options extension }; }; XRDOUC_ENUM_OPERATORS( PrepareFlags::Flags ) //---------------------------------------------------------------------------- //! Forward declaration of the implementation class holding the data members //---------------------------------------------------------------------------- struct FileSystemImpl; //---------------------------------------------------------------------------- //! Send file/filesystem queries to an XRootD cluster //---------------------------------------------------------------------------- class FileSystem { friend class AssignLBHandler; friend class ForkHandler; public: typedef std::vector LocationList; //!< Location list //------------------------------------------------------------------------ //! Constructor //! //! @param url URL of the entry-point server to be contacted //! @param enablePlugIns enable the plug-in mechanism for this object //------------------------------------------------------------------------ FileSystem( const URL &url, bool enablePlugIns = true ); //------------------------------------------------------------------------ //! Destructor //------------------------------------------------------------------------ ~FileSystem(); //------------------------------------------------------------------------ //! Locate a file - async //! //! @param path path to the file to be located //! @param flags some of the OpenFlags::Flags //! @param handler handler to be notified when the response arrives, //! the response parameter will hold a Buffer object //! if the procedure is successful //! @param timeout timeout value, if 0 the environment default will //! be used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus Locate( const std::string &path, OpenFlags::Flags flags, ResponseHandler *handler, uint16_t timeout = 0 ) XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ //! Locate a file - sync //! //! @param path path to the file to be located //! @param flags some of the OpenFlags::Flags //! @param response the response (to be deleted by the user) //! @param timeout timeout value, if 0 the environment default will //! be used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus Locate( const std::string &path, OpenFlags::Flags flags, LocationInfo *&response, uint16_t timeout = 0 ) XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ //! Locate a file, recursively locate all disk servers - async //! //! @param path path to the file to be located //! @param flags some of the OpenFlags::Flags //! @param handler handler to be notified when the response arrives, //! the response parameter will hold a Buffer object //! if the procedure is successful //! @param timeout timeout value, if 0 the environment default will //! be used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus DeepLocate( const std::string &path, OpenFlags::Flags flags, ResponseHandler *handler, uint16_t timeout = 0 ) XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ //! Locate a file, recursively locate all disk servers - sync //! //! @param path path to the file to be located //! @param flags some of the OpenFlags::Flags //! @param response the response (to be deleted by the user) //! @param timeout timeout value, if 0 the environment default will //! be used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus DeepLocate( const std::string &path, OpenFlags::Flags flags, LocationInfo *&response, uint16_t timeout = 0 ) XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ //! Move a directory or a file - async //! //! @param source the file or directory to be moved //! @param dest the new name //! @param handler handler to be notified when the response arrives, //! @param timeout timeout value, if 0 the environment default will //! be used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus Mv( const std::string &source, const std::string &dest, ResponseHandler *handler, uint16_t timeout = 0 ) XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ //! Move a directory or a file - sync //! //! @param source the file or directory to be moved //! @param dest the new name //! @param timeout timeout value, if 0 the environment default will //! be used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus Mv( const std::string &source, const std::string &dest, uint16_t timeout = 0 ) XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ //! Obtain server information - async //! //! @param queryCode the query code as specified in the QueryCode struct //! @param arg query argument //! @param handler handler to be notified when the response arrives, //! the response parameter will hold a Buffer object //! if the procedure is successful //! @param timeout timeout value, if 0 the environment default will //! be used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus Query( QueryCode::Code queryCode, const Buffer &arg, ResponseHandler *handler, uint16_t timeout = 0 ) XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ //! Obtain server information - sync //! //! @param queryCode the query code as specified in the QueryCode struct //! @param arg query argument //! @param response the response (to be deleted by the user) //! @param timeout timeout value, if 0 the environment default will //! be used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus Query( QueryCode::Code queryCode, const Buffer &arg, Buffer *&response, uint16_t timeout = 0 ) XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ //! Truncate a file - async //! //! @param path path to the file to be truncated //! @param size file size //! @param handler handler to be notified when the response arrives //! @param timeout timeout value, if 0 the environment default will //! be used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus Truncate( const std::string &path, uint64_t size, ResponseHandler *handler, uint16_t timeout = 0 ) XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ //! Truncate a file - sync //! //! @param path path to the file to be truncated //! @param size file size //! @param timeout timeout value, if 0 the environment default will //! be used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus Truncate( const std::string &path, uint64_t size, uint16_t timeout = 0 ) XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ //! Remove a file - async //! //! @param path path to the file to be removed //! @param handler handler to be notified when the response arrives //! @param timeout timeout value, if 0 the environment default will //! be used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus Rm( const std::string &path, ResponseHandler *handler, uint16_t timeout = 0 ) XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ //! Remove a file - sync //! //! @param path path to the file to be removed //! @param timeout timeout value, if 0 the environment default will //! be used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus Rm( const std::string &path, uint16_t timeout = 0 ) XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ //! Create a directory - async //! //! @param path path to the directory //! @param flags or'd MkDirFlags //! @param mode access mode, or'd Access::Mode //! @param handler handler to be notified when the response arrives //! @param timeout timeout value, if 0 the environment default will //! be used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus MkDir( const std::string &path, MkDirFlags::Flags flags, Access::Mode mode, ResponseHandler *handler, uint16_t timeout = 0 ) XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ //! Create a directory - sync //! //! @param path path to the directory //! @param flags or'd MkDirFlags //! @param mode access mode, or'd Access::Mode //! @param timeout timeout value, if 0 the environment default will //! be used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus MkDir( const std::string &path, MkDirFlags::Flags flags, Access::Mode mode, uint16_t timeout = 0 ) XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ //! Remove a directory - async //! //! @param path path to the directory to be removed //! @param handler handler to be notified when the response arrives //! @param timeout timeout value, if 0 the environment default will //! be used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus RmDir( const std::string &path, ResponseHandler *handler, uint16_t timeout = 0 ) XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ //! Remove a directory - sync //! //! @param path path to the directory to be removed //! @param timeout timeout value, if 0 the environment default will //! be used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus RmDir( const std::string &path, uint16_t timeout = 0 ) XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ //! Change access mode on a directory or a file - async //! //! @param path file/directory path //! @param mode access mode, or'd Access::Mode //! @param handler handler to be notified when the response arrives //! @param timeout timeout value, if 0 the environment default will //! be used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus ChMod( const std::string &path, Access::Mode mode, ResponseHandler *handler, uint16_t timeout = 0 ) XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ //! Change access mode on a directory or a file - sync //! //! @param path file/directory path //! @param mode access mode, or'd Access::Mode //! @param timeout timeout value, if 0 the environment default will //! be used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus ChMod( const std::string &path, Access::Mode mode, uint16_t timeout = 0 ) XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ //! Check if the server is alive - async //! //! @param handler handler to be notified when the response arrives //! @param timeout timeout value, if 0 the environment default will //! be used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus Ping( ResponseHandler *handler, uint16_t timeout = 0 ) XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ //! Check if the server is alive - sync //! //! @param timeout timeout value, if 0 the environment default will //! be used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus Ping( uint16_t timeout = 0 ) XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ //! Obtain status information for a path - async //! //! @param path file/directory path //! @param handler handler to be notified when the response arrives, //! the response parameter will hold a StatInfo object //! if the procedure is successful //! @param timeout timeout value, if 0 the environment default will //! be used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus Stat( const std::string &path, ResponseHandler *handler, uint16_t timeout = 0 ) XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ //! Obtain status information for a path - sync //! //! @param path file/directory path //! @param response the response (to be deleted by the user only if the //! procedure is successful) //! @param timeout timeout value, if 0 the environment default will //! be used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus Stat( const std::string &path, StatInfo *&response, uint16_t timeout = 0 ) XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ //! Obtain status information for a Virtual File System - async //! //! @param path file/directory path //! @param handler handler to be notified when the response arrives, //! the response parameter will hold a StatInfoVFS object //! if the procedure is successful //! @param timeout timeout value, if 0 the environment default will //! be used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus StatVFS( const std::string &path, ResponseHandler *handler, uint16_t timeout = 0 ) XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ //! Obtain status information for a Virtual File System - sync //! //! @param path file/directory path //! @param response the response (to be deleted by the user) //! @param timeout timeout value, if 0 the environment default will //! be used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus StatVFS( const std::string &path, StatInfoVFS *&response, uint16_t timeout = 0 ) XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ //! Obtain server protocol information - async //! //! @param handler handler to be notified when the response arrives, //! the response parameter will hold a ProtocolInfo object //! if the procedure is successful //! @param timeout timeout value, if 0 the environment default will //! be used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus Protocol( ResponseHandler *handler, uint16_t timeout = 0 ) XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ //! Obtain server protocol information - sync //! //! @param response the response (to be deleted by the user) //! @param timeout timeout value, if 0 the environment default will //! be used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus Protocol( ProtocolInfo *&response, uint16_t timeout = 0 ) XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ //! List entries of a directory - async //! //! @param path directory path //! @param flags currently unused //! @param handler handler to be notified when the response arrives, //! the response parameter will hold a DirectoryList //! object if the procedure is successful //! @param timeout timeout value, if 0 the environment default will //! be used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus DirList( const std::string &path, DirListFlags::Flags flags, ResponseHandler *handler, uint16_t timeout = 0 ) XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ //! List entries of a directory - sync //! //! @param path directory path //! @param flags DirListFlags //! @param response the response (to be deleted by the user) //! @param timeout timeout value, if 0 the environment default will //! be used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus DirList( const std::string &path, DirListFlags::Flags flags, DirectoryList *&response, uint16_t timeout = 0 ) XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ //! Send cache into the server - async //! //! @param info the info string to be sent //! @param handler handler to be notified when the response arrives, //! the response parameter will hold a Buffer object //! if the procedure is successful //! @param timeout timeout value, if 0 the environment default will //! be used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus SendCache( const std::string &info, ResponseHandler *handler, uint16_t timeout = 0 ) XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ //! Send cache into the server - sync //! //! @param info the info string to be sent //! @param response the response (to be deleted by the user) //! @param timeout timeout value, if 0 the environment default will //! be used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus SendCache( const std::string &info, Buffer *&response, uint16_t timeout = 0 ) XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ //! Send info to the server (up to 1024 characters)- async //! //! @param info the info string to be sent //! @param handler handler to be notified when the response arrives, //! the response parameter will hold a Buffer object //! if the procedure is successful //! @param timeout timeout value, if 0 the environment default will //! be used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus SendInfo( const std::string &info, ResponseHandler *handler, uint16_t timeout = 0 ) XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ //! Send info to the server (up to 1024 characters) - sync //! //! @param info the info string to be sent //! @param response the response (to be deleted by the user) //! @param timeout timeout value, if 0 the environment default will //! be used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus SendInfo( const std::string &info, Buffer *&response, uint16_t timeout = 0 ) XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ //! Prepare one or more files for access - async //! //! @param fileList list of files to be prepared //! @param flags PrepareFlags::Flags //! @param priority priority of the request 0 (lowest) - 3 (highest) //! @param handler handler to be notified when the response arrives, //! the response parameter will hold a Buffer object //! if the procedure is successful //! @param timeout timeout value, if 0 the environment default will //! be used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus Prepare( const std::vector &fileList, PrepareFlags::Flags flags, uint8_t priority, ResponseHandler *handler, uint16_t timeout = 0 ) XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ //! Prepare one or more files for access - sync //! //! @param fileList list of files to be prepared //! @param flags PrepareFlags::Flags //! @param priority priority of the request 0 (lowest) - 3 (highest) //! @param response the response (to be deleted by the user) //! @param timeout timeout value, if 0 the environment default will //! be used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus Prepare( const std::vector &fileList, PrepareFlags::Flags flags, uint8_t priority, Buffer *&response, uint16_t timeout = 0 ) XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ //! Set extended attributes - async //! //! @param attrs : list of extended attributes to set //! @param handler : handler to be notified when the response arrives, //! the response parameter will hold a std::vector of //! XAttrStatus objects //! @param timeout : timeout value, if 0 the environment default will //! be used //! //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus SetXAttr( const std::string &path, const std::vector &attrs, ResponseHandler *handler, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! Set extended attributes - sync //! //! @param attrs : list of extended attributes to set //! @param result : result of the operation //! @param timeout : timeout value, if 0 the environment default will //! be used //! //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus SetXAttr( const std::string &path, const std::vector &attrs, std::vector &result, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! Get extended attributes - async //! //! @param attrs : list of extended attributes to get //! @param handler : handler to be notified when the response arrives, //! the response parameter will hold a std::vector of //! XAttr objects //! @param timeout : timeout value, if 0 the environment default will //! be used //! //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus GetXAttr( const std::string &path, const std::vector &attrs, ResponseHandler *handler, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! Get extended attributes - sync //! //! @param attrs : list of extended attributes to get //! @param result : result of the operation //! @param timeout : timeout value, if 0 the environment default will //! be used //! //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus GetXAttr( const std::string &path, const std::vector &attrs, std::vector &result, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! Delete extended attributes - async //! //! @param attrs : list of extended attributes to set //! @param handler : handler to be notified when the response arrives, //! the response parameter will hold a std::vector of //! XAttrStatus objects //! @param timeout : timeout value, if 0 the environment default will //! be used //! //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus DelXAttr( const std::string &path, const std::vector &attrs, ResponseHandler *handler, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! Delete extended attributes - sync //! //! @param attrs : list of extended attributes to set //! @param result : result of the operation //! @param timeout : timeout value, if 0 the environment default will //! be used //! //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus DelXAttr( const std::string &path, const std::vector &attrs, std::vector &result, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! List extended attributes - async //! //! @param handler : handler to be notified when the response arrives, //! the response parameter will hold a std::vector of //! XAttr objects //! @param timeout : timeout value, if 0 the environment default will //! be used //! //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus ListXAttr( const std::string &path, ResponseHandler *handler, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! List extended attributes - sync //! //! @param result : result of the operation //! @param timeout : timeout value, if 0 the environment default will //! be used //! //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus ListXAttr( const std::string &path, std::vector &result, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! Set filesystem property //! //! Filesystem properties: //! FollowRedirects [true/false] - enable/disable following redirections //------------------------------------------------------------------------ bool SetProperty( const std::string &name, const std::string &value ); //------------------------------------------------------------------------ //! Get filesystem property //! //! @see FileSystem::SetProperty for property list //------------------------------------------------------------------------ bool GetProperty( const std::string &name, std::string &value ) const; private: FileSystem(const FileSystem &other); FileSystem &operator = (const FileSystem &other); //------------------------------------------------------------------------ // Lock the internal lock //------------------------------------------------------------------------ void Lock(); //------------------------------------------------------------------------ // Unlock the internal lock //------------------------------------------------------------------------ void UnLock(); //------------------------------------------------------------------------ //! Generic implementation of SendCache and SendInfo //! //! @param info : the info string to be sent //! @param handler : handler to be notified when the response arrives. //! @param timeout : timeout value or 0 for default. //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus SendSet( const char *prefix, const std::string &info, ResponseHandler *handler, uint16_t timeout = 0 ) XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ //! Generic implementation of xattr operation //! //! @param subcode : xattr operation code //! @param path : path to the file //! @param attrs : operation argument //! @param handler : operation handler //! @param timeout : operation timeout //------------------------------------------------------------------------ template Status XAttrOperationImpl( kXR_char subcode, kXR_char options, const std::string &path, const std::vector &attrs, ResponseHandler *handler, uint16_t timeout = 0 ); FileSystemImpl *pImpl; //< pointer to implementation (TODO: once we can break ABI we can use a shared pointer here, and then we can drop the FileSystemData in source file) FileSystemPlugIn *pPlugIn; //< file system plug-in }; } #endif // __XRD_CL_FILE_SYSTEM_HH__ xrootd-5.6.9/src/XrdCl/XrdClFileSystemOperations.hh000066400000000000000000001613711457266313600223110ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2017 by European Organization for Nuclear Research (CERN) // Author: Krzysztof Jamrog , // Michal Simon //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #ifndef __XRD_CL_FILE_SYSTEM_OPERATIONS_HH__ #define __XRD_CL_FILE_SYSTEM_OPERATIONS_HH__ #include "XrdCl/XrdClFileSystem.hh" #include "XrdCl/XrdClOperations.hh" #include "XrdCl/XrdClOperationHandlers.hh" #include "XrdCl/XrdClCtx.hh" namespace XrdCl { //---------------------------------------------------------------------------- //! Base class for all file system releated operations //! //! @arg Derived : the class that derives from this template (CRTP) //! @arg HasHndl : true if operation has a handler, false otherwise //! @arg Args : operation arguments //---------------------------------------------------------------------------- template class Derived, bool HasHndl, typename Response, typename ... Args> class FileSystemOperation: public ConcreteOperation { template class, bool, typename, typename ...> friend class FileSystemOperation; public: //------------------------------------------------------------------------ //! Constructor //! //! @param fs : file system on which the operation will be performed //! @param args : file operation arguments //------------------------------------------------------------------------ FileSystemOperation( Ctx fs, Args... args): ConcreteOperation( std::move( args )... ), filesystem( std::move( fs ) ) { } //------------------------------------------------------------------------ //! Move constructor from other states //! //! @arg from : state from which the object is being converted //! //! @param op : the object that is being converted //------------------------------------------------------------------------ template FileSystemOperation( FileSystemOperation && op ): ConcreteOperation( std::move( op ) ), filesystem( op.filesystem ) { } //------------------------------------------------------------------------ //! Destructor //------------------------------------------------------------------------ virtual ~FileSystemOperation() { } protected: //------------------------------------------------------------------------ //! The file system object itself. //------------------------------------------------------------------------ Ctx filesystem; }; //---------------------------------------------------------------------------- //! Locate operation (@see FileSystemOperation) //---------------------------------------------------------------------------- template class LocateImpl: public FileSystemOperation, Arg, Arg> { public: //------------------------------------------------------------------------ //! Inherit constructors from FileSystemOperation (@see FileSystemOperation) //------------------------------------------------------------------------ using FileSystemOperation, Arg, Arg>::FileSystemOperation; //------------------------------------------------------------------------ //! Argument indexes in the args tuple //------------------------------------------------------------------------ enum { PathArg, FlagsArg }; //------------------------------------------------------------------------ //! @return : name of the operation (@see Operation) //------------------------------------------------------------------------ std::string ToString() { return "Locate"; } protected: //------------------------------------------------------------------------ //! RunImpl operation (@see Operation) //! //! @param params : container with parameters forwarded from //! previous operation //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus RunImpl( PipelineHandler *handler, uint16_t pipelineTimeout ) { std::string &path = std::get( this->args ).Get(); OpenFlags::Flags flags = std::get( this->args ).Get(); uint16_t timeout = pipelineTimeout < this->timeout ? pipelineTimeout : this->timeout; return this->filesystem->Locate( path, flags, handler, timeout ); } }; typedef LocateImpl Locate; //---------------------------------------------------------------------------- //! DeepLocate operation (@see FileSystemOperation) //---------------------------------------------------------------------------- template class DeepLocateImpl: public FileSystemOperation, Arg, Arg> { public: //------------------------------------------------------------------------ //! Inherit constructors from FileSystemOperation (@see FileSystemOperation) //------------------------------------------------------------------------ using FileSystemOperation, Arg, Arg>::FileSystemOperation; //------------------------------------------------------------------------ //! Argument indexes in the args tuple //------------------------------------------------------------------------ enum { PathArg, FlagsArg }; //------------------------------------------------------------------------ //! @return : name of the operation (@see Operation) //------------------------------------------------------------------------ std::string ToString() { return "DeepLocate"; } protected: //------------------------------------------------------------------------ //! RunImpl operation (@see Operation) //! //! @param params : container with parameters forwarded from //! previous operation //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus RunImpl( PipelineHandler *handler, uint16_t pipelineTimeout ) { std::string &path = std::get( this->args ).Get(); OpenFlags::Flags flags = std::get( this->args ).Get(); uint16_t timeout = pipelineTimeout < this->timeout ? pipelineTimeout : this->timeout; return this->filesystem->DeepLocate( path, flags, handler, timeout ); } }; typedef DeepLocateImpl DeepLocate; //---------------------------------------------------------------------------- //! Mv operation (@see FileSystemOperation) //---------------------------------------------------------------------------- template class MvImpl: public FileSystemOperation, Arg, Arg> { public: //------------------------------------------------------------------------ //! Inherit constructors from FileSystemOperation (@see FileSystemOperation) //------------------------------------------------------------------------ using FileSystemOperation, Arg, Arg>::FileSystemOperation; //------------------------------------------------------------------------ //! Argument indexes in the args tuple //------------------------------------------------------------------------ enum { SourceArg, DestArg }; //------------------------------------------------------------------------ //! @return : name of the operation (@see Operation) //------------------------------------------------------------------------ std::string ToString() { return "Mv"; } protected: //------------------------------------------------------------------------ //! RunImpl operation (@see Operation) //! //! @param params : container with parameters forwarded from //! previous operation //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus RunImpl( PipelineHandler *handler, uint16_t pipelineTimeout ) { std::string &source = std::get( this->args ).Get(); std::string &dest = std::get( this->args ).Get(); uint16_t timeout = pipelineTimeout < this->timeout ? pipelineTimeout : this->timeout; return this->filesystem->Mv( source, dest, handler, timeout ); } }; typedef MvImpl Mv; //---------------------------------------------------------------------------- //! Query operation (@see FileSystemOperation) //---------------------------------------------------------------------------- template class QueryImpl: public FileSystemOperation, Arg, Arg> { public: //------------------------------------------------------------------------ //! Inherit constructors from FileSystemOperation (@see FileSystemOperation) //------------------------------------------------------------------------ using FileSystemOperation, Arg, Arg>::FileSystemOperation; //------------------------------------------------------------------------ //! Argument indexes in the args tuple //------------------------------------------------------------------------ enum { QueryCodeArg, BufferArg }; //------------------------------------------------------------------------ //! @return : name of the operation (@see Operation) //------------------------------------------------------------------------ std::string ToString() { return "Query"; } protected: //------------------------------------------------------------------------ //! RunImpl operation (@see Operation) //! //! @param params : container with parameters forwarded from //! previous operation //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus RunImpl( PipelineHandler *handler, uint16_t pipelineTimeout ) { QueryCode::Code queryCode = std::get( this->args ).Get(); const Buffer &buffer( std::get( this->args ).Get() ); uint16_t timeout = pipelineTimeout < this->timeout ? pipelineTimeout : this->timeout; return this->filesystem->Query( queryCode, buffer, handler, timeout ); } }; typedef QueryImpl Query; //---------------------------------------------------------------------------- //! Truncate operation (@see FileSystemOperation) //---------------------------------------------------------------------------- template class TruncateFsImpl: public FileSystemOperation, Arg, Arg> { public: //------------------------------------------------------------------------ //! Inherit constructors from FileSystemOperation (@see FileSystemOperation) //------------------------------------------------------------------------ using FileSystemOperation, Arg, Arg>::FileSystemOperation; //------------------------------------------------------------------------ //! Argument indexes in the args tuple //------------------------------------------------------------------------ enum { PathArg, SizeArg }; //------------------------------------------------------------------------ //! @return : name of the operation (@see Operation) //------------------------------------------------------------------------ std::string ToString() { return "Truncate"; } protected: //------------------------------------------------------------------------ //! RunImpl operation (@see Operation) //! //! @param params : container with parameters forwarded from //! previous operation //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus RunImpl( PipelineHandler *handler, uint16_t pipelineTimeout ) { std::string &path = std::get( this->args ).Get(); uint64_t size = std::get( this->args ).Get(); uint16_t timeout = pipelineTimeout < this->timeout ? pipelineTimeout : this->timeout; return this->filesystem->Truncate( path, size, handler, timeout ); } }; inline TruncateFsImpl Truncate( Ctx fs, Arg path, Arg size ) { return TruncateFsImpl( std::move( fs ), std::move( path ), std::move( size ) ); } //---------------------------------------------------------------------------- //! Rm operation (@see FileSystemOperation) //---------------------------------------------------------------------------- template class RmImpl: public FileSystemOperation, Arg> { public: //------------------------------------------------------------------------ //! Inherit constructors from FileSystemOperation (@see FileSystemOperation) //------------------------------------------------------------------------ using FileSystemOperation, Arg>::FileSystemOperation; //------------------------------------------------------------------------ //! Argument indexes in the args tuple //------------------------------------------------------------------------ enum { PathArg }; //------------------------------------------------------------------------ //! @return : name of the operation (@see Operation) //------------------------------------------------------------------------ std::string ToString() { return "Rm"; } protected: //------------------------------------------------------------------------ //! RunImpl operation (@see Operation) //! //! @param params : container with parameters forwarded from //! previous operation //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus RunImpl( PipelineHandler *handler, uint16_t pipelineTimeout ) { std::string &path = std::get( this->args ).Get(); uint16_t timeout = pipelineTimeout < this->timeout ? pipelineTimeout : this->timeout; return this->filesystem->Rm( path, handler, timeout ); } }; typedef RmImpl Rm; //---------------------------------------------------------------------------- //! MkDir operation (@see FileSystemOperation) //---------------------------------------------------------------------------- template class MkDirImpl: public FileSystemOperation, Arg, Arg, Arg> { public: //------------------------------------------------------------------------ //! Inherit constructors from FileSystemOperation (@see FileSystemOperation) //------------------------------------------------------------------------ using FileSystemOperation, Arg, Arg, Arg>::FileSystemOperation; //------------------------------------------------------------------------ //! Argument indexes in the args tuple //------------------------------------------------------------------------ enum { PathArg, FlagsArg, ModeArg }; //------------------------------------------------------------------------ //! @return : name of the operation (@see Operation) //------------------------------------------------------------------------ std::string ToString() { return "MkDir"; } protected: //------------------------------------------------------------------------ //! RunImpl operation (@see Operation) //! //! @param params : container with parameters forwarded from //! previous operation //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus RunImpl( PipelineHandler *handler, uint16_t pipelineTimeout ) { std::string &path = std::get( this->args ).Get(); MkDirFlags::Flags flags = std::get( this->args ).Get(); Access::Mode mode = std::get( this->args ).Get(); uint16_t timeout = pipelineTimeout < this->timeout ? pipelineTimeout : this->timeout; return this->filesystem->MkDir( path, flags, mode, handler, timeout ); } }; typedef MkDirImpl MkDir; //---------------------------------------------------------------------------- //! RmDir operation (@see FileSystemOperation) //---------------------------------------------------------------------------- template class RmDirImpl: public FileSystemOperation, Arg> { public: //------------------------------------------------------------------------ //! Inherit constructors from FileSystemOperation (@see FileSystemOperation) //------------------------------------------------------------------------ using FileSystemOperation, Arg>::FileSystemOperation; //------------------------------------------------------------------------ //! Argument indexes in the args tuple //------------------------------------------------------------------------ enum { PathArg }; //------------------------------------------------------------------------ //! @return : name of the operation (@see Operation) //------------------------------------------------------------------------ std::string ToString() { return "RmDir"; } protected: //------------------------------------------------------------------------ //! RunImpl operation (@see Operation) //! //! @param params : container with parameters forwarded from //! previous operation //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus RunImpl( PipelineHandler *handler, uint16_t pipelineTimeout ) { std::string &path = std::get( this->args ).Get(); uint16_t timeout = pipelineTimeout < this->timeout ? pipelineTimeout : this->timeout; return this->filesystem->RmDir( path, handler, timeout ); } }; typedef RmDirImpl RmDir; //---------------------------------------------------------------------------- //! ChMod operation (@see FileSystemOperation) //---------------------------------------------------------------------------- template class ChModImpl: public FileSystemOperation, Arg, Arg> { public: //------------------------------------------------------------------------ //! Inherit constructors from FileSystemOperation (@see FileSystemOperation) //------------------------------------------------------------------------ using FileSystemOperation, Arg, Arg>::FileSystemOperation; //------------------------------------------------------------------------ //! Argument indexes in the args tuple //------------------------------------------------------------------------ enum { PathArg, ModeArg }; //------------------------------------------------------------------------ //! @return : name of the operation (@see Operation) //------------------------------------------------------------------------ std::string ToString() { return "ChMod"; } protected: //------------------------------------------------------------------------ //! RunImpl operation (@see Operation) //! //! @param params : container with parameters forwarded from //! previous operation //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus RunImpl( PipelineHandler *handler, uint16_t pipelineTimeout ) { std::string &path = std::get( this->args ).Get(); Access::Mode mode = std::get( this->args ).Get(); uint16_t timeout = pipelineTimeout < this->timeout ? pipelineTimeout : this->timeout; return this->filesystem->ChMod( path, mode, handler, timeout ); } }; typedef ChModImpl ChMod; //---------------------------------------------------------------------------- //! Ping operation (@see FileSystemOperation) //---------------------------------------------------------------------------- template class PingImpl: public FileSystemOperation> { public: //------------------------------------------------------------------------ //! Inherit constructors from FileSystemOperation (@see FileSystemOperation) //------------------------------------------------------------------------ using FileSystemOperation>::FileSystemOperation; //------------------------------------------------------------------------ //! @return : name of the operation (@see Operation) //------------------------------------------------------------------------ std::string ToString() { return "Ping"; } protected: //------------------------------------------------------------------------ //! RunImpl operation (@see Operation) //! //! @param params : container with parameters forwarded from //! previous operation //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus RunImpl( PipelineHandler *handler, uint16_t pipelineTimeout ) { uint16_t timeout = pipelineTimeout < this->timeout ? pipelineTimeout : this->timeout; return this->filesystem->Ping( handler, timeout ); } }; typedef PingImpl Ping; //---------------------------------------------------------------------------- //! Stat operation (@see FileSystemOperation) //---------------------------------------------------------------------------- template class StatFsImpl: public FileSystemOperation, Arg> { public: //------------------------------------------------------------------------ //! Inherit constructors from FileSystemOperation (@see FileSystemOperation) //------------------------------------------------------------------------ using FileSystemOperation, Arg>::FileSystemOperation; //------------------------------------------------------------------------ //! Argument indexes in the args tuple //------------------------------------------------------------------------ enum { PathArg }; //------------------------------------------------------------------------ //! @return : name of the operation (@see Operation) //------------------------------------------------------------------------ std::string ToString() { return "Stat"; } protected: //------------------------------------------------------------------------ //! RunImpl operation (@see Operation) //! //! @param params : container with parameters forwarded from //! previous operation //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus RunImpl( PipelineHandler *handler, uint16_t pipelineTimeout ) { std::string &path = std::get( this->args ).Get(); uint16_t timeout = pipelineTimeout < this->timeout ? pipelineTimeout : this->timeout; return this->filesystem->Stat( path, handler, timeout ); } }; inline StatFsImpl Stat( Ctx fs, Arg path ) { return StatFsImpl( std::move( fs ), std::move( path ) ); } //---------------------------------------------------------------------------- //! StatVS operation (@see FileSystemOperation) //---------------------------------------------------------------------------- template class StatVFSImpl: public FileSystemOperation, Arg> { public: //------------------------------------------------------------------------ //! Inherit constructors from FileSystemOperation (@see FileSystemOperation) //------------------------------------------------------------------------ using FileSystemOperation, Arg>::FileSystemOperation; //------------------------------------------------------------------------ //! Argument indexes in the args tuple //------------------------------------------------------------------------ enum { PathArg }; //------------------------------------------------------------------------ //! @return : name of the operation (@see Operation) //------------------------------------------------------------------------ std::string ToString() { return "StatVFS"; } protected: //------------------------------------------------------------------------ //! RunImpl operation (@see Operation) //! //! @param params : container with parameters forwarded from //! previous operation //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus RunImpl( PipelineHandler *handler, uint16_t pipelineTimeout ) { std::string &path = std::get( this->args ).Get(); uint16_t timeout = pipelineTimeout < this->timeout ? pipelineTimeout : this->timeout; return this->filesystem->StatVFS( path, handler, timeout ); } }; typedef StatVFSImpl StatVFS; //---------------------------------------------------------------------------- //! Protocol operation (@see FileSystemOperation) //---------------------------------------------------------------------------- template class ProtocolImpl: public FileSystemOperation> { public: //------------------------------------------------------------------------ //! Inherit constructors from FileSystemOperation (@see FileSystemOperation) //------------------------------------------------------------------------ using FileSystemOperation>::FileSystemOperation; //------------------------------------------------------------------------ //! @return : name of the operation (@see Operation) //------------------------------------------------------------------------ std::string ToString() { return "Protocol"; } protected: //------------------------------------------------------------------------ //! RunImpl operation (@see Operation) //! //! @param params : container with parameters forwarded from //! previous operation //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus RunImpl( PipelineHandler *handler, uint16_t pipelineTimeout ) { uint16_t timeout = pipelineTimeout < this->timeout ? pipelineTimeout : this->timeout; return this->filesystem->Protocol( handler, timeout ); } }; typedef ProtocolImpl Protocol; //---------------------------------------------------------------------------- //! DirList operation (@see FileSystemOperation) //---------------------------------------------------------------------------- template class DirListImpl: public FileSystemOperation, Arg, Arg> { public: //------------------------------------------------------------------------ //! Inherit constructors from FileSystemOperation (@see FileSystemOperation) //------------------------------------------------------------------------ using FileSystemOperation, Arg, Arg>::FileSystemOperation; //------------------------------------------------------------------------ //! Argument indexes in the args tuple //------------------------------------------------------------------------ enum { PathArg, FlagsArg }; //------------------------------------------------------------------------ //! @return : name of the operation (@see Operation) //------------------------------------------------------------------------ std::string ToString() { return "DirList"; } protected: //------------------------------------------------------------------------ //! RunImpl operation (@see Operation) //! //! @param params : container with parameters forwarded from //! previous operation //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus RunImpl( PipelineHandler *handler, uint16_t pipelineTimeout ) { std::string &path = std::get( this->args ).Get(); DirListFlags::Flags flags = std::get( this->args ).Get(); uint16_t timeout = pipelineTimeout < this->timeout ? pipelineTimeout : this->timeout; return this->filesystem->DirList( path, flags, handler, timeout ); } }; typedef DirListImpl DirList; //---------------------------------------------------------------------------- //! SendInfo operation (@see FileSystemOperation) //---------------------------------------------------------------------------- template class SendInfoImpl: public FileSystemOperation, Arg> { public: //------------------------------------------------------------------------ //! Inherit constructors from FileSystemOperation (@see FileSystemOperation) //------------------------------------------------------------------------ using FileSystemOperation, Arg>::FileSystemOperation; //------------------------------------------------------------------------ //! Argument indexes in the args tuple //------------------------------------------------------------------------ enum { InfoArg }; //------------------------------------------------------------------------ //! @return : name of the operation (@see Operation) //------------------------------------------------------------------------ std::string ToString() { return "SendInfo"; } protected: //------------------------------------------------------------------------ //! RunImpl operation (@see Operation) //! //! @param params : container with parameters forwarded from //! previous operation //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus RunImpl( PipelineHandler *handler, uint16_t pipelineTimeout ) { std::string &info = std::get( this->args ).Get(); uint16_t timeout = pipelineTimeout < this->timeout ? pipelineTimeout : this->timeout; return this->filesystem->SendInfo( info, handler, timeout ); } }; typedef SendInfoImpl SendInfo; //---------------------------------------------------------------------------- //! Prepare operation (@see FileSystemOperation) //---------------------------------------------------------------------------- template class PrepareImpl: public FileSystemOperation, Arg>, Arg, Arg> { public: //------------------------------------------------------------------------ //! Inherit constructors from FileSystemOperation (@see FileSystemOperation) //------------------------------------------------------------------------ using FileSystemOperation, Arg>, Arg, Arg>::FileSystemOperation; //------------------------------------------------------------------------ //! Argument indexes in the args tuple //------------------------------------------------------------------------ enum { FileListArg, FlagsArg, PriorityArg }; //------------------------------------------------------------------------ //! @return : name of the operation (@see Operation) //------------------------------------------------------------------------ std::string ToString() { return "Prepare"; } protected: //------------------------------------------------------------------------ //! RunImpl operation (@see Operation) //! //! @param params : container with parameters forwarded from //! previous operation //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus RunImpl( PipelineHandler *handler, uint16_t pipelineTimeout ) { std::vector &fileList = std::get( this->args ).Get(); PrepareFlags::Flags flags = std::get( this->args ).Get(); uint8_t priority = std::get( this->args ).Get(); uint16_t timeout = pipelineTimeout < this->timeout ? pipelineTimeout : this->timeout; return this->filesystem->Prepare( fileList, flags, priority, handler, timeout ); } }; typedef PrepareImpl Prepare; //---------------------------------------------------------------------------- //! SetXAttr operation (@see FileOperation) //---------------------------------------------------------------------------- template class SetXAttrFsImpl: public FileSystemOperation, Arg, Arg, Arg> { public: //------------------------------------------------------------------------ //! Inherit constructors from FileOperation (@see FileOperation) //------------------------------------------------------------------------ using FileSystemOperation, Arg, Arg, Arg>::FileSystemOperation; //------------------------------------------------------------------------ //! Argument indexes in the args tuple //------------------------------------------------------------------------ enum { PathArg, NameArg, ValueArg }; //------------------------------------------------------------------------ //! @return : name of the operation (@see Operation) //------------------------------------------------------------------------ std::string ToString() { return "SetXAttrFsImpl"; } protected: //------------------------------------------------------------------------ //! RunImpl operation (@see Operation) //! //! @param params : container with parameters forwarded from //! previous operation //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus RunImpl( PipelineHandler *handler, uint16_t pipelineTimeout ) { std::string &path = std::get( this->args ).Get(); std::string &name = std::get( this->args ).Get(); std::string &value = std::get( this->args ).Get(); // wrap the arguments with a vector std::vector attrs; attrs.push_back( xattr_t( name, value ) ); // wrap the PipelineHandler so the response gets unpacked properly UnpackXAttrStatus *h = new UnpackXAttrStatus( handler ); uint16_t timeout = pipelineTimeout < this->timeout ? pipelineTimeout : this->timeout; XRootDStatus st = this->filesystem->SetXAttr( path, attrs, h, timeout ); if( !st.IsOK() ) delete h; return st; } }; //---------------------------------------------------------------------------- //! Factory for creating SetXAttrFsImpl objects (as there is another SetXAttr //! in File there would be a clash of typenames). //---------------------------------------------------------------------------- inline SetXAttrFsImpl SetXAttr( Ctx fs, Arg path, Arg name, Arg value ) { return SetXAttrFsImpl( std::move( fs ), std::move( path ), std::move( name ), std::move( value ) ); } //---------------------------------------------------------------------------- //! SetXAttr bulk operation (@see FileOperation) //---------------------------------------------------------------------------- template class SetXAttrFsBulkImpl: public FileSystemOperation>, Arg, Arg>> { public: //------------------------------------------------------------------------ //! Inherit constructors from FileOperation (@see FileOperation) //------------------------------------------------------------------------ using FileSystemOperation>, Arg, Arg>>::FileSystemOperation; //------------------------------------------------------------------------ //! Argument indexes in the args tuple //------------------------------------------------------------------------ enum { PathArg, AttrsArg }; //------------------------------------------------------------------------ //! @return : name of the operation (@see Operation) //------------------------------------------------------------------------ std::string ToString() { return "SetXAttrBulkImpl"; } protected: //------------------------------------------------------------------------ //! RunImpl operation (@see Operation) //! //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus RunImpl( PipelineHandler *handler, uint16_t pipelineTimeout ) { std::string &path = std::get( this->args ).Get(); std::vector &attrs = std::get( this->args ).Get(); uint16_t timeout = pipelineTimeout < this->timeout ? pipelineTimeout : this->timeout; return this->filesystem->SetXAttr( path, attrs, handler, timeout ); } }; //---------------------------------------------------------------------------- //! Factory for creating SetXAttrFsBulkImpl objects (as there is another SetXAttr //! in FileSystem there would be a clash of typenames). //---------------------------------------------------------------------------- inline SetXAttrFsBulkImpl SetXAttr( Ctx fs, Arg path, Arg> attrs ) { return SetXAttrFsBulkImpl( std::move( fs ), std::move( path ), std::move( attrs ) ); } //---------------------------------------------------------------------------- //! GetXAttr operation (@see FileOperation) //---------------------------------------------------------------------------- template class GetXAttrFsImpl: public FileSystemOperation, Arg, Arg> { public: //------------------------------------------------------------------------ //! Inherit constructors from FileOperation (@see FileOperation) //------------------------------------------------------------------------ using FileSystemOperation, Arg, Arg>::FileSystemOperation; //------------------------------------------------------------------------ //! Argument indexes in the args tuple //------------------------------------------------------------------------ enum { PathArg, NameArg }; //------------------------------------------------------------------------ //! @return : name of the operation (@see Operation) //------------------------------------------------------------------------ std::string ToString() { return "GetXAttrFsImpl"; } protected: //------------------------------------------------------------------------ //! RunImpl operation (@see Operation) //! //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus RunImpl( PipelineHandler *handler, uint16_t pipelineTimeout ) { std::string &path = std::get( this->args ).Get(); std::string &name = std::get( this->args ).Get(); // wrap the argument with a vector std::vector attrs; attrs.push_back( name ); // wrap the PipelineHandler so the response gets unpacked properly UnpackXAttr *h = new UnpackXAttr( handler ); uint16_t timeout = pipelineTimeout < this->timeout ? pipelineTimeout : this->timeout; XRootDStatus st = this->filesystem->GetXAttr( path, attrs, h, timeout ); if( !st.IsOK() ) delete h; return st; } }; //---------------------------------------------------------------------------- //! Factory for creating GetXAttrFsImpl objects (as there is another GetXAttr //! in File there would be a clash of typenames). //---------------------------------------------------------------------------- inline GetXAttrFsImpl GetXAttr( Ctx fs, Arg path, Arg name ) { return GetXAttrFsImpl( std::move( fs ), std::move( path ), std::move( name ) ); } //---------------------------------------------------------------------------- //! GetXAttr bulk operation (@see FileOperation) //---------------------------------------------------------------------------- template class GetXAttrFsBulkImpl: public FileSystemOperation>, Arg, Arg>> { public: //------------------------------------------------------------------------ //! Inherit constructors from FileOperation (@see FileOperation) //------------------------------------------------------------------------ using FileSystemOperation>, Arg, Arg>>::FileSystemOperation; //------------------------------------------------------------------------ //! Argument indexes in the args tuple //------------------------------------------------------------------------ enum { PathArg, NamesArg }; //------------------------------------------------------------------------ //! @return : name of the operation (@see Operation) //------------------------------------------------------------------------ std::string ToString() { return "GetXAttrFsBulkImpl"; } protected: //------------------------------------------------------------------------ //! RunImpl operation (@see Operation) //! //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus RunImpl( PipelineHandler *handler, uint16_t pipelineTimeout ) { std::string &path = std::get( this->args ).Get(); std::vector &attrs = std::get( this->args ).Get(); uint16_t timeout = pipelineTimeout < this->timeout ? pipelineTimeout : this->timeout; return this->filesystem->GetXAttr( path, attrs, handler, timeout ); } }; //---------------------------------------------------------------------------- //! Factory for creating GetXAttrFsBulkImpl objects (as there is another GetXAttr in //! FileSystem there would be a clash of typenames). //---------------------------------------------------------------------------- inline GetXAttrFsBulkImpl GetXAttr( Ctx fs, Arg path, Arg> attrs ) { return GetXAttrFsBulkImpl( std::move( fs ), std::move( path ), std::move( attrs ) ); } //---------------------------------------------------------------------------- //! DelXAttr operation (@see FileOperation) //---------------------------------------------------------------------------- template class DelXAttrFsImpl: public FileSystemOperation, Arg, Arg> { public: //------------------------------------------------------------------------ //! Inherit constructors from FileOperation (@see FileOperation) //------------------------------------------------------------------------ using FileSystemOperation, Arg, Arg>::FileSystemOperation; //------------------------------------------------------------------------ //! Argument indexes in the args tuple //------------------------------------------------------------------------ enum { PathArg, NameArg }; //------------------------------------------------------------------------ //! @return : name of the operation (@see Operation) //------------------------------------------------------------------------ std::string ToString() { return "DelXAttrFsImpl"; } protected: //------------------------------------------------------------------------ //! RunImpl operation (@see Operation) //! //! @param params : container with parameters forwarded from //! previous operation //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus RunImpl( PipelineHandler *handler, uint16_t pipelineTimeout ) { std::string &path = std::get( this->args ).Get(); std::string &name = std::get( this->args ).Get(); // wrap the argument with a vector std::vector attrs; attrs.push_back( name ); // wrap the PipelineHandler so the response gets unpacked properly UnpackXAttrStatus *h = new UnpackXAttrStatus( handler ); uint16_t timeout = pipelineTimeout < this->timeout ? pipelineTimeout : this->timeout; XRootDStatus st = this->filesystem->DelXAttr( path, attrs, h, timeout ); if( !st.IsOK() ) delete h; return st; } }; //---------------------------------------------------------------------------- //! Factory for creating DelXAttrFsImpl objects (as there is another DelXAttr //! in File there would be a clash of typenames). //---------------------------------------------------------------------------- inline DelXAttrFsImpl DelXAttr( Ctx fs, Arg path, Arg name ) { return DelXAttrFsImpl( std::move( fs ), std::move( path ), std::move( name ) ); } //---------------------------------------------------------------------------- //! DelXAttr bulk operation (@see FileOperation) //---------------------------------------------------------------------------- template class DelXAttrFsBulkImpl: public FileSystemOperation>, Arg, Arg>> { public: //------------------------------------------------------------------------ //! Inherit constructors from FileOperation (@see FileOperation) //------------------------------------------------------------------------ using FileSystemOperation>, Arg, Arg>>::FileSystemOperation; //------------------------------------------------------------------------ //! Argument indexes in the args tuple //------------------------------------------------------------------------ enum { PathArg, NamesArg }; //------------------------------------------------------------------------ //! @return : name of the operation (@see Operation) //------------------------------------------------------------------------ std::string ToString() { return "DelXAttrBulkImpl"; } protected: //------------------------------------------------------------------------ //! RunImpl operation (@see Operation) //! //! @param params : container with parameters forwarded from //! previous operation //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus RunImpl( PipelineHandler *handler, uint16_t pipelineTimeout ) { std::string &path = std::get( this->args ).Get(); std::vector &attrs = std::get( this->args ).Get(); uint16_t timeout = pipelineTimeout < this->timeout ? pipelineTimeout : this->timeout; return this->filesystem->DelXAttr( path, attrs, handler, timeout ); } }; //---------------------------------------------------------------------------- //! Factory for creating DelXAttrFsBulkImpl objects (as there is another DelXAttr //! in FileSystem there would be a clash of typenames). //---------------------------------------------------------------------------- inline DelXAttrFsBulkImpl DelXAttr( Ctx fs, Arg path, Arg> attrs ) { return DelXAttrFsBulkImpl( std::move( fs ), std::move( path ), std::move( attrs ) ); } //---------------------------------------------------------------------------- //! ListXAttr bulk operation (@see FileOperation) //---------------------------------------------------------------------------- template class ListXAttrFsImpl: public FileSystemOperation>, Arg> { public: //------------------------------------------------------------------------ //! Inherit constructors from FileOperation (@see FileOperation) //------------------------------------------------------------------------ using FileSystemOperation>, Arg>::FileSystemOperation; //------------------------------------------------------------------------ //! Argument indexes in the args tuple //------------------------------------------------------------------------ enum { PathArg }; //------------------------------------------------------------------------ //! @return : name of the operation (@see Operation) //------------------------------------------------------------------------ std::string ToString() { return "ListXAttrFsImpl"; } protected: //------------------------------------------------------------------------ //! RunImpl operation (@see Operation) //! //! @param params : container with parameters forwarded from //! previous operation //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus RunImpl( PipelineHandler *handler, uint16_t pipelineTimeout ) { std::string &path = std::get( this->args ).Get(); uint16_t timeout = pipelineTimeout < this->timeout ? pipelineTimeout : this->timeout; return this->filesystem->ListXAttr( path, handler, timeout ); } }; //---------------------------------------------------------------------------- //! Factory for creating ListXAttrFsImpl objects (as there is another ListXAttr //! in FileSystem there would be a clash of typenames). //---------------------------------------------------------------------------- inline ListXAttrFsImpl ListXAttr( Ctx fs, Arg path ) { return ListXAttrFsImpl( std::move( fs ), std::move( path ) ); } } #endif // __XRD_CL_FILE_SYSTEM_OPERATIONS_HH__ xrootd-5.6.9/src/XrdCl/XrdClFileSystemUtils.cc000066400000000000000000000162631457266313600212530ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2014 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #include "XrdCl/XrdClFileSystemUtils.hh" #include "XrdCl/XrdClFileSystem.hh" #include "XrdCl/XrdClFileSystem.hh" #include "XrdCl/XrdClURL.hh" #include #include #include namespace XrdCl { //---------------------------------------------------------------------------- // The data holder implementation //---------------------------------------------------------------------------- struct FileSystemUtils::SpaceInfoImpl { SpaceInfoImpl( uint64_t total, uint64_t free, uint64_t used, uint64_t largestChunk ): pTotal( total ), pFree( free ), pUsed( used ), pLargestChunk( largestChunk ) { } uint64_t pTotal; uint64_t pFree; uint64_t pUsed; uint64_t pLargestChunk; }; //---------------------------------------------------------------------------- // Constructor //---------------------------------------------------------------------------- FileSystemUtils::SpaceInfo::SpaceInfo( uint64_t total, uint64_t free, uint64_t used, uint64_t largestChunk ): pImpl( new SpaceInfoImpl( total, free, used, largestChunk ) ) { } //--------------------------------------------------------------------------- // Destructor (needs to be here due to the unique_ptr guarding PIMPL) //--------------------------------------------------------------------------- FileSystemUtils::SpaceInfo::~SpaceInfo() { } //---------------------------------------------------------------------------- // Amount of total space in MB //---------------------------------------------------------------------------- uint64_t FileSystemUtils::SpaceInfo::GetTotal() const { return pImpl->pTotal; } //---------------------------------------------------------------------------- // Amount of free space in MB //---------------------------------------------------------------------------- uint64_t FileSystemUtils::SpaceInfo::GetFree() const { return pImpl->pFree; } //---------------------------------------------------------------------------- // Amount of used space in MB //---------------------------------------------------------------------------- uint64_t FileSystemUtils::SpaceInfo::GetUsed() const { return pImpl->pUsed; } //---------------------------------------------------------------------------- // Largest single chunk of free space //---------------------------------------------------------------------------- uint64_t FileSystemUtils::SpaceInfo::GetLargestFreeChunk() const { return pImpl->pLargestChunk; } //---------------------------------------------------------------------------- // Recursively get space information for given path //---------------------------------------------------------------------------- XRootDStatus FileSystemUtils::GetSpaceInfo( SpaceInfo *&result, FileSystem *fs, const std::string &path ) { //-------------------------------------------------------------------------- // Locate all the disk servers containing the space //-------------------------------------------------------------------------- LocationInfo *locationInfo = 0; XRootDStatus st = fs->DeepLocate( path, OpenFlags::Compress, locationInfo ); if( !st.IsOK() ) return st; std::unique_ptr locationInfoPtr( locationInfo ); bool partial = st.code == suPartial ? true : false; std::vector > resp; resp.push_back( std::make_pair( std::string("oss.space"), (uint64_t)0 ) ); resp.push_back( std::make_pair( std::string("oss.free"), (uint64_t)0 ) ); resp.push_back( std::make_pair( std::string("oss.used"), (uint64_t)0 ) ); resp.push_back( std::make_pair( std::string("oss.maxf"), (uint64_t)0 ) ); //-------------------------------------------------------------------------- // Loop over the file servers and get the space info from each of them //-------------------------------------------------------------------------- LocationInfo::Iterator it; Buffer pathArg; pathArg.FromString( path ); for( it = locationInfo->Begin(); it != locationInfo->End(); ++it ) { //------------------------------------------------------------------------ // Query the server //------------------------------------------------------------------------ Buffer *spaceInfo = 0; FileSystem fs1( it->GetAddress() ); st = fs1.Query( QueryCode::Space, pathArg, spaceInfo ); if( !st.IsOK() ) return st; std::unique_ptr spaceInfoPtr( spaceInfo ); //------------------------------------------------------------------------ // Parse the cgi //------------------------------------------------------------------------ std::string fakeUrl = "root://fake/fake?" + spaceInfo->ToString(); URL url( fakeUrl ); if( !url.IsValid() ) return XRootDStatus( stError, errInvalidResponse ); URL::ParamsMap params = url.GetParams(); //------------------------------------------------------------------------ // Convert and add up the params //------------------------------------------------------------------------ st = XRootDStatus( stError, errInvalidResponse ); for( size_t i = 0; i < resp.size(); ++i ) { URL::ParamsMap::iterator paramIt = params.find( resp[i].first ); if( paramIt == params.end() ) return st; char *res; uint64_t num = ::strtoll( paramIt->second.c_str(), &res, 0 ); if( *res != 0 ) return st; if( resp[i].first == "oss.maxf" ) { if( num > resp[i].second ) resp[i].second = num; } else resp[i].second += num; } } result = new SpaceInfo( resp[0].second, resp[1].second, resp[2].second, resp[3].second ); st = XRootDStatus(); if( partial ) st.code = suPartial; return st; } } xrootd-5.6.9/src/XrdCl/XrdClFileSystemUtils.hh000066400000000000000000000075151457266313600212650ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2014 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #ifndef __XRD_CL_FILE_SYSTEM_UTILS_HH__ #define __XRD_CL_FILE_SYSTEM_UTILS_HH__ #include "XrdCl/XrdClXRootDResponses.hh" #include #include #include namespace XrdCl { class FileSystem; //---------------------------------------------------------------------------- //! A container for file system utility functions that do not belong in //! FileSystem //---------------------------------------------------------------------------- class FileSystemUtils { //------------------------------------------------------------------------ // Forward declaration for PIMPL //------------------------------------------------------------------------ struct SpaceInfoImpl; public: //------------------------------------------------------------------------ //! Container for space information //------------------------------------------------------------------------ class SpaceInfo { public: SpaceInfo( uint64_t total, uint64_t free, uint64_t used, uint64_t largestChunk ); ~SpaceInfo(); //-------------------------------------------------------------------- //! Amount of total space in MB //-------------------------------------------------------------------- uint64_t GetTotal() const; //-------------------------------------------------------------------- //! Amount of free space in MB //-------------------------------------------------------------------- uint64_t GetFree() const; //-------------------------------------------------------------------- //! Amount of used space in MB //-------------------------------------------------------------------- uint64_t GetUsed() const; //-------------------------------------------------------------------- //! Largest single chunk of free space //-------------------------------------------------------------------- uint64_t GetLargestFreeChunk() const; private: std::unique_ptr pImpl; }; //------------------------------------------------------------------------ //! Recursively get space information for given path //------------------------------------------------------------------------ static XRootDStatus GetSpaceInfo( SpaceInfo *&result, FileSystem *fs, const std::string &path ); }; } #endif // __XRD_CL_FILE_SYSTEM_UTILS HH__ xrootd-5.6.9/src/XrdCl/XrdClFileTimer.cc000066400000000000000000000033571457266313600200260ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2013 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . //------------------------------------------------------------------------------ #include "XrdCl/XrdClFileTimer.hh" #include "XrdCl/XrdClDefaultEnv.hh" #include "XrdCl/XrdClConstants.hh" #include "XrdCl/XrdClFileStateHandler.hh" namespace XrdCl { //---------------------------------------------------------------------------- // Perform the task's action //---------------------------------------------------------------------------- time_t FileTimer::Run( time_t now ) { pMutex.Lock(); std::set::iterator it; for( it = pFileObjects.begin(); it != pFileObjects.end(); ++it ) (*it)->Tick(now); pMutex.UnLock(); Env *env = DefaultEnv::GetEnv(); int timeoutResolution = DefaultTimeoutResolution; env->GetInt( "TimeoutResolution", timeoutResolution ); return now+timeoutResolution; } } xrootd-5.6.9/src/XrdCl/XrdClFileTimer.hh000066400000000000000000000067511457266313600200410ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2013 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . //------------------------------------------------------------------------------ #ifndef __XRD_CL_FILE_TIMER_HH__ #define __XRD_CL_FILE_TIMER_HH__ #include "XrdSys/XrdSysPthread.hh" #include "XrdCl/XrdClTaskManager.hh" namespace XrdCl { class FileStateHandler; //---------------------------------------------------------------------------- //! Task generating timeout events for FileStateHandlers in recovery mode //---------------------------------------------------------------------------- class FileTimer: public Task { public: //------------------------------------------------------------------------ //! Constructor //------------------------------------------------------------------------ FileTimer() { SetName( "FileTimer task" ); } //------------------------------------------------------------------------ //! Destructor //------------------------------------------------------------------------ virtual ~FileTimer() { } //------------------------------------------------------------------------ //! Register a file state handler //------------------------------------------------------------------------ void RegisterFileObject( FileStateHandler *file ) { XrdSysMutexHelper scopedLock( pMutex ); pFileObjects.insert( file ); } //------------------------------------------------------------------------ //! Un-register a file state handler //------------------------------------------------------------------------ void UnRegisterFileObject( FileStateHandler *file ) { XrdSysMutexHelper scopedLock( pMutex ); pFileObjects.erase( file ); } //------------------------------------------------------------------------ //! Lock the task //------------------------------------------------------------------------ void Lock() { pMutex.Lock(); } //------------------------------------------------------------------------ //! Un-lock the task //------------------------------------------------------------------------ void UnLock() { pMutex.UnLock(); } //------------------------------------------------------------------------ //! Perform the task's action //------------------------------------------------------------------------ virtual time_t Run( time_t now ); private: std::set pFileObjects; XrdSysMutex pMutex; }; } #endif // __XRD_CL_FILE_TIMER_HH__ xrootd-5.6.9/src/XrdCl/XrdClFinalOperation.hh000066400000000000000000000053311457266313600210640ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2017 by European Organization for Nuclear Research (CERN) // Author: Krzysztof Jamrog , // Michal Simon //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #ifndef SRC_XRDCL_XRDCLFINALOPERATION_HH_ #define SRC_XRDCL_XRDCLFINALOPERATION_HH_ #include namespace XrdCl { class XRootDStatus; //--------------------------------------------------------------------------- //! Final operation in the pipeline, always executed, no matter if the //! pipeline failed or not. //! //! Used to manage resources. //--------------------------------------------------------------------------- class FinalOperation { //declare friendship with other operations template class Derived, bool HasHndl, typename HdlrFactory, typename ... Args> friend class ConcreteOperation; public: //----------------------------------------------------------------------- //! Constructor //! //! @param final : the routine that should be called in order to finalize //! the pipeline //----------------------------------------------------------------------- FinalOperation( std::function final ) : final( std::move( final ) ) { } private: //----------------------------------------------------------------------- //! finalization routine //----------------------------------------------------------------------- std::function final; }; typedef FinalOperation Final; } #endif /* SRC_XRDCL_XRDCLFINALOPERATION_HH_ */ xrootd-5.6.9/src/XrdCl/XrdClForkHandler.cc000066400000000000000000000112211457266313600203320ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2012 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . //------------------------------------------------------------------------------ #include "XrdCl/XrdClForkHandler.hh" #include "XrdCl/XrdClConstants.hh" #include "XrdCl/XrdClLog.hh" #include "XrdCl/XrdClDefaultEnv.hh" #include "XrdCl/XrdClPostMaster.hh" #include "XrdCl/XrdClFileTimer.hh" #include "XrdCl/XrdClFileStateHandler.hh" namespace XrdCl { //---------------------------------------------------------------------------- // Constructor //---------------------------------------------------------------------------- ForkHandler::ForkHandler(): pPostMaster(0), pFileTimer(0) { } //---------------------------------------------------------------------------- // Handle the preparation part of the forking process //---------------------------------------------------------------------------- void ForkHandler::Prepare() { Log *log = DefaultEnv::GetLog(); pid_t pid = getpid(); log->Debug( UtilityMsg, "Running the prepare fork handler for process %d", pid ); pMutex.Lock(); if( pPostMaster ) pPostMaster->Stop(); pFileTimer->Lock(); //-------------------------------------------------------------------------- // Lock the user-level objects //-------------------------------------------------------------------------- log->Debug( UtilityMsg, "Locking File and FileSystem objects for process: " "%d", pid ); std::set::iterator itFile; for( itFile = pFileObjects.begin(); itFile != pFileObjects.end(); ++itFile ) (*itFile)->Lock(); std::set::iterator itFs; for( itFs = pFileSystemObjects.begin(); itFs != pFileSystemObjects.end(); ++itFs ) (*itFs)->Lock(); } //---------------------------------------------------------------------------- // Handle the parent post-fork //---------------------------------------------------------------------------- void ForkHandler::Parent() { Log *log = DefaultEnv::GetLog(); pid_t pid = getpid(); log->Debug( UtilityMsg, "Running the parent fork handler for process %d", pid ); log->Debug( UtilityMsg, "Unlocking File and FileSystem objects for " "process: %d", pid ); std::set::iterator itFile; for( itFile = pFileObjects.begin(); itFile != pFileObjects.end(); ++itFile ) (*itFile)->UnLock(); std::set::iterator itFs; for( itFs = pFileSystemObjects.begin(); itFs != pFileSystemObjects.end(); ++itFs ) (*itFs)->UnLock(); pFileTimer->UnLock(); if( pPostMaster ) pPostMaster->Start(); pMutex.UnLock(); } //------------------------------------------------------------------------ //! Handler the child post-fork //------------------------------------------------------------------------ void ForkHandler::Child() { Log *log = DefaultEnv::GetLog(); pid_t pid = getpid(); log->Debug( UtilityMsg, "Running the child fork handler for process %d", pid ); log->Debug( UtilityMsg, "Unlocking File and FileSystem objects for " "process: %d", pid ); std::set::iterator itFile; for( itFile = pFileObjects.begin(); itFile != pFileObjects.end(); ++itFile ) { (*itFile)->AfterForkChild(); (*itFile)->UnLock(); } std::set::iterator itFs; for( itFs = pFileSystemObjects.begin(); itFs != pFileSystemObjects.end(); ++itFs ) (*itFs)->UnLock(); pFileTimer->UnLock(); if( pPostMaster ) { pPostMaster->Finalize(); pPostMaster->Initialize(); pPostMaster->Start(); pPostMaster->GetTaskManager()->RegisterTask( pFileTimer, time(0), false ); } pMutex.UnLock(); } } xrootd-5.6.9/src/XrdCl/XrdClForkHandler.hh000066400000000000000000000105051457266313600203500ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2012 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . //------------------------------------------------------------------------------ #ifndef __XRD_CL_FORK_HANDLER_HH__ #define __XRD_CL_FORK_HANDLER_HH__ #include #include namespace XrdCl { class FileStateHandler; class FileSystem; class PostMaster; class FileTimer; //---------------------------------------------------------------------------- // Helper class for handling forking //---------------------------------------------------------------------------- class ForkHandler { public: ForkHandler(); //------------------------------------------------------------------------ //! Register a file object //------------------------------------------------------------------------ void RegisterFileObject( FileStateHandler *file ) { XrdSysMutexHelper scopedLock( pMutex ); pFileObjects.insert( file ); } //------------------------------------------------------------------------ // Un-register a file object //------------------------------------------------------------------------ void UnRegisterFileObject( FileStateHandler *file ) { XrdSysMutexHelper scopedLock( pMutex ); pFileObjects.erase( file ); } //------------------------------------------------------------------------ // Register a file system object //------------------------------------------------------------------------ void RegisterFileSystemObject( FileSystem *fs ) { XrdSysMutexHelper scopedLock( pMutex ); pFileSystemObjects.insert( fs ); } //------------------------------------------------------------------------ //! Un-register a file system object //------------------------------------------------------------------------ void UnRegisterFileSystemObject( FileSystem *fs ) { XrdSysMutexHelper scopedLock( pMutex ); pFileSystemObjects.erase( fs ); } //------------------------------------------------------------------------ //! Register a post master object //------------------------------------------------------------------------ void RegisterPostMaster( PostMaster *postMaster ) { XrdSysMutexHelper scopedLock( pMutex ); pPostMaster = postMaster; } void RegisterFileTimer( FileTimer *fileTimer ) { XrdSysMutexHelper scopedLock( pMutex ); pFileTimer = fileTimer; } //------------------------------------------------------------------------ //! Handle the preparation part of the forking process //------------------------------------------------------------------------ void Prepare(); //------------------------------------------------------------------------ //! Handle the parent post-fork //------------------------------------------------------------------------ void Parent(); //------------------------------------------------------------------------ //! Handler the child post-fork //------------------------------------------------------------------------ void Child(); private: std::set pFileObjects; std::set pFileSystemObjects; PostMaster *pPostMaster; FileTimer *pFileTimer; XrdSysMutex pMutex; }; } #endif // __XRD_CL_FORK_HANDLER_HH__ xrootd-5.6.9/src/XrdCl/XrdClFwd.hh000066400000000000000000000232531457266313600166750ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2017 by European Organization for Nuclear Research (CERN) // Author: Krzysztof Jamrog , // Michal Simon //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #ifndef SRC_XRDCL_XRDCLFWD_HH_ #define SRC_XRDCL_XRDCLFWD_HH_ #include #include namespace XrdCl { //---------------------------------------------------------------------------- //! Helper class for storing forwarded values //! Allocates memory respectively aligned for T but constructs the object //! only on assignment. //! //! @arg T : type of the value //---------------------------------------------------------------------------- template struct FwdStorage { //-------------------------------------------------------------------------- //! Default constructor //-------------------------------------------------------------------------- FwdStorage() : ptr( nullptr ) { } //-------------------------------------------------------------------------- //! Constructor from T. //! @param value : value for forwarding //-------------------------------------------------------------------------- FwdStorage( const T &value ) : ptr( new( &storage.memory ) T( value ) ) { } //-------------------------------------------------------------------------- //! Assignment operator from T //! @param value : value for forwarding //-------------------------------------------------------------------------- FwdStorage& operator=( const T &value ) { ptr = new( &storage.memory ) T( value ); return *this; } //-------------------------------------------------------------------------- //! Move constructor from T. //! @param value : value for forwarding //-------------------------------------------------------------------------- FwdStorage( T && value ) : ptr( new( &storage.memory ) T( std::move( value ) ) ) { } //-------------------------------------------------------------------------- //! Move assignment operator from T //! @param value : value for forwarding //-------------------------------------------------------------------------- FwdStorage& operator=( T && value ) { ptr = new( &storage.memory ) T( std::move( value ) ); return *this; } //-------------------------------------------------------------------------- //! Destructor //-------------------------------------------------------------------------- ~FwdStorage() { if( ptr ) ptr->~T(); } //-------------------------------------------------------------------------- //! Memory for the value //-------------------------------------------------------------------------- union Memory { //------------------------------------------------------------------------ //! Make sure the default constructor of T won't be called //------------------------------------------------------------------------ Memory() { } //------------------------------------------------------------------------ //! Make sure the destrutor of T won't be called //------------------------------------------------------------------------ ~Memory() { } //------------------------------------------------------------------------ //! The memory for storing forwarded value //------------------------------------------------------------------------ T memory; }; //-------------------------------------------------------------------------- //! The memory for storying forwarded value //-------------------------------------------------------------------------- Memory storage; //-------------------------------------------------------------------------- //! Pointer to the forwarded value //-------------------------------------------------------------------------- T *ptr; }; //---------------------------------------------------------------------------- //! A helper class for forwarding arguments between operations. //! In practice it's a wrapper around std::shared_ptr using FwdStorage as //! underlying memory. //! //! @arg T : type of forwarded value //---------------------------------------------------------------------------- template struct Fwd : protected std::shared_ptr> { //------------------------------------------------------------------------ //! Default constructor. //! //! Allocates memory for the underlying value object without callying //! its constructor. //------------------------------------------------------------------------ Fwd() : std::shared_ptr>( std::make_shared>() ) { } //------------------------------------------------------------------------ //! Copy constructor. //------------------------------------------------------------------------ Fwd( const Fwd &fwd ) : std::shared_ptr>( fwd ) { } //------------------------------------------------------------------------ //! Move constructor. //------------------------------------------------------------------------ Fwd( Fwd && fwd ) : std::shared_ptr>( std::move( fwd ) ) { } //------------------------------------------------------------------------ //! Initialize from shared_ptr //------------------------------------------------------------------------ Fwd( std::shared_ptr> && ptr ) : std::shared_ptr>( std::move( ptr ) ) { } //------------------------------------------------------------------------ //! Constructor from value //------------------------------------------------------------------------ explicit Fwd( const T &value ) { *this->get() = value; } //------------------------------------------------------------------------ //! Move construct from value //------------------------------------------------------------------------ explicit Fwd( T &&value ) { *this->get() = std::move( value ); } //------------------------------------------------------------------------ //! Assignment operator. //! //! @param value : forwarded value //! @throws : std::logic_error //------------------------------------------------------------------------ Fwd& operator=( const T &value ) { *this->get() = value; return *this; } //------------------------------------------------------------------------ //! Move assignment operator. //! //! @param value : forwarded value //! @throws : std::logic_error //------------------------------------------------------------------------ Fwd& operator=( T && value ) { *this->get() = std::move( value ); return *this; } //------------------------------------------------------------------------ //! Dereferencing operator. Note if Fwd has not been assigned with //! a value this will trigger an exception //! //! @return : reference to the underlying value //! @throws : std::logic_error //------------------------------------------------------------------------ T& operator*() const { if( !bool( this->get()->ptr ) ) throw std::logic_error( "XrdCl::Fwd contains no value!" ); return *this->get()->ptr; } //------------------------------------------------------------------------ //! Dereferencing member operator. Note if Fwd has not been assigned with //! a value this will trigger an exception //! //! @return : pointer to the underlying value //! @throws : std::logic_error //------------------------------------------------------------------------ T* operator->() const { if( !bool( this->get()->ptr ) ) throw std::logic_error( "XrdCl::Fwd contains no value!" ); return this->get()->ptr; } //------------------------------------------------------------------------ //! Check if it contains a valid value //------------------------------------------------------------------------ bool Valid() const { return bool( this->get()->ptr ); } }; //-------------------------------------------------------------------------- // Utility function for creating forwardable objects //-------------------------------------------------------------------------- template inline std::shared_ptr> make_fwd( Args&&... args ) { return std::make_shared>( std::forward( args )... ); } } #endif /* SRC_XRDCL_XRDCLFWD_HH_ */ xrootd-5.6.9/src/XrdCl/XrdClInQueue.cc000066400000000000000000000143451457266313600175200ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2012 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . //------------------------------------------------------------------------------ #include "XProtocol/XProtocol.hh" #include "XrdCl/XrdClInQueue.hh" #include "XrdCl/XrdClPostMasterInterfaces.hh" #include "XrdCl/XrdClMessage.hh" #include "XrdCl/XrdClLog.hh" #include "XrdCl/XrdClDefaultEnv.hh" #include "XrdCl/XrdClConstants.hh" #include // for network unmarshalling stuff namespace XrdCl { //---------------------------------------------------------------------------- // Filter messages //---------------------------------------------------------------------------- bool InQueue::DiscardMessage( Message& msg, uint16_t& sid) const { if( msg.GetSize() < 8 ) return true; ServerResponse *rsp = (ServerResponse *)msg.GetBuffer(); // We only care about async responses, but those are extracted now // in the SocketHandler if( rsp->hdr.status == kXR_attn ) return true; else sid = ((uint16_t)rsp->hdr.streamid[1] << 8) | (uint16_t)rsp->hdr.streamid[0]; return false; } //---------------------------------------------------------------------------- // Add a listener that should be notified about incoming messages //---------------------------------------------------------------------------- void InQueue::AddMessageHandler( MsgHandler *handler, time_t expires, bool &rmMsg ) { uint16_t handlerSid = handler->GetSid(); XrdSysMutexHelper scopedLock( pMutex ); pHandlers[handlerSid] = HandlerAndExpire( handler, expires ); } //---------------------------------------------------------------------------- // Get a message handler interested in receiving message whose header // is stored in msg //---------------------------------------------------------------------------- MsgHandler *InQueue::GetHandlerForMessage( std::shared_ptr &msg, time_t &expires, uint16_t &action ) { time_t exp = 0; uint16_t act = 0; uint16_t msgSid = 0; MsgHandler* handler = 0; if (DiscardMessage(*msg, msgSid)) { return handler; } XrdSysMutexHelper scopedLock( pMutex ); HandlerMap::iterator it = pHandlers.find(msgSid); if (it != pHandlers.end()) { Log *log = DefaultEnv::GetLog(); handler = it->second.first; act = handler->Examine( msg ); exp = it->second.second; log->Debug( ExDbgMsg, "[msg: 0x%x] Assigned MsgHandler: 0x%x.", msg.get(), handler ); if( act & MsgHandler::RemoveHandler ) { pHandlers.erase( it ); log->Debug( ExDbgMsg, "[handler: 0x%x] Removed MsgHandler: 0x%x from the in-queue.", handler, handler ); } } if( handler ) { expires = exp; action = act; } return handler; } //---------------------------------------------------------------------------- // Re-insert the handler without scanning the cached messages //---------------------------------------------------------------------------- void InQueue::ReAddMessageHandler( MsgHandler *handler, time_t expires ) { uint16_t handlerSid = handler->GetSid(); XrdSysMutexHelper scopedLock( pMutex ); pHandlers[handlerSid] = HandlerAndExpire( handler, expires ); } //---------------------------------------------------------------------------- // Remove a listener //---------------------------------------------------------------------------- void InQueue::RemoveMessageHandler( MsgHandler *handler ) { uint16_t handlerSid = handler->GetSid(); XrdSysMutexHelper scopedLock( pMutex ); pHandlers.erase(handlerSid); Log *log = DefaultEnv::GetLog(); log->Debug( ExDbgMsg, "[handler: 0x%x] Removed MsgHandler: 0x%x from the in-queue.", handler, handler ); } //---------------------------------------------------------------------------- // Report an event to the handlers //---------------------------------------------------------------------------- void InQueue::ReportStreamEvent( MsgHandler::StreamEvent event, XRootDStatus status ) { uint8_t action = 0; XrdSysMutexHelper scopedLock( pMutex ); for( HandlerMap::iterator it = pHandlers.begin(); it != pHandlers.end(); ) { action = it->second.first->OnStreamEvent( event, status ); if( action & MsgHandler::RemoveHandler ) { auto next = it; ++next; pHandlers.erase( it ); it = next; } else ++it; } } //---------------------------------------------------------------------------- // Timeout handlers //---------------------------------------------------------------------------- void InQueue::ReportTimeout( time_t now ) { if( !now ) now = ::time(0); XrdSysMutexHelper scopedLock( pMutex ); HandlerMap::iterator it = pHandlers.begin(); while( it != pHandlers.end() ) { if( it->second.second <= now ) { uint8_t act = it->second.first->OnStreamEvent( MsgHandler::Timeout, Status( stError, errOperationExpired ) ); auto next = it; ++next; if( act & MsgHandler::RemoveHandler ) pHandlers.erase( it ); it = next; } else ++it; } } } xrootd-5.6.9/src/XrdCl/XrdClInQueue.hh000066400000000000000000000111741457266313600175270ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2012 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . //------------------------------------------------------------------------------ #ifndef __XRD_CL_IN_QUEUE_HH__ #define __XRD_CL_IN_QUEUE_HH__ #include #include #include #include #include "XrdCl/XrdClXRootDResponses.hh" #include "XrdCl/XrdClPostMasterInterfaces.hh" namespace XrdCl { class Message; //---------------------------------------------------------------------------- //! A synchronize queue for incoming data //---------------------------------------------------------------------------- class InQueue { public: //------------------------------------------------------------------------ //! Add a listener that should be notified about incoming messages //! //! @param handler message handler //! @param expires time when the message handler expires //! @param rmMsg will be set to true if a left over message matching the //! request has been removed from the queue //------------------------------------------------------------------------ void AddMessageHandler( MsgHandler *handler, time_t expires, bool &rmMsg ); //------------------------------------------------------------------------ //! Get a message handler interested in receiving message whose header //! is stored in msg //! //! @param msg message header //! @param expires handle's expiration timestamp //! @param action the action declared by the handler //! //! @return handler or 0 if none is interested //------------------------------------------------------------------------ MsgHandler *GetHandlerForMessage( std::shared_ptr &msg, time_t &expires, uint16_t &action ); //------------------------------------------------------------------------ //! Re-insert the handler without scanning the cached messages //------------------------------------------------------------------------ void ReAddMessageHandler( MsgHandler *handler, time_t expires ); //------------------------------------------------------------------------ //! Remove a listener //------------------------------------------------------------------------ void RemoveMessageHandler( MsgHandler *handler ); //------------------------------------------------------------------------ //! Report an event to the handlers //------------------------------------------------------------------------ void ReportStreamEvent( MsgHandler::StreamEvent event, XRootDStatus status ); //------------------------------------------------------------------------ //! Timeout handlers //------------------------------------------------------------------------ void ReportTimeout( time_t now = 0 ); private: //------------------------------------------------------------------------ //! Discard messages that don't meet basic criteria and extract the //! message sid //! //! @param msg message object //! @param sid extracted message sid used later for matching with the //! handler //! //! @return true if message discarded, otherwise false //------------------------------------------------------------------------ bool DiscardMessage(Message& msg, uint16_t& sid) const; typedef std::pair HandlerAndExpire; typedef std::map HandlerMap; HandlerMap pHandlers; XrdSysRecMutex pMutex; }; } #endif // __XRD_CL_IN_QUEUE_HH__ xrootd-5.6.9/src/XrdCl/XrdClJobManager.cc000066400000000000000000000120061457266313600201420ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2013 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . //------------------------------------------------------------------------------ #include "XrdCl/XrdClJobManager.hh" #include "XrdCl/XrdClLog.hh" #include "XrdCl/XrdClDefaultEnv.hh" #include "XrdCl/XrdClConstants.hh" #include "XrdSys/XrdSysE2T.hh" //------------------------------------------------------------------------------ // The thread //------------------------------------------------------------------------------ extern "C" { static void *RunRunnerThread( void *arg ) { using namespace XrdCl; JobManager *mgr = (JobManager*)arg; mgr->RunJobs(); return 0; } } namespace XrdCl { //---------------------------------------------------------------------------- // Initialize the job manager //---------------------------------------------------------------------------- bool JobManager::Initialize() { return true; } //---------------------------------------------------------------------------- // Finalize the job manager, clear the queues //---------------------------------------------------------------------------- bool JobManager::Finalize() { pJobs.Clear(); return true; } //---------------------------------------------------------------------------- // Start the workers //---------------------------------------------------------------------------- bool JobManager::Start() { XrdSysMutexHelper scopedLock( pMutex ); Log *log = DefaultEnv::GetLog(); log->Debug( JobMgrMsg, "Starting the job manager..." ); if( pRunning ) { log->Error( JobMgrMsg, "The job manager is already running" ); return false; } for( uint32_t i = 0; i < pWorkers.size(); ++i ) { int ret = ::pthread_create( &pWorkers[i], 0, ::RunRunnerThread, this ); if( ret != 0 ) { log->Error( JobMgrMsg, "Unable to spawn a job worker thread: %s", XrdSysE2T( errno ) ); if( i > 0 ) StopWorkers( i ); return false; } } pRunning = true; log->Debug( JobMgrMsg, "Job manager started, %d workers", pWorkers.size() ); return true; } //---------------------------------------------------------------------------- // Stop the workers //---------------------------------------------------------------------------- bool JobManager::Stop() { XrdSysMutexHelper scopedLock( pMutex ); Log *log = DefaultEnv::GetLog(); log->Debug( JobMgrMsg, "Stopping the job manager..." ); if( !pRunning ) { log->Error( JobMgrMsg, "The job manager is not running" ); return false; } StopWorkers( pWorkers.size() ); pRunning = false; log->Debug( JobMgrMsg, "Job manager stopped" ); return true; } //---------------------------------------------------------------------------- // Stop all workers up to n'th //---------------------------------------------------------------------------- void JobManager::StopWorkers( uint32_t n ) { Log *log = DefaultEnv::GetLog(); for( uint32_t i = 0; i < n; ++i ) { void *threadRet; log->Dump( JobMgrMsg, "Stopping worker #%d...", i ); int rc = pthread_cancel( pWorkers[i] ); if( rc != 0 ) { log->Error( TaskMgrMsg, "Unable to cancel worker #%d: %s", i, XrdSysE2T( errno ) ); if( rc == ESRCH ) continue; abort(); } rc = pthread_join( pWorkers[i], (void**)&threadRet ); if( rc != 0 ) { log->Error( TaskMgrMsg, "Unable to join worker #%d: %s", i, XrdSysE2T( errno ) ); if( rc == ESRCH ) continue; abort(); } log->Dump( JobMgrMsg, "Worker #%d stopped", i ); } } //---------------------------------------------------------------------------- // Initialize the job manager //---------------------------------------------------------------------------- void JobManager::RunJobs() { pthread_setcanceltype( PTHREAD_CANCEL_DEFERRED, 0 ); for( ;; ) { JobHelper h = pJobs.Get(); pthread_setcancelstate( PTHREAD_CANCEL_DISABLE, 0 ); h.job->Run( h.arg ); pthread_setcancelstate( PTHREAD_CANCEL_ENABLE, 0 ); } } } xrootd-5.6.9/src/XrdCl/XrdClJobManager.hh000066400000000000000000000113741457266313600201630ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2013 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . //------------------------------------------------------------------------------ #ifndef __XRD_CL_JOB_MANAGER_HH__ #define __XRD_CL_JOB_MANAGER_HH__ #include #include #include #include #include "XrdCl/XrdClSyncQueue.hh" namespace XrdCl { //---------------------------------------------------------------------------- //! Interface for a job to be run by the job manager //---------------------------------------------------------------------------- class Job { public: //------------------------------------------------------------------------ //! Virtual destructor //------------------------------------------------------------------------ virtual ~Job() {}; //------------------------------------------------------------------------ //! The job logic //------------------------------------------------------------------------ virtual void Run( void *arg ) = 0; }; //---------------------------------------------------------------------------- //! A synchronized queue //---------------------------------------------------------------------------- class JobManager { public: //------------------------------------------------------------------------ //! Constructor //------------------------------------------------------------------------ JobManager( uint32_t workers ) { pRunning = false; pWorkers.resize( workers ); } //------------------------------------------------------------------------ //! Destructor //------------------------------------------------------------------------ ~JobManager() { } //------------------------------------------------------------------------ //! Initialize the job manager //------------------------------------------------------------------------ bool Initialize(); //------------------------------------------------------------------------ //! Finalize the job manager, clear the queues //------------------------------------------------------------------------ bool Finalize(); //------------------------------------------------------------------------ //! Start the workers //------------------------------------------------------------------------ bool Start(); //------------------------------------------------------------------------ //! Stop the workers //------------------------------------------------------------------------ bool Stop(); //------------------------------------------------------------------------ //! Add a job to be run //------------------------------------------------------------------------ void QueueJob( Job *job, void *arg = 0 ) { pJobs.Put( JobHelper( job, arg ) ); } //------------------------------------------------------------------------ //! Run the jobs //------------------------------------------------------------------------ void RunJobs(); bool IsWorker() { pthread_t thread = pthread_self(); std::vector::iterator itr = std::find( pWorkers.begin(), pWorkers.end(), thread ); return itr != pWorkers.end(); } private: //------------------------------------------------------------------------ //! Stop all workers up to n'th //------------------------------------------------------------------------ void StopWorkers( uint32_t n ); struct JobHelper { JobHelper( Job *j = 0, void *a = 0 ): job(j), arg(a) {} Job *job; void *arg; }; std::vector pWorkers; SyncQueue pJobs; XrdSysMutex pMutex; bool pRunning; }; } #endif // __XRD_CL_ANY_OBJECT_HH__ xrootd-5.6.9/src/XrdCl/XrdClLocalFileHandler.cc000066400000000000000000001074251457266313600212770ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2017 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH // Author: Paul-Niklas Kramp // Michal Simon //------------------------------------------------------------------------------ // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . //------------------------------------------------------------------------------ #include "XrdCl/XrdClLocalFileHandler.hh" #include "XrdCl/XrdClConstants.hh" #include "XrdCl/XrdClPostMaster.hh" #include "XrdCl/XrdClURL.hh" #include "XrdCl/XrdClMessageUtils.hh" #include "XrdCl/XrdClFileSystem.hh" #include "XProtocol/XProtocol.hh" #include "XrdSys/XrdSysE2T.hh" #include "XrdSys/XrdSysXAttr.hh" #include "XrdSys/XrdSysFAttr.hh" #include "XrdSys/XrdSysFD.hh" #include #include #include #include #include #include #include #include #include #include namespace { class AioCtx { public: enum Opcode { None, Read, Write, Sync }; AioCtx( const XrdCl::HostList &hostList, XrdCl::ResponseHandler *handler ) : opcode( None ), hosts( new XrdCl::HostList( hostList ) ), handler( handler ) { aiocb *ptr = new aiocb(); memset( ptr, 0, sizeof( aiocb ) ); XrdCl::Env *env = XrdCl::DefaultEnv::GetEnv(); int useSignals = XrdCl::DefaultAioSignal; env->GetInt( "AioSignal", useSignals ); if( useSignals ) { static SignalHandlerRegistrator registrator; // registers the signal handler ptr->aio_sigevent.sigev_notify = SIGEV_SIGNAL; ptr->aio_sigevent.sigev_signo = SIGUSR1; } else { ptr->aio_sigevent.sigev_notify = SIGEV_THREAD; ptr->aio_sigevent.sigev_notify_function = ThreadHandler; } ptr->aio_sigevent.sigev_value.sival_ptr = this; cb.reset( ptr ); } void SetWrite( int fd, size_t offset, size_t size, const void *buffer ) { cb->aio_fildes = fd; cb->aio_offset = offset; cb->aio_buf = const_cast( buffer ); cb->aio_nbytes = size; opcode = Opcode::Write; } void SetRead( int fd, size_t offset, size_t size, void *buffer ) { cb->aio_fildes = fd; cb->aio_offset = offset; cb->aio_buf = buffer; cb->aio_nbytes = size; opcode = Opcode::Read; } void SetFsync( int fd ) { cb->aio_fildes = fd; opcode = Opcode::Sync; } static void ThreadHandler( sigval arg ) { std::unique_ptr me( reinterpret_cast( arg.sival_ptr ) ); Handler( std::move( me ) ); } static void SignalHandler( int sig, siginfo_t *info, void *ucontext ) { std::unique_ptr me( reinterpret_cast( info->si_value.sival_ptr ) ); Handler( std::move( me ) ); } operator aiocb*() { return cb.get(); } private: struct SignalHandlerRegistrator { SignalHandlerRegistrator() { struct sigaction newact, oldact; newact.sa_sigaction = SignalHandler; sigemptyset( &newact.sa_mask ); newact.sa_flags = SA_SIGINFO; int rc = sigaction( SIGUSR1, &newact, &oldact ); if( rc < 0 ) throw std::runtime_error( XrdSysE2T( errno ) ); } }; static void Handler( std::unique_ptr me ) { if( me->opcode == Opcode::None ) return; using namespace XrdCl; int rc = aio_return( me->cb.get() ); if( rc < 0 ) { int errcode = aio_error( me->cb.get() ); Log *log = DefaultEnv::GetLog(); log->Error( FileMsg, GetErrMsg( me->opcode ), XrdSysE2T( errcode ) ); XRootDStatus *error = new XRootDStatus( stError, errLocalError, errcode ) ; QueueTask( error, 0, me->hosts, me->handler ); } else { AnyObject *resp = 0; if( me->opcode == Opcode::Read ) { ChunkInfo *chunk = new ChunkInfo( me->cb->aio_offset, rc, const_cast( me->cb->aio_buf ) ); resp = new AnyObject(); resp->Set( chunk ); } QueueTask( new XRootDStatus(), resp, me->hosts, me->handler ); } } static const char* GetErrMsg( Opcode opcode ) { static const char readmsg[] = "Read: failed %s"; static const char writemsg[] = "Write: failed %s"; static const char syncmsg[] = "Sync: failed %s"; switch( opcode ) { case Opcode::Read: return readmsg; case Opcode::Write: return writemsg; case Opcode::Sync: return syncmsg; default: return 0; } } static void QueueTask( XrdCl::XRootDStatus *status, XrdCl::AnyObject *resp, XrdCl::HostList *hosts, XrdCl::ResponseHandler *handler ) { using namespace XrdCl; // if it is simply the sync handler we can release the semaphore // and return there is no need to execute this in the thread-pool SyncResponseHandler *syncHandler = dynamic_cast( handler ); if( syncHandler || DefaultEnv::GetPostMaster() == nullptr ) { syncHandler->HandleResponse( status, resp ); } else { JobManager *jmngr = DefaultEnv::GetPostMaster()->GetJobManager(); LocalFileTask *task = new LocalFileTask( status, resp, hosts, handler ); jmngr->QueueJob( task ); } } std::unique_ptr cb; Opcode opcode; XrdCl::HostList *hosts; XrdCl::ResponseHandler *handler; }; }; namespace XrdCl { //------------------------------------------------------------------------ // Constructor //------------------------------------------------------------------------ LocalFileHandler::LocalFileHandler() : fd( -1 ) { } //------------------------------------------------------------------------ // Destructor //------------------------------------------------------------------------ LocalFileHandler::~LocalFileHandler() { } //------------------------------------------------------------------------ // Open //------------------------------------------------------------------------ XRootDStatus LocalFileHandler::Open( const std::string& url, uint16_t flags, uint16_t mode, ResponseHandler* handler, uint16_t timeout ) { AnyObject *resp = 0; XRootDStatus st = OpenImpl( url, flags, mode, resp ); if( !st.IsOK() && st.code != errLocalError ) return st; return QueueTask( new XRootDStatus( st ), resp, handler ); } XRootDStatus LocalFileHandler::Open( const URL *url, const Message *req, AnyObject *&resp ) { const ClientOpenRequest* request = reinterpret_cast( req->GetBuffer() ); uint16_t flags = ntohs( request->options ); uint16_t mode = ntohs( request->mode ); return OpenImpl( url->GetURL(), flags, mode, resp ); } //------------------------------------------------------------------------ // Close //------------------------------------------------------------------------ XRootDStatus LocalFileHandler::Close( ResponseHandler* handler, uint16_t timeout ) { if( close( fd ) == -1 ) { Log *log = DefaultEnv::GetLog(); log->Error( FileMsg, "Close: file fd: %i %s", fd, XrdSysE2T( errno ) ); XRootDStatus *error = new XRootDStatus( stError, errLocalError, errno ); return QueueTask( error, 0, handler ); } return QueueTask( new XRootDStatus(), 0, handler ); } //------------------------------------------------------------------------ // Stat //------------------------------------------------------------------------ XRootDStatus LocalFileHandler::Stat( ResponseHandler* handler, uint16_t timeout ) { Log *log = DefaultEnv::GetLog(); struct stat ssp; if( fstat( fd, &ssp ) == -1 ) { log->Error( FileMsg, "Stat: failed fd: %i %s", fd, XrdSysE2T( errno ) ); XRootDStatus *error = new XRootDStatus( stError, errLocalError, errno ); return QueueTask( error, 0, handler ); } std::ostringstream data; data << ssp.st_dev << " " << ssp.st_size << " " << ssp.st_mode << " " << ssp.st_mtime; log->Debug( FileMsg, data.str().c_str() ); StatInfo *statInfo = new StatInfo(); if( !statInfo->ParseServerResponse( data.str().c_str() ) ) { log->Error( FileMsg, "Stat: ParseServerResponse failed." ); delete statInfo; return QueueTask( new XRootDStatus( stError, errLocalError, kXR_FSError ), 0, handler ); } AnyObject *resp = new AnyObject(); resp->Set( statInfo ); return QueueTask( new XRootDStatus(), resp, handler ); } //------------------------------------------------------------------------ // Read //------------------------------------------------------------------------ XRootDStatus LocalFileHandler::Read( uint64_t offset, uint32_t size, void* buffer, ResponseHandler* handler, uint16_t timeout ) { #if defined(__APPLE__) Log *log = DefaultEnv::GetLog(); int read = 0; if( ( read = pread( fd, buffer, size, offset ) ) == -1 ) { log->Error( FileMsg, "Read: failed %s", XrdSysE2T( errno ) ); XRootDStatus *error = new XRootDStatus( stError, errLocalError, errno ); return QueueTask( error, 0, handler ); } ChunkInfo *chunk = new ChunkInfo( offset, read, buffer ); AnyObject *resp = new AnyObject(); resp->Set( chunk ); return QueueTask( new XRootDStatus(), resp, handler ); #else AioCtx *ctx = new AioCtx( pHostList, handler ); ctx->SetRead( fd, offset, size, buffer ); int rc = aio_read( *ctx ); if( rc < 0 ) { Log *log = DefaultEnv::GetLog(); log->Error( FileMsg, "Read: failed %s", XrdSysE2T( errno ) ); return XRootDStatus( stError, errLocalError, errno ); } return XRootDStatus(); #endif } //------------------------------------------------------------------------ // ReadV //------------------------------------------------------------------------ XRootDStatus LocalFileHandler::ReadV( uint64_t offset, struct iovec *iov, int iovcnt, ResponseHandler *handler, uint16_t timeout ) { Log *log = DefaultEnv::GetLog(); #if defined(__APPLE__) ssize_t ret = lseek( fd, offset, SEEK_SET ); if( ret >= 0 ) ret = readv( fd, iov, iovcnt ); #else ssize_t ret = preadv( fd, iov, iovcnt, offset ); #endif if( ret == -1 ) { log->Error( FileMsg, "ReadV: failed %s", XrdSysE2T( errno ) ); XRootDStatus *error = new XRootDStatus( stError, errLocalError, errno ); return QueueTask( error, 0, handler ); } VectorReadInfo *info = new VectorReadInfo(); info->SetSize( ret ); uint64_t choff = offset; uint32_t left = ret; for( int i = 0; i < iovcnt; ++i ) { uint32_t chlen = iov[i].iov_len; if( chlen > left ) chlen = left; info->GetChunks().emplace_back( choff, chlen, iov[i].iov_base); left -= chlen; choff += chlen; } AnyObject *resp = new AnyObject(); resp->Set( info ); return QueueTask( new XRootDStatus(), resp, handler ); } //------------------------------------------------------------------------ // Write //------------------------------------------------------------------------ XRootDStatus LocalFileHandler::Write( uint64_t offset, uint32_t size, const void* buffer, ResponseHandler* handler, uint16_t timeout ) { #if defined(__APPLE__) const char *buff = reinterpret_cast( buffer ); size_t bytesWritten = 0; while( bytesWritten < size ) { ssize_t ret = pwrite( fd, buff, size, offset ); if( ret < 0 ) { Log *log = DefaultEnv::GetLog(); log->Error( FileMsg, "Write: failed %s", XrdSysE2T( errno ) ); XRootDStatus *error = new XRootDStatus( stError, errLocalError, errno ); return QueueTask( error, 0, handler ); } offset += ret; buff += ret; bytesWritten += ret; } return QueueTask( new XRootDStatus(), 0, handler ); #else AioCtx *ctx = new AioCtx( pHostList, handler ); ctx->SetWrite( fd, offset, size, buffer ); int rc = aio_write( *ctx ); if( rc < 0 ) { Log *log = DefaultEnv::GetLog(); log->Error( FileMsg, "Write: failed %s", XrdSysE2T( errno ) ); return XRootDStatus( stError, errLocalError, errno ); } return XRootDStatus(); #endif } //------------------------------------------------------------------------ // Sync //------------------------------------------------------------------------ XRootDStatus LocalFileHandler::Sync( ResponseHandler* handler, uint16_t timeout ) { #if defined(__APPLE__) if( fsync( fd ) ) { Log *log = DefaultEnv::GetLog(); log->Error( FileMsg, "Sync: failed %s", XrdSysE2T( errno ) ); XRootDStatus *error = new XRootDStatus( stError, errOSError, XProtocol::mapError( errno ), XrdSysE2T( errno ) ); return QueueTask( error, 0, handler ); } return QueueTask( new XRootDStatus(), 0, handler ); #else AioCtx *ctx = new AioCtx( pHostList, handler ); ctx->SetFsync( fd ); int rc = aio_fsync( O_SYNC, *ctx ); if( rc < 0 ) { Log *log = DefaultEnv::GetLog(); log->Error( FileMsg, "Sync: failed %s", XrdSysE2T( errno ) ); return XRootDStatus( stError, errLocalError, errno ); } #endif return XRootDStatus(); } //------------------------------------------------------------------------ // Truncate //------------------------------------------------------------------------ XRootDStatus LocalFileHandler::Truncate( uint64_t size, ResponseHandler* handler, uint16_t timeout ) { if( ftruncate( fd, size ) ) { Log *log = DefaultEnv::GetLog(); log->Error( FileMsg, "Truncate: failed, file descriptor: %i, %s", fd, XrdSysE2T( errno ) ); XRootDStatus *error = new XRootDStatus( stError, errLocalError, errno ); return QueueTask( error, 0, handler ); } return QueueTask( new XRootDStatus( stOK ), 0, handler ); } //------------------------------------------------------------------------ // VectorRead //------------------------------------------------------------------------ XRootDStatus LocalFileHandler::VectorRead( const ChunkList& chunks, void* buffer, ResponseHandler* handler, uint16_t timeout ) { std::unique_ptr info( new VectorReadInfo() ); size_t totalSize = 0; bool useBuffer( buffer ); for( auto itr = chunks.begin(); itr != chunks.end(); ++itr ) { auto &chunk = *itr; if( !useBuffer ) buffer = chunk.buffer; ssize_t bytesRead = pread( fd, buffer, chunk.length, chunk.offset ); if( bytesRead < 0 ) { Log *log = DefaultEnv::GetLog(); log->Error( FileMsg, "VectorRead: failed, file descriptor: %i, %s", fd, XrdSysE2T( errno ) ); XRootDStatus *error = new XRootDStatus( stError, errLocalError, errno ); return QueueTask( error, 0, handler ); } totalSize += bytesRead; info->GetChunks().push_back( ChunkInfo( chunk.offset, bytesRead, buffer ) ); if( useBuffer ) buffer = reinterpret_cast( buffer ) + bytesRead; } info->SetSize( totalSize ); AnyObject *resp = new AnyObject(); resp->Set( info.release() ); return QueueTask( new XRootDStatus(), resp, handler ); } //------------------------------------------------------------------------ // VectorWrite //------------------------------------------------------------------------ XRootDStatus LocalFileHandler::VectorWrite( const ChunkList &chunks, ResponseHandler *handler, uint16_t timeout ) { for( auto itr = chunks.begin(); itr != chunks.end(); ++itr ) { auto &chunk = *itr; ssize_t bytesWritten = pwrite( fd, chunk.buffer, chunk.length, chunk.offset ); if( bytesWritten < 0 ) { Log *log = DefaultEnv::GetLog(); log->Error( FileMsg, "VectorWrite: failed, file descriptor: %i, %s", fd, XrdSysE2T( errno ) ); XRootDStatus *error = new XRootDStatus( stError, errLocalError, errno ); return QueueTask( error, 0, handler ); } } return QueueTask( new XRootDStatus(), 0, handler ); } //------------------------------------------------------------------------ // WriteV //------------------------------------------------------------------------ XRootDStatus LocalFileHandler::WriteV( uint64_t offset, ChunkList *chunks, ResponseHandler *handler, uint16_t timeout ) { size_t iovcnt = chunks->size(); iovec iovcp[iovcnt]; ssize_t size = 0; for( size_t i = 0; i < iovcnt; ++i ) { iovcp[i].iov_base = (*chunks)[i].buffer; iovcp[i].iov_len = (*chunks)[i].length; size += (*chunks)[i].length; } iovec *iovptr = iovcp; ssize_t bytesWritten = 0; while( bytesWritten < size ) { #ifdef __APPLE__ ssize_t ret = lseek( fd, offset, SEEK_SET ); if( ret >= 0 ) ret = writev( fd, iovptr, iovcnt ); #else ssize_t ret = pwritev( fd, iovptr, iovcnt, offset ); #endif if( ret < 0 ) { Log *log = DefaultEnv::GetLog(); log->Error( FileMsg, "WriteV: failed %s", XrdSysE2T( errno ) ); XRootDStatus *error = new XRootDStatus( stError, errLocalError, errno ); return QueueTask( error, 0, handler ); } bytesWritten += ret; while( ret ) { if( size_t( ret ) > iovptr[0].iov_len ) { ret -= iovptr[0].iov_len; --iovcnt; ++iovptr; } else { iovptr[0].iov_len -= ret; iovptr[0].iov_base = reinterpret_cast( iovptr[0].iov_base ) + ret; ret = 0; } } } return QueueTask( new XRootDStatus(), 0, handler ); } //------------------------------------------------------------------------ // Fcntl //------------------------------------------------------------------------ XRootDStatus LocalFileHandler::Fcntl( const Buffer &arg, ResponseHandler *handler, uint16_t timeout ) { return XRootDStatus( stError, errNotSupported ); } //------------------------------------------------------------------------ // Visa //------------------------------------------------------------------------ XRootDStatus LocalFileHandler::Visa( ResponseHandler *handler, uint16_t timeout ) { return XRootDStatus( stError, errNotSupported ); } //------------------------------------------------------------------------ // Set extended attributes - async //------------------------------------------------------------------------ XRootDStatus LocalFileHandler::SetXAttr( const std::vector &attrs, ResponseHandler *handler, uint16_t timeout ) { XrdSysXAttr *xattr = XrdSysFAttr::Xat; std::vector response; auto itr = attrs.begin(); for( ; itr != attrs.end(); ++itr ) { std::string name = std::get( *itr ); std::string value = std::get( *itr ); int err = xattr->Set( name.c_str(), value.c_str(), value.size(), 0, fd ); XRootDStatus status = err < 0 ? XRootDStatus( stError, errLocalError, -err ) : XRootDStatus(); response.push_back( XAttrStatus( name, status ) ); } AnyObject *resp = new AnyObject(); resp->Set( new std::vector( std::move( response ) ) ); return QueueTask( new XRootDStatus(), resp, handler ); } //------------------------------------------------------------------------ // Get extended attributes - async //------------------------------------------------------------------------ XRootDStatus LocalFileHandler::GetXAttr( const std::vector &attrs, ResponseHandler *handler, uint16_t timeout ) { XrdSysXAttr *xattr = XrdSysFAttr::Xat; std::vector response; auto itr = attrs.begin(); for( ; itr != attrs.end(); ++itr ) { std::string name = *itr; std::unique_ptr buffer; int size = xattr->Get( name.c_str(), 0, 0, 0, fd ); if( size < 0 ) { XRootDStatus status( stError, errLocalError, -size ); response.push_back( XAttr( *itr, "", status ) ); continue; } buffer.reset( new char[size] ); int ret = xattr->Get( name.c_str(), buffer.get(), size, 0, fd ); XRootDStatus status; std::string value; if( ret >= 0 ) value.append( buffer.get(), ret ); else if( ret < 0 ) status = XRootDStatus( stError, errLocalError, -ret ); response.push_back( XAttr( *itr, value, status ) ); } AnyObject *resp = new AnyObject(); resp->Set( new std::vector( std::move( response ) ) ); return QueueTask( new XRootDStatus(), resp, handler ); } //------------------------------------------------------------------------ // Delete extended attributes - async //------------------------------------------------------------------------ XRootDStatus LocalFileHandler::DelXAttr( const std::vector &attrs, ResponseHandler *handler, uint16_t timeout ) { XrdSysXAttr *xattr = XrdSysFAttr::Xat; std::vector response; auto itr = attrs.begin(); for( ; itr != attrs.end(); ++itr ) { std::string name = *itr; int err = xattr->Del( name.c_str(), 0, fd ); XRootDStatus status = err < 0 ? XRootDStatus( stError, errLocalError, -err ) : XRootDStatus(); response.push_back( XAttrStatus( name, status ) ); } AnyObject *resp = new AnyObject(); resp->Set( new std::vector( std::move( response ) ) ); return QueueTask( new XRootDStatus(), resp, handler ); } //------------------------------------------------------------------------ // List extended attributes - async //------------------------------------------------------------------------ XRootDStatus LocalFileHandler::ListXAttr( ResponseHandler *handler, uint16_t timeout ) { XrdSysXAttr *xattr = XrdSysFAttr::Xat; std::vector response; XrdSysXAttr::AList *alist = 0; int err = xattr->List( &alist, 0, fd, 1 ); if( err < 0 ) { XRootDStatus *status = new XRootDStatus( stError, XProtocol::mapError( -err ) ); return QueueTask( status, 0, handler ); } XrdSysXAttr::AList *ptr = alist; while( ptr ) { std::string name( ptr->Name, ptr->Nlen ); int vlen = ptr->Vlen; ptr = ptr->Next; std::unique_ptr buffer( new char[vlen] ); int ret = xattr->Get( name.c_str(), buffer.get(), vlen, 0, fd ); std::string value = ret >= 0 ? std::string( buffer.get(), ret ) : std::string(); XRootDStatus status = ret >= 0 ? XRootDStatus() : XRootDStatus( stError, errLocalError, -ret ); response.push_back( XAttr( name, value, status ) ); } xattr->Free( alist ); AnyObject *resp = new AnyObject(); resp->Set( new std::vector( std::move( response ) ) ); return QueueTask( new XRootDStatus(), resp, handler ); } //------------------------------------------------------------------------ // QueueTask - queues error/success tasks for all operations. // Must always return stOK. // Is always creating the same HostList containing only localhost. //------------------------------------------------------------------------ XRootDStatus LocalFileHandler::QueueTask( XRootDStatus *st, AnyObject *resp, ResponseHandler *handler ) { // if it is simply the sync handler we can release the semaphore // and return there is no need to execute this in the thread-pool SyncResponseHandler *syncHandler = dynamic_cast( handler ); if( syncHandler || DefaultEnv::GetPostMaster() == nullptr ) { syncHandler->HandleResponse( st, resp ); return XRootDStatus(); } HostList *hosts = pHostList.empty() ? 0 : new HostList( pHostList ); LocalFileTask *task = new LocalFileTask( st, resp, hosts, handler ); DefaultEnv::GetPostMaster()->GetJobManager()->QueueJob( task ); return XRootDStatus(); } //------------------------------------------------------------------------ // MkdirPath - creates the folders specified in file_path // called if kXR_mkdir flag is set //------------------------------------------------------------------------ XRootDStatus LocalFileHandler::MkdirPath( const std::string &path ) { // first find the most up-front component that exists size_t pos = path.rfind( '/' ); while( pos != std::string::npos && pos != 0 ) { std::string tmp = path.substr( 0, pos ); struct stat st; int rc = lstat( tmp.c_str(), &st ); if( rc == 0 ) break; if( errno != ENOENT ) return XRootDStatus( stError, errLocalError, errno ); pos = path.rfind( '/', pos - 1 ); } pos = path.find( '/', pos + 1 ); while( pos != std::string::npos && pos != 0 ) { std::string tmp = path.substr( 0, pos ); if( mkdir( tmp.c_str(), 0755 ) ) { if( errno != EEXIST ) return XRootDStatus( stError, errLocalError, errno ); } pos = path.find( '/', pos + 1 ); } return XRootDStatus(); } XRootDStatus LocalFileHandler::OpenImpl( const std::string &url, uint16_t flags, uint16_t mode, AnyObject *&resp) { Log *log = DefaultEnv::GetLog(); // safe the file URL for the HostList for later pUrl = url; URL fileUrl( url ); if( !fileUrl.IsValid() ) return XRootDStatus( stError, errInvalidArgs ); if( fileUrl.GetHostName() != "localhost" ) return XRootDStatus( stError, errNotSupported ); std::string path = fileUrl.GetPath(); //--------------------------------------------------------------------- // Prepare Flags //--------------------------------------------------------------------- uint16_t openflags = 0; if( flags & kXR_new ) openflags |= O_CREAT | O_EXCL; if( flags & kXR_open_wrto ) openflags |= O_WRONLY; else if( flags & kXR_open_updt ) openflags |= O_RDWR; else openflags |= O_RDONLY; if( flags & kXR_delete ) openflags |= O_CREAT | O_TRUNC; if( flags & kXR_mkdir ) { XRootDStatus st = MkdirPath( path ); if( !st.IsOK() ) { log->Error( FileMsg, "Open MkdirPath failed %s: %s", path.c_str(), XrdSysE2T( st.errNo ) ); return st; } } //--------------------------------------------------------------------- // Open File //--------------------------------------------------------------------- if( mode == Access::Mode::None) mode = 0644; fd = XrdSysFD_Open( path.c_str(), openflags, mode ); if( fd == -1 ) { log->Error( FileMsg, "Open: open failed: %s: %s", path.c_str(), XrdSysE2T( errno ) ); return XRootDStatus( stError, errLocalError, XProtocol::mapError( errno ) ); } //--------------------------------------------------------------------- // Stat File and cache statInfo in openInfo //--------------------------------------------------------------------- struct stat ssp; if( fstat( fd, &ssp ) == -1 ) { log->Error( FileMsg, "Open: stat failed." ); return XRootDStatus( stError, errLocalError, XProtocol::mapError( errno ) ); } std::ostringstream data; data << ssp.st_dev << " " << ssp.st_size << " " << ssp.st_mode << " " << ssp.st_mtime; StatInfo *statInfo = new StatInfo(); if( !statInfo->ParseServerResponse( data.str().c_str() ) ) { log->Error( FileMsg, "Open: ParseServerResponse failed." ); delete statInfo; return XRootDStatus( stError, errLocalError, kXR_FSError ); } // add the URL to hosts list pHostList.push_back( HostInfo( pUrl, false ) ); //All went well uint32_t ufd = fd; OpenInfo *openInfo = new OpenInfo( (uint8_t*)&ufd, 1, statInfo ); resp = new AnyObject(); resp->Set( openInfo ); return XRootDStatus(); } //------------------------------------------------------------------------ // Parses kXR_fattr request and calls respective XAttr operation //------------------------------------------------------------------------ XRootDStatus LocalFileHandler::XAttrImpl( kXR_char code, kXR_char numattr, size_t bodylen, char *body, ResponseHandler *handler ) { // shift body by 1 to omit the empty path if( bodylen > 0 ) { ++body; --bodylen; } switch( code ) { case kXR_fattrGet: case kXR_fattrDel: { std::vector attrs; // parse namevec for( kXR_char i = 0; i < numattr; ++i ) { if( bodylen < sizeof( kXR_unt16 ) ) return XRootDStatus( stError, errDataError ); // shift by RC size body += sizeof( kXR_unt16 ); bodylen -= sizeof( kXR_unt16 ); // get the size of attribute name size_t len = strlen( body ); if( len > bodylen ) return XRootDStatus( stError, errDataError ); attrs.push_back( std::string( body, len ) ); body += len + 1; // +1 for the null terminating the string bodylen -= len + 1; // +1 for the null terminating the string } if( code == kXR_fattrGet ) return GetXAttr( attrs, handler ); return DelXAttr( attrs, handler ); } case kXR_fattrSet: { std::vector attrs; // parse namevec for( kXR_char i = 0; i < numattr; ++i ) { if( bodylen < sizeof( kXR_unt16 ) ) return XRootDStatus( stError, errDataError ); // shift by RC size body += sizeof( kXR_unt16 ); bodylen -= sizeof( kXR_unt16 ); // get the size of attribute name char *name = 0; body = ClientFattrRequest::NVecRead( body, name ); attrs.push_back( std::make_tuple( std::string( name ), std::string() ) ); bodylen -= strlen( name ) + 1; // +1 for the null terminating the string free( name ); } // parse valuevec for( kXR_char i = 0; i < numattr; ++i ) { // get value length if( bodylen < sizeof( kXR_int32 ) ) return XRootDStatus( stError, errDataError ); kXR_int32 len = 0; body = ClientFattrRequest::VVecRead( body, len ); bodylen -= sizeof( kXR_int32 ); // get value if( size_t( len ) > bodylen ) return XRootDStatus( stError, errDataError ); char *value = 0; body = ClientFattrRequest::VVecRead( body, len, value ); bodylen -= len; std::get( attrs[i] ) = value; free( value ); } return SetXAttr( attrs, handler ); } case kXR_fattrList: { return ListXAttr( handler ); } default: return XRootDStatus( stError, errInvalidArgs ); } return XRootDStatus(); } XRootDStatus LocalFileHandler::ExecRequest( const URL &url, Message *msg, ResponseHandler *handler, MessageSendParams &sendParams ) { ClientRequest *req = reinterpret_cast( msg->GetBuffer() ); switch( req->header.requestid ) { case kXR_open: { XRootDStatus st = Open( url.GetURL(), req->open.options, req->open.mode, handler, sendParams.timeout ); delete msg; // in case of other operations msg is owned by the handler return st; } case kXR_close: { return Close( handler, sendParams.timeout ); } case kXR_stat: { return Stat( handler, sendParams.timeout ); } case kXR_read: { if( msg->GetVirtReqID() == kXR_virtReadv ) { auto &chunkList = *sendParams.chunkList; struct iovec iov[chunkList.size()]; for( size_t i = 0; i < chunkList.size() ; ++i ) { iov[i].iov_base = chunkList[i].buffer; iov[i].iov_len = chunkList[i].length; } return ReadV( chunkList.front().offset, iov, chunkList.size(), handler, sendParams.timeout ); } return Read( req->read.offset, req->read.rlen, sendParams.chunkList->front().buffer, handler, sendParams.timeout ); } case kXR_write: { ChunkList *chunks = sendParams.chunkList; if( chunks->size() == 1 ) { // it's an ordinary write return Write( req->write.offset, req->write.dlen, chunks->front().buffer, handler, sendParams.timeout ); } // it's WriteV call return WriteV( req->write.offset, sendParams.chunkList, handler, sendParams.timeout ); } case kXR_sync: { return Sync( handler, sendParams.timeout ); } case kXR_truncate: { return Truncate( req->truncate.offset, handler, sendParams.timeout ); } case kXR_writev: { return VectorWrite( *sendParams.chunkList, handler, sendParams.timeout ); } case kXR_readv: { return VectorRead( *sendParams.chunkList, 0, handler, sendParams.timeout ); } case kXR_fattr: { return XAttrImpl( req->fattr.subcode, req->fattr.numattr, req->fattr.dlen, msg->GetBuffer( sizeof(ClientRequestHdr ) ), handler ); } default: { return XRootDStatus( stError, errNotSupported ); } } } } xrootd-5.6.9/src/XrdCl/XrdClLocalFileHandler.hh000066400000000000000000000423141457266313600213040ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2017 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH // Author: Paul-Niklas Kramp //------------------------------------------------------------------------------ // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . //------------------------------------------------------------------------------ #ifndef __XRD_CL_LOCAL_FILE_HANDLER_HH__ #define __XRD_CL_LOCAL_FILE_HANDLER_HH__ #include "XrdCl/XrdClJobManager.hh" #include "XrdCl/XrdClLocalFileTask.hh" #include "XrdCl/XrdClDefaultEnv.hh" #include "XrdCl/XrdClLog.hh" #include namespace XrdCl { class Message; struct MessageSendParams; class LocalFileHandler { public: LocalFileHandler(); ~LocalFileHandler(); //------------------------------------------------------------------------ //! Open the file pointed to by the given URL //! //! @param url url of the file to be opened //! @param flags OpenFlags::Flags //! @param mode Access::Mode for new files, 0 otherwise //! @param handler handler to be notified about the status of the operation //! @param timeout timeout value, if 0 the environment default will be //! used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus Open( const std::string &url, uint16_t flags, uint16_t mode, ResponseHandler *handler, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! Handle local redirect to given URL triggered by the given request //------------------------------------------------------------------------ XRootDStatus Open( const URL *url, const Message *req, AnyObject *&resp ); //------------------------------------------------------------------------ //! Close the file object //! //! @param handler handler to be notified about the status of the operation //! @param timeout timeout value, if 0 the environment default will be //! used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus Close( ResponseHandler *handler, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! Obtain status information for this file - async //! //! @param handler handler to be notified when the response arrives, //! the response parameter will hold a StatInfo object //! if the procedure is successful //! @param timeout timeout value, if 0 the environment default will //! be used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus Stat( ResponseHandler *handler, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! Read a data chunk at a given offset - sync //! //! @param offset offset from the beginning of the file //! @param size number of bytes to be read //! @param buffer a pointer to a buffer big enough to hold the data //! or 0 if the buffer should be allocated by the system //! @param handler handler to be notified when the response arrives, //! the response parameter will hold a buffer object if //! the procedure was successful, if a preallocated //! buffer was specified then the buffer object will //! "wrap" this buffer //! @param timeout timeout value, if 0 the environment default will be //! used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus Read( uint64_t offset, uint32_t size, void *buffer, ResponseHandler *handler, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! Read data into scattered buffers in one operation - async //! //! @param offset offset from the beginning of the file //! @param iov list of the buffers to be written //! @param iovcnt number of buffers //! @param handler handler to be notified when the response arrives //! @param timeout timeout value, if 0 then the environment default //! will be used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus ReadV( uint64_t offset, struct iovec *iov, int iovcnt, ResponseHandler *handler, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! Write a data chunk at a given offset - async //! //! @param offset offset from the beginning of the file //! @param size number of bytes to be written //! @param buffer a pointer to the buffer holding the data to be written //! @param handler handler to be notified when the response arrives //! @param timeout timeout value, if 0 the environment default will be //! used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus Write( uint64_t offset, uint32_t size, const void *buffer, ResponseHandler *handler, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! Commit all pending disk writes - async //! //! @param handler handler to be notified when the response arrives //! @param timeout timeout value, if 0 the environment default will be //! used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus Sync( ResponseHandler *handler, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! Truncate the file to a particular size - async //! //! @param size desired size of the file //! @param handler handler to be notified when the response arrives //! @param timeout timeout value, if 0 the environment default will be //! used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus Truncate( uint64_t size, ResponseHandler *handler, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! Read scattered data chunks in one operation - async //! //! @param chunks list of the chunks to be read //! @param buffer a pointer to a buffer big enough to hold the data //! @param handler handler to be notified when the response arrives //! @param timeout timeout value, if 0 then the environment default //! will be used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus VectorRead( const ChunkList &chunks, void *buffer, ResponseHandler *handler, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! Write scattered data chunks in one operation - async //! //! @param chunks list of the chunks to be read //! @param handler handler to be notified when the response arrives //! @param timeout timeout value, if 0 then the environment default //! will be used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus VectorWrite( const ChunkList &chunks, ResponseHandler *handler, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! Write scattered buffers in one operation - async //! //! @param offset offset from the beginning of the file //! @param chunks list of the chunks to be read //! @param handler handler to be notified when the response arrives //! @param timeout timeout value, if 0 then the environment default //! will be used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus WriteV( uint64_t offset, ChunkList *chunks, ResponseHandler *handler, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! Queues a task to the jobmanager //! //! @param st the status of the file operation //! @param obj the object holding data like open-, chunk- or vreadinfo //! @param handler handler to be notified when the response arrives //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus QueueTask( XRootDStatus *st, AnyObject *obj, ResponseHandler *handler ); //------------------------------------------------------------------------ //! Performs a custom operation on an open file - async //! //! @param arg query argument //! @param handler handler to be notified when the response arrives, //! the response parameter will hold a Buffer object //! if the procedure is successful //! @param timeout timeout value, if 0 the environment default will //! be used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus Fcntl( const Buffer &arg, ResponseHandler *handler, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! Get access token to a file - async //! //! @param handler handler to be notified when the response arrives, //! the response parameter will hold a Buffer object //! if the procedure is successful //! @param timeout timeout value, if 0 the environment default will //! be used //! @return status of the operation //------------------------------------------------------------------------ XRootDStatus Visa( ResponseHandler *handler, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! Set extended attributes - async //! //! @param attrs : list of extended attributes to set //! @param handler : handler to be notified when the response arrives, //! the response parameter will hold a std::vector of //! XAttrStatus objects //! @param timeout : timeout value, if 0 the environment default will //! be used //! //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus SetXAttr( const std::vector &attrs, ResponseHandler *handler, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! Get extended attributes - async //! //! @param attrs : list of extended attributes to get //! @param handler : handler to be notified when the response arrives, //! the response parameter will hold a std::vector of //! XAttr objects //! @param timeout : timeout value, if 0 the environment default will //! be used //! //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus GetXAttr( const std::vector &attrs, ResponseHandler *handler, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! Delete extended attributes - async //! //! @param attrs : list of extended attributes to set //! @param handler : handler to be notified when the response arrives, //! the response parameter will hold a std::vector of //! XAttrStatus objects //! @param timeout : timeout value, if 0 the environment default will //! be used //! //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus DelXAttr( const std::vector &attrs, ResponseHandler *handler, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! List extended attributes - async //! //! @param handler : handler to be notified when the response arrives, //! the response parameter will hold a std::vector of //! XAttr objects //! @param timeout : timeout value, if 0 the environment default will //! be used //! //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus ListXAttr( ResponseHandler *handler, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! creates the directories specified in path //! //! @param path specifies which directories are to be created //! @return status of the mkdir system call //------------------------------------------------------------------------ static XRootDStatus MkdirPath( const std::string &path ); void SetHostList( const HostList &hostList ) { pHostList = hostList; } const HostList& GetHostList() { return pHostList; } //------------------------------------------------------------------------ //! Translate an XRootD request into LocalFileHandler call //------------------------------------------------------------------------ XRootDStatus ExecRequest( const URL &url, Message *msg, ResponseHandler *handler, MessageSendParams &sendParams ); private: XRootDStatus OpenImpl( const std::string &url, uint16_t flags, uint16_t mode, AnyObject *&resp ); //------------------------------------------------------------------------ //! Parses kXR_fattr request and calls respective XAttr operation //------------------------------------------------------------------------ XRootDStatus XAttrImpl( kXR_char code, kXR_char numattr, size_t bodylen, char *body, ResponseHandler *handler ); //--------------------------------------------------------------------- // Internal filedescriptor, which is used by all operations after open //--------------------------------------------------------------------- int fd; //--------------------------------------------------------------------- // The file URL //--------------------------------------------------------------------- std::string pUrl; //--------------------------------------------------------------------- // The host list returned in the user callback //--------------------------------------------------------------------- HostList pHostList; }; } #endif xrootd-5.6.9/src/XrdCl/XrdClLocalFileTask.cc000066400000000000000000000031041457266313600206110ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2017 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH // Author: Paul-Niklas Kramp //------------------------------------------------------------------------------ // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . //------------------------------------------------------------------------------ #include "XrdCl/XrdClLocalFileTask.hh" namespace XrdCl { LocalFileTask::LocalFileTask( XRootDStatus *st, AnyObject *obj, HostList *hosts, ResponseHandler *responsehandler ) { this->st = st; this->obj = obj; this->hosts = hosts; this->responsehandler = responsehandler; } LocalFileTask::~LocalFileTask(){} void LocalFileTask::Run( void *arg ) { if( responsehandler ) responsehandler->HandleResponseWithHosts( st, obj, hosts ); else{ delete st; delete obj; delete hosts; } delete this; } } xrootd-5.6.9/src/XrdCl/XrdClLocalFileTask.hh000066400000000000000000000031101457266313600206200ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2017 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH // Author: Paul-Niklas Kramp //------------------------------------------------------------------------------ // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . //------------------------------------------------------------------------------ #ifndef __XRD_CL_LOCAL_FILE_TASK_HH__ #define __XRD_CL_LOCAL_FILE_TASK_HH__ #include "XrdCl/XrdClStatus.hh" #include "XrdCl/XrdClAnyObject.hh" #include "XrdCl/XrdClJobManager.hh" #include "XrdCl/XrdClXRootDResponses.hh" namespace XrdCl{ class LocalFileTask : public Job{ public: LocalFileTask( XRootDStatus *st, AnyObject *obj, HostList *hosts, ResponseHandler *responsehandler ); ~LocalFileTask(); virtual void Run( void *arg ); private: XRootDStatus *st; AnyObject *obj; HostList *hosts; ResponseHandler *responsehandler; }; } #endif xrootd-5.6.9/src/XrdCl/XrdClLog.cc000066400000000000000000000230571457266313600166660ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2012 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . //------------------------------------------------------------------------------ #include #include #include #include #include #include #include #include #include #include #include "XrdOuc/XrdOucTokenizer.hh" #include "XrdSys/XrdSysE2T.hh" #include "XrdCl/XrdClOptimizers.hh" #include "XrdCl/XrdClLog.hh" namespace XrdCl { //---------------------------------------------------------------------------- // Open a file //---------------------------------------------------------------------------- bool LogOutFile::Open( const std::string &filename ) { int fd = open( filename.c_str(), O_WRONLY | O_APPEND | O_CREAT, S_IRUSR | S_IWUSR ); if( fd < 0 ) { std::cerr << "Unable to open " << filename << " " << XrdSysE2T( errno ); std::cerr << std::endl; return false; } pFileDes = fd; return true; } //---------------------------------------------------------------------------- // Destructor //---------------------------------------------------------------------------- void LogOutFile::Close() { if( pFileDes != -1 ) { close( pFileDes ); pFileDes = -1; } } //---------------------------------------------------------------------------- // Message handler //---------------------------------------------------------------------------- void LogOutFile::Write( const std::string &message ) { if( unlikely( pFileDes == -1 ) ) { std::cerr << "Log file not opened" << std::endl; return; } int ret = write( pFileDes, message.c_str(), message.length() ); if( ret < 0 ) { std::cerr << "Unable to write to the log file: " << XrdSysE2T( errno ); std::cerr << std::endl; return; } } //---------------------------------------------------------------------------- // Message handler //---------------------------------------------------------------------------- void LogOutCerr::Write( const std::string &message ) { pMutex.Lock(); std::cerr << message; pMutex.UnLock(); } //---------------------------------------------------------------------------- // Print an error message //---------------------------------------------------------------------------- void Log::Say( LogLevel level, uint64_t topic, const char *format, va_list list ) { //-------------------------------------------------------------------------- // Build the user message //-------------------------------------------------------------------------- int size = 1024; int ret = 0; char *buffer = 0; while(1) { va_list cp; va_copy( cp, list ); buffer = new char[size]; ret = vsnprintf( buffer, size, format, cp ); va_end( cp ); if( ret < 0 ) { snprintf( buffer, size, "Error while processing a log message \"%s\" \n", format); pOutput->Write(buffer); delete [] buffer; return; } else if( ret < size ) break; size *= 2; delete [] buffer; } //-------------------------------------------------------------------------- // Add time and error level //-------------------------------------------------------------------------- char now[48]; char ts[32]; char tz[8]; tm tsNow; timeval ttNow; gettimeofday( &ttNow, 0 ); localtime_r( &ttNow.tv_sec, &tsNow ); strftime( ts, 32, "%Y-%m-%d %H:%M:%S", &tsNow ); strftime( tz, 8, "%z", &tsNow ); snprintf( now, 48, "%s.%06ld %s", ts, (long int)ttNow.tv_usec, tz ); XrdOucTokenizer tok( buffer ); char *line = 0; std::ostringstream out; while( (line = tok.GetLine()) ) { out << "[" << now << "][" << LogLevelToString( level ) << "]"; out << "[" << TopicToString( topic ) << "]"; if(pPid) out << "[" << std::setw(5) << pPid << "]"; out << " " << line << std::endl; } pOutput->Write( out.str() ); delete [] buffer; } //---------------------------------------------------------------------------- // Map a topic number to a string //---------------------------------------------------------------------------- void Log::SetTopicName( uint64_t topic, std::string name ) { uint32_t len = name.length(); if( len > pTopicMaxLength ) { pTopicMaxLength = len; TopicMap::iterator it; for( it = pTopicMap.begin(); it != pTopicMap.end(); ++it ) it->second.append( len-it->second.length(), ' ' ); } else name.append( pTopicMaxLength-len, ' ' ); pTopicMap[topic] = name; } //---------------------------------------------------------------------------- // Convert log level to string //---------------------------------------------------------------------------- std::string Log::LogLevelToString( LogLevel level ) { switch( level ) { case ErrorMsg: return "Error "; case WarningMsg: return "Warning"; case InfoMsg: return "Info "; case DebugMsg: return "Debug "; case DumpMsg: return "Dump "; default: return "Unknown Level"; } } //---------------------------------------------------------------------------- // Convert a string to LogLevel //---------------------------------------------------------------------------- bool Log::StringToLogLevel( const std::string &strLevel, LogLevel &level ) { if( strLevel == "Error" ) level = ErrorMsg; else if( strLevel == "Warning" ) level = WarningMsg; else if( strLevel == "Info" ) level = InfoMsg; else if( strLevel == "Debug" ) level = DebugMsg; else if( strLevel == "Dump" ) level = DumpMsg; else return false; return true; } //---------------------------------------------------------------------------- // Convert a topic number to a string //---------------------------------------------------------------------------- std::string Log::TopicToString( uint64_t topic ) { TopicMap::iterator it = pTopicMap.find( topic ); if( it != pTopicMap.end() ) return it->second; std::ostringstream o; o << "0x" << std::setw(pTopicMaxLength-2) << std::setfill( '0' ); o << std::setbase(16) << topic; return o.str(); } //---------------------------------------------------------------------------- // Report an error //---------------------------------------------------------------------------- void Log::Error( uint64_t topic, const char *format, ... ) { if( unlikely( GetLevel() < ErrorMsg ) ) return; if( unlikely( (topic & pMask[ErrorMsg]) == 0 ) ) return; va_list argList; va_start( argList, format ); Say( ErrorMsg, topic, format, argList ); va_end( argList ); } //---------------------------------------------------------------------------- // Report a warning //---------------------------------------------------------------------------- void Log::Warning( uint64_t topic, const char *format, ... ) { if( unlikely( GetLevel() < WarningMsg ) ) return; if( unlikely( (topic & pMask[WarningMsg]) == 0 ) ) return; va_list argList; va_start( argList, format ); Say( WarningMsg, topic, format, argList ); va_end( argList ); } //---------------------------------------------------------------------------- // Print an info //---------------------------------------------------------------------------- void Log::Info( uint64_t topic, const char *format, ... ) { if( likely( GetLevel() < InfoMsg ) ) return; if( unlikely( (topic & pMask[InfoMsg]) == 0 ) ) return; va_list argList; va_start( argList, format ); Say( InfoMsg, topic, format, argList ); va_end( argList ); } //---------------------------------------------------------------------------- // Print a debug message //---------------------------------------------------------------------------- void Log::Debug( uint64_t topic, const char *format, ... ) { if( likely( GetLevel() < DebugMsg ) ) return; if( unlikely( (topic & pMask[DebugMsg]) == 0 ) ) return; va_list argList; va_start( argList, format ); Say( DebugMsg, topic, format, argList ); va_end( argList ); } //---------------------------------------------------------------------------- // Print a dump message //---------------------------------------------------------------------------- void Log::Dump( uint64_t topic, const char *format, ... ) { if( likely( GetLevel() < DumpMsg ) ) return; if( unlikely( (topic & pMask[DumpMsg]) == 0 ) ) return; va_list argList; va_start( argList, format ); Say( DumpMsg, topic, format, argList ); va_end( argList ); } } xrootd-5.6.9/src/XrdCl/XrdClLog.hh000066400000000000000000000243641457266313600167020ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2014 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #ifndef __XRD_CL_LOG_HH__ #define __XRD_CL_LOG_HH__ #include #include #include #include #include "XrdSys/XrdSysPthread.hh" //------------------------------------------------------------------------------ // C++11 atomics are used to avoid illegal behavior when setting/getting the // log level. To minimize costs across all platforms, we use // std::memory_order_relaxed; this means threads may reorder SetLogLevel writes // and the visibility is relatively undefined. However, we know the stores are // at least atomic. //------------------------------------------------------------------------------ #include namespace XrdCl { //---------------------------------------------------------------------------- //! Interface for logger outputs //---------------------------------------------------------------------------- class LogOut { public: virtual ~LogOut() {} //------------------------------------------------------------------------ //! Write a message to the destination //! //! @param message message to be written //------------------------------------------------------------------------ virtual void Write( const std::string &message ) = 0; }; //---------------------------------------------------------------------------- //! Write log messages to a file //---------------------------------------------------------------------------- class LogOutFile: public LogOut { public: LogOutFile(): pFileDes(-1) {}; virtual ~LogOutFile() { Close(); }; //------------------------------------------------------------------------ //! Open the log file //------------------------------------------------------------------------ bool Open( const std::string &fileName ); //------------------------------------------------------------------------ //! Close the log file //------------------------------------------------------------------------ void Close(); virtual void Write( const std::string &message ); private: int pFileDes; }; //---------------------------------------------------------------------------- //! Write log messages to stderr //---------------------------------------------------------------------------- class LogOutCerr: public LogOut { public: virtual void Write( const std::string &message ); virtual ~LogOutCerr() {} private: XrdSysMutex pMutex; }; //---------------------------------------------------------------------------- //! Handle diagnostics //---------------------------------------------------------------------------- class Log { public: //------------------------------------------------------------------------ //! Log levels //------------------------------------------------------------------------ enum LogLevel { NoMsg = 0, //!< report nothing ErrorMsg = 1, //!< report errors WarningMsg = 2, //!< report warnings InfoMsg = 3, //!< print info DebugMsg = 4, //!< print debug info DumpMsg = 5 //!< print details of the request and responses }; //------------------------------------------------------------------------ //! Constructor //------------------------------------------------------------------------ Log(): pLevel( NoMsg ), pTopicMaxLength( 18 ), pPid(0) { pOutput = new LogOutCerr(); int maxMask = (int)DumpMsg+1; for( int i = 0; i < maxMask; ++i ) pMask[i] = 0xffffffffffffffffULL; } //------------------------------------------------------------------------ // Destructor //------------------------------------------------------------------------ ~Log() { delete pOutput; } //------------------------------------------------------------------------ //! Report an error //------------------------------------------------------------------------ void Error( uint64_t topic, const char *format, ... ); //------------------------------------------------------------------------ //! Report a warning //------------------------------------------------------------------------ void Warning( uint64_t topic, const char *format, ... ); //------------------------------------------------------------------------ //! Print an info //------------------------------------------------------------------------ void Info( uint64_t topic, const char *format, ... ); //------------------------------------------------------------------------ //! Print a debug message //------------------------------------------------------------------------ void Debug( uint64_t topic, const char *format, ... ); //------------------------------------------------------------------------ //! Print a dump message //------------------------------------------------------------------------ void Dump( uint64_t topic, const char *format, ... ); //------------------------------------------------------------------------ //! Always print the message //! //! @param level log level //! @param topic topic of the message //! @param format format string - the same as in printf //! @param list list of arguments //------------------------------------------------------------------------ void Say( LogLevel level, uint64_t topic, const char *format, va_list list ); //------------------------------------------------------------------------ //! Set the level of the messages that should be sent to the destination //------------------------------------------------------------------------ void SetLevel( LogLevel level ) { #if __cplusplus >= 201103L pLevel.store(level, std::memory_order_relaxed); #else pLevel = level; #endif } //------------------------------------------------------------------------ //! Set the level of the messages that should be sent to the destination //------------------------------------------------------------------------ void SetLevel( const std::string &level ) { LogLevel lvl; if( StringToLogLevel( level, lvl ) ) SetLevel( lvl ); } //------------------------------------------------------------------------ //! Set the output that should be used. //------------------------------------------------------------------------ void SetOutput( LogOut *output ) { delete pOutput; pOutput = output; } //------------------------------------------------------------------------ //! Sets the mask for the topics of messages that should be printed //------------------------------------------------------------------------ void SetMask( LogLevel level, uint64_t mask ) { pMask[level] = mask; } //------------------------------------------------------------------------ //! Sets the mask for the topics of messages that should be printed //------------------------------------------------------------------------ void SetMask( const std::string &level, uint64_t mask ) { LogLevel lvl; if( StringToLogLevel( level, lvl ) ) pMask[lvl] = mask; } //------------------------------------------------------------------------ //! Map a topic number to a string //------------------------------------------------------------------------ void SetTopicName( uint64_t topic, std::string name ); //------------------------------------------------------------------------ //! Register new topic //------------------------------------------------------------------------ inline uint64_t RegisterTopic( const std::string &topic ) { uint64_t tpcnb = pTopicMap.rbegin()->first << 1; SetTopicName( tpcnb, topic ); return tpcnb; } //------------------------------------------------------------------------ //! Get the log level //------------------------------------------------------------------------ LogLevel GetLevel() const { LogLevel lvl = pLevel.load(std::memory_order_relaxed); return lvl; } //------------------------------------------------------------------------ //! Set pid //------------------------------------------------------------------------ void SetPid(pid_t pid) { pPid = pid; } private: typedef std::map TopicMap; std::string LogLevelToString( LogLevel level ); bool StringToLogLevel( const std::string &strLevel, LogLevel &level ); std::string TopicToString( uint64_t topic ); std::atomic pLevel; uint64_t pMask[DumpMsg+1]; LogOut *pOutput; TopicMap pTopicMap; uint32_t pTopicMaxLength; pid_t pPid; }; } #endif // __XRD_CL_LOG_HH__ xrootd-5.6.9/src/XrdCl/XrdClMessage.hh000066400000000000000000000124201457266313600175330ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2012 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . //------------------------------------------------------------------------------ #ifndef __XRD_CL_MESSAGE_HH__ #define __XRD_CL_MESSAGE_HH__ #include "XrdCl/XrdClBuffer.hh" namespace XrdCl { //---------------------------------------------------------------------------- //! The message representation used throughout the system //---------------------------------------------------------------------------- class Message: public Buffer { public: //------------------------------------------------------------------------ //! Constructor //------------------------------------------------------------------------ Message( uint32_t size = 0 ): Buffer( size ), pIsMarshalled( false ), pSessionId(0), pVirtReqID( 0 ) { if( size ) Zero(); } //------------------------------------------------------------------------ //! Move Constructor //------------------------------------------------------------------------ Message( Message && msg ): Buffer( std::move( msg ) ), pIsMarshalled( msg.pIsMarshalled ), pSessionId( std::move( msg.pSessionId ) ), pVirtReqID( msg.pVirtReqID ) { } //------------------------------------------------------------------------ //! Move assignment operator //------------------------------------------------------------------------ Message& operator=( Message && msg ) { Steal( std::move( msg ) ); pIsMarshalled = msg.pIsMarshalled; pSessionId = std::move( msg.pSessionId ); pVirtReqID = msg.pVirtReqID; return *this; } //------------------------------------------------------------------------ //! Destructor //------------------------------------------------------------------------ virtual ~Message() {} //------------------------------------------------------------------------ //! Check if the message is marshalled //------------------------------------------------------------------------ bool IsMarshalled() const { return pIsMarshalled; } //------------------------------------------------------------------------ //! Set the marshalling status //------------------------------------------------------------------------ void SetIsMarshalled( bool isMarshalled ) { pIsMarshalled = isMarshalled; } //------------------------------------------------------------------------ //! Set the description of the message //------------------------------------------------------------------------ void SetDescription( const std::string &description ) { pDescription = description; } //------------------------------------------------------------------------ //! Get the description of the message //------------------------------------------------------------------------ const std::string &GetDescription() const { return pDescription; } //------------------------------------------------------------------------ //! Set the session ID which this message is meant for //------------------------------------------------------------------------ void SetSessionId( uint64_t sessionId ) { pSessionId = sessionId; } //------------------------------------------------------------------------ //! Get the session ID the message is meant for //------------------------------------------------------------------------ uint64_t GetSessionId() const { return pSessionId; } //------------------------------------------------------------------------ //! Set virtual request ID for the message //------------------------------------------------------------------------ void SetVirtReqID( uint16_t virtReqID ) { pVirtReqID = virtReqID; } //------------------------------------------------------------------------ //! Get virtual request ID for the message //------------------------------------------------------------------------ uint16_t GetVirtReqID() const { return pVirtReqID; } private: bool pIsMarshalled; uint64_t pSessionId; std::string pDescription; uint16_t pVirtReqID; }; } #endif // __XRD_CL_MESSAGE_HH__ xrootd-5.6.9/src/XrdCl/XrdClMessageUtils.cc000066400000000000000000000471031457266313600205500ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2014 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #include "XrdCl/XrdClMessageUtils.hh" #include "XrdCl/XrdClLog.hh" #include "XrdCl/XrdClDefaultEnv.hh" #include "XrdCl/XrdClConstants.hh" #include "XrdCl/XrdClAnyObject.hh" #include "XrdCl/XrdClSIDManager.hh" #include "XrdCl/XrdClPostMaster.hh" #include "XrdCl/XrdClXRootDTransport.hh" #include "XrdCl/XrdClXRootDMsgHandler.hh" #include "XrdClRedirectorRegistry.hh" #include "XProtocol/XProtocol.hh" namespace XrdCl { //---------------------------------------------------------------------------- // Send a message //---------------------------------------------------------------------------- XRootDStatus MessageUtils::SendMessage( const URL &url, Message *msg, ResponseHandler *handler, MessageSendParams &sendParams, LocalFileHandler *lFileHandler ) { //-------------------------------------------------------------------------- // Get the stuff needed to send the message //-------------------------------------------------------------------------- Log *log = DefaultEnv::GetLog(); PostMaster *postMaster = DefaultEnv::GetPostMaster(); XRootDStatus st; if( !postMaster ) return XRootDStatus( stError, errUninitialized ); log->Dump( XRootDMsg, "[%s] Sending message %s", url.GetHostId().c_str(), msg->GetDescription().c_str() ); //-------------------------------------------------------------------------- // Get an instance of SID manager object //-------------------------------------------------------------------------- std::shared_ptr sidMgr( SIDMgrPool::Instance().GetSIDMgr( url ) ); ClientRequestHdr *req = (ClientRequestHdr*)msg->GetBuffer(); //-------------------------------------------------------------------------- // Allocate the SID and marshall the message //-------------------------------------------------------------------------- st = sidMgr->AllocateSID( req->streamid ); if( !st.IsOK() ) { log->Error( XRootDMsg, "[%s] Unable to allocate stream id", url.GetHostId().c_str() ); return st; } //-------------------------------------------------------------------------- // Make sure that in case of checkpoint xeq request the embedded request // SID is matching //-------------------------------------------------------------------------- if( req->requestid == kXR_chkpoint ) { ClientRequest *r = (ClientRequest*)req; if( r->chkpoint.opcode == kXR_ckpXeq ) { ClientRequest *xeq = (ClientRequest*) msg->GetBuffer( sizeof( ClientChkPointRequest ) ); xeq->header.streamid[0] = req->streamid[0]; xeq->header.streamid[1] = req->streamid[1]; } } XRootDTransport::MarshallRequest( msg ); //-------------------------------------------------------------------------- // Create and set up the message handler //-------------------------------------------------------------------------- XRootDMsgHandler *msgHandler; msgHandler = new XRootDMsgHandler( msg, handler, &url, sidMgr, lFileHandler ); msgHandler->SetExpiration( sendParams.expires ); msgHandler->SetRedirectAsAnswer( !sendParams.followRedirects ); msgHandler->SetOksofarAsAnswer( sendParams.chunkedResponse ); msgHandler->SetChunkList( sendParams.chunkList ); msgHandler->SetKernelBuffer( sendParams.kbuff ); msgHandler->SetRedirectCounter( sendParams.redirectLimit ); msgHandler->SetStateful( sendParams.stateful ); msgHandler->SetCrc32cDigests( std::move( sendParams.crc32cDigests ) ); if( sendParams.loadBalancer.url.IsValid() ) msgHandler->SetLoadBalancer( sendParams.loadBalancer ); HostList *list = 0; if( sendParams.hostList ) { list = sendParams.hostList; sendParams.hostList = nullptr; } else list = new HostList(); list->push_back( url ); msgHandler->SetHostList( list ); //-------------------------------------------------------------------------- // Send the message //-------------------------------------------------------------------------- st = postMaster->Send( url, msg, msgHandler, sendParams.stateful, sendParams.expires ); if( !st.IsOK() ) { XRootDTransport::UnMarshallRequest( msg ); log->Error( XRootDMsg, "[%s] Unable to send the message %s: %s", url.GetHostId().c_str(), msg->GetDescription().c_str(), st.ToString().c_str() ); // we need to reassign req as its current value might have been // invalidated in the meanwhile due to a realloc req = (ClientRequestHdr*)msg->GetBuffer(); // Release the SID as the request was never send sidMgr->ReleaseSID( req->streamid ); delete msgHandler; return st; } return XRootDStatus(); } //---------------------------------------------------------------------------- // Redirect a message //---------------------------------------------------------------------------- Status MessageUtils::RedirectMessage( const URL &url, Message *msg, ResponseHandler *handler, MessageSendParams &sendParams, LocalFileHandler *lFileHandler ) { //-------------------------------------------------------------------------- // Register a new virtual redirector //-------------------------------------------------------------------------- RedirectorRegistry& registry = RedirectorRegistry::Instance(); Status st = registry.Register( url ); if( !st.IsOK() ) return st; //-------------------------------------------------------------------------- // Get the stuff needed to send the message //-------------------------------------------------------------------------- Log *log = DefaultEnv::GetLog(); PostMaster *postMaster = DefaultEnv::GetPostMaster(); if( !postMaster ) return Status( stError, errUninitialized ); log->Dump( XRootDMsg, "[%s] Redirecting message %s", url.GetHostId().c_str(), msg->GetDescription().c_str() ); XRootDTransport::MarshallRequest( msg ); //-------------------------------------------------------------------------- // Create and set up the message handler //-------------------------------------------------------------------------- XRootDMsgHandler *msgHandler; msgHandler = new XRootDMsgHandler( msg, handler, &url, std::shared_ptr(), lFileHandler ); msgHandler->SetExpiration( sendParams.expires ); msgHandler->SetRedirectAsAnswer( !sendParams.followRedirects ); msgHandler->SetOksofarAsAnswer( sendParams.chunkedResponse ); msgHandler->SetChunkList( sendParams.chunkList ); msgHandler->SetRedirectCounter( sendParams.redirectLimit ); msgHandler->SetFollowMetalink( true ); HostInfo info( url, true ); info.flags = kXR_isManager | kXR_attrMeta | kXR_attrVirtRdr; sendParams.loadBalancer = info; msgHandler->SetLoadBalancer( info ); HostList *list = 0; list = new HostList(); list->push_back( info ); msgHandler->SetHostList( list ); //-------------------------------------------------------------------------- // Redirect the message //-------------------------------------------------------------------------- st = postMaster->Redirect( url, msg, msgHandler ); if( !st.IsOK() ) { XRootDTransport::UnMarshallRequest( msg ); log->Error( XRootDMsg, "[%s] Unable to send the message %s: %s", url.GetHostId().c_str(), msg->GetDescription().c_str(), st.ToString().c_str() ); delete msgHandler; delete list; return st; } return Status(); } //---------------------------------------------------------------------------- // Process sending params //---------------------------------------------------------------------------- void MessageUtils::ProcessSendParams( MessageSendParams &sendParams ) { //-------------------------------------------------------------------------- // Timeout //-------------------------------------------------------------------------- Env *env = DefaultEnv::GetEnv(); if( sendParams.timeout == 0 ) { int requestTimeout = DefaultRequestTimeout; env->GetInt( "RequestTimeout", requestTimeout ); sendParams.timeout = requestTimeout; } if( sendParams.expires == 0 ) sendParams.expires = ::time(0)+sendParams.timeout; //-------------------------------------------------------------------------- // Redirect limit //-------------------------------------------------------------------------- if( sendParams.redirectLimit == 0 ) { int redirectLimit = DefaultRedirectLimit; env->GetInt( "RedirectLimit", redirectLimit ); sendParams.redirectLimit = redirectLimit; } } //---------------------------------------------------------------------------- //! Append cgi to the one already present in the message //---------------------------------------------------------------------------- void MessageUtils::RewriteCGIAndPath( Message *msg, const URL::ParamsMap &newCgi, bool replace, const std::string &newPath ) { ClientRequest *req = (ClientRequest *)msg->GetBuffer(); switch( req->header.requestid ) { case kXR_chmod: case kXR_mkdir: case kXR_mv: case kXR_open: case kXR_rm: case kXR_rmdir: case kXR_stat: case kXR_truncate: { //---------------------------------------------------------------------- // Get the pointer to the appropriate path //---------------------------------------------------------------------- char *path = msg->GetBuffer( 24 ); size_t length = req->header.dlen; if( req->header.requestid == kXR_mv ) { for( int i = 0; i < req->header.dlen; ++i, ++path, --length ) if( *path == ' ' ) break; ++path; --length; } //---------------------------------------------------------------------- // Create a fake URL from an existing CGI //---------------------------------------------------------------------- char *pathWithNull = new char[length+1]; memcpy( pathWithNull, path, length ); pathWithNull[length] = 0; std::ostringstream o; o << "fake://fake:111/" << pathWithNull; delete [] pathWithNull; URL currentPath( o.str() ); URL::ParamsMap currentCgi = currentPath.GetParams(); MergeCGI( currentCgi, newCgi, replace ); currentPath.SetParams( currentCgi ); if( !newPath.empty() ) currentPath.SetPath( newPath ); std::string newPathWitParams = currentPath.GetPathWithFilteredParams(); //---------------------------------------------------------------------- // Write the path with the new cgi appended to the message //---------------------------------------------------------------------- uint32_t newDlen = req->header.dlen - length + newPathWitParams.size(); msg->ReAllocate( 24+newDlen ); req = (ClientRequest *)msg->GetBuffer(); path = msg->GetBuffer( 24 ); if( req->header.requestid == kXR_mv ) { for( int i = 0; i < req->header.dlen; ++i, ++path ) if( *path == ' ' ) break; ++path; } memcpy( path, newPathWitParams.c_str(), newPathWitParams.size() ); req->header.dlen = newDlen; break; } case kXR_locate: { Env *env = DefaultEnv::GetEnv(); int preserveLocateTried = DefaultPreserveLocateTried; env->GetInt( "PreserveLocateTried", preserveLocateTried ); if( !preserveLocateTried ) break; //---------------------------------------------------------------------- // In case of locate we only want to preserve tried/triedrc CGI info //---------------------------------------------------------------------- URL::ParamsMap triedCgi; URL::ParamsMap::const_iterator itr = newCgi.find( "triedrc" ); if( itr != newCgi.end() ) triedCgi[itr->first] = itr->second; itr = newCgi.find( "tried" ); if( itr != newCgi.end() ) triedCgi[itr->first] = itr->second; //---------------------------------------------------------------------- // Is there anything to do? //---------------------------------------------------------------------- if( triedCgi.empty() ) break; //---------------------------------------------------------------------- // Get the pointer to the appropriate path //---------------------------------------------------------------------- char *path = msg->GetBuffer( 24 ); size_t length = req->header.dlen; //---------------------------------------------------------------------- // Create a fake URL from an existing CGI //---------------------------------------------------------------------- std::string strpath( path, length ); std::ostringstream o; o << "fake://fake:111/" << strpath; URL currentPath( o.str() ); URL::ParamsMap currentCgi = currentPath.GetParams(); MergeCGI( currentCgi, triedCgi, replace ); currentPath.SetParams( currentCgi ); std::string pathWitParams = currentPath.GetPathWithFilteredParams(); //---------------------------------------------------------------------- // Write the path with the new cgi appended to the message //---------------------------------------------------------------------- uint32_t newDlen = pathWitParams.size(); msg->ReAllocate( 24+newDlen ); req = (ClientRequest *)msg->GetBuffer(); path = msg->GetBuffer( 24 ); memcpy( path, pathWitParams.c_str(), pathWitParams.size() ); req->header.dlen = newDlen; break; } } XRootDTransport::SetDescription( msg ); } //------------------------------------------------------------------------ //! Merge cgi2 into cgi1 //------------------------------------------------------------------------ void MessageUtils::MergeCGI( URL::ParamsMap &cgi1, const URL::ParamsMap &cgi2, bool replace ) { URL::ParamsMap::const_iterator it; for( it = cgi2.begin(); it != cgi2.end(); ++it ) { if( replace || cgi1.find( it->first ) == cgi1.end() ) cgi1[it->first] = it->second; else { std::string &v = cgi1[it->first]; if( v.empty() ) v = it->second; else { v += ','; v += it->second; } } } } //------------------------------------------------------------------------ //! Create xattr vector //------------------------------------------------------------------------ Status MessageUtils::CreateXAttrVec( const std::vector &attrs, std::vector &avec ) { if( attrs.empty() ) return Status(); if( attrs.size() > xfaLimits::kXR_faMaxVars ) return Status( stError, errInvalidArgs ); //---------------------------------------------------------------------- // Calculate the name and value vector lengths //---------------------------------------------------------------------- // 2 bytes for rc + 1 byte for null character at the end static const int name_overhead = 3; // 4 bytes for value length static const int value_overhead = 4; size_t nlen = 0, vlen = 0; for( auto itr = attrs.begin(); itr != attrs.end(); ++itr ) { nlen += std::get( *itr ).size() + name_overhead; vlen += std::get( *itr ).size() + value_overhead; } if( nlen > xfaLimits::kXR_faMaxNlen ) return Status( stError, errInvalidArgs ); if( vlen > xfaLimits::kXR_faMaxVlen ) return Status( stError, errInvalidArgs ); //---------------------------------------------------------------------- // Create name and value vectors //---------------------------------------------------------------------- avec.resize( nlen + vlen, 0 ); char *nvec = avec.data(), *vvec = avec.data() + nlen; for( auto itr = attrs.begin(); itr != attrs.end(); ++itr ) { const std::string &name = std::get( *itr ); nvec = ClientFattrRequest::NVecInsert( name.c_str(), nvec ); const std::string &value = std::get( *itr ); vvec = ClientFattrRequest::VVecInsert( value.c_str(), vvec ); } return Status(); } //------------------------------------------------------------------------ // Create xattr name vector vector //------------------------------------------------------------------------ Status MessageUtils::CreateXAttrVec( const std::vector &attrs, std::vector &nvec ) { if( attrs.empty() ) return Status(); if( attrs.size() > xfaLimits::kXR_faMaxVars ) return Status( stError, errInvalidArgs ); //---------------------------------------------------------------------- // Calculate the name and value vector lengths //---------------------------------------------------------------------- // 2 bytes for rc + 1 byte for null character at the end static const int name_overhead = 3; size_t nlen = 0; for( auto itr = attrs.begin(); itr != attrs.end(); ++itr ) nlen += itr->size() + name_overhead; if( nlen > xfaLimits::kXR_faMaxNlen ) return Status( stError, errInvalidArgs ); //---------------------------------------------------------------------- // Create name vector //---------------------------------------------------------------------- nvec.resize( nlen, 0 ); char *nptr = nvec.data(); for( auto itr = attrs.begin(); itr != attrs.end(); ++itr ) nptr = ClientFattrRequest::NVecInsert( itr->c_str(), nptr ); return Status(); } } xrootd-5.6.9/src/XrdCl/XrdClMessageUtils.hh000066400000000000000000000305311457266313600205570ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2014 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #ifndef __XRD_CL_MESSAGE_UTILS_HH__ #define __XRD_CL_MESSAGE_UTILS_HH__ #include "XrdCl/XrdClXRootDResponses.hh" #include "XrdCl/XrdClURL.hh" #include "XrdCl/XrdClMessage.hh" #include "XrdSys/XrdSysKernelBuffer.hh" #include "XrdSys/XrdSysPthread.hh" namespace XrdCl { class LocalFileHandler; class XAttr; //---------------------------------------------------------------------------- //! Synchronize the response //---------------------------------------------------------------------------- class SyncResponseHandler: public ResponseHandler { public: //------------------------------------------------------------------------ //! Constructor //------------------------------------------------------------------------ SyncResponseHandler(): pStatus(0), pResponse(0), pCondVar(0) {} //------------------------------------------------------------------------ //! Destructor //------------------------------------------------------------------------ virtual ~SyncResponseHandler() { } //------------------------------------------------------------------------ //! Handle the response //------------------------------------------------------------------------ virtual void HandleResponse( XRootDStatus *status, AnyObject *response ) { XrdSysCondVarHelper scopedLock(pCondVar); pStatus = status; pResponse = response; pCondVar.Broadcast(); } //------------------------------------------------------------------------ //! Get the status //------------------------------------------------------------------------ XRootDStatus *GetStatus() { return pStatus; } //------------------------------------------------------------------------ //! Get the response //------------------------------------------------------------------------ AnyObject *GetResponse() { return pResponse; } //------------------------------------------------------------------------ //! Wait for the arrival of the response //------------------------------------------------------------------------ void WaitForResponse() { XrdSysCondVarHelper scopedLock(pCondVar); while (pStatus == 0) { pCondVar.Wait(); } } private: SyncResponseHandler(const SyncResponseHandler &other); SyncResponseHandler &operator = (const SyncResponseHandler &other); XRootDStatus *pStatus; AnyObject *pResponse; XrdSysCondVar pCondVar; }; //---------------------------------------------------------------------------- // We're not interested in the response just commit suicide //---------------------------------------------------------------------------- class NullResponseHandler: public XrdCl::ResponseHandler { public: //------------------------------------------------------------------------ // Handle the response //------------------------------------------------------------------------ virtual void HandleResponseWithHosts( XrdCl::XRootDStatus *status, XrdCl::AnyObject *response, XrdCl::HostList *hostList ) { delete this; } }; //---------------------------------------------------------------------------- // Sending parameters //---------------------------------------------------------------------------- struct MessageSendParams { MessageSendParams(): timeout(0), expires(0), followRedirects(true), chunkedResponse(false), stateful(true), hostList(0), chunkList(0), redirectLimit(0), kbuff(0){} uint16_t timeout; time_t expires; HostInfo loadBalancer; bool followRedirects; bool chunkedResponse; bool stateful; HostList *hostList; ChunkList *chunkList; uint16_t redirectLimit; XrdSys::KernelBuffer *kbuff; std::vector crc32cDigests; }; class MessageUtils { public: //------------------------------------------------------------------------ //! Wait and return the status of the query //------------------------------------------------------------------------ static XRootDStatus WaitForStatus( SyncResponseHandler *handler ) { handler->WaitForResponse(); XRootDStatus *status = handler->GetStatus(); XRootDStatus ret( *status ); delete status; return ret; } //------------------------------------------------------------------------ //! Wait for the response //------------------------------------------------------------------------ template static XrdCl::XRootDStatus WaitForResponse( SyncResponseHandler *handler, Type *&response ) { handler->WaitForResponse(); AnyObject *resp = handler->GetResponse(); XRootDStatus *status = handler->GetStatus(); XRootDStatus ret( *status ); delete status; if( ret.IsOK() ) { if( !resp ) return XRootDStatus( stError, errInternal ); resp->Get( response ); resp->Set( (int *)0 ); delete resp; if( !response ) return XRootDStatus( stError, errInternal ); } return ret; } //------------------------------------------------------------------------ //! Create a message //------------------------------------------------------------------------ template static void CreateRequest( Message *&msg, Request *&req, uint32_t payloadSize = 0 ) { msg = new Message( sizeof(Request) + payloadSize ); req = (Request*)msg->GetBuffer(); msg->Zero(); } //------------------------------------------------------------------------ //! Send message //------------------------------------------------------------------------ static XRootDStatus SendMessage( const URL &url, Message *msg, ResponseHandler *handler, MessageSendParams &sendParams, LocalFileHandler *lFileHandler ); //------------------------------------------------------------------------ //! Redirect message //------------------------------------------------------------------------ static Status RedirectMessage( const URL &url, Message *msg, ResponseHandler *handler, MessageSendParams &sendParams, LocalFileHandler *lFileHandler ); //------------------------------------------------------------------------ //! Process sending params //------------------------------------------------------------------------ static void ProcessSendParams( MessageSendParams &sendParams ); //------------------------------------------------------------------------ //! Rewrite CGI and path if necessary //! //! @param msg message concerned //! @param newCgi the new cgi //! @param replace indicates whether, in case of a conflict, the new CGI //! parameter should replace an existing one or be //! appended to it using a comma //! @param newPath will be used as the new destination path if it is //! not empty //------------------------------------------------------------------------ static void RewriteCGIAndPath( Message *msg, const URL::ParamsMap &newCgi, bool replace, const std::string &newPath ); //------------------------------------------------------------------------ //! Merge cgi2 into cgi1 //! //! @param cgi1 cgi to be merged into //! @param cgi2 cgi to be merged in //! @param replace indicates whether, in case of a conflict, the new CGI //! parameter should replace an existing one or be //! appended to it using a comma //------------------------------------------------------------------------ static void MergeCGI( URL::ParamsMap &cgi1, const URL::ParamsMap &cgi2, bool replace ); //------------------------------------------------------------------------ //! Create xattr vector //! //! @param attrs : extended attribute list //! @param avec : vector containing the name vector //! and the value vector //------------------------------------------------------------------------ static Status CreateXAttrVec( const std::vector &attrs, std::vector &avec ); //------------------------------------------------------------------------ //! Create xattr name vector vector //! //! @param attrs : extended attribute name list //! @param nvec : vector containing the name vector //------------------------------------------------------------------------ static Status CreateXAttrVec( const std::vector &attrs, std::vector &nvec ); //------------------------------------------------------------------------ //! Create body of xattr request and set the body size //! //! @param msg : the request //! @param vec : the argument //! @param path : file path //------------------------------------------------------------------------ template static Status CreateXAttrBody( Message *msg, const std::vector &vec, const std::string &path = "" ) { ClientRequestHdr *hdr = reinterpret_cast( msg->GetBuffer() ); std::vector xattrvec; Status st = MessageUtils::CreateXAttrVec( vec, xattrvec ); if( !st.IsOK() ) return st; // update body size in the header hdr->dlen = path.size() + 1; hdr->dlen += xattrvec.size(); // append the body size_t offset = sizeof( ClientRequestHdr ); msg->Append( path.c_str(), path.size() + 1, offset ); offset += path.size() + 1; msg->Append( xattrvec.data(), xattrvec.size(), offset ); return Status(); } }; } #endif // __XRD_CL_MESSAGE_UTILS_HH__ xrootd-5.6.9/src/XrdCl/XrdClMetalinkRedirector.cc000066400000000000000000000441231457266313600217310ustar00rootroot00000000000000/* * XrdClMetalinkRedirector.cc * * Created on: May 2, 2016 * Author: simonm */ #include "XrdClMetalinkRedirector.hh" #include "XrdCl/XrdClConstants.hh" #include "XrdCl/XrdClDefaultEnv.hh" #include "XrdCl/XrdClLog.hh" #include "XrdCl/XrdClMessageUtils.hh" #include "XrdCl/XrdClFile.hh" #include "XrdCl/XrdClFileStateHandler.hh" #include "XrdCl/XrdClURL.hh" #include "XrdCl/XrdClJobManager.hh" #include "XrdCl/XrdClUtils.hh" #include "XrdCl/XrdClPostMasterInterfaces.hh" #include "XrdCl/XrdClPostMaster.hh" #include "XrdXml/XrdXmlMetaLink.hh" #include "XProtocol/XProtocol.hh" #include #include namespace XrdCl { void DeallocArgs( XRootDStatus *status, AnyObject *response, HostList *hostList ) { delete status; delete response; delete hostList; } //---------------------------------------------------------------------------- // Read metalink response handler. // // If the whole file has been read triggers parsing and // initializes the Redirector, otherwise triggers another // read. //---------------------------------------------------------------------------- class MetalinkReadHandler: public ResponseHandler { public: //---------------------------------------------------------------------------- // Constructor //---------------------------------------------------------------------------- MetalinkReadHandler( MetalinkRedirector *mr, ResponseHandler *userHandler, const std::string &content = "" ) : pRedirector( mr ), pUserHandler( userHandler ), pBuffer( new char[DefaultCPChunkSize] ), pContent( content ) { } //---------------------------------------------------------------------------- // Destructor //---------------------------------------------------------------------------- virtual ~MetalinkReadHandler() { delete[] pBuffer; } //---------------------------------------------------------------------------- // Handle the response //---------------------------------------------------------------------------- virtual void HandleResponse( XRootDStatus *status, AnyObject *response ) { try { // check the status if( !status->IsOK() ) throw status; // we don't need it anymore delete status; // make sure we got a response if( !response ) throw new XRootDStatus( stError, errInternal ); // make sure the response is not empty ChunkInfo * info = 0; response->Get( info ); if( !info ) throw new XRootDStatus( stError, errInternal ); uint32_t bytesRead = info->length; uint64_t offset = info->offset + bytesRead; pContent += std::string( pBuffer, bytesRead ); // are we done ? if( bytesRead > 0 ) { // lets try to read another chunk MetalinkReadHandler * mrh = new MetalinkReadHandler( pRedirector, pUserHandler, pContent ); XRootDStatus st = pRedirector->pFile->Read( offset, DefaultCPChunkSize, mrh->GetBuffer(), mrh ); if( !st.IsOK() ) { delete mrh; throw new XRootDStatus( st ); } } else // we have the whole metalink file { // we don't need the File object anymore delete pRedirector->pFile; pRedirector->pFile = 0; // now we can parse the metalink file XRootDStatus st = pRedirector->Parse( pContent ); // now with the redirector fully initialized we can handle pending requests pRedirector->FinalizeInitialization(); // we are done, pass the status to the user (whatever it is) if( pUserHandler ) pUserHandler->HandleResponse( new XRootDStatus( st ), 0 ); } // clean up delete response; } catch( XRootDStatus *status ) { pRedirector->FinalizeInitialization( *status ); // if we were not able to read from the metalink, // propagate the error to the user handler if( pUserHandler ) pUserHandler->HandleResponse( status, 0 ); else DeallocArgs( status, response, 0 ); } delete this; } //---------------------------------------------------------------------------- // Get the receive-buffer //---------------------------------------------------------------------------- char * GetBuffer() { return pBuffer; } private: MetalinkRedirector *pRedirector; ResponseHandler *pUserHandler; char *pBuffer; std::string pContent; }; //---------------------------------------------------------------------------- // Open metalink response handler. // // After successful open trrigers a read. //---------------------------------------------------------------------------- class MetalinkOpenHandler: public ResponseHandler { public: //---------------------------------------------------------------------------- // Constructor //---------------------------------------------------------------------------- MetalinkOpenHandler( MetalinkRedirector *mr, ResponseHandler *userHandler ) : pRedirector( mr ), pUserHandler( userHandler ) { } //---------------------------------------------------------------------------- // Handle the response //---------------------------------------------------------------------------- virtual void HandleResponseWithHosts( XRootDStatus *status, AnyObject *response, HostList *hostList ) { try { if( status->IsOK() ) { delete status; // download the content MetalinkReadHandler *mrh = new MetalinkReadHandler( pRedirector, pUserHandler ); XRootDStatus st = pRedirector->pFile->Read( 0, DefaultCPChunkSize, mrh->GetBuffer(), mrh ); if( !st.IsOK() ) { delete mrh; throw new XRootDStatus( stError, errInternal ); } else { delete response; delete hostList; } } else throw status; } catch( XRootDStatus *status ) { pRedirector->FinalizeInitialization( *status ); // if we were not able to schedule a read // pass an error to the user handler if( pUserHandler ) pUserHandler->HandleResponseWithHosts( status, response, hostList ); else DeallocArgs( status, response, hostList ); } delete this; } private: MetalinkRedirector *pRedirector; ResponseHandler *pUserHandler; }; //---------------------------------------------------------------------------- // Constructor //---------------------------------------------------------------------------- MetalinkRedirector::MetalinkRedirector( const std::string & url ) : pUrl( url ), pFile( new File( File::DisableVirtRedirect ) ), pReady( false ), pFileSize( -1 ) { } //---------------------------------------------------------------------------- // Destructor //---------------------------------------------------------------------------- MetalinkRedirector::~MetalinkRedirector() { delete pFile; } //---------------------------------------------------------------------------- // Initializes the object with the content of the metalink file //---------------------------------------------------------------------------- XRootDStatus MetalinkRedirector::Load( ResponseHandler *userHandler ) { MetalinkOpenHandler *handler = new MetalinkOpenHandler( this, userHandler ); XRootDStatus st = pFile->Open( pUrl, OpenFlags::Read, Access::None, handler, 0 ); if( !st.IsOK() ) delete handler; return st; } //---------------------------------------------------------------------------- // Parses the metalink file //---------------------------------------------------------------------------- XRootDStatus MetalinkRedirector::Parse( const std::string &metalink ) { Log *log = DefaultEnv::GetLog(); Env *env = DefaultEnv::GetEnv(); std::string glfnRedirector; env->GetString( "GlfnRedirector", glfnRedirector ); // parse the metalink XrdXmlMetaLink parser( "root:xroot:roots:xroots:file:", "xroot:", glfnRedirector.empty() ? 0 : glfnRedirector.c_str() ); int size = 0; XrdOucFileInfo **fileInfos = parser.ConvertAll( metalink.c_str(), size, metalink.size() ); if( !fileInfos ) { int ecode; const char * etxt = parser.GetStatus( ecode ); log->Error( UtilityMsg, "Failed to parse the metalink file: %s (error code: %d)", etxt, ecode ); return XRootDStatus( stError, errDataError, 0, "Malformed or corrupted metalink file." ); } // we are expecting just one file per metalink (as agreed with RUCIO) if( size != 1 ) { log->Error( UtilityMsg, "Expected only one file per metalink." ); return XRootDStatus( stError, errDataError ); } InitCksum( fileInfos ); InitReplicas( fileInfos ); pTarget = fileInfos[0]->GetTargetName(); pFileSize = fileInfos[0]->GetSize(); XrdXmlMetaLink::DeleteAll( fileInfos, size ); return XRootDStatus(); } //---------------------------------------------------------------------------- // Finalize the initialization process: // - mark as ready // - setup the status // - and handle pending redirects //---------------------------------------------------------------------------- void MetalinkRedirector::FinalizeInitialization( const XRootDStatus &status ) { XrdSysMutexHelper scopedLock( pMutex ); pReady = true; pStatus = status; // Handle pending redirects (those that were // submitted before the metalink has been loaded) while( !pPendingRedirects.empty() ) { const Message *msg = pPendingRedirects.front().first; MsgHandler *handler = pPendingRedirects.front().second; pPendingRedirects.pop_front(); if( !handler || !msg ) continue; HandleRequestImpl( msg, handler ); } } //---------------------------------------------------------------------------- // Generates redirect response for the given request //---------------------------------------------------------------------------- std::shared_ptr MetalinkRedirector::GetResponse( const Message *msg ) const { if( !pStatus.IsOK() ) return GetErrorMsg( msg, "Could not load the Metalink file.", static_cast( XProtocol::mapError( pStatus.errNo ) ) ); const ClientRequestHdr *req = reinterpret_cast( msg->GetBuffer() ); // get the redirect location std::string replica; if( !GetReplica( *msg, replica ).IsOK() ) return GetErrorMsg( msg, "Metalink: no more replicas to try.", kXR_noReplicas ); auto resp = std::make_shared( sizeof(ServerResponse) ); ServerResponse* response = reinterpret_cast( resp->GetBuffer() ); response->hdr.status = kXR_redirect; response->hdr.streamid[0] = req->streamid[0]; response->hdr.streamid[1] = req->streamid[1]; response->hdr.dlen = 4 + replica.size(); // 4 bytes are reserved for port number response->body.redirect.port = -1; // this indicates that the full URL will be given in the 'host' field memcpy( response->body.redirect.host, replica.c_str(), replica.size() ); return resp; } //---------------------------------------------------------------------------- // Generates error response for the given request //---------------------------------------------------------------------------- std::shared_ptr MetalinkRedirector::GetErrorMsg( const Message *msg, const std::string &errMsg, XErrorCode code ) const { const ClientRequestHdr *req = reinterpret_cast( msg->GetBuffer() ); auto resp = std::make_shared( sizeof(ServerResponse) ); ServerResponse* response = reinterpret_cast( resp->GetBuffer() ); response->hdr.status = kXR_error; response->hdr.streamid[0] = req->streamid[0]; response->hdr.streamid[1] = req->streamid[1]; response->hdr.dlen = 4 + errMsg.size(); response->body.error.errnum = htonl( code ); memcpy( response->body.error.errmsg, errMsg.c_str(), errMsg.size() ); return resp; } //---------------------------------------------------------------------------- // Creates an instant redirect response for the given message // or an error response if there are no more replicas to try. // The virtual response is being handled by the given handler. //---------------------------------------------------------------------------- XRootDStatus MetalinkRedirector::HandleRequestImpl( const Message *msg, MsgHandler *handler ) { auto resp = GetResponse( msg ); JobManager *jobMan = DefaultEnv::GetPostMaster()->GetJobManager(); RedirectJob *job = new RedirectJob( handler, std::move( resp ) ); jobMan->QueueJob( job ); return XRootDStatus(); } //---------------------------------------------------------------------------- // If the MetalinkRedirector is initialized creates an instant // redirect response, otherwise queues the request until initialization // is done. //---------------------------------------------------------------------------- XRootDStatus MetalinkRedirector::HandleRequest( const Message *msg, MsgHandler *handler ) { XrdSysMutexHelper scopedLck( pMutex ); // if the metalink data haven't been loaded yet, make it pending if( !pReady ) { pPendingRedirects.push_back( std::make_pair( msg, handler ) );; return XRootDStatus(); } // otherwise generate a virtual response return HandleRequestImpl( msg, handler ); } //---------------------------------------------------------------------------- //! Count how many replicas do we have left to try for given request //---------------------------------------------------------------------------- int MetalinkRedirector::Count( Message &req ) const { ReplicaList::const_iterator itr = GetReplica( req ); return pReplicas.end() - itr; } //---------------------------------------------------------------------------- // Gets the file checksum if specified in the metalink //---------------------------------------------------------------------------- void MetalinkRedirector::InitCksum( XrdOucFileInfo **fileInfos ) { const char *chvalue = 0, *chtype = 0; while( ( chtype = fileInfos[0]->GetDigest( chvalue ) ) ) { pChecksums[chtype] = chvalue; } } //---------------------------------------------------------------------------- // Initializes replica list //---------------------------------------------------------------------------- void MetalinkRedirector::InitReplicas( XrdOucFileInfo **fileInfos ) { URL replica; const char *url = 0; while( ( url = fileInfos[0]->GetUrl() ) ) { replica = URL( url ); if( !replica.IsValid() || replica.GetURL().size() > 4096 ) continue; // this is the internal limit (defined in the protocol) pReplicas.push_back( replica.GetURL() ); } } //---------------------------------------------------------------------------- //! Get the next replica for the given message //---------------------------------------------------------------------------- XRootDStatus MetalinkRedirector::GetReplica( const Message &msg, std::string &replica ) const { ReplicaList::const_iterator itr = GetReplica( msg ); if( itr == pReplicas.end() ) return XRootDStatus( stError, errNotFound ); XrdCl::Env *env = XrdCl::DefaultEnv::GetEnv(); int tlsmtl = DefaultTlsMetalink; env->GetInt( "TlsMetalink", tlsmtl ); URL url( *itr ); if( tlsmtl && ( url.GetProtocol() == "root" || url.GetProtocol() == "xroot" ) ) url.SetProtocol( "roots" ); replica = url.GetURL(); return XRootDStatus(); } //---------------------------------------------------------------------------- //! Get the next replica for the given message //---------------------------------------------------------------------------- MetalinkRedirector::ReplicaList::const_iterator MetalinkRedirector::GetReplica( const Message &msg ) const { if( pReplicas.empty() ) return pReplicas.cend(); std::string tried; if( !GetCgiInfo( msg, "tried", tried ).IsOK() ) return pReplicas.cbegin(); ReplicaList triedList; Utils::splitString( triedList, tried, "," ); std::set triedSet( triedList.begin(), triedList.end() ); ReplicaList::const_iterator itr = pReplicas.begin(); for( ; itr != pReplicas.end(); ++itr ) { URL url( *itr ); if( !triedSet.count( url.GetHostName() ) ) break; } return itr; } //---------------------------------------------------------------------------- //! Extracts an element from url cgi //---------------------------------------------------------------------------- XRootDStatus MetalinkRedirector::GetCgiInfo( const Message &msg, const std::string &key, std::string &value ) const { const ClientRequestHdr *req = reinterpret_cast( msg.GetBuffer() ); kXR_int32 dlen = msg.IsMarshalled() ? ntohl( req->dlen ) : req->dlen; std::string url( msg.GetBuffer( 24 ), dlen ); size_t pos = url.find( '?' ); if( pos == std::string::npos ) return XRootDStatus( stError ); size_t start = url.find( key, pos ); if( start == std::string::npos ) return XRootDStatus( stError ); start += key.size() + 1; // the +1 stands for the '=' sign that is not part of the key size_t end = url.find( '&', start ); if( end == std::string::npos ) end = url.size(); value = url.substr( start, end - start ); return XRootDStatus(); } } /* namespace XrdCl */ xrootd-5.6.9/src/XrdCl/XrdClMetalinkRedirector.hh000066400000000000000000000203151457266313600217400ustar00rootroot00000000000000/* * XrdClMetalinkRedirector.hh * * Created on: May 2, 2016 * Author: simonm */ #ifndef SRC_XRDCL_XRDCLMETALINKREDIRECTOR_HH_ #define SRC_XRDCL_XRDCLMETALINKREDIRECTOR_HH_ #include "XrdCl/XrdClMessageUtils.hh" #include "XrdCl/XrdClRedirectorRegistry.hh" #include #include #include class XrdOucFileInfo; namespace XrdCl { class File; class Message; //---------------------------------------------------------------------------- //! An abstraction representing a virtual //! redirector based on a metalink file //---------------------------------------------------------------------------- class MetalinkRedirector : public VirtualRedirector { friend class MetalinkOpenHandler; friend class MetalinkReadHandler; public: //---------------------------------------------------------------------------- //! Constructor //! @param url : URL to the metalink file //---------------------------------------------------------------------------- MetalinkRedirector( const std::string &url ); //---------------------------------------------------------------------------- //! Destructor //---------------------------------------------------------------------------- virtual ~MetalinkRedirector(); //---------------------------------------------------------------------------- //! Initializes the object with the content of the metalink file //! @param userHandler : the response handler provided by end user //---------------------------------------------------------------------------- XRootDStatus Load( ResponseHandler *userHandler ); //---------------------------------------------------------------------------- //! If the MetalinkRedirector is initialized creates an instant //! redirect response, otherwise queues the request until initialization //! is done. //---------------------------------------------------------------------------- XRootDStatus HandleRequest( const Message *msg, MsgHandler *handler ); //---------------------------------------------------------------------------- //! Gets the file name as specified in the metalink //---------------------------------------------------------------------------- std::string GetTargetName() const { return pTarget; } //---------------------------------------------------------------------------- //! Returns the checksum of the given type if specified //! in the metalink file, or an empty string otherwise //---------------------------------------------------------------------------- std::string GetCheckSum( const std::string &type ) const { std::string t = type != "adler32" ? type : "a32"; CksumMap::const_iterator it = pChecksums.find( t ); if( it == pChecksums.end() ) return std::string(); return type + ":" + it->second; } //---------------------------------------------------------------------------- //! Returns the first (in alphabetical order) checksum type available in the //! metalink file, if no checksum is available returns an empty string //---------------------------------------------------------------------------- std::vector GetSupportedCheckSums() const { std::vector ret; CksumMap::const_iterator itr = pChecksums.begin(); for( ; itr != pChecksums.end(); ++itr ) { if( itr->first == "a32" ) ret.push_back( "adler32" ); else ret.push_back( itr->first ); } return ret; } //---------------------------------------------------------------------------- //! Returns the file size if specified in the metalink file, //! otherwise a negative number //---------------------------------------------------------------------------- long long GetSize() const { return pFileSize; } //---------------------------------------------------------------------------- //! Returns a vector with replicas as given in the meatlink file //---------------------------------------------------------------------------- const std::vector& GetReplicas() { return pReplicas; } //---------------------------------------------------------------------------- //! Count how many replicas do we have left to try for given request //---------------------------------------------------------------------------- virtual int Count( Message &req ) const; private: //---------------------------------------------------------------------------- //! Creates an instant redirect response for the given message //! or an error response if there are no more replicas to try. //! The virtual response is being handled by the given handler //! in the thread-pool. //---------------------------------------------------------------------------- XRootDStatus HandleRequestImpl( const Message *msg, MsgHandler *handler ); //---------------------------------------------------------------------------- //! Parses the metalink file //! @param metalink : the content of the metalink file //---------------------------------------------------------------------------- XRootDStatus Parse( const std::string &metalink ); //---------------------------------------------------------------------------- //! Finalize the initialization process: //! - mark as ready //! - setup the status //! - and handle pending redirects //---------------------------------------------------------------------------- void FinalizeInitialization( const XRootDStatus &status = XRootDStatus() ); //---------------------------------------------------------------------------- //! Generates redirect response for the given request //---------------------------------------------------------------------------- std::shared_ptr GetResponse( const Message *msg ) const; //---------------------------------------------------------------------------- //! Generates error response for the given request //---------------------------------------------------------------------------- std::shared_ptr GetErrorMsg( const Message *msg, const std::string &errMsg, XErrorCode code ) const; //---------------------------------------------------------------------------- //! Initializes checksum map //---------------------------------------------------------------------------- void InitCksum( XrdOucFileInfo **fileInfos ); //---------------------------------------------------------------------------- //! Initializes replica list //---------------------------------------------------------------------------- void InitReplicas( XrdOucFileInfo **fileInfos ); //---------------------------------------------------------------------------- //! Get the next replica for the given message //---------------------------------------------------------------------------- XRootDStatus GetReplica( const Message &msg, std::string &replica ) const; //---------------------------------------------------------------------------- //! Extracts an element from URL cgi //---------------------------------------------------------------------------- XRootDStatus GetCgiInfo( const Message &msg, const std::string &key, std::string &out ) const; typedef std::list< std::pair > RedirectList; typedef std::map CksumMap; typedef std::vector ReplicaList; //---------------------------------------------------------------------------- //! Get the next replica for the given message //---------------------------------------------------------------------------- ReplicaList::const_iterator GetReplica( const Message &msg ) const; RedirectList pPendingRedirects; std::string pUrl; File *pFile; CksumMap pChecksums; ReplicaList pReplicas; bool pReady; XRootDStatus pStatus; std::string pTarget; long long pFileSize; XrdSysMutex pMutex; static const std::string LocalFile; }; } /* namespace XrdCl */ #endif /* SRC_XRDCL_XRDCLMETALINKREDIRECTOR_HH_ */ xrootd-5.6.9/src/XrdCl/XrdClMonitor.hh000066400000000000000000000244561457266313600176120ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2012 by the Board of Trustees of the Leland Stanford, Jr., // University // Copyright (c) 2012 by European Organization for Nuclear Research (CERN) // Author: Andrew Hanushevsky // Author: Lukasz Janyst //------------------------------------------------------------------------------ // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . //------------------------------------------------------------------------------ //------------------------------------------------------------------------------ //! When the envar XRD_CLIENTMONITOR is set to the libpath/libname.so that //! holds the monitoring object, it is automatically loaded. The following //! "C" external symbols must exist in the monitor plug-in shared library. //! It is called to obtain an instance of the XrdCl::Monitor object. //! //! @param exec full path name to executable provided by XrdSysUtils::ExecName //! //! @param parms Value of XRD_CLIENTMONITOPARAM envar or null if it is not set. //! //! @return Pointer to an instance of XrdCl::Monitor or null which causes //! monitoring to be disabled. //! //! extern "C" //! { //! XrdCl::Monitor *XrdClGetMonitor(const char *exec, const char *parms); //! } //------------------------------------------------------------------------------ #ifndef __XRD_CL_MONITOR_HH__ #define __XRD_CL_MONITOR_HH__ #include "XrdCl/XrdClFileSystem.hh" #include namespace XrdCl { class URL; //---------------------------------------------------------------------------- //! An abstract class to describe the client-side monitoring plugin interface. //---------------------------------------------------------------------------- class Monitor { public: //------------------------------------------------------------------------ //! Constructor //------------------------------------------------------------------------ Monitor() {} //------------------------------------------------------------------------ //! Destructor //------------------------------------------------------------------------ virtual ~Monitor() {} //------------------------------------------------------------------------ //! Describe a server login event //------------------------------------------------------------------------ struct ConnectInfo { ConnectInfo(): streams( 0 ) { sTOD.tv_sec = 0; sTOD.tv_usec = 0; eTOD.tv_sec = 0; eTOD.tv_usec = 0; } std::string server; //!< "user@host:port" std::string auth; //!< authentication protocol used or empty if none timeval sTOD; //!< gettimeofday() when login started timeval eTOD; //!< gettimeofday() when login ended uint16_t streams; //!< Number of streams }; //------------------------------------------------------------------------ //! Describe a server logout event //------------------------------------------------------------------------ struct DisconnectInfo { DisconnectInfo(): rBytes(0), sBytes(0), cTime(0) {} std::string server; //!< "user@host:port" uint64_t rBytes; //!< Number of bytes received uint64_t sBytes; //!< Number of bytes sent time_t cTime; //!< Seconds connected to the server Status status; //!< Disconnection status }; //------------------------------------------------------------------------ //! Describe a file open event to the monitor //------------------------------------------------------------------------ struct OpenInfo { OpenInfo(): file(0), fSize(0), oFlags(0) {} const URL *file; //!< File in question std::string dataServer; //!< Actual fata server uint64_t fSize; //!< File size in bytes uint16_t oFlags; //!< OpenFlags }; //------------------------------------------------------------------------ //! Describe a file close event //------------------------------------------------------------------------ struct CloseInfo { CloseInfo(): file(0), rBytes(0), vrBytes(0), wBytes(0), vwBytes(0), vSegs(0), rCount(0), vCount(0), wCount(0), status(0) { oTOD.tv_sec = 0; oTOD.tv_usec = 0; cTOD.tv_sec = 0; cTOD.tv_usec = 0; } const URL *file; //!< The file in question timeval oTOD; //!< gettimeofday() when file was opened timeval cTOD; //!< gettimeofday() when file was closed uint64_t rBytes; //!< Total number of bytes read via read uint64_t vrBytes; //!< Total number of bytes read via readv uint64_t wBytes; //!< Total number of bytes written uint64_t vwBytes; //!< Total number of bytes written vie writev uint64_t vSegs; //!< Total count of readv segments uint32_t rCount; //!< Total count of reads uint32_t vCount; //!< Total count of readv uint32_t wCount; //!< Total count of writes const XRootDStatus *status; //!< Close status }; //------------------------------------------------------------------------ //! Describe an encountered file-based error //------------------------------------------------------------------------ struct ErrorInfo { enum Operation { ErrOpen = 0, //!< Open (ditto) ErrRead, //!< Read ErrReadV, //!< Readv ErrWrite, //!< Write ErrWriteV, //!< WriteV ErrUnc //!< Unclassified operation }; ErrorInfo(): file(0), status(0), opCode( ErrUnc ) {} const URL *file; //!< The file in question const XRootDStatus *status; //!< Status code Operation opCode; //!< The associated operation }; //------------------------------------------------------------------------ //! Describe the transfer //------------------------------------------------------------------------ struct TransferInfo { TransferInfo(): origin(0), target(0) {} const URL *origin; //!< URL of the origin const URL *target; //!< URL of the target }; //------------------------------------------------------------------------ //! Describe a start of copy event. Copy events are sequential by nature. //! a copybeg event is followed by a number of open and close events. When //! the copy finishes, all files are closed and a copyend event occurs. //------------------------------------------------------------------------ struct CopyBInfo { TransferInfo transfer; //!< The transfer in question }; //------------------------------------------------------------------------ //! Describe an end of copy event //------------------------------------------------------------------------ struct CopyEInfo { CopyEInfo(): sources(0), status(0) { bTOD.tv_sec = 0; bTOD.tv_usec = 0; eTOD.tv_sec = 0; eTOD.tv_usec = 0; } TransferInfo transfer; //!< The transfer in question int sources; //!< Number of sources used for the copy timeval bTOD; //!< Copy start time timeval eTOD; //!< Copy end time const XRootDStatus *status; //!< Status of the copy }; //------------------------------------------------------------------------ //! Describe a checksum event //------------------------------------------------------------------------ struct CheckSumInfo { CheckSumInfo(): oTime(0), tTime(0), isOK(false) {} TransferInfo transfer; //!< The transfer in question std::string cksum; //!< Checksum as "type:value" uint64_t oTime; //!< Microseconds to obtain cksum from origin uint64_t tTime; //!< Microseconds to obtain cksum from target bool isOK; //!< True if checksum matched, false otherwise }; //------------------------------------------------------------------------ //! Event codes passed to the Event() method. Event code values not //! listed here, if encountered, should be ignored. //------------------------------------------------------------------------ enum EventCode { EvCopyBeg, //!< CopyBInfo: Copy operation started EvCopyEnd, //!< CopyEInfo: Copy operation ended EvCheckSum, //!< CheckSumInfo: File checksummed EvOpen, //!< OpenInfo: File opened EvClose, //!< CloseInfo: File closed EvErrIO, //!< ErrorInfo: An I/O error occurred EvConnect, //!< ConnectInfo: Login into a server EvDisconnect //!< DisconnectInfo: Logout from a server }; //------------------------------------------------------------------------ //! Inform the monitor of an event. //! //! @param evCode is the event that occurred (see enum evNum) //! @param evData is the event information structure describing the event //! it is cast to (void *) so that one method can be used //! and should be recast to the correct corresponding struct //------------------------------------------------------------------------ virtual void Event( EventCode evCode, void *evData ) = 0; }; } #endif // __XRD_CL_MONITOR_HH xrootd-5.6.9/src/XrdCl/XrdClOperationHandlers.hh000066400000000000000000000712621457266313600216010ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2017 by European Organization for Nuclear Research (CERN) // Author: Krzysztof Jamrog , // Michal Simon //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #ifndef __XRD_CL_OPERATION_HANDLERS_HH__ #define __XRD_CL_OPERATION_HANDLERS_HH__ #include "XrdCl/XrdClFile.hh" #include "XrdCl/XrdClCtx.hh" #include #include #include namespace XrdCl { //---------------------------------------------------------------------------- //! Helper class for unpacking single XAttrStatus from bulk response //---------------------------------------------------------------------------- class UnpackXAttrStatus : public ResponseHandler { public: UnpackXAttrStatus( ResponseHandler *handler ) : handler( handler ) { } //------------------------------------------------------------------------ //! Callback method. //------------------------------------------------------------------------ void HandleResponse( XRootDStatus *status, AnyObject *response ) { // status maybe error for old servers not supporting xattrs if( !status->IsOK() ) { handler->HandleResponse( status, nullptr ); return; } std::vector *bulk = nullptr; response->Get( bulk ); *status = bulk->front().status; handler->HandleResponse( status, nullptr ); delete response; } private: ResponseHandler *handler; }; //---------------------------------------------------------------------------- //! Helper class for unpacking single XAttr from bulk response //---------------------------------------------------------------------------- class UnpackXAttr : public ResponseHandler { public: UnpackXAttr( ResponseHandler *handler ) : handler( handler ) { } //------------------------------------------------------------------------ //! Callback method. //------------------------------------------------------------------------ void HandleResponse( XRootDStatus *status, AnyObject *response ) { // status is always OK for bulk response std::vector *bulk = nullptr; response->Get( bulk ); *status = bulk->front().status; std::string *rsp = new std::string( std::move( bulk->front().value ) ); delete bulk; response->Set( rsp ); handler->HandleResponse( status, response ); } private: ResponseHandler *handler; }; //---------------------------------------------------------------------------- // Helper class for creating null references for particular types // // @arg Response : type for which we need a null reference //---------------------------------------------------------------------------- template struct NullRef { static Response value; }; //---------------------------------------------------------------------------- // Initialize the 'null-reference' //---------------------------------------------------------------------------- template Response NullRef::value; //---------------------------------------------------------------------------- //! Unpack response //! //! @param rsp : AnyObject holding response //! @return : the response //---------------------------------------------------------------------------- template inline Response* GetResponse( AnyObject *rsp ) { Response *ret = nullptr; rsp->Get( ret ); return ret; } //---------------------------------------------------------------------------- //! Unpack response //! //! @param rsp : AnyObject holding response //! @param status : //! @return : the response //---------------------------------------------------------------------------- template inline Response* GetResponse( XRootDStatus *status, AnyObject *rsp ) { if( !status->IsOK() ) return &NullRef::value; return GetResponse( rsp ); } //---------------------------------------------------------------------------- //! Lambda wrapper //! //! @arg ResponseType : type of response returned by the server //---------------------------------------------------------------------------- template class FunctionWrapper: public ResponseHandler { public: //------------------------------------------------------------------------ //! Constructor. // //! @param func : function, functor or lambda (2 arguments) //------------------------------------------------------------------------ FunctionWrapper( std::function handleFunction ) : fun( [handleFunction]( XRootDStatus &s, Response &r, HostList& ){ handleFunction( s, r ); } ) { } //------------------------------------------------------------------------ //! Constructor. // //! @param func : function, functor or lambda (3 arguments) //------------------------------------------------------------------------ FunctionWrapper( std::function handleFunction ) : fun( handleFunction ) { } //------------------------------------------------------------------------ //! Callback method. //------------------------------------------------------------------------ void HandleResponseWithHosts( XRootDStatus *status, AnyObject *response, HostList *hostList ) { std::unique_ptr delst( status ); std::unique_ptr delrsp( response ); std::unique_ptr delhl( hostList ); Response *res = GetResponse( status, response ); fun( *status, *res, *hostList ); } private: //------------------------------------------------------------------------ //! user defined function, functor or lambda //------------------------------------------------------------------------ std::function fun; }; //---------------------------------------------------------------------------- //! Lambda wrapper //! //! Template specialization for responses that return no value (void) //---------------------------------------------------------------------------- template<> class FunctionWrapper : public ResponseHandler { public: //------------------------------------------------------------------------ //! Constructor. // //! @param func : function, functor or lambda (1 argument) //------------------------------------------------------------------------ FunctionWrapper( std::function handleFunction ) : fun( [handleFunction]( XRootDStatus& s, HostList& ){ handleFunction( s ); } ) { } //------------------------------------------------------------------------ //! Constructor. // //! @param func : function, functor or lambda (2 arguments) //------------------------------------------------------------------------ FunctionWrapper( std::function handleFunction ) : fun( handleFunction ) { } //------------------------------------------------------------------------ //! Callback method. //------------------------------------------------------------------------ void HandleResponseWithHosts( XRootDStatus *status, AnyObject *response, HostList *hostList ) { std::unique_ptr delst( status ); std::unique_ptr delrsp( response ); std::unique_ptr delhl( hostList ); fun( *status, *hostList ); } private: //------------------------------------------------------------------------ //! user defined function, functor or lambda //------------------------------------------------------------------------ std::function fun; }; //---------------------------------------------------------------------------- //! Packaged Task wrapper //! //! @arg Response : type of response returned by the server //! @arg Return : type of the value returned by the task //---------------------------------------------------------------------------- template class TaskWrapper: public ResponseHandler { public: //------------------------------------------------------------------------ //! Constructor. // //! @param task : a std::packaged_task //------------------------------------------------------------------------ TaskWrapper( std::packaged_task && task ) : task( std::move( task ) ) { } //------------------------------------------------------------------------ //! Callback method. //------------------------------------------------------------------------ void HandleResponse( XRootDStatus *status, AnyObject *response ) { std::unique_ptr delst( status ); std::unique_ptr delrsp( response ); Response *resp = GetResponse( status, response ); task( *status, *resp ); } private: //------------------------------------------------------------------------ //! user defined task //------------------------------------------------------------------------ std::packaged_task task; }; //---------------------------------------------------------------------------- //! Packaged Task wrapper, specialization for requests that have no response //! except for status. //! //! @arg Response : type of response returned by the server //! @arg Return : type of the value returned by the task //---------------------------------------------------------------------------- template class TaskWrapper: public ResponseHandler { public: //------------------------------------------------------------------------ //! Constructor. // //! @param task : a std::packaged_task //------------------------------------------------------------------------ TaskWrapper( std::packaged_task && task ) : task( std::move( task ) ) { } //------------------------------------------------------------------------ //! Callback method. //------------------------------------------------------------------------ void HandleResponse( XRootDStatus *status, AnyObject *response ) { std::unique_ptr delst( status ); std::unique_ptr delrsp( response ); task( *status ); } private: //------------------------------------------------------------------------ //! user defined task //------------------------------------------------------------------------ std::packaged_task task; }; //---------------------------------------------------------------------------- //! Lambda wrapper //---------------------------------------------------------------------------- class ExOpenFuncWrapper: public ResponseHandler { public: //------------------------------------------------------------------------ //! Constructor. // //! @param func : function, functor or lambda (2 arguments) //------------------------------------------------------------------------ ExOpenFuncWrapper( const Ctx &f, std::function handleFunction ) : f( f ), fun( [handleFunction]( XRootDStatus &s, StatInfo &i, HostList& ){ handleFunction( s, i ); } ) { } //------------------------------------------------------------------------ //! Constructor. // //! @param func : function, functor or lambda (3 arguments) //------------------------------------------------------------------------ ExOpenFuncWrapper( const Ctx &f, std::function handleFunction ) : f( f ), fun( handleFunction ) { } //------------------------------------------------------------------------ //! Callback method. //------------------------------------------------------------------------ void HandleResponseWithHosts( XRootDStatus *status, AnyObject *response, HostList *hostList ) { delete response; std::unique_ptr delst( status ); std::unique_ptr delrsp; std::unique_ptr delhl; StatInfo *info = nullptr; if( status->IsOK() ) { XRootDStatus st = f->Stat( false, info ); delrsp.reset( info ); } else info = &NullRef::value; fun( *status, *info, *hostList ); } private: Ctx f; //------------------------------------------------------------------------ //! user defined function, functor or lambda //------------------------------------------------------------------------ std::function fun; }; //---------------------------------------------------------------------------- //! Pipeline exception, wrapps an XRootDStatus //---------------------------------------------------------------------------- class PipelineException : public std::exception { public: //------------------------------------------------------------------------ //! Constructor from XRootDStatus //------------------------------------------------------------------------ PipelineException( const XRootDStatus &error ) : error( error ), strerr( error.ToString() ) { } //------------------------------------------------------------------------ //! Copy constructor. //------------------------------------------------------------------------ PipelineException( const PipelineException &ex ) : error( ex.error ), strerr( ex.error.ToString() ) { } //------------------------------------------------------------------------ //! Assigment operator //------------------------------------------------------------------------ PipelineException& operator=( const PipelineException &ex ) { error = ex.error; strerr = ex.strerr; return *this; } //------------------------------------------------------------------------ //! inherited from std::exception //------------------------------------------------------------------------ const char* what() const noexcept { return strerr.c_str(); } //------------------------------------------------------------------------ //! @return : the XRootDStatus //------------------------------------------------------------------------ const XRootDStatus& GetError() const { return error; } private: //------------------------------------------------------------------------ //! the XRootDStatus associated with this exception //------------------------------------------------------------------------ XRootDStatus error; std::string strerr; }; //---------------------------------------------------------------------------- //! A wrapper handler for a std::promise / std::future. //! //! @arg Response : response type //---------------------------------------------------------------------------- template class FutureWrapperBase : public ResponseHandler { public: //------------------------------------------------------------------------ //! Constructor, initializes the std::future argument from its //! own std::promise //! //! @param ftr : the future to be linked with this handler //------------------------------------------------------------------------ FutureWrapperBase( std::future &ftr ) : fulfilled( false ) { ftr = prms.get_future(); } //------------------------------------------------------------------------ //! Destructor //------------------------------------------------------------------------ virtual ~FutureWrapperBase() { if( !fulfilled ) SetException( XRootDStatus( stError, errPipelineFailed ) ); } protected: //------------------------------------------------------------------------ //! Set exception in the std::promise / std::future //! //! @param err : the error //------------------------------------------------------------------------ inline void SetException( const XRootDStatus &err ) { std::exception_ptr ex = std::make_exception_ptr( PipelineException( err ) ); prms.set_exception( ex ); fulfilled = true; } //------------------------------------------------------------------------ //! promise that corresponds to the future //------------------------------------------------------------------------ std::promise prms; bool fulfilled; }; //---------------------------------------------------------------------------- //! A wrapper handler for a std::promise / std::future. //! //! @arg Response : response type //---------------------------------------------------------------------------- template class FutureWrapper : public FutureWrapperBase { public: //------------------------------------------------------------------------ //! Constructor, @see FutureWrapperBase //! //! @param ftr : the future to be linked with this handler //------------------------------------------------------------------------ FutureWrapper( std::future &ftr ) : FutureWrapperBase( ftr ) { } //------------------------------------------------------------------------ //! Callback method. //------------------------------------------------------------------------ void HandleResponse( XRootDStatus *status, AnyObject *response ) { std::unique_ptr delst( status ); std::unique_ptr delrsp( response ); if( status->IsOK() ) { Response *resp = GetResponse( response ); if( resp == &NullRef::value ) this->SetException( XRootDStatus( stError, errInternal ) ); else { this->prms.set_value( std::move( *resp ) ); this->fulfilled = true; } } else this->SetException( *status ); } }; //---------------------------------------------------------------------------- //! A wrapper handler for a std::promise / std::future, overload for void type //---------------------------------------------------------------------------- template<> class FutureWrapper : public FutureWrapperBase { public: //------------------------------------------------------------------------ //! Constructor, @see FutureWrapperBase //! //! @param ftr : the future to be linked with this handler //------------------------------------------------------------------------ FutureWrapper( std::future &ftr ) : FutureWrapperBase( ftr ) { } //------------------------------------------------------------------------ //! Callback method. //------------------------------------------------------------------------ void HandleResponse( XRootDStatus *status, AnyObject *response ) { std::unique_ptr delst( status ); std::unique_ptr delrsp( response ); if( status->IsOK() ) { prms.set_value(); fulfilled = true; } else SetException( *status ); } }; //---------------------------------------------------------------------------- //! Wrapper class for raw response handler (ResponseHandler). //---------------------------------------------------------------------------- class RawWrapper : public ResponseHandler { public: //------------------------------------------------------------------------ //! Constructor //! //! @param handler : the actual operation handler //------------------------------------------------------------------------ RawWrapper( ResponseHandler *handler ) : handler( handler ) { } //------------------------------------------------------------------------ //! Callback method (@see ResponseHandler) //! //! Note: does not delete itself because it is assumed that it is owned //! by the PipelineHandler (@see PipelineHandler) //------------------------------------------------------------------------ virtual void HandleResponseWithHosts( XRootDStatus *status, AnyObject *response, HostList *hostList ) { handler->HandleResponseWithHosts( status, response, hostList ); } private: //------------------------------------------------------------------------ //! The actual operation handler (we don't own the pointer) //------------------------------------------------------------------------ ResponseHandler *handler; }; //---------------------------------------------------------------------------- //! A base class for factories, creates ForwardingHandlers from //! ResponseHandler*, ResponseHandler& and std::future //! //! @arg Response : response type //---------------------------------------------------------------------------- template struct RespBase { //------------------------------------------------------------------------ //! A factory method, simply forwards the given handler //! //! @param hdlr : the ResponseHandler that should be wrapped //! @return : a ForwardingHandler instance //------------------------------------------------------------------------ inline static ResponseHandler* Create( ResponseHandler *hdlr ) { return new RawWrapper( hdlr ); } //------------------------------------------------------------------------ //! A factory method, simply forwards the given handler //! //! @param hdlr : the ResponseHandler that should be wrapped //! @return : a ForwardingHandler instance //------------------------------------------------------------------------ inline static ResponseHandler* Create( ResponseHandler &hdlr ) { return new RawWrapper( &hdlr ); } //------------------------------------------------------------------------ //! A factory method //! //! @arg Response : response type //! @param ftr : the std::future that should be wrapped //------------------------------------------------------------------------ inline static ResponseHandler* Create( std::future &ftr ) { return new FutureWrapper( ftr ); } }; //---------------------------------------------------------------------------- //! Factory class, creates ForwardingHandler from std::function, in addition //! to what RespBase provides (@see RespBase) //! //! @arg Response : response type //---------------------------------------------------------------------------- template struct Resp: RespBase { //------------------------------------------------------------------------ //! A factory method //! //! @param func : the function/functor/lambda that should be wrapped //! @return : FunctionWrapper instance //------------------------------------------------------------------------ inline static ResponseHandler* Create( std::function func ) { return new FunctionWrapper( func ); } //------------------------------------------------------------------------ //! A factory method //! //! @param func : the function/functor/lambda that should be wrapped //! @return : FunctionWrapper instance //------------------------------------------------------------------------ inline static ResponseHandler* Create( std::function func ) { return new FunctionWrapper( func ); } //------------------------------------------------------------------------ //! A factory method //! //! @param task : the task that should be wrapped //! @return : TaskWrapper instance //------------------------------------------------------------------------ template inline static ResponseHandler* Create( std::packaged_task &task ) { return new TaskWrapper( std::move( task ) ); } //------------------------------------------------------------------------ //! Make the Create overloads from RespBase visible //------------------------------------------------------------------------ using RespBase::Create; }; //---------------------------------------------------------------------------- //! Factory class, overloads Resp for void type //! //! @arg Response : response type //---------------------------------------------------------------------------- template<> struct Resp: RespBase { //------------------------------------------------------------------------ //! A factory method //! //! @param func : the function/functor/lambda that should be wrapped //! @return : SimpleFunctionWrapper instance //------------------------------------------------------------------------ inline static ResponseHandler* Create( std::function func ) { return new FunctionWrapper( func ); } //------------------------------------------------------------------------ //! A factory method //! //! @param func : the function/functor/lambda that should be wrapped //! @return : SimpleFunctionWrapper instance //------------------------------------------------------------------------ inline static ResponseHandler* Create( std::function func ) { return new FunctionWrapper( func ); } //------------------------------------------------------------------------ //! A factory method //! //! @param task : the task that should be wrapped //! @return : TaskWrapper instance //------------------------------------------------------------------------ template inline static ResponseHandler* Create( std::packaged_task &task ) { return new TaskWrapper( std::move( task ) ); } //------------------------------------------------------------------------ //! Make the Create overloads from RespBase visible //------------------------------------------------------------------------ using RespBase::Create; }; } #endif // __XRD_CL_OPERATIONS_HANDLERS_HH__ xrootd-5.6.9/src/XrdCl/XrdClOperationTimeout.hh000066400000000000000000000020521457266313600214560ustar00rootroot00000000000000/* * XrdClOperationTimeout.hh * * Created on: 4 Nov 2020 * Author: simonm */ #ifndef SRC_XRDCL_XRDCLOPERATIONTIMEOUT_HH_ #define SRC_XRDCL_XRDCLOPERATIONTIMEOUT_HH_ #include #include #include namespace XrdCl { class operation_expired : public std::exception {}; class Timeout { public: Timeout(): timeout( 0 ), start( 0 ) { } Timeout( uint16_t timeout ): timeout( timeout ), start( time( 0 ) ) { } Timeout& operator=( const Timeout &to ) { timeout = to.timeout; start = to.start; return *this; } Timeout( const Timeout &to ) : timeout( to.timeout ), start( to.start ) { } operator uint16_t() const { if( !timeout ) return 0; time_t elapsed = time( 0 ) - start; if( timeout < elapsed) throw operation_expired(); return timeout - elapsed; } private: uint16_t timeout; time_t start; }; } #endif /* SRC_XRDCL_XRDCLOPERATIONTIMEOUT_HH_ */ xrootd-5.6.9/src/XrdCl/XrdClOperations.cc000066400000000000000000000233431457266313600202660ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2017 by European Organization for Nuclear Research (CERN) // Author: Krzysztof Jamrog , // Michal Simon //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #include #include #include "XrdCl/XrdClOperations.hh" #include "XrdCl/XrdClLog.hh" #include "XrdCl/XrdClDefaultEnv.hh" #include "XrdCl/XrdClConstants.hh" namespace { //---------------------------------------------------------------------------- //! An exception type used to (force) stop a pipeline //---------------------------------------------------------------------------- struct StopPipeline { StopPipeline( const XrdCl::XRootDStatus &status ) : status( status ) { } XrdCl::XRootDStatus status; }; //---------------------------------------------------------------------------- //! An exception type used to repeat an operation //---------------------------------------------------------------------------- struct RepeatOpeation { }; //---------------------------------------------------------------------------- //! An exception type used to replace an operation //---------------------------------------------------------------------------- struct ReplaceOperation { ReplaceOperation( XrdCl::Operation &&opr ) : opr( opr.ToHandled() ) { } std::unique_ptr> opr; }; //---------------------------------------------------------------------------- //! An exception type used to replace whole pipeline //---------------------------------------------------------------------------- struct ReplacePipeline { ReplacePipeline( XrdCl::Pipeline p ) : pipeline( std::move( p ) ) { } XrdCl::Pipeline pipeline; }; //---------------------------------------------------------------------------- //! An exception type used to ignore a failed operation //---------------------------------------------------------------------------- struct IgnoreError { }; } namespace XrdCl { //---------------------------------------------------------------------------- // OperationHandler Constructor. //---------------------------------------------------------------------------- PipelineHandler::PipelineHandler( ResponseHandler *handler ) : responseHandler( handler ) { } //---------------------------------------------------------------------------- // OperationHandler::AddOperation //---------------------------------------------------------------------------- void PipelineHandler::AddOperation( Operation *operation ) { if( nextOperation ) { nextOperation->AddOperation( operation ); } else { nextOperation.reset( operation ); } } //---------------------------------------------------------------------------- // OperationHandler::HandleResponseImpl //---------------------------------------------------------------------------- void PipelineHandler::HandleResponseImpl( XRootDStatus *status, AnyObject *response, HostList *hostList ) { std::unique_ptr myself( this ); // We need to copy status as original status object is destroyed in // HandleResponse function XRootDStatus st( *status ); if( responseHandler ) { try { responseHandler->HandleResponseWithHosts( status, response, hostList ); } catch( const StopPipeline &stop ) { if( final ) final( stop.status ); prms.set_value( stop.status ); return; } catch( const RepeatOpeation &repeat ) { Operation *opr = currentOperation.release(); opr->handler.reset( myself.release() ); opr->Run( timeout, std::move( prms ), std::move( final ) ); return; } catch( ReplaceOperation &replace ) { Operation *opr = replace.opr.release(); opr->handler.reset( myself.release() ); opr->Run( timeout, std::move( prms ), std::move( final ) ); return; } catch( ReplacePipeline &replace ) { Pipeline p = std::move( replace.pipeline ); Operation *opr = p.operation.release(); opr->Run( timeout, std::move( prms ), std::move( final ) ); return; } catch( const IgnoreError &ignore ) { st = XRootDStatus(); } } else dealloc( status, response, hostList ); if( !st.IsOK() || !nextOperation ) { if( final ) final( st ); prms.set_value( st ); return; } Operation *opr = nextOperation.release(); opr->Run( timeout, std::move( prms ), std::move( final ) ); } //---------------------------------------------------------------------------- // OperationHandler::HandleResponseWithHosts //---------------------------------------------------------------------------- void PipelineHandler::HandleResponseWithHosts( XRootDStatus *status, AnyObject *response, HostList *hostList ) { HandleResponseImpl( status, response, hostList ); } //---------------------------------------------------------------------------- // OperationHandler::HandleResponse //---------------------------------------------------------------------------- void PipelineHandler::HandleResponse( XRootDStatus *status, AnyObject *response ) { HandleResponseImpl( status, response ); } //---------------------------------------------------------------------------- // OperationHandler::AssignToWorkflow //---------------------------------------------------------------------------- void PipelineHandler::Assign( const Timeout &t, std::promise p, std::function f, Operation *opr ) { timeout = t; prms = std::move( p ); if( !final ) final = std::move( f ); else if( f ) { auto f1 = std::move( final ); final = [f1, f]( const XRootDStatus &st ){ f1( st ); f( st ); }; } currentOperation.reset( opr ); } //------------------------------------------------------------------------ // Assign the finalization routine //------------------------------------------------------------------------ void PipelineHandler::Assign( std::function f ) { final = std::move( f ); } //------------------------------------------------------------------------ // Called by a pipeline on the handler of its first operation before Run //------------------------------------------------------------------------ void PipelineHandler::PreparePipelineStart() { // Move any final-function from the handler of the last operaiton to the // first. It will be moved along the pipeline of handlers while the // pipeline is run. if( final || !nextOperation ) return; PipelineHandler *last = nextOperation->handler.get(); while( last ) { Operation *nextop = last->nextOperation.get(); if( !nextop ) break; last = nextop->handler.get(); } if( last ) { // swap-then-move rather than only move as we need to guarantee that // last->final is left without target. std::function f; f.swap( last->final ); Assign( std::move( f ) ); } } //------------------------------------------------------------------------ // Stop the current pipeline //------------------------------------------------------------------------ void Pipeline::Stop( const XRootDStatus &status ) { throw StopPipeline( status ); } //------------------------------------------------------------------------ // Repeat current operation //------------------------------------------------------------------------ void Pipeline::Repeat() { throw RepeatOpeation(); } //------------------------------------------------------------------------ // Replace current operation //------------------------------------------------------------------------ void Pipeline::Replace( Operation &&opr ) { throw ReplaceOperation( std::move( opr ) ); } //------------------------------------------------------------------------ // Replace with pipeline //------------------------------------------------------------------------ void Pipeline::Replace( Pipeline p ) { throw ReplacePipeline( std::move( p ) ); } //------------------------------------------------------------------------ // Ignore error and proceed with the pipeline //------------------------------------------------------------------------ void Pipeline::Ignore() { throw IgnoreError(); } } xrootd-5.6.9/src/XrdCl/XrdClOperations.hh000066400000000000000000000747301457266313600203060ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2017 by European Organization for Nuclear Research (CERN) // Author: Krzysztof Jamrog , // Michal Simon //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #ifndef __XRD_CL_OPERATIONS_HH__ #define __XRD_CL_OPERATIONS_HH__ #include #include #include #include #include #include "XrdCl/XrdClXRootDResponses.hh" #include "XrdCl/XrdClOperationHandlers.hh" #include "XrdCl/XrdClArg.hh" #include "XrdCl/XrdClOperationTimeout.hh" #include "XrdCl/XrdClFinalOperation.hh" #include "XrdSys/XrdSysPthread.hh" #include "XrdCl/XrdClResponseJob.hh" #include "XrdCl/XrdClJobManager.hh" #include "XrdCl/XrdClPostMaster.hh" #include "XrdCl/XrdClDefaultEnv.hh" namespace XrdCl { template class Operation; class Pipeline; //---------------------------------------------------------------------------- //! Type of the recovery function to be provided by the user //---------------------------------------------------------------------------- typedef std::function*(const XRootDStatus&)> rcvry_func; //---------------------------------------------------------------------------- //! Wrapper for ResponseHandler, used only internally to run next operation //! after previous one is finished //---------------------------------------------------------------------------- class PipelineHandler: public ResponseHandler { template friend class Operation; public: //------------------------------------------------------------------------ //! Constructor. //! //! @param handler : the handler of our operation //------------------------------------------------------------------------ PipelineHandler( ResponseHandler *handler ); //------------------------------------------------------------------------ //! Default Constructor. //------------------------------------------------------------------------ PipelineHandler() { } //------------------------------------------------------------------------ //! Callback function. //------------------------------------------------------------------------ void HandleResponseWithHosts( XRootDStatus *status, AnyObject *response, HostList *hostList ); //------------------------------------------------------------------------ //! Callback function. //------------------------------------------------------------------------ void HandleResponse( XRootDStatus *status, AnyObject *response ); //------------------------------------------------------------------------ //! Destructor. //------------------------------------------------------------------------ ~PipelineHandler() { } //------------------------------------------------------------------------ //! Add new operation to the pipeline //! //! @param operation : operation to add //------------------------------------------------------------------------ void AddOperation( Operation *operation ); //------------------------------------------------------------------------ //! Set workflow to this and all next handlers. In the last handler //! it is used to finish workflow execution //! //! @param prms : a promis that the pipeline will have a result //! @param final : a callable that should be called at the end of //! pipeline //------------------------------------------------------------------------ void Assign( const Timeout &timeout, std::promise prms, std::function final, Operation *opr ); //------------------------------------------------------------------------ //! Assign the finalization routine //------------------------------------------------------------------------ void Assign( std::function final ); //------------------------------------------------------------------------ //! Called by a pipeline on the handler of its first operation before Run //------------------------------------------------------------------------ void PreparePipelineStart(); private: //------------------------------------------------------------------------ //! Callback function implementation; //------------------------------------------------------------------------ void HandleResponseImpl( XRootDStatus *status, AnyObject *response, HostList *hostList = nullptr ); inline void dealloc( XRootDStatus *status, AnyObject *response, HostList *hostList ) { delete status; delete response; delete hostList; } //------------------------------------------------------------------------ //! The handler of our operation //------------------------------------------------------------------------ std::unique_ptr responseHandler; //------------------------------------------------------------------------ //! The operation the handler is assigned to //------------------------------------------------------------------------ std::unique_ptr> currentOperation; //------------------------------------------------------------------------ //! Next operation in the pipeline //------------------------------------------------------------------------ std::unique_ptr> nextOperation; //------------------------------------------------------------------------ //! Pipeline timeout //------------------------------------------------------------------------ Timeout timeout; //------------------------------------------------------------------------ //! The promise that there will be a result (traveling along the pipeline) //------------------------------------------------------------------------ std::promise prms; //------------------------------------------------------------------------ //! The lambda/function/functor that should be called at the end of the //! pipeline (traveling along the pipeline) //------------------------------------------------------------------------ std::function final; }; //---------------------------------------------------------------------------- //! Operation template. An Operation is a once-use-only object - once executed //! by a Workflow engine it is invalidated. Also if used as an argument for //! >> or | the original object gets invalidated. //! //! @arg HasHndl : true if operation has a handler, false otherwise //---------------------------------------------------------------------------- template class Operation { // Declare friendship between templates template friend class Operation; friend std::future Async( Pipeline, uint16_t ); friend class Pipeline; friend class PipelineHandler; public: //------------------------------------------------------------------------ //! Constructor //------------------------------------------------------------------------ Operation() : valid( true ) { } //------------------------------------------------------------------------ //! Move constructor between template instances. //------------------------------------------------------------------------ template Operation( Operation && op ) : handler( std::move( op.handler ) ), valid( true ) { if( !op.valid ) throw std::invalid_argument( "Cannot construct " "Operation from an invalid Operation!" ); op.valid = false; } //------------------------------------------------------------------------ //! Destructor //------------------------------------------------------------------------ virtual ~Operation() { } //------------------------------------------------------------------------ //! Name of the operation. //------------------------------------------------------------------------ virtual std::string ToString() = 0; //------------------------------------------------------------------------ //! Move current object into newly allocated instance //! //! @return : the new instance //------------------------------------------------------------------------ virtual Operation* Move() = 0; //------------------------------------------------------------------------ //! Move current object into newly allocated instance, and convert //! it into 'handled' operation. //! //! @return : the new instance //------------------------------------------------------------------------ virtual Operation* ToHandled() = 0; protected: //------------------------------------------------------------------------ //! Run operation //! //! @param prms : the promise that we will have a result //! @param final : the object to call at the end of pipeline //------------------------------------------------------------------------ void Run( Timeout timeout, std::promise prms, std::function final ) { static_assert(HasHndl, "Only an operation that has a handler can be assigned to workflow"); handler->Assign( timeout, std::move( prms ), std::move( final ), this ); PipelineHandler *h = handler.release(); XRootDStatus st; try { st = RunImpl( h, timeout ); } catch( const operation_expired& ex ) { st = XRootDStatus( stError, errOperationExpired ); } catch( const PipelineException& ex ) // probably not needed { st = ex.GetError(); } catch( const std::exception& ex ) { st = XRootDStatus( stError, errInternal, 0, ex.what() ); } if( !st.IsOK() ){ ResponseJob *job = new ResponseJob(h, new XRootDStatus(st), 0, nullptr); DefaultEnv::GetPostMaster()->GetJobManager()->QueueJob(job); } } //------------------------------------------------------------------------ //! Run the actual operation //! //! @param params : container with parameters forwarded from //! previous operation //! @return : status of the operation //------------------------------------------------------------------------ virtual XRootDStatus RunImpl( PipelineHandler *handler, uint16_t timeout ) = 0; //------------------------------------------------------------------------ //! Add next operation in the pipeline //! //! @param op : operation to add //------------------------------------------------------------------------ void AddOperation( Operation *op ) { if( handler ) handler->AddOperation( op ); } //------------------------------------------------------------------------ //! Operation handler //------------------------------------------------------------------------ std::unique_ptr handler; //------------------------------------------------------------------------ //! Flag indicating if it is a valid object //------------------------------------------------------------------------ bool valid; }; //---------------------------------------------------------------------------- //! A wrapper around operation pipeline. A Pipeline is a once-use-only //! object - once executed by a Workflow engine it is invalidated. //! //! Takes ownership of given operation pipeline (which is in most would //! be a temporary object) //---------------------------------------------------------------------------- class Pipeline { template friend class ParallelOperation; friend std::future Async( Pipeline, uint16_t ); friend class PipelineHandler; public: //------------------------------------------------------------------------ //! Default constructor //------------------------------------------------------------------------ Pipeline() { } //------------------------------------------------------------------------ //! Constructor //------------------------------------------------------------------------ Pipeline( Operation *op ) : operation( op->Move() ) { } //------------------------------------------------------------------------ //! Constructor //------------------------------------------------------------------------ Pipeline( Operation &op ) : operation( op.Move() ) { } //------------------------------------------------------------------------ //! Constructor //------------------------------------------------------------------------ Pipeline( Operation &&op ) : operation( op.Move() ) { } Pipeline( Operation *op ) : operation( op->ToHandled() ) { } //------------------------------------------------------------------------ //! Constructor //------------------------------------------------------------------------ Pipeline( Operation &op ) : operation( op.ToHandled() ) { } //------------------------------------------------------------------------ //! Constructor //------------------------------------------------------------------------ Pipeline( Operation &&op ) : operation( op.ToHandled() ) { } Pipeline( Pipeline &&pipe ) : operation( std::move( pipe.operation ) ) { } //------------------------------------------------------------------------ //! Constructor //------------------------------------------------------------------------ Pipeline& operator=( Pipeline &&pipe ) { operation = std::move( pipe.operation ); return *this; } //------------------------------------------------------------------------ //! Extend pipeline //------------------------------------------------------------------------ Pipeline& operator|=( Operation&& op ) { operation->AddOperation( op.Move() ); return *this; } //------------------------------------------------------------------------ //! Extend pipeline //------------------------------------------------------------------------ Pipeline& operator|=( Operation&& op ) { operation->AddOperation( op.ToHandled() ); return *this; } //------------------------------------------------------------------------ //! Conversion to Operation //! //! @throws : std::logic_error if pipeline is invalid //------------------------------------------------------------------------ operator Operation&() { if( !bool( operation ) ) throw std::logic_error( "Invalid pipeline." ); return *operation.get(); } //------------------------------------------------------------------------ //! Conversion to boolean //! //! @return : true if it's a valid pipeline, false otherwise //------------------------------------------------------------------------ operator bool() { return bool( operation ); } //------------------------------------------------------------------------ //! Stop the current pipeline //! //! @param status : the final status for the pipeline //------------------------------------------------------------------------ static void Stop( const XRootDStatus &status = XrdCl::XRootDStatus() ); //------------------------------------------------------------------------ //! Repeat current operation //------------------------------------------------------------------------ static void Repeat(); //------------------------------------------------------------------------ //! Replace current operation //------------------------------------------------------------------------ static void Replace( Operation &&opr ); //------------------------------------------------------------------------ //! Replace with pipeline //------------------------------------------------------------------------ static void Replace( Pipeline p ); //------------------------------------------------------------------------ //! Ignore error and proceed with the pipeline //------------------------------------------------------------------------ static void Ignore(); private: //------------------------------------------------------------------------ //! Member access declaration, provides access to the underlying //! operation. //! //! @return : pointer to the underlying //------------------------------------------------------------------------ Operation* operator->() { return operation.get(); } //------------------------------------------------------------------------ //! Schedules the underlying pipeline for execution. //! //! @param timeout : pipeline timeout value //! @param final : to be called at the end of the pipeline //------------------------------------------------------------------------ void Run( Timeout timeout, std::function final = nullptr ) { if( ftr.valid() ) throw std::logic_error( "Pipeline is already running!" ); // a promise that the pipe will have a result std::promise prms; ftr = prms.get_future(); if( !operation ) std::logic_error( "Empty pipeline!" ); Operation *opr = operation.release(); PipelineHandler *h = opr->handler.get(); if( h ) h->PreparePipelineStart(); opr->Run( timeout, std::move( prms ), std::move( final ) ); } //------------------------------------------------------------------------ //! First operation in the pipeline //------------------------------------------------------------------------ std::unique_ptr> operation; //------------------------------------------------------------------------ //! The future result of the pipeline //------------------------------------------------------------------------ std::future ftr; }; //---------------------------------------------------------------------------- //! Helper function, schedules execution of given pipeline //! //! @param pipeline : the pipeline to be executed //! @param timeout : the pipeline timeout //! //! @return : future status of the operation //---------------------------------------------------------------------------- inline std::future Async( Pipeline pipeline, uint16_t timeout = 0 ) { pipeline.Run( timeout ); return std::move( pipeline.ftr ); } //---------------------------------------------------------------------------- //! Helper function, schedules execution of given pipeline and waits for //! the status //! //! @param pipeline : the pipeline to be executed //! @param timeout : the pipeline timeout //! //! @return : status of the operation //---------------------------------------------------------------------------- inline XRootDStatus WaitFor( Pipeline pipeline, uint16_t timeout = 0 ) { return Async( std::move( pipeline ), timeout ).get(); } //---------------------------------------------------------------------------- //! Concrete Operation template //! Defines | and >> operator as well as operation arguments. //! //! @arg Derived : the class that derives from this template (CRTP) //! @arg HasHndl : true if operation has a handler, false otherwise //! @arg Args : operation arguments //---------------------------------------------------------------------------- template class Derived, bool HasHndl, typename HdlrFactory, typename ... Args> class ConcreteOperation: public Operation { template class, bool, typename, typename ...> friend class ConcreteOperation; public: //------------------------------------------------------------------------ //! Constructor //! //! @param args : operation arguments //------------------------------------------------------------------------ ConcreteOperation( Args&&... args ) : args( std::tuple( std::move( args )... ) ), timeout( 0 ) { static_assert( !HasHndl, "It is only possible to construct operation without handler" ); } //------------------------------------------------------------------------ //! Move constructor from other states //! //! @arg from : state from which the object is being converted //! //! @param op : the object that is being converted //------------------------------------------------------------------------ template ConcreteOperation( ConcreteOperation && op ) : Operation( std::move( op ) ), args( std::move( op.args ) ), timeout( 0 ) { } //------------------------------------------------------------------------ //! Adds ResponseHandler/function/functor/lambda/future handler for //! the operation. //! //! Note: due to reference collapsing this covers both l-value and //! r-value references. //! //! @param hdlr : function/functor/lambda //------------------------------------------------------------------------ template Derived operator>>( Hdlr &&hdlr ) { return this->StreamImpl( HdlrFactory::Create( hdlr ) ); } //------------------------------------------------------------------------ //! Creates a pipeline of 2 or more operations //! //! @param op : operation to add //! //! @return : handled operation //------------------------------------------------------------------------ Derived operator|( Operation &op ) { return PipeImpl( *this, op ); } //------------------------------------------------------------------------ //! Creates a pipeline of 2 or more operations //! //! @param op : operation to add //! //! @return : handled operation //------------------------------------------------------------------------ Derived operator|( Operation &&op ) { return PipeImpl( *this, op ); } //------------------------------------------------------------------------ //! Creates a pipeline of 2 or more operations //! //! @param op operation to add //! //! @return handled operation //------------------------------------------------------------------------ Derived operator|( Operation &op ) { return PipeImpl( *this, op ); } //------------------------------------------------------------------------ //! Creates a pipeline of 2 or more operations //! //! @param op : operation to add //! //! @return : handled operation //------------------------------------------------------------------------ Derived operator|( Operation &&op ) { return PipeImpl( *this, op ); } //------------------------------------------------------------------------ //! Adds a final operation to the pipeline //------------------------------------------------------------------------ Derived operator|( FinalOperation &&fo ) { AllocHandler( *this ); this->handler->Assign( fo.final ); return this->template Transform(); } //------------------------------------------------------------------------ //! Move current object into newly allocated instance //! //! @return : the new instance //------------------------------------------------------------------------ inline Operation* Move() { Derived *me = static_cast*>( this ); return new Derived( std::move( *me ) ); } //------------------------------------------------------------------------ //! Transform operation to handled //! //! @return Operation& //------------------------------------------------------------------------ inline Operation* ToHandled() { this->handler.reset( new PipelineHandler() ); Derived *me = static_cast*>( this ); return new Derived( std::move( *me ) ); } //------------------------------------------------------------------------ //! Set operation timeout //------------------------------------------------------------------------ Derived Timeout( uint16_t timeout ) { this->timeout = timeout; Derived *me = static_cast*>( this ); return std::move( *me ); } protected: //------------------------------------------------------------------------ //! Transform into a new instance with desired state //! //! @return : new instance in the desired state //------------------------------------------------------------------------ template inline Derived Transform() { Derived *me = static_cast*>( this ); return Derived( std::move( *me ) ); } //------------------------------------------------------------------------ //! Implements operator>> functionality //! //! @param handler : handler to be added //! //! @return : return an instance of Derived //------------------------------------------------------------------------ inline Derived StreamImpl( ResponseHandler *handler ) { static_assert( !HasHndl, "Operator >> is available only for operation without handler" ); this->handler.reset( new PipelineHandler( handler ) ); return Transform(); } //------------------------------------------------------------------------ // Allocate handler if necessary //------------------------------------------------------------------------ inline static void AllocHandler( ConcreteOperation &me ) { // nothing to do } //------------------------------------------------------------------------ // Allocate handler if necessary //------------------------------------------------------------------------ inline static void AllocHandler( ConcreteOperation &me ) { me.handler.reset( new PipelineHandler() ); } //------------------------------------------------------------------------ //! Implements operator| functionality //! //! @param me : reference to myself (*this) //! @param op : reference to the other operation //! //! @return : move-copy of myself //------------------------------------------------------------------------ inline static Derived PipeImpl( ConcreteOperation &me, Operation &op ) { AllocHandler( me ); // if HasHndl is false allocate handler me.AddOperation( op.Move() ); return me.template Transform(); } //------------------------------------------------------------------------ //! Implements operator| functionality //! //! @param me : reference to myself (*this) //! @param op : reference to the other operation //! //! @return : move-copy of myself //------------------------------------------------------------------------ inline static Derived PipeImpl( ConcreteOperation &me, Operation &op ) { AllocHandler( me ); // if HasHndl is false allocate handler me.AddOperation( op.ToHandled() ); return me.template Transform(); } //------------------------------------------------------------------------ //! Operation arguments //------------------------------------------------------------------------ std::tuple args; //------------------------------------------------------------------------ //! Operation timeout //------------------------------------------------------------------------ uint16_t timeout; }; } #endif // __XRD_CL_OPERATIONS_HH__ xrootd-5.6.9/src/XrdCl/XrdClOptimizers.hh000066400000000000000000000024171457266313600203210ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2012 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . //------------------------------------------------------------------------------ #ifndef __XRD_CL_OPTIMIZERS_HH__ #define __XRD_CL_OPTIMIZERS_HH__ #ifdef __GNUC__ #define likely(x) __builtin_expect(!!(x), 1) #define unlikely(x) __builtin_expect(!!(x), 0) #else #define likely(x) x #define unlikely(x) x #endif #endif // __XRD_CL_OPTIMIZERS_HH__ xrootd-5.6.9/src/XrdCl/XrdClOptional.hh000066400000000000000000000145431457266313600177440ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2017 by European Organization for Nuclear Research (CERN) // Author: Michal Simon //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #ifndef __XRD_CL_OPTIONAL_HH__ #define __XRD_CL_OPTIONAL_HH__ #include namespace XrdCl { //---------------------------------------------------------------------------- //! none object for initializing empty Optional //---------------------------------------------------------------------------- static struct None{ } none; //---------------------------------------------------------------------------- //! The Optional class //! //! @arg T : type of the optional parameter //---------------------------------------------------------------------------- template class Optional { public: //------------------------------------------------------------------------ //! Constructor for value //------------------------------------------------------------------------ Optional( const T& t ) : optional( false ) { new( &memory.value ) T( t ); } //------------------------------------------------------------------------ //! Default constructor //------------------------------------------------------------------------ Optional( const None& n = none ) : optional( true ) { (void)n; } //------------------------------------------------------------------------ //! Copy constructor //------------------------------------------------------------------------ Optional( const Optional& opt ) : optional( opt.optional ) { if( !optional ) new( &memory.value ) T( opt.memory.value ); } //------------------------------------------------------------------------ //! Move constructor //------------------------------------------------------------------------ Optional( Optional && opt ) : optional( opt.optional ) { if( !optional ) new( &memory.value ) T( std::move( opt.memory.value ) ); } //------------------------------------------------------------------------ // Destructor //------------------------------------------------------------------------ ~Optional() { if( optional ) memory.value.~T(); } //------------------------------------------------------------------------ //! Copy assignment operator //------------------------------------------------------------------------ Optional& operator=( const Optional& opt ) { if( this != &opt ) { optional = opt.optional; if( !optional ) memory.value = opt.memory.value; } return *this; } //------------------------------------------------------------------------ //! Move assignment operator //------------------------------------------------------------------------ Optional& operator=( Optional&& opt ) { if( this != &opt ) { optional = opt.optional; if( !optional ) memory.value = std::move( opt.memory.value ); } return *this; } //------------------------------------------------------------------------ //! Conversion to boolean //------------------------------------------------------------------------ operator bool() const { return optional; } //------------------------------------------------------------------------ //! Dereference operator //------------------------------------------------------------------------ T& operator*() { return memory.value; } //------------------------------------------------------------------------ //! Dereference operator //------------------------------------------------------------------------ const T& operator*() const { return memory.value; } private: //------------------------------------------------------------------------ //! true if the value is optional, false otherwise //------------------------------------------------------------------------ bool optional; //------------------------------------------------------------------------ //! we use union as this is the only way to obtain memory with correct //! alignment and don't actually construct the object //------------------------------------------------------------------------ union Storage { //---------------------------------------------------------------------- //! value of the optional variable, if the variable is optional is //! remains uninitialized //---------------------------------------------------------------------- T value; //---------------------------------------------------------------------- //! Default constructor //---------------------------------------------------------------------- inline Storage(){ } //---------------------------------------------------------------------- // Destructor //---------------------------------------------------------------------- inline ~Storage(){ }; } memory; //> memory storage for the optional variable }; } #endif // __XRD_CL_OPTIONAL_HH__ xrootd-5.6.9/src/XrdCl/XrdClOutQueue.cc000066400000000000000000000124241457266313600177150ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2012 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . //------------------------------------------------------------------------------ #include "XrdCl/XrdClOutQueue.hh" #include "XrdCl/XrdClPostMasterInterfaces.hh" namespace XrdCl { //---------------------------------------------------------------------------- // Add a message to the back of the queue //---------------------------------------------------------------------------- void OutQueue::PushBack( Message *msg, MsgHandler *handler, time_t expires, bool stateful ) { pMessages.push_back( MsgHelper( msg, handler, expires, stateful ) ); } //---------------------------------------------------------------------------- // Add a message to the front the queue //---------------------------------------------------------------------------- void OutQueue::PushFront( Message *msg, MsgHandler *handler, time_t expires, bool stateful ) { pMessages.push_front( MsgHelper( msg, handler, expires, stateful ) ); } //---------------------------------------------------------------------------- //! Get a message from the front of the queue //---------------------------------------------------------------------------- Message *OutQueue::PopMessage( MsgHandler *&handler, time_t &expires, bool &stateful ) { if( pMessages.empty() ) return 0; MsgHelper m = pMessages.front(); handler = m.handler; expires = m.expires; stateful = m.stateful; pMessages.pop_front(); return m.msg; } //---------------------------------------------------------------------------- // Remove a message from the front //---------------------------------------------------------------------------- void OutQueue::PopFront() { pMessages.pop_front(); } //---------------------------------------------------------------------------- // Report status to all handlers //---------------------------------------------------------------------------- void OutQueue::Report( XRootDStatus status ) { MessageList::iterator it; for( it = pMessages.begin(); it != pMessages.end(); ++it ) it->handler->OnStatusReady( it->msg, status ); } //------------------------------------------------------------------------ // Return the size of the queue counting only the stateless messages //------------------------------------------------------------------------ uint64_t OutQueue::GetSizeStateless() const { uint64_t size = 0; MessageList::const_iterator it; for( it = pMessages.begin(); it != pMessages.end(); ++it ) if( !it->stateful ) ++size; return size; } //---------------------------------------------------------------------------- // Remove all the expired messages from the queue and put them in // this one //---------------------------------------------------------------------------- void OutQueue::GrabExpired( OutQueue &queue, time_t exp ) { MessageList::iterator it; for( it = queue.pMessages.begin(); it != queue.pMessages.end(); ) { if( it->expires > exp ) { ++it; continue; } pMessages.push_back( *it ); it = queue.pMessages.erase( it ); } } //---------------------------------------------------------------------------- // Remove all the stateful messages from the queue and put them in this // one //---------------------------------------------------------------------------- void OutQueue::GrabStateful( OutQueue &queue ) { MessageList::iterator it; for( it = queue.pMessages.begin(); it != queue.pMessages.end(); ) { if( !it->stateful ) { ++it; continue; } pMessages.push_back( *it ); it = queue.pMessages.erase( it ); } } //---------------------------------------------------------------------------- // Take all the items from the queue and put them in this one //---------------------------------------------------------------------------- void OutQueue::GrabItems( OutQueue &queue ) { MessageList::iterator it; for( it = queue.pMessages.begin(); it != queue.pMessages.end(); ++it ) pMessages.push_back( *it ); queue.pMessages.clear(); } } xrootd-5.6.9/src/XrdCl/XrdClOutQueue.hh000066400000000000000000000150631457266313600177310ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2012 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . //------------------------------------------------------------------------------ #ifndef __XRD_CL_OUT_QUEUE_HH__ #define __XRD_CL_OUT_QUEUE_HH__ #include #include #include "XrdCl/XrdClXRootDResponses.hh" namespace XrdCl { class Message; class MsgHandler; //---------------------------------------------------------------------------- //! A synchronized queue for the outgoing data //---------------------------------------------------------------------------- class OutQueue { public: //------------------------------------------------------------------------ //! Add a message to the back the queue //! //! @param msg message to be sent //! @param handler handler to be notified about the status of the //! operation //! @param expires timeout //! @param stateful if true a disconnection will cause an error and //! removing from the queue, otherwise sending //! wil be re-attempted //------------------------------------------------------------------------ void PushBack( Message *msg, MsgHandler *handler, time_t expires, bool stateful ); //------------------------------------------------------------------------ //! Add a message to the front the queue //! //! @param msg message to be sent //! @param handler handler to be notified about the status of the //! operation //! @param expires timeout //! @param stateful if true a disconnection will cause an error and //! removing from the queue, otherwise sending //! wil be re-attempted //------------------------------------------------------------------------ void PushFront( Message *msg, MsgHandler *handler, time_t expires, bool stateful ); //------------------------------------------------------------------------ //! Pop a message from the front of the queue //! //! @return 0 if there is no message message //------------------------------------------------------------------------ Message *PopMessage( MsgHandler *&handler, time_t &expires, bool &stateful ); //------------------------------------------------------------------------ //! Remove a message from the front //------------------------------------------------------------------------ void PopFront(); //------------------------------------------------------------------------ //! Report status to all the handlers //------------------------------------------------------------------------ void Report( XRootDStatus status ); //------------------------------------------------------------------------ //! Check if the queue is empty //------------------------------------------------------------------------ bool IsEmpty() const { return pMessages.empty(); } //------------------------------------------------------------------------ // Return the size of the queue //------------------------------------------------------------------------ uint64_t GetSize() const { return pMessages.size(); } //------------------------------------------------------------------------ //! Return the size of the queue counting only the stateless messages //------------------------------------------------------------------------ uint64_t GetSizeStateless() const; //------------------------------------------------------------------------ //! Remove all the expired messages from the queue and put them in //! this one //! //! @param queue queue to take the message from //! @param exp expiration timestamp //------------------------------------------------------------------------ void GrabExpired( OutQueue &queue, time_t exp = 0 ); //------------------------------------------------------------------------ //! Remove all the stateful messages from the queue and put them in this //! one //! //! @param queue the queue to take the messages from //------------------------------------------------------------------------ void GrabStateful( OutQueue &queue ); //------------------------------------------------------------------------ //! Take all the items from the queue and put them in this one //! //! @param queue queue to take the message //------------------------------------------------------------------------ void GrabItems( OutQueue &queue ); //------------------------------------------------------------------------ // Helper struct holding all the message data //------------------------------------------------------------------------ struct MsgHelper { MsgHelper( Message *m = nullptr, MsgHandler *h = nullptr, time_t r = 0, bool s = false ): msg( m ), handler( h ), expires( r ), stateful( s ) {} void Reset() { msg = 0; handler = 0; expires = 0; stateful = 0; } Message *msg; MsgHandler *handler; time_t expires; bool stateful; }; private: typedef std::list MessageList; MessageList pMessages; }; } #endif // __XRD_CL_OUT_QUEUE_HH__ xrootd-5.6.9/src/XrdCl/XrdClParallelOperation.hh000066400000000000000000000530251457266313600215720ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2017 by European Organization for Nuclear Research (CERN) // Author: Krzysztof Jamrog , // Michal Simon //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #ifndef __XRD_CL_PARALLELOPERATION_HH__ #define __XRD_CL_PARALLELOPERATION_HH__ #include "XrdCl/XrdClOperations.hh" #include "XrdCl/XrdClOperationHandlers.hh" #include "XrdCl/XrdClDefaultEnv.hh" #include "XrdCl/XrdClPostMaster.hh" #include "XrdCl/XrdClJobManager.hh" #include #include #include namespace XrdCl { //---------------------------------------------------------------------------- // Interface for different execution policies: // - all : all operations need to succeed in order for the parallel // operation to be successful // - any : just one of the operations needs to succeed in order for // the parallel operation to be successful // - some : n (user defined) operations need to succeed in order for // the parallel operation to be successful // - at least : at least n (user defined) operations need to succeed in // order for the parallel operation to be successful (the // user handler will be called only when all operations are // resolved) // // @param status : status returned by one of the aggregated operations // // @return : true if the status should be passed to the user handler, // false otherwise. //---------------------------------------------------------------------------- struct PolicyExecutor { virtual ~PolicyExecutor() { } virtual bool Examine( const XrdCl::XRootDStatus &status ) = 0; virtual XRootDStatus Result() = 0; }; //---------------------------------------------------------------------------- //! Parallel operations, allows to execute two or more pipelines in //! parallel. //! //! @arg state : describes current operation configuration state //! (@see Operation) //---------------------------------------------------------------------------- template class ParallelOperation: public ConcreteOperation> { template friend class ParallelOperation; public: //------------------------------------------------------------------------ //! Constructor: copy-move a ParallelOperation in different state //------------------------------------------------------------------------ template ParallelOperation( ParallelOperation &&obj ) : ConcreteOperation>( std::move( obj ) ), pipelines( std::move( obj.pipelines ) ), policy( std::move( obj.policy ) ) { } //------------------------------------------------------------------------ //! Constructor //! //! @arg Container : iterable container type //! //! @param container : iterable container with pipelines //------------------------------------------------------------------------ template ParallelOperation( Container &&container ) { static_assert( !HasHndl, "Constructor is available only operation without handler"); pipelines.reserve( container.size() ); auto begin = std::make_move_iterator( container.begin() ); auto end = std::make_move_iterator( container.end() ); std::copy( begin, end, std::back_inserter( pipelines ) ); container.clear(); // there's junk inside so we clear it } ~ParallelOperation() { } //------------------------------------------------------------------------ //! @return : operation name //------------------------------------------------------------------------ std::string ToString() { std::ostringstream oss; oss << "Parallel("; for( size_t i = 0; i < pipelines.size(); i++ ) { oss << pipelines[i]->ToString(); if( i + 1 != pipelines.size() ) { oss << " && "; } } oss << ")"; return oss.str(); } //------------------------------------------------------------------------ //! Set policy to `All` (default) //! //! All operations need to succeed in order for the parallel operation to //! be successful. //------------------------------------------------------------------------ ParallelOperation All() { policy.reset( new AllPolicy() ); return std::move( *this ); } //------------------------------------------------------------------------ //! Set policy to `Any` //! //! Just one of the operations needs to succeed in order for the parallel //! operation to be successful. //------------------------------------------------------------------------ ParallelOperation Any() { policy.reset( new AnyPolicy( pipelines.size() ) ); return std::move( *this ); } //------------------------------------------------------------------------ // Set policy to `Some` //! //! n (user defined) operations need to succeed in order for the parallel //! operation to be successful. //------------------------------------------------------------------------ ParallelOperation Some( size_t threshold ) { policy.reset( new SomePolicy( pipelines.size(), threshold ) ); return std::move( *this ); } //------------------------------------------------------------------------ //! Set policy to `At Least`. //! //! At least n (user defined) operations need to succeed in order for the //! parallel operation to be successful (the user handler will be called //! only when all operations are resolved). //------------------------------------------------------------------------ ParallelOperation AtLeast( size_t threshold ) { policy.reset( new AtLeastPolicy( pipelines.size(), threshold ) ); return std::move( *this ); } private: //------------------------------------------------------------------------ //! `All` policy implementation //! //! All operations need to succeed in order for the parallel operation to //! be successful. //------------------------------------------------------------------------ struct AllPolicy : public PolicyExecutor { bool Examine( const XrdCl::XRootDStatus &status ) { // keep the status in case this is the final result res = status; if( status.IsOK() ) return false; // we require all request to succeed return true; } XRootDStatus Result() { return res; } XRootDStatus res; }; //------------------------------------------------------------------------ //! `Any` policy implementation //! //! Just one of the operations needs to succeed in order for the parallel //! operation to be successful. //------------------------------------------------------------------------ struct AnyPolicy : public PolicyExecutor { AnyPolicy( size_t size) : cnt( size ) { } bool Examine( const XrdCl::XRootDStatus &status ) { // keep the status in case this is the final result res = status; // decrement the counter size_t nb = cnt.fetch_sub( 1, std::memory_order_relaxed ); // we require just one operation to be successful if( status.IsOK() ) return true; // lets see if this is the last one? if( nb == 1 ) return true; // we still have a chance there will be one that is successful return false; } XRootDStatus Result() { return res; } private: std::atomic cnt; XRootDStatus res; }; //------------------------------------------------------------------------ //! `Some` policy implementation //! //! n (user defined) operations need to succeed in order for the parallel //! operation to be successful. //------------------------------------------------------------------------ struct SomePolicy : PolicyExecutor { SomePolicy( size_t size, size_t threshold ) : failed( 0 ), succeeded( 0 ), threshold( threshold ), size( size ) { } bool Examine( const XrdCl::XRootDStatus &status ) { // keep the status in case this is the final result res = status; if( status.IsOK() ) { size_t s = succeeded.fetch_add( 1, std::memory_order_relaxed ); if( s + 1 == threshold ) return true; // we reached the threshold // we are not yet there return false; } size_t f = failed.fetch_add( 1, std::memory_order_relaxed ); // did we drop below the threshold if( f == size - threshold ) return true; // we still have a chance there will be enough of successful operations return false; } XRootDStatus Result() { return res; } private: std::atomic failed; std::atomic succeeded; const size_t threshold; const size_t size; XRootDStatus res; }; //------------------------------------------------------------------------ //! `At Least` policy implementation //! //! At least n (user defined) operations need to succeed in order for the //! parallel operation to be successful (the user handler will be called //! only when all operations are resolved). //------------------------------------------------------------------------ struct AtLeastPolicy : PolicyExecutor { AtLeastPolicy( size_t size, size_t threshold ) : pending_cnt( size ), failed_cnt( 0 ), failed_threshold( size - threshold ) { } bool Examine( const XrdCl::XRootDStatus &status ) { // update number of pending operations size_t pending = pending_cnt.fetch_sub( 1, std::memory_order_relaxed ) - 1; // although we might have the minimum to succeed we wait for the rest if( status.IsOK() ) return ( pending == 0 ); size_t nb = failed_cnt.fetch_add( 1, std::memory_order_relaxed ); if( nb == failed_threshold ) res = status; // we dropped below the threshold // if we still have to wait for pending operations return false, // otherwise all is done, return true return ( pending == 0 ); } XRootDStatus Result() { return res; } private: std::atomic pending_cnt; std::atomic failed_cnt; const size_t failed_threshold; XRootDStatus res; }; //------------------------------------------------------------------------ //! A wait barrier helper class //------------------------------------------------------------------------ struct barrier_t { barrier_t() : on( true ) { } void wait() { std::unique_lock lck( mtx ); if( on ) cv.wait( lck ); } void lift() { std::unique_lock lck( mtx ); on = false; cv.notify_all(); } private: std::condition_variable cv; std::mutex mtx; bool on; }; //------------------------------------------------------------------------ //! Helper class for handling the PipelineHandler of the //! ParallelOperation (RAII). //! //! Guarantees that the handler will be executed exactly once. //------------------------------------------------------------------------ struct Ctx { //---------------------------------------------------------------------- //! Constructor. //! //! @param handler : the PipelineHandler of the Parallel operation //---------------------------------------------------------------------- Ctx( PipelineHandler *handler, PolicyExecutor *policy ): handler( handler ), policy( policy ) { } //---------------------------------------------------------------------- //! Destructor. //---------------------------------------------------------------------- ~Ctx() { Handle( XRootDStatus() ); } //---------------------------------------------------------------------- //! Forwards the status to the PipelineHandler if the handler haven't //! been called yet. //! //! @param st : status //---------------------------------------------------------------------- inline void Examine( const XRootDStatus &st ) { if( policy->Examine( st ) ) Handle( policy->Result() ); } //---------------------------------------------------------------------- //! Forwards the status to the PipelineHandler if the handler haven't //! been called yet. //! //! @param st : status //--------------------------------------------------------------------- inline void Handle( const XRootDStatus &st ) { PipelineHandler* hdlr = handler.exchange( nullptr, std::memory_order_relaxed ); if( hdlr ) { barrier.wait(); hdlr->HandleResponse( new XRootDStatus( st ), nullptr ); } } //---------------------------------------------------------------------- //! PipelineHandler of the ParallelOperation //---------------------------------------------------------------------- std::atomic handler; //---------------------------------------------------------------------- //! Policy defining when the user handler should be called //---------------------------------------------------------------------- std::unique_ptr policy; //---------------------------------------------------------------------- //! wait barrier that assures handler is called only after RunImpl //! started all pipelines //---------------------------------------------------------------------- barrier_t barrier; }; //------------------------------------------------------------------------ //! The thread-pool job for schedule Ctx::Examine //------------------------------------------------------------------------ struct PipelineEnd : public Job { //---------------------------------------------------------------------- // Constructor //---------------------------------------------------------------------- PipelineEnd( std::shared_ptr &ctx, const XrdCl::XRootDStatus &st ) : ctx( ctx ), st( st ) { } //---------------------------------------------------------------------- // Run Ctx::Examine in the thread-pool //---------------------------------------------------------------------- void Run( void* ) { ctx->Examine( st ); delete this; } private: std::shared_ptr ctx; //< ParallelOperaion context XrdCl::XRootDStatus st; //< final status of the ParallelOperation }; //------------------------------------------------------------------------ //! Schedule Ctx::Examine to be executed in the client thread-pool //------------------------------------------------------------------------ inline static void Schedule( std::shared_ptr &ctx, const XrdCl::XRootDStatus &st) { XrdCl::JobManager *mgr = XrdCl::DefaultEnv::GetPostMaster()->GetJobManager(); PipelineEnd *end = new PipelineEnd( ctx, st ); mgr->QueueJob( end, nullptr ); } //------------------------------------------------------------------------ //! Run operation //! //! @param params : container with parameters forwarded from //! previous operation //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus RunImpl( PipelineHandler *handler, uint16_t pipelineTimeout ) { // make sure we have a valid policy for the parallel operation if( !policy ) policy.reset( new AllPolicy() ); std::shared_ptr ctx = std::make_shared( handler, policy.release() ); uint16_t timeout = pipelineTimeout < this->timeout ? pipelineTimeout : this->timeout; for( size_t i = 0; i < pipelines.size(); ++i ) { if( !pipelines[i] ) continue; pipelines[i].Run( timeout, [ctx]( const XRootDStatus &st ) mutable { Schedule( ctx, st ); } ); } ctx->barrier.lift(); return XRootDStatus(); } std::vector pipelines; std::unique_ptr policy; }; //---------------------------------------------------------------------------- //! Factory function for creating parallel operation from a vector //---------------------------------------------------------------------------- template inline ParallelOperation Parallel( Container &&container ) { return ParallelOperation( container ); } //---------------------------------------------------------------------------- //! Helper function for converting parameter pack into a vector //---------------------------------------------------------------------------- inline void PipesToVec( std::vector& ) { // base case } //---------------------------------------------------------------------------- // Declare PipesToVec (we need to do declare those functions ahead of // definitions, as they may call each other. //---------------------------------------------------------------------------- template inline void PipesToVec( std::vector &v, Operation &operation, Others&... others ); template inline void PipesToVec( std::vector &v, Operation &operation, Others&... others ); template inline void PipesToVec( std::vector &v, Pipeline &pipeline, Others&... others ); //---------------------------------------------------------------------------- // Define PipesToVec //---------------------------------------------------------------------------- template void PipesToVec( std::vector &v, Operation &operation, Others&... others ) { v.emplace_back( operation ); PipesToVec( v, others... ); } template void PipesToVec( std::vector &v, Operation &operation, Others&... others ) { v.emplace_back( operation ); PipesToVec( v, others... ); } template void PipesToVec( std::vector &v, Pipeline &pipeline, Others&... others ) { v.emplace_back( std::move( pipeline ) ); PipesToVec( v, others... ); } //---------------------------------------------------------------------------- //! Factory function for creating parallel operation from //! a given number of operations //! (we use && reference since due to reference collapsing this will fit //! both r- and l-value references) //---------------------------------------------------------------------------- template inline ParallelOperation Parallel( Operations&& ... operations ) { constexpr size_t size = sizeof...( operations ); std::vector v; v.reserve( size ); PipesToVec( v, operations... ); return Parallel( v ); } } #endif // __XRD_CL_OPERATIONS_HH__ xrootd-5.6.9/src/XrdCl/XrdClPlugInInterface.hh000066400000000000000000000642231457266313600211760ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2014 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #ifndef __XRD_CL_PLUGIN_INTERFACE__ #define __XRD_CL_PLUGIN_INTERFACE__ #include "XrdCl/XrdClFile.hh" #include "XrdCl/XrdClFileSystem.hh" #include "XrdCl/XrdClOptional.hh" namespace XrdCl { //---------------------------------------------------------------------------- //! An interface for file plug-ins //---------------------------------------------------------------------------- class FilePlugIn { public: //------------------------------------------------------------------------ //! Destructor //------------------------------------------------------------------------ virtual ~FilePlugIn() {} //------------------------------------------------------------------------ //! @see XrdCl::File::Open //------------------------------------------------------------------------ virtual XRootDStatus Open( const std::string &url, OpenFlags::Flags flags, Access::Mode mode, ResponseHandler *handler, uint16_t timeout ) { (void)url; (void)flags; (void)mode; (void)handler; (void)timeout; return XRootDStatus( stError, errNotImplemented ); } //------------------------------------------------------------------------ //! @see XrdCl::File::Close //------------------------------------------------------------------------ virtual XRootDStatus Close( ResponseHandler *handler, uint16_t timeout ) { (void)handler; (void)timeout; return XRootDStatus( stError, errNotImplemented ); } //------------------------------------------------------------------------ //! @see XrdCl::File::Stat //------------------------------------------------------------------------ virtual XRootDStatus Stat( bool force, ResponseHandler *handler, uint16_t timeout ) { (void)force; (void)handler; (void)timeout; return XRootDStatus( stError, errNotImplemented ); } //------------------------------------------------------------------------ //! @see XrdCl::File::Read //------------------------------------------------------------------------ virtual XRootDStatus Read( uint64_t offset, uint32_t size, void *buffer, ResponseHandler *handler, uint16_t timeout ) { (void)offset; (void)size; (void)buffer; (void)handler; (void)timeout; return XRootDStatus( stError, errNotImplemented ); } //------------------------------------------------------------------------ //! @see XrdCl::File::Read //------------------------------------------------------------------------ virtual XRootDStatus Read( uint64_t offset, uint32_t size, Optional fdoff, int fd, ResponseHandler *handler, uint16_t timeout = 0 ) { (void)offset; (void)size; (void)fdoff; (void)fd, (void)handler; (void)timeout; return XRootDStatus( stError, errNotImplemented ); } //------------------------------------------------------------------------ //! @see XrdCl:File PgRead //------------------------------------------------------------------------ virtual XRootDStatus PgRead( uint64_t offset, uint32_t size, void *buffer, ResponseHandler *handler, uint16_t timeout ) { (void)offset; (void)size; (void)buffer; (void)handler; (void)timeout; return XRootDStatus( stError, errNotImplemented ); } //------------------------------------------------------------------------ //! @see XrdCl::File::Write //------------------------------------------------------------------------ virtual XRootDStatus Write( uint64_t offset, uint32_t size, const void *buffer, ResponseHandler *handler, uint16_t timeout ) { (void)offset; (void)size; (void)buffer; (void)handler; (void)timeout; return XRootDStatus( stError, errNotImplemented ); } //------------------------------------------------------------------------ //! @see XrdCl::File::Write //------------------------------------------------------------------------ virtual XRootDStatus Write( uint64_t offset, Buffer &&buffer, ResponseHandler *handler, uint16_t timeout = 0 ) { (void)offset; (void)buffer; (void)handler; (void)timeout; return XRootDStatus( stError, errNotImplemented ); } //------------------------------------------------------------------------ //! @see XrdCl::File::Write //------------------------------------------------------------------------ virtual XRootDStatus Write( uint64_t offset, uint32_t size, Optional fdoff, int fd, ResponseHandler *handler, uint16_t timeout = 0 ) { (void)offset; (void)size; (void)fdoff; (void)fd, (void)handler; (void)timeout; return XRootDStatus( stError, errNotImplemented ); } //------------------------------------------------------------------------ //! @see XrdCl::File::PgWrite //------------------------------------------------------------------------ virtual XRootDStatus PgWrite( uint64_t offset, uint32_t nbpgs, const void *buffer, std::vector &cksums, ResponseHandler *handler, uint16_t timeout ) { (void)offset; (void)nbpgs; (void)buffer; (void)cksums, (void)handler; (void)timeout; return XRootDStatus( stError, errNotImplemented ); } //------------------------------------------------------------------------ //! @see XrdCl::File::Sync //------------------------------------------------------------------------ virtual XRootDStatus Sync( ResponseHandler *handler, uint16_t timeout ) { (void)handler; (void)timeout; return XRootDStatus( stError, errNotImplemented ); } //------------------------------------------------------------------------ //! @see XrdCl::File::Truncate //------------------------------------------------------------------------ virtual XRootDStatus Truncate( uint64_t size, ResponseHandler *handler, uint16_t timeout ) { (void)size; (void)handler; (void)timeout; return XRootDStatus( stError, errNotImplemented ); } //------------------------------------------------------------------------ //! @see XrdCl::File::VectorRead //------------------------------------------------------------------------ virtual XRootDStatus VectorRead( const ChunkList &chunks, void *buffer, ResponseHandler *handler, uint16_t timeout ) { (void)chunks; (void)buffer; (void)handler; (void)timeout; return XRootDStatus( stError, errNotImplemented ); } //------------------------------------------------------------------------ //! @see XrdCl::File::VectorWrite //------------------------------------------------------------------------ virtual XRootDStatus VectorWrite( const ChunkList &chunks, ResponseHandler *handler, uint16_t timeout = 0 ) { (void)chunks; (void)handler; (void)timeout; return XRootDStatus( stError, errNotImplemented ); } //------------------------------------------------------------------------ //! @see XrdCl::File::WriteV //------------------------------------------------------------------------ virtual XRootDStatus WriteV( uint64_t offset, const struct iovec *iov, int iovcnt, ResponseHandler *handler, uint16_t timeout = 0 ) { (void)offset; (void)iov; (void)iovcnt; (void)handler; (void)timeout; return XRootDStatus( stError, errNotImplemented ); } //------------------------------------------------------------------------ //! @see XrdCl::File::Fcntl //------------------------------------------------------------------------ virtual XRootDStatus Fcntl( const Buffer &arg, ResponseHandler *handler, uint16_t timeout ) { (void)arg; (void)handler; (void)timeout; return XRootDStatus( stError, errNotImplemented ); } //------------------------------------------------------------------------ //! @see XrdCl::File::Visa //------------------------------------------------------------------------ virtual XRootDStatus Visa( ResponseHandler *handler, uint16_t timeout ) { (void)handler; (void)timeout; return XRootDStatus( stError, errNotImplemented ); } //------------------------------------------------------------------------ //! @see XrdCl::File::IsOpen //------------------------------------------------------------------------ virtual bool IsOpen() const { return false; } //------------------------------------------------------------------------ //! @see XrdCl::File::SetProperty //------------------------------------------------------------------------ virtual bool SetProperty( const std::string &name, const std::string &value ) { (void)name; (void)value; return false; } //------------------------------------------------------------------------ //! @see XrdCl::File::GetProperty //------------------------------------------------------------------------ virtual bool GetProperty( const std::string &name, std::string &value ) const { (void)name; (void)value; return false; } }; //---------------------------------------------------------------------------- //! An interface for file plug-ins //---------------------------------------------------------------------------- class FileSystemPlugIn { public: //------------------------------------------------------------------------ //! Destructor //------------------------------------------------------------------------ virtual ~FileSystemPlugIn() {} //------------------------------------------------------------------------ //! @see XrdCl::FileSystem::Locate //------------------------------------------------------------------------ virtual XRootDStatus Locate( const std::string &path, OpenFlags::Flags flags, ResponseHandler *handler, uint16_t timeout ) { (void)path; (void)flags; (void)handler; (void)timeout; return XRootDStatus( stError, errNotImplemented ); } //------------------------------------------------------------------------ //! @see XrdCl::FileSystem::DeepLocate //------------------------------------------------------------------------ virtual XRootDStatus DeepLocate( const std::string &path, OpenFlags::Flags flags, ResponseHandler *handler, uint16_t timeout ) { (void)path; (void)flags; (void)handler; (void)timeout; return XRootDStatus( stError, errNotImplemented ); } //------------------------------------------------------------------------ //! @see XrdCl::FileSystem::Mv //------------------------------------------------------------------------ virtual XRootDStatus Mv( const std::string &source, const std::string &dest, ResponseHandler *handler, uint16_t timeout ) { (void)source; (void)dest; (void)handler; (void)timeout; return XRootDStatus( stError, errNotImplemented ); } //------------------------------------------------------------------------ //! @see XrdCl::FileSystem::Query //------------------------------------------------------------------------ virtual XRootDStatus Query( QueryCode::Code queryCode, const Buffer &arg, ResponseHandler *handler, uint16_t timeout ) { (void)queryCode; (void)arg; (void)handler; (void)timeout; return XRootDStatus( stError, errNotImplemented ); } //------------------------------------------------------------------------ //! @see XrdCl::FileSystem::Truncate //------------------------------------------------------------------------ virtual XRootDStatus Truncate( const std::string &path, uint64_t size, ResponseHandler *handler, uint16_t timeout ) { (void)path; (void)size; (void)handler; (void)timeout; return XRootDStatus( stError, errNotImplemented ); } //------------------------------------------------------------------------ //! @see XrdCl::FileSystem::Rm //------------------------------------------------------------------------ virtual XRootDStatus Rm( const std::string &path, ResponseHandler *handler, uint16_t timeout ) { (void)path; (void)handler; (void)timeout; return XRootDStatus( stError, errNotImplemented ); } //------------------------------------------------------------------------ //! @see XrdCl::FileSystem::MkDir //------------------------------------------------------------------------ virtual XRootDStatus MkDir( const std::string &path, MkDirFlags::Flags flags, Access::Mode mode, ResponseHandler *handler, uint16_t timeout ) { (void)path; (void)flags; (void)mode; (void)handler; (void)timeout; return XRootDStatus( stError, errNotImplemented ); } //------------------------------------------------------------------------ //! @see XrdCl::FileSystem::RmDir //------------------------------------------------------------------------ virtual XRootDStatus RmDir( const std::string &path, ResponseHandler *handler, uint16_t timeout ) { (void)path; (void)handler; (void)timeout; return XRootDStatus( stError, errNotImplemented ); } //------------------------------------------------------------------------ //! @see XrdCl::FileSystem::ChMod //------------------------------------------------------------------------ virtual XRootDStatus ChMod( const std::string &path, Access::Mode mode, ResponseHandler *handler, uint16_t timeout ) { (void)path; (void)mode; (void)handler; (void)timeout; return XRootDStatus( stError, errNotImplemented ); } //------------------------------------------------------------------------ //! @see XrdCl::FileSystem::Ping //------------------------------------------------------------------------ virtual XRootDStatus Ping( ResponseHandler *handler, uint16_t timeout ) { (void)handler; (void)timeout; return XRootDStatus( stError, errNotImplemented ); } //------------------------------------------------------------------------ //! @see XrdCl::FileSystem::Stat //------------------------------------------------------------------------ virtual XRootDStatus Stat( const std::string &path, ResponseHandler *handler, uint16_t timeout ) { (void)path; (void)handler; (void)timeout; return XRootDStatus( stError, errNotImplemented ); } //------------------------------------------------------------------------ //! @see XrdCl::FileSystem::StatVFS //------------------------------------------------------------------------ virtual XRootDStatus StatVFS( const std::string &path, ResponseHandler *handler, uint16_t timeout ) { (void)path; (void)handler; (void)timeout; return XRootDStatus( stError, errNotImplemented ); } //------------------------------------------------------------------------ //! @see XrdCl::FileSystem::Protocol //------------------------------------------------------------------------ virtual XRootDStatus Protocol( ResponseHandler *handler, uint16_t timeout = 0 ) { (void)handler; (void)timeout; return XRootDStatus( stError, errNotImplemented ); } //------------------------------------------------------------------------ //! @see XrdCl::FileSystem::DirlList //------------------------------------------------------------------------ virtual XRootDStatus DirList( const std::string &path, DirListFlags::Flags flags, ResponseHandler *handler, uint16_t timeout ) { (void)path; (void)flags; (void)handler; (void)timeout; return XRootDStatus( stError, errNotImplemented ); } //------------------------------------------------------------------------ //! @see XrdCl::FileSystem::SendInfo //------------------------------------------------------------------------ virtual XRootDStatus SendInfo( const std::string &info, ResponseHandler *handler, uint16_t timeout ) { (void)info; (void)handler; (void)timeout; return XRootDStatus( stError, errNotImplemented ); } //------------------------------------------------------------------------ //! @see XrdCl::FileSystem::Prepare //------------------------------------------------------------------------ virtual XRootDStatus Prepare( const std::vector &fileList, PrepareFlags::Flags flags, uint8_t priority, ResponseHandler *handler, uint16_t timeout ) { (void)fileList; (void)flags; (void)priority; (void)handler; (void)timeout; return XRootDStatus( stError, errNotImplemented ); } //------------------------------------------------------------------------ //! @see XrdCl::FileSystem::SetXAttr //------------------------------------------------------------------------ virtual XRootDStatus SetXAttr( const std::string &path, const std::vector &attrs, ResponseHandler *handler, uint16_t timeout ) { (void)path; (void)attrs; (void)handler; (void)timeout; return XRootDStatus( stError, errNotImplemented ); } //------------------------------------------------------------------------ //! @see XrdCl::FileSystem::GetXAttr //------------------------------------------------------------------------ virtual XRootDStatus GetXAttr( const std::string &path, const std::vector &attrs, ResponseHandler *handler, uint16_t timeout ) { (void)path; (void)attrs; (void)handler; (void)timeout; return XRootDStatus( stError, errNotImplemented ); } //------------------------------------------------------------------------ //! @see XrdCl::FileSystem::DelXAttr //------------------------------------------------------------------------ virtual XRootDStatus DelXAttr( const std::string &path, const std::vector &attrs, ResponseHandler *handler, uint16_t timeout ) { (void)path; (void)attrs; (void)handler; (void)timeout; return XRootDStatus( stError, errNotImplemented ); } //------------------------------------------------------------------------ //! @see XrdCl::FileSystem::ListXAttr //------------------------------------------------------------------------ virtual XRootDStatus ListXAttr( const std::string &path, ResponseHandler *handler, uint16_t timeout ) { (void)path; (void)handler; (void)timeout; return XRootDStatus( stError, errNotImplemented ); } //------------------------------------------------------------------------ //! @see XrdCl::FileSystem::SetProperty //------------------------------------------------------------------------ virtual bool SetProperty( const std::string &name, const std::string &value ) { (void)name; (void)value; return false; } //------------------------------------------------------------------------ //! @see XrdCl::FileSystem::GetProperty //------------------------------------------------------------------------ virtual bool GetProperty( const std::string &name, std::string &value ) const { (void)name; (void)value; return false; } }; //---------------------------------------------------------------------------- //! Plugin factory //---------------------------------------------------------------------------- class PlugInFactory { public: //------------------------------------------------------------------------ //! Destructor //------------------------------------------------------------------------ virtual ~PlugInFactory() {} //------------------------------------------------------------------------ //! Create a file plug-in for the given URL //------------------------------------------------------------------------ virtual FilePlugIn *CreateFile( const std::string &url ) = 0; //------------------------------------------------------------------------ //! Create a file system plug-in for the given URL //------------------------------------------------------------------------ virtual FileSystemPlugIn *CreateFileSystem( const std::string &url ) = 0; }; } #endif // __XRD_CL_PLUGIN_INTERFACE__ xrootd-5.6.9/src/XrdCl/XrdClPlugInManager.cc000066400000000000000000000432321457266313600206330ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2014 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #include "XrdCl/XrdClPlugInManager.hh" #include "XrdCl/XrdClDefaultEnv.hh" #include "XrdCl/XrdClLog.hh" #include "XrdCl/XrdClConstants.hh" #include "XrdCl/XrdClUtils.hh" #include "XrdCl/XrdClURL.hh" #include "XrdSys/XrdSysPwd.hh" #include "XrdVersion.hh" #ifdef WITH_XRDEC #include "XrdCl/XrdClEcHandler.hh" #endif #include #include #include #include #include XrdVERSIONINFOREF( XrdCl ); namespace XrdCl { //---------------------------------------------------------------------------- // Constructor //---------------------------------------------------------------------------- PlugInManager::PlugInManager(): pDefaultFactory(0) { } //---------------------------------------------------------------------------- // Destructor //---------------------------------------------------------------------------- PlugInManager:: ~PlugInManager() { std::map::iterator it; for( it = pFactoryMap.begin(); it != pFactoryMap.end(); ++it ) { it->second->counter--; if( it->second->counter == 0 ) delete it->second; } delete pDefaultFactory; } //---------------------------------------------------------------------------- // Register a plug-in favtory for the given url //---------------------------------------------------------------------------- bool PlugInManager::RegisterFactory( const std::string &url, PlugInFactory *factory ) { Log *log = DefaultEnv::GetLog(); XrdSysMutexHelper scopedLock( pMutex ); std::string normUrl = NormalizeURL( url ); if( normUrl == "" ) return false; std::map::iterator it; it = pFactoryMap.find( normUrl ); if( it != pFactoryMap.end() ) { if( it->second->isEnv ) return false; // we don't need to check the counter because it's valid only // for environment plugins which cannot be replaced via // this method delete it->second; } if( !factory ) { log->Debug( PlugInMgrMsg, "Removing the factory for %s", normUrl.c_str() ); pFactoryMap.erase( it ); return true; } log->Debug( PlugInMgrMsg, "Registering a factory for %s", normUrl.c_str() ); FactoryHelper *h = new FactoryHelper(); h->factory = factory; h->counter = 1; pFactoryMap[normUrl] = h; return true; } //------------------------------------------------------------------------ //! Register a plug-in factory applying to all URLs //------------------------------------------------------------------------ bool PlugInManager::RegisterDefaultFactory( PlugInFactory *factory ) { Log *log = DefaultEnv::GetLog(); XrdSysMutexHelper scopedLock( pMutex ); if( pDefaultFactory && pDefaultFactory->isEnv ) return false; delete pDefaultFactory; pDefaultFactory = 0; if( factory ) { log->Debug( PlugInMgrMsg, "Registering a default factory" ); pDefaultFactory = new FactoryHelper; pDefaultFactory->factory = factory; } else log->Debug( PlugInMgrMsg, "Removing the default factory" ); return true; } //---------------------------------------------------------------------------- // Retrieve the plug-in factory for the given URL //---------------------------------------------------------------------------- PlugInFactory *PlugInManager::GetFactory( const std::string url ) { XrdSysMutexHelper scopedLock( pMutex ); if( pDefaultFactory && pDefaultFactory->isEnv ) return pDefaultFactory->factory; std::string normUrl = NormalizeURL( url ); if( normUrl.empty() ) { if( pDefaultFactory ) return pDefaultFactory->factory; return 0; } std::map::iterator it; it = pFactoryMap.find( normUrl ); if( it != pFactoryMap.end() && it->second->isEnv ) return it->second->factory; std::string protocol = URL( url ).GetProtocol(); std::map::iterator itProt; itProt = pFactoryMap.find( protocol ); if( itProt != pFactoryMap.end() && itProt->second->isEnv ) return itProt->second->factory; if( pDefaultFactory ) return pDefaultFactory->factory; if( it != pFactoryMap.end() ) return it->second->factory; if( itProt != pFactoryMap.end() ) return itProt->second->factory; return 0; } //---------------------------------------------------------------------------- // Process user environment to load plug-in settings. //---------------------------------------------------------------------------- void PlugInManager::ProcessEnvironmentSettings() { XrdSysMutexHelper scopedLock( pMutex ); Log *log = DefaultEnv::GetLog(); Env *env = DefaultEnv::GetEnv(); log->Debug( PlugInMgrMsg, "Initializing plug-in manager..." ); //-------------------------------------------------------------------------- // Check if a default plug-in has been specified in the environment //-------------------------------------------------------------------------- bool loadConfigs = true; std::string defaultPlugIn = DefaultPlugIn; env->GetString( "PlugIn", defaultPlugIn ); if( !defaultPlugIn.empty() ) { loadConfigs = false; log->Debug( PlugInMgrMsg, "Loading default plug-in from %s...", defaultPlugIn.c_str()); std::pair pg = LoadFactory( defaultPlugIn, std::map() ); if( !pg.first ) { log->Debug( PlugInMgrMsg, "Failed to load default plug-in from %s", defaultPlugIn.c_str()); loadConfigs = false; } pDefaultFactory = new FactoryHelper(); pDefaultFactory->factory = pg.second; pDefaultFactory->plugin = pg.first; pDefaultFactory->isEnv = true; } //-------------------------------------------------------------------------- // If there is no default plug-in or it is invalid then load plug-in config // files //-------------------------------------------------------------------------- if( loadConfigs ) { log->Debug( PlugInMgrMsg, "No default plug-in, loading plug-in configs..." ); ProcessConfigDir( "/etc/xrootd/client.plugins.d" ); XrdSysPwd pwdHandler; passwd *pwd = pwdHandler.Get( getuid() ); if( pwd ) { std::string userPlugIns = pwd->pw_dir; userPlugIns += "/.xrootd/client.plugins.d"; ProcessConfigDir( userPlugIns ); } std::string customPlugIns = DefaultPlugInConfDir; env->GetString( "PlugInConfDir", customPlugIns ); if( !customPlugIns.empty() ) ProcessConfigDir( customPlugIns ); } } //---------------------------------------------------------------------------- // Process the configuration directory and load plug in definitions //---------------------------------------------------------------------------- void PlugInManager::ProcessConfigDir( const std::string &dir ) { Log *log = DefaultEnv::GetLog(); log->Debug( PlugInMgrMsg, "Processing plug-in definitions in %s...", dir.c_str()); std::vector entries; std::vector::iterator it; Status st = Utils::GetDirectoryEntries( entries, dir ); if( !st.IsOK() ) { log->Debug( PlugInMgrMsg, "Unable to process directory %s: %s", dir.c_str(), st.ToString().c_str() ); return; } std::sort( entries.begin(), entries.end() ); for( it = entries.begin(); it != entries.end(); ++it ) { std::string confFile = dir + "/" + *it; std::string suffix = ".conf"; if( confFile.length() <= suffix.length() ) continue; if( !std::equal( suffix.rbegin(), suffix.rend(), confFile.rbegin() ) ) continue; ProcessPlugInConfig( confFile ); } } //---------------------------------------------------------------------------- // Process a plug-in config file and load the plug-in if possible //---------------------------------------------------------------------------- void PlugInManager::ProcessPlugInConfig( const std::string &confFile ) { Log *log = DefaultEnv::GetLog(); log->Dump( PlugInMgrMsg, "Processing: %s", confFile.c_str() ); //-------------------------------------------------------------------------- // Read the config //-------------------------------------------------------------------------- std::map config; Status st = Utils::ProcessConfig( config, confFile ); if( !st.IsOK() ) { log->Debug( PlugInMgrMsg, "Unable process config %s: %s", confFile.c_str(), st.ToString().c_str() ); return; } const char *keys[] = { "url", "lib", "enable", 0 }; for( int i = 0; keys[i]; ++i ) { if( config.find( keys[i] ) == config.end() ) { log->Debug( PlugInMgrMsg, "Unable to find '%s' key in the config file " "%s, ignoring this config", keys[i], confFile.c_str() ); return; } } //-------------------------------------------------------------------------- // Attempt to load the plug in and place it in the map //-------------------------------------------------------------------------- std::string url = config["url"]; std::string lib = config["lib"]; std::string enable = config["enable"]; log->Dump( PlugInMgrMsg, "Settings from '%s': url='%s', lib='%s', " "enable='%s'", confFile.c_str(), url.c_str(), lib.c_str(), enable.c_str() ); std::pair pg; pg.first = 0; pg.second = 0; if( enable == "true" ) { log->Debug( PlugInMgrMsg, "Trying to load a plug-in for '%s' from '%s'", url.c_str(), lib.c_str() ); pg = LoadFactory( lib, config ); if( !pg.second ) return; } else log->Debug( PlugInMgrMsg, "Trying to disable plug-in for '%s'", url.c_str() ); if( !RegisterFactory( url, lib, pg.second, pg.first ) ) { delete pg.first; delete pg.second; } } //---------------------------------------------------------------------------- // Load the plug-in and create the factory //---------------------------------------------------------------------------- std::pair PlugInManager::LoadFactory( const std::string &lib, const std::map &config ) { Log *log = DefaultEnv::GetLog(); #ifdef WITH_XRDEC if( lib == "XrdEcDefault" ) { auto itr = config.find( "nbdta" ); if( itr == config.end() ) return std::make_pair( nullptr, nullptr ); uint8_t nbdta = std::stoul( itr->second ); itr = config.find( "nbprt" ); if( itr == config.end() ) return std::make_pair( nullptr, nullptr ); uint8_t nbprt = std::stoul( itr->second ); itr = config.find( "chsz" ); if( itr == config.end() ) return std::make_pair( nullptr, nullptr ); uint64_t chsz = std::stoul( itr->second ); std::vector plgr; itr = config.find( "plgr" ); if( itr != config.end() ) Utils::splitString( plgr, itr->second, "," ); std::string xrdclECenv = std::to_string(nbdta) + "," + std::to_string(nbprt) + "," + std::to_string(chsz); setenv("XRDCL_EC", xrdclECenv.c_str(), 1); EcPlugInFactory *ecHandler = new EcPlugInFactory( nbdta, nbprt, chsz, std::move( plgr ) ); return std::make_pair( nullptr, ecHandler ); } #endif char errorBuff[1024]; XrdOucPinLoader *pgHandler = new XrdOucPinLoader( errorBuff, 1024, &XrdVERSIONINFOVAR( XrdCl ), "client", lib.c_str() ); PlugInFunc_t pgFunc = (PlugInFunc_t)pgHandler->Resolve("XrdClGetPlugIn", -1); if( !pgFunc ) { log->Debug( PlugInMgrMsg, "Error while loading %s: %s", lib.c_str(), errorBuff ); pgHandler->Unload(); delete pgHandler; return std::make_pair( 0, 0 ); } PlugInFactory *f = (PlugInFactory*)pgFunc( &config ); if( !f ) { delete pgHandler; return std::make_pair( 0, 0 ); } return std::make_pair( pgHandler, f ); } //---------------------------------------------------------------------------- // Handle factory - register it or free all the memory //---------------------------------------------------------------------------- bool PlugInManager::RegisterFactory( const std::string &urlString, const std::string &lib, PlugInFactory *factory, XrdOucPinLoader *plugin ) { //-------------------------------------------------------------------------- // Process and normalize the URLs //-------------------------------------------------------------------------- Log *log = DefaultEnv::GetLog(); std::vector urls; std::vector normalizedURLs; std::vector::iterator it; if (urlString == "*") { if (pDefaultFactory) { if (pDefaultFactory->isEnv) { log->Debug(PlugInMgrMsg, "There is already an env default plugin " "loaded, skipping %s", lib.c_str()); return false; } else { log->Debug(PlugInMgrMsg, "There can be only one default plugin " "loaded, skipping %s", lib.c_str()); return false; } } else { pDefaultFactory = new FactoryHelper(); pDefaultFactory->factory = factory; pDefaultFactory->plugin = plugin; pDefaultFactory->isEnv = false; return true; } } Utils::splitString( urls, urlString, ";" ); for( it = urls.begin(); it != urls.end(); ++it ) { std::string normURL = NormalizeURL( *it ); if( normURL == "" ) { log->Debug( PlugInMgrMsg, "Url cannot be normalized: '%s', ignoring", it->c_str() ); continue; } normalizedURLs.push_back( normURL ); } std::sort( normalizedURLs.begin(), normalizedURLs.end() ); auto last = std::unique( normalizedURLs.begin(), normalizedURLs.end() ); normalizedURLs.erase( last, normalizedURLs.end() ); if( normalizedURLs.empty() ) return false; //-------------------------------------------------------------------------- // Insert or remove from the map //-------------------------------------------------------------------------- FactoryHelper *h = 0; if( factory ) { h = new FactoryHelper(); h->isEnv = true; h->counter = normalizedURLs.size(); h->plugin = plugin; h->factory = factory; } std::map::iterator mapIt; for( it = normalizedURLs.begin(); it != normalizedURLs.end(); ++it ) { mapIt = pFactoryMap.find( *it ); if( mapIt != pFactoryMap.end() ) { mapIt->second->counter--; if( mapIt->second->counter == 0 ) delete mapIt->second; } if( h ) { log->Debug( PlugInMgrMsg, "Registering a factory for %s from %s", it->c_str(), lib.c_str() ); pFactoryMap[*it] = h; } else { if( mapIt != pFactoryMap.end() ) { log->Debug( PlugInMgrMsg, "Removing the factory for %s", it->c_str() ); pFactoryMap.erase( mapIt ); } } } return true; } //---------------------------------------------------------------------------- // Normalize a URL //---------------------------------------------------------------------------- std::string PlugInManager::NormalizeURL( const std::string url ) { URL urlObj = url; if( !urlObj.IsValid() ) return ""; std::string protocol = urlObj.GetProtocol(); std::string hostname = urlObj.GetHostName(); if( hostname == "*" ) return protocol; std::ostringstream o; o << protocol << "://" << hostname << ":"; o << urlObj.GetPort(); return o.str(); } }; xrootd-5.6.9/src/XrdCl/XrdClPlugInManager.hh000066400000000000000000000164721457266313600206530ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2014 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #ifndef __XRD_CL_PLUGIN_MANAGER__ #define __XRD_CL_PLUGIN_MANAGER__ #include "XrdCl/XrdClPlugInInterface.hh" #include "XrdOuc/XrdOucPinLoader.hh" #include "XrdSys/XrdSysPthread.hh" #include #include #include namespace XrdCl { //---------------------------------------------------------------------------- //! Manage client-side plug-ins and match them agains URLs //---------------------------------------------------------------------------- class PlugInManager { public: //------------------------------------------------------------------------ //! Constructor //------------------------------------------------------------------------ PlugInManager(); //------------------------------------------------------------------------ //! Destructor //------------------------------------------------------------------------ ~PlugInManager(); //------------------------------------------------------------------------ //! Register a plug-in factory for the given url, registering a 0 pointer //! removes the factory for the url //------------------------------------------------------------------------ bool RegisterFactory( const std::string &url, PlugInFactory *factory ); //------------------------------------------------------------------------ //! Register a plug-in factory applying to all URLs, registering //! a 0 pointer removes the factory //------------------------------------------------------------------------ bool RegisterDefaultFactory( PlugInFactory *factory ); //------------------------------------------------------------------------ //! Retrieve the plug-in factory for the given URL //! //! @return you do not own the returned memory //------------------------------------------------------------------------ PlugInFactory *GetFactory( const std::string url ); //------------------------------------------------------------------------ //! Process user environment to load plug-in settings. //! //! This will try to load a default plug-in from a library pointed to //! by the XRD_PLUGIN envvar. If this fails it will scan the configuration //! files located in: //! //! 1) system directory: /etc/xrootd/client.plugins.d/ //! 2) user direvtory: ~/.xrootd/client.plugins.d/ //! 3) directory pointed to by XRD_PLUGINCONFDIR envvar //! //! In that order. //! //! The configuration files contain lines with key-value pairs in the //! form of 'key=value'. //! //! Mandatory keys are: //! url - a semicolon separated list of URLs the plug-in applies to //! lib - plugin library to be loaded //! enabled - determines whether the plug-in should be enabled or not //! //! You may use any other keys for your own purposes. //! //! The config files are processed in alphabetic order, any satteing //! found later superseeds the previous one. Any setting applied via //! environment or config files superseeds any setting done //! programatically. //! //! The plug-in library must implement the following C function: //! //! @code{.cpp} //! extern "C" //! { //! void *XrdClGetPlugIn( const void *arg ) //! { //! return __your_plug_in_factory__; //! } //! } //! @endcode //! //! where arg is a const pointer to std::map //! containing the plug-in configuration. //------------------------------------------------------------------------ void ProcessEnvironmentSettings(); private: typedef void *(*PlugInFunc_t)( const void *arg ); struct FactoryHelper { FactoryHelper(): plugin(0), factory(0), isEnv(false), counter(0) {} ~FactoryHelper() { delete factory; if(plugin) plugin->Unload(); delete plugin; } XrdOucPinLoader *plugin; PlugInFactory *factory; bool isEnv; uint32_t counter; }; //------------------------------------------------------------------------ //! Process the configuration directory and load plug in definitions //------------------------------------------------------------------------ void ProcessConfigDir( const std::string &dir ); //------------------------------------------------------------------------ //! Process a plug-in config file and load the plug-in if possible //------------------------------------------------------------------------ void ProcessPlugInConfig( const std::string &confFile ); //------------------------------------------------------------------------ //! Load the plug-in and create the factory //------------------------------------------------------------------------ std::pair LoadFactory( const std::string &lib, const std::map &config ); //------------------------------------------------------------------------ //! Register factory, if successful it actuires ownership of the objects //! @return true if successfully registered //------------------------------------------------------------------------ bool RegisterFactory( const std::string &urlString, const std::string &lib, PlugInFactory *factory, XrdOucPinLoader *plugin ); //------------------------------------------------------------------------ //! Normalize a URL //------------------------------------------------------------------------ std::string NormalizeURL( const std::string url ); std::map pFactoryMap; FactoryHelper *pDefaultFactory; XrdSysMutex pMutex; }; } #endif // __XRD_CL_PLUGIN_MANAGER__ xrootd-5.6.9/src/XrdCl/XrdClPoller.hh000066400000000000000000000160611457266313600174110ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2012 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . //------------------------------------------------------------------------------ #ifndef __XRD_CL_POLLER_HH__ #define __XRD_CL_POLLER_HH__ #include #include namespace XrdCl { class Socket; class Poller; //---------------------------------------------------------------------------- //! Interface //---------------------------------------------------------------------------- class SocketHandler { public: //------------------------------------------------------------------------ //! Event type //------------------------------------------------------------------------ enum EventType { ReadyToRead = 0x01, //!< New data has arrived ReadTimeOut = 0x02, //!< Read timeout ReadyToWrite = 0x04, //!< Writing won't block WriteTimeOut = 0x08 //!< Write timeout }; //------------------------------------------------------------------------ // Destructor //------------------------------------------------------------------------ virtual ~SocketHandler() {} //------------------------------------------------------------------------ //! Initializer //------------------------------------------------------------------------ virtual void Initialize( Poller * ) {} //------------------------------------------------------------------------ //! Finalizer //------------------------------------------------------------------------ virtual void Finalize() {}; //------------------------------------------------------------------------ //! Called when an event occurred on a given socket //------------------------------------------------------------------------ virtual void Event( uint8_t type, Socket *socket ) = 0; //------------------------------------------------------------------------ //! Translate the event type to a string //------------------------------------------------------------------------ static std::string EventTypeToString( uint8_t event ) { std::string ev; if( event & ReadyToRead ) ev += "ReadyToRead|"; if( event & ReadTimeOut ) ev += "ReadTimeOut|"; if( event & ReadyToWrite ) ev += "ReadyToWrite|"; if( event & WriteTimeOut ) ev += "WriteTimeOut|"; ev.erase( ev.length()-1, 1) ; return ev; } }; //---------------------------------------------------------------------------- //! Interface for socket pollers //---------------------------------------------------------------------------- class Poller { public: //------------------------------------------------------------------------ //! Destructor //------------------------------------------------------------------------ virtual ~Poller() {} //------------------------------------------------------------------------ //! Initialize the poller //------------------------------------------------------------------------ virtual bool Initialize() = 0; //------------------------------------------------------------------------ //! Finalize the poller //------------------------------------------------------------------------ virtual bool Finalize() = 0; //------------------------------------------------------------------------ //! Start polling //------------------------------------------------------------------------ virtual bool Start() = 0; //------------------------------------------------------------------------ //! Stop polling //------------------------------------------------------------------------ virtual bool Stop() = 0; //------------------------------------------------------------------------ //! Add socket to the polling loop //! //! @param socket the socket //! @param handler object handling the events //------------------------------------------------------------------------ virtual bool AddSocket( Socket *socket, SocketHandler *handler ) = 0; //------------------------------------------------------------------------ //! Remove the socket //------------------------------------------------------------------------ virtual bool RemoveSocket( Socket *socket ) = 0; //------------------------------------------------------------------------ //! Notify the handler about read events //! //! @param socket the socket //! @param notify specify if the handler should be notified //! @param timeout if no read event occurred after this time a timeout //! event will be generated //------------------------------------------------------------------------ virtual bool EnableReadNotification( Socket *socket, bool notify, uint16_t timeout = 60 ) = 0; //------------------------------------------------------------------------ //! Notify the handler about write events //! @param socket the socket //! @param notify specify if the handler should be notified //! @param timeout if no write event occurred after this time a timeout //! event will be generated //------------------------------------------------------------------------ virtual bool EnableWriteNotification( Socket *socket, bool notify, uint16_t timeout = 60 ) = 0; //------------------------------------------------------------------------ //! Check whether the socket is registered with the poller //------------------------------------------------------------------------ virtual bool IsRegistered( Socket *socket ) = 0; //------------------------------------------------------------------------ //! Is the event loop running? //------------------------------------------------------------------------ virtual bool IsRunning() const = 0; }; } #endif // __XRD_CL_POLLER_HH__ xrootd-5.6.9/src/XrdCl/XrdClPollerBuiltIn.cc000066400000000000000000000501721457266313600206670ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2014 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #include "XrdCl/XrdClPollerBuiltIn.hh" #include "XrdCl/XrdClLog.hh" #include "XrdCl/XrdClDefaultEnv.hh" #include "XrdCl/XrdClConstants.hh" #include "XrdCl/XrdClSocket.hh" #include "XrdCl/XrdClOptimizers.hh" #include "XrdSys/XrdSysE2T.hh" #include "XrdSys/XrdSysIOEvents.hh" namespace { //---------------------------------------------------------------------------- // A helper struct passed to the callback as a custom arg //---------------------------------------------------------------------------- struct PollerHelper { PollerHelper(): channel(0), callBack(0), readEnabled(false), writeEnabled(false), readTimeout(0), writeTimeout(0) {} XrdSys::IOEvents::Channel *channel; XrdSys::IOEvents::CallBack *callBack; bool readEnabled; bool writeEnabled; uint16_t readTimeout; uint16_t writeTimeout; }; //---------------------------------------------------------------------------- // Call back implementation //---------------------------------------------------------------------------- class SocketCallBack: public XrdSys::IOEvents::CallBack { public: SocketCallBack( XrdCl::Socket *sock, XrdCl::SocketHandler *sh ): pSocket( sock ), pHandler( sh ) {} virtual ~SocketCallBack() {}; virtual bool Event( XrdSys::IOEvents::Channel *chP, void *cbArg, int evFlags ) { using namespace XrdCl; uint8_t ev = 0; if( evFlags & ReadyToRead ) ev |= SocketHandler::ReadyToRead; if( evFlags & ReadTimeOut ) ev |= SocketHandler::ReadTimeOut; if( evFlags & ReadyToWrite ) ev |= SocketHandler::ReadyToWrite; if( evFlags & WriteTimeOut ) ev |= SocketHandler::WriteTimeOut; Log *log = DefaultEnv::GetLog(); if( unlikely(log->GetLevel() >= Log::DumpMsg) ) { log->Dump( PollerMsg, "%s Got an event: %s", pSocket->GetName().c_str(), SocketHandler::EventTypeToString( ev ).c_str() ); } pHandler->Event( ev, pSocket ); return true; } private: XrdCl::Socket *pSocket; XrdCl::SocketHandler *pHandler; }; } namespace XrdCl { //---------------------------------------------------------------------------- // Initialize the poller //---------------------------------------------------------------------------- bool PollerBuiltIn::Initialize() { return true; } //---------------------------------------------------------------------------- // Finalize the poller //---------------------------------------------------------------------------- bool PollerBuiltIn::Finalize() { //-------------------------------------------------------------------------- // Clean up the channels //-------------------------------------------------------------------------- SocketMap::iterator it; for( it = pSocketMap.begin(); it != pSocketMap.end(); ++it ) { PollerHelper *helper = (PollerHelper*)it->second; if( helper->channel ) helper->channel->Delete(); delete helper->callBack; delete helper; } pSocketMap.clear(); return true; } //------------------------------------------------------------------------ // Start polling //------------------------------------------------------------------------ bool PollerBuiltIn::Start() { //-------------------------------------------------------------------------- // Start the poller //-------------------------------------------------------------------------- using namespace XrdSys; Log *log = DefaultEnv::GetLog(); log->Debug( PollerMsg, "Creating and starting the built-in poller..." ); XrdSysMutexHelper scopedLock( pMutex ); int errNum = 0; const char *errMsg = 0; for( int i = 0; i < pNbPoller; ++i ) { XrdSys::IOEvents::Poller* poller = IOEvents::Poller::Create( errNum, &errMsg ); if( !poller ) { log->Error( PollerMsg, "Unable to create the internal poller object: ", "%s (%s)", XrdSysE2T( errno ), errMsg ); return false; } pPollerPool.push_back( poller ); } pNext = pPollerPool.begin(); log->Debug( PollerMsg, "Using %d poller threads", pNbPoller ); //-------------------------------------------------------------------------- // Check if we have any descriptors to reinsert from the last time we // were started //-------------------------------------------------------------------------- SocketMap::iterator it; for( it = pSocketMap.begin(); it != pSocketMap.end(); ++it ) { PollerHelper *helper = (PollerHelper*)it->second; Socket *socket = it->first; helper->channel = new IOEvents::Channel( RegisterAndGetPoller( socket ), socket->GetFD(), helper->callBack ); if( helper->readEnabled ) { bool status = helper->channel->Enable( IOEvents::Channel::readEvents, helper->readTimeout, &errMsg ); if( !status ) { log->Error( PollerMsg, "Unable to enable read notifications ", "while re-starting %s (%s)", XrdSysE2T( errno ), errMsg ); return false; } } if( helper->writeEnabled ) { bool status = helper->channel->Enable( IOEvents::Channel::writeEvents, helper->writeTimeout, &errMsg ); if( !status ) { log->Error( PollerMsg, "Unable to enable write notifications ", "while re-starting %s (%s)", XrdSysE2T( errno ), errMsg ); return false; } } } return true; } //------------------------------------------------------------------------ // Stop polling //------------------------------------------------------------------------ bool PollerBuiltIn::Stop() { using namespace XrdSys::IOEvents; Log *log = DefaultEnv::GetLog(); log->Debug( PollerMsg, "Stopping the poller..." ); XrdSysMutexHelper scopedLock( pMutex ); if( pPollerPool.empty() ) { log->Debug( PollerMsg, "Stopping a poller that has not been started" ); return true; } while( !pPollerPool.empty() ) { XrdSys::IOEvents::Poller *poller = pPollerPool.back(); pPollerPool.pop_back(); if( !poller ) continue; scopedLock.UnLock(); poller->Stop(); delete poller; scopedLock.Lock( &pMutex ); } pNext = pPollerPool.end(); pPollerMap.clear(); SocketMap::iterator it; const char *errMsg = 0; for( it = pSocketMap.begin(); it != pSocketMap.end(); ++it ) { PollerHelper *helper = (PollerHelper*)it->second; if( !helper->channel ) continue; bool status = helper->channel->Disable( Channel::allEvents, &errMsg ); if( !status ) { Socket *socket = it->first; log->Error( PollerMsg, "%s Unable to disable write notifications: %s", socket->GetName().c_str(), errMsg ); } helper->channel->Delete(); helper->channel = 0; } return true; } //------------------------------------------------------------------------ // Add socket to the polling queue //------------------------------------------------------------------------ bool PollerBuiltIn::AddSocket( Socket *socket, SocketHandler *handler ) { Log *log = DefaultEnv::GetLog(); XrdSysMutexHelper scopedLock( pMutex ); if( !socket ) { log->Error( PollerMsg, "Invalid socket, impossible to poll" ); return false; } if( socket->GetStatus() != Socket::Connected && socket->GetStatus() != Socket::Connecting ) { log->Error( PollerMsg, "Socket is not in a state valid for polling" ); return false; } log->Debug( PollerMsg, "Adding socket 0x%x to the poller", socket ); //-------------------------------------------------------------------------- // Check if the socket is already registered //-------------------------------------------------------------------------- SocketMap::const_iterator it = pSocketMap.find( socket ); if( it != pSocketMap.end() ) { log->Warning( PollerMsg, "%s Already registered with this poller", socket->GetName().c_str() ); return false; } //-------------------------------------------------------------------------- // Create the socket helper //-------------------------------------------------------------------------- XrdSys::IOEvents::Poller* poller = RegisterAndGetPoller( socket ); PollerHelper *helper = new PollerHelper(); helper->callBack = new ::SocketCallBack( socket, handler ); if( poller ) { helper->channel = new XrdSys::IOEvents::Channel( poller, socket->GetFD(), helper->callBack ); } handler->Initialize( this ); pSocketMap[socket] = helper; return true; } //------------------------------------------------------------------------ // Remove the socket //------------------------------------------------------------------------ bool PollerBuiltIn::RemoveSocket( Socket *socket ) { using namespace XrdSys::IOEvents; Log *log = DefaultEnv::GetLog(); //-------------------------------------------------------------------------- // Find the right socket //-------------------------------------------------------------------------- XrdSysMutexHelper scopedLock( pMutex ); SocketMap::iterator it = pSocketMap.find( socket ); if( it == pSocketMap.end() ) return true; log->Debug( PollerMsg, "%s Removing socket from the poller", socket->GetName().c_str() ); // unregister from the poller it's currently associated with UnregisterFromPoller( socket ); //-------------------------------------------------------------------------- // Remove the socket //-------------------------------------------------------------------------- PollerHelper *helper = (PollerHelper*)it->second; pSocketMap.erase( it ); scopedLock.UnLock(); if( helper->channel ) { const char *errMsg; bool status = helper->channel->Disable( Channel::allEvents, &errMsg ); if( !status ) { log->Error( PollerMsg, "%s Unable to disable write notifications: %s", socket->GetName().c_str(), errMsg ); return false; } helper->channel->Delete(); } delete helper->callBack; delete helper; return true; } //---------------------------------------------------------------------------- // Notify the handler about read events //---------------------------------------------------------------------------- bool PollerBuiltIn::EnableReadNotification( Socket *socket, bool notify, uint16_t timeout ) { using namespace XrdSys::IOEvents; Log *log = DefaultEnv::GetLog(); if( !socket ) { log->Error( PollerMsg, "Invalid socket, read events unavailable" ); return false; } //-------------------------------------------------------------------------- // Check if the socket is registered //-------------------------------------------------------------------------- XrdSysMutexHelper scopedLock( pMutex ); SocketMap::const_iterator it = pSocketMap.find( socket ); if( it == pSocketMap.end() ) { log->Warning( PollerMsg, "%s Socket is not registered", socket->GetName().c_str() ); return false; } PollerHelper *helper = (PollerHelper*)it->second; XrdSys::IOEvents::Poller *poller = GetPoller( socket ); //-------------------------------------------------------------------------- // Enable read notifications //-------------------------------------------------------------------------- if( notify ) { if( helper->readEnabled ) return true; helper->readTimeout = timeout; log->Dump( PollerMsg, "%s Enable read notifications, timeout: %d", socket->GetName().c_str(), timeout ); if( poller ) { const char *errMsg; bool status = helper->channel->Enable( Channel::readEvents, timeout, &errMsg ); if( !status ) { log->Error( PollerMsg, "%s Unable to enable read notifications: %s", socket->GetName().c_str(), errMsg ); return false; } } helper->readEnabled = true; } //-------------------------------------------------------------------------- // Disable read notifications //-------------------------------------------------------------------------- else { if( !helper->readEnabled ) return true; log->Dump( PollerMsg, "%s Disable read notifications", socket->GetName().c_str() ); if( poller ) { const char *errMsg; bool status = helper->channel->Disable( Channel::readEvents, &errMsg ); if( !status ) { log->Error( PollerMsg, "%s Unable to disable read notifications: %s", socket->GetName().c_str(), errMsg ); return false; } } helper->readEnabled = false; } return true; } //---------------------------------------------------------------------------- // Notify the handler about write events //---------------------------------------------------------------------------- bool PollerBuiltIn::EnableWriteNotification( Socket *socket, bool notify, uint16_t timeout ) { using namespace XrdSys::IOEvents; Log *log = DefaultEnv::GetLog(); if( !socket ) { log->Error( PollerMsg, "Invalid socket, write events unavailable" ); return false; } //-------------------------------------------------------------------------- // Check if the socket is registered //-------------------------------------------------------------------------- XrdSysMutexHelper scopedLock( pMutex ); SocketMap::const_iterator it = pSocketMap.find( socket ); if( it == pSocketMap.end() ) { log->Warning( PollerMsg, "%s Socket is not registered", socket->GetName().c_str() ); return false; } PollerHelper *helper = (PollerHelper*)it->second; XrdSys::IOEvents::Poller *poller = GetPoller( socket ); //-------------------------------------------------------------------------- // Enable write notifications //-------------------------------------------------------------------------- if( notify ) { if( helper->writeEnabled ) return true; helper->writeTimeout = timeout; log->Dump( PollerMsg, "%s Enable write notifications, timeout: %d", socket->GetName().c_str(), timeout ); if( poller ) { const char *errMsg; bool status = helper->channel->Enable( Channel::writeEvents, timeout, &errMsg ); if( !status ) { log->Error( PollerMsg, "%s Unable to enable write notifications: %s", socket->GetName().c_str(), errMsg ); return false; } } helper->writeEnabled = true; } //-------------------------------------------------------------------------- // Disable read notifications //-------------------------------------------------------------------------- else { if( !helper->writeEnabled ) return true; log->Dump( PollerMsg, "%s Disable write notifications", socket->GetName().c_str() ); if( poller ) { const char *errMsg; bool status = helper->channel->Disable( Channel::writeEvents, &errMsg ); if( !status ) { log->Error( PollerMsg, "%s Unable to disable write notifications: %s", socket->GetName().c_str(), errMsg ); return false; } } helper->writeEnabled = false; } return true; } //---------------------------------------------------------------------------- // Check whether the socket is registered with the poller //---------------------------------------------------------------------------- bool PollerBuiltIn::IsRegistered( Socket *socket ) { XrdSysMutexHelper scopedLock( pMutex ); SocketMap::iterator it = pSocketMap.find( socket ); return it != pSocketMap.end(); } //---------------------------------------------------------------------------- // Return poller threads in round-robin fashion //---------------------------------------------------------------------------- XrdSys::IOEvents::Poller* PollerBuiltIn::GetNextPoller() { if( pPollerPool.empty() ) return 0; PollerPool::iterator ret = pNext; ++pNext; if( pNext == pPollerPool.end() ) pNext = pPollerPool.begin(); return *ret; } //---------------------------------------------------------------------------- // Return the poller associated with the respective channel //---------------------------------------------------------------------------- XrdSys::IOEvents::Poller* PollerBuiltIn::RegisterAndGetPoller(const Socket * socket) { PollerMap::iterator itr = pPollerMap.find( socket->GetChannelID() ); if( itr == pPollerMap.end() ) { XrdSys::IOEvents::Poller* poller = GetNextPoller(); if( poller ) pPollerMap[socket->GetChannelID()] = std::make_pair( poller, size_t( 1 ) ); return poller; } ++( itr->second.second ); return itr->second.first; } void PollerBuiltIn::UnregisterFromPoller( const Socket *socket ) { PollerMap::iterator itr = pPollerMap.find( socket->GetChannelID() ); if( itr == pPollerMap.end() ) return; --itr->second.second; if( itr->second.second == 0 ) pPollerMap.erase( itr ); } XrdSys::IOEvents::Poller* PollerBuiltIn::GetPoller(const Socket * socket) { PollerMap::iterator itr = pPollerMap.find( socket->GetChannelID() ); if( itr == pPollerMap.end() ) return 0; return itr->second.first; } //---------------------------------------------------------------------------- // Get the initial value for pNbPoller //---------------------------------------------------------------------------- int PollerBuiltIn::GetNbPollerInit() { Env * env = DefaultEnv::GetEnv(); int ret = XrdCl::DefaultParallelEvtLoop; env->GetInt("ParallelEvtLoop", ret); return ret; } } xrootd-5.6.9/src/XrdCl/XrdClPollerBuiltIn.hh000066400000000000000000000161231457266313600206770ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2012 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . //------------------------------------------------------------------------------ #ifndef __XRD_CL_POLLER_BUILT_IN_HH__ #define __XRD_CL_POLLER_BUILT_IN_HH__ #include "XrdSys/XrdSysPthread.hh" #include "XrdCl/XrdClPoller.hh" #include #include namespace XrdSys { namespace IOEvents { class Poller; }; }; namespace XrdCl { class AnyObject; //---------------------------------------------------------------------------- //! A poller implementation using the build-in XRootD poller //---------------------------------------------------------------------------- class PollerBuiltIn: public Poller { public: //------------------------------------------------------------------------ //! Constructor //------------------------------------------------------------------------ PollerBuiltIn() : pNbPoller( GetNbPollerInit() ){} ~PollerBuiltIn() {} //------------------------------------------------------------------------ //! Initialize the poller //------------------------------------------------------------------------ virtual bool Initialize(); //------------------------------------------------------------------------ //! Finalize the poller //------------------------------------------------------------------------ virtual bool Finalize(); //------------------------------------------------------------------------ //! Start polling //------------------------------------------------------------------------ virtual bool Start(); //------------------------------------------------------------------------ //! Stop polling //------------------------------------------------------------------------ virtual bool Stop(); //------------------------------------------------------------------------ //! Add socket to the polling loop //! //! @param socket the socket //! @param handler object handling the events //------------------------------------------------------------------------ virtual bool AddSocket( Socket *socket, SocketHandler *handler ); //------------------------------------------------------------------------ //! Remove the socket //------------------------------------------------------------------------ virtual bool RemoveSocket( Socket *socket ); //------------------------------------------------------------------------ //! Notify the handler about read events //! //! @param socket the socket //! @param notify specify if the handler should be notified //! @param timeout if no read event occurred after this time a timeout //! event will be generated //------------------------------------------------------------------------ virtual bool EnableReadNotification( Socket *socket, bool notify, uint16_t timeout = 60 ); //------------------------------------------------------------------------ //! Notify the handler about write events //! //! @param socket the socket //! @param notify specify if the handler should be notified //! @param timeout if no write event occurred after this time a timeout //! event will be generated //------------------------------------------------------------------------ virtual bool EnableWriteNotification( Socket *socket, bool notify, uint16_t timeout = 60); //------------------------------------------------------------------------ //! Check whether the socket is registered with the poller //------------------------------------------------------------------------ virtual bool IsRegistered( Socket *socket ); //------------------------------------------------------------------------ //! Is the event loop running? //------------------------------------------------------------------------ virtual bool IsRunning() const { return !pPollerPool.empty(); } private: //------------------------------------------------------------------------ //! Goes over poller threads in round robin fashion //------------------------------------------------------------------------ XrdSys::IOEvents::Poller* GetNextPoller(); //------------------------------------------------------------------------ //! Registers given socket as a poller user and returns the poller object //------------------------------------------------------------------------ XrdSys::IOEvents::Poller* RegisterAndGetPoller(const Socket *socket); //------------------------------------------------------------------------ //! Unregisters given socket from poller object //------------------------------------------------------------------------ void UnregisterFromPoller( const Socket *socket); //------------------------------------------------------------------------ //! Returns the poller object associated with the given socket //------------------------------------------------------------------------ XrdSys::IOEvents::Poller* GetPoller(const Socket *socket); //------------------------------------------------------------------------ //! Gets the initial value for 'pNbPoller' //------------------------------------------------------------------------ static int GetNbPollerInit(); // associates channel ID to a pair: poller and count (how many sockets where mapped to this poller) typedef std::map > PollerMap; typedef std::map SocketMap; typedef std::vector PollerPool; SocketMap pSocketMap; PollerMap pPollerMap; PollerPool pPollerPool; PollerPool::iterator pNext; const int pNbPoller; XrdSysMutex pMutex; }; } #endif // __XRD_CL_POLLER_BUILT_IN_HH__ xrootd-5.6.9/src/XrdCl/XrdClPollerFactory.cc000066400000000000000000000071651457266313600207340ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2012 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . //------------------------------------------------------------------------------ #include "XrdCl/XrdClPollerFactory.hh" #include "XrdCl/XrdClPollerBuiltIn.hh" #include "XrdCl/XrdClConstants.hh" #include "XrdCl/XrdClLog.hh" #include "XrdCl/XrdClUtils.hh" #include "XrdCl/XrdClDefaultEnv.hh" #include #include //------------------------------------------------------------------------------ // Poller creators //------------------------------------------------------------------------------ namespace { XrdCl::Poller *createBuiltIn() { return new XrdCl::PollerBuiltIn(); } }; namespace XrdCl { //------------------------------------------------------------------------ // Create a poller object, try in order of preference //------------------------------------------------------------------------ Poller *PollerFactory::CreatePoller( const std::string &preference ) { Log *log = DefaultEnv::GetLog(); //-------------------------------------------------------------------------- // Create a list of known pollers //-------------------------------------------------------------------------- typedef std::map PollerMap; PollerMap pollerMap; pollerMap["built-in"] = createBuiltIn; //-------------------------------------------------------------------------- // Print the list of available pollers //-------------------------------------------------------------------------- PollerMap::iterator it; std::string available; for( it = pollerMap.begin(); it != pollerMap.end(); ++it ) { available += it->first; available += ", "; } if( !available.empty() ) available.erase( available.length()-2, 2 ); log->Debug( PollerMsg, "Available pollers: %s", available.c_str() ); //-------------------------------------------------------------------------- // Try to create a poller //-------------------------------------------------------------------------- if( preference.empty() ) { log->Error( PollerMsg, "Poller preference list is empty" ); return 0; } log->Debug( PollerMsg, "Attempting to create a poller according to " "preference: %s", preference.c_str() ); std::vector prefs; std::vector::iterator itP; Utils::splitString( prefs, preference, "," ); for( itP = prefs.begin(); itP != prefs.end(); ++itP ) { it = pollerMap.find( *itP ); if( it == pollerMap.end() ) { log->Debug( PollerMsg, "Unable to create poller: %s", itP->c_str() ); continue; } log->Debug( PollerMsg, "Creating poller: %s", itP->c_str() ); return (*it->second)(); } return 0; } } xrootd-5.6.9/src/XrdCl/XrdClPollerFactory.hh000066400000000000000000000037361457266313600207460ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2012 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . //------------------------------------------------------------------------------ #ifndef __XRD_CL_POLLER_FACTORY_HH__ #define __XRD_CL_POLLER_FACTORY_HH__ #include "XrdCl/XrdClPoller.hh" namespace XrdCl { //---------------------------------------------------------------------------- //! Helper for creating poller objects //---------------------------------------------------------------------------- class PollerFactory { public: //------------------------------------------------------------------------ //! Create a poller object, try in order of preference, if none of the //! poller types is known then return 0 //! //! @param preference comma separated list of poller types in order of //! preference //! @return poller object or 0 if non of the poller types //! is known //------------------------------------------------------------------------ static Poller *CreatePoller( const std::string &preference ); }; } #endif // __XRD_CL_POLLER_FACTORY_HH__ xrootd-5.6.9/src/XrdCl/XrdClPostMaster.cc000066400000000000000000000434231457266313600202450ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2012 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . //------------------------------------------------------------------------------ #include "XrdCl/XrdClPostMaster.hh" #include "XrdCl/XrdClPollerFactory.hh" #include "XrdCl/XrdClXRootDTransport.hh" #include "XrdCl/XrdClMessage.hh" #include "XrdCl/XrdClConstants.hh" #include "XrdCl/XrdClDefaultEnv.hh" #include "XrdCl/XrdClPoller.hh" #include "XrdCl/XrdClTaskManager.hh" #include "XrdCl/XrdClJobManager.hh" #include "XrdCl/XrdClTransportManager.hh" #include "XrdCl/XrdClChannel.hh" #include "XrdCl/XrdClConstants.hh" #include "XrdCl/XrdClLog.hh" #include "XrdCl/XrdClRedirectorRegistry.hh" #include "XrdSys/XrdSysPthread.hh" namespace XrdCl { struct ConnErrJob : public Job { ConnErrJob( const URL &url, const XRootDStatus &status, std::function handler) : url( url ), status( status ), handler( handler ) { } void Run( void *arg ) { handler( url, status ); delete this; } URL url; XRootDStatus status; std::function handler; }; struct PostMasterImpl { PostMasterImpl() : pPoller( 0 ), pInitialized( false ), pRunning( false ) { Env *env = DefaultEnv::GetEnv(); int workerThreads = DefaultWorkerThreads; env->GetInt( "WorkerThreads", workerThreads ); pTaskManager = new TaskManager(); pJobManager = new JobManager(workerThreads); } ~PostMasterImpl() { delete pPoller; delete pTaskManager; delete pJobManager; } typedef std::map ChannelMap; Poller *pPoller; TaskManager *pTaskManager; ChannelMap pChannelMap; XrdSysMutex pChannelMapMutex; bool pInitialized; bool pRunning; JobManager *pJobManager; XrdSysMutex pMtx; std::unique_ptr pOnConnJob; std::function pOnConnErrCB; XrdSysRWLock pDisconnectLock; }; //---------------------------------------------------------------------------- // Constructor //---------------------------------------------------------------------------- PostMaster::PostMaster(): pImpl( new PostMasterImpl() ) { } //---------------------------------------------------------------------------- // Destructor //---------------------------------------------------------------------------- PostMaster::~PostMaster() { } //---------------------------------------------------------------------------- // Initializer //---------------------------------------------------------------------------- bool PostMaster::Initialize() { Env *env = DefaultEnv::GetEnv(); std::string pollerPref = DefaultPollerPreference; env->GetString( "PollerPreference", pollerPref ); pImpl->pPoller = PollerFactory::CreatePoller( pollerPref ); if( !pImpl->pPoller ) return false; bool st = pImpl->pPoller->Initialize(); if( !st ) { delete pImpl->pPoller; return false; } pImpl->pJobManager->Initialize(); pImpl->pInitialized = true; return true; } //---------------------------------------------------------------------------- // Finalizer //---------------------------------------------------------------------------- bool PostMaster::Finalize() { //-------------------------------------------------------------------------- // Clean up the channels //-------------------------------------------------------------------------- if( !pImpl->pInitialized ) return true; pImpl->pInitialized = false; pImpl->pJobManager->Finalize(); PostMasterImpl::ChannelMap::iterator it; for( it = pImpl->pChannelMap.begin(); it != pImpl->pChannelMap.end(); ++it ) delete it->second; pImpl->pChannelMap.clear(); return pImpl->pPoller->Finalize(); } //---------------------------------------------------------------------------- // Start the post master //---------------------------------------------------------------------------- bool PostMaster::Start() { if( !pImpl->pInitialized ) return false; if( !pImpl->pPoller->Start() ) return false; if( !pImpl->pTaskManager->Start() ) { pImpl->pPoller->Stop(); return false; } if( !pImpl->pJobManager->Start() ) { pImpl->pPoller->Stop(); pImpl->pTaskManager->Stop(); return false; } pImpl->pRunning = true; return true; } //---------------------------------------------------------------------------- // Stop the postmaster //---------------------------------------------------------------------------- bool PostMaster::Stop() { if( !pImpl->pInitialized || !pImpl->pRunning ) return true; if( !pImpl->pJobManager->Stop() ) return false; if( !pImpl->pPoller->Stop() ) return false; if( !pImpl->pTaskManager->Stop() ) return false; pImpl->pRunning = false; return true; } //---------------------------------------------------------------------------- // Reinitialize after fork //---------------------------------------------------------------------------- bool PostMaster::Reinitialize() { return true; } //---------------------------------------------------------------------------- // Send the message asynchronously //---------------------------------------------------------------------------- XRootDStatus PostMaster::Send( const URL &url, Message *msg, MsgHandler *handler, bool stateful, time_t expires ) { XrdSysRWLockHelper scopedLock( pImpl->pDisconnectLock ); Channel *channel = GetChannel( url ); if( !channel ) return XRootDStatus( stError, errNotSupported ); return channel->Send( msg, handler, stateful, expires ); } Status PostMaster::Redirect( const URL &url, Message *msg, MsgHandler *inHandler ) { RedirectorRegistry ®istry = RedirectorRegistry::Instance(); VirtualRedirector *redirector = registry.Get( url ); if( !redirector ) return Status( stError, errInvalidOp ); return redirector->HandleRequest( msg, inHandler ); } //---------------------------------------------------------------------------- // Query the transport handler //---------------------------------------------------------------------------- Status PostMaster::QueryTransport( const URL &url, uint16_t query, AnyObject &result ) { XrdSysRWLockHelper scopedLock( pImpl->pDisconnectLock ); Channel *channel = 0; { XrdSysMutexHelper scopedLock2( pImpl->pChannelMapMutex ); PostMasterImpl::ChannelMap::iterator it = pImpl->pChannelMap.find( url.GetChannelId() ); if( it == pImpl->pChannelMap.end() ) return Status( stError, errInvalidOp ); channel = it->second; } if( !channel ) return Status( stError, errNotSupported ); return channel->QueryTransport( query, result ); } //---------------------------------------------------------------------------- // Register channel event handler //---------------------------------------------------------------------------- Status PostMaster::RegisterEventHandler( const URL &url, ChannelEventHandler *handler ) { XrdSysRWLockHelper scopedLock( pImpl->pDisconnectLock ); Channel *channel = GetChannel( url ); if( !channel ) return Status( stError, errNotSupported ); channel->RegisterEventHandler( handler ); return Status(); } //---------------------------------------------------------------------------- // Remove a channel event handler //---------------------------------------------------------------------------- Status PostMaster::RemoveEventHandler( const URL &url, ChannelEventHandler *handler ) { XrdSysRWLockHelper scopedLock( pImpl->pDisconnectLock ); Channel *channel = GetChannel( url ); if( !channel ) return Status( stError, errNotSupported ); channel->RemoveEventHandler( handler ); return Status(); } //------------------------------------------------------------------------ // Get the task manager object user by the post master //------------------------------------------------------------------------ TaskManager* PostMaster::GetTaskManager() { return pImpl->pTaskManager; } //------------------------------------------------------------------------ // Get the job manager object user by the post master //------------------------------------------------------------------------ JobManager* PostMaster::GetJobManager() { return pImpl->pJobManager; } //------------------------------------------------------------------------ // Shut down a channel //------------------------------------------------------------------------ Status PostMaster::ForceDisconnect( const URL &url ) { XrdSysRWLockHelper scopedLock( pImpl->pDisconnectLock, false ); PostMasterImpl::ChannelMap::iterator it = pImpl->pChannelMap.find( url.GetChannelId() ); if( it == pImpl->pChannelMap.end() ) return Status( stError, errInvalidOp ); it->second->ForceDisconnect(); delete it->second; pImpl->pChannelMap.erase( it ); return Status(); } Status PostMaster::ForceReconnect( const URL &url ) { XrdSysRWLockHelper scopedLock( pImpl->pDisconnectLock, false ); PostMasterImpl::ChannelMap::iterator it = pImpl->pChannelMap.find( url.GetChannelId() ); if( it == pImpl->pChannelMap.end() ) return Status( stError, errInvalidOp ); it->second->ForceReconnect(); return Status(); } //------------------------------------------------------------------------ // Get the number of connected data streams //------------------------------------------------------------------------ uint16_t PostMaster::NbConnectedStrm( const URL &url ) { XrdSysRWLockHelper scopedLock( pImpl->pDisconnectLock ); Channel *channel = GetChannel( url ); if( !channel ) return 0; return channel->NbConnectedStrm(); } //------------------------------------------------------------------------ //! Set the on-connect handler for data streams //------------------------------------------------------------------------ void PostMaster::SetOnDataConnectHandler( const URL &url, std::shared_ptr onConnJob ) { XrdSysRWLockHelper scopedLock( pImpl->pDisconnectLock ); Channel *channel = GetChannel( url ); if( !channel ) return; channel->SetOnDataConnectHandler( onConnJob ); } //------------------------------------------------------------------------ //! Set the global on-connect handler for control streams //------------------------------------------------------------------------ void PostMaster::SetOnConnectHandler( std::unique_ptr onConnJob ) { XrdSysMutexHelper lck( pImpl->pMtx ); pImpl->pOnConnJob = std::move( onConnJob ); } //------------------------------------------------------------------------ // Set the global connection error handler //------------------------------------------------------------------------ void PostMaster::SetConnectionErrorHandler( std::function handler ) { XrdSysMutexHelper lck( pImpl->pMtx ); pImpl->pOnConnErrCB = std::move( handler ); } //------------------------------------------------------------------------ // Notify the global on-connect handler //------------------------------------------------------------------------ void PostMaster::NotifyConnectHandler( const URL &url ) { XrdSysMutexHelper lck( pImpl->pMtx ); if( pImpl->pOnConnJob ) { URL *ptr = new URL( url ); pImpl->pJobManager->QueueJob( pImpl->pOnConnJob.get(), ptr ); } } //------------------------------------------------------------------------ // Notify the global error connection handler //------------------------------------------------------------------------ void PostMaster::NotifyConnErrHandler( const URL &url, const XRootDStatus &status ) { XrdSysMutexHelper lck( pImpl->pMtx ); if( pImpl->pOnConnErrCB ) { ConnErrJob *job = new ConnErrJob( url, status, pImpl->pOnConnErrCB ); pImpl->pJobManager->QueueJob( job, nullptr ); } } //---------------------------------------------------------------------------- //! Collapse channel URL - replace the URL of the channel //---------------------------------------------------------------------------- void PostMaster::CollapseRedirect( const URL &alias, const URL &url ) { XrdSysMutexHelper scopedLock( pImpl->pChannelMapMutex ); //-------------------------------------------------------------------------- // Get the passive channel //-------------------------------------------------------------------------- PostMasterImpl::ChannelMap::iterator it = pImpl->pChannelMap.find( alias.GetChannelId() ); Channel *passive = 0; if( it != pImpl->pChannelMap.end() ) passive = it->second; //-------------------------------------------------------------------------- // If the channel does not exist there's nothing to do //-------------------------------------------------------------------------- else return; //-------------------------------------------------------------------------- // Check if this URL is eligible for collapsing //-------------------------------------------------------------------------- if( !passive->CanCollapse( url ) ) return; //-------------------------------------------------------------------------- // Create the active channel //-------------------------------------------------------------------------- TransportManager *trManager = DefaultEnv::GetTransportManager(); TransportHandler *trHandler = trManager->GetHandler( url.GetProtocol() ); if( !trHandler ) { Log *log = DefaultEnv::GetLog(); log->Error( PostMasterMsg, "Unable to get transport handler for %s " "protocol", url.GetProtocol().c_str() ); return; } Log *log = DefaultEnv::GetLog(); log->Info( PostMasterMsg, "Label channel %s with alias %s.", url.GetHostId().c_str(), alias.GetHostId().c_str() ); Channel *active = new Channel( alias, pImpl->pPoller, trHandler, pImpl->pTaskManager, pImpl->pJobManager, url ); pImpl->pChannelMap[alias.GetChannelId()] = active; //-------------------------------------------------------------------------- // The passive channel will be deallocated by TTL //-------------------------------------------------------------------------- } //------------------------------------------------------------------------ // Decrement file object instance count bound to this channel //------------------------------------------------------------------------ void PostMaster::DecFileInstCnt( const URL &url ) { XrdSysRWLockHelper scopedLock( pImpl->pDisconnectLock ); Channel *channel = GetChannel( url ); if( !channel ) return; return channel->DecFileInstCnt(); } //------------------------------------------------------------------------ //true if underlying threads are running, false otherwise //------------------------------------------------------------------------ bool PostMaster::IsRunning() { return pImpl->pRunning; } //---------------------------------------------------------------------------- // Get the channel //---------------------------------------------------------------------------- Channel *PostMaster::GetChannel( const URL &url ) { XrdSysMutexHelper scopedLock( pImpl->pChannelMapMutex ); Channel *channel = 0; PostMasterImpl::ChannelMap::iterator it = pImpl->pChannelMap.find( url.GetChannelId() ); if( it == pImpl->pChannelMap.end() ) { TransportManager *trManager = DefaultEnv::GetTransportManager(); TransportHandler *trHandler = trManager->GetHandler( url.GetProtocol() ); if( !trHandler ) { Log *log = DefaultEnv::GetLog(); log->Error( PostMasterMsg, "Unable to get transport handler for %s " "protocol", url.GetProtocol().c_str() ); return 0; } channel = new Channel( url, pImpl->pPoller, trHandler, pImpl->pTaskManager, pImpl->pJobManager ); pImpl->pChannelMap[url.GetChannelId()] = channel; } else channel = it->second; return channel; } } xrootd-5.6.9/src/XrdCl/XrdClPostMaster.hh000066400000000000000000000232301457266313600202510ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2012 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . //------------------------------------------------------------------------------ #ifndef __XRD_CL_POST_MASTER_HH__ #define __XRD_CL_POST_MASTER_HH__ #include #include #include #include #include #include "XrdCl/XrdClStatus.hh" #include "XrdCl/XrdClURL.hh" #include "XrdCl/XrdClPostMasterInterfaces.hh" #include "XrdSys/XrdSysPthread.hh" namespace XrdCl { class Poller; class TaskManager; class Channel; class JobManager; class Job; struct PostMasterImpl; //---------------------------------------------------------------------------- //! A hub for dispatching and receiving messages //---------------------------------------------------------------------------- class PostMaster { public: //------------------------------------------------------------------------ //! Constructor //------------------------------------------------------------------------ PostMaster(); //------------------------------------------------------------------------ //! Destructor //------------------------------------------------------------------------ virtual ~PostMaster(); //------------------------------------------------------------------------ //! Initializer //------------------------------------------------------------------------ bool Initialize(); //------------------------------------------------------------------------ //! Finalizer //------------------------------------------------------------------------ bool Finalize(); //------------------------------------------------------------------------ //! Start the post master //------------------------------------------------------------------------ bool Start(); //------------------------------------------------------------------------ //! Stop the postmaster //------------------------------------------------------------------------ bool Stop(); //------------------------------------------------------------------------ //! Reinitialize after fork //------------------------------------------------------------------------ bool Reinitialize(); //------------------------------------------------------------------------ //! Send the message asynchronously - the message is inserted into the //! send queue and a listener is called when the message is succesfsully //! pushed through the wire or when the timeout elapses //! //! DEADLOCK WARNING: no lock should be taken while calling this method //! that are used in the callback as well. //! //! @param url recipient of the message //! @param msg message to be sent //! @param expires unix timestamp after which a failure is reported //! to the handler //! @param handler handler will be notified about the status //! @param stateful physical stream disconnection causes an error //! @return success if the message was successfully inserted //! into the send queues, failure otherwise //------------------------------------------------------------------------ XRootDStatus Send( const URL &url, Message *msg, MsgHandler *handler, bool stateful, time_t expires ); //------------------------------------------------------------------------ //! //------------------------------------------------------------------------ Status Redirect( const URL &url, Message *msg, MsgHandler *handler); //------------------------------------------------------------------------ //! Query the transport handler for a given URL //! //! @param url the channel to be queried //! @param query the query as defined in the TransportQuery struct or //! others that may be recognized by the protocol transport //! @param result the result of the query //! @return status of the query //------------------------------------------------------------------------ Status QueryTransport( const URL &url, uint16_t query, AnyObject &result ); //------------------------------------------------------------------------ //! Register channel event handler //------------------------------------------------------------------------ Status RegisterEventHandler( const URL &url, ChannelEventHandler *handler ); //------------------------------------------------------------------------ //! Remove a channel event handler //------------------------------------------------------------------------ Status RemoveEventHandler( const URL &url, ChannelEventHandler *handler ); //------------------------------------------------------------------------ //! Get the task manager object user by the post master //------------------------------------------------------------------------ TaskManager *GetTaskManager(); //------------------------------------------------------------------------ //! Get the job manager object user by the post master //------------------------------------------------------------------------ JobManager *GetJobManager(); //------------------------------------------------------------------------ //! Shut down a channel //------------------------------------------------------------------------ Status ForceDisconnect( const URL &url ); //------------------------------------------------------------------------ //! Reconnect the channel //------------------------------------------------------------------------ Status ForceReconnect( const URL &url ); //------------------------------------------------------------------------ //! Get the number of connected data streams //------------------------------------------------------------------------ uint16_t NbConnectedStrm( const URL &url ); //------------------------------------------------------------------------ //! Set the on-connect handler for data streams //------------------------------------------------------------------------ void SetOnDataConnectHandler( const URL &url, std::shared_ptr onConnJob ); //------------------------------------------------------------------------ //! Set the global connection error handler //------------------------------------------------------------------------ void SetOnConnectHandler( std::unique_ptr onConnJob ); //------------------------------------------------------------------------ //! Set the global on-error on-connect handler for control streams //------------------------------------------------------------------------ void SetConnectionErrorHandler( std::function handler ); //------------------------------------------------------------------------ //! Notify the global on-connect handler //------------------------------------------------------------------------ void NotifyConnectHandler( const URL &url ); //------------------------------------------------------------------------ //! Notify the global error connection handler //------------------------------------------------------------------------ void NotifyConnErrHandler( const URL &url, const XRootDStatus &status ); //------------------------------------------------------------------------ //! Collapse channel URL - replace the URL of the channel //------------------------------------------------------------------------ void CollapseRedirect( const URL &oldurl, const URL &newURL ); //------------------------------------------------------------------------ //! Decrement file object instance count bound to this channel //------------------------------------------------------------------------ void DecFileInstCnt( const URL &url ); //------------------------------------------------------------------------ //! @return : true if underlying threads are running, false otherwise //------------------------------------------------------------------------ bool IsRunning(); private: Channel *GetChannel( const URL &url ); std::unique_ptr pImpl; }; } #endif // __XRD_CL_POST_MASTER_HH__ xrootd-5.6.9/src/XrdCl/XrdClPostMasterInterfaces.hh000066400000000000000000000572601457266313600222670ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2014 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #ifndef __XRD_CL_POST_MASTER_INTERFACES_HH__ #define __XRD_CL_POST_MASTER_INTERFACES_HH__ #include "XrdCl/XrdClXRootDResponses.hh" #include #include #include #include "XrdCl/XrdClStatus.hh" #include "XrdCl/XrdClAnyObject.hh" #include "XrdCl/XrdClURL.hh" class XrdNetAddr; namespace XrdCl { class Channel; class Message; class URL; class Socket; //---------------------------------------------------------------------------- //! Message handler //---------------------------------------------------------------------------- class MsgHandler { public: //------------------------------------------------------------------------ //! Actions to be taken after a message is processed by the handler //------------------------------------------------------------------------ enum Action { None = 0x0000, Nop = 0x0001, //!< A place holder Ignore = 0x0002, //!< Ignore the message RemoveHandler = 0x0004, //!< Remove the handler from the notification //!< list Raw = 0x0008, //!< the handler is interested in reading //!< the message body directly from the //!< socket NoProcess = 0x0010, //!< don't call the processing callback //!< even if the message belongs to this //!< handler Corrupted = 0x0020, //!< the handler discovered that the message //!< header is corrupted, we will have to //!< tear down the socket More = 0x0040 //!< there are more (non-raw) data to be read }; //------------------------------------------------------------------------ //! Events that may have occurred to the stream //------------------------------------------------------------------------ enum StreamEvent { Ready = 1, //!< The stream has become connected Broken = 2, //!< The stream is broken Timeout = 3, //!< The declared timeout has occurred FatalError = 4 //!< Stream has been broken and won't be recovered }; //------------------------------------------------------------------------ //! Event types that the message handler may receive //------------------------------------------------------------------------ virtual ~MsgHandler() {} //------------------------------------------------------------------------ //! Examine an incoming message, and decide on the action to be taken //! //! @param msg the message, may be zero if receive failed //! @return action type that needs to be take wrt the message and //! the handler //------------------------------------------------------------------------ virtual uint16_t Examine( std::shared_ptr &msg ) = 0; //------------------------------------------------------------------------ //! Reexamine the incoming message, and decide on the action to be taken //! //! In case of kXR_status the message can be only fully examined after //! reading the whole body (without raw data). //! //! @param msg the message, may be zero if receive failed //! @return action type that needs to be take wrt the message and //! the handler //------------------------------------------------------------------------ virtual uint16_t InspectStatusRsp() = 0; //------------------------------------------------------------------------ //! Get handler sid //! //! return sid of the corresponding request, otherwise 0 //------------------------------------------------------------------------ virtual uint16_t GetSid() const = 0; //------------------------------------------------------------------------ //! Process the message if it was "taken" by the examine action //! //! @param msg the message to be processed //------------------------------------------------------------------------ virtual void Process() {}; //------------------------------------------------------------------------ //! Read message body directly from a socket - called if Examine returns //! Raw flag - only socket related errors may be returned here //! //! @param msg the corresponding message header //! @param socket the socket to read from //! @param bytesRead number of bytes read by the method //! @return stOK & suDone if the whole body has been processed //! stOK & suRetry if more data is needed //! stError on failure //------------------------------------------------------------------------ virtual XRootDStatus ReadMessageBody( Message *msg, Socket *socket, uint32_t &bytesRead ) { (void)msg; (void)socket; (void)bytesRead; return Status( stOK, suDone ); }; //------------------------------------------------------------------------ //! Handle an event other that a message arrival //! //! @param event type of the event //! @param status status info //! @return Action::RemoveHandler or 0 //------------------------------------------------------------------------ virtual uint8_t OnStreamEvent( StreamEvent event, XRootDStatus status ) { (void)event; (void)status; return 0; }; //------------------------------------------------------------------------ //! The requested action has been performed and the status is available //------------------------------------------------------------------------ virtual void OnStatusReady( const Message *message, XRootDStatus status ) = 0; //------------------------------------------------------------------------ //! Called just before the message is going to be sent through //! a valid connection, so that the user can still make some //! modifications that were impossible before (ie. protocol version //! dependent adjustments) //! //! @param msg message concerned //------------------------------------------------------------------------ virtual void OnReadyToSend( Message *msg ) { (void)msg; }; //------------------------------------------------------------------------ //! Determines whether the handler wants to write some data directly //! to the socket after the message (or message header) has been sent, //! WriteMessageBody will be called //------------------------------------------------------------------------ virtual bool IsRaw() const { return false; } //------------------------------------------------------------------------ //! Write message body directly to a socket - called if IsRaw returns //! true - only socket related errors may be returned here //! //! @param socket the socket to read from //! @param bytesWritten number of bytes written by the method //! @return stOK & suDone if the whole body has been processed //! stOK & suRetry if more data needs to be written //! stError on failure //------------------------------------------------------------------------ virtual XRootDStatus WriteMessageBody( Socket *socket, uint32_t &bytesWritten ) { (void)socket; (void)bytesWritten; return Status(); } virtual time_t GetExpiration() = 0; }; //---------------------------------------------------------------------------- //! Channel event handler //---------------------------------------------------------------------------- class ChannelEventHandler { public: //------------------------------------------------------------------------ //! Events that may have occurred to the channel //------------------------------------------------------------------------ enum ChannelEvent { StreamReady = 1, //!< The stream has become connected StreamBroken = 2, //!< The stream is broken FatalError = 4 //!< Stream has been broken and won't be recovered }; //------------------------------------------------------------------------ //! Destructor //------------------------------------------------------------------------ virtual ~ChannelEventHandler() {}; //------------------------------------------------------------------------ //! Event callback //! //! @param event the event that has occurred //! @param status the status info //! @return true if the handler should be kept //! false if it should be removed from further consideration //------------------------------------------------------------------------ virtual bool OnChannelEvent( ChannelEvent event, Status status ) = 0; }; //---------------------------------------------------------------------------- //! Data structure that carries the handshake information //---------------------------------------------------------------------------- struct HandShakeData { //-------------------------------------------------------------------------- //! Constructor //-------------------------------------------------------------------------- HandShakeData( const URL *addr, uint16_t subStream ): step(0), out(0), in(0), url(addr), subStreamId( subStream ), startTime( time(0) ), serverAddr(0) {} uint16_t step; //!< Handshake step Message *out; //!< Message to be sent out Message *in; //!< Message that has been received const URL *url; //!< Destination URL uint16_t subStreamId; //!< Sub-stream id time_t startTime; //!< Timestamp of when the handshake started const XrdNetAddr *serverAddr; //!< Server address std::string clientName; //!< Client name (an IPv6 representation) std::string streamName; //!< Name of the stream }; //---------------------------------------------------------------------------- //! Path ID - a pair of integers describing the up and down stream //! for given interaction //---------------------------------------------------------------------------- struct PathID { PathID( uint16_t u = 0, uint16_t d = 0 ): up(u), down(d) {} uint16_t up; uint16_t down; }; //---------------------------------------------------------------------------- //! Transport query definitions //! The transports may support other queries, with ids > 1000 //---------------------------------------------------------------------------- struct TransportQuery { static const uint16_t Name = 1; //!< Transport name, returns const char * static const uint16_t Auth = 2; //!< Transport name, returns std::string * }; //---------------------------------------------------------------------------- //! XRootD related protocol queries //---------------------------------------------------------------------------- struct XRootDQuery { static const uint16_t ServerFlags = 1002; //!< returns server flags static const uint16_t ProtocolVersion = 1003; //!< returns the protocol version static const uint16_t IsEncrypted = 1004; //!< returns true if the channel is encrypted }; //---------------------------------------------------------------------------- //! Stream query definitions //! The transports may support other queries, with ids > 2999 and ids < 2000 //---------------------------------------------------------------------------- struct StreamQuery { static const uint16_t IpAddr = 2001; static const uint16_t IpStack = 2002; static const uint16_t HostName = 2003; }; //---------------------------------------------------------------------------- //! Perform the handshake and the authentication for each physical stream //---------------------------------------------------------------------------- class TransportHandler { public: //------------------------------------------------------------------------ //! Stream actions that may be triggered by incoming control messages //------------------------------------------------------------------------ enum StreamAction { NoAction = 0x0000, //!< No action DigestMsg = 0x0001, //!< Digest the incoming message so that it won't //!< be passed to the user handlers AbortStream = 0x0002, //!< Disconnect, abort all the on-going //!< operations and mark the stream as //!< permanently broken [not yet implemented] CloseStream = 0x0004, //!< Disconnect and attempt reconnection later //!< [not yet implemented] ResumeStream = 0x0008, //!< Resume sending requests //!< [not yet implemented] HoldStream = 0x0010, //!< Stop sending requests [not yet implemented] RequestClose = 0x0020 //!< Send a close request }; virtual ~TransportHandler() {} //------------------------------------------------------------------------ //! Read a message header from the socket, the socket is non-blocking, //! so if there is not enough data the function should return errRetry //! in which case it will be called again when more data arrives, with //! the data previously read stored in the message buffer //! //! @param message the message buffer //! @param socket the socket //! @return stOK & suDone if the whole message has been processed //! stOK & suRetry if more data is needed //! stError on failure //------------------------------------------------------------------------ virtual XRootDStatus GetHeader( Message &message, Socket *socket ) = 0; //------------------------------------------------------------------------ //! Read the message body from the socket, the socket is non-blocking, //! the method may be called multiple times - see GetHeader for details //! //! @param message the message buffer containing the header //! @param socket the socket //! @return stOK & suDone if the whole message has been processed //! stOK & suRetry if more data is needed //! stError on failure //------------------------------------------------------------------------ virtual XRootDStatus GetBody( Message &message, Socket *socket ) = 0; //------------------------------------------------------------------------ //! Read more of the message body from the socket, the socket is //! non-blocking the method may be called multiple times - see GetHeader //! for details //! //! @param message the message buffer containing the header //! @param socket the socket //! @return stOK & suDone if the whole message has been processed //! stOK & suRetry if more data is needed //! stError on failure //------------------------------------------------------------------------ virtual XRootDStatus GetMore( Message &message, Socket *socket ) = 0; //------------------------------------------------------------------------ //! Initialize channel //------------------------------------------------------------------------ virtual void InitializeChannel( const URL &url, AnyObject &channelData ) = 0; //------------------------------------------------------------------------ //! Finalize channel //------------------------------------------------------------------------ virtual void FinalizeChannel( AnyObject &channelData ) = 0; //------------------------------------------------------------------------ //! HandHake //------------------------------------------------------------------------ virtual XRootDStatus HandShake( HandShakeData *handShakeData, AnyObject &channelData ) = 0; //------------------------------------------------------------------------ // @return true if handshake has been done and stream is connected, // false otherwise //------------------------------------------------------------------------ virtual bool HandShakeDone( HandShakeData *handShakeData, AnyObject &channelData ) = 0; //------------------------------------------------------------------------ //! Check if the stream should be disconnected //------------------------------------------------------------------------ virtual bool IsStreamTTLElapsed( time_t inactiveTime, AnyObject &channelData ) = 0; //------------------------------------------------------------------------ //! Check the stream is broken - ie. TCP connection got broken and //! went undetected by the TCP stack //------------------------------------------------------------------------ virtual Status IsStreamBroken( time_t inactiveTime, AnyObject &channelData ) = 0; //------------------------------------------------------------------------ //! Return the ID for the up stream this message should be sent by //! and the down stream which the answer should be expected at. //! Modify the message itself if necessary. //! If hint is non-zero then the message should be modified such that //! the answer will be returned via the hinted stream. //------------------------------------------------------------------------ virtual PathID Multiplex( Message *msg, AnyObject &channelData, PathID *hint = 0 ) = 0; //------------------------------------------------------------------------ //! Return the ID for the up substream this message should be sent by //! and the down substream which the answer should be expected at. //! Modify the message itself if necessary. //! If hint is non-zero then the message should be modified such that //! the answer will be returned via the hinted stream. //------------------------------------------------------------------------ virtual PathID MultiplexSubStream( Message *msg, AnyObject &channelData, PathID *hint = 0 ) = 0; //------------------------------------------------------------------------ //! Return a number of substreams per stream that should be created //------------------------------------------------------------------------ virtual uint16_t SubStreamNumber( AnyObject &channelData ) = 0; //------------------------------------------------------------------------ //! The stream has been disconnected, do the cleanups //------------------------------------------------------------------------ virtual void Disconnect( AnyObject &channelData, uint16_t subStreamId ) = 0; //------------------------------------------------------------------------ //! Query the channel //------------------------------------------------------------------------ virtual Status Query( uint16_t query, AnyObject &result, AnyObject &channelData ) = 0; //------------------------------------------------------------------------ //! Check if the message invokes a stream action //------------------------------------------------------------------------ virtual uint32_t MessageReceived( Message &msg, uint16_t subStream, AnyObject &channelData ) = 0; //------------------------------------------------------------------------ //! Notify the transport about a message having been sent //------------------------------------------------------------------------ virtual void MessageSent( Message *msg, uint16_t subStream, uint32_t bytesSent, AnyObject &channelData ) = 0; //------------------------------------------------------------------------ //! Wait before exit //------------------------------------------------------------------------ virtual void WaitBeforeExit() = 0; //------------------------------------------------------------------------ //! @return : true if encryption should be turned on, false otherwise //------------------------------------------------------------------------ virtual bool NeedEncryption( HandShakeData *handShakeData, AnyObject &channelData ) = 0; //------------------------------------------------------------------------ //! Get signature for given message //------------------------------------------------------------------------ virtual Status GetSignature( Message *toSign, Message *&sign, AnyObject &channelData ) = 0; //------------------------------------------------------------------------ //! Decrement file object instance count bound to this channel //------------------------------------------------------------------------ virtual void DecFileInstCnt( AnyObject &channelData ) = 0; //------------------------------------------------------------------------ //! Get bind preference for the next data stream //------------------------------------------------------------------------ virtual URL GetBindPreference( const URL &url, AnyObject &channelData ) = 0; }; } #endif // __XRD_CL_POST_MASTER_INTERFACES_HH__ xrootd-5.6.9/src/XrdCl/XrdClPropertyList.hh000066400000000000000000000260511457266313600206340ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2014 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //----------------------------------------------------------------------------- // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #ifndef __XRD_CL_PROPERTY_LIST_HH__ #define __XRD_CL_PROPERTY_LIST_HH__ #include #include #include #include #include "XrdCl/XrdClXRootDResponses.hh" namespace XrdCl { //---------------------------------------------------------------------------- //! A key-value pair map storing both keys and values as strings //---------------------------------------------------------------------------- class PropertyList { public: typedef std::map PropertyMap; //------------------------------------------------------------------------ //! Associate a value with a key //! //! @param name must not contain spaces //! @param value needs to be convertible to std::string //------------------------------------------------------------------------ template void Set( const std::string &name, const Item &value ) { std::ostringstream o; o << value; pProperties[name] = o.str(); } //------------------------------------------------------------------------ //! Get the value associated with a name //! //! @return true if the name was found, false otherwise //------------------------------------------------------------------------ template bool Get( const std::string &name, Item &item ) const { PropertyMap::const_iterator it; it = pProperties.find( name ); if( it == pProperties.end() ) return false; std::istringstream i; i.str( it->second ); i >> item; if( i.bad() ) return false; return true; } //------------------------------------------------------------------------ //! Get the value associated with a name //! //! @return the value or Item() if the key does not exist //------------------------------------------------------------------------ template Item Get( const std::string &name ) const { PropertyMap::const_iterator it; it = pProperties.find( name ); if( it == pProperties.end() ) return Item(); std::istringstream i; i.str( it->second ); Item item; i >> item; if( i.bad() ) return Item(); return item; } //------------------------------------------------------------------------ //! Set a value with a name and an index //! //! @param name must not contain spaces //! @param index //! @param value must be convertible to std::string //------------------------------------------------------------------------ template void Set( const std::string &name, uint32_t index, const Item &value ) { std::ostringstream o; o << name << " " << index; Set( o.str(), value ); } //------------------------------------------------------------------------ //! Get the value associated with a key and an index //! //! @return true if the key and index were found, false otherwise //------------------------------------------------------------------------ template bool Get( const std::string &name, uint32_t index, Item &item ) const { std::ostringstream o; o << name << " " << index; return Get( o.str(), item ); } //------------------------------------------------------------------------ //! Get the value associated with a key and an index //! //! @return the value or Item() if the key does not exist //------------------------------------------------------------------------ template Item Get( const std::string &name, uint32_t index ) const { std::ostringstream o; o << name << " " << index; return Get( o.str() ); } //------------------------------------------------------------------------ //! Check if we now about the given name //------------------------------------------------------------------------ bool HasProperty( const std::string &name ) const { return pProperties.find( name ) != pProperties.end(); } //------------------------------------------------------------------------ //! Check if we know about the given name and index //------------------------------------------------------------------------ bool HasProperty( const std::string &name, uint32_t index ) const { std::ostringstream o; o << name << " " << index; return HasProperty( o.str() ); } //------------------------------------------------------------------------ //! Get the begin iterator //------------------------------------------------------------------------ PropertyMap::const_iterator begin() const { return pProperties.begin(); } //------------------------------------------------------------------------ //! Get the end iterator //------------------------------------------------------------------------ PropertyMap::const_iterator end() const { return pProperties.end(); } //------------------------------------------------------------------------ //! Clear the property list //------------------------------------------------------------------------ void Clear() { pProperties.clear(); } private: PropertyMap pProperties; }; //---------------------------------------------------------------------------- // Specialize get for strings //---------------------------------------------------------------------------- template<> inline bool PropertyList::Get( const std::string &name, std::string &item ) const { PropertyMap::const_iterator it; it = pProperties.find( name ); if( it == pProperties.end() ) return false; item = it->second; return true; } template<> inline std::string PropertyList::Get( const std::string &name ) const { PropertyMap::const_iterator it; it = pProperties.find( name ); if( it == pProperties.end() ) return std::string(); return it->second; } //---------------------------------------------------------------------------- // Specialize set for XRootDStatus //---------------------------------------------------------------------------- template<> inline void PropertyList::Set( const std::string &name, const XRootDStatus &item ) { std::ostringstream o; o << item.status << ";" << item.code << ";" << item.errNo << "#"; o << item.GetErrorMessage(); Set( name, o.str() ); } //---------------------------------------------------------------------------- // Specialize get for XRootDStatus //---------------------------------------------------------------------------- template<> inline bool PropertyList::Get( const std::string &name, XRootDStatus &item ) const { std::string str, msg, tmp; if( !Get( name, str ) ) return false; std::string::size_type i; i = str.find( '#' ); if( i == std::string::npos ) return false; item.SetErrorMessage( str.substr( i+1, str.length()-i-1 ) ); str.erase( i, str.length()-i ); std::replace( str.begin(), str.end(), ';', ' ' ); std::istringstream is; is.str( str ); is >> item.status; if( is.bad() ) return false; is >> item.code; if( is.bad() ) return false; is >> item.errNo; if( is.bad() ) return false; return true; } template<> inline XRootDStatus PropertyList::Get( const std::string &name ) const { XRootDStatus st; if( !Get( name, st ) ) return XRootDStatus(); return st; } //---------------------------------------------------------------------------- // Specialize set for URL //---------------------------------------------------------------------------- template<> inline void PropertyList::Set( const std::string &name, const URL &item ) { Set( name, item.GetURL() ); } //---------------------------------------------------------------------------- // Specialize get for URL //---------------------------------------------------------------------------- template<> inline bool PropertyList::Get( const std::string &name, URL &item ) const { std::string tmp; if( !Get( name, tmp ) ) return false; item = tmp; return true; } //---------------------------------------------------------------------------- // Specialize set for vector //---------------------------------------------------------------------------- template<> inline void PropertyList::Set >( const std::string &name, const std::vector &item ) { std::vector::const_iterator it; int i = 0; for( it = item.begin(); it != item.end(); ++it, ++i ) Set( name, i, *it ); } //---------------------------------------------------------------------------- // Specialize get for XRootDStatus //---------------------------------------------------------------------------- template<> inline bool PropertyList::Get >( const std::string &name, std::vector &item ) const { std::string tmp; item.clear(); for( int i = 0; HasProperty( name, i ); ++i ) { if( !Get( name, i, tmp ) ) return false; item.push_back( tmp ); } return true; } } #endif // __XRD_OUC_PROPERTY_LIST_HH__ xrootd-5.6.9/src/XrdCl/XrdClRedirectorRegistry.cc000066400000000000000000000107101457266313600217700ustar00rootroot00000000000000/* * XrdClRedirectorRegister.cc * * Created on: May 23, 2016 * Author: simonm */ #include "XrdClRedirectorRegistry.hh" #include "XrdCl/XrdClMetalinkRedirector.hh" #include "XrdCl/XrdClPostMasterInterfaces.hh" #include "XrdCl/XrdClDefaultEnv.hh" #include "XrdCl/XrdClConstants.hh" #include "XrdCl/XrdClLog.hh" #include namespace XrdCl { void RedirectJob::Run( void* ) { // this makes sure the handler takes ownership of the new message if( pHandler->Examine( msg ) != MsgHandler::Action::Ignore ) pHandler->Process(); delete this; } RedirectorRegistry& RedirectorRegistry::Instance() { static RedirectorRegistry redirector; return redirector; } RedirectorRegistry::~RedirectorRegistry() { RedirectorMap::iterator itr; for( itr = pRegistry.begin(); itr != pRegistry.end(); ++itr ) delete itr->second.first; } XRootDStatus RedirectorRegistry::RegisterImpl( const URL &u, ResponseHandler *handler ) { URL url = ConvertLocalfile( u ); // we can only create a virtual redirector if // a path to a metadata file has been provided if( url.GetPath().empty() ) return XRootDStatus( stError, errNotSupported ); // regarding file protocol we only support localhost if( url.GetProtocol() == "file" && url.GetHostName() != "localhost" ) return XRootDStatus( stError, errNotSupported ); XrdSysMutexHelper scopedLock( pMutex ); // get the key and check if it is already in the registry const std::string key = url.GetLocation(); RedirectorMap::iterator itr = pRegistry.find( key ); if( itr != pRegistry.end() ) { // increment user counter ++itr->second.second; if( handler ) handler->HandleResponseWithHosts( new XRootDStatus(), 0, 0 ); return XRootDStatus( stOK, suAlreadyDone ); } // If it is a Metalink create a MetalinkRedirector if( url.IsMetalink() ) { MetalinkRedirector *redirector = new MetalinkRedirector( key ); XRootDStatus st = redirector->Load( handler ); if( !st.IsOK() ) delete redirector; else pRegistry[key] = std::pair( redirector, 1 ); return st; } else // so far we only support Metalink metadata format return XRootDStatus( stError, errNotSupported ); } URL RedirectorRegistry::ConvertLocalfile( const URL &url ) { int localml = DefaultLocalMetalinkFile; DefaultEnv::GetEnv()->GetInt( "LocalMetalinkFile", localml ); if( localml && url.GetProtocol() == "root" && url.GetHostName() == "localfile" ) { Log *log = DefaultEnv::GetLog(); log->Warning( PostMasterMsg, "Please note that the 'root://localfile//path/filename.meta4' " "semantic is now deprecated, use 'file://localhost/path/filename.meta4'" "instead!" ); URL copy( url ); copy.SetHostName( "localhost" ); copy.SetProtocol( "file" ); return copy; } return url; } XRootDStatus RedirectorRegistry::Register( const URL &url ) { return RegisterImpl( url, 0 ); } XRootDStatus RedirectorRegistry::RegisterAndWait( const URL &url ) { SyncResponseHandler handler; Status st = RegisterImpl( url, &handler ); if( !st.IsOK() ) return st; return MessageUtils::WaitForStatus( &handler ); } VirtualRedirector* RedirectorRegistry::Get( const URL &u ) const { URL url = ConvertLocalfile( u ); XrdSysMutexHelper scopedLock( pMutex ); // get the key and return the value if it is in the registry // offset 24 is where the path has been stored const std::string key = url.GetLocation(); RedirectorMap::const_iterator itr = pRegistry.find( key ); if( itr != pRegistry.end() ) return itr->second.first; // otherwise return null return 0; } //---------------------------------------------------------------------------- // Release the virtual redirector associated with the given URL //---------------------------------------------------------------------------- void RedirectorRegistry::Release( const URL &u ) { URL url = ConvertLocalfile( u ); XrdSysMutexHelper scopedLock( pMutex ); // get the key and return the value if it is in the registry // offset 24 is where the path has been stored const std::string key = url.GetLocation(); RedirectorMap::iterator itr = pRegistry.find( key ); if( itr == pRegistry.end() ) return; // decrement user counter --itr->second.second; // if nobody is using it delete the object // and remove it from the registry if( !itr->second.second ) { delete itr->second.first; pRegistry.erase( itr ); } } } /* namespace XrdCl */ xrootd-5.6.9/src/XrdCl/XrdClRedirectorRegistry.hh000066400000000000000000000201021457266313600217760ustar00rootroot00000000000000/* * XrdClRedirectorRegister.hh * * Created on: May 23, 2016 * Author: simonm */ #ifndef SRC_XRDCL_XRDCLREDIRECTORREGISTRY_HH_ #define SRC_XRDCL_XRDCLREDIRECTORREGISTRY_HH_ #include "XrdCl/XrdClXRootDResponses.hh" #include "XrdCl/XrdClURL.hh" #include "XrdCl/XrdClJobManager.hh" #include "XrdSys/XrdSysPthread.hh" #include #include #include namespace XrdCl { class Message; class MsgHandler; //-------------------------------------------------------------------------------- //! A job class for redirect handling in the thread-pool //-------------------------------------------------------------------------------- class RedirectJob: public Job { public: //------------------------------------------------------------------------ //! Constructor //------------------------------------------------------------------------ RedirectJob( MsgHandler *handler, std::shared_ptr msg ) : pHandler( handler ), msg( msg ) { } //------------------------------------------------------------------------ //! Destructor //------------------------------------------------------------------------ virtual ~RedirectJob() { } //------------------------------------------------------------------------ //! Run the user handler //------------------------------------------------------------------------ virtual void Run( void *arg ); private: MsgHandler *pHandler; std::shared_ptr msg; }; //-------------------------------------------------------------------------------- //! An interface for metadata redirectors. //-------------------------------------------------------------------------------- class VirtualRedirector { public: //---------------------------------------------------------------------------- //! Destructor. //---------------------------------------------------------------------------- virtual ~VirtualRedirector(){} //---------------------------------------------------------------------------- //! Creates an instant redirect response for the given message //! or an error response if there are no more replicas to try. //! The virtual response is being handled by the given handler //! in the thread-pool. //---------------------------------------------------------------------------- virtual XRootDStatus HandleRequest( const Message *msg, MsgHandler *handler ) = 0; //---------------------------------------------------------------------------- //! Initializes the object with the content of the metalink file //---------------------------------------------------------------------------- virtual XRootDStatus Load( ResponseHandler *userHandler ) = 0; //---------------------------------------------------------------------------- //! Gets the file name as specified in the metalink //---------------------------------------------------------------------------- virtual std::string GetTargetName() const = 0; //---------------------------------------------------------------------------- //! Returns the checksum of the given type if specified //! in the metalink file, or an empty string otherwise //---------------------------------------------------------------------------- virtual std::string GetCheckSum( const std::string &type ) const = 0; //---------------------------------------------------------------------------- //! Returns the default checksum type (the first one given in the metalink), //! if no checksum is available returns an empty string //---------------------------------------------------------------------------- virtual std::vector GetSupportedCheckSums() const = 0; //---------------------------------------------------------------------------- //! Returns the file size as specified in the metalink, //! or a negative number if size was not specified //---------------------------------------------------------------------------- virtual long long GetSize() const = 0; //---------------------------------------------------------------------------- //! Returns a vector with replicas as given in the meatlink file //---------------------------------------------------------------------------- virtual const std::vector& GetReplicas() = 0; //---------------------------------------------------------------------------- //! Count how many replicas do we have left to try for given request //---------------------------------------------------------------------------- virtual int Count( Message &req ) const = 0; }; //-------------------------------------------------------------------------------- //! Singleton access to URL to virtual redirector mapping. //-------------------------------------------------------------------------------- class RedirectorRegistry { public: //---------------------------------------------------------------------------- //! Returns reference to the single instance. //---------------------------------------------------------------------------- static RedirectorRegistry& Instance(); //---------------------------------------------------------------------------- //! Destructor //---------------------------------------------------------------------------- ~RedirectorRegistry(); //---------------------------------------------------------------------------- //! Creates a new virtual redirector and registers it (async). //---------------------------------------------------------------------------- XRootDStatus Register( const URL &url ); //---------------------------------------------------------------------------- //! Creates a new virtual redirector and registers it (sync). //---------------------------------------------------------------------------- XRootDStatus RegisterAndWait( const URL &url ); //---------------------------------------------------------------------------- //! Get a virtual redirector associated with the given URL. //---------------------------------------------------------------------------- VirtualRedirector* Get( const URL &url ) const; //---------------------------------------------------------------------------- //! Release the virtual redirector associated with the given URL //---------------------------------------------------------------------------- void Release( const URL &url ); private: typedef std::map< std::string, std::pair > RedirectorMap; //---------------------------------------------------------------------------- //! Register implementation. //---------------------------------------------------------------------------- XRootDStatus RegisterImpl( const URL &url, ResponseHandler *handler ); //---------------------------------------------------------------------------- //! Convert the old convention for accessing local metalink files: //! root://localfile//path/metalink.meta4 //! into: //! file://localhost/path/metalink.meta4 //---------------------------------------------------------------------------- static URL ConvertLocalfile( const URL &url ); //---------------------------------------------------------------------------- // Constructor (private!). //---------------------------------------------------------------------------- RedirectorRegistry() {} //---------------------------------------------------------------------------- // Copy constructor (private!). //---------------------------------------------------------------------------- RedirectorRegistry( const RedirectorRegistry & ); //---------------------------------------------------------------------------- // Assignment operator (private!). //---------------------------------------------------------------------------- RedirectorRegistry& operator=( const RedirectorRegistry & ); RedirectorMap pRegistry; mutable XrdSysMutex pMutex; }; } /* namespace XrdCl */ #endif /* SRC_XRDCL_XRDCLREDIRECTORREGISTRY_HH_ */ xrootd-5.6.9/src/XrdCl/XrdClRequestSync.hh000066400000000000000000000101321457266313600204320ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2014 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #ifndef __XRD_CL_REQUEST_SYNC_HH__ #define __XRD_CL_REQUEST_SYNC_HH__ #include "XrdSys/XrdSysPthread.hh" namespace XrdCl { //---------------------------------------------------------------------------- //! A helper running a fixed number of requests at a given time //---------------------------------------------------------------------------- class RequestSync { public: //------------------------------------------------------------------------ //! Constructor //! //! @param reqTotal total number of requests //! @param reqQuota number of requests to be run in parallel //------------------------------------------------------------------------ RequestSync( uint32_t reqTotal, uint32_t reqQuota ): pQuotaSem( new XrdSysSemaphore( reqQuota ) ), pTotalSem( new XrdSysSemaphore( 0 ) ), pRequestsLeft( reqTotal ), pFailureCounter( 0 ) { if( !reqTotal ) pTotalSem->Post(); } //------------------------------------------------------------------------ //! Destructor //------------------------------------------------------------------------ ~RequestSync() { delete pQuotaSem; delete pTotalSem; } //------------------------------------------------------------------------ //! Wait for the request quota //------------------------------------------------------------------------ void WaitForQuota() { pQuotaSem->Wait(); } //------------------------------------------------------------------------ //! Wait for all the requests to be finished //------------------------------------------------------------------------ void WaitForAll() { pTotalSem->Wait(); } //------------------------------------------------------------------------ //! Report the request finish //------------------------------------------------------------------------ void TaskDone( bool success = true ) { XrdSysMutexHelper scopedLock( pMutex ); if( !success ) ++pFailureCounter; --pRequestsLeft; pQuotaSem->Post(); if( !pRequestsLeft ) pTotalSem->Post(); } //------------------------------------------------------------------------ //! Number of tasks finishing with an error //------------------------------------------------------------------------ uint32_t FailureCount() const { return pFailureCounter; } private: RequestSync(const RequestSync &other); RequestSync &operator = (const RequestSync &other); XrdSysMutex pMutex; XrdSysSemaphore *pQuotaSem; XrdSysSemaphore *pTotalSem; uint32_t pRequestsLeft; uint32_t pFailureCounter; }; } #endif // __XRD_CL_REQUEST_SYNC_HH__ xrootd-5.6.9/src/XrdCl/XrdClResponseJob.hh000066400000000000000000000051431457266313600204040ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2013 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . //------------------------------------------------------------------------------ #ifndef __XRD_CL_RESPONSE_JOB_HH__ #define __XRD_CL_RESPONSE_JOB_HH__ #include "XrdCl/XrdClJobManager.hh" #include "XrdCl/XrdClXRootDResponses.hh" namespace XrdCl { //---------------------------------------------------------------------------- //! Call the user callback //---------------------------------------------------------------------------- class ResponseJob: public Job { public: //------------------------------------------------------------------------ //! Constructor //------------------------------------------------------------------------ ResponseJob( ResponseHandler *handler, XRootDStatus *status, AnyObject *response, HostList *hostList ): pHandler( handler ), pStatus( status ), pResponse( response ), pHostList( hostList ) { } //------------------------------------------------------------------------ //! Destructor //------------------------------------------------------------------------ virtual ~ResponseJob() { } //------------------------------------------------------------------------ //! Run the user handler //------------------------------------------------------------------------ virtual void Run( void *arg ) { pHandler->HandleResponseWithHosts( pStatus, pResponse, pHostList ); delete this; } private: ResponseHandler *pHandler; XRootDStatus *pStatus; AnyObject *pResponse; HostList *pHostList; }; } #endif // __XRD_CL_RESPONSE_JOB_HH__ xrootd-5.6.9/src/XrdCl/XrdClSIDManager.cc000066400000000000000000000167141457266313600200610ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2012 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . //------------------------------------------------------------------------------ #include "XrdCl/XrdClSIDManager.hh" #include namespace XrdCl { //---------------------------------------------------------------------------- // Allocate a SID //--------------------------------------------------------------------------- Status SIDManager::AllocateSID( uint8_t sid[2] ) { XrdSysMutexHelper scopedLock( pMutex ); uint16_t allocSID = 1; //-------------------------------------------------------------------------- // Get a SID from the list of free SIDs if it's not empty //-------------------------------------------------------------------------- if( !pFreeSIDs.empty() ) { allocSID = pFreeSIDs.front(); pFreeSIDs.pop_front(); } //-------------------------------------------------------------------------- // Allocate a new SID if possible //-------------------------------------------------------------------------- else { if( pSIDCeiling == 0xffff ) return Status( stError, errNoMoreFreeSIDs ); allocSID = pSIDCeiling++; } memcpy( sid, &allocSID, 2 ); pAllocTime[allocSID] = time(0); return Status(); } //---------------------------------------------------------------------------- // Release the SID that is no longer needed //---------------------------------------------------------------------------- void SIDManager::ReleaseSID( uint8_t sid[2] ) { XrdSysMutexHelper scopedLock( pMutex ); uint16_t relSID = 0; memcpy( &relSID, sid, 2 ); pFreeSIDs.push_back( relSID ); pAllocTime.erase( relSID ); } //---------------------------------------------------------------------------- // Register a SID of a request that timed out //---------------------------------------------------------------------------- void SIDManager::TimeOutSID( uint8_t sid[2] ) { XrdSysMutexHelper scopedLock( pMutex ); uint16_t tiSID = 0; memcpy( &tiSID, sid, 2 ); pTimeOutSIDs.insert( tiSID ); pAllocTime.erase( tiSID ); } //---------------------------------------------------------------------------- // Check if any SID was allocated at or before a given time //---------------------------------------------------------------------------- bool SIDManager::IsAnySIDOldAs( const time_t tlim ) const { XrdSysMutexHelper scopedLock( pMutex ); return std::any_of( pAllocTime.begin(), pAllocTime.end(), [tlim](const auto& p) { return p.second <= tlim; } ); } //---------------------------------------------------------------------------- // Check if a SID is timed out //---------------------------------------------------------------------------- bool SIDManager::IsTimedOut( uint8_t sid[2] ) { XrdSysMutexHelper scopedLock( pMutex ); uint16_t tiSID = 0; memcpy( &tiSID, sid, 2 ); std::set::iterator it = pTimeOutSIDs.find( tiSID ); if( it != pTimeOutSIDs.end() ) return true; return false; } //---------------------------------------------------------------------------- // Release a timed out SID //----------------------------------------------------------------------------- void SIDManager::ReleaseTimedOut( uint8_t sid[2] ) { XrdSysMutexHelper scopedLock( pMutex ); uint16_t tiSID = 0; memcpy( &tiSID, sid, 2 ); pTimeOutSIDs.erase( tiSID ); pFreeSIDs.push_back( tiSID ); } //------------------------------------------------------------------------ // Release all timed out SIDs //------------------------------------------------------------------------ void SIDManager::ReleaseAllTimedOut() { XrdSysMutexHelper scopedLock( pMutex ); std::set::iterator it; for( it = pTimeOutSIDs.begin(); it != pTimeOutSIDs.end(); ++it ) pFreeSIDs.push_back( *it ); pTimeOutSIDs.clear(); } //---------------------------------------------------------------------------- // Get number of allocated SIDs //---------------------------------------------------------------------------- uint16_t SIDManager::GetNumberOfAllocatedSIDs() const { XrdSysMutexHelper scopedLock( pMutex ); return pSIDCeiling - pFreeSIDs.size() - pTimeOutSIDs.size() - 1; } //---------------------------------------------------------------------------- // Returns a pointer to the SIDManager object //---------------------------------------------------------------------------- std::shared_ptr SIDMgrPool::GetSIDMgr( const URL &url ) { //-------------------------------------------------------------------------- // Look for an instance of SID manager in the pool //-------------------------------------------------------------------------- XrdSysMutexHelper lck1( mtx ); SIDManager *mgr = 0; auto itr = pool.find( url.GetChannelId() ); if( itr == pool.end() ) { mgr = new SIDManager(); pool[url.GetChannelId()] = mgr; } else mgr = itr->second; //-------------------------------------------------------------------------- // Update the reference counter //-------------------------------------------------------------------------- XrdSysMutexHelper lck2( mgr->pMutex ); ++mgr->pRefCount; //-------------------------------------------------------------------------- // Create a shared pointer that will recycle the SID manager //-------------------------------------------------------------------------- RecycleSidMgr deleter; std::shared_ptr ptr( mgr, deleter ); return ptr; } void SIDMgrPool::Recycle( SIDManager *mgr ) { //-------------------------------------------------------------------------- // Lock the pool, we need to do it in the same order as in 'GetSIDMgr' //-------------------------------------------------------------------------- XrdSysMutexHelper lck1( mtx ); //-------------------------------------------------------------------------- // Lock the SID manager object //-------------------------------------------------------------------------- XrdSysMutexHelper lck2( mgr->pMutex ); --mgr->pRefCount; if( !mgr->pRefCount ) { //------------------------------------------------------------------------ // Remove the SID manager from the pool //------------------------------------------------------------------------ auto itr = pool.begin(); for( ; itr != pool.end() ; ++itr ) if( itr->second == mgr ) { pool.erase( itr ); break; } lck2.UnLock(); delete mgr; } } } xrootd-5.6.9/src/XrdCl/XrdClSIDManager.hh000066400000000000000000000205531457266313600200670ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2012 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . //------------------------------------------------------------------------------ #ifndef __XRD_CL_SID_MANAGER_HH__ #define __XRD_CL_SID_MANAGER_HH__ #include #include #include #include #include #include #include "XrdSys/XrdSysPthread.hh" #include "XrdCl/XrdClStatus.hh" #include "XrdCl/XrdClURL.hh" namespace XrdCl { //---------------------------------------------------------------------------- // We need the forward declaration for the friendship to work properly //---------------------------------------------------------------------------- class SIDMgrPool; //---------------------------------------------------------------------------- //! Handle XRootD stream IDs //---------------------------------------------------------------------------- class SIDManager { friend class SIDMgrPool; private: //------------------------------------------------------------------------ //! Constructor //------------------------------------------------------------------------ SIDManager(): pSIDCeiling(1), pRefCount(0) { } #if __cplusplus < 201103L //------------------------------------------------------------------------ // For older complilers we have to make the destructor public, although // the shared_pointer is using a custom deleter. It will go away once // we drop SLC6 support. //------------------------------------------------------------------------ public: #endif //------------------------------------------------------------------------ //! Destructor //------------------------------------------------------------------------ ~SIDManager() { } public: //------------------------------------------------------------------------ //! Allocate a SID //! //! @param sid a two byte array where the allocated SID should be stored //! @return stOK on success, stError on error //------------------------------------------------------------------------ Status AllocateSID( uint8_t sid[2] ); //------------------------------------------------------------------------ //! Release the SID that is no longer needed //------------------------------------------------------------------------ void ReleaseSID( uint8_t sid[2] ); //------------------------------------------------------------------------ //! Register a SID of a request that timed out //------------------------------------------------------------------------ void TimeOutSID( uint8_t sid[2] ); //---------------------------------------------------------------------------- //! Check if any SID was allocated at or before a given time //---------------------------------------------------------------------------- bool IsAnySIDOldAs( const time_t tlim ) const; //------------------------------------------------------------------------ //! Check if a SID is timed out //------------------------------------------------------------------------ bool IsTimedOut( uint8_t sid[2] ); //------------------------------------------------------------------------ //! Release a timed out SID //------------------------------------------------------------------------ void ReleaseTimedOut( uint8_t sid[2] ); //------------------------------------------------------------------------ //! Release all timed out SIDs //------------------------------------------------------------------------ void ReleaseAllTimedOut(); //------------------------------------------------------------------------ //! Number of timeout sids //------------------------------------------------------------------------ uint32_t NumberOfTimedOutSIDs() const { XrdSysMutexHelper scopedLock( pMutex ); return pTimeOutSIDs.size(); } //------------------------------------------------------------------------ //! Number of allocated streams //------------------------------------------------------------------------ uint16_t GetNumberOfAllocatedSIDs() const; private: std::unordered_map pAllocTime; std::list pFreeSIDs; std::set pTimeOutSIDs; uint16_t pSIDCeiling; mutable XrdSysMutex pMutex; mutable size_t pRefCount; }; //---------------------------------------------------------------------------- //! Pool of SID manager objects //---------------------------------------------------------------------------- class SIDMgrPool { public: //------------------------------------------------------------------------ //! @return : instance of SIDMgrPool //------------------------------------------------------------------------ static SIDMgrPool& Instance() { //---------------------------------------------------------------------- // We could also use a nifty counter but this is simpler and will do! //---------------------------------------------------------------------- static SIDMgrPool *instance = new SIDMgrPool(); return *instance; } //------------------------------------------------------------------------ //! Destructor //------------------------------------------------------------------------ ~SIDMgrPool() { } //------------------------------------------------------------------------ //! @param url : URL for which we need a SIDManager //! @return : a shared pointer to SIDManager object, the pointer has // a custom deleter that will return the object to the pool //------------------------------------------------------------------------ std::shared_ptr GetSIDMgr( const URL &url ); //------------------------------------------------------------------------ //! @param mgr : the SIDManager object to be recycled //------------------------------------------------------------------------ void Recycle( SIDManager *mgr ); private: //------------------------------------------------------------------------ //! A functional object for handling the deletion of SIDManager objects //------------------------------------------------------------------------ struct RecycleSidMgr { inline void operator()( SIDManager *mgr ) { SIDMgrPool &pool = SIDMgrPool::Instance(); pool.Recycle( mgr ); } }; //------------------------------------------------------------------------ //! Constructor //------------------------------------------------------------------------ SIDMgrPool() { } //------------------------------------------------------------------------ //! Deleted constructors //------------------------------------------------------------------------ SIDMgrPool( const SIDMgrPool& ) = delete; SIDMgrPool( SIDMgrPool&& ) = delete; //------------------------------------------------------------------------ //! Deleted assigment operators //------------------------------------------------------------------------ SIDMgrPool& operator=( const SIDMgrPool& ) = delete; SIDMgrPool& operator=( SIDMgrPool&& ) = delete; XrdSysMutex mtx; std::unordered_map pool; }; } #endif // __XRD_CL_SID_MANAGER_HH__ xrootd-5.6.9/src/XrdCl/XrdClSocket.cc000066400000000000000000000740241457266313600173750ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2012 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . //------------------------------------------------------------------------------ #include "XrdCl/XrdClSocket.hh" #include "XrdCl/XrdClUtils.hh" #include "XrdCl/XrdClConstants.hh" #include "XrdCl/XrdClMessage.hh" #include "XrdCl/XrdClDefaultEnv.hh" #include "XrdCl/XrdClTls.hh" #include "XrdNet/XrdNetConnect.hh" #include "XrdSys/XrdSysFD.hh" #include #include #include #include #include #include #include #include #include #include #include #include #include namespace XrdCl { Socket::Socket( int socket, SocketStatus status ): pSocket(socket), pStatus( status ), pProtocolFamily( AF_INET ), pChannelID( 0 ), pCorked( false ) { }; //------------------------------------------------------------------------ // Desctuctor //------------------------------------------------------------------------ Socket::~Socket() { Close(); }; //---------------------------------------------------------------------------- // Initialize //---------------------------------------------------------------------------- XRootDStatus Socket::Initialize( int family ) { if( pSocket != -1 ) return XRootDStatus( stError, errInvalidOp ); pSocket = XrdSysFD_Socket( family, SOCK_STREAM, 0 ); if( pSocket < 0 ) { pSocket = -1; return XRootDStatus( stError, errSocketError ); } pProtocolFamily = family; //-------------------------------------------------------------------------- // Make the socket non blocking and disable the Nagle algorithm since // we will be using this for transmitting messages not handling streams //-------------------------------------------------------------------------- int flags; if( (flags = ::fcntl( pSocket, F_GETFL, 0 )) == -1 ) flags = 0; if( ::fcntl( pSocket, F_SETFL, flags | O_NONBLOCK | O_NDELAY ) == -1 ) { Close(); return XRootDStatus( stError, errFcntl, errno ); } XrdCl::Env *env = XrdCl::DefaultEnv::GetEnv(); flags = DefaultNoDelay; env->GetInt( "NoDelay", flags ); if( setsockopt( pSocket, IPPROTO_TCP, TCP_NODELAY, &flags, sizeof( int ) ) < 0 ) { Close(); return XRootDStatus( stError, errFcntl, errno ); } //-------------------------------------------------------------------------- // We use send with MSG_NOSIGNAL to avoid SIGPIPEs on Linux, on MacOSX // we set SO_NOSIGPIPE option, on Solaris we ignore the SIGPIPE //-------------------------------------------------------------------------- #ifdef __APPLE__ int set = 1; XRootDStatus st = SetSockOpt( SOL_SOCKET, SO_NOSIGPIPE, &set, sizeof(int) ); if( !st.IsOK() ) { Close(); return st; } #elif __solaris__ struct sigaction act; act.sa_handler = SIG_IGN; sigaction( SIGPIPE, &act, NULL ); #endif return XRootDStatus(); } //---------------------------------------------------------------------------- // Set the socket flags //---------------------------------------------------------------------------- XRootDStatus Socket::SetFlags( int flags ) { if( pSocket == -1 ) return XRootDStatus( stError, errInvalidOp ); int st = ::fcntl( pSocket, F_SETFL, flags ); if( st == -1 ) return XRootDStatus( stError, errSocketError, errno ); return XRootDStatus(); } //---------------------------------------------------------------------------- // Get the socket flags //---------------------------------------------------------------------------- XRootDStatus Socket::GetFlags( int &flags ) { if( pSocket == -1 ) return XRootDStatus( stError, errInvalidOp ); int st = ::fcntl( pSocket, F_GETFL, 0 ); if( st == -1 ) return XRootDStatus( stError, errSocketError, errno ); flags = st; return XRootDStatus(); } //---------------------------------------------------------------------------- // Get socket options //---------------------------------------------------------------------------- XRootDStatus Socket::GetSockOpt( int level, int optname, void *optval, socklen_t *optlen ) { if( pSocket == -1 ) return XRootDStatus( stError, errInvalidOp ); if( ::getsockopt( pSocket, level, optname, optval, optlen ) != 0 ) return XRootDStatus( stError, errSocketOptError, errno ); return XRootDStatus(); } //------------------------------------------------------------------------ // Set socket options //------------------------------------------------------------------------ XRootDStatus Socket::SetSockOpt( int level, int optname, const void *optval, socklen_t optlen ) { if( pSocket == -1 ) return XRootDStatus( stError, errInvalidOp ); if( ::setsockopt( pSocket, level, optname, optval, optlen ) != 0 ) return XRootDStatus( stError, errSocketOptError, errno ); return XRootDStatus(); } //---------------------------------------------------------------------------- // Connect to the given host name //---------------------------------------------------------------------------- XRootDStatus Socket::Connect( const std::string &host, uint16_t port, uint16_t timeout ) { if( pSocket == -1 || pStatus == Connected || pStatus == Connecting ) return XRootDStatus( stError, errInvalidOp ); std::vector addrs; std::ostringstream o; o << host << ":" << port; XRootDStatus st; if( pProtocolFamily == AF_INET6 ) st = Utils::GetHostAddresses( addrs, URL( o.str() ), Utils::IPAll ); else st = Utils::GetHostAddresses( addrs, URL( o.str() ), Utils::IPv4 ); if( !st.IsOK() ) return st; Utils::LogHostAddresses( DefaultEnv::GetLog(), PostMasterMsg, o.str(), addrs ); return ConnectToAddress( addrs[0], timeout ); } //---------------------------------------------------------------------------- // Connect to the given host //---------------------------------------------------------------------------- XRootDStatus Socket::ConnectToAddress( const XrdNetAddr &addr, uint16_t timeout ) { if( pSocket == -1 || pStatus == Connected || pStatus == Connecting ) return XRootDStatus( stError, errInvalidOp ); pServerAddr.reset( new XrdNetAddr( addr ) );; //-------------------------------------------------------------------------- // Make sure TLS is off when the physical connection is newly established //-------------------------------------------------------------------------- pTls.reset(); //-------------------------------------------------------------------------- // Connect //-------------------------------------------------------------------------- int status = XrdNetConnect::Connect( pSocket, pServerAddr->SockAddr(), pServerAddr->SockSize(), timeout ); if( status != 0 ) { XRootDStatus st( stError ); //------------------------------------------------------------------------ // If we connect asynchronously this is not really an error //------------------------------------------------------------------------ if( !timeout && status == EINPROGRESS ) { pStatus = Connecting; return XRootDStatus(); } //------------------------------------------------------------------------ // Errors //------------------------------------------------------------------------ else if( status == ETIMEDOUT ) st.code = errSocketTimeout; else st.code = errSocketError; st.errNo = status; Close(); return st; } pStatus = Connected; return XRootDStatus(); } //---------------------------------------------------------------------------- // Disconnect //---------------------------------------------------------------------------- void Socket::Close() { if( pTls ) pTls->Shutdown(); if( pSocket != -1 ) { close( pSocket ); pStatus = Disconnected; pSocket = -1; pSockName = ""; pPeerName = ""; pName = ""; } } //---------------------------------------------------------------------------- //! Read raw bytes from the socket //---------------------------------------------------------------------------- XRootDStatus Socket::ReadRaw( void *buffer, uint32_t size, int32_t timeout, uint32_t &bytesRead ) { //-------------------------------------------------------------------------- // Check if we're connected //-------------------------------------------------------------------------- if( pStatus != Connected ) return XRootDStatus( stError, errInvalidOp ); //-------------------------------------------------------------------------- // Some useful variables //-------------------------------------------------------------------------- bytesRead = 0; char *current = (char *)buffer; bool useTimeout = (timeout!=-1); time_t now = 0; time_t newNow = 0; XRootDStatus sc; if( useTimeout ) now = ::time(0); //-------------------------------------------------------------------------- // Repeat the following until we have read all the requested data //-------------------------------------------------------------------------- while ( bytesRead < size ) { //------------------------------------------------------------------------ // Check if we can read something //------------------------------------------------------------------------ sc = Poll( true, false, useTimeout ? timeout : -1 ); //------------------------------------------------------------------------ // It looks like we've got an event. Let's check if we can read something. //------------------------------------------------------------------------ if( sc.status == stOK ) { ssize_t n = ::read( pSocket, current, (size-bytesRead) ); if( n > 0 ) { bytesRead += n; current += n; } //---------------------------------------------------------------------- // We got a close here - this means that there is no more data in // the buffer so we disconnect //---------------------------------------------------------------------- if( n == 0 ) { Close(); return XRootDStatus( stError, errSocketDisconnected ); } //---------------------------------------------------------------------- // Error //---------------------------------------------------------------------- if( (n < 0) && (errno != EAGAIN) && (errno != EWOULDBLOCK) ) { Close(); return XRootDStatus( stError, errSocketError, errno ); } } else { Close(); return sc; } //------------------------------------------------------------------------ // Do we still have time to wait for data? //------------------------------------------------------------------------ if( useTimeout ) { newNow = ::time(0); timeout -= (newNow-now); now = newNow; if( timeout < 0 ) break; } } //-------------------------------------------------------------------------- // Have we managed to read everything? //-------------------------------------------------------------------------- if( bytesRead < size ) return XRootDStatus( stError, errSocketTimeout ); return XRootDStatus( stOK ); } //---------------------------------------------------------------------------- // Write raw bytes to the socket //---------------------------------------------------------------------------- XRootDStatus Socket::WriteRaw( void *buffer, uint32_t size, int32_t timeout, uint32_t &bytesWritten ) { //-------------------------------------------------------------------------- // Check if we're connected //-------------------------------------------------------------------------- if( pStatus != Connected ) return XRootDStatus( stError, errInvalidOp ); //-------------------------------------------------------------------------- // Some useful variables //-------------------------------------------------------------------------- bytesWritten = 0; char *current = (char *)buffer; bool useTimeout = (timeout!=-1); time_t now = 0; time_t newNow = 0; XRootDStatus sc; if( useTimeout ) now = ::time(0); //-------------------------------------------------------------------------- // Repeat the following until we have written everything //-------------------------------------------------------------------------- while ( bytesWritten < size ) { //------------------------------------------------------------------------ // Check if we can read something //------------------------------------------------------------------------ sc = Poll( false, true, useTimeout ? timeout : -1 ); //------------------------------------------------------------------------ // Let's write //------------------------------------------------------------------------ if( sc.status == stOK ) { ssize_t n = ::write( pSocket, current, (size-bytesWritten) ); if( n > 0 ) { bytesWritten += n; current += n; } //---------------------------------------------------------------------- // Error //---------------------------------------------------------------------- if( (n <= 0) && (errno != EAGAIN) && (errno != EWOULDBLOCK) ) { Close(); return XRootDStatus( stError, errSocketError, errno ); } } else { Close(); return sc; } //------------------------------------------------------------------------ // Do we still have time to wait for data? //------------------------------------------------------------------------ if( useTimeout ) { newNow = ::time(0); timeout -= (newNow-now); now = newNow; if( timeout < 0 ) break; } } //-------------------------------------------------------------------------- // Have we managed to read everything? //-------------------------------------------------------------------------- if( bytesWritten < size ) return XRootDStatus( stError, errSocketTimeout ); return XRootDStatus( stOK ); } //------------------------------------------------------------------------ // Portable wrapper around SIGPIPE free send //---------------------------------------------------------------------------- XRootDStatus Socket::Send( const char *buffer, size_t size, int &bytesWritten ) { if( pTls ) return pTls->Send( buffer, size, bytesWritten ); //-------------------------------------------------------------------------- // We use send with MSG_NOSIGNAL to avoid SIGPIPEs on Linux //-------------------------------------------------------------------------- #if defined(__linux__) || defined(__GNU__) || (defined(__FreeBSD_kernel__) && defined(__GLIBC__)) int status = ::send( pSocket, buffer, size, MSG_NOSIGNAL ); #else int status = ::write( pSocket, buffer, size ); #endif if( status <= 0 ) return ClassifyErrno( errno ); bytesWritten = status; return XRootDStatus(); } //------------------------------------------------------------------------ //! Write data from a kernel buffer to the socket //! //! @param kbuff : data to be written //! @return : the amount of data actually written //------------------------------------------------------------------------ XRootDStatus Socket::Send( XrdSys::KernelBuffer &kbuff, int &bytesWritten ) { if( pTls ) return XRootDStatus( stError, errNotSupported, 0, "Cannot send a kernel-buffer over TLS." ); ssize_t status = XrdSys::Send( pSocket, kbuff ); if( status <= 0 ) return ClassifyErrno( errno ); bytesWritten += status; return XRootDStatus(); } //------------------------------------------------------------------------ // Write message to the socket //------------------------------------------------------------------------ XRootDStatus Socket::Send( Message &msg, const std::string &strmname ) { //---------------------------------------------------------------------- // Try to write down the current message //---------------------------------------------------------------------- size_t btsleft = msg.GetSize() - msg.GetCursor(); if( !btsleft ) return XRootDStatus(); while( btsleft ) { int wrtcnt = 0; XRootDStatus st = Send( msg.GetBufferAtCursor(), btsleft, wrtcnt ); if( !st.IsOK() ) { msg.SetCursor( 0 ); return st; } if( st.code == suRetry ) return st; msg.AdvanceCursor( wrtcnt ); btsleft -= wrtcnt; } //---------------------------------------------------------------------- // We have written the message successfully //---------------------------------------------------------------------- Log *log = DefaultEnv::GetLog(); log->Dump( AsyncSockMsg, "[%s] Wrote a message: %s (0x%x), %d bytes", strmname.c_str(), msg.GetDescription().c_str(), &msg, msg.GetSize() ); return XRootDStatus(); } //---------------------------------------------------------------------------- // Poll the descriptor //---------------------------------------------------------------------------- XRootDStatus Socket::Poll( bool readyForReading, bool readyForWriting, int32_t timeout ) { //-------------------------------------------------------------------------- // Check if we're connected //-------------------------------------------------------------------------- if( pStatus != Connected ) return XRootDStatus( stError, errInvalidOp ); //-------------------------------------------------------------------------- // Prepare the stuff //-------------------------------------------------------------------------- pollfd pollDesc; int pollRet; bool useTimeout = (timeout!=-1); time_t now = 0; time_t newNow = 0; short hupEvents = POLLHUP; #ifdef __linux__ hupEvents |= POLLRDHUP; #endif if( useTimeout ) now = ::time(0); pollDesc.fd = pSocket; pollDesc.events = POLLERR | POLLNVAL | hupEvents; if( readyForReading ) pollDesc.events |= (POLLIN | POLLPRI); if( readyForWriting ) pollDesc.events |= POLLOUT; //-------------------------------------------------------------------------- // We loop on poll because it may return -1 even thought no fatal error // has occurred, these may be: // * a signal interrupting the execution (errno == EINTR) // * a failure to initialize some internal structures (Solaris only) // (errno == EAGAIN) //-------------------------------------------------------------------------- do { pollRet = poll( &pollDesc, 1, (useTimeout ? timeout*1000 : -1) ); if( (pollRet < 0) && (errno != EINTR) && (errno != EAGAIN) ) return XRootDStatus( stError, errPoll, errno ); //------------------------------------------------------------------------ // Check if we did not time out in the case where we are not supposed // to wait indefinitely //------------------------------------------------------------------------ if( useTimeout ) { newNow = time(0); timeout -= (newNow-now); now = newNow; if( timeout < 0 ) return XRootDStatus( stError, errSocketTimeout ); } } while( pollRet == -1 ); //-------------------------------------------------------------------------- // Check if we have timed out //-------------------------------------------------------------------------- if( pollRet == 0 ) return XRootDStatus( stError, errSocketTimeout ); //-------------------------------------------------------------------------- // We have some events //-------------------------------------------------------------------------- if( pollDesc.revents & (POLLIN | POLLPRI | POLLOUT) ) return XRootDStatus( stOK ); //-------------------------------------------------------------------------- // We've been hang up on //-------------------------------------------------------------------------- if( pollDesc.revents & hupEvents ) return XRootDStatus( stError, errSocketDisconnected ); //-------------------------------------------------------------------------- // We're messed up, either because we messed up ourselves (POLLNVAL) or // got messed up by the network (POLLERR) //-------------------------------------------------------------------------- return XRootDStatus( stError, errSocketError ); } //---------------------------------------------------------------------------- // Get the name of the socket //---------------------------------------------------------------------------- std::string Socket::GetSockName() const { if( pStatus != Connected ) return ""; if( pSockName.length() ) return pSockName; char nameBuff[256]; int len = XrdNetUtils::IPFormat( -pSocket, nameBuff, sizeof(nameBuff) ); if( len == 0 ) return ""; pSockName = nameBuff; return pSockName; } //---------------------------------------------------------------------------- // Get the name of the remote peer //---------------------------------------------------------------------------- std::string Socket::GetPeerName() const { if( pStatus != Connected ) return ""; if( pPeerName.length() ) return pPeerName; char nameBuff[256]; int len = XrdNetUtils::IPFormat( pSocket, nameBuff, sizeof(nameBuff) ); if( len == 0 ) return ""; pPeerName = nameBuff; return pPeerName; } //---------------------------------------------------------------------------- // Get the string representation of the socket //---------------------------------------------------------------------------- std::string Socket::GetName() const { if( pStatus != Connected ) return "<-->"; if( pName.length() ) return pName; pName = "<"; pName += GetSockName(); pName += "><--><"; pName += GetPeerName(); pName += ">"; return pName; } //------------------------------------------------------------------------ // Classify errno while reading/writing //------------------------------------------------------------------------ XRootDStatus Socket::ClassifyErrno( int error ) { switch( errno ) { case EAGAIN: #if EAGAIN != EWOULDBLOCK case EWOULDBLOCK: #endif { //------------------------------------------------------------------ // Reading/writing operation would block! So we are done for now, // but we will be back ;-) //------------------------------------------------------------------ return XRootDStatus( stOK, suRetry ); } case ECONNRESET: case EDESTADDRREQ: case EMSGSIZE: case ENOTCONN: case ENOTSOCK: { //------------------------------------------------------------------ // Actual socket error error! //------------------------------------------------------------------ return XRootDStatus( stError, errSocketError, errno ); } case EFAULT: { //------------------------------------------------------------------ // The buffer provided by the user for reading/writing is invalid //------------------------------------------------------------------ return XRootDStatus( stError, errInvalidArgs ); } default: { //------------------------------------------------------------------ // Not a socket error //------------------------------------------------------------------ return XRootDStatus( stError, errInternal, errno ); } } } //---------------------------------------------------------------------------- // Read helper from raw socket helper //---------------------------------------------------------------------------- XRootDStatus Socket::Read( char *buffer, size_t size, int &bytesRead ) { if( pTls ) return pTls->Read( buffer, size, bytesRead ); int status = ::read( pSocket, buffer, size ); // if the server shut down the socket declare a socket error (it // will trigger a re-connect) if( status == 0 ) return XRootDStatus( stError, errSocketError, errno ); if( status < 0 ) return ClassifyErrno( errno ); bytesRead = status; return XRootDStatus(); } //---------------------------------------------------------------------------- // ReadV helper for raw socket //---------------------------------------------------------------------------- XRootDStatus Socket::ReadV( iovec *iov, int iovcnt, int &bytesRead ) { if( pTls ) return pTls->ReadV( iov, iovcnt, bytesRead ); int status = ::readv( pSocket, iov, iovcnt ); // if the server shut down the socket declare a socket error (it // will trigger a re-connect) if( status == 0 ) return XRootDStatus( stError, errSocketError, errno ); if( status < 0 ) return ClassifyErrno( errno ); bytesRead = status; return XRootDStatus(); } //------------------------------------------------------------------------ // Cork the underlying socket //------------------------------------------------------------------------ XRootDStatus Socket::Cork() { #if defined(TCP_CORK) && !defined(__GNU__) // it's not defined on mac, we might want explore the possibility of using TCP_NOPUSH if( pCorked ) return XRootDStatus(); int state = 1; int rc = setsockopt( pSocket, IPPROTO_TCP, TCP_CORK, &state, sizeof( state ) ); if( rc != 0 ) return XRootDStatus( stFatal, errSocketOptError, errno ); #endif pCorked = true; return XRootDStatus(); } //------------------------------------------------------------------------ // Uncork the underlying socket //------------------------------------------------------------------------ XRootDStatus Socket::Uncork() { #if defined(TCP_CORK) && !defined(__GNU__) // it's not defined on mac, we might want explore the possibility of using TCP_NOPUSH if( !pCorked ) return XRootDStatus(); int state = 0; int rc = setsockopt( pSocket, IPPROTO_TCP, TCP_CORK, &state, sizeof( state ) ); if( rc != 0 ) return XRootDStatus( stFatal, errSocketOptError, errno ); #endif pCorked = false; return XRootDStatus(); } //------------------------------------------------------------------------ // Flash the underlying socket //------------------------------------------------------------------------ XRootDStatus Socket::Flash() { //---------------------------------------------------------------------- // Uncork the socket in order to flash the socket //---------------------------------------------------------------------- XRootDStatus st = Uncork(); if( !st.IsOK() ) return st; //---------------------------------------------------------------------- // Once the data has been flashed we can cork the socket back //---------------------------------------------------------------------- return Cork(); } //------------------------------------------------------------------------ // Do special event mapping if applicable //------------------------------------------------------------------------ uint8_t Socket::MapEvent( uint8_t event ) { if( pTls ) return pTls->MapEvent( event ); return event; } //------------------------------------------------------------------------ // Enable encryption //------------------------------------------------------------------------ XRootDStatus Socket::TlsHandShake( AsyncSocketHandler *socketHandler, const std::string &thehost ) { try { if( !pServerAddr ) return XRootDStatus( stError, errInvalidOp ); if( !pTls ) pTls.reset( new Tls( this, socketHandler ) ); return pTls->Connect( thehost, pServerAddr.get() ); } catch( std::exception& ex ) { // the exception has been thrown when we tried to create // the TLS context return XRootDStatus( stFatal, errTlsError, 0, ex.what() ); } return XRootDStatus(); } //------------------------------------------------------------------------ // @return : true if socket is using TLS layer for encryption, // false otherwise //------------------------------------------------------------------------ bool Socket::IsEncrypted() { return bool( pTls.get() ); } } xrootd-5.6.9/src/XrdCl/XrdClSocket.hh000066400000000000000000000364701457266313600174120ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2012 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . //------------------------------------------------------------------------------ #ifndef __XRD_CL_SOCKET_HH__ #define __XRD_CL_SOCKET_HH__ #include #include #include #include #include "XrdCl/XrdClXRootDResponses.hh" #include "XrdNet/XrdNetAddr.hh" #include "XrdSys/XrdSysKernelBuffer.hh" namespace XrdCl { class AnyObject; class Tls; class AsyncSocketHandler; class Message; //---------------------------------------------------------------------------- //! A network socket //---------------------------------------------------------------------------- class Socket { public: //------------------------------------------------------------------------ //! Status of the socket //------------------------------------------------------------------------ enum SocketStatus { Disconnected = 1, //!< The socket is disconnected Connected = 2, //!< The socket is connected Connecting = 3 //!< The connection process is in progress }; //------------------------------------------------------------------------ //! Constructor //! //! @param socket already connected socket if available, -1 otherwise //! @param status status of a socket if available //------------------------------------------------------------------------ Socket( int socket = -1, SocketStatus status = Disconnected ); //------------------------------------------------------------------------ //! Desctuctor //------------------------------------------------------------------------ virtual ~Socket(); //------------------------------------------------------------------------ //! Initialize the socket //------------------------------------------------------------------------ XRootDStatus Initialize( int family = AF_INET ); //------------------------------------------------------------------------ //! Set the socket flags (man fcntl) //------------------------------------------------------------------------ XRootDStatus SetFlags( int flags ); //------------------------------------------------------------------------ //! Get the socket flags (man fcntl) //------------------------------------------------------------------------ XRootDStatus GetFlags( int &flags ); //------------------------------------------------------------------------ //! Get socket options //------------------------------------------------------------------------ XRootDStatus GetSockOpt( int level, int optname, void *optval, socklen_t *optlen ); //------------------------------------------------------------------------ //! Set socket options //------------------------------------------------------------------------ XRootDStatus SetSockOpt( int level, int optname, const void *optval, socklen_t optlen ); //------------------------------------------------------------------------ //! Connect to the given host name //! //! @param host name of the host to connect to //! @param port port to connect to //! @param timout timeout in seconds, 0 for no timeout handling (may be //! used for non blocking IO) //------------------------------------------------------------------------ XRootDStatus Connect( const std::string &host, uint16_t port, uint16_t timout = 10 ); //------------------------------------------------------------------------ //! Connect to the given host address //! //! @param addr address of the host to connect to //! @param timout timeout in seconds, 0 for no timeout handling (may be //! used for non blocking IO) //------------------------------------------------------------------------ XRootDStatus ConnectToAddress( const XrdNetAddr &addr, uint16_t timout = 10 ); //------------------------------------------------------------------------ //! Disconnect //------------------------------------------------------------------------ void Close(); //------------------------------------------------------------------------ //! Get the socket status //------------------------------------------------------------------------ SocketStatus GetStatus() const { return pStatus; } //------------------------------------------------------------------------ //! Set socket status - do not use unless you know what you're doing //------------------------------------------------------------------------ void SetStatus( SocketStatus status ) { pStatus = status; } //------------------------------------------------------------------------ //! Read raw bytes from the socket //! //! @param buffer data to be sent //! @param size size of the data buffer //! @param timeout timout value in seconds, -1 to wait indefinitely //! @param bytesRead the amount of data actually read //------------------------------------------------------------------------ XRootDStatus ReadRaw( void *buffer, uint32_t size, int32_t timeout, uint32_t &bytesRead ); //------------------------------------------------------------------------ //! Write raw bytes to the socket //! //! @param buffer data to be written //! @param size size of the data buffer //! @param timeout timeout value in seconds, -1 to wait indefinitely //! @param bytesWritten the amount of data actually written //------------------------------------------------------------------------ XRootDStatus WriteRaw( void *buffer, uint32_t size, int32_t timeout, uint32_t &bytesWritten ); //------------------------------------------------------------------------ //! Portable wrapper around SIGPIPE free send //! //! @param buffer : data to be written //! @param size : size of the data buffer //! @param bytesWritten : the amount of data actually written //------------------------------------------------------------------------ virtual XRootDStatus Send( const char *buffer, size_t size, int &bytesWritten ); //------------------------------------------------------------------------ //! Write data from a kernel buffer to the socket //! //! @param kbuff : data to be written //! @param bytesWritten : the amount of data actually written //------------------------------------------------------------------------ XRootDStatus Send( XrdSys::KernelBuffer &kbuff, int &bytesWritten ); //------------------------------------------------------------------------ //! Write message to the socket //! //! @param msg : message (request) to be sent //! @param strmname : stream name (for logging purposes) //------------------------------------------------------------------------ XRootDStatus Send( Message &msg, const std::string &strmname ); //---------------------------------------------------------------------------- //! Read helper for raw socket //! //! @param buffer : the sink for the data //! @param size : size of the sink //! @param bytesRead : number of bytes actually written into the sink //! @return : success : ( stOK ) //! EAGAIN : ( stOK, suRetry ) //! EWOULDBLOCK : ( stOK, suRetry ) //! other error : ( stError, errSocketError ) //---------------------------------------------------------------------------- virtual XRootDStatus Read( char *buffer, size_t size, int &bytesRead ); //---------------------------------------------------------------------------- //! ReadV helper for raw socket //! //! @param iov : the buffers for the data //! @param iocnt : number of buffers //! @param bytesRead : number of bytes actually written into the sink //! @return : success : ( stOK ) //! EAGAIN : ( stOK, suRetry ) //! EWOULDBLOCK : ( stOK, suRetry ) //! other error : ( stError, errSocketError ) //---------------------------------------------------------------------------- XRootDStatus ReadV( iovec *iov, int iocnt, int &bytesRead ); //------------------------------------------------------------------------ //! Get the file descriptor //------------------------------------------------------------------------ int GetFD() { return pSocket; } //------------------------------------------------------------------------ //! Get the name of the socket //------------------------------------------------------------------------ std::string GetSockName() const; //------------------------------------------------------------------------ //! Get the name of the remote peer //------------------------------------------------------------------------ std::string GetPeerName() const; //------------------------------------------------------------------------ //! Get the string representation of the socket //------------------------------------------------------------------------ std::string GetName() const; //------------------------------------------------------------------------ //! Get the server address //------------------------------------------------------------------------ const XrdNetAddr* GetServerAddress() const { return pServerAddr.get(); } //------------------------------------------------------------------------ //! Set Channel ID //! (an object that allows to identify all sockets corresponding to the same channel) //------------------------------------------------------------------------ void SetChannelID( AnyObject *channelID ) { pChannelID = channelID; } //------------------------------------------------------------------------ //! Get Channel ID //! (an object that allows to identify all sockets corresponding to the same channel) //------------------------------------------------------------------------ const AnyObject* GetChannelID() const { return pChannelID; } //------------------------------------------------------------------------ // Classify errno while reading/writing //------------------------------------------------------------------------ static XRootDStatus ClassifyErrno( int error ); //------------------------------------------------------------------------ // Cork the underlying socket // // As there is no way to do vector writes with SSL/TLS we need to cork // the socket and then flash it when appropriate //------------------------------------------------------------------------ XRootDStatus Cork(); //------------------------------------------------------------------------ // Uncork the underlying socket //------------------------------------------------------------------------ XRootDStatus Uncork(); //------------------------------------------------------------------------ // Flash the underlying socket //------------------------------------------------------------------------ XRootDStatus Flash(); //------------------------------------------------------------------------ // Check if the socket is corked //------------------------------------------------------------------------ inline bool IsCorked() const { return pCorked; } //------------------------------------------------------------------------ // Do special event mapping if applicable //------------------------------------------------------------------------ uint8_t MapEvent( uint8_t event ); //------------------------------------------------------------------------ // Enable encryption // // @param socketHandler : the socket handler that is handling the socket // @param the host : host name for verification //------------------------------------------------------------------------ XRootDStatus TlsHandShake( AsyncSocketHandler *socketHandler, const std::string &thehost = std::string() ); //------------------------------------------------------------------------ // @return : true if socket is using TLS layer for encryption, // false otherwise //------------------------------------------------------------------------ bool IsEncrypted(); protected: //------------------------------------------------------------------------ //! Poll the socket to see whether it is ready for IO //! //! @param readyForReading poll for readiness to read //! @param readyForWriting poll for readiness to write //! @param timeout timeout in seconds, -1 to wait indefinitely //! @return stOK - ready for IO //! errSocketDisconnected - on disconnection //! errSocketError - on socket error //! errSocketTimeout - on socket timeout //! errInvalidOp - when called on a non connected socket //------------------------------------------------------------------------ XRootDStatus Poll( bool readyForReading, bool readyForWriting, int32_t timeout ); int pSocket; SocketStatus pStatus; std::unique_ptr pServerAddr; mutable std::string pSockName; // mutable because it's for caching mutable std::string pPeerName; mutable std::string pName; int pProtocolFamily; AnyObject *pChannelID; bool pCorked; std::unique_ptr pTls; }; } #endif // __XRD_CL_SOCKET_HH__ xrootd-5.6.9/src/XrdCl/XrdClStatus.cc000066400000000000000000000130761457266313600174300ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2012 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . //------------------------------------------------------------------------------ #include "XrdCl/XrdClStatus.hh" #include "XrdSys/XrdSysE2T.hh" #include "XProtocol/XProtocol.hh" #include namespace { using namespace XrdCl; struct ErrorMap { uint16_t code; const char *msg; }; ErrorMap errors[] = { { errUnknown, "Unknown error" }, { errInvalidOp, "Invalid operation" }, { errFcntl, "Fcntl error" }, { errPoll, "Poll error" }, { errConfig, "Configuration error" }, { errInternal, "Internal error" }, { errUnknownCommand, "Command not found" }, { errInvalidArgs, "Invalid arguments" }, { errInProgress, "Operation in progress" }, { errUninitialized, "Initialization error" }, { errOSError, "OS Error" }, { errNotSupported, "Operation not supported" }, { errDataError, "Received corrupted data" }, { errNotImplemented, "Operation is not implemented" }, { errNoMoreReplicas, "No more replicas to try" }, { errPipelineFailed, "Pipeline failed" }, { errInvalidAddr, "Invalid address" }, { errSocketError, "Socket error" }, { errSocketTimeout, "Socket timeout" }, { errSocketDisconnected, "Socket disconnected" }, { errPollerError, "Poller error" }, { errSocketOptError, "Socket opt error" }, { errStreamDisconnect, "Stream disconnect" }, { errConnectionError, "Connection error" }, { errInvalidSession, "Invalid session" }, { errTlsError, "TLS error" }, { errInvalidMessage, "Invalid message" }, { errNotFound, "Resource not found" }, { errCheckSumError, "CheckSum error" }, { errRedirectLimit, "Redirect limit has been reached" }, { errHandShakeFailed, "Hand shake failed" }, { errLoginFailed, "Login failed" }, { errAuthFailed, "Auth failed" }, { errQueryNotSupported, "Query not supported" }, { errOperationExpired, "Operation expired" }, { errOperationInterrupted, "Operation interrupted" }, { errThresholdExceeded, "Threshold exceeded" }, { errNoMoreFreeSIDs, "No more free SIDs" }, { errInvalidRedirectURL, "Invalid redirect URL" }, { errInvalidResponse, "Invalid response" }, { errRedirect, "Unhandled redirect" }, { errErrorResponse, "Error response" }, { errLocalError, "Local error" }, { errResponseNegative, "Query response negative" }, { 0, 0 } }; //---------------------------------------------------------------------------- // Get error code //---------------------------------------------------------------------------- std::string GetErrorMessage( uint16_t code ) { for( int i = 0; errors[i].msg != 0; ++i ) if( errors[i].code == code ) return errors[i].msg; return "Unknown error code"; } } namespace XrdCl { //---------------------------------------------------------------------------- // Create a string representation //---------------------------------------------------------------------------- std::string Status::ToString() const { std::ostringstream o; //-------------------------------------------------------------------------- // The status is OK //-------------------------------------------------------------------------- if( IsOK() ) { o << "[SUCCESS] "; if( code == suContinue ) o << "Continue"; else if( code == suRetry ) o << "Retry"; return o.str(); } //-------------------------------------------------------------------------- // We have an error //-------------------------------------------------------------------------- if( IsFatal() ) o << "[FATAL] "; else o << "[ERROR] "; o << GetErrorMessage( code ); //-------------------------------------------------------------------------- // Add errno //-------------------------------------------------------------------------- if( errNo >= kXR_ArgInvalid ) // kXR_ArgInvalid is the first (lowest) xrootd error code // it is used in an inconsistent way sometimes it is // xrootd error code and sometimes it is a plain errno o << ": " << XrdSysE2T( XProtocol::toErrno( errNo ) ); else if ( errNo ) o << ": " << XrdSysE2T( errNo ); return o.str(); } } xrootd-5.6.9/src/XrdCl/XrdClStatus.hh000066400000000000000000000157441457266313600174460ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2012 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . //------------------------------------------------------------------------------ #ifndef __XRD_CL_STATUS_HH__ #define __XRD_CL_STATUS_HH__ #include #include #include namespace XrdCl { //---------------------------------------------------------------------------- // Constants //---------------------------------------------------------------------------- const uint16_t stOK = 0x0000; //!< Everything went OK const uint16_t stError = 0x0001; //!< An error occurred that could potentially be retried const uint16_t stFatal = 0x0003; //!< Fatal error, it's still an error //---------------------------------------------------------------------------- // Additional info for the stOK status //---------------------------------------------------------------------------- const uint16_t suDone = 0; const uint16_t suContinue = 1; const uint16_t suRetry = 2; const uint16_t suPartial = 3; const uint16_t suAlreadyDone = 4; const uint16_t suNotStarted = 5; //---------------------------------------------------------------------------- // Generic errors //---------------------------------------------------------------------------- const uint16_t errNone = 0; //!< No error const uint16_t errRetry = 1; //!< Try again for whatever reason const uint16_t errUnknown = 2; //!< Unknown error const uint16_t errInvalidOp = 3; //!< The operation cannot be performed in the //!< given circumstances const uint16_t errFcntl = 4; //!< failed manipulate file descriptor const uint16_t errPoll = 5; //!< error while polling descriptors const uint16_t errConfig = 6; //!< System misconfigured const uint16_t errInternal = 7; //!< Internal error const uint16_t errUnknownCommand = 8; const uint16_t errInvalidArgs = 9; const uint16_t errInProgress = 10; const uint16_t errUninitialized = 11; const uint16_t errOSError = 12; const uint16_t errNotSupported = 13; const uint16_t errDataError = 14; //!< data is corrupted const uint16_t errNotImplemented = 15; //!< Operation is not implemented const uint16_t errNoMoreReplicas = 16; //!< No more replicas to try const uint16_t errPipelineFailed = 17; //!< Pipeline failed and operation couldn't be executed //---------------------------------------------------------------------------- // Socket related errors //---------------------------------------------------------------------------- const uint16_t errInvalidAddr = 101; const uint16_t errSocketError = 102; const uint16_t errSocketTimeout = 103; const uint16_t errSocketDisconnected = 104; const uint16_t errPollerError = 105; const uint16_t errSocketOptError = 106; const uint16_t errStreamDisconnect = 107; const uint16_t errConnectionError = 108; const uint16_t errInvalidSession = 109; const uint16_t errTlsError = 110; //---------------------------------------------------------------------------- // Post Master related errors //---------------------------------------------------------------------------- const uint16_t errInvalidMessage = 201; const uint16_t errHandShakeFailed = 202; const uint16_t errLoginFailed = 203; const uint16_t errAuthFailed = 204; const uint16_t errQueryNotSupported = 205; const uint16_t errOperationExpired = 206; const uint16_t errOperationInterrupted = 207; const uint16_t errThresholdExceeded = 208; //---------------------------------------------------------------------------- // XRootD related errors //---------------------------------------------------------------------------- const uint16_t errNoMoreFreeSIDs = 301; const uint16_t errInvalidRedirectURL = 302; const uint16_t errInvalidResponse = 303; const uint16_t errNotFound = 304; const uint16_t errCheckSumError = 305; const uint16_t errRedirectLimit = 306; const uint16_t errCorruptedHeader = 307; const uint16_t errErrorResponse = 400; const uint16_t errRedirect = 401; const uint16_t errLocalError = 402; const uint16_t errResponseNegative = 500; //!< Query response was negative //---------------------------------------------------------------------------- //! Procedure execution status //---------------------------------------------------------------------------- struct Status { //-------------------------------------------------------------------------- //! Constructor //-------------------------------------------------------------------------- Status( uint16_t st = stOK, uint16_t cod = errNone, uint32_t errN = 0 ): status(st), code(cod), errNo( errN ) {} bool IsError() const { return status & stError; } //!< Error bool IsFatal() const { return (status&0x0002) & stFatal; } //!< Fatal error bool IsOK() const { return status == stOK; } //!< We're fine //-------------------------------------------------------------------------- //! Get the status code that may be returned to the shell //-------------------------------------------------------------------------- int GetShellCode() const { if( IsOK() ) return 0; return (code/100)+50; } inline static bool IsSocketError( uint16_t code ) { return int( code / 100 ) == 1; } //-------------------------------------------------------------------------- //! Create a string representation //-------------------------------------------------------------------------- std::string ToString() const; uint16_t status; //!< Status of the execution uint16_t code; //!< Error type, or additional hints on what to do uint32_t errNo; //!< Errno, if any }; } #endif // __XRD_CL_STATUS_HH__ xrootd-5.6.9/src/XrdCl/XrdClStream.cc000066400000000000000000001357361457266313600174100ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2014 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #include "XrdCl/XrdClStream.hh" #include "XrdCl/XrdClSocket.hh" #include "XrdCl/XrdClChannel.hh" #include "XrdCl/XrdClConstants.hh" #include "XrdCl/XrdClLog.hh" #include "XrdCl/XrdClMessage.hh" #include "XrdCl/XrdClDefaultEnv.hh" #include "XrdCl/XrdClUtils.hh" #include "XrdCl/XrdClOutQueue.hh" #include "XrdCl/XrdClMonitor.hh" #include "XrdCl/XrdClMessageUtils.hh" #include "XrdCl/XrdClXRootDTransport.hh" #include "XrdCl/XrdClXRootDMsgHandler.hh" #include "XrdClAsyncSocketHandler.hh" #include #include #include #include namespace XrdCl { //---------------------------------------------------------------------------- // Statics //---------------------------------------------------------------------------- RAtomic_uint64_t Stream::sSessCntGen{0}; //---------------------------------------------------------------------------- // Incoming message helper //---------------------------------------------------------------------------- struct InMessageHelper { InMessageHelper( Message *message = 0, MsgHandler *hndlr = 0, time_t expir = 0, uint16_t actio = 0 ): msg( message ), handler( hndlr ), expires( expir ), action( actio ) {} void Reset() { msg = 0; handler = 0; expires = 0; action = 0; } Message *msg; MsgHandler *handler; time_t expires; uint16_t action; }; //---------------------------------------------------------------------------- // Sub stream helper //---------------------------------------------------------------------------- struct SubStreamData { SubStreamData(): socket( 0 ), status( Socket::Disconnected ) { outQueue = new OutQueue(); } ~SubStreamData() { delete socket; delete outQueue; } AsyncSocketHandler *socket; OutQueue *outQueue; OutQueue::MsgHelper outMsgHelper; InMessageHelper inMsgHelper; Socket::SocketStatus status; }; //---------------------------------------------------------------------------- // Constructor //---------------------------------------------------------------------------- Stream::Stream( const URL *url, const URL &prefer ): pUrl( url ), pPrefer( prefer ), pTransport( 0 ), pPoller( 0 ), pTaskManager( 0 ), pJobManager( 0 ), pIncomingQueue( 0 ), pChannelData( 0 ), pLastStreamError( 0 ), pConnectionCount( 0 ), pConnectionInitTime( 0 ), pAddressType( Utils::IPAll ), pSessionId( 0 ), pBytesSent( 0 ), pBytesReceived( 0 ) { pConnectionStarted.tv_sec = 0; pConnectionStarted.tv_usec = 0; pConnectionDone.tv_sec = 0; pConnectionDone.tv_usec = 0; std::ostringstream o; o << pUrl->GetHostId(); pStreamName = o.str(); pConnectionWindow = Utils::GetIntParameter( *url, "ConnectionWindow", DefaultConnectionWindow ); pConnectionRetry = Utils::GetIntParameter( *url, "ConnectionRetry", DefaultConnectionRetry ); pStreamErrorWindow = Utils::GetIntParameter( *url, "StreamErrorWindow", DefaultStreamErrorWindow ); std::string netStack = Utils::GetStringParameter( *url, "NetworkStack", DefaultNetworkStack ); pAddressType = Utils::String2AddressType( netStack ); if( pAddressType == Utils::AddressType::IPAuto ) { XrdNetUtils::NetProt stacks = XrdNetUtils::NetConfig( XrdNetUtils::NetType::qryINIF ); if( !( stacks & XrdNetUtils::hasIP64 ) ) { if( stacks & XrdNetUtils::hasIPv4 ) pAddressType = Utils::AddressType::IPv4; else if( stacks & XrdNetUtils::hasIPv6 ) pAddressType = Utils::AddressType::IPv6; } } Log *log = DefaultEnv::GetLog(); log->Debug( PostMasterMsg, "[%s] Stream parameters: Network Stack: %s, " "Connection Window: %d, ConnectionRetry: %d, Stream Error " "Window: %d", pStreamName.c_str(), netStack.c_str(), pConnectionWindow, pConnectionRetry, pStreamErrorWindow ); } //---------------------------------------------------------------------------- // Destructor //---------------------------------------------------------------------------- Stream::~Stream() { Disconnect( true ); Log *log = DefaultEnv::GetLog(); log->Debug( PostMasterMsg, "[%s] Destroying stream", pStreamName.c_str() ); MonitorDisconnection( XRootDStatus() ); SubStreamList::iterator it; for( it = pSubStreams.begin(); it != pSubStreams.end(); ++it ) delete *it; } //---------------------------------------------------------------------------- // Initializer //---------------------------------------------------------------------------- XRootDStatus Stream::Initialize() { if( !pTransport || !pPoller || !pChannelData ) return XRootDStatus( stError, errUninitialized ); AsyncSocketHandler *s = new AsyncSocketHandler( *pUrl, pPoller, pTransport, pChannelData, 0, this ); pSubStreams.push_back( new SubStreamData() ); pSubStreams[0]->socket = s; return XRootDStatus(); } //------------------------------------------------------------------------ // Make sure that the underlying socket handler gets write readiness // events //------------------------------------------------------------------------ XRootDStatus Stream::EnableLink( PathID &path ) { XrdSysMutexHelper scopedLock( pMutex ); //-------------------------------------------------------------------------- // We are in the process of connecting the main stream, so we do nothing // because when the main stream connection is established it will connect // all the other streams //-------------------------------------------------------------------------- if( pSubStreams[0]->status == Socket::Connecting ) return XRootDStatus(); //-------------------------------------------------------------------------- // The main stream is connected, so we can verify whether we have // the up and the down stream connected and ready to handle data. // If anything is not right we fall back to stream 0. //-------------------------------------------------------------------------- if( pSubStreams[0]->status == Socket::Connected ) { if( pSubStreams[path.down]->status != Socket::Connected ) path.down = 0; if( pSubStreams[path.up]->status == Socket::Disconnected ) { path.up = 0; return pSubStreams[0]->socket->EnableUplink(); } if( pSubStreams[path.up]->status == Socket::Connected ) return pSubStreams[path.up]->socket->EnableUplink(); return XRootDStatus(); } //-------------------------------------------------------------------------- // The main stream is not connected, we need to check whether enough time // has passed since we last encountered an error (if any) so that we could // re-attempt the connection //-------------------------------------------------------------------------- Log *log = DefaultEnv::GetLog(); time_t now = ::time(0); if( now-pLastStreamError < pStreamErrorWindow ) return pLastFatalError; gettimeofday( &pConnectionStarted, 0 ); ++pConnectionCount; //-------------------------------------------------------------------------- // Resolve all the addresses of the host we're supposed to connect to //-------------------------------------------------------------------------- XRootDStatus st = Utils::GetHostAddresses( pAddresses, *pUrl, pAddressType ); if( !st.IsOK() ) { log->Error( PostMasterMsg, "[%s] Unable to resolve IP address for " "the host", pStreamName.c_str() ); pLastStreamError = now; st.status = stFatal; pLastFatalError = st; return st; } if( pPrefer.IsValid() ) { std::vector addrresses; XRootDStatus st = Utils::GetHostAddresses( addrresses, pPrefer, pAddressType ); if( !st.IsOK() ) { log->Error( PostMasterMsg, "[%s] Unable to resolve IP address for %s", pStreamName.c_str(), pPrefer.GetHostName().c_str() ); } else { std::vector tmp; tmp.reserve( pAddresses.size() ); // first add all remaining addresses auto itr = pAddresses.begin(); for( ; itr != pAddresses.end() ; ++itr ) { if( !HasNetAddr( *itr, addrresses ) ) tmp.push_back( *itr ); } // then copy all 'preferred' addresses std::copy( addrresses.begin(), addrresses.end(), std::back_inserter( tmp ) ); // and keep the result pAddresses.swap( tmp ); } } Utils::LogHostAddresses( log, PostMasterMsg, pUrl->GetHostId(), pAddresses ); while( !pAddresses.empty() ) { pSubStreams[0]->socket->SetAddress( pAddresses.back() ); pAddresses.pop_back(); pConnectionInitTime = ::time( 0 ); st = pSubStreams[0]->socket->Connect( pConnectionWindow ); if( st.IsOK() ) { pSubStreams[0]->status = Socket::Connecting; break; } } return st; } //---------------------------------------------------------------------------- // Queue the message for sending //---------------------------------------------------------------------------- XRootDStatus Stream::Send( Message *msg, MsgHandler *handler, bool stateful, time_t expires ) { XrdSysMutexHelper scopedLock( pMutex ); Log *log = DefaultEnv::GetLog(); //-------------------------------------------------------------------------- // Check the session ID and bounce if needed //-------------------------------------------------------------------------- if( msg->GetSessionId() && (pSubStreams[0]->status != Socket::Connected || pSessionId != msg->GetSessionId()) ) return XRootDStatus( stError, errInvalidSession ); //-------------------------------------------------------------------------- // Decide on the path to send the message //-------------------------------------------------------------------------- PathID path = pTransport->MultiplexSubStream( msg, *pChannelData ); if( pSubStreams.size() <= path.up ) { log->Warning( PostMasterMsg, "[%s] Unable to send message %s through " "substream %d, using 0 instead", pStreamName.c_str(), msg->GetDescription().c_str(), path.up ); path.up = 0; } log->Dump( PostMasterMsg, "[%s] Sending message %s (0x%x) through " "substream %d expecting answer at %d", pStreamName.c_str(), msg->GetDescription().c_str(), msg, path.up, path.down ); //-------------------------------------------------------------------------- // Enable *a* path and insert the message to the right queue //-------------------------------------------------------------------------- XRootDStatus st = EnableLink( path ); if( st.IsOK() ) { pTransport->MultiplexSubStream( msg, *pChannelData, &path ); pSubStreams[path.up]->outQueue->PushBack( msg, handler, expires, stateful ); } else st.status = stFatal; return st; } //---------------------------------------------------------------------------- // Force connection //---------------------------------------------------------------------------- void Stream::ForceConnect() { XrdSysMutexHelper scopedLock( pMutex ); if( pSubStreams[0]->status == Socket::Connecting ) { pSubStreams[0]->status = Socket::Disconnected; XrdCl::PathID path( 0, 0 ); XrdCl::XRootDStatus st = EnableLink( path ); if( !st.IsOK() ) OnConnectError( 0, st ); } } //---------------------------------------------------------------------------- // Disconnect the stream //---------------------------------------------------------------------------- void Stream::Disconnect( bool /*force*/ ) { XrdSysMutexHelper scopedLock( pMutex ); SubStreamList::iterator it; for( it = pSubStreams.begin(); it != pSubStreams.end(); ++it ) { (*it)->socket->Close(); (*it)->status = Socket::Disconnected; } } //---------------------------------------------------------------------------- // Handle a clock event //---------------------------------------------------------------------------- void Stream::Tick( time_t now ) { //-------------------------------------------------------------------------- // Check for timed-out requests and incoming handlers //-------------------------------------------------------------------------- pMutex.Lock(); OutQueue q; SubStreamList::iterator it; for( it = pSubStreams.begin(); it != pSubStreams.end(); ++it ) q.GrabExpired( *(*it)->outQueue, now ); pMutex.UnLock(); q.Report( XRootDStatus( stError, errOperationExpired ) ); pIncomingQueue->ReportTimeout( now ); } } //------------------------------------------------------------------------------ // Handle message timeouts and reconnection in the future //------------------------------------------------------------------------------ namespace { class StreamConnectorTask: public XrdCl::Task { public: //------------------------------------------------------------------------ // Constructor //------------------------------------------------------------------------ StreamConnectorTask( const XrdCl::URL &url, const std::string &n ): url( url ) { std::string name = "StreamConnectorTask for "; name += n; SetName( name ); } //------------------------------------------------------------------------ // Run the task //------------------------------------------------------------------------ time_t Run( time_t ) { XrdCl::DefaultEnv::GetPostMaster()->ForceReconnect( url ); return 0; } private: XrdCl::URL url; }; } namespace XrdCl { XRootDStatus Stream::RequestClose( Message &response ) { ServerResponse *rsp = reinterpret_cast( response.GetBuffer() ); if( rsp->hdr.dlen < 4 ) return XRootDStatus( stError ); Message *msg; ClientCloseRequest *req; MessageUtils::CreateRequest( msg, req ); req->requestid = kXR_close; memcpy( req->fhandle, reinterpret_cast( rsp->body.buffer.data ), 4 ); XRootDTransport::SetDescription( msg ); msg->SetSessionId( pSessionId ); NullResponseHandler *handler = new NullResponseHandler(); MessageSendParams params; params.timeout = 0; params.followRedirects = false; params.stateful = true; MessageUtils::ProcessSendParams( params ); return MessageUtils::SendMessage( *pUrl, msg, handler, params, 0 ); } //------------------------------------------------------------------------ // Check if message is a partial response //------------------------------------------------------------------------ bool Stream::IsPartial( Message &msg ) { ServerResponseHeader *rsphdr = (ServerResponseHeader*)msg.GetBuffer(); if( rsphdr->status == kXR_oksofar ) return true; if( rsphdr->status == kXR_status ) { ServerResponseStatus *rspst = (ServerResponseStatus*)msg.GetBuffer(); if( rspst->bdy.resptype == XrdProto::kXR_PartialResult ) return true; } return false; } //---------------------------------------------------------------------------- // Call back when a message has been reconstructed //---------------------------------------------------------------------------- void Stream::OnIncoming( uint16_t subStream, std::shared_ptr msg, uint32_t bytesReceived ) { msg->SetSessionId( pSessionId ); pBytesReceived += bytesReceived; MsgHandler *handler = nullptr; uint16_t action = 0; { InMessageHelper &mh = pSubStreams[subStream]->inMsgHelper; handler = mh.handler; action = mh.action; mh.Reset(); } if( !IsPartial( *msg ) ) { uint32_t streamAction = pTransport->MessageReceived( *msg, subStream, *pChannelData ); if( streamAction & TransportHandler::DigestMsg ) return; if( streamAction & TransportHandler::RequestClose ) { RequestClose( *msg ); return; } } Log *log = DefaultEnv::GetLog(); //-------------------------------------------------------------------------- // No handler, we discard the message ... //-------------------------------------------------------------------------- if( !handler ) { ServerResponse *rsp = (ServerResponse*)msg->GetBuffer(); log->Warning( PostMasterMsg, "[%s] Discarding received message: 0x%x " "(status=%d, SID=[%d,%d]), no MsgHandler found.", pStreamName.c_str(), msg.get(), rsp->hdr.status, rsp->hdr.streamid[0], rsp->hdr.streamid[1] ); return; } //-------------------------------------------------------------------------- // We have a handler, so we call the callback //-------------------------------------------------------------------------- log->Dump( PostMasterMsg, "[%s] Handling received message: 0x%x.", pStreamName.c_str(), msg.get() ); if( action & (MsgHandler::NoProcess|MsgHandler::Ignore) ) { log->Dump( PostMasterMsg, "[%s] Ignoring the processing handler for: 0x%x.", pStreamName.c_str(), msg->GetDescription().c_str() ); // if we are handling partial response we have to take down the timeout fence if( IsPartial( *msg ) ) { XRootDMsgHandler *xrdHandler = dynamic_cast( handler ); if( xrdHandler ) xrdHandler->PartialReceived(); } return; } Job *job = new HandleIncMsgJob( handler ); pJobManager->QueueJob( job ); } //---------------------------------------------------------------------------- // Call when one of the sockets is ready to accept a new message //---------------------------------------------------------------------------- std::pair Stream::OnReadyToWrite( uint16_t subStream ) { XrdSysMutexHelper scopedLock( pMutex ); Log *log = DefaultEnv::GetLog(); if( pSubStreams[subStream]->outQueue->IsEmpty() ) { log->Dump( PostMasterMsg, "[%s] Nothing to write, disable uplink", pSubStreams[subStream]->socket->GetStreamName().c_str() ); pSubStreams[subStream]->socket->DisableUplink(); return std::make_pair( (Message *)0, (MsgHandler *)0 ); } OutQueue::MsgHelper &h = pSubStreams[subStream]->outMsgHelper; h.msg = pSubStreams[subStream]->outQueue->PopMessage( h.handler, h.expires, h.stateful ); scopedLock.UnLock(); if( h.handler ) h.handler->OnReadyToSend( h.msg ); return std::make_pair( h.msg, h.handler ); } void Stream::DisableIfEmpty( uint16_t subStream ) { XrdSysMutexHelper scopedLock( pMutex ); Log *log = DefaultEnv::GetLog(); if( pSubStreams[subStream]->outQueue->IsEmpty() ) { log->Dump( PostMasterMsg, "[%s] All messages consumed, disable uplink", pSubStreams[subStream]->socket->GetStreamName().c_str() ); pSubStreams[subStream]->socket->DisableUplink(); } } //---------------------------------------------------------------------------- // Call when a message is written to the socket //---------------------------------------------------------------------------- void Stream::OnMessageSent( uint16_t subStream, Message *msg, uint32_t bytesSent ) { pTransport->MessageSent( msg, subStream, bytesSent, *pChannelData ); OutQueue::MsgHelper &h = pSubStreams[subStream]->outMsgHelper; pBytesSent += bytesSent; if( h.handler ) { h.handler->OnStatusReady( msg, XRootDStatus() ); bool rmMsg = false; pIncomingQueue->AddMessageHandler( h.handler, h.handler->GetExpiration(), rmMsg ); if( rmMsg ) { Log *log = DefaultEnv::GetLog(); log->Warning( PostMasterMsg, "[%s] Removed a leftover msg from the in-queue.", pStreamName.c_str(), subStream ); } } pSubStreams[subStream]->outMsgHelper.Reset(); } //---------------------------------------------------------------------------- // Call back when a message has been reconstructed //---------------------------------------------------------------------------- void Stream::OnConnect( uint16_t subStream ) { XrdSysMutexHelper scopedLock( pMutex ); pSubStreams[subStream]->status = Socket::Connected; std::string ipstack( pSubStreams[0]->socket->GetIpStack() ); Log *log = DefaultEnv::GetLog(); log->Debug( PostMasterMsg, "[%s] Stream %d connected (%s).", pStreamName.c_str(), subStream, ipstack.c_str() ); if( subStream == 0 ) { pLastStreamError = 0; pLastFatalError = XRootDStatus(); pConnectionCount = 0; uint16_t numSub = pTransport->SubStreamNumber( *pChannelData ); pSessionId = ++sSessCntGen; //------------------------------------------------------------------------ // Create the streams if they don't exist yet //------------------------------------------------------------------------ if( pSubStreams.size() == 1 && numSub > 1 ) { for( uint16_t i = 1; i < numSub; ++i ) { URL url = pTransport->GetBindPreference( *pUrl, *pChannelData ); AsyncSocketHandler *s = new AsyncSocketHandler( url, pPoller, pTransport, pChannelData, i, this ); pSubStreams.push_back( new SubStreamData() ); pSubStreams[i]->socket = s; } } //------------------------------------------------------------------------ // Connect the extra streams, if we fail we move all the outgoing items // to stream 0, we don't need to enable the uplink here, because it // should be already enabled after the handshaking process is completed. //------------------------------------------------------------------------ if( pSubStreams.size() > 1 ) { log->Debug( PostMasterMsg, "[%s] Attempting to connect %d additional " "streams.", pStreamName.c_str(), pSubStreams.size()-1 ); for( size_t i = 1; i < pSubStreams.size(); ++i ) { pSubStreams[i]->socket->SetAddress( pSubStreams[0]->socket->GetAddress() ); XRootDStatus st = pSubStreams[i]->socket->Connect( pConnectionWindow ); if( !st.IsOK() ) { pSubStreams[0]->outQueue->GrabItems( *pSubStreams[i]->outQueue ); pSubStreams[i]->socket->Close(); } else { pSubStreams[i]->status = Socket::Connecting; } } } //------------------------------------------------------------------------ // Inform monitoring //------------------------------------------------------------------------ pBytesSent = 0; pBytesReceived = 0; gettimeofday( &pConnectionDone, 0 ); Monitor *mon = DefaultEnv::GetMonitor(); if( mon ) { Monitor::ConnectInfo i; i.server = pUrl->GetHostId(); i.sTOD = pConnectionStarted; i.eTOD = pConnectionDone; i.streams = pSubStreams.size(); AnyObject qryResult; std::string *qryResponse = 0; pTransport->Query( TransportQuery::Auth, qryResult, *pChannelData ); qryResult.Get( qryResponse ); i.auth = *qryResponse; delete qryResponse; mon->Event( Monitor::EvConnect, &i ); } //------------------------------------------------------------------------ // For every connected control-stream call the global on-connect handler //------------------------------------------------------------------------ XrdCl::DefaultEnv::GetPostMaster()->NotifyConnectHandler( *pUrl ); } else if( pOnDataConnJob ) { //------------------------------------------------------------------------ // For every connected data-stream call the on-connect handler //------------------------------------------------------------------------ pJobManager->QueueJob( pOnDataConnJob.get(), 0 ); } } //---------------------------------------------------------------------------- // On connect error //---------------------------------------------------------------------------- void Stream::OnConnectError( uint16_t subStream, XRootDStatus status ) { XrdSysMutexHelper scopedLock( pMutex ); Log *log = DefaultEnv::GetLog(); pSubStreams[subStream]->socket->Close(); time_t now = ::time(0); //-------------------------------------------------------------------------- // For every connection error call the global connection error handler //-------------------------------------------------------------------------- XrdCl::DefaultEnv::GetPostMaster()->NotifyConnErrHandler( *pUrl, status ); //-------------------------------------------------------------------------- // If we connected subStream == 0 and cannot connect >0 then we just give // up and move the outgoing messages to another queue //-------------------------------------------------------------------------- if( subStream > 0 ) { pSubStreams[subStream]->status = Socket::Disconnected; pSubStreams[0]->outQueue->GrabItems( *pSubStreams[subStream]->outQueue ); if( pSubStreams[0]->status == Socket::Connected ) { XRootDStatus st = pSubStreams[0]->socket->EnableUplink(); if( !st.IsOK() ) OnFatalError( 0, st, scopedLock ); return; } if( pSubStreams[0]->status == Socket::Connecting ) return; OnFatalError( subStream, status, scopedLock ); return; } //-------------------------------------------------------------------------- // Check if we still have time to try and do something in the current window //-------------------------------------------------------------------------- time_t elapsed = now-pConnectionInitTime; log->Error( PostMasterMsg, "[%s] elapsed = %d, pConnectionWindow = %d " "seconds.", pStreamName.c_str(), elapsed, pConnectionWindow ); //------------------------------------------------------------------------ // If we have some IP addresses left we try them //------------------------------------------------------------------------ if( !pAddresses.empty() ) { XRootDStatus st; do { pSubStreams[0]->socket->SetAddress( pAddresses.back() ); pAddresses.pop_back(); pConnectionInitTime = ::time( 0 ); st = pSubStreams[0]->socket->Connect( pConnectionWindow ); } while( !pAddresses.empty() && !st.IsOK() ); if( !st.IsOK() ) OnFatalError( subStream, st, scopedLock ); return; } //------------------------------------------------------------------------ // If we still can retry with the same host name, we sleep until the end // of the connection window and try //------------------------------------------------------------------------ else if( elapsed < pConnectionWindow && pConnectionCount < pConnectionRetry && !status.IsFatal() ) { log->Info( PostMasterMsg, "[%s] Attempting reconnection in %d " "seconds.", pStreamName.c_str(), pConnectionWindow-elapsed ); Task *task = new ::StreamConnectorTask( *pUrl, pStreamName ); pTaskManager->RegisterTask( task, pConnectionInitTime+pConnectionWindow ); return; } //-------------------------------------------------------------------------- // We are out of the connection window, the only thing we can do here // is re-resolving the host name and retrying if we still can //-------------------------------------------------------------------------- else if( pConnectionCount < pConnectionRetry && !status.IsFatal() ) { pAddresses.clear(); pSubStreams[0]->status = Socket::Disconnected; PathID path( 0, 0 ); XRootDStatus st = EnableLink( path ); if( !st.IsOK() ) OnFatalError( subStream, st, scopedLock ); return; } //-------------------------------------------------------------------------- // Else, we fail //-------------------------------------------------------------------------- OnFatalError( subStream, status, scopedLock ); } //---------------------------------------------------------------------------- // Call back when an error has occurred //---------------------------------------------------------------------------- void Stream::OnError( uint16_t subStream, XRootDStatus status ) { XrdSysMutexHelper scopedLock( pMutex ); Log *log = DefaultEnv::GetLog(); pSubStreams[subStream]->socket->Close(); pSubStreams[subStream]->status = Socket::Disconnected; log->Debug( PostMasterMsg, "[%s] Recovering error for stream #%d: %s.", pStreamName.c_str(), subStream, status.ToString().c_str() ); //-------------------------------------------------------------------------- // Reinsert the stuff that we have failed to sent //-------------------------------------------------------------------------- if( pSubStreams[subStream]->outMsgHelper.msg ) { OutQueue::MsgHelper &h = pSubStreams[subStream]->outMsgHelper; pSubStreams[subStream]->outQueue->PushFront( h.msg, h.handler, h.expires, h.stateful ); pSubStreams[subStream]->outMsgHelper.Reset(); } //-------------------------------------------------------------------------- // Reinsert the receiving handler and reset any partially read partial //-------------------------------------------------------------------------- if( pSubStreams[subStream]->inMsgHelper.handler ) { InMessageHelper &h = pSubStreams[subStream]->inMsgHelper; pIncomingQueue->ReAddMessageHandler( h.handler, h.expires ); XRootDMsgHandler *xrdHandler = dynamic_cast( h.handler ); if( xrdHandler ) xrdHandler->PartialReceived(); h.Reset(); } //-------------------------------------------------------------------------- // We are dealing with an error of a peripheral stream. If we don't have // anything to send don't bother recovering. Otherwise move the requests // to stream 0 if possible. //-------------------------------------------------------------------------- if( subStream > 0 ) { if( pSubStreams[subStream]->outQueue->IsEmpty() ) return; if( pSubStreams[0]->status != Socket::Disconnected ) { pSubStreams[0]->outQueue->GrabItems( *pSubStreams[subStream]->outQueue ); if( pSubStreams[0]->status == Socket::Connected ) { XRootDStatus st = pSubStreams[0]->socket->EnableUplink(); if( !st.IsOK() ) OnFatalError( 0, st, scopedLock ); return; } } OnFatalError( subStream, status, scopedLock ); return; } //-------------------------------------------------------------------------- // If we lost the stream 0 we have lost the session, we re-enable the // stream if we still have things in one of the outgoing queues, otherwise // there is not point to recover at this point. //-------------------------------------------------------------------------- if( subStream == 0 ) { MonitorDisconnection( status ); SubStreamList::iterator it; size_t outstanding = 0; for( it = pSubStreams.begin(); it != pSubStreams.end(); ++it ) outstanding += (*it)->outQueue->GetSizeStateless(); if( outstanding ) { PathID path( 0, 0 ); XRootDStatus st = EnableLink( path ); if( !st.IsOK() ) { OnFatalError( 0, st, scopedLock ); return; } } //------------------------------------------------------------------------ // We're done here, unlock the stream mutex to avoid deadlocks and // report the disconnection event to the handlers //------------------------------------------------------------------------ log->Debug( PostMasterMsg, "[%s] Reporting disconnection to queued " "message handlers.", pStreamName.c_str() ); OutQueue q; for( it = pSubStreams.begin(); it != pSubStreams.end(); ++it ) q.GrabStateful( *(*it)->outQueue ); scopedLock.UnLock(); q.Report( status ); pIncomingQueue->ReportStreamEvent( MsgHandler::Broken, status ); pChannelEvHandlers.ReportEvent( ChannelEventHandler::StreamBroken, status ); return; } } //------------------------------------------------------------------------ // Force error //------------------------------------------------------------------------ void Stream::ForceError( XRootDStatus status ) { XrdSysMutexHelper scopedLock( pMutex ); Log *log = DefaultEnv::GetLog(); for( size_t substream = 0; substream < pSubStreams.size(); ++substream ) { if( pSubStreams[substream]->status != Socket::Connected ) continue; pSubStreams[substream]->socket->Close(); pSubStreams[substream]->status = Socket::Disconnected; log->Error( PostMasterMsg, "[%s] Forcing error on disconnect: %s.", pStreamName.c_str(), status.ToString().c_str() ); //-------------------------------------------------------------------- // Reinsert the stuff that we have failed to sent //-------------------------------------------------------------------- if( pSubStreams[substream]->outMsgHelper.msg ) { OutQueue::MsgHelper &h = pSubStreams[substream]->outMsgHelper; pSubStreams[substream]->outQueue->PushFront( h.msg, h.handler, h.expires, h.stateful ); pSubStreams[substream]->outMsgHelper.Reset(); } //-------------------------------------------------------------------- // Reinsert the receiving handler and reset any partially read partial //-------------------------------------------------------------------- if( pSubStreams[substream]->inMsgHelper.handler ) { InMessageHelper &h = pSubStreams[substream]->inMsgHelper; pIncomingQueue->ReAddMessageHandler( h.handler, h.expires ); XRootDMsgHandler *xrdHandler = dynamic_cast( h.handler ); if( xrdHandler ) xrdHandler->PartialReceived(); h.Reset(); } } pConnectionCount = 0; //------------------------------------------------------------------------ // We're done here, unlock the stream mutex to avoid deadlocks and // report the disconnection event to the handlers //------------------------------------------------------------------------ log->Debug( PostMasterMsg, "[%s] Reporting disconnection to queued " "message handlers.", pStreamName.c_str() ); SubStreamList::iterator it; OutQueue q; for( it = pSubStreams.begin(); it != pSubStreams.end(); ++it ) q.GrabItems( *(*it)->outQueue ); scopedLock.UnLock(); q.Report( status ); pIncomingQueue->ReportStreamEvent( MsgHandler::Broken, status ); pChannelEvHandlers.ReportEvent( ChannelEventHandler::StreamBroken, status ); } //---------------------------------------------------------------------------- // On fatal error //---------------------------------------------------------------------------- void Stream::OnFatalError( uint16_t subStream, XRootDStatus status, XrdSysMutexHelper &lock ) { Log *log = DefaultEnv::GetLog(); pSubStreams[subStream]->status = Socket::Disconnected; log->Error( PostMasterMsg, "[%s] Unable to recover: %s.", pStreamName.c_str(), status.ToString().c_str() ); //-------------------------------------------------------------------------- // Don't set the stream error windows for authentication errors as the user // may refresh his credential at any time //-------------------------------------------------------------------------- if( status.code != errAuthFailed ) { pConnectionCount = 0; pLastStreamError = ::time(0); pLastFatalError = status; } SubStreamList::iterator it; OutQueue q; for( it = pSubStreams.begin(); it != pSubStreams.end(); ++it ) q.GrabItems( *(*it)->outQueue ); lock.UnLock(); status.status = stFatal; q.Report( status ); pIncomingQueue->ReportStreamEvent( MsgHandler::FatalError, status ); pChannelEvHandlers.ReportEvent( ChannelEventHandler::FatalError, status ); } //---------------------------------------------------------------------------- // Inform monitoring about disconnection //---------------------------------------------------------------------------- void Stream::MonitorDisconnection( XRootDStatus status ) { Monitor *mon = DefaultEnv::GetMonitor(); if( mon ) { Monitor::DisconnectInfo i; i.server = pUrl->GetHostId(); i.rBytes = pBytesReceived; i.sBytes = pBytesSent; i.cTime = ::time(0) - pConnectionDone.tv_sec; i.status = status; mon->Event( Monitor::EvDisconnect, &i ); } } //---------------------------------------------------------------------------- // Call back when a message has been reconstructed //---------------------------------------------------------------------------- bool Stream::OnReadTimeout( uint16_t substream ) { //-------------------------------------------------------------------------- // We only take the main stream into account //-------------------------------------------------------------------------- if( substream != 0 ) return true; //-------------------------------------------------------------------------- // Check if there is no outgoing messages and if the stream TTL is elapesed. // It is assumed that the underlying transport makes sure that there is no // pending requests that are not answered, ie. all possible virtual streams // are de-allocated //-------------------------------------------------------------------------- Log *log = DefaultEnv::GetLog(); SubStreamList::iterator it; time_t now = time(0); XrdSysMutexHelper scopedLock( pMutex ); uint32_t outgoingMessages = 0; time_t lastActivity = 0; for( it = pSubStreams.begin(); it != pSubStreams.end(); ++it ) { outgoingMessages += (*it)->outQueue->GetSize(); time_t sockLastActivity = (*it)->socket->GetLastActivity(); if( lastActivity < sockLastActivity ) lastActivity = sockLastActivity; } if( !outgoingMessages ) { bool disconnect = pTransport->IsStreamTTLElapsed( now-lastActivity, *pChannelData ); if( disconnect ) { log->Debug( PostMasterMsg, "[%s] Stream TTL elapsed, disconnecting...", pStreamName.c_str() ); scopedLock.UnLock(); //---------------------------------------------------------------------- // Important note! // // This destroys the Stream object itself, the underlined // AsyncSocketHandler object (that called this method) and the Channel // object that aggregates this Stream. //---------------------------------------------------------------------- DefaultEnv::GetPostMaster()->ForceDisconnect( *pUrl ); return false; } } //-------------------------------------------------------------------------- // Check if the stream is broken //-------------------------------------------------------------------------- XRootDStatus st = pTransport->IsStreamBroken( now-lastActivity, *pChannelData ); if( !st.IsOK() ) { scopedLock.UnLock(); OnError( substream, st ); return false; } return true; } //---------------------------------------------------------------------------- // Call back when a message has been reconstru //---------------------------------------------------------------------------- bool Stream::OnWriteTimeout( uint16_t /*substream*/ ) { return true; } //---------------------------------------------------------------------------- // Register channel event handler //---------------------------------------------------------------------------- void Stream::RegisterEventHandler( ChannelEventHandler *handler ) { pChannelEvHandlers.AddHandler( handler ); } //---------------------------------------------------------------------------- // Remove a channel event handler //---------------------------------------------------------------------------- void Stream::RemoveEventHandler( ChannelEventHandler *handler ) { pChannelEvHandlers.RemoveHandler( handler ); } //---------------------------------------------------------------------------- // Install a incoming message handler //---------------------------------------------------------------------------- MsgHandler* Stream::InstallIncHandler( std::shared_ptr &msg, uint16_t stream ) { InMessageHelper &mh = pSubStreams[stream]->inMsgHelper; if( !mh.handler ) mh.handler = pIncomingQueue->GetHandlerForMessage( msg, mh.expires, mh.action ); if( !mh.handler ) return nullptr; if( mh.action & MsgHandler::Raw ) return mh.handler; return nullptr; } //---------------------------------------------------------------------------- //! In case the message is a kXR_status response it needs further attention //! //! @return : a MsgHandler in case we need to read out raw data //---------------------------------------------------------------------------- uint16_t Stream::InspectStatusRsp( uint16_t stream, MsgHandler *&incHandler ) { InMessageHelper &mh = pSubStreams[stream]->inMsgHelper; if( !mh.handler ) return MsgHandler::RemoveHandler; uint16_t action = mh.handler->InspectStatusRsp(); mh.action |= action; if( action & MsgHandler::RemoveHandler ) pIncomingQueue->RemoveMessageHandler( mh.handler ); if( action & MsgHandler::Raw ) { incHandler = mh.handler; return MsgHandler::Raw; } if( action & MsgHandler::Corrupted ) return MsgHandler::Corrupted; if( action & MsgHandler::More ) return MsgHandler::More; return MsgHandler::None; } //---------------------------------------------------------------------------- // Check if channel can be collapsed using given URL //---------------------------------------------------------------------------- bool Stream::CanCollapse( const URL &url ) { Log *log = DefaultEnv::GetLog(); //-------------------------------------------------------------------------- // Resolve all the addresses of the host we're supposed to connect to //-------------------------------------------------------------------------- std::vector prefaddrs; XRootDStatus st = Utils::GetHostAddresses( prefaddrs, url, pAddressType ); if( !st.IsOK() ) { log->Error( PostMasterMsg, "[%s] Unable to resolve IP address for %s." , pStreamName.c_str(), url.GetHostName().c_str() ); return false; } //-------------------------------------------------------------------------- // Resolve all the addresses of the alias //-------------------------------------------------------------------------- std::vector aliasaddrs; st = Utils::GetHostAddresses( aliasaddrs, *pUrl, pAddressType ); if( !st.IsOK() ) { log->Error( PostMasterMsg, "[%s] Unable to resolve IP address for %s." , pStreamName.c_str(), pUrl->GetHostName().c_str() ); return false; } //-------------------------------------------------------------------------- // Now check if the preferred host is part of the alias //-------------------------------------------------------------------------- auto itr = prefaddrs.begin(); for( ; itr != prefaddrs.end() ; ++itr ) { auto itr2 = aliasaddrs.begin(); for( ; itr2 != aliasaddrs.end() ; ++itr2 ) if( itr->Same( &*itr2 ) ) return true; } return false; } //------------------------------------------------------------------------ // Query the stream //------------------------------------------------------------------------ Status Stream::Query( uint16_t query, AnyObject &result ) { switch( query ) { case StreamQuery::IpAddr: { result.Set( new std::string( pSubStreams[0]->socket->GetIpAddr() ), false ); return Status(); } case StreamQuery::IpStack: { result.Set( new std::string( pSubStreams[0]->socket->GetIpStack() ), false ); return Status(); } case StreamQuery::HostName: { result.Set( new std::string( pSubStreams[0]->socket->GetHostName() ), false ); return Status(); } default: return Status( stError, errQueryNotSupported ); } } } xrootd-5.6.9/src/XrdCl/XrdClStream.hh000066400000000000000000000407531457266313600174140ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2012 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . //------------------------------------------------------------------------------ #ifndef __XRD_CL_STREAM_HH__ #define __XRD_CL_STREAM_HH__ #include "XrdCl/XrdClPoller.hh" #include "XrdCl/XrdClStatus.hh" #include "XrdCl/XrdClURL.hh" #include "XrdCl/XrdClPostMasterInterfaces.hh" #include "XrdCl/XrdClChannelHandlerList.hh" #include "XrdCl/XrdClJobManager.hh" #include "XrdCl/XrdClInQueue.hh" #include "XrdCl/XrdClUtils.hh" #include "XrdSys/XrdSysPthread.hh" #include "XrdSys/XrdSysRAtomic.hh" #include "XrdNet/XrdNetAddr.hh" #include "XrdOuc/XrdOucCompiler.hh" #include #include #include #include namespace XrdCl { class Message; class Channel; class TransportHandler; class TaskManager; struct SubStreamData; //---------------------------------------------------------------------------- //! Stream //---------------------------------------------------------------------------- class Stream { public: //------------------------------------------------------------------------ //! Status of the stream //------------------------------------------------------------------------ enum StreamStatus { Disconnected = 0, //!< Not connected Connected = 1, //!< Connected Connecting = 2, //!< In the process of being connected Error = 3 //!< Broken }; //------------------------------------------------------------------------ //! Constructor //------------------------------------------------------------------------ Stream( const URL *url, const URL &prefer = URL() ); //------------------------------------------------------------------------ //! Destructor //------------------------------------------------------------------------ ~Stream(); //------------------------------------------------------------------------ //! Initializer //------------------------------------------------------------------------ XRootDStatus Initialize(); //------------------------------------------------------------------------ //! Queue the message for sending //------------------------------------------------------------------------ XRootDStatus Send( Message *msg, MsgHandler *handler, bool stateful, time_t expires ); //------------------------------------------------------------------------ //! Set the transport //------------------------------------------------------------------------ void SetTransport( TransportHandler *transport ) { pTransport = transport; } //------------------------------------------------------------------------ //! Set the poller //------------------------------------------------------------------------ void SetPoller( Poller *poller ) { pPoller = poller; } //------------------------------------------------------------------------ //! Set the incoming queue //------------------------------------------------------------------------ void SetIncomingQueue( InQueue *incomingQueue ) { pIncomingQueue = incomingQueue; } //------------------------------------------------------------------------ //! Set the channel data //------------------------------------------------------------------------ void SetChannelData( AnyObject *channelData ) { pChannelData = channelData; } //------------------------------------------------------------------------ //! Set task manager //------------------------------------------------------------------------ void SetTaskManager( TaskManager *taskManager ) { pTaskManager = taskManager; } //------------------------------------------------------------------------ //! Set job manager //------------------------------------------------------------------------ void SetJobManager( JobManager *jobManager ) { pJobManager = jobManager; } //------------------------------------------------------------------------ //! Connect if needed, otherwise make sure that the underlying socket //! handler gets write readiness events, it will update the path with //! what it has actually enabled //------------------------------------------------------------------------ XRootDStatus EnableLink( PathID &path ); //------------------------------------------------------------------------ //! Disconnect the stream //------------------------------------------------------------------------ void Disconnect( bool force = false ); //------------------------------------------------------------------------ //! Handle a clock event generated either by socket timeout, or by //! the task manager event //------------------------------------------------------------------------ void Tick( time_t now ); //------------------------------------------------------------------------ //! Get the URL //------------------------------------------------------------------------ const URL *GetURL() const { return pUrl; } //------------------------------------------------------------------------ //! Force connection //------------------------------------------------------------------------ void ForceConnect(); //------------------------------------------------------------------------ //! Return stream name //------------------------------------------------------------------------ const std::string &GetName() const { return pStreamName; } //------------------------------------------------------------------------ //! Disables respective uplink if empty //------------------------------------------------------------------------ void DisableIfEmpty( uint16_t subStream ); //------------------------------------------------------------------------ //! Call back when a message has been reconstructed //------------------------------------------------------------------------ void OnIncoming( uint16_t subStream, std::shared_ptr msg, uint32_t bytesReceived ); //------------------------------------------------------------------------ // Call when one of the sockets is ready to accept a new message //------------------------------------------------------------------------ std::pair OnReadyToWrite( uint16_t subStream ); //------------------------------------------------------------------------ // Call when a message is written to the socket //------------------------------------------------------------------------ void OnMessageSent( uint16_t subStream, Message *msg, uint32_t bytesSent ); //------------------------------------------------------------------------ //! Call back when a message has been reconstructed //------------------------------------------------------------------------ void OnConnect( uint16_t subStream ); //------------------------------------------------------------------------ //! On connect error //------------------------------------------------------------------------ void OnConnectError( uint16_t subStream, XRootDStatus status ); //------------------------------------------------------------------------ //! On error //------------------------------------------------------------------------ void OnError( uint16_t subStream, XRootDStatus status ); //------------------------------------------------------------------------ //! Force error //------------------------------------------------------------------------ void ForceError( XRootDStatus status ); //------------------------------------------------------------------------ //! On read timeout //------------------------------------------------------------------------ bool OnReadTimeout( uint16_t subStream ) XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ //! On write timeout //------------------------------------------------------------------------ bool OnWriteTimeout( uint16_t subStream ) XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ //! Register channel event handler //------------------------------------------------------------------------ void RegisterEventHandler( ChannelEventHandler *handler ); //------------------------------------------------------------------------ //! Remove a channel event handler //------------------------------------------------------------------------ void RemoveEventHandler( ChannelEventHandler *handler ); //------------------------------------------------------------------------ //! Install a message handler for the given message if there is one //! available, if the handler want's to be called in the raw mode //! it will be returned, the message ownership flag is returned //! in any case //! //! @param msg message header //! @param stream stream concerned //! @return a pair containing the handler and ownership flag //------------------------------------------------------------------------ MsgHandler* InstallIncHandler( std::shared_ptr &msg, uint16_t stream ); //------------------------------------------------------------------------ //! In case the message is a kXR_status response it needs further attention //! //! @return : a MsgHandler in case we need to read out raw data //------------------------------------------------------------------------ uint16_t InspectStatusRsp( uint16_t stream, MsgHandler *&incHandler ); //------------------------------------------------------------------------ //! Set the on-connect handler for data streams //------------------------------------------------------------------------ void SetOnDataConnectHandler( std::shared_ptr &onConnJob ) { XrdSysMutexHelper scopedLock( pMutex ); pOnDataConnJob = onConnJob; } //------------------------------------------------------------------------ //! @return : true is this channel can be collapsed using this URL, false //! otherwise //------------------------------------------------------------------------ bool CanCollapse( const URL &url ); //------------------------------------------------------------------------ //! Query the stream //------------------------------------------------------------------------ Status Query( uint16_t query, AnyObject &result ); private: //------------------------------------------------------------------------ //! Check if message is a partial response //------------------------------------------------------------------------ static bool IsPartial( Message &msg ); //------------------------------------------------------------------------ //! Check if addresses contains given address //------------------------------------------------------------------------ inline static bool HasNetAddr( const XrdNetAddr &addr, std::vector &addresses ) { auto itr = addresses.begin(); for( ; itr != addresses.end() ; ++itr ) { if( itr->Same( &addr ) ) return true; } return false; } //------------------------------------------------------------------------ // Job handling the incoming messages //------------------------------------------------------------------------ class HandleIncMsgJob: public Job { public: HandleIncMsgJob( MsgHandler *handler ): pHandler( handler ) {}; virtual ~HandleIncMsgJob() {}; virtual void Run( void* ) { pHandler->Process(); delete this; } private: MsgHandler *pHandler; }; //------------------------------------------------------------------------ //! On fatal error - unlocks the stream //------------------------------------------------------------------------ void OnFatalError( uint16_t subStream, XRootDStatus status, XrdSysMutexHelper &lock ); //------------------------------------------------------------------------ //! Inform the monitoring about disconnection //------------------------------------------------------------------------ void MonitorDisconnection( XRootDStatus status ); //------------------------------------------------------------------------ //! Send close after an open request timed out //------------------------------------------------------------------------ XRootDStatus RequestClose( Message &resp ); typedef std::vector SubStreamList; //------------------------------------------------------------------------ // Data members //------------------------------------------------------------------------ const URL *pUrl; const URL pPrefer; std::string pStreamName; TransportHandler *pTransport; Poller *pPoller; TaskManager *pTaskManager; JobManager *pJobManager; XrdSysRecMutex pMutex; InQueue *pIncomingQueue; AnyObject *pChannelData; uint32_t pLastStreamError; XRootDStatus pLastFatalError; uint16_t pStreamErrorWindow; uint16_t pConnectionCount; uint16_t pConnectionRetry; time_t pConnectionInitTime; uint16_t pConnectionWindow; SubStreamList pSubStreams; std::vector pAddresses; Utils::AddressType pAddressType; ChannelHandlerList pChannelEvHandlers; uint64_t pSessionId; //------------------------------------------------------------------------ // Monitoring info //------------------------------------------------------------------------ timeval pConnectionStarted; timeval pConnectionDone; uint64_t pBytesSent; uint64_t pBytesReceived; //------------------------------------------------------------------------ // Data stream on-connect handler //------------------------------------------------------------------------ std::shared_ptr pOnDataConnJob; //------------------------------------------------------------------------ // Track last assigned Id across all Streams, to ensure unique sessionId //------------------------------------------------------------------------ static RAtomic_uint64_t sSessCntGen; }; } #endif // __XRD_CL_STREAM_HH__ xrootd-5.6.9/src/XrdCl/XrdClSyncQueue.hh000066400000000000000000000070331457266313600200740ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2013 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . //------------------------------------------------------------------------------ #ifndef __XRD_CL_SYNC_QUEUE_HH__ #define __XRD_CL_SYNC_QUEUE_HH__ #include #include "XrdSys/XrdSysPthread.hh" namespace XrdCl { //---------------------------------------------------------------------------- //! A synchronized queue //---------------------------------------------------------------------------- template class SyncQueue { public: //------------------------------------------------------------------------ //! Constructor //------------------------------------------------------------------------ SyncQueue() { pSem = new XrdSysSemaphore(0); }; //------------------------------------------------------------------------ //! Destructor //------------------------------------------------------------------------ ~SyncQueue() { delete pSem; } //------------------------------------------------------------------------ //! Put the item in the queue //------------------------------------------------------------------------ void Put( const Item &item ) { XrdSysMutexHelper scopedLock( pMutex ); pQueue.push( item ); pSem->Post(); } //------------------------------------------------------------------------ //! Get the item from the front of the queue //------------------------------------------------------------------------ Item Get() { pSem->Wait(); XrdSysMutexHelper scopedLock( pMutex ); // this is not possible, so when it happens we commit a suicide if( pQueue.empty() ) abort(); Item i = pQueue.front(); pQueue.pop(); return i; } //------------------------------------------------------------------------ //! Clear the queue //------------------------------------------------------------------------ void Clear() { XrdSysMutexHelper scopedLock( pMutex ); while( !pQueue.empty() ) pQueue.pop(); delete pSem; pSem = new XrdSysSemaphore(0); } //------------------------------------------------------------------------ //! Check if the queue is empty //------------------------------------------------------------------------ bool IsEmpty() { XrdSysMutexHelper scopedLock( pMutex ); return pQueue.empty(); } protected: std::queue pQueue; XrdSysMutex pMutex; XrdSysSemaphore *pSem; }; } #endif // __XRD_CL_ANY_OBJECT_HH__ xrootd-5.6.9/src/XrdCl/XrdClTPFallBackCopyJob.cc000066400000000000000000000070311457266313600213300ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2014 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #include "XrdCl/XrdClTPFallBackCopyJob.hh" #include "XrdCl/XrdClThirdPartyCopyJob.hh" #include "XrdCl/XrdClClassicCopyJob.hh" #include "XrdCl/XrdClConstants.hh" #include "XrdCl/XrdClLog.hh" #include "XrdCl/XrdClDefaultEnv.hh" #include namespace XrdCl { //---------------------------------------------------------------------------- // Constructor //---------------------------------------------------------------------------- TPFallBackCopyJob::TPFallBackCopyJob( uint16_t jobId, PropertyList *jobProperties, PropertyList *jobResults ): CopyJob( jobId, jobProperties, jobResults ), pJob( 0 ) { Log *log = DefaultEnv::GetLog(); log->Debug( UtilityMsg, "Creating a third party fall back copy job, " "from %s to %s", GetSource().GetURL().c_str(), GetTarget().GetURL().c_str() ); } //------------------------------------------------------------------------ // Destructor //------------------------------------------------------------------------ TPFallBackCopyJob::~TPFallBackCopyJob() { delete pJob; } //---------------------------------------------------------------------------- // Run the copy job //---------------------------------------------------------------------------- XRootDStatus TPFallBackCopyJob::Run( CopyProgressHandler *progress ) { //-------------------------------------------------------------------------- // Set up the job //-------------------------------------------------------------------------- std::string tmp; bool tpcFallBack = false; pProperties->Get( "thirdParty", tmp ); if( tmp == "first" ) tpcFallBack = true; pJob = new ThirdPartyCopyJob( pJobId, pProperties, pResults ); XRootDStatus st = pJob->Run( progress ); if( st.IsOK() ) return st; // we are done // check if we can fall back to streaming if( tpcFallBack && ( st.code == errNotSupported || st.code == errOperationExpired ) ) { Log *log = DefaultEnv::GetLog(); log->Debug( UtilityMsg, "TPC is not supported, falling back to streaming mode." ); delete pJob; pJob = new ClassicCopyJob( pJobId, pProperties, pResults ); return pJob->Run( progress ); } return st; } } xrootd-5.6.9/src/XrdCl/XrdClTPFallBackCopyJob.hh000066400000000000000000000047551457266313600213540ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2014 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #ifndef __XRD_CL_TP_FALLBACK_COPY_JOB_HH__ #define __XRD_CL_TP_FALLBACK_COPY_JOB_HH__ #include "XrdCl/XrdClCopyProcess.hh" #include "XrdCl/XrdClCopyJob.hh" namespace XrdCl { class TPFallBackCopyJob: public CopyJob { public: //------------------------------------------------------------------------ //! Constructor //------------------------------------------------------------------------ TPFallBackCopyJob( uint16_t jobId, PropertyList *jobProperties, PropertyList *jobResults ); //------------------------------------------------------------------------ //! Destructor //------------------------------------------------------------------------ virtual ~TPFallBackCopyJob(); //------------------------------------------------------------------------ //! Run the copy job //! //! @param progress the handler to be notified about the copy progress //! @return status of the copy operation //------------------------------------------------------------------------ virtual XRootDStatus Run( CopyProgressHandler *progress = 0 ); private: CopyJob *pJob; }; } #endif // __XRD_CL_TP_FALLBACK_COPY_JOB__ xrootd-5.6.9/src/XrdCl/XrdClTaskManager.cc000066400000000000000000000214321457266313600203350ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2012 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . //------------------------------------------------------------------------------ #include "XrdCl/XrdClTaskManager.hh" #include "XrdCl/XrdClLog.hh" #include "XrdCl/XrdClUtils.hh" #include "XrdCl/XrdClDefaultEnv.hh" #include "XrdCl/XrdClConstants.hh" #include "XrdSys/XrdSysE2T.hh" #include "XrdSys/XrdSysTimer.hh" #include //------------------------------------------------------------------------------ // The thread //------------------------------------------------------------------------------ extern "C" { static void *RunRunnerThread( void *arg ) { using namespace XrdCl; TaskManager *mgr = (TaskManager*)arg; mgr->RunTasks(); return 0; } } namespace XrdCl { //---------------------------------------------------------------------------- // Constructor //---------------------------------------------------------------------------- TaskManager::TaskManager(): pResolution(1), pRunnerThread(0), pRunning(false) {} //---------------------------------------------------------------------------- // Destructor //---------------------------------------------------------------------------- TaskManager::~TaskManager() { TaskSet::iterator it, itE; for( it = pTasks.begin(); it != pTasks.end(); ++it ) if( it->own ) delete it->task; } //---------------------------------------------------------------------------- // Start the manager //---------------------------------------------------------------------------- bool TaskManager::Start() { XrdSysMutexHelper scopedLock( pOpMutex ); Log *log = DefaultEnv::GetLog(); log->Debug( TaskMgrMsg, "Starting the task manager..." ); if( pRunning ) { log->Error( TaskMgrMsg, "The task manager is already running" ); return false; } int ret = ::pthread_create( &pRunnerThread, 0, ::RunRunnerThread, this ); if( ret != 0 ) { log->Error( TaskMgrMsg, "Unable to spawn the task runner thread: %s", XrdSysE2T( errno ) ); return false; } pRunning = true; log->Debug( TaskMgrMsg, "Task manager started" ); return true; } //---------------------------------------------------------------------------- // Stop the manager //---------------------------------------------------------------------------- bool TaskManager::Stop() { XrdSysMutexHelper scopedLock( pOpMutex ); Log *log = DefaultEnv::GetLog(); log->Debug( TaskMgrMsg, "Stopping the task manager..." ); if( !pRunning ) { log->Error( TaskMgrMsg, "The task manager is not running" ); return false; } if( ::pthread_cancel( pRunnerThread ) != 0 ) { log->Error( TaskMgrMsg, "Unable to cancel the task runner thread: %s", XrdSysE2T( errno ) ); return false; } void *threadRet; int ret = pthread_join( pRunnerThread, (void **)&threadRet ); if( ret != 0 ) { log->Error( TaskMgrMsg, "Failed to join the task runner thread: %s", XrdSysE2T( errno ) ); return false; } pRunning = false; log->Debug( TaskMgrMsg, "Task manager stopped" ); return true; } //---------------------------------------------------------------------------- // Run the given task at the given time //---------------------------------------------------------------------------- void TaskManager::RegisterTask( Task *task, time_t time, bool own ) { Log *log = DefaultEnv::GetLog(); log->Debug( TaskMgrMsg, "Registering task: \"%s\" to be run at: [%s]", task->GetName().c_str(), Utils::TimeToString(time).c_str() ); XrdSysMutexHelper scopedLock( pMutex ); pTasks.insert( TaskHelper( task, time, own ) ); } //-------------------------------------------------------------------------- // Remove a task if it hasn't run yet //-------------------------------------------------------------------------- void TaskManager::UnregisterTask( Task *task ) { Log *log = DefaultEnv::GetLog(); log->Debug( TaskMgrMsg, "Requesting unregistration of: \"%s\"", task->GetName().c_str() ); XrdSysMutexHelper scopedLock( pMutex ); pToBeUnregistered.push_back( task ); } //---------------------------------------------------------------------------- // Run tasks //---------------------------------------------------------------------------- void TaskManager::RunTasks() { Log *log = DefaultEnv::GetLog(); //-------------------------------------------------------------------------- // We want the thread to be cancelable only when we sleep between tasks // execution //-------------------------------------------------------------------------- pthread_setcanceltype( PTHREAD_CANCEL_DEFERRED, 0 ); for(;;) { pthread_setcancelstate( PTHREAD_CANCEL_DISABLE, 0 ); pMutex.Lock(); //------------------------------------------------------------------------ // Remove the tasks from the active set - super inefficient, // but, hopefully, never really necessary. We first need to build a list // of iterators because it is impossible to remove elements from // a multiset when iterating over it //------------------------------------------------------------------------ TaskList::iterator listIt = pToBeUnregistered.begin(); TaskSet::iterator it, itE; std::list iteratorList; std::list::iterator itRem; for( ; listIt != pToBeUnregistered.end(); ++listIt ) { for( it = pTasks.begin(); it != pTasks.end(); ++it ) { if( it->task == *listIt ) iteratorList.push_back( it ); } } for( itRem = iteratorList.begin(); itRem != iteratorList.end(); ++itRem ) { Task *tsk = (*itRem)->task; bool own = (*itRem)->own; log->Debug( TaskMgrMsg, "Removing task: \"%s\"", tsk->GetName().c_str() ); pTasks.erase( *itRem ); if( own ) delete tsk; } pToBeUnregistered.clear(); //------------------------------------------------------------------------ // Select the tasks to be run //------------------------------------------------------------------------ time_t now = time(0); std::list toRun; std::list::iterator trIt; it = pTasks.begin(); itE = pTasks.upper_bound( TaskHelper( 0, now ) ); for( ; it != itE; ++it ) toRun.push_back( TaskHelper( it->task, 0, it->own ) ); pTasks.erase( pTasks.begin(), itE ); pMutex.UnLock(); //------------------------------------------------------------------------ // Run the tasks and reinsert them if necessary //------------------------------------------------------------------------ for( trIt = toRun.begin(); trIt != toRun.end(); ++trIt ) { log->Dump( TaskMgrMsg, "Running task: \"%s\"", trIt->task->GetName().c_str() ); time_t schedule = trIt->task->Run( now ); if( schedule ) { log->Dump( TaskMgrMsg, "Will rerun task \"%s\" at [%s]", trIt->task->GetName().c_str(), Utils::TimeToString(schedule).c_str() ); pMutex.Lock(); pTasks.insert( TaskHelper( trIt->task, schedule, trIt->own ) ); pMutex.UnLock(); } else { log->Debug( TaskMgrMsg, "Done with task: \"%s\"", trIt->task->GetName().c_str() ); if( trIt->own ) delete trIt->task; } } //------------------------------------------------------------------------ // Enable the cancellation and go to sleep //------------------------------------------------------------------------ pthread_setcancelstate( PTHREAD_CANCEL_ENABLE, 0 ); pthread_testcancel(); XrdSysTimer::Wait( pResolution*1000 ); } } } xrootd-5.6.9/src/XrdCl/XrdClTaskManager.hh000066400000000000000000000140671457266313600203550ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2012 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . //------------------------------------------------------------------------------ #ifndef __XRD_CL_TASK_MANAGER_HH__ #define __XRD_CL_TASK_MANAGER_HH__ #include #include #include #include #include #include #include "XrdSys/XrdSysPthread.hh" namespace XrdCl { //---------------------------------------------------------------------------- //! Interface for a task to be run by the TaskManager //---------------------------------------------------------------------------- class Task { public: virtual ~Task() {}; //------------------------------------------------------------------------ //! Perform the task //! //! @param now current timestamp //! @return 0 if the task is completed and should no longer be run or //! the time at which it should be run again //------------------------------------------------------------------------ virtual time_t Run( time_t now ) = 0; //------------------------------------------------------------------------ //! Name of the task //------------------------------------------------------------------------ const std::string &GetName() const { return pName; } //------------------------------------------------------------------------ //! Set name of the task //------------------------------------------------------------------------ void SetName( const std::string &name ) { pName = name; } private: std::string pName; }; //---------------------------------------------------------------------------- //! Run short tasks at a given time in the future //! //! The task manager just runs one extra thread so the execution of one tasks //! may interfere with the execution of another //---------------------------------------------------------------------------- class TaskManager { public: //------------------------------------------------------------------------ //! Constructor //------------------------------------------------------------------------ TaskManager(); //------------------------------------------------------------------------ //! Destructor //------------------------------------------------------------------------ ~TaskManager(); //------------------------------------------------------------------------ //! Start the manager //------------------------------------------------------------------------ bool Start(); //------------------------------------------------------------------------ //! Stop the manager //! //! Will wait until the currently running task completes //------------------------------------------------------------------------ bool Stop(); //------------------------------------------------------------------------ //! Run the given task at the given time. //! //! @param task task to be run //! @param time time at which the task should be run //! @param own determines whether the task object should be destroyed //! when no longer needed //------------------------------------------------------------------------ void RegisterTask( Task *task, time_t time, bool own = true ); //------------------------------------------------------------------------ //! Remove a task, the unregistration process is asynchronous and may //! be performed at any point in the future, the function just queues //! the request. Unregistered task gets destroyed if it was owned by //! the task manager. //------------------------------------------------------------------------ void UnregisterTask( Task *task ); //------------------------------------------------------------------------ //! Run the tasks - this loops infinitely //------------------------------------------------------------------------ void RunTasks(); private: //------------------------------------------------------------------------ // Task set helpers //------------------------------------------------------------------------ struct TaskHelper { TaskHelper( Task *tsk, time_t tme, bool ow = true ): task(tsk), execTime(tme), own(ow) {} Task *task; time_t execTime; bool own; }; struct TaskHelperCmp { bool operator () ( const TaskHelper &th1, const TaskHelper &th2 ) const { return th1.execTime < th2.execTime; } }; typedef std::multiset TaskSet; typedef std::list TaskList; //------------------------------------------------------------------------ // Private variables //------------------------------------------------------------------------ uint16_t pResolution; TaskSet pTasks; TaskList pToBeUnregistered; pthread_t pRunnerThread; bool pRunning; XrdSysMutex pMutex; XrdSysMutex pOpMutex; }; } #endif // __XRD_CL_TASK_MANAGER_HH__ xrootd-5.6.9/src/XrdCl/XrdClThirdPartyCopyJob.cc000066400000000000000000001033121457266313600215160ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2014 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #include "XrdCl/XrdClThirdPartyCopyJob.hh" #include "XrdCl/XrdClConstants.hh" #include "XrdCl/XrdClLog.hh" #include "XrdCl/XrdClDefaultEnv.hh" #include "XrdCl/XrdClUtils.hh" #include "XrdCl/XrdClMessageUtils.hh" #include "XrdCl/XrdClMonitor.hh" #include "XrdCl/XrdClRedirectorRegistry.hh" #include "XrdCl/XrdClDlgEnv.hh" #include "XrdOuc/XrdOucTPC.hh" #include "XrdSys/XrdSysPthread.hh" #include "XrdSys/XrdSysTimer.hh" #include #include #include #include #include #include #include #include #include namespace { //---------------------------------------------------------------------------- //! Handle an async response //---------------------------------------------------------------------------- class TPCStatusHandler: public XrdCl::ResponseHandler { public: //------------------------------------------------------------------------ // Constructor //------------------------------------------------------------------------ TPCStatusHandler(): pSem( new XrdSysSemaphore(0) ), pStatus(0) { } //------------------------------------------------------------------------ // Destructor //------------------------------------------------------------------------ virtual ~TPCStatusHandler() { delete pStatus; delete pSem; } //------------------------------------------------------------------------ // Handle Response //------------------------------------------------------------------------ virtual void HandleResponse( XrdCl::XRootDStatus *status, XrdCl::AnyObject *response ) { delete response; pStatus = status; pSem->Post(); } //------------------------------------------------------------------------ // Get Mutex //------------------------------------------------------------------------ XrdSysSemaphore *GetXrdSysSemaphore() { return pSem; } //------------------------------------------------------------------------ // Get status //------------------------------------------------------------------------ XrdCl::XRootDStatus *GetStatus() { return pStatus; } private: TPCStatusHandler(const TPCStatusHandler &other); TPCStatusHandler &operator = (const TPCStatusHandler &other); XrdSysSemaphore *pSem; XrdCl::XRootDStatus *pStatus; }; class InitTimeoutCalc { public: InitTimeoutCalc( uint16_t timeLeft ) : hasInitTimeout( timeLeft ), start( time( 0 ) ), timeLeft( timeLeft ) { } XrdCl::XRootDStatus operator()() { if( !hasInitTimeout ) return XrdCl::XRootDStatus(); time_t now = time( 0 ); if( now - start > timeLeft ) return XrdCl::XRootDStatus( XrdCl::stError, XrdCl::errOperationExpired ); timeLeft -= ( now - start ); return XrdCl::XRootDStatus(); } operator uint16_t() { return timeLeft; } private: bool hasInitTimeout; time_t start; uint16_t timeLeft; }; static XrdCl::XRootDStatus& UpdateErrMsg( XrdCl::XRootDStatus &status, const std::string &str ) { std::string msg = status.GetErrorMessage(); msg += " (" + str + ")"; status.SetErrorMessage( msg ); return status; } } namespace XrdCl { //---------------------------------------------------------------------------- // Constructor //---------------------------------------------------------------------------- ThirdPartyCopyJob::ThirdPartyCopyJob( uint16_t jobId, PropertyList *jobProperties, PropertyList *jobResults ): CopyJob( jobId, jobProperties, jobResults ), dstFile( File::DisableVirtRedirect ), sourceSize( 0 ), initTimeout( 0 ), force( false ), coerce( false ), delegate( false ), nbStrm( 0 ), tpcLite( false ) { Log *log = DefaultEnv::GetLog(); log->Debug( UtilityMsg, "Creating a third party copy job, from %s to %s", GetSource().GetURL().c_str(), GetTarget().GetURL().c_str() ); } //---------------------------------------------------------------------------- // Run the copy job //---------------------------------------------------------------------------- XRootDStatus ThirdPartyCopyJob::Run( CopyProgressHandler *progress ) { Log *log = DefaultEnv::GetLog(); XRootDStatus st = CanDo(); if( !st.IsOK() ) return st; if( tpcLite ) { //------------------------------------------------------------------------ // Run TPC-lite algorithm //------------------------------------------------------------------------ XRootDStatus st = RunLite( progress ); if( !st.IsOK() ) return st; } else { //------------------------------------------------------------------------ // Run vanilla TPC algorithm //------------------------------------------------------------------------ XRootDStatus st = RunTPC( progress ); if( !st.IsOK() ) return st; } //-------------------------------------------------------------------------- // Verify the checksums if needed //-------------------------------------------------------------------------- if( checkSumMode != "none" ) { log->Debug( UtilityMsg, "Attempting checksum calculation." ); std::string sourceCheckSum; std::string targetCheckSum; //------------------------------------------------------------------------ // Get the check sum at source //------------------------------------------------------------------------ timeval oStart, oEnd; XRootDStatus st; if( checkSumMode == "end2end" || checkSumMode == "source" || !checkSumPreset.empty() ) { gettimeofday( &oStart, 0 ); if( !checkSumPreset.empty() ) { sourceCheckSum = checkSumType + ":"; sourceCheckSum += Utils::NormalizeChecksum( checkSumType, checkSumPreset ); } else { VirtualRedirector *redirector = 0; std::string vrCheckSum; if( GetSource().IsMetalink() && ( redirector = RedirectorRegistry::Instance().Get( GetSource() ) ) && !( vrCheckSum = redirector->GetCheckSum( checkSumType ) ).empty() ) sourceCheckSum = vrCheckSum; else st = Utils::GetRemoteCheckSum( sourceCheckSum, checkSumType, tpcSource ); } gettimeofday( &oEnd, 0 ); if( !st.IsOK() ) return UpdateErrMsg( st, "source" ); pResults->Set( "sourceCheckSum", sourceCheckSum ); } //------------------------------------------------------------------------ // Get the check sum at destination //------------------------------------------------------------------------ timeval tStart, tEnd; if( checkSumMode == "end2end" || checkSumMode == "target" ) { gettimeofday( &tStart, 0 ); st = Utils::GetRemoteCheckSum( targetCheckSum, checkSumType, realTarget ); gettimeofday( &tEnd, 0 ); if( !st.IsOK() ) return UpdateErrMsg( st, "destination" ); pResults->Set( "targetCheckSum", targetCheckSum ); } //------------------------------------------------------------------------ // Make sure the checksums are both lower case //------------------------------------------------------------------------ auto sanitize_cksum = []( char c ) { std::locale loc; if( std::isalpha( c ) ) return std::tolower( c, loc ); return c; }; std::transform( sourceCheckSum.begin(), sourceCheckSum.end(), sourceCheckSum.begin(), sanitize_cksum ); std::transform( targetCheckSum.begin(), targetCheckSum.end(), targetCheckSum.begin(), sanitize_cksum ); //------------------------------------------------------------------------ // Compare and inform monitoring //------------------------------------------------------------------------ if( !sourceCheckSum.empty() && !targetCheckSum.empty() ) { bool match = false; if( sourceCheckSum == targetCheckSum ) match = true; Monitor *mon = DefaultEnv::GetMonitor(); if( mon ) { Monitor::CheckSumInfo i; i.transfer.origin = &GetSource(); i.transfer.target = &GetTarget(); i.cksum = sourceCheckSum; i.oTime = Utils::GetElapsedMicroSecs( oStart, oEnd ); i.tTime = Utils::GetElapsedMicroSecs( tStart, tEnd ); i.isOK = match; mon->Event( Monitor::EvCheckSum, &i ); } if( !match ) return XRootDStatus( stError, errCheckSumError, 0 ); log->Info(UtilityMsg, "Checksum verification: succeeded." ); } } return XRootDStatus(); } //---------------------------------------------------------------------------- // Check whether doing a third party copy is feasible for given // job descriptor //---------------------------------------------------------------------------- XRootDStatus ThirdPartyCopyJob::CanDo() { const URL &source = GetSource(); const URL &target = GetTarget(); //-------------------------------------------------------------------------- // We can only do a TPC if both source and destination are remote files //-------------------------------------------------------------------------- if( source.IsLocalFile() || target.IsLocalFile() ) return XRootDStatus( stError, errNotSupported, 0, "Cannot do a third-party-copy for local file." ); //-------------------------------------------------------------------------- // Check the initial settings //-------------------------------------------------------------------------- Log *log = DefaultEnv::GetLog(); log->Debug( UtilityMsg, "Check if third party copy between %s and %s " "is possible", source.GetURL().c_str(), target.GetURL().c_str() ); if( target.GetProtocol() != "root" && target.GetProtocol() != "xroot" && target.GetProtocol() != "roots" && target.GetProtocol() != "xroots" ) return XRootDStatus( stError, errNotSupported, 0, "Third-party-copy " "is only supported for root/xroot protocol." ); pProperties->Get( "initTimeout", initTimeout ); InitTimeoutCalc timeLeft( initTimeout ); pProperties->Get( "checkSumMode", checkSumMode ); pProperties->Get( "checkSumType", checkSumType ); pProperties->Get( "checkSumPreset", checkSumPreset ); pProperties->Get( "force", force ); pProperties->Get( "coerce", coerce ); pProperties->Get( "delegate", delegate ); XrdCl::Env *env = XrdCl::DefaultEnv::GetEnv(); env->GetInt( "SubStreamsPerChannel", nbStrm ); // account for the control stream if (nbStrm > 0) --nbStrm; bool tpcLiteOnly = false; if( !delegate ) log->Info( UtilityMsg, "We are NOT using delegation" ); //-------------------------------------------------------------------------- // Resolve the 'auto' checksum type. //-------------------------------------------------------------------------- if( checkSumType == "auto" ) { checkSumType = Utils::InferChecksumType( GetSource(), GetTarget() ); if( checkSumType.empty() ) log->Info( UtilityMsg, "Could not infer checksum type." ); else log->Info( UtilityMsg, "Using inferred checksum type: %s.", checkSumType.c_str() ); } //-------------------------------------------------------------------------- // Check if we can open the source. Note in TPC-lite scenario it is optional // for this step to be successful. //-------------------------------------------------------------------------- File sourceFile; XRootDStatus st; URL sourceURL = source; URL::ParamsMap params; // set WriteRecovery property std::string value; DefaultEnv::GetEnv()->GetString( "ReadRecovery", value ); sourceFile.SetProperty( "ReadRecovery", value ); // save the original opaque parameter list as specified by the user for later const URL::ParamsMap &srcparams = sourceURL.GetParams(); //-------------------------------------------------------------------------- // Do the facultative step at source only if the protocol is root/xroot, // otherwise don't bother //-------------------------------------------------------------------------- if( sourceURL.GetProtocol() == "root" || sourceURL.GetProtocol() == "xroot" || sourceURL.GetProtocol() == "roots" || sourceURL.GetProtocol() == "xroots" ) { params = sourceURL.GetParams(); params["tpc.stage"] = "placement"; sourceURL.SetParams( params ); log->Debug( UtilityMsg, "Trying to open %s for reading", sourceURL.GetURL().c_str() ); st = sourceFile.Open( sourceURL.GetURL(), OpenFlags::Read, Access::None, timeLeft ); } else st = XRootDStatus( stError, errNotSupported ); if( st.IsOK() ) { std::string sourceUrl; sourceFile.GetProperty( "LastURL", sourceUrl ); tpcSource = sourceUrl; VirtualRedirector *redirector = 0; long long size = -1; if( source.IsMetalink() && ( redirector = RedirectorRegistry::Instance().Get( tpcSource ) ) && ( size = redirector->GetSize() ) >= 0 ) sourceSize = size; else { StatInfo *statInfo; st = sourceFile.Stat( false, statInfo ); if (st.IsOK()) sourceSize = statInfo->GetSize(); delete statInfo; } } else { log->Info( UtilityMsg, "Cannot open source file %s: %s", source.GetURL().c_str(), st.ToStr().c_str() ); if( !delegate ) { //---------------------------------------------------------------------- // If we cannot contact the source and there is no credential to delegate // it cannot possibly work //---------------------------------------------------------------------- st.status = stFatal; return st; } tpcSource = sourceURL; tpcLiteOnly = true; } // get the opaque parameters as returned by the redirector URL tpcSourceUrl = tpcSource; URL::ParamsMap tpcsrcparams = tpcSourceUrl.GetParams(); // merge the original cgi with the one returned by the redirector, // the original values take precedence URL::ParamsMap::const_iterator itr = srcparams.begin(); for( ; itr != srcparams.end(); ++itr ) tpcsrcparams[itr->first] = itr->second; tpcSourceUrl.SetParams( tpcsrcparams ); // save the merged opaque parameter list for later std::string scgi; const URL::ParamsMap &scgiparams = tpcSourceUrl.GetParams(); itr = scgiparams.begin(); for( ; itr != scgiparams.end(); ++itr ) if( itr->first.compare( 0, 6, "xrdcl." ) != 0 ) { if( !scgi.empty() ) scgi += '\t'; scgi += itr->first + '=' + itr->second; } if( !timeLeft().IsOK() ) { // we still want to send a close, but we time it out quickly st = sourceFile.Close( 1 ); return XRootDStatus( stError, errOperationExpired ); } st = sourceFile.Close( timeLeft ); if( !timeLeft().IsOK() ) return XRootDStatus( stError, errOperationExpired ); //-------------------------------------------------------------------------- // Now we need to check the destination !!! //-------------------------------------------------------------------------- if( delegate ) DlgEnv::Instance().Enable(); else DlgEnv::Instance().Disable(); //-------------------------------------------------------------------------- // Generate the destination CGI //-------------------------------------------------------------------------- log->Debug( UtilityMsg, "Generating the destination TPC URL" ); tpcKey = GenerateKey(); char *cgiBuff = new char[2048]; const char *cgiP = XrdOucTPC::cgiC2Dst( tpcKey.c_str(), tpcSource.GetHostId().c_str(), tpcSource.GetPath().c_str(), 0, cgiBuff, 2048, nbStrm, GetSource().GetHostId().c_str(), GetSource().GetProtocol().c_str(), GetTarget().GetProtocol().c_str(), delegate ); if( *cgiP == '!' ) { log->Error( UtilityMsg, "Unable to setup target url: %s", cgiP+1 ); delete [] cgiBuff; return XRootDStatus( stError, errNotSupported ); } URL cgiURL; cgiURL.SetParams( cgiBuff ); delete [] cgiBuff; realTarget = GetTarget(); params = realTarget.GetParams(); MessageUtils::MergeCGI( params, cgiURL.GetParams(), true ); if( !tpcLiteOnly ) // we only append oss.asize if it source file size is actually known { std::ostringstream o; o << sourceSize; params["oss.asize"] = o.str(); } params["tpc.stage"] = "copy"; // forward source cgi info to the destination in case we are going to do delegation if( !scgi.empty() && delegate ) params["tpc.scgi"] = scgi; realTarget.SetParams( params ); log->Debug( UtilityMsg, "Target url is: %s", realTarget.GetURL().c_str() ); //-------------------------------------------------------------------------- // Open the target file //-------------------------------------------------------------------------- // set WriteRecovery property DefaultEnv::GetEnv()->GetString( "WriteRecovery", value ); dstFile.SetProperty( "WriteRecovery", value ); OpenFlags::Flags targetFlags = OpenFlags::Update; if( force ) targetFlags |= OpenFlags::Delete; else targetFlags |= OpenFlags::New; if( coerce ) targetFlags |= OpenFlags::Force; Access::Mode mode = Access::UR|Access::UW|Access::GR|Access::OR; st = dstFile.Open( realTarget.GetURL(), targetFlags, mode, timeLeft ); if( !st.IsOK() ) { log->Error( UtilityMsg, "Unable to open target %s: %s", realTarget.GetURL().c_str(), st.ToStr().c_str() ); if( st.code == errErrorResponse && st.errNo == kXR_FSError && st.GetErrorMessage().find( "tpc not supported" ) != std::string::npos ) return XRootDStatus( stError, errNotSupported, 0, // the open failed due to lack of TPC support on the server side "Destination does not support third-party-copy." ); return UpdateErrMsg( st, "destination" ); } std::string lastUrl; dstFile.GetProperty( "LastURL", lastUrl ); realTarget = lastUrl; if( !timeLeft().IsOK() ) { // we still want to send a close, but we time it out fast st = dstFile.Close( 1 ); return XRootDStatus( stError, errOperationExpired ); } //-------------------------------------------------------------------------- // Verify if the destination supports TPC / TPC-lite //-------------------------------------------------------------------------- st = Utils::CheckTPCLite( realTarget.GetHostId() ); if( !st.IsOK() ) { // we still want to send a close, but we time it out fast st = dstFile.Close( 1 ); return XRootDStatus( stError, errNotSupported, 0, // doesn't support TPC "Destination does not support third-party-copy."); } //-------------------------------------------------------------------------- // if target supports TPC-lite and we have a credential to delegate we can // go ahead and use TPC-lite //-------------------------------------------------------------------------- tpcLite = ( st.code != suPartial ) && delegate; if( !tpcLite && tpcLiteOnly ) // doesn't support TPC-lite and it was our only hope { st = dstFile.Close( 1 ); return XRootDStatus( stError, errNotSupported, 0, "Destination does not " "support delegation." ); } //-------------------------------------------------------------------------- // adjust the InitTimeout //-------------------------------------------------------------------------- if( !timeLeft().IsOK() ) { // we still want to send a close, but we time it out fast st = dstFile.Close( 1 ); return XRootDStatus( stError, errOperationExpired ); } //-------------------------------------------------------------------------- // If we don't use delegation the source has to support TPC //-------------------------------------------------------------------------- if( !tpcLite ) { st = Utils::CheckTPC( URL( tpcSource ).GetHostId(), timeLeft ); if( !st.IsOK() ) { log->Error( UtilityMsg, "Source (%s) does not support TPC", tpcSource.GetHostId().c_str() ); return XRootDStatus( stError, errNotSupported, 0, "Source does not " "support third-party-copy" ); } if( !timeLeft().IsOK() ) { // we still want to send a close, but we time it out quickly st = sourceFile.Close( 1 ); return XRootDStatus( stError, errOperationExpired ); } } initTimeout = uint16_t( timeLeft ); return XRootDStatus(); } //---------------------------------------------------------------------------- // Run vanilla copy job //---------------------------------------------------------------------------- XRootDStatus ThirdPartyCopyJob::RunTPC( CopyProgressHandler *progress ) { Log *log = DefaultEnv::GetLog(); //-------------------------------------------------------------------------- // Generate the source CGI //-------------------------------------------------------------------------- char *cgiBuff = new char[2048]; const char *cgiP = XrdOucTPC::cgiC2Src( tpcKey.c_str(), realTarget.GetHostName().c_str(), -1, cgiBuff, 2048 ); if( *cgiP == '!' ) { log->Error( UtilityMsg, "Unable to setup source url: %s", cgiP+1 ); delete [] cgiBuff; return XRootDStatus( stError, errInvalidArgs ); } URL cgiURL; cgiURL.SetParams( cgiBuff ); delete [] cgiBuff; URL::ParamsMap params = tpcSource.GetParams(); MessageUtils::MergeCGI( params, cgiURL.GetParams(), true ); params["tpc.stage"] = "copy"; tpcSource.SetParams( params ); log->Debug( UtilityMsg, "Source url is: %s", tpcSource.GetURL().c_str() ); // Set the close timeout to the default value of the stream timeout int closeTimeout = 0; (void) DefaultEnv::GetEnv()->GetInt( "StreamTimeout", closeTimeout); //-------------------------------------------------------------------------- // Set up the rendez-vous and open the source //-------------------------------------------------------------------------- InitTimeoutCalc timeLeft( initTimeout ); XRootDStatus st = dstFile.Sync( timeLeft ); if( !st.IsOK() ) { log->Error( UtilityMsg, "Unable set up rendez-vous: %s", st.ToStr().c_str() ); XRootDStatus status = dstFile.Close( closeTimeout ); return UpdateErrMsg( st, "destination" ); } //-------------------------------------------------------------------------- // Calculate the time we have left to perform source open //-------------------------------------------------------------------------- if( !timeLeft().IsOK() ) { XRootDStatus status = dstFile.Close( closeTimeout ); return XRootDStatus( stError, errOperationExpired ); } File sourceFile( File::DisableVirtRedirect ); // set ReadRecovery property std::string value; DefaultEnv::GetEnv()->GetString( "ReadRecovery", value ); sourceFile.SetProperty( "ReadRecovery", value ); st = sourceFile.Open( tpcSource.GetURL(), OpenFlags::Read, Access::None, timeLeft ); if( !st.IsOK() ) { log->Error( UtilityMsg, "Unable to open source %s: %s", tpcSource.GetURL().c_str(), st.ToStr().c_str() ); XRootDStatus status = dstFile.Close( closeTimeout ); return UpdateErrMsg( st, "source" ); } //-------------------------------------------------------------------------- // Do the copy and follow progress //-------------------------------------------------------------------------- TPCStatusHandler statusHandler; XrdSysSemaphore *sem = statusHandler.GetXrdSysSemaphore(); StatInfo *info = 0; uint16_t tpcTimeout = 0; pProperties->Get( "tpcTimeout", tpcTimeout ); st = dstFile.Sync( &statusHandler, tpcTimeout ); if( !st.IsOK() ) { log->Error( UtilityMsg, "Unable start the copy: %s", st.ToStr().c_str() ); XRootDStatus statusS = sourceFile.Close( closeTimeout ); XRootDStatus statusT = dstFile.Close( closeTimeout ); return UpdateErrMsg( st, "destination" ); } //-------------------------------------------------------------------------- // Stat the file every second until sync returns //-------------------------------------------------------------------------- bool canceled = false; while( 1 ) { XrdSysTimer::Wait( 2500 ); if( progress ) { st = dstFile.Stat( true, info ); if( st.IsOK() ) { progress->JobProgress( pJobId, info->GetSize(), sourceSize ); delete info; info = 0; } bool shouldCancel = progress->ShouldCancel( pJobId ); if( shouldCancel ) { log->Debug( UtilityMsg, "Cancellation requested by progress handler" ); Buffer arg, *response = 0; arg.FromString( "ofs.tpc cancel" ); XRootDStatus st = dstFile.Fcntl( arg, response ); if( !st.IsOK() ) log->Debug( UtilityMsg, "Error while trying to cancel tpc: %s", st.ToStr().c_str() ); delete response; canceled = true; break; } } if( sem->CondWait() ) break; } //-------------------------------------------------------------------------- // Sync has returned so we can check if it was successful //-------------------------------------------------------------------------- if( canceled ) sem->Wait(); st = *statusHandler.GetStatus(); if( !st.IsOK() ) { log->Error( UtilityMsg, "Third party copy from %s to %s failed: %s", GetSource().GetURL().c_str(), GetTarget().GetURL().c_str(), st.ToStr().c_str() ); // Ignore close response XRootDStatus statusS = sourceFile.Close( closeTimeout ); XRootDStatus statusT = dstFile.Close( closeTimeout ); return st; } XRootDStatus statusS = sourceFile.Close( closeTimeout ); XRootDStatus statusT = dstFile.Close( closeTimeout ); if ( !statusS.IsOK() || !statusT.IsOK() ) { st = (statusS.IsOK() ? statusT : statusS); log->Error( UtilityMsg, "Third party copy from %s to %s failed during " "close of %s: %s", GetSource().GetURL().c_str(), GetTarget().GetURL().c_str(), (statusS.IsOK() ? "destination" : "source"), st.ToStr().c_str() ); return UpdateErrMsg( st, statusS.IsOK() ? "source" : "destination" ); } log->Debug( UtilityMsg, "Third party copy from %s to %s successful", GetSource().GetURL().c_str(), GetTarget().GetURL().c_str() ); pResults->Set( "size", sourceSize ); return XRootDStatus(); } XRootDStatus ThirdPartyCopyJob::RunLite( CopyProgressHandler *progress ) { Log *log = DefaultEnv::GetLog(); // Set the close timeout to the default value of the stream timeout int closeTimeout = 0; (void) DefaultEnv::GetEnv()->GetInt( "StreamTimeout", closeTimeout); //-------------------------------------------------------------------------- // Set up the rendez-vous //-------------------------------------------------------------------------- InitTimeoutCalc timeLeft( initTimeout ); XRootDStatus st = dstFile.Sync( timeLeft ); if( !st.IsOK() ) { log->Error( UtilityMsg, "Unable set up rendez-vous: %s", st.ToStr().c_str() ); XRootDStatus status = dstFile.Close( closeTimeout ); return UpdateErrMsg( st, "destination" ); } //-------------------------------------------------------------------------- // Do the copy and follow progress //-------------------------------------------------------------------------- TPCStatusHandler statusHandler; XrdSysSemaphore *sem = statusHandler.GetXrdSysSemaphore(); StatInfo *info = 0; uint16_t tpcTimeout = 0; pProperties->Get( "tpcTimeout", tpcTimeout ); st = dstFile.Sync( &statusHandler, tpcTimeout ); if( !st.IsOK() ) { log->Error( UtilityMsg, "Unable start the copy: %s", st.ToStr().c_str() ); XRootDStatus statusT = dstFile.Close( closeTimeout ); return UpdateErrMsg( st, "destination" ); } //-------------------------------------------------------------------------- // Stat the file every second until sync returns //-------------------------------------------------------------------------- bool canceled = false; while( 1 ) { XrdSysTimer::Wait( 2500 ); if( progress ) { st = dstFile.Stat( true, info ); if( st.IsOK() ) { progress->JobProgress( pJobId, info->GetSize(), sourceSize ); delete info; info = 0; } bool shouldCancel = progress->ShouldCancel( pJobId ); if( shouldCancel ) { log->Debug( UtilityMsg, "Cancellation requested by progress handler" ); Buffer arg, *response = 0; arg.FromString( "ofs.tpc cancel" ); XRootDStatus st = dstFile.Fcntl( arg, response ); if( !st.IsOK() ) log->Debug( UtilityMsg, "Error while trying to cancel tpc: %s", st.ToStr().c_str() ); delete response; canceled = true; break; } } if( sem->CondWait() ) break; } //-------------------------------------------------------------------------- // Sync has returned so we can check if it was successful //-------------------------------------------------------------------------- if( canceled ) sem->Wait(); st = *statusHandler.GetStatus(); if( !st.IsOK() ) { log->Error( UtilityMsg, "Third party copy from %s to %s failed: %s", GetSource().GetURL().c_str(), GetTarget().GetURL().c_str(), st.ToStr().c_str() ); // Ignore close response XRootDStatus statusT = dstFile.Close( closeTimeout ); return st; } st = dstFile.Close( closeTimeout ); if ( !st.IsOK() ) { log->Error( UtilityMsg, "Third party copy from %s to %s failed during " "close of %s: %s", GetSource().GetURL().c_str(), GetTarget().GetURL().c_str(), "destination", st.ToStr().c_str() ); return UpdateErrMsg( st, "destination" ); } log->Debug( UtilityMsg, "Third party copy from %s to %s successful", GetSource().GetURL().c_str(), GetTarget().GetURL().c_str() ); pResults->Set( "size", sourceSize ); return XRootDStatus(); } //---------------------------------------------------------------------------- // Generate a rendez-vous key //---------------------------------------------------------------------------- std::string ThirdPartyCopyJob::GenerateKey() { static const int _10to9 = 1000000000; char tpcKey[25]; auto tp = std::chrono::high_resolution_clock::now(); auto d = tp.time_since_epoch(); auto ns = std::chrono::duration_cast( d ); auto s = std::chrono::duration_cast( d ); uint32_t k1 = ns.count() - s.count() * _10to9; uint32_t k2 = getpid() | (getppid() << 16); uint32_t k3 = s.count(); snprintf( tpcKey, 25, "%08x%08x%08x", k1, k2, k3 ); return std::string(tpcKey); } } xrootd-5.6.9/src/XrdCl/XrdClThirdPartyCopyJob.hh000066400000000000000000000072341457266313600215360ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2012 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . //------------------------------------------------------------------------------ #ifndef __XRD_CL_THIRD_PARTY_COPY_JOB_HH__ #define __XRD_CL_THIRD_PARTY_COPY_JOB_HH__ #include "XrdCl/XrdClCopyProcess.hh" #include "XrdCl/XrdClCopyJob.hh" #include "XrdCl/XrdClFile.hh" namespace XrdCl { class File; class ThirdPartyCopyJob: public CopyJob { public: //------------------------------------------------------------------------ //! Constructor //------------------------------------------------------------------------ ThirdPartyCopyJob( uint16_t jobId, PropertyList *jobProperties, PropertyList *jobResults ); //------------------------------------------------------------------------ //! Run the copy job //! //! @param progress the handler to be notified about the copy progress //! @return status of the copy operation //------------------------------------------------------------------------ virtual XRootDStatus Run( CopyProgressHandler *progress = 0 ); private: //------------------------------------------------------------------------ //! Check whether doing a third party copy is feasible for given //! job descriptor //! //! @param property list - may be extended by info needed for TPC //! @return error when a third party copy cannot be performed and //! fatal error when no copy can be performed //------------------------------------------------------------------------ XRootDStatus CanDo(); //------------------------------------------------------------------------ //! Run vanilla copy job //------------------------------------------------------------------------ XRootDStatus RunTPC( CopyProgressHandler *progress ); //------------------------------------------------------------------------ //! Run TPC-lite copy job //------------------------------------------------------------------------ XRootDStatus RunLite( CopyProgressHandler *progress ); //------------------------------------------------------------------------ //! Generate TPC key //------------------------------------------------------------------------ static std::string GenerateKey(); XrdCl::File dstFile; URL tpcSource; URL realTarget; std::string tpcKey; std::string checkSumMode; std::string checkSumType; std::string checkSumPreset; uint64_t sourceSize; uint16_t initTimeout; bool force; bool coerce; bool delegate; int nbStrm; bool tpcLite; }; } #endif // __XRD_CL_THIRD_PARTY_COPY_JOB_HH__ xrootd-5.6.9/src/XrdCl/XrdClTls.cc000066400000000000000000000372161457266313600167110ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2012 by European Organization for Nuclear Research (CERN) // Author: Michal Simon //------------------------------------------------------------------------------ // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . //------------------------------------------------------------------------------ #include "XrdCl/XrdClTls.hh" #include "XrdCl/XrdClPoller.hh" #include "XrdCl/XrdClSocket.hh" #include "XrdCl/XrdClDefaultEnv.hh" #include "XrdCl/XrdClLog.hh" #include "XrdCl/XrdClConstants.hh" #include "XrdTls/XrdTls.hh" #include "XrdTls/XrdTlsContext.hh" #include "XrdOuc/XrdOucUtils.hh" #include #include static std::unique_ptr tlsContext = nullptr; namespace { //------------------------------------------------------------------------ // Helper class for setting the message callback for the TLS layer for // logging purposes //------------------------------------------------------------------------ struct SetTlsMsgCB { //---------------------------------------------------------------------- // The message callback //---------------------------------------------------------------------- static void MsgCallBack(const char *tid, const char *msg, bool sslmsg) { XrdCl::Log *log = XrdCl::DefaultEnv::GetLog(); if( sslmsg ) log->Debug( XrdCl::TlsMsg, "[%s] %s", tid, msg ); else log->Error( XrdCl::TlsMsg, "[%s] %s", tid, msg ); } inline static void Once() { static SetTlsMsgCB instance; } private: //-------------------------------------------------------------------- // Constructor. Sets the callback, there should be only one static // instance //-------------------------------------------------------------------- inline SetTlsMsgCB() { XrdTls::SetMsgCB( MsgCallBack ); XrdTls::SetDebug( TlsDbgLvl(), MsgCallBack ); } //-------------------------------------------------------------------- // Get TLS debug level //-------------------------------------------------------------------- static int TlsDbgLvl() { XrdCl::Env *env = XrdCl::DefaultEnv::GetEnv(); std::string tlsDbgLvl; env->GetString( "TlsDbgLvl", tlsDbgLvl ); if( tlsDbgLvl == "OFF" ) return XrdTls::dbgOFF; if( tlsDbgLvl == "CTX" ) return XrdTls::dbgCTX; if( tlsDbgLvl == "SOK" ) return XrdTls::dbgSOK; if( tlsDbgLvl == "SIO" ) return XrdTls::dbgSIO; if( tlsDbgLvl == "ALL" ) return XrdTls::dbgALL; if( tlsDbgLvl == "OUT" ) return XrdTls::dbgOUT; return XrdTls::dbgOFF; } }; } namespace XrdCl { bool InitTLS() { if (tlsContext) return true; XrdCl::Env *env = XrdCl::DefaultEnv::GetEnv(); XrdCl::Log *log = XrdCl::DefaultEnv::GetLog(); int notls = false; env->GetInt("NoTlsOK", notls); if (notls) return false; const char *cadir = getenv("X509_CERT_DIR"); const char *cafile = getenv("X509_CERT_FILE"); if (!cadir && !cafile) cadir = "/etc/grid-security/certificates"; const char *msg; const mode_t camode = S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH; if (cadir && (msg = XrdOucUtils::ValPath(cadir, camode, true))) { log->Error(XrdCl::TlsMsg, "Failed to initialize TLS context: CA directory %s", msg); env->PutInt("NoTlsOK", 1); return false; } std::string emsg = "unknown error"; tlsContext = std::make_unique(nullptr, nullptr, cadir, cafile, 0ul, &emsg); if (!tlsContext || !tlsContext->isOK()) { tlsContext.reset(nullptr); log->Error(XrdCl::TlsMsg, "Failed to initialize TLS context: %s", emsg.c_str()); env->PutInt("NoTlsOK", 1); return false; } return true; } //------------------------------------------------------------------------ // Constructor //------------------------------------------------------------------------ Tls::Tls( Socket *socket, AsyncSocketHandler *socketHandler ) : pSocket( socket ), pTlsHSRevert( None ), pSocketHandler( socketHandler ) { //---------------------------------------------------------------------- // Set the message callback for TLS layer //---------------------------------------------------------------------- SetTlsMsgCB::Once(); if( !InitTLS() ) throw std::runtime_error( "Failed to initialize TLS" ); pTls.reset( new XrdTlsSocket( *tlsContext, pSocket->GetFD(), XrdTlsSocket::TLS_RNB_WNB, XrdTlsSocket::TLS_HS_NOBLK, true ) ); } //------------------------------------------------------------------------ // Establish a TLS/SSL session and perform host verification. //------------------------------------------------------------------------ XRootDStatus Tls::Connect( const std::string &thehost, XrdNetAddrInfo *netInfo ) { std::string errmsg; const char *verhost = 0; if( thehost != "localhost" && thehost != "127.0.0.1" && thehost != "[::1]" ) verhost = thehost.c_str(); XrdTls::RC error = pTls->Connect( verhost, &errmsg ); XRootDStatus status = ToStatus( error ); if( !status.IsOK() ) status.SetErrorMessage( errmsg ); //-------------------------------------------------------------------------- // There's no follow up if the read simply failed //-------------------------------------------------------------------------- if( !status.IsOK() ) { XrdCl::Log *log = XrdCl::DefaultEnv::GetLog(); log->Error( XrdCl::TlsMsg, "Failed to do TLS connect: %s", errmsg.c_str() ); return status; } if( pTls->NeedHandShake() ) { //------------------------------------------------------------------------ // Make sure the socket is uncorked so the TLS hand-shake can go through //------------------------------------------------------------------------ if( pSocket->IsCorked() ) { XRootDStatus st = pSocket->Uncork(); if( !st.IsOK() ) return st; } //---------------------------------------------------------------------- // Check if TLS hand-shake wants to write something //---------------------------------------------------------------------- if( error == XrdTls::TLS_WantWrite ) { XRootDStatus st = pSocketHandler->EnableUplink(); if( !st.IsOK() ) return st; } //---------------------------------------------------------------------- // Otherwise disable uplink //---------------------------------------------------------------------- else if( error == XrdTls::TLS_WantRead ) { XRootDStatus st = pSocketHandler->DisableUplink(); if( !st.IsOK() ) return st; } } return status; } XRootDStatus Tls::Read( char *buffer, size_t size, int &bytesRead ) { //-------------------------------------------------------------------------- // If necessary, TLS_read() will negotiate a TLS/SSL session, so we don't // have to explicitly call connect or do_handshake. //-------------------------------------------------------------------------- XrdTls::RC error = pTls->Read( buffer, size, bytesRead ); XRootDStatus status = ToStatus( error ); //-------------------------------------------------------------------------- // There's no follow up if the read simply failed //-------------------------------------------------------------------------- if( !status.IsOK() ) return status; if( pTls->NeedHandShake() ) { //------------------------------------------------------------------------ // Make sure the socket is uncorked so the TLS hand-shake can go through //------------------------------------------------------------------------ if( pSocket->IsCorked() ) { XRootDStatus st = pSocket->Uncork(); if( !st.IsOK() ) return st; } //---------------------------------------------------------------------- // Check if we need to switch on a revert state //---------------------------------------------------------------------- if( error == XrdTls::TLS_WantWrite ) { pTlsHSRevert = ReadOnWrite; XRootDStatus st = pSocketHandler->EnableUplink(); if( !st.IsOK() ) status = st; //-------------------------------------------------------------------- // Return early so the revert state won't get cleared //-------------------------------------------------------------------- return status; } } //------------------------------------------------------------------------ // If we got up until here we need to clear the revert state //------------------------------------------------------------------------ if( pTlsHSRevert == ReadOnWrite ) { XRootDStatus st = pSocketHandler->DisableUplink(); if( !st.IsOK() ) status = st; } pTlsHSRevert = None; //------------------------------------------------------------------------ // If we didn't manage to read any data wait for another read event //------------------------------------------------------------------------ if( bytesRead == 0 ) return XRootDStatus( stOK, suRetry ); return status; } //------------------------------------------------------------------------ //! (Fake) ReadV through the TLS layer from the socket //! If necessary, will establish a TLS/SSL session. //------------------------------------------------------------------------ XRootDStatus Tls::ReadV( iovec *iov, int iocnt, int &bytesRead ) { bytesRead = 0; for( int i = 0; i < iocnt; ++i ) { int btsread = 0; auto st = Read( static_cast( iov[i].iov_base ), iov[i].iov_len, btsread ); if( !st.IsOK() ) return st; bytesRead += btsread; if( st.code == suRetry ) return st; } return XRootDStatus(); } XRootDStatus Tls::Send( const char *buffer, size_t size, int &bytesWritten ) { //-------------------------------------------------------------------------- // If necessary, TLS_write() will negotiate a TLS/SSL session, so we don't // have to explicitly call connect or do_handshake. //-------------------------------------------------------------------------- XrdTls::RC error = pTls->Write( buffer, size, bytesWritten ); XRootDStatus status = ToStatus( error ); //-------------------------------------------------------------------------- // There's no follow up if the write simply failed //-------------------------------------------------------------------------- if( !status.IsOK() ) return status; //-------------------------------------------------------------------------- // We are in the middle of a TLS hand-shake //-------------------------------------------------------------------------- if( pTls->NeedHandShake() ) { //------------------------------------------------------------------------ // Make sure the socket is uncorked so the TLS hand-shake can go through //------------------------------------------------------------------------ if( pSocket->IsCorked() ) { XRootDStatus st = pSocket->Uncork(); if( !st.IsOK() ) return st; } //------------------------------------------------------------------------ // Check if we need to switch on a revert state //------------------------------------------------------------------------ if( error == XrdTls::TLS_WantRead ) { pTlsHSRevert = WriteOnRead; XRootDStatus st = pSocketHandler->DisableUplink(); if( !st.IsOK() ) status = st; //---------------------------------------------------------------------- // Return early so the revert state won't get cleared //---------------------------------------------------------------------- return status; } } //-------------------------------------------------------------------------- // If we got up until here we need to clear the revert state //-------------------------------------------------------------------------- if( pTlsHSRevert == WriteOnRead ) { XRootDStatus st = pSocketHandler->EnableUplink(); if( !st.IsOK() ) status = st; } pTlsHSRevert = None; //------------------------------------------------------------------------ // If we didn't manage to read any data wait for another write event // // Adding this by symmetry, never actually experienced this (for reads // it has been experienced) //------------------------------------------------------------------------ if( bytesWritten == 0 ) return XRootDStatus( stOK, suRetry ); return status; } //------------------------------------------------------------------------ // Shutdown the TLS/SSL connection //------------------------------------------------------------------------ void Tls::Shutdown() { pTls->Shutdown(); } XRootDStatus Tls::ToStatus( XrdTls::RC rc ) { std::string msg = XrdTls::RC2Text( rc, true ); switch( rc ) { case XrdTls::TLS_AOK: return XRootDStatus(); case XrdTls::TLS_WantConnect: case XrdTls::TLS_WantWrite: case XrdTls::TLS_WantRead: return XRootDStatus( stOK, suRetry, 0, msg ); case XrdTls::TLS_UNK_Error: case XrdTls::TLS_SYS_Error: return XRootDStatus( stError, errTlsError, 0, msg ); case XrdTls::TLS_SSL_Error: return XRootDStatus( stFatal, errTlsError, EAGAIN, msg ); case XrdTls::TLS_VER_Error: case XrdTls::TLS_HNV_Error: return XRootDStatus( stFatal, errTlsError, 0, msg ); // the connection was closed by the server, treat this as a socket error case XrdTls::TLS_CON_Closed: return XRootDStatus( stError, errSocketError ); default: return XRootDStatus( stError, errTlsError, 0, msg ); } } //------------------------------------------------------------------------ // Map: // * in case the TLS layer requested reads on writes map // ReadyToWrite to ReadyToRead // * in case the TLS layer requested writes on reads map // ReadyToRead to ReadyToWrite //------------------------------------------------------------------------ uint8_t Tls::MapEvent( uint8_t event ) { if( pTlsHSRevert == ReadOnWrite ) { //------------------------------------------------------------------------ // In this case we would like to call the OnRead routine on the Write event //------------------------------------------------------------------------ if( event & SocketHandler::ReadyToWrite ) return SocketHandler::ReadyToRead; } else if( pTlsHSRevert == WriteOnRead ) { //------------------------------------------------------------------------ // In this case we would like to call the OnWrite routine on the Read event //------------------------------------------------------------------------ if( event & SocketHandler::ReadyToRead ) return SocketHandler::ReadyToWrite; } return event; } void Tls::ClearErrorQueue() { XrdTls::ClearErrorQueue(); } } xrootd-5.6.9/src/XrdCl/XrdClTls.hh000066400000000000000000000147451457266313600167250ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2012 by European Organization for Nuclear Research (CERN) // Author: Michal Simon //------------------------------------------------------------------------------ // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . //------------------------------------------------------------------------------ #ifndef __XRD_CL_TLS_HH__ #define __XRD_CL_TLS_HH__ #include #include "XrdTls/XrdTlsSocket.hh" #include "XrdCl/XrdClXRootDResponses.hh" #include "XrdCl/XrdClAsyncSocketHandler.hh" namespace XrdCl { class Socket; /** Initialize TLS context, returns false on failure */ bool InitTLS(); //---------------------------------------------------------------------------- //! TLS layer for socket connection //---------------------------------------------------------------------------- class Tls { public: //------------------------------------------------------------------------ //! Constructor - creates async TLS layer for given socker file descriptor //------------------------------------------------------------------------ Tls( Socket *socket, AsyncSocketHandler *socketHandler ); //------------------------------------------------------------------------ //! Destructor //------------------------------------------------------------------------ ~Tls() { } //------------------------------------------------------------------------ //! Establish a TLS/SSL session and perform host verification. //------------------------------------------------------------------------ XRootDStatus Connect( const std::string &thehost, XrdNetAddrInfo *netInfo ); //------------------------------------------------------------------------ //! Read through the TLS layer from the socket //! If necessary, will establish a TLS/SSL session. //------------------------------------------------------------------------ XRootDStatus Read( char *buffer, size_t size, int &bytesRead ); //------------------------------------------------------------------------ //! (Fake) ReadV through the TLS layer from the socket //! If necessary, will establish a TLS/SSL session. //------------------------------------------------------------------------ XRootDStatus ReadV( iovec *iov, int iocnt, int &bytesRead ); //------------------------------------------------------------------------ //! Write through the TLS layer to the socket //! If necessary, will establish a TLS/SSL session. //------------------------------------------------------------------------ XRootDStatus Send( const char *buffer, size_t size, int &bytesWritten ); //------------------------------------------------------------------------ //! Shutdown the TLS/SSL connection //------------------------------------------------------------------------ void Shutdown(); //------------------------------------------------------------------------ //! Map: //! * in case the TLS layer requested reads on writes map //! ReadyToWrite to ReadyToRead //! * in case the TLS layer requested writes on reads map //! ReadyToRead to ReadyToWrite //------------------------------------------------------------------------ uint8_t MapEvent( uint8_t event ); //------------------------------------------------------------------------ //! Clear the error queue for the calling thread //------------------------------------------------------------------------ static void ClearErrorQueue(); private: //------------------------------------------------------------------------ //! Flags to indicate what is the TLS hand-shake revert state //! //! - None : there is no revert state //! - ReadOnWrite : OnRead routines will be called on write event due to //! TLS handshake //! - WriteOnRead : OnWrite routines will be called on read event due to //! TLS handshake //------------------------------------------------------------------------ enum TlsHSRevert{ None, ReadOnWrite, WriteOnRead }; //------------------------------------------------------------------------ //! Translate OPEN SSL error code into XRootD Status //------------------------------------------------------------------------ XRootDStatus ToStatus( XrdTls::RC rc ); //------------------------------------------------------------------------ //! The underlying vanilla socket //------------------------------------------------------------------------ Socket *pSocket; //------------------------------------------------------------------------ //! The TSL I/O wrapper over socket //------------------------------------------------------------------------ std::unique_ptr pTls; //------------------------------------------------------------------------ // In case during TLS hand-shake WantRead has been returned on write or // WantWrite has been returned on read we need to flip the following events. // // None : all events should be processed normally // ReadOnWrite : on write event the OnRead routines should be called // WriteOnRead : on read event the OnWrite routines should be called //------------------------------------------------------------------------ TlsHSRevert pTlsHSRevert; //------------------------------------------------------------------------ //! Socket handler (for enabling/disabling write notification) //------------------------------------------------------------------------ AsyncSocketHandler *pSocketHandler; }; } #endif // __XRD_CL_TLS_HH__ xrootd-5.6.9/src/XrdCl/XrdClTransportManager.cc000066400000000000000000000057601457266313600214350ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2012 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . //------------------------------------------------------------------------------ #include "XrdCl/XrdClTransportManager.hh" #include "XrdCl/XrdClXRootDTransport.hh" namespace XrdCl { //---------------------------------------------------------------------------- // Constructor //---------------------------------------------------------------------------- TransportManager::TransportManager() { pHandlers["root"] = new XRootDTransport(); pHandlers["xroot"] = new XRootDTransport(); pHandlers["roots"] = new XRootDTransport(); pHandlers["xroots"] = new XRootDTransport(); } //---------------------------------------------------------------------------- // Destructor //---------------------------------------------------------------------------- TransportManager::~TransportManager() { HandlerMap::iterator it; for( it = pHandlers.begin(); it != pHandlers.end(); ++it ) delete it->second; } //---------------------------------------------------------------------------- // Register a transport factory function for a given protocol //---------------------------------------------------------------------------- bool TransportManager::RegisterFactory( const std::string &protocol, TransportFactory factory ) { FactoryMap::iterator it = pFactories.find( protocol ); if( it == pFactories.end() ) return false; pFactories[protocol] = factory; return true; } //---------------------------------------------------------------------------- // Get a transport handler object for a given protocol //---------------------------------------------------------------------------- TransportHandler *TransportManager::GetHandler( const std::string &protocol ) { HandlerMap::iterator itH = pHandlers.find( protocol ); if( itH != pHandlers.end() ) return itH->second; FactoryMap::iterator itF = pFactories.find( protocol ); if( itF == pFactories.end() ) return 0; TransportHandler *handler = (*itF->second)(); pHandlers[protocol] = handler; return handler; } } xrootd-5.6.9/src/XrdCl/XrdClTransportManager.hh000066400000000000000000000053421457266313600214430ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2012 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . //------------------------------------------------------------------------------ #ifndef __XRD_CL_TRANSPORT_MANAGER_HH__ #define __XRD_CL_TRANSPORT_MANAGER_HH__ #include #include namespace XrdCl { class TransportHandler; //---------------------------------------------------------------------------- //! Manage transport handler objects //---------------------------------------------------------------------------- class TransportManager { public: typedef TransportHandler *(*TransportFactory)(); //------------------------------------------------------------------------ //! Constructor //------------------------------------------------------------------------ TransportManager(); //------------------------------------------------------------------------ // Destructor //------------------------------------------------------------------------ virtual ~TransportManager(); //------------------------------------------------------------------------ //! Register a transport factory function for a given protocol //------------------------------------------------------------------------ bool RegisterFactory( const std::string &protocol, TransportFactory factory ); //------------------------------------------------------------------------ //! Get a transport handler object for a given protocol //------------------------------------------------------------------------ TransportHandler *GetHandler( const std::string &protocol ); private: typedef std::map HandlerMap; typedef std::map FactoryMap; HandlerMap pHandlers; FactoryMap pFactories; }; } #endif // __XRD_CL_TRANSPORT_MANAGER_HH__ xrootd-5.6.9/src/XrdCl/XrdClURL.cc000066400000000000000000000413571457266313600166120ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2012 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . //------------------------------------------------------------------------------ #include "XrdCl/XrdClLog.hh" #include "XrdCl/XrdClDefaultEnv.hh" #include "XrdCl/XrdClConstants.hh" #include "XrdCl/XrdClURL.hh" #include "XrdCl/XrdClUtils.hh" #include #include #include #include namespace XrdCl { //---------------------------------------------------------------------------- // Constructor //---------------------------------------------------------------------------- URL::URL(): pPort( 1094 ) { } //---------------------------------------------------------------------------- // Constructor //---------------------------------------------------------------------------- URL::URL( const std::string &url ): pPort( 1094 ) { FromString( url ); } URL::URL( const char *url ) : pPort( 1094 ) { FromString( url ); } //---------------------------------------------------------------------------- // Parse URL - it is rather trivial and horribly slow but probably there // is not need to have anything more fancy //---------------------------------------------------------------------------- bool URL::FromString( const std::string &url ) { Log *log = DefaultEnv::GetLog(); Clear(); if( url.length() == 0 ) { log->Error( UtilityMsg, "The given URL is empty" ); return false; } //-------------------------------------------------------------------------- // Extract the protocol, assume file:// if none found //-------------------------------------------------------------------------- size_t pos = url.find( "://" ); std::string current; if( pos != std::string::npos ) { pProtocol = url.substr( 0, pos ); current = url.substr( pos+3 ); } else if( url[0] == '/' ) { pProtocol = "file"; current = url; } else if( url[0] == '-' ) { pProtocol = "stdio"; current = "-"; pPort = 0; } else { pProtocol = "root"; current = url; } //-------------------------------------------------------------------------- // If the protocol is HTTP or HTTPS, change the default port number //-------------------------------------------------------------------------- if (pProtocol == "http") { pPort = 80; } if (pProtocol == "https") { pPort = 443; } //-------------------------------------------------------------------------- // Extract host info and path //-------------------------------------------------------------------------- std::string path; std::string hostInfo; if( pProtocol == "stdio" ) path = current; else if( pProtocol == "file") { if( current[0] == '/' ) current = "localhost" + current; pos = current.find( '/' ); if( pos == std::string::npos ) hostInfo = current; else { hostInfo = current.substr( 0, pos ); path = current.substr( pos ); } } else { pos = current.find( '/' ); if( pos == std::string::npos ) hostInfo = current; else { hostInfo = current.substr( 0, pos ); path = current.substr( pos+1 ); } } if( !ParseHostInfo( hostInfo ) ) { Clear(); return false; } if( !ParsePath( path ) ) { Clear(); return false; } ComputeURL(); //-------------------------------------------------------------------------- // Dump the url //-------------------------------------------------------------------------- log->Dump( UtilityMsg, "URL: %s\n" "Protocol: %s\n" "User Name: %s\n" "Password: %s\n" "Host Name: %s\n" "Port: %d\n" "Path: %s\n", url.c_str(), pProtocol.c_str(), pUserName.c_str(), pPassword.c_str(), pHostName.c_str(), pPort, pPath.c_str() ); return true; } //---------------------------------------------------------------------------- // Parse host info //---------------------------------------------------------------------------- bool URL::ParseHostInfo( const std::string hostInfo ) { if( pProtocol == "stdio" ) return true; if( pProtocol.empty() || hostInfo.empty() ) return false; size_t pos = hostInfo.find( "@" ); std::string hostPort; //-------------------------------------------------------------------------- // We have found username-password //-------------------------------------------------------------------------- if( pos != std::string::npos ) { std::string userPass = hostInfo.substr( 0, pos ); hostPort = hostInfo.substr( pos+1 ); pos = userPass.find( ":" ); //------------------------------------------------------------------------ // It's both username and password //------------------------------------------------------------------------ if( pos != std::string::npos ) { pUserName = userPass.substr( 0, pos ); pPassword = userPass.substr( pos+1 ); if( pPassword.empty() ) return false; } //------------------------------------------------------------------------ // It's just the user name //------------------------------------------------------------------------ else pUserName = userPass; if( pUserName.empty() ) return false; } //-------------------------------------------------------------------------- // No username-password //-------------------------------------------------------------------------- else hostPort = hostInfo; //-------------------------------------------------------------------------- // Deal with hostname - IPv6 encoded address RFC 2732 //-------------------------------------------------------------------------- if( hostPort.length() >= 3 && hostPort[0] == '[' ) { pos = hostPort.find( "]" ); if( pos != std::string::npos ) { pHostName = hostPort.substr( 0, pos+1 ); hostPort.erase( 0, pos+2 ); //---------------------------------------------------------------------- // Check if we're IPv6 encoded IPv4 //---------------------------------------------------------------------- pos = pHostName.find( "." ); size_t pos2 = pHostName.find( "[::ffff" ); size_t pos3 = pHostName.find( "[::" ); if( pos != std::string::npos && pos3 != std::string::npos && pos2 == std::string::npos ) { pHostName.erase( 0, 3 ); pHostName.erase( pHostName.length()-1, 1 ); } } } else { pos = hostPort.find( ":" ); if( pos != std::string::npos ) { pHostName = hostPort.substr( 0, pos ); hostPort.erase( 0, pos+1 ); } else { pHostName = hostPort; hostPort = ""; } if( pHostName.empty() ) return false; } //-------------------------------------------------------------------------- // Deal with port number //-------------------------------------------------------------------------- if( !hostPort.empty() ) { char *result; pPort = ::strtol( hostPort.c_str(), &result, 0 ); if( *result != 0 ) return false; } ComputeHostId(); return true; } //---------------------------------------------------------------------------- // Parse path //---------------------------------------------------------------------------- bool URL::ParsePath( const std::string &path ) { size_t pos = path.find( "?" ); if( pos != std::string::npos ) { pPath = path.substr( 0, pos ); SetParams( path.substr( pos+1, path.length() ) ); } else pPath = path; if( !pPath.empty() ) { std::string::iterator back = pPath.end() - 1; if( pProtocol == "file" && *back == '/' ) pPath.erase( back ); } ComputeURL(); return true; } //---------------------------------------------------------------------------- // Get path with params //---------------------------------------------------------------------------- std::string URL::GetPathWithParams() const { std::ostringstream o; if( !pPath.empty() ) o << pPath; o << GetParamsAsString(); return o.str(); } //------------------------------------------------------------------------ //! Get the path with params, filteres out 'xrdcl.' //------------------------------------------------------------------------ std::string URL::GetPathWithFilteredParams() const { std::ostringstream o; if( !pPath.empty() ) o << pPath; o << GetParamsAsString( true ); return o.str(); } //------------------------------------------------------------------------ //! Get protocol://host:port/path //------------------------------------------------------------------------ std::string URL::GetLocation() const { std::ostringstream o; o << pProtocol << "://"; if( pProtocol == "file" ) o << pHostName; else o << pHostName << ":" << pPort << "/"; o << pPath; return o.str(); } //------------------------------------------------------------------------ // Get the URL params as string //------------------------------------------------------------------------ std::string URL::GetParamsAsString() const { return GetParamsAsString( false ); } //------------------------------------------------------------------------ // Get the login token if present in the opaque info //------------------------------------------------------------------------ std::string URL::GetLoginToken() const { auto itr = pParams.find( "xrd.logintoken" ); if( itr == pParams.end() ) return ""; return itr->second; } //------------------------------------------------------------------------ //! Get the URL params as string //------------------------------------------------------------------------ std::string URL::GetParamsAsString( bool filter ) const { if( pParams.empty() ) return ""; std::ostringstream o; o << "?"; ParamsMap::const_iterator it; for( it = pParams.begin(); it != pParams.end(); ++it ) { // we filter out client specific parameters if( filter && it->first.compare( 0, 6, "xrdcl." ) == 0 ) continue; if( it != pParams.begin() ) o << "&"; o << it->first << "=" << it->second; } std::string ret = o.str(); if( ret == "?" ) ret.clear(); return ret; } //------------------------------------------------------------------------ // Set params //------------------------------------------------------------------------ void URL::SetParams( const std::string ¶ms ) { pParams.clear(); std::string p = params; if( p.empty() ) return; if( p[0] == '?' ) p.erase( 0, 1 ); std::vector paramsVect; std::vector::iterator it; Utils::splitString( paramsVect, p, "&" ); for( it = paramsVect.begin(); it != paramsVect.end(); ++it ) { if( it->empty() ) continue; size_t qpos = it->find( '?' ); if( qpos != std::string::npos ) // we have login token { pParams["xrd.logintoken"] = it->substr( qpos + 1 ); it->erase( qpos ); } size_t pos = it->find( "=" ); if( pos == std::string::npos ) pParams[*it] = ""; else pParams[it->substr(0, pos)] = it->substr( pos+1, it->length() ); } } //---------------------------------------------------------------------------- // Clear the fields //---------------------------------------------------------------------------- void URL::Clear() { pHostId.clear(); pProtocol.clear(); pUserName.clear(); pPassword.clear(); pHostName.clear(); pPort = 1094; pPath.clear(); pParams.clear(); pURL.clear(); } //---------------------------------------------------------------------------- // Check validity //---------------------------------------------------------------------------- bool URL::IsValid() const { if( pProtocol.empty() ) return false; if( pProtocol == "file" && pPath.empty() ) return false; if( pProtocol == "stdio" && pPath != "-" ) return false; if( pProtocol != "file" && pProtocol != "stdio" && pHostName.empty() ) return false; return true; } bool URL::IsMetalink() const { Env *env = DefaultEnv::GetEnv(); int mlProcessing = DefaultMetalinkProcessing; env->GetInt( "MetalinkProcessing", mlProcessing ); if( !mlProcessing ) return false; return PathEndsWith( ".meta4" ) || PathEndsWith( ".metalink" ); } bool URL::IsLocalFile() const { return pProtocol == "file" && pHostName == "localhost"; } //------------------------------------------------------------------------ // Does the protocol indicate encryption //------------------------------------------------------------------------ bool URL::IsSecure() const { return ( pProtocol == "roots" || pProtocol == "xroots" ); } //------------------------------------------------------------------------ // Is the URL used in TPC context //------------------------------------------------------------------------ bool URL::IsTPC() const { ParamsMap::const_iterator itr = pParams.find( "xrdcl.intent" ); if( itr != pParams.end() ) return itr->second == "tpc"; return false; } bool URL::PathEndsWith(const std::string & sufix) const { if (sufix.size() > pPath.size()) return false; return std::equal(sufix.rbegin(), sufix.rend(), pPath.rbegin() ); } //------------------------------------------------------------------------ //Get the host part of the URL (user:password\@host:port) plus channel //specific CGI (xrdcl.identity & xrd.gsiusrpxy) //------------------------------------------------------------------------ std::string URL::GetChannelId() const { std::string ret = pProtocol + "://" + pHostId + "/"; bool hascgi = false; std::string keys[] = { "xrdcl.intent", "xrd.gsiusrpxy", "xrd.gsiusrcrt", "xrd.gsiusrkey", "xrd.sss", "xrd.k5ccname" }; size_t size = sizeof( keys ) / sizeof( std::string ); for( size_t i = 0; i < size; ++i ) { ParamsMap::const_iterator itr = pParams.find( keys[i] ); if( itr != pParams.end() ) { ret += hascgi ? '&' : '?'; ret += itr->first; ret += '='; ret += itr->second; hascgi = true; } } return ret; } //---------------------------------------------------------------------------- // Recompute the host id //---------------------------------------------------------------------------- void URL::ComputeHostId() { std::ostringstream o; if( !pUserName.empty() ) { o << pUserName; if( !pPassword.empty() ) o << ":" << pPassword; o << "@"; } if( pProtocol == "file" ) o << pHostName; else o << pHostName << ":" << pPort; pHostId = o.str(); } //---------------------------------------------------------------------------- // Recreate the url //---------------------------------------------------------------------------- void URL::ComputeURL() { if( !IsValid() ) pURL = ""; std::ostringstream o; if( !pProtocol.empty() ) o << pProtocol << "://"; if( !pUserName.empty() ) { o << pUserName; if( !pPassword.empty() ) o << ":" << pPassword; o << "@"; } if( !pHostName.empty() ) { if( pProtocol == "file" ) o << pHostName; else o << pHostName << ":" << pPort << "/"; } o << GetPathWithParams(); pURL = o.str(); } } xrootd-5.6.9/src/XrdCl/XrdClURL.hh000066400000000000000000000267601457266313600166250ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2012 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . //------------------------------------------------------------------------------ #ifndef __XRD_CL_URL_HH__ #define __XRD_CL_URL_HH__ #include #include namespace XrdCl { //---------------------------------------------------------------------------- //! URL representation //---------------------------------------------------------------------------- class URL { public: typedef std::map ParamsMap; //!< Map of get //!< params //------------------------------------------------------------------------ //! Default constructor //------------------------------------------------------------------------ URL(); //------------------------------------------------------------------------ //! Constructor //! //! @param url an url in format: //! protocol://user:password\@host:port/path?param1=x¶m2=y //------------------------------------------------------------------------ URL( const std::string &url ); //------------------------------------------------------------------------ //! Constructor //! //! @param url an url in format: //! protocol://user:password\@host:port/path?param1=x¶m2=y //------------------------------------------------------------------------ URL( const char *url ); //------------------------------------------------------------------------ //! Is the url valid //------------------------------------------------------------------------ bool IsValid() const; //------------------------------------------------------------------------ //! Is it a URL to a metalink //------------------------------------------------------------------------ bool IsMetalink() const; //------------------------------------------------------------------------ //! Is it a URL to a local file //! (file://localhost //------------------------------------------------------------------------ bool IsLocalFile() const; //------------------------------------------------------------------------ //! Does the protocol indicate encryption //------------------------------------------------------------------------ bool IsSecure() const; //------------------------------------------------------------------------ //! Is the URL used in TPC context //------------------------------------------------------------------------ bool IsTPC() const; //------------------------------------------------------------------------ //! Get the URL //------------------------------------------------------------------------ std::string GetURL() const { return pURL; } //------------------------------------------------------------------------ //! Get the host part of the URL (user:password\@host:port) //------------------------------------------------------------------------ std::string GetHostId() const { return pHostId; } //------------------------------------------------------------------------ //! Get the host part of the URL (user:password\@host:port) plus channel //! specific CGI (xrdcl.identity & xrd.gsiusrpxy) //------------------------------------------------------------------------ std::string GetChannelId() const; //------------------------------------------------------------------------ //! Get location (protocol://host:port/path) //------------------------------------------------------------------------ std::string GetLocation() const; //------------------------------------------------------------------------ //! Get the protocol //------------------------------------------------------------------------ const std::string &GetProtocol() const { return pProtocol; } //------------------------------------------------------------------------ //! Set protocol //------------------------------------------------------------------------ void SetProtocol( const std::string &protocol ) { pProtocol = protocol; ComputeURL(); } //------------------------------------------------------------------------ //! Get the username //------------------------------------------------------------------------ const std::string &GetUserName() const { return pUserName; } //------------------------------------------------------------------------ //! Set the username //------------------------------------------------------------------------ void SetUserName( const std::string &userName ) { pUserName = userName; ComputeHostId(); ComputeURL(); } //------------------------------------------------------------------------ //! Get the password //------------------------------------------------------------------------ const std::string &GetPassword() const { return pPassword; } //------------------------------------------------------------------------ //! Set the password //------------------------------------------------------------------------ void SetPassword( const std::string &password ) { pPassword = password; ComputeURL(); } //------------------------------------------------------------------------ //! Get the name of the target host //------------------------------------------------------------------------ const std::string &GetHostName() const { return pHostName; } //------------------------------------------------------------------------ //! Set the host name //------------------------------------------------------------------------ void SetHostName( const std::string &hostName ) { pHostName = hostName; ComputeHostId(); ComputeURL(); } //------------------------------------------------------------------------ //! Get the target port //------------------------------------------------------------------------ int GetPort() const { return pPort; } //------------------------------------------------------------------------ // Set port //------------------------------------------------------------------------ void SetPort( int port ) { pPort = port; ComputeHostId(); ComputeURL(); } //------------------------------------------------------------------------ // Set host and port //------------------------------------------------------------------------ void SetHostPort( const std::string &hostName, int port ) { pHostName = hostName; pPort = port; ComputeHostId(); ComputeURL(); } //------------------------------------------------------------------------ //! Get the path //------------------------------------------------------------------------ const std::string &GetPath() const { return pPath; } //------------------------------------------------------------------------ //! Set the path //------------------------------------------------------------------------ void SetPath( const std::string &path ) { pPath = path; ComputeURL(); } //------------------------------------------------------------------------ //! Get the path with params //------------------------------------------------------------------------ std::string GetPathWithParams() const; //------------------------------------------------------------------------ //! Get the path with params, filteres out 'xrdcl.' //------------------------------------------------------------------------ std::string GetPathWithFilteredParams() const; //------------------------------------------------------------------------ //! Get the URL params //------------------------------------------------------------------------ const ParamsMap &GetParams() const { return pParams; } //------------------------------------------------------------------------ //! Get the URL params as string //------------------------------------------------------------------------ std::string GetParamsAsString() const; //------------------------------------------------------------------------ //! Get the login token if present in the opaque info //------------------------------------------------------------------------ std::string GetLoginToken() const; //------------------------------------------------------------------------ //! Get the URL params as string //! //! @param filter : if set to true filters out 'xrdcl.' //------------------------------------------------------------------------ std::string GetParamsAsString( bool filter ) const; //------------------------------------------------------------------------ //! Set params //------------------------------------------------------------------------ void SetParams( const std::string ¶ms ); //------------------------------------------------------------------------ //! Set params //------------------------------------------------------------------------ void SetParams( const ParamsMap ¶ms ) { pParams = params; ComputeURL(); } //------------------------------------------------------------------------ //! Parse a string and fill the URL fields //------------------------------------------------------------------------ bool FromString( const std::string &url ); //------------------------------------------------------------------------ //! Clear the url //------------------------------------------------------------------------ void Clear(); private: bool ParseHostInfo( const std::string hhostInfo ); bool ParsePath( const std::string &path ); void ComputeHostId(); void ComputeURL(); bool PathEndsWith( const std::string & sufix ) const; std::string pHostId; std::string pProtocol; std::string pUserName; std::string pPassword; std::string pHostName; int pPort; std::string pPath; ParamsMap pParams; std::string pURL; }; } #endif // __XRD_CL_URL_HH__ xrootd-5.6.9/src/XrdCl/XrdClUtils.cc000066400000000000000000001002021457266313600172310ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2012 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . //------------------------------------------------------------------------------ #include "XrdCl/XrdClUtils.hh" #include "XrdCl/XrdClFileSystem.hh" #include "XrdCl/XrdClDefaultEnv.hh" #include "XrdCl/XrdClConstants.hh" #include "XrdCl/XrdClCheckSumManager.hh" #include "XrdCl/XrdClRedirectorRegistry.hh" #include "XrdCl/XrdClMessage.hh" #include "XrdNet/XrdNetAddr.hh" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #if __cplusplus < 201103L #include #endif namespace { bool isNotSpace( char c ) { return c != ' '; } //---------------------------------------------------------------------------- // Ordering function for sorting IP addresses //---------------------------------------------------------------------------- struct PreferIPv6 { bool operator() ( const XrdNetAddr &l, const XrdNetAddr &r ) { bool rIsIPv4 = false; if( r.isIPType( XrdNetAddrInfo::IPv4 ) || (r.isIPType( XrdNetAddrInfo::IPv6 ) && r.isMapped()) ) rIsIPv4 = true; if( l.isIPType( XrdNetAddrInfo::IPv6 ) && rIsIPv4 ) return true; return false; } }; } namespace XrdCl { //---------------------------------------------------------------------------- // Get a parameter either from the environment or URL //---------------------------------------------------------------------------- int Utils::GetIntParameter( const URL &url, const std::string &name, int defaultVal ) { Env *env = DefaultEnv::GetEnv(); int value = defaultVal; char *endPtr; URL::ParamsMap::const_iterator it; env->GetInt( name, value ); it = url.GetParams().find( std::string("XrdCl.") + name ); if( it != url.GetParams().end() ) { int urlValue = (int)strtol( it->second.c_str(), &endPtr, 0 ); if( !*endPtr ) value = urlValue; } return value; } //---------------------------------------------------------------------------- // Get a parameter either from the environment or URL //---------------------------------------------------------------------------- std::string Utils::GetStringParameter( const URL &url, const std::string &name, const std::string &defaultVal ) { Env *env = DefaultEnv::GetEnv(); std::string value = defaultVal; URL::ParamsMap::const_iterator it; env->GetString( name, value ); it = url.GetParams().find( std::string("XrdCl.") + name ); if( it != url.GetParams().end() ) value = it->second; return value; } //---------------------------------------------------------------------------- // Interpret a string as address type, default to IPAll //---------------------------------------------------------------------------- Utils::AddressType Utils::String2AddressType( const std::string &addressType ) { if( addressType == "IPv6" ) return IPv6; else if( addressType == "IPv4" ) return IPv4; else if( addressType == "IPv4Mapped6" ) return IPv4Mapped6; else if( addressType == "IPAll" ) return IPAll; else return IPAuto; } //---------------------------------------------------------------------------- // Resolve IP addresses //---------------------------------------------------------------------------- Status Utils::GetHostAddresses( std::vector &addresses, const URL &url, Utils::AddressType type ) { Log *log = DefaultEnv::GetLog(); const char *err = 0; int ordn; //-------------------------------------------------------------------------- // Resolve all the addresses //-------------------------------------------------------------------------- std::ostringstream o; o << url.GetHostName() << ":" << url.GetPort(); XrdNetUtils::AddrOpts opts; if( type == IPv6 ) opts = XrdNetUtils::onlyIPv6; else if( type == IPv4 ) opts = XrdNetUtils::onlyIPv4; else if( type == IPv4Mapped6 ) opts = XrdNetUtils::allV4Map; else if( type == IPAll ) opts = XrdNetUtils::allIPMap; else opts = XrdNetUtils::prefAuto; //-------------------------------------------------------------------------- // Check what are the preferences IPv6 or IPv4 //-------------------------------------------------------------------------- int preferIPv4 = DefaultPreferIPv4; DefaultEnv::GetEnv()->GetInt( "PreferIPv4", preferIPv4 ); //-------------------------------------------------------------------------- // Partition the addresses according to the preferences // // The preferred IP family goes to the back as it is easier to remove // items from the back of the vector //-------------------------------------------------------------------------- opts |= (preferIPv4 ? XrdNetUtils::order64 : XrdNetUtils::order46); //-------------------------------------------------------------------------- // Now get all of the properly partitioned addresses; ordn will hold the // number of non-preferred addresses at the front of the table. //-------------------------------------------------------------------------- err = XrdNetUtils::GetAddrs( o.str(), addresses, &ordn, opts ); if( err ) { log->Error( UtilityMsg, "Unable to resolve %s: %s", o.str().c_str(), err ); return Status( stError, errInvalidAddr ); } if( addresses.size() == 0 ) { log->Error( UtilityMsg, "No addresses for %s were found", o.str().c_str() ); return Status( stError, errInvalidAddr ); } //-------------------------------------------------------------------------- // Shuffle each partition //-------------------------------------------------------------------------- int ipNoShuffle = DefaultIPNoShuffle; Env *env = DefaultEnv::GetEnv(); env->GetInt( "IPNoShuffle", ipNoShuffle ); if( !ipNoShuffle ) { #if __cplusplus < 201103L // initialize the random generator only once static struct only_once_t { only_once_t() { std::srand ( unsigned ( std::time(0) ) ); } } only_once; std::random_shuffle( addresses.begin(), addresses.begin() + ordn ); std::random_shuffle( addresses.begin() + ordn, addresses.end() ); #else static std::default_random_engine rand_engine( std::chrono::system_clock::now().time_since_epoch().count() ); std::shuffle( addresses.begin(), addresses.begin() + ordn, rand_engine ); std::shuffle( addresses.begin() + ordn, addresses.end(), rand_engine ); #endif } //-------------------------------------------------------------------------- // Return status as the result is already in the output parameter //-------------------------------------------------------------------------- return Status(); } //---------------------------------------------------------------------------- // Log all the addresses on the list //---------------------------------------------------------------------------- void Utils::LogHostAddresses( Log *log, uint64_t type, const std::string &hostId, std::vector &addresses ) { std::string addrStr; std::vector::iterator it; for( it = addresses.begin(); it != addresses.end(); ++it ) { char nameBuff[256]; it->Format( nameBuff, 256, XrdNetAddrInfo::fmtAdv6 ); addrStr += nameBuff; addrStr += ", "; } addrStr.erase( addrStr.length()-2, 2 ); log->Debug( type, "[%s] Found %d address(es): %s", hostId.c_str(), addresses.size(), addrStr.c_str() ); } //---------------------------------------------------------------------------- // Convert timestamp to a string //---------------------------------------------------------------------------- std::string Utils::TimeToString( time_t timestamp ) { char now[30]; tm tsNow; time_t ttNow = timestamp; localtime_r( &ttNow, &tsNow ); strftime( now, 30, "%Y-%m-%d %H:%M:%S %z", &tsNow ); return now; } //---------------------------------------------------------------------------- // Get the elapsed microseconds between two timevals //---------------------------------------------------------------------------- uint64_t Utils::GetElapsedMicroSecs( timeval start, timeval end ) { uint64_t startUSec = start.tv_sec*1000000 + start.tv_usec; uint64_t endUSec = end.tv_sec*1000000 + end.tv_usec; return endUSec-startUSec; } //---------------------------------------------------------------------------- // Get remote checksum //---------------------------------------------------------------------------- XRootDStatus Utils::GetRemoteCheckSum( std::string &checkSum, const std::string &checkSumType, const URL &url ) { FileSystem *fs = new FileSystem( url ); // add the 'cks.type' cgi tag in order to // select the proper checksum type in case // the server supports more than one checksum size_t pos = url.GetPath().find( '?' ); std::string cksPath = url.GetPath() + ( pos == std::string::npos ? '?' : '&' ) + "cks.type=" + checkSumType; Buffer arg; arg.FromString( cksPath ); Buffer *cksResponse = 0; XRootDStatus st; Log *log = DefaultEnv::GetLog(); st = fs->Query( QueryCode::Checksum, arg, cksResponse ); delete fs; if( !st.IsOK() ) { std::string msg = st.GetErrorMessage(); msg += " Got an error while querying the checksum!"; st.SetErrorMessage( msg ); return st; } if( !cksResponse ) return XRootDStatus( stError, errInternal, 0, "Got invalid response while querying the checksum!" ); std::vector elems; Utils::splitString( elems, cksResponse->ToString(), " " ); delete cksResponse; if( elems.size() != 2 ) return XRootDStatus( stError, errInvalidResponse, 0, "Got invalid response while querying the checksum!" ); if( elems[0] != checkSumType ) return XRootDStatus( stError, errCheckSumError ); checkSum = elems[0] + ":"; checkSum += NormalizeChecksum( elems[0], elems[1] ); log->Dump( UtilityMsg, "Checksum for %s checksum: %s", url.GetPath().c_str(), checkSum.c_str() ); return XRootDStatus(); } //------------------------------------------------------------------------ // Get a checksum from local file //------------------------------------------------------------------------ XRootDStatus Utils::GetLocalCheckSum( std::string &checkSum, const std::string &checkSumType, const std::string &path ) { Log *log = DefaultEnv::GetLog(); CheckSumManager *cksMan = DefaultEnv::GetCheckSumManager(); if( !cksMan ) { log->Error( UtilityMsg, "Unable to get the checksum manager" ); return XRootDStatus( stError, errInternal ); } XrdCksData ckSum; ckSum.Set( checkSumType.c_str() ); bool status = cksMan->Calculate( ckSum, checkSumType, path.c_str() ); if( !status ) { log->Error( UtilityMsg, "Error while calculating checksum for %s", path.c_str() ); return XRootDStatus( stError, errCheckSumError ); } char *cksBuffer = new char[265]; ckSum.Get( cksBuffer, 256 ); checkSum = checkSumType + ":"; checkSum += NormalizeChecksum( checkSumType, cksBuffer ); delete [] cksBuffer; log->Dump( UtilityMsg, "Checksum for %s is: %s", path.c_str(), checkSum.c_str() ); return XRootDStatus(); } //---------------------------------------------------------------------------- // Convert bytes to a human readable string //---------------------------------------------------------------------------- std::string Utils::BytesToString( uint64_t bytes ) { double final = bytes; int i = 0; char suf[3] = { 'k', 'M', 'G' }; for( i = 0; i < 3 && final > 1024; ++i, final /= 1024 ) {}; std::ostringstream o; o << std::setprecision(4) << final; if( i > 0 ) o << suf[i-1]; return o.str(); } //---------------------------------------------------------------------------- // Check if peer supports tpc //---------------------------------------------------------------------------- XRootDStatus Utils::CheckTPC( const std::string &server, uint16_t timeout ) { Log *log = DefaultEnv::GetLog(); log->Debug( UtilityMsg, "Checking if the data server %s supports tpc", server.c_str() ); FileSystem sourceDSFS( server ); Buffer queryArg; queryArg.FromString( "tpc" ); Buffer *queryResponse = 0; XRootDStatus st; st = sourceDSFS.Query( QueryCode::Config, queryArg, queryResponse, timeout ); if( !st.IsOK() ) { log->Error( UtilityMsg, "Cannot query source data server %s: %s", server.c_str(), st.ToStr().c_str() ); st.status = stFatal; return st; } if( !queryResponse ) { log->Error( UtilityMsg, "Cannot query source data server: empty response." ); st.status = stFatal; return st; } std::string answer = queryResponse->ToString(); delete queryResponse; if( answer.length() == 1 || !isdigit( answer[0] ) || atoi(answer.c_str()) == 0) { log->Debug( UtilityMsg, "Third party copy not supported at: %s", server.c_str() ); return XRootDStatus( stError, errNotSupported ); } log->Debug( UtilityMsg, "Third party copy supported at: %s", server.c_str() ); return XRootDStatus(); } //------------------------------------------------------------------------ // Check if peer supports tpc / tpc lite //------------------------------------------------------------------------ XRootDStatus Utils::CheckTPCLite( const std::string &server, uint16_t timeout ) { Log *log = DefaultEnv::GetLog(); log->Debug( UtilityMsg, "Checking if the data server %s supports tpc / tpc lite", server.c_str() ); FileSystem sourceDSFS( server ); Buffer queryArg; queryArg.FromString( "tpc tpcdlg" ); Buffer *queryResponse = 0; XRootDStatus st; st = sourceDSFS.Query( QueryCode::Config, queryArg, queryResponse, timeout ); if( !st.IsOK() ) { log->Error( UtilityMsg, "Cannot query source data server %s: %s", server.c_str(), st.ToStr().c_str() ); st.status = stFatal; return st; } if( !queryResponse ) { log->Error( UtilityMsg, "Cannot query source data server: empty response." ); st.status = stFatal; return st; } std::string answer = queryResponse->ToString(); delete queryResponse; if( answer.empty() ) { log->Error( UtilityMsg, "Cannot query source data server: empty response." ); st.status = stFatal; return st; } std::vector resp; Utils::splitString( resp, answer, "\n" ); if( resp.empty() || resp[0].empty() || !isdigit( resp[0][0]) || atoi( resp[0].c_str() ) == 0 ) { log->Debug( UtilityMsg, "Third party copy not supported at: %s", server.c_str() ); return XRootDStatus( stError, errNotSupported ); } if( resp.size() == 1 || resp[1] == "tpcdlg" ) { log->Debug( UtilityMsg, "TPC lite not supported at: %s", server.c_str() ); return XRootDStatus( stOK, suPartial ); } log->Debug( UtilityMsg, "TPC lite supported at: %s", server.c_str() ); return XRootDStatus(); } //---------------------------------------------------------------------------- // Convert the fully qualified host name to country code //---------------------------------------------------------------------------- std::string Utils::FQDNToCC( const std::string &fqdn ) { std::vector el; Utils::splitString( el, fqdn, "." ); if( el.size() < 2 ) return "us"; std::string cc = *el.rbegin(); if( cc.length() == 2 ) return cc; return "us"; } //---------------------------------------------------------------------------- // Get directory entries //---------------------------------------------------------------------------- Status Utils::GetDirectoryEntries( std::vector &entries, const std::string &path ) { DIR *dp = opendir( path.c_str() ); if( !dp ) return Status( stError, errOSError, errno ); dirent *dirEntry; while( (dirEntry = readdir(dp)) != 0 ) { std::string entryName = dirEntry->d_name; if( !entryName.compare( 0, 2, "..") ) continue; if( !entryName.compare( 0, 1, ".") ) continue; entries.push_back( dirEntry->d_name ); } closedir(dp); return Status(); } //---------------------------------------------------------------------------- // Process a config file and return key-value pairs //---------------------------------------------------------------------------- Status Utils::ProcessConfig( std::map &config, const std::string &file ) { config.clear(); std::ifstream inFile( file.c_str() ); if( !inFile.good() ) return Status( stError, errOSError, errno ); errno = 0; std::string line; while( std::getline( inFile, line ) ) { if( line.empty() || line[0] == '#' ) continue; std::vector elems; splitString( elems, line, "=" ); if( elems.size() != 2 ) return Status( stError, errConfig ); std::string key = elems[0]; Trim( key ); std::string value = elems[1]; Trim( value ); config[key] = value; } if( errno ) return Status( stError, errOSError, errno ); return Status(); } //------------------------------------------------------------------------ //! Process a config directory and return key-value pairs //------------------------------------------------------------------------ Status Utils::ProcessConfigDir( std::map &config, const std::string &dir ) { Log *log = DefaultEnv::GetLog(); log->Debug( UtilityMsg, "Processing configuration files in %s...", dir.c_str()); std::vector entries; Status st = Utils::GetDirectoryEntries( entries, dir ); if( !st.IsOK() ) { log->Debug( UtilityMsg, "Unable to process directory %s: %s", dir.c_str(), st.ToString().c_str() ); return st; } static const std::string suffix = ".conf"; for( auto &entry : entries ) { std::string confFile = dir + "/" + entry; if( confFile.length() <= suffix.length() ) continue; if( !std::equal( suffix.rbegin(), suffix.rend(), confFile.rbegin() ) ) continue; st = ProcessConfig( config, confFile ); if( !st.IsOK() ) { log->Debug( UtilityMsg, "Unable to process configuration file %s: %s", confFile.c_str(), st.ToString().c_str() ); } } return Status(); } //---------------------------------------------------------------------------- // Trim a string //---------------------------------------------------------------------------- void Utils::Trim( std::string &str ) { str.erase( str.begin(), std::find_if( str.begin(), str.end(), isNotSpace ) ); str.erase( std::find_if( str.rbegin(), str.rend(), isNotSpace ).base(), str.end() ); } //---------------------------------------------------------------------------- // Log property list //---------------------------------------------------------------------------- void Utils::LogPropertyList( Log *log, uint64_t topic, const char *format, const PropertyList &list ) { PropertyList::PropertyMap::const_iterator it; std::string keyVals; for( it = list.begin(); it != list.end(); ++it ) keyVals += "'" + it->first + "' = '" + it->second + "', "; keyVals.erase( keyVals.length()-2, 2 ); log->Dump( topic, format, keyVals.c_str() ); } //---------------------------------------------------------------------------- // Print a char array as hex //---------------------------------------------------------------------------- std::string Utils::Char2Hex( uint8_t *array, uint16_t size ) { char *hex = new char[2*size+1]; for( uint16_t i = 0; i < size; ++i ) snprintf( hex+(2*i), 3, "%02x", (int)array[i] ); std::string result = hex; delete [] hex; return result; } //---------------------------------------------------------------------------- // Normalize checksum //---------------------------------------------------------------------------- std::string Utils::NormalizeChecksum( const std::string &name, const std::string &checksum ) { if( name == "adler32" || name == "crc32" ) { size_t i; for( i = 0; i < checksum.length(); ++i ) if( checksum[i] != '0' ) break; return checksum.substr(i); } return checksum; } //---------------------------------------------------------------------------- // Get supported checksum types for given URL //---------------------------------------------------------------------------- std::vector Utils::GetSupportedCheckSums( const XrdCl::URL &url ) { std::vector ret; FileSystem fs( url ); Buffer arg; arg.FromString( "chksum" ); Buffer *resp = 0; XRootDStatus st = fs.Query( QueryCode::Config, arg, resp ); if( st.IsOK() ) { std::string response = resp->ToString(); if( response != "chksum" ) { // we are expecting a response of format: '0:zcrc32,1:adler32' std::vector result; Utils::splitString( result, response, "," ); std::vector::iterator itr = result.begin(); for( ; itr != result.end(); ++itr ) { size_t pos = itr->find( ':' ); if( pos == std::string::npos ) continue; std::string cksname = itr->substr( pos + 1 ); // remove all white spaces cksname.erase( std::remove_if( cksname.begin(), cksname.end(), ::isspace ), cksname.end() ); ret.push_back( std::move( cksname ) ); } } } return ret; } //------------------------------------------------------------------------ //! Check if this client can support given EC redirect //------------------------------------------------------------------------ bool Utils::CheckEC( const Message *req, const URL &url ) { #ifdef WITH_XRDEC // make sure that if we will be writing it is a new file ClientRequest *request = (ClientRequest*)req->GetBuffer(); uint16_t options = ntohs( request->open.options ); bool open_wrt = ( options & kXR_open_updt ) || ( options & kXR_open_wrto ); bool open_new = ( options & kXR_new ); if( open_wrt && !open_new ) return false; const URL::ParamsMap ¶ms = url.GetParams(); // make sure all the xrdec. tokens are present and the values are sane URL::ParamsMap::const_iterator itr = params.find( "xrdec.nbdta" ); if( itr == params.end() ) return false; size_t nbdta = std::stoul( itr->second ); itr = params.find( "xrdec.nbprt" ); if( itr == params.end() ) return false; size_t nbprt = std::stoul( itr->second ); itr = params.find( "xrdec.blksz" ); if( itr == params.end() ) return false; itr = params.find( "xrdec.plgr" ); if( itr == params.end() ) return false; std::vector plgr; splitString( plgr, itr->second, "," ); if( plgr.size() < nbdta + nbprt ) return false; itr = params.find( "xrdec.objid" ); if( itr == params.end() ) return false; itr = params.find( "xrdec.format" ); if( itr == params.end() ) return false; size_t format = std::stoul( itr->second ); if( format != 1 ) return false; // TODO use constant itr = params.find( "xrdec.dtacgi" ); if( itr != params.end() ) { std::vector dtacgi; splitString( dtacgi, itr->second, "," ); if( plgr.size() != dtacgi.size() ) return false; } itr = params.find( "xrdec.mdtacgi" ); if( itr != params.end() ) { std::vector mdtacgi; splitString( mdtacgi, itr->second, "," ); if( plgr.size() != mdtacgi.size() ) return false; } itr = params.find( "xrdec.cosc" ); if( itr == params.end() ) return false; std::string cosc = itr->second; if( cosc != "true" && cosc != "false" ) return false; return true; #else return false; #endif } //---------------------------------------------------------------------------- //! Automatically infer the right checksum type //---------------------------------------------------------------------------- std::string Utils::InferChecksumType( const XrdCl::URL &source, const XrdCl::URL &destination, bool zip) { //-------------------------------------------------------------------------- // If both files are local we won't be checksumming at all //-------------------------------------------------------------------------- if( source.IsLocalFile() && !source.IsMetalink() && destination.IsLocalFile() ) return std::string(); // checksums supported by local files std::set local_supported; local_supported.insert( "adler32" ); local_supported.insert( "crc32" ); local_supported.insert( "md5" ); local_supported.insert( "zcrc32" ); std::vector srccks; if( source.IsMetalink() ) { int useMtlnCksum = DefaultZipMtlnCksum; Env *env = DefaultEnv::GetEnv(); env->GetInt( "ZipMtlnCksum", useMtlnCksum ); //------------------------------------------------------------------------ // In case of ZIP use other checksums than zcrc32 only if the user // requested it explicitly. //------------------------------------------------------------------------ if( !zip || ( zip && useMtlnCksum ) ) { RedirectorRegistry ®istry = RedirectorRegistry::Instance(); VirtualRedirector *redirector = registry.Get( source ); std::vector cks = redirector->GetSupportedCheckSums(); srccks.insert( srccks.end(), cks.begin(), cks.end() ); } } if( zip ) { //------------------------------------------------------------------------ // In case of ZIP we can always extract the checksum from the archive //------------------------------------------------------------------------ srccks.push_back( "zcrc32" ); } else if( source.GetProtocol() == "root" || source.GetProtocol() == "xroot" ) { //------------------------------------------------------------------------ // If the source is a remote endpoint query the supported checksums //------------------------------------------------------------------------ std::vector cks = GetSupportedCheckSums( source ); srccks.insert( srccks.end(), cks.begin(), cks.end() ); } std::vector dstcks; if( destination.GetProtocol() == "root" || destination.GetProtocol() == "xroot" ) { //------------------------------------------------------------------------ // If the destination is a remote endpoint query the supported checksums //------------------------------------------------------------------------ std::vector cks = GetSupportedCheckSums( destination ); dstcks.insert( dstcks.end(), cks.begin(), cks.end() ); } //-------------------------------------------------------------------------- // Now we have all the information we need, we can infer the right checksum // type!!! // // First check if source is local //-------------------------------------------------------------------------- if( source.IsLocalFile() && !source.IsMetalink() ) { std::vector::iterator itr = dstcks.begin(); for( ; itr != dstcks.end(); ++itr ) if( local_supported.count( *itr ) ) return *itr; return std::string(); } //-------------------------------------------------------------------------- // then check if destination is local //-------------------------------------------------------------------------- if( destination.IsLocalFile() ) { std::vector::iterator itr = srccks.begin(); for( ; itr != srccks.end(); ++itr ) if( local_supported.count( *itr ) ) return *itr; return std::string(); } //-------------------------------------------------------------------------- // if both source and destination are remote look for a checksum that can // satisfy both //-------------------------------------------------------------------------- std::set dst_supported( dstcks.begin(), dstcks.end() ); std::vector::iterator itr = srccks.begin(); for( ; itr != srccks.end(); ++itr ) if( dst_supported.count( *itr ) ) return *itr; return std::string(); } //---------------------------------------------------------------------------- //! Split chunks in a ChunkList into one or more ChunkLists //---------------------------------------------------------------------------- void Utils::SplitChunks( std::vector &listsvec, const ChunkList &chunks, const uint32_t maxcs, const size_t maxc ) { listsvec.clear(); if( !chunks.size() ) return; listsvec.emplace_back(); ChunkList *c = &listsvec.back(); const size_t cs = chunks.size(); size_t idx = 0; size_t nc = 0; ChunkInfo tmpc; c->reserve( cs ); while( idx < cs ) { if( maxc && nc >= maxc ) { listsvec.emplace_back(); c = &listsvec.back(); c->reserve( cs - idx ); nc = 0; } if( tmpc.length == 0 ) tmpc = chunks[idx]; if( maxcs && tmpc.length > maxcs ) { c->emplace_back( tmpc.offset, maxcs, tmpc.buffer ); tmpc.offset += maxcs; tmpc.length -= maxcs; tmpc.buffer = static_cast( tmpc.buffer ) + maxcs; } else { c->emplace_back( tmpc.offset, tmpc.length, tmpc.buffer ); tmpc.length = 0; ++idx; } ++nc; } } } xrootd-5.6.9/src/XrdCl/XrdClUtils.hh000066400000000000000000000431651457266313600172610ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2012 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . //------------------------------------------------------------------------------ #ifndef __XRD_CL_UTILS_HH__ #define __XRD_CL_UTILS_HH__ #include #include #include "XrdCl/XrdClStatus.hh" #include "XrdCl/XrdClLog.hh" #include "XrdCl/XrdClURL.hh" #include "XrdCl/XrdClXRootDResponses.hh" #include "XrdCl/XrdClPropertyList.hh" #include "XrdCl/XrdClDefaultEnv.hh" #include "XrdCl/XrdClConstants.hh" #include "XrdCl/XrdClPostMaster.hh" #include "XrdCl/XrdClXRootDTransport.hh" #include "XrdNet/XrdNetUtils.hh" #include "XrdOuc/XrdOucTUtils.hh" #include #ifdef __linux__ #include #endif namespace XrdCl { class Message; //---------------------------------------------------------------------------- //! Random utilities //---------------------------------------------------------------------------- class Utils { public: //------------------------------------------------------------------------ //! Split a string //------------------------------------------------------------------------ template static void splitString( Container &result, const std::string &input, const std::string &delimiter ) { /* * This was done in order to not duplicate code as this method * is also used in XrdHttp * TODO: Maybe this method could be collapsed * to avoid this middle-man call here */ XrdOucTUtils::splitString(result,input,delimiter); } //------------------------------------------------------------------------ //! Get a parameter either from the environment or URL //------------------------------------------------------------------------ static int GetIntParameter( const URL &url, const std::string &name, int defaultVal ); //------------------------------------------------------------------------ //! Get a parameter either from the environment or URL //------------------------------------------------------------------------ static std::string GetStringParameter( const URL &url, const std::string &name, const std::string &defaultVal ); //------------------------------------------------------------------------ //! Address type //------------------------------------------------------------------------ enum AddressType { IPAuto = 0, IPAll = 1, IPv6 = 2, IPv4 = 3, IPv4Mapped6 = 4 }; //------------------------------------------------------------------------ //! Interpret a string as address type, default to IPAll //------------------------------------------------------------------------ static AddressType String2AddressType( const std::string &addressType ); //------------------------------------------------------------------------ //! Resolve IP addresses //------------------------------------------------------------------------ static Status GetHostAddresses( std::vector &addresses, const URL &url, AddressType type ); //------------------------------------------------------------------------ //! Log all the addresses on the list //------------------------------------------------------------------------ static void LogHostAddresses( Log *log, uint64_t type, const std::string &hostId, std::vector &addresses ); //------------------------------------------------------------------------ //! Convert timestamp to a string //------------------------------------------------------------------------ static std::string TimeToString( time_t timestamp ); //------------------------------------------------------------------------ //! Get the elapsed microseconds between two timevals //------------------------------------------------------------------------ static uint64_t GetElapsedMicroSecs( timeval start, timeval end ); //------------------------------------------------------------------------ //! Get a checksum from a remote xrootd server //------------------------------------------------------------------------ static XRootDStatus GetRemoteCheckSum( std::string &checkSum, const std::string &checkSumType, const URL &url ); //------------------------------------------------------------------------ //! Get a checksum from local file //------------------------------------------------------------------------ static XRootDStatus GetLocalCheckSum( std::string &checkSum, const std::string &checkSumType, const std::string &path ); //------------------------------------------------------------------------ //! Convert bytes to a human readable string //------------------------------------------------------------------------ static std::string BytesToString( uint64_t bytes ); //------------------------------------------------------------------------ //! Check if peer supports tpc //------------------------------------------------------------------------ static XRootDStatus CheckTPC( const std::string &server, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! Check if peer supports tpc / tpc lite //! //! @return : suDone if TPC lite is supported, suPartial if plain TPC is //! supported, stError otherwise //------------------------------------------------------------------------ static XRootDStatus CheckTPCLite( const std::string &server, uint16_t timeout = 0 ); //------------------------------------------------------------------------ //! Convert the fully qualified host name to country code //------------------------------------------------------------------------ static std::string FQDNToCC( const std::string &fqdn ); //------------------------------------------------------------------------ //! Get directory entries //------------------------------------------------------------------------ static Status GetDirectoryEntries( std::vector &entries, const std::string &path ); //------------------------------------------------------------------------ //! Process a config file and return key-value pairs //------------------------------------------------------------------------ static Status ProcessConfig( std::map &config, const std::string &file ); //------------------------------------------------------------------------ //! Process a config directory and return key-value pairs //------------------------------------------------------------------------ static Status ProcessConfigDir( std::map &config, const std::string &dir ); //------------------------------------------------------------------------ //! Trim a string //------------------------------------------------------------------------ static void Trim( std::string &str ); //------------------------------------------------------------------------ //! Log property list //------------------------------------------------------------------------ static void LogPropertyList( Log *log, uint64_t topic, const char *format, const PropertyList &list ); //------------------------------------------------------------------------ //! Print a char array as hex //------------------------------------------------------------------------ static std::string Char2Hex( uint8_t *array, uint16_t size ); //------------------------------------------------------------------------ //! Normalize checksum //------------------------------------------------------------------------ static std::string NormalizeChecksum( const std::string &name, const std::string &checksum ); //------------------------------------------------------------------------ //! Get supported checksum types for given URL //------------------------------------------------------------------------ static std::vector GetSupportedCheckSums( const XrdCl::URL &url ); //------------------------------------------------------------------------ //! Automatically infer the right checksum type //! //! @param source : source URL //! @param destination : destination URL //! @param zip : true if the source file is being extracted from //! a ZIP archive, false otherwise //! @return : checksum type //------------------------------------------------------------------------ static std::string InferChecksumType( const XrdCl::URL &source, const XrdCl::URL &destination, bool zip = false ); //------------------------------------------------------------------------ //! Check if this client can support given EC redirect //------------------------------------------------------------------------ static bool CheckEC( const Message *req, const URL &url ); //------------------------------------------------------------------------ //! Get protocol version of the given server //! @param url : URL pointing to the server //! @param protver : protocol version (output parameter) //! @return : operation status //------------------------------------------------------------------------ inline static XrdCl::XRootDStatus GetProtocolVersion( const XrdCl::URL url, int &protver ) { XrdCl::AnyObject qryResult; XrdCl::XRootDStatus st = XrdCl::DefaultEnv::GetPostMaster()-> QueryTransport( url, XrdCl::XRootDQuery::ProtocolVersion, qryResult ); if( !st.IsOK() ) return st; int *tmp = 0; qryResult.Get( tmp ); protver = *tmp; delete tmp; return XrdCl::XRootDStatus(); } //------------------------------------------------------------------------ //! Check if given server supports extended file attributes //! @param url : URL pointing to the server //! @return : true if yes, false otherwise //------------------------------------------------------------------------ inline static bool HasXAttr( const XrdCl::URL &url ) { if( url.IsLocalFile() ) return true; int protver = 0; auto st = GetProtocolVersion( url, protver ); if( !st.IsOK() ) return false; return protver >= kXR_PROTXATTVERSION; } //------------------------------------------------------------------------ //! Check if given server supports pgread/pgwrite //! @param url : URL pointing to the server //! @return : true if yes, false otherwise //------------------------------------------------------------------------ inline static bool HasPgRW( const XrdCl::URL &url ) { if( url.IsLocalFile() ) return false; int protver = 0; auto st = GetProtocolVersion( url, protver ); if( !st.IsOK() ) return false; return protver >= kXR_PROTPGRWVERSION; } //------------------------------------------------------------------------ //! Split chunks in a ChunkList into one or more ChunkLists //! @param listsvec : output vector of ChunkLists //! @param chunks : input ChunkLisits //! @param maxcs : maximum size of a ChunkInfo in output //! @param maxc : maximum number of ChunkInfo in each ChunkList //------------------------------------------------------------------------ static void SplitChunks( std::vector &listsvec, const ChunkList &chunks, const uint32_t maxcs, const size_t maxc ); }; //---------------------------------------------------------------------------- //! Smart descriptor - closes the descriptor on destruction //---------------------------------------------------------------------------- class ScopedDescriptor { public: //------------------------------------------------------------------------ //! Constructor //------------------------------------------------------------------------ ScopedDescriptor( int descriptor ): pDescriptor( descriptor ) {} //------------------------------------------------------------------------ //! Destructor //------------------------------------------------------------------------ ~ScopedDescriptor() { if( pDescriptor >= 0 ) close( pDescriptor ); } //------------------------------------------------------------------------ //! Release the descriptor being held //------------------------------------------------------------------------ int Release() { int desc = pDescriptor; pDescriptor = -1; return desc; } //------------------------------------------------------------------------ //! Get the descriptor //------------------------------------------------------------------------ int GetDescriptor() { return pDescriptor; } private: int pDescriptor; }; #ifdef __linux__ //---------------------------------------------------------------------------- //! Scoped fsuid and fsgid setter, restoring original values on destruction //---------------------------------------------------------------------------- class ScopedFsUidSetter { public: //------------------------------------------------------------------------ //! Constructor //------------------------------------------------------------------------ ScopedFsUidSetter(uid_t fsuid, gid_t fsgid, const std::string &streamName) : pFsUid(fsuid), pFsGid(fsgid), pStreamName(streamName) { pOk = true; pPrevFsUid = -1; pPrevFsGid = -1; //---------------------------------------------------------------------- //! Set fsuid //---------------------------------------------------------------------- if(pFsUid >= 0) { pPrevFsUid = setfsuid(pFsUid); if(setfsuid(pFsUid) != pFsUid) { pOk = false; return; } } //---------------------------------------------------------------------- //! Set fsgid //---------------------------------------------------------------------- if(pFsGid >= 0) { pPrevFsGid = setfsgid(pFsGid); if(setfsgid(pFsGid) != pFsGid) { pOk = false; return; } } } //------------------------------------------------------------------------ //! Destructor //------------------------------------------------------------------------ ~ScopedFsUidSetter() { Log *log = DefaultEnv::GetLog(); if(pPrevFsUid >= 0) { int retcode = setfsuid(pPrevFsUid); log->Dump(XRootDTransportMsg, "[%s] Restored fsuid from %d to %d", pStreamName.c_str(), retcode, pPrevFsUid); } if(pPrevFsGid >= 0) { int retcode = setfsgid(pPrevFsGid); log->Dump(XRootDTransportMsg, "[%s] Restored fsgid from %d to %d", pStreamName.c_str(), retcode, pPrevFsGid); } } bool IsOk() const { return pOk; } private: int pFsUid; int pFsGid; const std::string &pStreamName; int pPrevFsUid; int pPrevFsGid; bool pOk; }; #endif } #endif // __XRD_CL_UTILS_HH__ xrootd-5.6.9/src/XrdCl/XrdClXCpCtx.cc000066400000000000000000000122611457266313600173110ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2017 by European Organization for Nuclear Research (CERN) // Author: Michal Simon //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #include "XrdCl/XrdClXCpCtx.hh" #include "XrdCl/XrdClXCpSrc.hh" #include "XrdCl/XrdClLog.hh" #include "XrdCl/XrdClDefaultEnv.hh" #include "XrdCl/XrdClConstants.hh" #include namespace XrdCl { XCpCtx::XCpCtx( const std::vector &urls, uint64_t blockSize, uint8_t parallelSrc, uint64_t chunkSize, uint64_t parallelChunks, int64_t fileSize ) : pUrls( std::deque( urls.begin(), urls.end() ) ), pBlockSize( blockSize ), pParallelSrc( parallelSrc ), pChunkSize( chunkSize ), pParallelChunks( parallelChunks ), pOffset( 0 ), pFileSize( -1 ), pFileSizeCV( 0 ), pDataReceived( 0 ), pDone( false ), pDoneCV( 0 ), pRefCount( 1 ) { SetFileSize( fileSize ); } XCpCtx::~XCpCtx() { // at this point there's no concurrency // this object dies as the last one while( !pSink.IsEmpty() ) { PageInfo *chunk = pSink.Get(); if( chunk ) XCpSrc::DeleteChunk( chunk ); } } bool XCpCtx::GetNextUrl( std::string & url ) { XrdSysMutexHelper lck( pMtx ); if( pUrls.empty() ) return false; url = pUrls.front(); pUrls.pop(); return true; } XCpSrc* XCpCtx::WeakestLink( XCpSrc *exclude ) { uint64_t transferRate = -1; // set transferRate to max uint64 value XCpSrc *ret = 0; std::list::iterator itr; for( itr = pSources.begin() ; itr != pSources.end() ; ++itr ) { XCpSrc *src = *itr; if( src == exclude ) continue; uint64_t tmp = src->TransferRate(); if( src->HasData() && tmp < transferRate ) { ret = src; transferRate = tmp; } } return ret; } void XCpCtx::PutChunk( PageInfo* chunk ) { pSink.Put( chunk ); } std::pair XCpCtx::GetBlock() { XrdSysMutexHelper lck( pMtx ); uint64_t blkSize = pBlockSize, offset = pOffset; if( pOffset + blkSize > uint64_t( pFileSize ) ) blkSize = pFileSize - pOffset; pOffset += blkSize; return std::make_pair( offset, blkSize ); } void XCpCtx::SetFileSize( int64_t size ) { XrdSysMutexHelper lck( pMtx ); if( pFileSize < 0 && size >= 0 ) { XrdSysCondVarHelper lck( pFileSizeCV ); pFileSize = size; pFileSizeCV.Broadcast(); if( pBlockSize > uint64_t( pFileSize ) / pParallelSrc ) pBlockSize = pFileSize / pParallelSrc; if( pBlockSize < pChunkSize ) pBlockSize = pChunkSize; } } XRootDStatus XCpCtx::Initialize() { for( uint8_t i = 0; i < pParallelSrc; ++i ) { XCpSrc *src = new XCpSrc( pChunkSize, pParallelChunks, pFileSize, this ); pSources.push_back( src ); src->Start(); } if( pSources.empty() ) { Log *log = DefaultEnv::GetLog(); log->Error( UtilityMsg, "Failed to initialize (failed to create new threads)" ); return XRootDStatus( stError, errInternal, EAGAIN, "XCpCtx: failed to create new threads." ); } return XRootDStatus(); } XRootDStatus XCpCtx::GetChunk( XrdCl::PageInfo &ci ) { // if we received all the data we are done here if( pDataReceived == uint64_t( pFileSize ) ) { XrdSysCondVarHelper lck( pDoneCV ); pDone = true; pDoneCV.Broadcast(); return XRootDStatus( stOK, suDone ); } // if we don't have active sources it means we failed if( GetRunning() == 0 ) { XrdSysCondVarHelper lck( pDoneCV ); pDone = true; pDoneCV.Broadcast(); return XRootDStatus( stError, errNoMoreReplicas ); } PageInfo *chunk = pSink.Get(); if( chunk ) { pDataReceived += chunk->GetLength(); ci = std::move( *chunk ); delete chunk; return XRootDStatus( stOK, suContinue ); } return XRootDStatus( stOK, suRetry ); } void XCpCtx::NotifyIdleSrc() { pDoneCV.Broadcast(); } bool XCpCtx::AllDone() { XrdSysCondVarHelper lck( pDoneCV ); if( !pDone ) pDoneCV.Wait( 60 ); return pDone; } size_t XCpCtx::GetRunning() { // count active sources size_t nbRunning = 0; std::list::iterator itr; for( itr = pSources.begin() ; itr != pSources.end() ; ++ itr) if( (*itr)->IsRunning() ) ++nbRunning; return nbRunning; } } /* namespace XrdCl */ xrootd-5.6.9/src/XrdCl/XrdClXCpCtx.hh000066400000000000000000000201411457266313600173170ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2017 by European Organization for Nuclear Research (CERN) // Author: Michal Simon //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #ifndef SRC_XRDCL_XRDCLXCPCTX_HH_ #define SRC_XRDCL_XRDCLXCPCTX_HH_ #include "XrdCl/XrdClSyncQueue.hh" #include "XrdCl/XrdClXRootDResponses.hh" #include "XrdSys/XrdSysPthread.hh" #include #include namespace XrdCl { class XCpSrc; class XCpCtx { public: /** * Constructor * * @param urls : list of replica urls * @param blockSize : the default block size * @param parallelSrc : maximum number of parallel sources * @param chunkSize : the default chunk size * @param parallelChunks : the default number of parallel chunks per source * @param fileSize : the file size if specified in the metalink file * (-1 indicates that the file size is not known and * a stat should be done) */ XCpCtx( const std::vector &urls, uint64_t blockSize, uint8_t parallelSrc, uint64_t chunkSize, uint64_t parallelChunks, int64_t fileSize ); /** * Deletes the instance if the reference counter reached 0. */ void Delete() { XrdSysMutexHelper lck( pMtx ); --pRefCount; if( !pRefCount ) { lck.UnLock(); delete this; } } /** * Increments the reference counter. * * @return : myself. */ XCpCtx* Self() { XrdSysMutexHelper lck( pMtx ); ++pRefCount; return this; } /** * Gets the next URL from the list of file replicas * * @param url : the output parameter * @return : true if a url has been written to the * url parameter, false otherwise */ bool GetNextUrl( std::string & url ); /** * Get the 'weakest' sources * * @param exclude : the source that is excluded from the * search * @return : the weakest source */ XCpSrc* WeakestLink( XCpSrc *exclude ); /** * Put a chunk into the sink * * @param chunk : the chunk */ void PutChunk( PageInfo* chunk ); /** * Get next block that has to be transferred * * @return : pair of offset and block size */ std::pair GetBlock(); /** * Set the file size (GetSize will block until * SetFileSize will be called). * Also calculates the block size. * * @param size : file size */ void SetFileSize( int64_t size ); /** * Get file size. The call blocks until the file * size is being set using SetFileSize. */ int64_t GetSize() { XrdSysCondVarHelper lck( pFileSizeCV ); while( pFileSize < 0 && GetRunning() > 0 ) pFileSizeCV.Wait(); return pFileSize; } /** * Starts one thread per source, each thread * tries to open a file, stat the file if necessary, * and then starts reading the file, all chunks read * go to the sink. * * @return Error if we were not able to create any threads */ XRootDStatus Initialize(); /** * Gets the next chunk from the sink, if the sink is empty blocks. * * @param ci : the chunk retrieved from sink (output parameter) * @return : stError if we failed to transfer the file, * stOK otherwise, with one of the following codes: * - suDone : the whole file has been transferred, * we are done * - suContinue : a chunk has been written into ci, * continue calling GetChunk in order * to retrieve remaining chunks * - suRetry : a chunk has not been written into ci, * try again. */ XRootDStatus GetChunk( XrdCl::PageInfo &ci ); /** * Remove given source * * @param src : the source to be removed */ void RemoveSrc( XCpSrc *src ) { XrdSysMutexHelper lck( pMtx ); pSources.remove( src ); } /** * Notify idle sources, used in two case: * - if one of the sources failed and an * idle source needs to take over * - or if we are done and all idle source * should be stopped */ void NotifyIdleSrc(); /** * Returns true if all chunks have been transferred, * otherwise blocks until NotifyIdleSrc is called, * or a 1 minute timeout occurs. * * @return : true is all chunks have been transferred, * false otherwise. */ bool AllDone(); /** * Notify those who are waiting for initialization. * In particular the GetSize() caller will be waiting * on the result of initialization. */ void NotifyInitExpectant() { pFileSizeCV.Broadcast(); } private: /** * Returns the number of active sources * * @return : number of active sources */ size_t GetRunning(); /** * Destructor (private). * * Use Delelte to destroy the object. */ virtual ~XCpCtx(); /** * The URLs of all the replicas that were provided * to us. */ std::queue pUrls; /** * The size of the block allocated to a single source. */ uint64_t pBlockSize; /** * Number of parallel sources. */ uint8_t pParallelSrc; /** * Chunk size. */ uint32_t pChunkSize; /** * Number of parallel chunks per source. */ uint8_t pParallelChunks; /** * Offset in the file (everything before the offset * has been allocated, everything after the offset * needs to be allocated) */ uint64_t pOffset; /** * File size. */ int64_t pFileSize; /** * File Size conditional variable. * (notifies waiters if the file size has been set) */ XrdSysCondVar pFileSizeCV; /** * List of sources. Those pointers are not owned by * this object. */ std::list pSources; /** * A queue shared between all the sources (producers), * and the extreme copy context (consumer). */ SyncQueue pSink; /** * Total amount of data received */ uint64_t pDataReceived; /** * A flag, true if all chunks have been received and we are done, * false otherwise */ bool pDone; /** * A condition variable, idle sources wait on this cond var until * we are done, or until one of the active sources fails. */ XrdSysCondVar pDoneCV; /** * A mutex guarding the object */ XrdSysMutex pMtx; /** * Reference counter */ size_t pRefCount; }; } /* namespace XrdCl */ #endif /* SRC_XRDCL_XRDCLXCPCTX_HH_ */ xrootd-5.6.9/src/XrdCl/XrdClXCpSrc.cc000066400000000000000000000416571457266313600173150ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2017 by European Organization for Nuclear Research (CERN) // Author: Michal Simon //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #include "XrdCl/XrdClXCpSrc.hh" #include "XrdCl/XrdClXCpCtx.hh" #include "XrdCl/XrdClLog.hh" #include "XrdCl/XrdClDefaultEnv.hh" #include "XrdCl/XrdClConstants.hh" #include "XrdCl/XrdClUtils.hh" #include #include namespace XrdCl { class ChunkHandler: public ResponseHandler { public: ChunkHandler( XCpSrc *src, uint64_t offset, uint64_t size, char *buffer, File *handle, bool usepgrd ) : pSrc( src->Self() ), pOffset( offset ), pSize( size ), pBuffer( buffer ), pHandle( handle ), pUsePgRead( usepgrd ) { } virtual ~ChunkHandler() { pSrc->Delete(); } virtual void HandleResponse( XRootDStatus *status, AnyObject *response ) { PageInfo *chunk = 0; if( response ) // get the response { ToPgInfo( response, chunk ); delete response; } if( !chunk && status->IsOK() ) // if the response is not there make sure the status is error { *status = XRootDStatus( stError, errInternal ); } if( status->IsOK() && chunk->GetLength() != pSize ) // the file size on the server is different { // than the one specified in metalink file *status = XRootDStatus( stError, errDataError ); } if( !status->IsOK() ) { delete[] pBuffer; delete chunk; chunk = 0; } pSrc->ReportResponse( status, chunk, pHandle ); delete this; } private: void ToPgInfo( AnyObject *response, PageInfo *&chunk ) { if( pUsePgRead ) { response->Get( chunk ); response->Set( ( int* )0 ); } else { ChunkInfo *rsp = nullptr; response->Get( rsp ); chunk = new PageInfo( rsp->offset, rsp->length, rsp->buffer ); } } XCpSrc *pSrc; uint64_t pOffset; uint64_t pSize; char *pBuffer; File *pHandle; bool pUsePgRead; }; XCpSrc::XCpSrc( uint32_t chunkSize, uint8_t parallel, int64_t fileSize, XCpCtx *ctx ) : pChunkSize( chunkSize ), pParallel( parallel ), pFileSize( fileSize ), pThread(), pCtx( ctx->Self() ), pFile( 0 ), pCurrentOffset( 0 ), pBlkEnd( 0 ), pDataTransfered( 0 ), pRefCount( 1 ), pRunning( false ), pStartTime( 0 ), pTransferTime( 0 ), pUsePgRead( false ) { } XCpSrc::~XCpSrc() { pCtx->RemoveSrc( this ); pCtx->Delete(); } void XCpSrc::Start() { pRunning = true; int rc = pthread_create( &pThread, 0, Run, this ); if( rc ) { pRunning = false; pCtx->RemoveSrc( this ); pCtx->Delete(); } } void* XCpSrc::Run( void* arg ) { XCpSrc *me = static_cast( arg ); me->StartDownloading(); me->Delete(); return 0; } void XCpSrc::StartDownloading() { XRootDStatus st = Initialize(); if( !st.IsOK() ) { pRunning = false; // notify those who wait for the file // size, they won't get it from this // source pCtx->NotifyInitExpectant(); // put a null chunk so we are sure // the main thread doesn't get stuck // at the sync queue pCtx->PutChunk( 0 ); return; } // start counting transfer time pStartTime = time( 0 ); while( pRunning ) { st = ReadChunks(); if( st.IsOK() && st.code == suPartial ) { // we have only ongoing transfers // so we can already ask for new block if( GetWork().IsOK() ) continue; } else if( st.IsOK() && st.code == suDone ) { // if we are done, try to get more work, // if successful continue if( GetWork().IsOK() ) continue; // keep track of the time before we go idle pTransferTime += time( 0 ) - pStartTime; // check if the overall download process is // done, this makes the thread wait until // either the download is done, or a source // went to error, or a 60s timeout has been // reached (the timeout is there so we can // check if a source degraded in the meanwhile // and now we can steal from it) if( !pCtx->AllDone() ) { // reset start time after pause pStartTime = time( 0 ); continue; } // stop counting // otherwise we are done here pRunning = false; return; } XRootDStatus *status = pReports.Get(); if( !status->IsOK() ) { Log *log = DefaultEnv::GetLog(); std::string myHost = URL( pUrl ).GetHostName(); log->Error( UtilityMsg, "Failed to read chunk from %s: %s", myHost.c_str(), status->GetErrorMessage().c_str() ); if( !Recover().IsOK() ) { delete status; pRunning = false; // notify idle sources, they might be // interested in taking over my workload pCtx->NotifyIdleSrc(); // put a null chunk so we are sure // the main thread doesn't get stuck // at the sync queue pCtx->PutChunk( 0 ); // if we have data we need to wait for someone to take over // unless the extreme copy is over, in this case we don't care while( HasData() && !pCtx->AllDone() ); return; } } delete status; } } XRootDStatus XCpSrc::Initialize() { Log *log = DefaultEnv::GetLog(); XRootDStatus st; do { if( !pCtx->GetNextUrl( pUrl ) ) { log->Error( UtilityMsg, "Failed to initialize XCp source, no more replicas to try" ); return XRootDStatus( stError ); } log->Debug( UtilityMsg, "Opening %s for reading", pUrl.c_str() ); std::string value; DefaultEnv::GetEnv()->GetString( "ReadRecovery", value ); pFile = new File(); pFile->SetProperty( "ReadRecovery", value ); st = pFile->Open( pUrl, OpenFlags::Read ); if( !st.IsOK() ) { log->Warning( UtilityMsg, "Failed to open %s for reading: %s", pUrl.c_str(), st.GetErrorMessage().c_str() ); DeletePtr( pFile ); continue; } URL url( pUrl ); if( ( !url.IsLocalFile() && !pFile->IsSecure() ) || ( url.IsLocalFile() && url.IsMetalink() ) ) { std::string datasrv; pFile->GetProperty( "DataServer", datasrv ); //-------------------------------------------------------------------- // Decide whether we can use PgRead //-------------------------------------------------------------------- int val = XrdCl::DefaultCpUsePgWrtRd; XrdCl::DefaultEnv::GetEnv()->GetInt( "CpUsePgWrtRd", val ); pUsePgRead = XrdCl::Utils::HasPgRW( datasrv ) && ( val == 1 ); } if( pFileSize < 0 ) { StatInfo *statInfo = 0; st = pFile->Stat( false, statInfo ); if( !st.IsOK() ) { log->Warning( UtilityMsg, "Failed to stat %s: %s", pUrl.c_str(), st.GetErrorMessage().c_str() ); DeletePtr( pFile ); continue; } pFileSize = statInfo->GetSize(); pCtx->SetFileSize( pFileSize ); delete statInfo; } } while( !st.IsOK() ); std::pair p = pCtx->GetBlock(); pCurrentOffset = p.first; pBlkEnd = p.second + p.first; return st; } XRootDStatus XCpSrc::Recover() { Log *log = DefaultEnv::GetLog(); XRootDStatus st; do { if( !pCtx->GetNextUrl( pUrl ) ) { log->Error( UtilityMsg, "Failed to initialize XCp source, no more replicas to try" ); return XRootDStatus( stError ); } log->Debug( UtilityMsg, "Opening %s for reading", pUrl.c_str() ); std::string value; DefaultEnv::GetEnv()->GetString( "ReadRecovery", value ); pFile = new File(); pFile->SetProperty( "ReadRecovery", value ); st = pFile->Open( pUrl, OpenFlags::Read ); if( !st.IsOK() ) { DeletePtr( pFile ); log->Warning( UtilityMsg, "Failed to open %s for reading: %s", pUrl.c_str(), st.GetErrorMessage().c_str() ); } URL url( pUrl ); if( ( !url.IsLocalFile() && pFile->IsSecure() ) || ( url.IsLocalFile() && url.IsMetalink() ) ) { std::string datasrv; pFile->GetProperty( "DataServer", datasrv ); //-------------------------------------------------------------------- // Decide whether we can use PgRead //-------------------------------------------------------------------- int val = XrdCl::DefaultCpUsePgWrtRd; XrdCl::DefaultEnv::GetEnv()->GetInt( "CpUsePgWrtRd", val ); pUsePgRead = XrdCl::Utils::HasPgRW( datasrv ) && ( val == 1 ); } } while( !st.IsOK() ); pRecovered.insert( pOngoing.begin(), pOngoing.end() ); pOngoing.clear(); // since we have a brand new source, we need // to restart transfer rate statistics pTransferTime = 0; pStartTime = time( 0 ); pDataTransfered = 0; return st; } XRootDStatus XCpSrc::ReadChunks() { XrdSysMutexHelper lck( pMtx ); while( pOngoing.size() < pParallel && !pRecovered.empty() ) { std::pair p; std::map::iterator itr = pRecovered.begin(); p = *itr; pOngoing.insert( p ); pRecovered.erase( itr ); char *buffer = new char[p.second]; ChunkHandler *handler = new ChunkHandler( this, p.first, p.second, buffer, pFile, pUsePgRead ); XRootDStatus st = pUsePgRead ? pFile->PgRead( p.first, p.second, buffer, handler ) : pFile->Read( p.first, p.second, buffer, handler ); if( !st.IsOK() ) { delete[] buffer; delete handler; ReportResponse( new XRootDStatus( st ), 0, pFile ); return st; } } while( pOngoing.size() < pParallel && pCurrentOffset < pBlkEnd ) { uint64_t chunkSize = pChunkSize; if( pCurrentOffset + chunkSize > pBlkEnd ) chunkSize = pBlkEnd - pCurrentOffset; pOngoing[pCurrentOffset] = chunkSize; char *buffer = new char[chunkSize]; ChunkHandler *handler = new ChunkHandler( this, pCurrentOffset, chunkSize, buffer, pFile, pUsePgRead ); XRootDStatus st = pUsePgRead ? pFile->PgRead( pCurrentOffset, chunkSize, buffer, handler ) : pFile->Read( pCurrentOffset, chunkSize, buffer, handler ); pCurrentOffset += chunkSize; if( !st.IsOK() ) { delete[] buffer; delete handler; ReportResponse( new XRootDStatus( st ), 0, pFile ); return st; } } if( pOngoing.empty() ) return XRootDStatus( stOK, suDone ); if( pRecovered.empty() && pCurrentOffset >= pBlkEnd ) return XRootDStatus( stOK, suPartial ); return XRootDStatus( stOK, suContinue ); } void XCpSrc::ReportResponse( XRootDStatus *status, PageInfo *chunk, File *handle ) { XrdSysMutexHelper lck( pMtx ); bool ignore = false; if( status->IsOK() ) { // if the status is OK remove it from // the list of ongoing transfers, if it // was not on the list we ignore the // response (this could happen due to // source change or stealing) ignore = !pOngoing.erase( chunk->GetOffset() ); } else if( FilesEqual( pFile, handle ) ) { // if the status is NOT OK, and pFile // match the handle it means that we see // an error for the first time, map the // broken file to the number of outstanding // asynchronous operations and reset the pointer pFailed[pFile] = pOngoing.size(); pFile = 0; } else DeletePtr( status ); if( !FilesEqual( pFile, handle ) ) { // if the pFile does not match the handle, // it means that this response came from // a broken source, decrement the count of // outstanding async operations for this src, --pFailed[handle]; if( pFailed[handle] == 0 ) { // if this was the last outstanding operation // close the file and delete it pFailed.erase( handle ); XRootDStatus st = handle->Close(); delete handle; } } lck.UnLock(); if( status ) pReports.Put( status ); if( ignore ) { DeleteChunk( chunk ); return; } if( chunk ) { pDataTransfered += chunk->GetLength(); pCtx->PutChunk( chunk ); } } void XCpSrc::Steal( XCpSrc *src ) { if( !src ) return; XrdSysMutexHelper lck1( pMtx ), lck2( src->pMtx ); Log *log = DefaultEnv::GetLog(); std::string myHost = URL( pUrl ).GetHostName(), srcHost = URL( src->pUrl ).GetHostName(); if( !src->pRunning ) { // the source we are stealing from is in error state, we can have everything pRecovered.insert( src->pOngoing.begin(), src->pOngoing.end() ); pRecovered.insert( src->pRecovered.begin(), src->pRecovered.end() ); pCurrentOffset = src->pCurrentOffset; pBlkEnd = src->pBlkEnd; src->pOngoing.clear(); src->pRecovered.clear(); src->pCurrentOffset = 0; src->pBlkEnd = 0; // a broken source might be waiting for // someone to take over his data, so we // need to notify pCtx->NotifyIdleSrc(); log->Debug( UtilityMsg, "s%: Stealing everything from %s", myHost.c_str(), srcHost.c_str() ); return; } // the source we are stealing from is just slower, only take part of its work // so we want a fraction of its work we want for ourself uint64_t myTransferRate = TransferRate(), srcTransferRate = src->TransferRate(); if( myTransferRate == 0 ) return; double fraction = double( myTransferRate ) / double( myTransferRate + srcTransferRate ); if( src->pCurrentOffset < src->pBlkEnd ) { // the source still has a block of data uint64_t blkSize = src->pBlkEnd - src->pCurrentOffset; uint64_t steal = static_cast( round( fraction * blkSize ) ); // if after stealing there will be less than one chunk // take everything if( blkSize - steal <= pChunkSize ) steal = blkSize; pCurrentOffset = src->pBlkEnd - steal; pBlkEnd = src->pBlkEnd; src->pBlkEnd -= steal; log->Debug( UtilityMsg, "s%: Stealing fraction (%f) of block from %s", myHost.c_str(), fraction, srcHost.c_str() ); return; } if( !src->pRecovered.empty() ) { size_t count = static_cast( round( fraction * src->pRecovered.size() ) ); while( count-- ) { std::map::iterator itr = src->pRecovered.begin(); pRecovered.insert( *itr ); src->pRecovered.erase( itr ); } log->Debug( UtilityMsg, "s%: Stealing fraction (%f) of recovered chunks from %s", myHost.c_str(), fraction, srcHost.c_str() ); return; } // * a fraction < 0.5 means that we are actually slower (so it does // not make sense to steal ongoing's from someone who's faster) // * a fraction ~ 0.5 means that we have more or less the same transfer // rate (similarly, it doesn't make sense to steal) // * the source needs to be really faster (though, this is an arbitrary // choice) to actually steal something if( !src->pOngoing.empty() && fraction > 0.7 ) { size_t count = static_cast( round( fraction * src->pOngoing.size() ) ); while( count-- ) { std::map::iterator itr = src->pOngoing.begin(); pRecovered.insert( *itr ); src->pOngoing.erase( itr ); } log->Debug( UtilityMsg, "s%: Stealing fraction (%f) of ongoing chunks from %s", myHost.c_str(), fraction, srcHost.c_str() ); } } XRootDStatus XCpSrc::GetWork() { std::pair p = pCtx->GetBlock(); if( p.second > 0 ) { XrdSysMutexHelper lck( pMtx ); pCurrentOffset = p.first; pBlkEnd = p.first + p.second; Log *log = DefaultEnv::GetLog(); std::string myHost = URL( pUrl ).GetHostName(); log->Debug( UtilityMsg, "s% got next block", myHost.c_str() ); return XRootDStatus(); } XCpSrc *wLink = pCtx->WeakestLink( this ); Steal( wLink ); // if we managed to steal something declare success if( pCurrentOffset < pBlkEnd || !pRecovered.empty() ) return XRootDStatus(); // otherwise return an error return XRootDStatus( stError, errInvalidOp ); } uint64_t XCpSrc::TransferRate() { time_t duration = pTransferTime + time( 0 ) - pStartTime; return pDataTransfered / ( duration + 1 ); // add one to avoid floating point exception } } /* namespace XrdCl */ xrootd-5.6.9/src/XrdCl/XrdClXCpSrc.hh000066400000000000000000000223631457266313600173200ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2017 by European Organization for Nuclear Research (CERN) // Author: Michal Simon //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #ifndef SRC_XRDCL_XRDCLXCPSRC_HH_ #define SRC_XRDCL_XRDCLXCPSRC_HH_ #include "XrdCl/XrdClFile.hh" #include "XrdCl/XrdClSyncQueue.hh" #include "XrdSys/XrdSysPthread.hh" namespace XrdCl { class XCpCtx; class XCpSrc { friend class ChunkHandler; public: /** * Constructor. * * @param chunkSize : default chunk size * @param parallel : number of parallel chunks * @param fileSize : file size if available (e.g. in metalink file), * should be set to -1 if not available, in this case * a stat will be performed during initialization * @param ctx : Extreme Copy context */ XCpSrc( uint32_t chunkSize, uint8_t parallel, int64_t fileSize, XCpCtx *ctx ); /** * Creates new thread with XCpSrc::Run as the start routine. */ void Start(); /** * Stops the thread. */ void Stop() { pRunning = false; } /** * Deletes the instance if the reference counter reached 0. */ void Delete() { XrdSysMutexHelper lck( pMtx ); --pRefCount; if( !pRefCount ) { lck.UnLock(); delete this; } } /** * Increments the reference counter. * * @return : myself. */ XCpSrc* Self() { XrdSysMutexHelper lck( pMtx ); ++pRefCount; return this; } /** * @return : true if the thread is running, false otherwise */ bool IsRunning() { return pRunning; } /** * @return true if the source has a block of non zero * size / some chunks allocated, false otherwise */ bool HasData() { XrdSysMutexHelper lck( pMtx ); return pCurrentOffset < pBlkEnd || !pRecovered.empty() || !pOngoing.empty(); } /** * Get the transfer rate for current source * * @return : transfer rate for current source [B/s] */ uint64_t TransferRate(); /** * Delete ChunkInfo object, and set the pointer to null. * * @param chunk : the chunk to be deleted */ static void DeleteChunk( PageInfo *&chunk ) { if( chunk ) { delete[] static_cast( chunk->GetBuffer() ); delete chunk; chunk = 0; } } private: /** * Destructor (private). * * Use Delelte() method to destroy the object. */ virtual ~XCpSrc(); /** * The start routine. */ static void* Run( void* arg ); /** * Initializes the object first. * Afterwards, starts the download. */ void StartDownloading(); /** * Initializes the object: * - Opens a file (retries with another * URL, in case of failure) * - Stats the file if necessary * - Gets the first block (offset and size) * for download * * @return : error in case the object could not be initialized */ XRootDStatus Initialize(); /** * Tries to open the file at the next available URL. * Moves all ongoing chunk to recovered. * * @return : error if run out of URLs to try, * success otherwise */ XRootDStatus Recover(); /** * Asynchronously reads consecutive chunks. * * @return : operation status: * - suContinue : I still have work to do * - suPartial : I only have ongoing transfers, * but the block has been consumed * - suDone : We are done, the block has been * consumed, there are no ongoing * transfers, and there are no new * data */ XRootDStatus ReadChunks(); /** * Steal work from given source. * * - if it is a failed source we can have everything * - otherwise, if the source has a block of size * greater than 0, steal respective fraction of * the block * - otherwise, if the source has recovered chunks, * steal respective fraction of those chunks * - otherwise, steal respective fraction of ongoing * chunks, if we are a faster source * * @param src : the source from whom we are stealing */ void Steal( XCpSrc *src ); /** * Get more work. * First try to get a new block. * If there are no blocks remaining, * try stealing from others. * * @return : error if didn't got any data to transfer */ XRootDStatus GetWork(); /** * This method is used by ChunkHandler to report the result of a write, * to the source object. * * @param status : operation status * @param chunk : the read chunk (if operation failed, should be null) * @param handle : the file object used to read the chunk */ void ReportResponse( XRootDStatus *status, PageInfo *chunk, File *handle ); /** * Delets a pointer and sets it to null. */ template static void DeletePtr( T *&obj ) { delete obj; obj = 0; } /** * Check if two file object point to the same URL. * * @return : true if both files point to the same URL, * false otherwise */ static bool FilesEqual( File *f1, File *f2 ) { if( !f1 || !f2 ) return false; const std::string lastURL = "LastURL"; std::string url1, url2; f1->GetProperty( lastURL, url1 ); f2->GetProperty( lastURL, url2 ); // remove cgi information size_t pos = url1.find( '?' ); if( pos != std::string::npos ) url1 = url1.substr( 0 , pos ); pos = url2.find( '?' ); if( pos != std::string::npos ) url2 = url2.substr( 0 , pos ); return url1 == url2; } /** * Default chunk size */ uint32_t pChunkSize; /** * Number of parallel chunks */ uint8_t pParallel; /** * The file size */ int64_t pFileSize; /** * Thread id */ pthread_t pThread; /** * Extreme Copy context */ XCpCtx *pCtx; /** * Source URL. */ std::string pUrl; /** * Handle to the file. */ File *pFile; std::map pFailed; /** * The offset of the next chunk to be transferred. */ uint64_t pCurrentOffset; /** * End of the our block. */ uint64_t pBlkEnd; /** * Total number of data transferred from this source. */ uint64_t pDataTransfered; /** * A map of ongoing transfers (the offset is the key, * the chunk size is the value). */ std::map pOngoing; /** * A map of stolen chunks (again the offset is the key, * the chunk size is the value). */ std::map pRecovered; /** * Sync queue with reports (statuses) from async reads * that have been issued. An error appears only once * per URL (independently of how many concurrent async * reads are allowed). */ SyncQueue pReports; /** * A mutex guarding the object. */ XrdSysRecMutex pMtx; /** * Reference counter */ size_t pRefCount; /** * A flag, true means the source is running, * false means the source has been stopped, * or failed. */ bool pRunning; /** * The time when we started / restarted chunks */ time_t pStartTime; /** * The total time we were transferring data, before * the restart */ time_t pTransferTime; /** * The total time we were transferring data, before * the restart */ bool pUsePgRead; }; } /* namespace XrdCl */ #endif /* SRC_XRDCL_XRDCLXCPSRC_HH_ */ xrootd-5.6.9/src/XrdCl/XrdClXRootDMsgHandler.cc000066400000000000000000002715661457266313600213030ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2014 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #include "XrdCl/XrdClXRootDMsgHandler.hh" #include "XrdCl/XrdClLog.hh" #include "XrdCl/XrdClDefaultEnv.hh" #include "XrdCl/XrdClConstants.hh" #include "XrdCl/XrdClXRootDTransport.hh" #include "XrdCl/XrdClMessage.hh" #include "XrdCl/XrdClURL.hh" #include "XrdCl/XrdClUtils.hh" #include "XrdCl/XrdClTaskManager.hh" #include "XrdCl/XrdClJobManager.hh" #include "XrdCl/XrdClSIDManager.hh" #include "XrdCl/XrdClMessageUtils.hh" #include "XrdCl/XrdClLocalFileHandler.hh" #include "XrdCl/XrdClRedirectorRegistry.hh" #include "XrdCl/XrdClSocket.hh" #include "XrdCl/XrdClTls.hh" #include "XrdOuc/XrdOucCRC.hh" #include "XrdSys/XrdSysPlatform.hh" // same as above #include "XrdSys/XrdSysAtomics.hh" #include "XrdSys/XrdSysPthread.hh" #include #include #include namespace { //---------------------------------------------------------------------------- // We need an extra task what will run the handler in the future, because // tasks get deleted and we need the handler //---------------------------------------------------------------------------- class WaitTask: public XrdCl::Task { public: WaitTask( XrdCl::XRootDMsgHandler *handler ): pHandler( handler ) { std::ostringstream o; o << "WaitTask for: 0x" << handler->GetRequest(); SetName( o.str() ); } virtual time_t Run( time_t now ) { pHandler->WaitDone( now ); return 0; } private: XrdCl::XRootDMsgHandler *pHandler; }; } namespace XrdCl { //---------------------------------------------------------------------------- // Delegate the response handling to the thread-pool //---------------------------------------------------------------------------- class HandleRspJob: public XrdCl::Job { public: HandleRspJob( XrdCl::XRootDMsgHandler *handler ): pHandler( handler ) { } virtual ~HandleRspJob() { } virtual void Run( void *arg ) { pHandler->HandleResponse(); delete this; } private: XrdCl::XRootDMsgHandler *pHandler; }; //---------------------------------------------------------------------------- // Examine an incoming message, and decide on the action to be taken //---------------------------------------------------------------------------- uint16_t XRootDMsgHandler::Examine( std::shared_ptr &msg ) { //-------------------------------------------------------------------------- // if the MsgHandler is already being used to process another request // (kXR_oksofar) we need to wait //-------------------------------------------------------------------------- if( pOksofarAsAnswer ) { XrdSysCondVarHelper lck( pCV ); while( pResponse ) pCV.Wait(); } else { if( pResponse ) { Log *log = DefaultEnv::GetLog(); log->Warning( ExDbgMsg, "[%s] MsgHandler is examining a response although " "it already owns a response: 0x%x (message: %s ).", pUrl.GetHostId().c_str(), this, pRequest->GetDescription().c_str() ); } } if( msg->GetSize() < 8 ) return Ignore; ServerResponse *rsp = (ServerResponse *)msg->GetBuffer(); ClientRequest *req = (ClientRequest *)pRequest->GetBuffer(); uint16_t status = 0; uint32_t dlen = 0; //-------------------------------------------------------------------------- // We only care about async responses, but those are extracted now // in the SocketHandler. //-------------------------------------------------------------------------- if( rsp->hdr.status == kXR_attn ) { return Ignore; } //-------------------------------------------------------------------------- // We got a sync message - check if it belongs to us //-------------------------------------------------------------------------- else { if( rsp->hdr.streamid[0] != req->header.streamid[0] || rsp->hdr.streamid[1] != req->header.streamid[1] ) return Ignore; status = rsp->hdr.status; dlen = rsp->hdr.dlen; } //-------------------------------------------------------------------------- // We take the ownership of the message and decide what we will do // with the handler itself, the options are: // 1) we want to either read in raw mode (the Raw flag) or have the message // body reconstructed for us by the TransportHandler by the time // Process() is called (default, no extra flag) // 2) we either got a full response in which case we don't want to be // notified about anything anymore (RemoveHandler) or we got a partial // answer and we need to wait for more (default, no extra flag) //-------------------------------------------------------------------------- pResponse = msg; pBodyReader->SetDataLength( dlen ); Log *log = DefaultEnv::GetLog(); switch( status ) { //------------------------------------------------------------------------ // Handle the cached cases //------------------------------------------------------------------------ case kXR_error: case kXR_redirect: case kXR_wait: return RemoveHandler; case kXR_waitresp: { log->Dump( XRootDMsg, "[%s] Got kXR_waitresp response to " "message %s", pUrl.GetHostId().c_str(), pRequest->GetDescription().c_str() ); pResponse.reset(); return Ignore; // This must be handled synchronously! } //------------------------------------------------------------------------ // Handle the potential raw cases //------------------------------------------------------------------------ case kXR_ok: { //---------------------------------------------------------------------- // For kXR_read we read in raw mode //---------------------------------------------------------------------- uint16_t reqId = ntohs( req->header.requestid ); if( reqId == kXR_read ) { return Raw | RemoveHandler; } //---------------------------------------------------------------------- // kXR_readv is the same as kXR_read //---------------------------------------------------------------------- if( reqId == kXR_readv ) { return Raw | RemoveHandler; } //---------------------------------------------------------------------- // For everything else we just take what we got //---------------------------------------------------------------------- return RemoveHandler; } //------------------------------------------------------------------------ // kXR_oksofars are special, they are not full responses, so we reset // the response pointer to 0 and add the message to the partial list //------------------------------------------------------------------------ case kXR_oksofar: { log->Dump( XRootDMsg, "[%s] Got a kXR_oksofar response to request " "%s", pUrl.GetHostId().c_str(), pRequest->GetDescription().c_str() ); if( !pOksofarAsAnswer ) { pPartialResps.emplace_back( std::move( pResponse ) ); } //---------------------------------------------------------------------- // For kXR_read we either read in raw mode if the message has not // been fully reconstructed already, if it has, we adjust // the buffer offset to prepare for the next one //---------------------------------------------------------------------- uint16_t reqId = ntohs( req->header.requestid ); if( reqId == kXR_read ) { pTimeoutFence.store( true, std::memory_order_relaxed ); return Raw | ( pOksofarAsAnswer ? None : NoProcess ); } //---------------------------------------------------------------------- // kXR_readv is similar to read, except that the payload is different //---------------------------------------------------------------------- if( reqId == kXR_readv ) { pTimeoutFence.store( true, std::memory_order_relaxed ); return Raw | ( pOksofarAsAnswer ? None : NoProcess ); } return ( pOksofarAsAnswer ? None : NoProcess ); } case kXR_status: { log->Dump( XRootDMsg, "[%s] Got a kXR_status response to request " "%s", pUrl.GetHostId().c_str(), pRequest->GetDescription().c_str() ); uint16_t reqId = ntohs( req->header.requestid ); if( reqId == kXR_pgwrite ) { //-------------------------------------------------------------------- // In case of pgwrite by definition this wont be a partial response // so we can already remove the handler from the in-queue //-------------------------------------------------------------------- return RemoveHandler; } //---------------------------------------------------------------------- // Otherwise (pgread), first of all we need to read the body of the // kXR_status response, we can handle the raw data (if any) only after // we have the whole kXR_status body //---------------------------------------------------------------------- pTimeoutFence.store( true, std::memory_order_relaxed ); return None; } //------------------------------------------------------------------------ // Default //------------------------------------------------------------------------ default: return RemoveHandler; } return RemoveHandler; } //---------------------------------------------------------------------------- // Reexamine the incoming message, and decide on the action to be taken //---------------------------------------------------------------------------- uint16_t XRootDMsgHandler::InspectStatusRsp() { if( !pResponse ) return 0; Log *log = DefaultEnv::GetLog(); ServerResponse *rsp = (ServerResponse *)pResponse->GetBuffer(); //-------------------------------------------------------------------------- // Additional action is only required for kXR_status //-------------------------------------------------------------------------- if( rsp->hdr.status != kXR_status ) return 0; //-------------------------------------------------------------------------- // Ignore malformed status response //-------------------------------------------------------------------------- if( pResponse->GetSize() < sizeof( ServerResponseStatus ) ) { log->Error( XRootDMsg, "[%s] kXR_status: invalid message size.", pUrl.GetHostId().c_str() ); return Corrupted; } ClientRequest *req = (ClientRequest *)pRequest->GetBuffer(); uint16_t reqId = ntohs( req->header.requestid ); //-------------------------------------------------------------------------- // Unmarshal the status body //-------------------------------------------------------------------------- XRootDStatus st = XRootDTransport::UnMarshalStatusBody( *pResponse, reqId ); if( !st.IsOK() && st.code == errDataError ) { log->Error( XRootDMsg, "[%s] %s", pUrl.GetHostId().c_str(), st.GetErrorMessage().c_str() ); return Corrupted; } if( !st.IsOK() ) { log->Error( XRootDMsg, "[%s] Failed to unmarshall status body.", pUrl.GetHostId().c_str() ); pStatus = st; HandleRspOrQueue(); return Ignore; } //-------------------------------------------------------------------------- // Common handling for partial results //-------------------------------------------------------------------------- ServerResponseV2 *rspst = (ServerResponseV2*)pResponse->GetBuffer(); if( rspst->status.bdy.resptype == XrdProto::kXR_PartialResult ) { pPartialResps.push_back( std::move( pResponse ) ); } //-------------------------------------------------------------------------- // Decide the actions that we need to take //-------------------------------------------------------------------------- uint16_t action = 0; if( reqId == kXR_pgread ) { //---------------------------------------------------------------------- // The message contains only Status header and body but no raw data //---------------------------------------------------------------------- if( !pPageReader ) pPageReader.reset( new AsyncPageReader( *pChunkList, pCrc32cDigests ) ); pPageReader->SetRsp( rspst ); action |= Raw; if( rspst->status.bdy.resptype == XrdProto::kXR_PartialResult ) action |= NoProcess; else action |= RemoveHandler; } else if( reqId == kXR_pgwrite ) { // if data corruption has been detected on the server side we will // send some additional data pointing to the pages that need to be // retransmitted if( size_t( sizeof( ServerResponseHeader ) + rspst->status.hdr.dlen + rspst->status.bdy.dlen ) > pResponse->GetCursor() ) action |= More; } return action; } //---------------------------------------------------------------------------- // Get handler sid //---------------------------------------------------------------------------- uint16_t XRootDMsgHandler::GetSid() const { ClientRequest* req = (ClientRequest*) pRequest->GetBuffer(); return ((uint16_t)req->header.streamid[1] << 8) | (uint16_t)req->header.streamid[0]; } //---------------------------------------------------------------------------- //! Process the message if it was "taken" by the examine action //---------------------------------------------------------------------------- void XRootDMsgHandler::Process() { Log *log = DefaultEnv::GetLog(); ServerResponse *rsp = (ServerResponse *)pResponse->GetBuffer(); ClientRequest *req = (ClientRequest *)pRequest->GetBuffer(); //-------------------------------------------------------------------------- // If it is a local file, it can be only a metalink redirector //-------------------------------------------------------------------------- if( pUrl.IsLocalFile() && pUrl.IsMetalink() ) pHosts->back().protocol = kXR_PROTOCOLVERSION; //-------------------------------------------------------------------------- // We got an answer, check who we were talking to //-------------------------------------------------------------------------- else { AnyObject qryResult; int *qryResponse = 0; pPostMaster->QueryTransport( pUrl, XRootDQuery::ServerFlags, qryResult ); qryResult.Get( qryResponse ); pHosts->back().flags = *qryResponse; delete qryResponse; qryResponse = 0; pPostMaster->QueryTransport( pUrl, XRootDQuery::ProtocolVersion, qryResult ); qryResult.Get( qryResponse ); pHosts->back().protocol = *qryResponse; delete qryResponse; } //-------------------------------------------------------------------------- // Process the message //-------------------------------------------------------------------------- Status st = XRootDTransport::UnMarshallBody( pResponse.get(), req->header.requestid ); if( !st.IsOK() ) { pStatus = Status( stFatal, errInvalidMessage ); HandleResponse(); return; } //-------------------------------------------------------------------------- // we have an response for the message so it's not in fly anymore //-------------------------------------------------------------------------- pMsgInFly = false; //-------------------------------------------------------------------------- // Reset the aggregated wait (used to omit wait response in case of Metalink // redirector) //-------------------------------------------------------------------------- if( rsp->hdr.status != kXR_wait ) pAggregatedWaitTime = 0; switch( rsp->hdr.status ) { //------------------------------------------------------------------------ // kXR_ok - we're done here //------------------------------------------------------------------------ case kXR_ok: { log->Dump( XRootDMsg, "[%s] Got a kXR_ok response to request %s", pUrl.GetHostId().c_str(), pRequest->GetDescription().c_str() ); pStatus = Status(); HandleResponse(); return; } case kXR_status: { log->Dump( XRootDMsg, "[%s] Got a kXR_status response to request %s", pUrl.GetHostId().c_str(), pRequest->GetDescription().c_str() ); pStatus = Status(); HandleResponse(); return; } //------------------------------------------------------------------------ // kXR_ok - we're serving partial result to the user //------------------------------------------------------------------------ case kXR_oksofar: { log->Dump( XRootDMsg, "[%s] Got a kXR_oksofar response to request %s", pUrl.GetHostId().c_str(), pRequest->GetDescription().c_str() ); pStatus = Status( stOK, suContinue ); HandleResponse(); return; } //------------------------------------------------------------------------ // kXR_error - we've got a problem //------------------------------------------------------------------------ case kXR_error: { char *errmsg = new char[rsp->hdr.dlen-3]; errmsg[rsp->hdr.dlen-4] = 0; memcpy( errmsg, rsp->body.error.errmsg, rsp->hdr.dlen-4 ); log->Dump( XRootDMsg, "[%s] Got a kXR_error response to request %s " "[%d] %s", pUrl.GetHostId().c_str(), pRequest->GetDescription().c_str(), rsp->body.error.errnum, errmsg ); delete [] errmsg; HandleError( Status(stError, errErrorResponse, rsp->body.error.errnum) ); return; } //------------------------------------------------------------------------ // kXR_redirect - they tell us to go elsewhere //------------------------------------------------------------------------ case kXR_redirect: { if( rsp->hdr.dlen <= 4 ) { log->Error( XRootDMsg, "[%s] Got invalid redirect response.", pUrl.GetHostId().c_str() ); pStatus = Status( stError, errInvalidResponse ); HandleResponse(); return; } char *urlInfoBuff = new char[rsp->hdr.dlen-3]; urlInfoBuff[rsp->hdr.dlen-4] = 0; memcpy( urlInfoBuff, rsp->body.redirect.host, rsp->hdr.dlen-4 ); std::string urlInfo = urlInfoBuff; delete [] urlInfoBuff; log->Dump( XRootDMsg, "[%s] Got kXR_redirect response to " "message %s: %s, port %d", pUrl.GetHostId().c_str(), pRequest->GetDescription().c_str(), urlInfo.c_str(), rsp->body.redirect.port ); //---------------------------------------------------------------------- // Check if we can proceed //---------------------------------------------------------------------- if( !pRedirectCounter ) { log->Warning( XRootDMsg, "[%s] Redirect limit has been reached for " "message %s, the last known error is: %s", pUrl.GetHostId().c_str(), pRequest->GetDescription().c_str(), pLastError.ToString().c_str() ); pStatus = Status( stFatal, errRedirectLimit ); HandleResponse(); return; } --pRedirectCounter; //---------------------------------------------------------------------- // Keep the info about this server if we still need to find a load // balancer //---------------------------------------------------------------------- uint32_t flags = pHosts->back().flags; if( !pHasLoadBalancer ) { if( flags & kXR_isManager ) { //------------------------------------------------------------------ // If the current server is a meta manager then it supersedes // any existing load balancer, otherwise we assign a load-balancer // only if it has not been already assigned //------------------------------------------------------------------ if( ( flags & kXR_attrMeta ) || !pLoadBalancer.url.IsValid() ) { pLoadBalancer = pHosts->back(); log->Dump( XRootDMsg, "[%s] Current server has been assigned " "as a load-balancer for message %s", pUrl.GetHostId().c_str(), pRequest->GetDescription().c_str() ); HostList::iterator it; for( it = pHosts->begin(); it != pHosts->end(); ++it ) it->loadBalancer = false; pHosts->back().loadBalancer = true; } } } //---------------------------------------------------------------------- // If the redirect comes from a data server safe the URL because // in case of a failure we will use it as the effective data server URL // for the tried CGI opaque info //---------------------------------------------------------------------- if( flags & kXR_isServer ) pEffectiveDataServerUrl = new URL( pHosts->back().url ); //---------------------------------------------------------------------- // Build the URL and check it's validity //---------------------------------------------------------------------- std::vector urlComponents; std::string newCgi; Utils::splitString( urlComponents, urlInfo, "?" ); std::ostringstream o; o << urlComponents[0]; if( rsp->body.redirect.port > 0 ) o << ":" << rsp->body.redirect.port << "/"; else if( rsp->body.redirect.port < 0 ) { //-------------------------------------------------------------------- // check if the manager wants to enforce write recovery at himself // (beware we are dealing here with negative flags) //-------------------------------------------------------------------- if( ~uint32_t( rsp->body.redirect.port ) & kXR_recoverWrts ) pHosts->back().flags |= kXR_recoverWrts; //-------------------------------------------------------------------- // check if the manager wants to collapse the communication channel // (the redirect host is to replace the current host) //-------------------------------------------------------------------- if( ~uint32_t( rsp->body.redirect.port ) & kXR_collapseRedir ) { std::string url( rsp->body.redirect.host, rsp->hdr.dlen-4 ); pPostMaster->CollapseRedirect( pUrl, url ); } if( ~uint32_t( rsp->body.redirect.port ) & kXR_ecRedir ) { std::string url( rsp->body.redirect.host, rsp->hdr.dlen-4 ); if( Utils::CheckEC( pRequest, url ) ) pRedirectAsAnswer = true; } } URL newUrl = URL( o.str() ); if( !newUrl.IsValid() ) { pStatus = Status( stError, errInvalidRedirectURL ); log->Error( XRootDMsg, "[%s] Got invalid redirection URL: %s", pUrl.GetHostId().c_str(), urlInfo.c_str() ); HandleResponse(); return; } if( pUrl.GetUserName() != "" && newUrl.GetUserName() == "" ) newUrl.SetUserName( pUrl.GetUserName() ); if( pUrl.GetPassword() != "" && newUrl.GetPassword() == "" ) newUrl.SetPassword( pUrl.GetPassword() ); //---------------------------------------------------------------------- // Forward any "xrd.*" params from the original client request also to // the new redirection url // Also, we need to preserve any "xrdcl.*' as they are important for // our internal workflows. //---------------------------------------------------------------------- std::ostringstream ossXrd; const URL::ParamsMap &urlParams = pUrl.GetParams(); for(URL::ParamsMap::const_iterator it = urlParams.begin(); it != urlParams.end(); ++it ) { if( it->first.compare( 0, 4, "xrd." ) && it->first.compare( 0, 6, "xrdcl." ) ) continue; ossXrd << it->first << '=' << it->second << '&'; } std::string xrdCgi = ossXrd.str(); pRedirectUrl = newUrl.GetURL(); URL cgiURL; if( urlComponents.size() > 1 ) { pRedirectUrl += "?"; pRedirectUrl += urlComponents[1]; std::ostringstream o; o << "fake://fake:111//fake?"; o << urlComponents[1]; if( urlComponents.size() == 3 ) o << '?' << urlComponents[2]; if (!xrdCgi.empty()) { o << '&' << xrdCgi; pRedirectUrl += '&'; pRedirectUrl += xrdCgi; } cgiURL = URL( o.str() ); } else { if (!xrdCgi.empty()) { std::ostringstream o; o << "fake://fake:111//fake?"; o << xrdCgi; cgiURL = URL( o.str() ); pRedirectUrl += '?'; pRedirectUrl += xrdCgi; } } //---------------------------------------------------------------------- // Check if we need to return the URL as a response //---------------------------------------------------------------------- if( newUrl.GetProtocol() != "root" && newUrl.GetProtocol() != "xroot" && newUrl.GetProtocol() != "roots" && newUrl.GetProtocol() != "xroots" && !newUrl.IsLocalFile() ) pRedirectAsAnswer = true; if( pRedirectAsAnswer ) { pStatus = Status( stError, errRedirect ); HandleResponse(); return; } //---------------------------------------------------------------------- // Rewrite the message in a way required to send it to another server //---------------------------------------------------------------------- newUrl.SetParams( cgiURL.GetParams() ); Status st = RewriteRequestRedirect( newUrl ); if( !st.IsOK() ) { pStatus = st; HandleResponse(); return; } //---------------------------------------------------------------------- // Make sure we don't change the protocol by accident (root vs roots) //---------------------------------------------------------------------- if( ( pUrl.GetProtocol() == "roots" || pUrl.GetProtocol() == "xroots" ) && ( newUrl.GetProtocol() == "root" || newUrl.GetProtocol() == "xroot" ) ) newUrl.SetProtocol( "roots" ); //---------------------------------------------------------------------- // Send the request to the new location //---------------------------------------------------------------------- HandleError( RetryAtServer( newUrl, RedirectEntry::EntryRedirect ) ); return; } //------------------------------------------------------------------------ // kXR_wait - we wait, and re-issue the request later //------------------------------------------------------------------------ case kXR_wait: { uint32_t waitSeconds = 0; if( rsp->hdr.dlen >= 4 ) { char *infoMsg = new char[rsp->hdr.dlen-3]; infoMsg[rsp->hdr.dlen-4] = 0; memcpy( infoMsg, rsp->body.wait.infomsg, rsp->hdr.dlen-4 ); log->Dump( XRootDMsg, "[%s] Got kXR_wait response of %d seconds to " "message %s: %s", pUrl.GetHostId().c_str(), rsp->body.wait.seconds, pRequest->GetDescription().c_str(), infoMsg ); delete [] infoMsg; waitSeconds = rsp->body.wait.seconds; } else { log->Dump( XRootDMsg, "[%s] Got kXR_wait response of 0 seconds to " "message %s", pUrl.GetHostId().c_str(), pRequest->GetDescription().c_str() ); } pAggregatedWaitTime += waitSeconds; // We need a special case if the data node comes from metalink // redirector. In this case it might make more sense to try the // next entry in the Metalink than wait. if( OmitWait( *pRequest, pLoadBalancer.url ) ) { int maxWait = DefaultMaxMetalinkWait; DefaultEnv::GetEnv()->GetInt( "MaxMetalinkWait", maxWait ); if( pAggregatedWaitTime > maxWait ) { UpdateTriedCGI(); HandleError( RetryAtServer( pLoadBalancer.url, RedirectEntry::EntryRedirectOnWait ) ); return; } } //---------------------------------------------------------------------- // Some messages require rewriting before they can be sent again // after wait //---------------------------------------------------------------------- Status st = RewriteRequestWait(); if( !st.IsOK() ) { pStatus = st; HandleResponse(); return; } //---------------------------------------------------------------------- // Register a task to resend the message in some seconds, if we still // have time to do that, and report a timeout otherwise //---------------------------------------------------------------------- time_t resendTime = ::time(0)+waitSeconds; if( resendTime < pExpiration ) { log->Debug( ExDbgMsg, "[%s] Scheduling WaitTask for MsgHandler: 0x%x (message: %s ).", pUrl.GetHostId().c_str(), this, pRequest->GetDescription().c_str() ); TaskManager *taskMgr = pPostMaster->GetTaskManager(); taskMgr->RegisterTask( new WaitTask( this ), resendTime ); } else { log->Debug( XRootDMsg, "[%s] Wait time is too long, timing out %s", pUrl.GetHostId().c_str(), pRequest->GetDescription().c_str() ); HandleError( Status( stError, errOperationExpired) ); } return; } //------------------------------------------------------------------------ // kXR_waitresp - the response will be returned in some seconds as an // unsolicited message. Currently all messages of this type are handled // one step before in the XrdClStream::OnIncoming as they need to be // processed synchronously. //------------------------------------------------------------------------ case kXR_waitresp: { if( rsp->hdr.dlen < 4 ) { log->Error( XRootDMsg, "[%s] Got invalid waitresp response.", pUrl.GetHostId().c_str() ); pStatus = Status( stError, errInvalidResponse ); HandleResponse(); return; } log->Dump( XRootDMsg, "[%s] Got kXR_waitresp response of %d seconds to " "message %s", pUrl.GetHostId().c_str(), rsp->body.waitresp.seconds, pRequest->GetDescription().c_str() ); return; } //------------------------------------------------------------------------ // Default - unrecognized/unsupported response, declare an error //------------------------------------------------------------------------ default: { log->Dump( XRootDMsg, "[%s] Got unrecognized response %d to " "message %s", pUrl.GetHostId().c_str(), rsp->hdr.status, pRequest->GetDescription().c_str() ); pStatus = Status( stError, errInvalidResponse ); HandleResponse(); return; } } return; } //---------------------------------------------------------------------------- // Handle an event other that a message arrival - may be timeout //---------------------------------------------------------------------------- uint8_t XRootDMsgHandler::OnStreamEvent( StreamEvent event, XRootDStatus status ) { Log *log = DefaultEnv::GetLog(); log->Dump( XRootDMsg, "[%s] Stream event reported for msg %s", pUrl.GetHostId().c_str(), pRequest->GetDescription().c_str() ); if( event == Ready ) return 0; if( pTimeoutFence.load( std::memory_order_relaxed ) ) return 0; HandleError( status ); return RemoveHandler; } //---------------------------------------------------------------------------- // Read message body directly from a socket //---------------------------------------------------------------------------- XRootDStatus XRootDMsgHandler::ReadMessageBody( Message*, Socket *socket, uint32_t &bytesRead ) { ClientRequest *req = (ClientRequest *)pRequest->GetBuffer(); uint16_t reqId = ntohs( req->header.requestid ); if( reqId == kXR_pgread ) return pPageReader->Read( *socket, bytesRead ); return pBodyReader->Read( *socket, bytesRead ); } //---------------------------------------------------------------------------- // We're here when we requested sending something over the wire // and there has been a status update on this action //---------------------------------------------------------------------------- void XRootDMsgHandler::OnStatusReady( const Message *message, XRootDStatus status ) { Log *log = DefaultEnv::GetLog(); //-------------------------------------------------------------------------- // We were successful, so we now need to listen for a response //-------------------------------------------------------------------------- if( status.IsOK() ) { log->Dump( XRootDMsg, "[%s] Message %s has been successfully sent.", pUrl.GetHostId().c_str(), message->GetDescription().c_str() ); log->Debug( ExDbgMsg, "[%s] Moving MsgHandler: 0x%x (message: %s ) from out-queue to in-queue.", pUrl.GetHostId().c_str(), this, pRequest->GetDescription().c_str() ); pMsgInFly = true; return; } //-------------------------------------------------------------------------- // We have failed, recover if possible //-------------------------------------------------------------------------- log->Error( XRootDMsg, "[%s] Impossible to send message %s. Trying to " "recover.", pUrl.GetHostId().c_str(), message->GetDescription().c_str() ); HandleError( status ); } //---------------------------------------------------------------------------- // Are we a raw writer or not? //---------------------------------------------------------------------------- bool XRootDMsgHandler::IsRaw() const { ClientRequest *req = (ClientRequest *)pRequest->GetBuffer(); uint16_t reqId = ntohs( req->header.requestid ); if( reqId == kXR_write || reqId == kXR_writev || reqId == kXR_pgwrite ) return true; // checkpoint + execute if( reqId == kXR_chkpoint && req->chkpoint.opcode == kXR_ckpXeq ) { ClientRequest *xeq = (ClientRequest*)pRequest->GetBuffer( sizeof( ClientRequest ) ); reqId = ntohs( xeq->header.requestid ); return reqId != kXR_truncate; // only checkpointed truncate does not have raw data } return false; } //---------------------------------------------------------------------------- // Write the message body //---------------------------------------------------------------------------- XRootDStatus XRootDMsgHandler::WriteMessageBody( Socket *socket, uint32_t &bytesWritten ) { //-------------------------------------------------------------------------- // First check if it is a PgWrite //-------------------------------------------------------------------------- if( !pChunkList->empty() && !pCrc32cDigests.empty() ) { //------------------------------------------------------------------------ // PgWrite will have just one chunk //------------------------------------------------------------------------ ChunkInfo chunk = pChunkList->front(); //------------------------------------------------------------------------ // Calculate the size of the first and last page (in case the chunk is not // 4KB aligned) //------------------------------------------------------------------------ int fLen = 0, lLen = 0; size_t nbpgs = XrdOucPgrwUtils::csNum( chunk.offset, chunk.length, fLen, lLen ); //------------------------------------------------------------------------ // Set the crc32c buffer if not ready yet //------------------------------------------------------------------------ if( pPgWrtCksumBuff.GetCursor() == 0 ) { uint32_t digest = htonl( pCrc32cDigests[pPgWrtCurrentPageNb] ); memcpy( pPgWrtCksumBuff.GetBuffer(), &digest, sizeof( uint32_t ) ); } uint32_t btsLeft = chunk.length - pAsyncOffset; uint32_t pglen = ( pPgWrtCurrentPageNb == 0 ? fLen : XrdSys::PageSize ) - pPgWrtCurrentPageOffset; if( pglen > btsLeft ) pglen = btsLeft; char* pgbuf = static_cast( chunk.buffer ) + pAsyncOffset; while( btsLeft > 0 ) { // first write the crc32c digest while( pPgWrtCksumBuff.GetCursor() < sizeof( uint32_t ) ) { uint32_t dgstlen = sizeof( uint32_t ) - pPgWrtCksumBuff.GetCursor(); char* dgstbuf = pPgWrtCksumBuff.GetBufferAtCursor(); int btswrt = 0; Status st = socket->Send( dgstbuf, dgstlen, btswrt ); if( !st.IsOK() ) return st; bytesWritten += btswrt; pPgWrtCksumBuff.AdvanceCursor( btswrt ); if( st.code == suRetry ) return st; } // then write the raw data (one page) int btswrt = 0; Status st = socket->Send( pgbuf, pglen, btswrt ); if( !st.IsOK() ) return st; pgbuf += btswrt; pglen -= btswrt; btsLeft -= btswrt; bytesWritten += btswrt; pAsyncOffset += btswrt; // update the offset to the raw data if( st.code == suRetry ) return st; // if we managed to write all the data ... if( pglen == 0 ) { // move to the next page ++pPgWrtCurrentPageNb; if( pPgWrtCurrentPageNb < nbpgs ) { // set the digest buffer pPgWrtCksumBuff.SetCursor( 0 ); uint32_t digest = htonl( pCrc32cDigests[pPgWrtCurrentPageNb] ); memcpy( pPgWrtCksumBuff.GetBuffer(), &digest, sizeof( uint32_t ) ); } // set the page length pglen = XrdSys::PageSize; if( pglen > btsLeft ) pglen = btsLeft; // reset offset in the current page pPgWrtCurrentPageOffset = 0; } else // otherwise just adjust the offset in the current page pPgWrtCurrentPageOffset += btswrt; } } else if( !pChunkList->empty() ) { size_t size = pChunkList->size(); for( size_t i = pAsyncChunkIndex ; i < size; ++i ) { char *buffer = (char*)(*pChunkList)[i].buffer; uint32_t size = (*pChunkList)[i].length; size_t leftToBeWritten = size - pAsyncOffset; while( leftToBeWritten ) { int btswrt = 0; Status st = socket->Send( buffer + pAsyncOffset, leftToBeWritten, btswrt ); bytesWritten += btswrt; if( !st.IsOK() || st.code == suRetry ) return st; pAsyncOffset += btswrt; leftToBeWritten -= btswrt; } //---------------------------------------------------------------------- // Remember that we have moved to the next chunk, also clear the offset // within the buffer as we are going to move to a new one //---------------------------------------------------------------------- ++pAsyncChunkIndex; pAsyncOffset = 0; } } else { Log *log = DefaultEnv::GetLog(); //------------------------------------------------------------------------ // If the socket is encrypted we cannot use a kernel buffer, we have to // convert to user space buffer //------------------------------------------------------------------------ if( socket->IsEncrypted() ) { log->Debug( XRootDMsg, "[%s] Channel is encrypted: cannot use kernel buffer.", pUrl.GetHostId().c_str() ); char *ubuff = 0; ssize_t ret = XrdSys::Move( *pKBuff, ubuff ); if( ret < 0 ) return Status( stError, errInternal ); pChunkList->push_back( ChunkInfo( 0, ret, ubuff ) ); return WriteMessageBody( socket, bytesWritten ); } //------------------------------------------------------------------------ // Send the data //------------------------------------------------------------------------ while( !pKBuff->Empty() ) { int btswrt = 0; Status st = socket->Send( *pKBuff, btswrt ); bytesWritten += btswrt; if( !st.IsOK() || st.code == suRetry ) return st; } log->Debug( XRootDMsg, "[%s] Request %s payload (kernel buffer) transferred to socket.", pUrl.GetHostId().c_str(), pRequest->GetDescription().c_str() ); } return Status(); } //---------------------------------------------------------------------------- // We're here when we got a time event. We needed to re-issue the request // in some time in the future, and that moment has arrived //---------------------------------------------------------------------------- void XRootDMsgHandler::WaitDone( time_t ) { HandleError( RetryAtServer( pUrl, RedirectEntry::EntryWait ) ); } //---------------------------------------------------------------------------- // Bookkeeping after partial response has been received. //---------------------------------------------------------------------------- void XRootDMsgHandler::PartialReceived() { pTimeoutFence.store( false, std::memory_order_relaxed ); // Take down the timeout fence } //---------------------------------------------------------------------------- // Unpack the message and call the response handler //---------------------------------------------------------------------------- void XRootDMsgHandler::HandleResponse() { //-------------------------------------------------------------------------- // Process the response and notify the listener //-------------------------------------------------------------------------- XRootDTransport::UnMarshallRequest( pRequest ); XRootDStatus *status = ProcessStatus(); AnyObject *response = 0; Log *log = DefaultEnv::GetLog(); log->Debug( ExDbgMsg, "[%s] Calling MsgHandler: 0x%x (message: %s ) " "with status: %s.", pUrl.GetHostId().c_str(), this, pRequest->GetDescription().c_str(), status->ToString().c_str() ); if( status->IsOK() ) { Status st = ParseResponse( response ); if( !st.IsOK() ) { delete status; delete response; status = new XRootDStatus( st ); response = 0; } } //-------------------------------------------------------------------------- // Close the redirect entry if necessary //-------------------------------------------------------------------------- if( pRdirEntry ) { pRdirEntry->status = *status; pRedirectTraceBack.push_back( std::move( pRdirEntry ) ); } //-------------------------------------------------------------------------- // Is it a final response? //-------------------------------------------------------------------------- bool finalrsp = !( pStatus.IsOK() && pStatus.code == suContinue ); //-------------------------------------------------------------------------- // Release the stream id //-------------------------------------------------------------------------- if( pSidMgr && finalrsp ) { ClientRequest *req = (ClientRequest *)pRequest->GetBuffer(); if( status->IsOK() || !pMsgInFly || !( status->code == errOperationExpired || status->code == errOperationInterrupted ) ) pSidMgr->ReleaseSID( req->header.streamid ); } HostList *hosts = pHosts.release(); if( !finalrsp ) pHosts.reset( new HostList( *hosts ) ); pResponseHandler->HandleResponseWithHosts( status, response, hosts ); //-------------------------------------------------------------------------- // if it is the final response there is nothing more to do ... //-------------------------------------------------------------------------- if( finalrsp ) delete this; //-------------------------------------------------------------------------- // on the other hand if it is not the final response, we have to keep the // MsgHandler and delete the current response //-------------------------------------------------------------------------- else { XrdSysCondVarHelper lck( pCV ); pResponse.reset(); pTimeoutFence.store( false, std::memory_order_relaxed ); pCV.Broadcast(); } } //---------------------------------------------------------------------------- // Extract the status information from the stuff that we got //---------------------------------------------------------------------------- XRootDStatus *XRootDMsgHandler::ProcessStatus() { XRootDStatus *st = new XRootDStatus( pStatus ); ServerResponse *rsp = 0; if( pResponse ) rsp = (ServerResponse *)pResponse->GetBuffer(); if( !pStatus.IsOK() && rsp ) { if( pStatus.code == errErrorResponse ) { st->errNo = rsp->body.error.errnum; // omit the last character as the string returned from the server // (acording to protocol specs) should be null-terminated std::string errmsg( rsp->body.error.errmsg, rsp->hdr.dlen-5 ); if( st->errNo == kXR_noReplicas && !pLastError.IsOK() ) errmsg += " Last seen error: " + pLastError.ToString(); st->SetErrorMessage( errmsg ); } else if( pStatus.code == errRedirect ) st->SetErrorMessage( pRedirectUrl ); } return st; } //------------------------------------------------------------------------ // Parse the response and put it in an object that could be passed to // the user //------------------------------------------------------------------------ Status XRootDMsgHandler::ParseResponse( AnyObject *&response ) { if( !pResponse ) return Status(); ServerResponse *rsp = (ServerResponse *)pResponse->GetBuffer(); ClientRequest *req = (ClientRequest *)pRequest->GetBuffer(); Log *log = DefaultEnv::GetLog(); //-------------------------------------------------------------------------- // Handle redirect as an answer //-------------------------------------------------------------------------- if( rsp->hdr.status == kXR_redirect ) { log->Error( XRootDMsg, "Internal Error: unable to process redirect" ); return 0; } Buffer buff; uint32_t length = 0; char *buffer = 0; //-------------------------------------------------------------------------- // We don't have any partial answers so pass what we have //-------------------------------------------------------------------------- if( pPartialResps.empty() ) { buffer = rsp->body.buffer.data; length = rsp->hdr.dlen; } //-------------------------------------------------------------------------- // Partial answers, we need to glue them together before parsing //-------------------------------------------------------------------------- else if( req->header.requestid != kXR_read && req->header.requestid != kXR_readv ) { for( uint32_t i = 0; i < pPartialResps.size(); ++i ) { ServerResponse *part = (ServerResponse*)pPartialResps[i]->GetBuffer(); length += part->hdr.dlen; } length += rsp->hdr.dlen; buff.Allocate( length ); uint32_t offset = 0; for( uint32_t i = 0; i < pPartialResps.size(); ++i ) { ServerResponse *part = (ServerResponse*)pPartialResps[i]->GetBuffer(); buff.Append( part->body.buffer.data, part->hdr.dlen, offset ); offset += part->hdr.dlen; } buff.Append( rsp->body.buffer.data, rsp->hdr.dlen, offset ); buffer = buff.GetBuffer(); } //-------------------------------------------------------------------------- // Right, but what was the question? //-------------------------------------------------------------------------- switch( req->header.requestid ) { //------------------------------------------------------------------------ // kXR_mv, kXR_truncate, kXR_rm, kXR_mkdir, kXR_rmdir, kXR_chmod, // kXR_ping, kXR_close, kXR_write, kXR_sync //------------------------------------------------------------------------ case kXR_mv: case kXR_truncate: case kXR_rm: case kXR_mkdir: case kXR_rmdir: case kXR_chmod: case kXR_ping: case kXR_close: case kXR_write: case kXR_writev: case kXR_sync: case kXR_chkpoint: return Status(); //------------------------------------------------------------------------ // kXR_locate //------------------------------------------------------------------------ case kXR_locate: { AnyObject *obj = new AnyObject(); char *nullBuffer = new char[length+1]; nullBuffer[length] = 0; memcpy( nullBuffer, buffer, length ); log->Dump( XRootDMsg, "[%s] Parsing the response to %s as " "LocateInfo: %s", pUrl.GetHostId().c_str(), pRequest->GetDescription().c_str(), nullBuffer ); LocationInfo *data = new LocationInfo(); if( data->ParseServerResponse( nullBuffer ) == false ) { delete obj; delete data; delete [] nullBuffer; return Status( stError, errInvalidResponse ); } delete [] nullBuffer; obj->Set( data ); response = obj; return Status(); } //------------------------------------------------------------------------ // kXR_stat //------------------------------------------------------------------------ case kXR_stat: { AnyObject *obj = new AnyObject(); //---------------------------------------------------------------------- // Virtual File System stat (kXR_vfs) //---------------------------------------------------------------------- if( req->stat.options & kXR_vfs ) { StatInfoVFS *data = new StatInfoVFS(); char *nullBuffer = new char[length+1]; nullBuffer[length] = 0; memcpy( nullBuffer, buffer, length ); log->Dump( XRootDMsg, "[%s] Parsing the response to %s as " "StatInfoVFS: %s", pUrl.GetHostId().c_str(), pRequest->GetDescription().c_str(), nullBuffer ); if( data->ParseServerResponse( nullBuffer ) == false ) { delete obj; delete data; delete [] nullBuffer; return Status( stError, errInvalidResponse ); } delete [] nullBuffer; obj->Set( data ); } //---------------------------------------------------------------------- // Normal stat //---------------------------------------------------------------------- else { StatInfo *data = new StatInfo(); char *nullBuffer = new char[length+1]; nullBuffer[length] = 0; memcpy( nullBuffer, buffer, length ); log->Dump( XRootDMsg, "[%s] Parsing the response to %s as StatInfo: " "%s", pUrl.GetHostId().c_str(), pRequest->GetDescription().c_str(), nullBuffer ); if( data->ParseServerResponse( nullBuffer ) == false ) { delete obj; delete data; delete [] nullBuffer; return Status( stError, errInvalidResponse ); } delete [] nullBuffer; obj->Set( data ); } response = obj; return Status(); } //------------------------------------------------------------------------ // kXR_protocol //------------------------------------------------------------------------ case kXR_protocol: { log->Dump( XRootDMsg, "[%s] Parsing the response to %s as ProtocolInfo", pUrl.GetHostId().c_str(), pRequest->GetDescription().c_str() ); if( rsp->hdr.dlen < 8 ) { log->Error( XRootDMsg, "[%s] Got invalid redirect response.", pUrl.GetHostId().c_str() ); return Status( stError, errInvalidResponse ); } AnyObject *obj = new AnyObject(); ProtocolInfo *data = new ProtocolInfo( rsp->body.protocol.pval, rsp->body.protocol.flags ); obj->Set( data ); response = obj; return Status(); } //------------------------------------------------------------------------ // kXR_dirlist //------------------------------------------------------------------------ case kXR_dirlist: { AnyObject *obj = new AnyObject(); log->Dump( XRootDMsg, "[%s] Parsing the response to %s as " "DirectoryList", pUrl.GetHostId().c_str(), pRequest->GetDescription().c_str() ); char *path = new char[req->dirlist.dlen+1]; path[req->dirlist.dlen] = 0; memcpy( path, pRequest->GetBuffer(24), req->dirlist.dlen ); DirectoryList *data = new DirectoryList(); data->SetParentName( path ); delete [] path; char *nullBuffer = new char[length+1]; nullBuffer[length] = 0; memcpy( nullBuffer, buffer, length ); bool invalidrsp = false; if( !pDirListStarted ) { pDirListWithStat = DirectoryList::HasStatInfo( nullBuffer ); pDirListStarted = true; invalidrsp = !data->ParseServerResponse( pUrl.GetHostId(), nullBuffer ); } else invalidrsp = !data->ParseServerResponse( pUrl.GetHostId(), nullBuffer, pDirListWithStat ); if( invalidrsp ) { delete data; delete obj; delete [] nullBuffer; return Status( stError, errInvalidResponse ); } delete [] nullBuffer; obj->Set( data ); response = obj; return Status(); } //------------------------------------------------------------------------ // kXR_open - if we got the statistics, otherwise return 0 //------------------------------------------------------------------------ case kXR_open: { log->Dump( XRootDMsg, "[%s] Parsing the response to %s as OpenInfo", pUrl.GetHostId().c_str(), pRequest->GetDescription().c_str() ); if( rsp->hdr.dlen < 4 ) { log->Error( XRootDMsg, "[%s] Got invalid open response.", pUrl.GetHostId().c_str() ); return Status( stError, errInvalidResponse ); } AnyObject *obj = new AnyObject(); StatInfo *statInfo = 0; //---------------------------------------------------------------------- // Handle StatInfo if requested //---------------------------------------------------------------------- if( req->open.options & kXR_retstat ) { log->Dump( XRootDMsg, "[%s] Parsing StatInfo in response to %s", pUrl.GetHostId().c_str(), pRequest->GetDescription().c_str() ); if( rsp->hdr.dlen >= 12 ) { char *nullBuffer = new char[rsp->hdr.dlen-11]; nullBuffer[rsp->hdr.dlen-12] = 0; memcpy( nullBuffer, buffer+12, rsp->hdr.dlen-12 ); statInfo = new StatInfo(); if( statInfo->ParseServerResponse( nullBuffer ) == false ) { delete statInfo; statInfo = 0; } delete [] nullBuffer; } if( rsp->hdr.dlen < 12 || !statInfo ) { log->Error( XRootDMsg, "[%s] Unable to parse StatInfo in response " "to %s", pUrl.GetHostId().c_str(), pRequest->GetDescription().c_str() ); delete obj; return Status( stError, errInvalidResponse ); } } OpenInfo *data = new OpenInfo( (uint8_t*)buffer, pResponse->GetSessionId(), statInfo ); obj->Set( data ); response = obj; return Status(); } //------------------------------------------------------------------------ // kXR_read //------------------------------------------------------------------------ case kXR_read: { log->Dump( XRootDMsg, "[%s] Parsing the response to %s as ChunkInfo", pUrl.GetHostId().c_str(), pRequest->GetDescription().c_str() ); for( uint32_t i = 0; i < pPartialResps.size(); ++i ) { //-------------------------------------------------------------------- // we are expecting to have only the header in the message, the raw // data have been readout into the user buffer //-------------------------------------------------------------------- if( pPartialResps[i]->GetSize() > 8 ) return Status( stOK, errInternal ); } //---------------------------------------------------------------------- // we are expecting to have only the header in the message, the raw // data have been readout into the user buffer //---------------------------------------------------------------------- if( pResponse->GetSize() > 8 ) return Status( stOK, errInternal ); //---------------------------------------------------------------------- // Get the response for the end user //---------------------------------------------------------------------- return pBodyReader->GetResponse( response ); } //------------------------------------------------------------------------ // kXR_pgread //------------------------------------------------------------------------ case kXR_pgread: { log->Dump( XRootDMsg, "[%s] Parsing the response to %s as PageInfo", pUrl.GetHostId().c_str(), pRequest->GetDescription().c_str() ); //---------------------------------------------------------------------- // Glue in the cached responses if necessary //---------------------------------------------------------------------- ChunkInfo chunk = pChunkList->front(); bool sizeMismatch = false; uint32_t currentOffset = 0; char *cursor = (char*)chunk.buffer; for( uint32_t i = 0; i < pPartialResps.size(); ++i ) { ServerResponseV2 *part = (ServerResponseV2*)pPartialResps[i]->GetBuffer(); //-------------------------------------------------------------------- // the actual size of the raw data without the crc32c checksums //-------------------------------------------------------------------- size_t datalen = part->status.bdy.dlen - NbPgPerRsp( part->info.pgread.offset, part->status.bdy.dlen ) * CksumSize; if( currentOffset + datalen > chunk.length ) { sizeMismatch = true; break; } currentOffset += datalen; cursor += datalen; } ServerResponseV2 *rspst = (ServerResponseV2*)pResponse->GetBuffer(); size_t datalen = rspst->status.bdy.dlen - NbPgPerRsp( rspst->info.pgread.offset, rspst->status.bdy.dlen ) * CksumSize; if( currentOffset + datalen <= chunk.length ) currentOffset += datalen; else sizeMismatch = true; //---------------------------------------------------------------------- // Overflow //---------------------------------------------------------------------- if( pChunkStatus.front().sizeError || sizeMismatch ) { log->Error( XRootDMsg, "[%s] Handling response to %s: user supplied " "buffer is too small for the received data.", pUrl.GetHostId().c_str(), pRequest->GetDescription().c_str() ); return Status( stError, errInvalidResponse ); } AnyObject *obj = new AnyObject(); PageInfo *pgInfo = new PageInfo( chunk.offset, currentOffset, chunk.buffer, std::move( pCrc32cDigests) ); obj->Set( pgInfo ); response = obj; return Status(); } //------------------------------------------------------------------------ // kXR_pgwrite //------------------------------------------------------------------------ case kXR_pgwrite: { std::vector> retries; ServerResponseV2 *rsp = (ServerResponseV2*)pResponse->GetBuffer(); if( rsp->status.bdy.dlen > 0 ) { ServerResponseBody_pgWrCSE *cse = (ServerResponseBody_pgWrCSE*)pResponse->GetBuffer( sizeof( ServerResponseV2 ) ); size_t pgcnt = ( rsp->status.bdy.dlen - 8 ) / sizeof( kXR_int64 ); retries.reserve( pgcnt ); kXR_int64 *pgoffs = (kXR_int64*)pResponse->GetBuffer( sizeof( ServerResponseV2 ) + sizeof( ServerResponseBody_pgWrCSE ) ); for( size_t i = 0; i < pgcnt; ++i ) { uint32_t len = XrdSys::PageSize; if( i == 0 ) len = cse->dlFirst; else if( i == pgcnt - 1 ) len = cse->dlLast; retries.push_back( std::make_tuple( pgoffs[i], len ) ); } } RetryInfo *info = new RetryInfo( std::move( retries ) ); AnyObject *obj = new AnyObject(); obj->Set( info ); response = obj; return Status(); } //------------------------------------------------------------------------ // kXR_readv - we need to pass the length of the buffer to the user code //------------------------------------------------------------------------ case kXR_readv: { log->Dump( XRootDMsg, "[%s] Parsing the response to 0x%x as " "VectorReadInfo", pUrl.GetHostId().c_str(), pRequest->GetDescription().c_str() ); for( uint32_t i = 0; i < pPartialResps.size(); ++i ) { //-------------------------------------------------------------------- // we are expecting to have only the header in the message, the raw // data have been readout into the user buffer //-------------------------------------------------------------------- if( pPartialResps[i]->GetSize() > 8 ) return Status( stOK, errInternal ); } //---------------------------------------------------------------------- // we are expecting to have only the header in the message, the raw // data have been readout into the user buffer //---------------------------------------------------------------------- if( pResponse->GetSize() > 8 ) return Status( stOK, errInternal ); //---------------------------------------------------------------------- // Get the response for the end user //---------------------------------------------------------------------- return pBodyReader->GetResponse( response ); } //------------------------------------------------------------------------ // kXR_fattr //------------------------------------------------------------------------ case kXR_fattr: { int len = rsp->hdr.dlen; char* data = rsp->body.buffer.data; return ParseXAttrResponse( data, len, response ); } //------------------------------------------------------------------------ // kXR_query //------------------------------------------------------------------------ case kXR_query: case kXR_set: case kXR_prepare: default: { AnyObject *obj = new AnyObject(); log->Dump( XRootDMsg, "[%s] Parsing the response to %s as BinaryData", pUrl.GetHostId().c_str(), pRequest->GetDescription().c_str() ); BinaryDataInfo *data = new BinaryDataInfo(); data->Allocate( length ); data->Append( buffer, length ); obj->Set( data ); response = obj; return Status(); } }; return Status( stError, errInvalidMessage ); } //------------------------------------------------------------------------ // Parse the response to kXR_fattr request and put it in an object that // could be passed to the user //------------------------------------------------------------------------ Status XRootDMsgHandler::ParseXAttrResponse( char *data, size_t len, AnyObject *&response ) { ClientRequest *req = (ClientRequest *)pRequest->GetBuffer(); // Log *log = DefaultEnv::GetLog(); //TODO switch( req->fattr.subcode ) { case kXR_fattrDel: case kXR_fattrSet: { Status status; kXR_char nerrs = 0; if( !( status = ReadFromBuffer( data, len, nerrs ) ).IsOK() ) return status; kXR_char nattr = 0; if( !( status = ReadFromBuffer( data, len, nattr ) ).IsOK() ) return status; std::vector resp; // read the namevec for( kXR_char i = 0; i < nattr; ++i ) { kXR_unt16 rc = 0; if( !( status = ReadFromBuffer( data, len, rc ) ).IsOK() ) return status; rc = ntohs( rc ); // count errors if( rc ) --nerrs; std::string name; if( !( status = ReadFromBuffer( data, len, name ) ).IsOK() ) return status; XRootDStatus st = rc ? XRootDStatus( stError, errErrorResponse, rc ) : XRootDStatus(); resp.push_back( XAttrStatus( name, st ) ); } // check if we read all the data and if the error count is OK if( len != 0 || nerrs != 0 ) return Status( stError, errDataError ); // set up the response object response = new AnyObject(); response->Set( new std::vector( std::move( resp ) ) ); return Status(); } case kXR_fattrGet: { Status status; kXR_char nerrs = 0; if( !( status = ReadFromBuffer( data, len, nerrs ) ).IsOK() ) return status; kXR_char nattr = 0; if( !( status = ReadFromBuffer( data, len, nattr ) ).IsOK() ) return status; std::vector resp; resp.reserve( nattr ); // read the name vec for( kXR_char i = 0; i < nattr; ++i ) { kXR_unt16 rc = 0; if( !( status = ReadFromBuffer( data, len, rc ) ).IsOK() ) return status; rc = ntohs( rc ); // count errors if( rc ) --nerrs; std::string name; if( !( status = ReadFromBuffer( data, len, name ) ).IsOK() ) return status; XRootDStatus st = rc ? XRootDStatus( stError, errErrorResponse, rc ) : XRootDStatus(); resp.push_back( XAttr( name, st ) ); } // read the value vec for( kXR_char i = 0; i < nattr; ++i ) { kXR_int32 vlen = 0; if( !( status = ReadFromBuffer( data, len, vlen ) ).IsOK() ) return status; vlen = ntohl( vlen ); std::string value; if( !( status = ReadFromBuffer( data, len, vlen, value ) ).IsOK() ) return status; resp[i].value.swap( value ); } // check if we read all the data and if the error count is OK if( len != 0 || nerrs != 0 ) return Status( stError, errDataError ); // set up the response object response = new AnyObject(); response->Set( new std::vector( std::move( resp ) ) ); return Status(); } case kXR_fattrList: { Status status; std::vector resp; while( len > 0 ) { std::string name; if( !( status = ReadFromBuffer( data, len, name ) ).IsOK() ) return status; kXR_int32 vlen = 0; if( !( status = ReadFromBuffer( data, len, vlen ) ).IsOK() ) return status; vlen = ntohl( vlen ); std::string value; if( !( status = ReadFromBuffer( data, len, vlen, value ) ).IsOK() ) return status; resp.push_back( XAttr( name, value ) ); } // set up the response object response = new AnyObject(); response->Set( new std::vector( std::move( resp ) ) ); return Status(); } default: return Status( stError, errDataError ); } } //---------------------------------------------------------------------------- // Perform the changes to the original request needed by the redirect // procedure - allocate new streamid, append redirection data and such //---------------------------------------------------------------------------- Status XRootDMsgHandler::RewriteRequestRedirect( const URL &newUrl ) { Log *log = DefaultEnv::GetLog(); Status st; // Append any "xrd.*" parameters present in newCgi so that any authentication // requirements are properly enforced const URL::ParamsMap &newCgi = newUrl.GetParams(); std::string xrdCgi = ""; std::ostringstream ossXrd; for(URL::ParamsMap::const_iterator it = newCgi.begin(); it != newCgi.end(); ++it ) { if( it->first.compare( 0, 4, "xrd." ) ) continue; ossXrd << it->first << '=' << it->second << '&'; } xrdCgi = ossXrd.str(); // Redirection URL containing also any original xrd.* opaque parameters XrdCl::URL authUrl; if (xrdCgi.empty()) { authUrl = newUrl; } else { std::string surl = newUrl.GetURL(); (surl.find('?') == std::string::npos) ? (surl += '?') : ((*surl.rbegin() != '&') ? (surl += '&') : (surl += "")); surl += xrdCgi; if (!authUrl.FromString(surl)) { log->Error( XRootDMsg, "[%s] Failed to build redirection URL from data:" "%s", surl.c_str()); return Status(stError, errInvalidRedirectURL); } } //-------------------------------------------------------------------------- // Rewrite particular requests //-------------------------------------------------------------------------- XRootDTransport::UnMarshallRequest( pRequest ); MessageUtils::RewriteCGIAndPath( pRequest, newCgi, true, newUrl.GetPath() ); XRootDTransport::MarshallRequest( pRequest ); return Status(); } //---------------------------------------------------------------------------- // Some requests need to be rewritten also after getting kXR_wait //---------------------------------------------------------------------------- Status XRootDMsgHandler::RewriteRequestWait() { ClientRequest *req = (ClientRequest *)pRequest->GetBuffer(); XRootDTransport::UnMarshallRequest( pRequest ); //------------------------------------------------------------------------ // For kXR_locate and kXR_open request the kXR_refresh bit needs to be // turned off after wait //------------------------------------------------------------------------ switch( req->header.requestid ) { case kXR_locate: { uint16_t refresh = kXR_refresh; req->locate.options &= (~refresh); break; } case kXR_open: { uint16_t refresh = kXR_refresh; req->locate.options &= (~refresh); break; } } XRootDTransport::SetDescription( pRequest ); XRootDTransport::MarshallRequest( pRequest ); return Status(); } //---------------------------------------------------------------------------- // Recover error //---------------------------------------------------------------------------- void XRootDMsgHandler::HandleError( XRootDStatus status ) { //-------------------------------------------------------------------------- // If there was no error then do nothing //-------------------------------------------------------------------------- if( status.IsOK() ) return; if( pSidMgr && pMsgInFly && ( status.code == errOperationExpired || status.code == errOperationInterrupted ) ) { ClientRequest *req = (ClientRequest *)pRequest->GetBuffer(); pSidMgr->TimeOutSID( req->header.streamid ); } bool noreplicas = ( status.code == errErrorResponse && status.errNo == kXR_noReplicas ); if( !noreplicas ) pLastError = status; Log *log = DefaultEnv::GetLog(); log->Debug( XRootDMsg, "[%s] Handling error while processing %s: %s.", pUrl.GetHostId().c_str(), pRequest->GetDescription().c_str(), status.ToString().c_str() ); //-------------------------------------------------------------------------- // Check if it is a fatal TLS error that has been marked as potentially // recoverable, if yes check if we can downgrade from fatal to error. //-------------------------------------------------------------------------- if( status.IsFatal() && status.code == errTlsError && status.errNo == EAGAIN ) { if( pSslErrCnt < MaxSslErrRetry ) { status.status &= ~stFatal; // switch off fatal&error bits status.status |= stError; // switch on error bit } ++pSslErrCnt; // count number of consecutive SSL errors } else pSslErrCnt = 0; //-------------------------------------------------------------------------- // We have got an error message, we can recover it at the load balancer if: // 1) we haven't got it from the load balancer // 2) we have a load balancer assigned // 3) the error is either one of: kXR_FSError, kXR_IOError, kXR_ServerError, // kXR_NotFound // 4) in the case of kXR_NotFound a kXR_refresh flags needs to be set //-------------------------------------------------------------------------- if( status.code == errErrorResponse ) { if( RetriableErrorResponse( status ) ) { UpdateTriedCGI(status.errNo); if( status.errNo == kXR_NotFound || status.errNo == kXR_Overloaded ) SwitchOnRefreshFlag(); HandleError( RetryAtServer( pLoadBalancer.url, RedirectEntry::EntryRetry ) ); return; } else { pStatus = status; HandleRspOrQueue(); return; } } //-------------------------------------------------------------------------- // Nothing can be done if: // 1) a user timeout has occurred // 2) has a non-zero session id // 3) if another error occurred and the validity of the message expired //-------------------------------------------------------------------------- if( status.code == errOperationExpired || pRequest->GetSessionId() || status.code == errOperationInterrupted || time(0) >= pExpiration ) { log->Error( XRootDMsg, "[%s] Unable to get the response to request %s", pUrl.GetHostId().c_str(), pRequest->GetDescription().c_str() ); pStatus = status; HandleRspOrQueue(); return; } //-------------------------------------------------------------------------- // At this point we're left with connection errors, we recover them // at a load balancer if we have one and if not on the current server // until we get a response, an unrecoverable error or a timeout //-------------------------------------------------------------------------- if( pLoadBalancer.url.IsValid() && pLoadBalancer.url.GetLocation() != pUrl.GetLocation() ) { UpdateTriedCGI( kXR_ServerError ); HandleError( RetryAtServer( pLoadBalancer.url, RedirectEntry::EntryRetry ) ); return; } else { if( !status.IsFatal() && IsRetriable() ) { log->Info( XRootDMsg, "[%s] Retrying request: %s.", pUrl.GetHostId().c_str(), pRequest->GetDescription().c_str() ); UpdateTriedCGI( kXR_ServerError ); HandleError( RetryAtServer( pUrl, RedirectEntry::EntryRetry ) ); return; } pStatus = status; HandleRspOrQueue(); return; } } //---------------------------------------------------------------------------- // Retry the message at another server //---------------------------------------------------------------------------- Status XRootDMsgHandler::RetryAtServer( const URL &url, RedirectEntry::Type entryType ) { pResponse.reset(); Log *log = DefaultEnv::GetLog(); //-------------------------------------------------------------------------- // Set up a redirect entry //-------------------------------------------------------------------------- if( pRdirEntry ) pRedirectTraceBack.push_back( std::move( pRdirEntry ) ); pRdirEntry.reset( new RedirectEntry( pUrl.GetLocation(), url.GetLocation(), entryType ) ); if( pUrl.GetLocation() != url.GetLocation() ) { pHosts->push_back( url ); //------------------------------------------------------------------------ // Assign a new stream id to the message //------------------------------------------------------------------------ // first release the old stream id // (though it could be a redirect from a local // metalink file, in this case there's no SID) ClientRequestHdr *req = (ClientRequestHdr*)pRequest->GetBuffer(); if( pSidMgr ) { pSidMgr->ReleaseSID( req->streamid ); pSidMgr.reset(); } // then get the new SIDManager // (again this could be a redirect to a local // file and in this case there is no SID) if( !url.IsLocalFile() ) { pSidMgr = SIDMgrPool::Instance().GetSIDMgr( url ); Status st = pSidMgr->AllocateSID( req->streamid ); if( !st.IsOK() ) { log->Error( XRootDMsg, "[%s] Impossible to send message %s.", pUrl.GetHostId().c_str(), pRequest->GetDescription().c_str() ); return st; } } pUrl = url; } if( pUrl.IsMetalink() && pFollowMetalink ) { log->Debug( ExDbgMsg, "[%s] Metaling redirection for MsgHandler: 0x%x (message: %s ).", pUrl.GetHostId().c_str(), this, pRequest->GetDescription().c_str() ); return pPostMaster->Redirect( pUrl, pRequest, this ); } else if( pUrl.IsLocalFile() ) { HandleLocalRedirect( &pUrl ); return Status(); } else { log->Debug( ExDbgMsg, "[%s] Retry at server MsgHandler: 0x%x (message: %s ).", pUrl.GetHostId().c_str(), this, pRequest->GetDescription().c_str() ); return pPostMaster->Send( pUrl, pRequest, this, true, pExpiration ); } } //---------------------------------------------------------------------------- // Update the "tried=" part of the CGI of the current message //---------------------------------------------------------------------------- void XRootDMsgHandler::UpdateTriedCGI(uint32_t errNo) { URL::ParamsMap cgi; std::string tried; //-------------------------------------------------------------------------- // In case a data server responded with a kXR_redirect and we fail at the // node where we were redirected to, the original data server should be // included in the tried CGI opaque info (instead of the current one). //-------------------------------------------------------------------------- if( pEffectiveDataServerUrl ) { tried = pEffectiveDataServerUrl->GetHostName(); delete pEffectiveDataServerUrl; pEffectiveDataServerUrl = 0; } //-------------------------------------------------------------------------- // Otherwise use the current URL. //-------------------------------------------------------------------------- else tried = pUrl.GetHostName(); // Report the reason for the failure to the next location // if (errNo) { if (errNo == kXR_NotFound) cgi["triedrc"] = "enoent"; else if (errNo == kXR_IOError) cgi["triedrc"] = "ioerr"; else if (errNo == kXR_FSError) cgi["triedrc"] = "fserr"; else if (errNo == kXR_ServerError) cgi["triedrc"] = "srverr"; } //-------------------------------------------------------------------------- // If our current load balancer is a metamanager and we failed either // at a diskserver or at an unidentified node we also exclude the last // known manager //-------------------------------------------------------------------------- if( pLoadBalancer.url.IsValid() && (pLoadBalancer.flags & kXR_attrMeta) ) { HostList::reverse_iterator it; for( it = pHosts->rbegin()+1; it != pHosts->rend(); ++it ) { if( it->loadBalancer ) break; tried += "," + it->url.GetHostName(); if( it->flags & kXR_isManager ) break; } } cgi["tried"] = tried; XRootDTransport::UnMarshallRequest( pRequest ); MessageUtils::RewriteCGIAndPath( pRequest, cgi, false, "" ); XRootDTransport::MarshallRequest( pRequest ); } //---------------------------------------------------------------------------- // Switch on the refresh flag for some requests //---------------------------------------------------------------------------- void XRootDMsgHandler::SwitchOnRefreshFlag() { XRootDTransport::UnMarshallRequest( pRequest ); ClientRequest *req = (ClientRequest *)pRequest->GetBuffer(); switch( req->header.requestid ) { case kXR_locate: { req->locate.options |= kXR_refresh; break; } case kXR_open: { req->locate.options |= kXR_refresh; break; } } XRootDTransport::SetDescription( pRequest ); XRootDTransport::MarshallRequest( pRequest ); } //------------------------------------------------------------------------ // If the current thread is a worker thread from our thread-pool // handle the response, otherwise submit a new task to the thread-pool //------------------------------------------------------------------------ void XRootDMsgHandler::HandleRspOrQueue() { JobManager *jobMgr = pPostMaster->GetJobManager(); if( jobMgr->IsWorker() ) HandleResponse(); else { Log *log = DefaultEnv::GetLog(); log->Debug( ExDbgMsg, "[%s] Passing to the thread-pool MsgHandler: 0x%x (message: %s ).", pUrl.GetHostId().c_str(), this, pRequest->GetDescription().c_str() ); jobMgr->QueueJob( new HandleRspJob( this ), 0 ); } } //------------------------------------------------------------------------ // Notify the FileStateHandler to retry Open() with new URL //------------------------------------------------------------------------ void XRootDMsgHandler::HandleLocalRedirect( URL *url ) { Log *log = DefaultEnv::GetLog(); log->Debug( ExDbgMsg, "[%s] Handling local redirect - MsgHandler: 0x%x (message: %s ).", pUrl.GetHostId().c_str(), this, pRequest->GetDescription().c_str() ); if( !pLFileHandler ) { HandleError( XRootDStatus( stFatal, errNotSupported ) ); return; } AnyObject *resp = 0; pLFileHandler->SetHostList( *pHosts ); XRootDStatus st = pLFileHandler->Open( url, pRequest, resp ); if( !st.IsOK() ) { HandleError( st ); return; } pResponseHandler->HandleResponseWithHosts( new XRootDStatus(), resp, pHosts.release() ); delete this; return; } //------------------------------------------------------------------------ // Check if it is OK to retry this request //------------------------------------------------------------------------ bool XRootDMsgHandler::IsRetriable() { std::string value; DefaultEnv::GetEnv()->GetString( "OpenRecovery", value ); if( value == "true" ) return true; // check if it is a mutable open (open + truncate or open + create) ClientRequest *req = reinterpret_cast( pRequest->GetBuffer() ); if( req->header.requestid == htons( kXR_open ) ) { bool _mutable = ( req->open.options & htons( kXR_delete ) ) || ( req->open.options & htons( kXR_new ) ); if( _mutable ) { Log *log = DefaultEnv::GetLog(); log->Debug( XRootDMsg, "[%s] Not allowed to retry open request (OpenRecovery disabled): %s.", pUrl.GetHostId().c_str(), pRequest->GetDescription().c_str() ); // disallow retry if it is a mutable open return false; } } return true; } //------------------------------------------------------------------------ // Check if for given request and Metalink redirector it is OK to omit // the kXR_wait and proceed straight to the next entry in the Metalink file //------------------------------------------------------------------------ bool XRootDMsgHandler::OmitWait( Message &request, const URL &url ) { // we can omit kXR_wait only if we have a Metalink redirector if( !url.IsMetalink() ) return false; // we can omit kXR_wait only for requests that can be redirected // (kXR_read is the only stateful request that can be redirected) ClientRequest *req = reinterpret_cast( request.GetBuffer() ); if( pStateful && req->header.requestid != kXR_read ) return false; // we can only omit kXR_wait if the Metalink redirect has more // replicas RedirectorRegistry ®istry = RedirectorRegistry::Instance(); VirtualRedirector *redirector = registry.Get( url ); // we need more than one server as the current one is not reflected // in tried CGI if( redirector->Count( request ) > 1 ) return true; return false; } //------------------------------------------------------------------------ // Checks if the given error returned by server is retriable. //------------------------------------------------------------------------ bool XRootDMsgHandler::RetriableErrorResponse( const Status &status ) { // we can only retry error response if we have a valid load-balancer and // it is not our current URL if( !( pLoadBalancer.url.IsValid() && pUrl.GetLocation() != pLoadBalancer.url.GetLocation() ) ) return false; // following errors are retriable at any load-balancer if( status.errNo == kXR_FSError || status.errNo == kXR_IOError || status.errNo == kXR_ServerError || status.errNo == kXR_NotFound || status.errNo == kXR_Overloaded || status.errNo == kXR_NoMemory ) return true; // check if the load-balancer is a meta-manager, if yes there are // more errors that can be recovered if( !( pLoadBalancer.flags & kXR_attrMeta ) ) return false; // those errors are retriable for meta-managers if( status.errNo == kXR_Unsupported || status.errNo == kXR_FileLocked ) return true; // in case of not-authorized error there is an imposed upper limit // on how many times we can retry this error if( status.errNo == kXR_NotAuthorized ) { int limit = DefaultNotAuthorizedRetryLimit; DefaultEnv::GetEnv()->GetInt( "NotAuthorizedRetryLimit", limit ); bool ret = pNotAuthorizedCounter < limit; ++pNotAuthorizedCounter; if( !ret ) { Log *log = DefaultEnv::GetLog(); log->Error( XRootDMsg, "[%s] Reached limit of NotAuthorized retries!", pUrl.GetHostId().c_str() ); } return ret; } // check if the load-balancer is a virtual (metalink) redirector, // if yes there are even more errors that can be recovered if( !( pLoadBalancer.flags & kXR_attrVirtRdr ) ) return false; // those errors are retriable for virtual (metalink) redirectors if( status.errNo == kXR_noserver || status.errNo == kXR_ArgTooLong ) return true; // otherwise it is a non-retriable error return false; } //------------------------------------------------------------------------ // Dump the redirect-trace-back into the log file //------------------------------------------------------------------------ void XRootDMsgHandler::DumpRedirectTraceBack() { if( pRedirectTraceBack.empty() ) return; std::stringstream sstrm; sstrm << "Redirect trace-back:\n"; int counter = 0; auto itr = pRedirectTraceBack.begin(); sstrm << '\t' << counter << ". " << (*itr)->ToString() << '\n'; auto prev = itr; ++itr; ++counter; for( ; itr != pRedirectTraceBack.end(); ++itr, ++prev, ++counter ) sstrm << '\t' << counter << ". " << (*itr)->ToString( (*prev)->status.IsOK() ) << '\n'; int authlimit = DefaultNotAuthorizedRetryLimit; DefaultEnv::GetEnv()->GetInt( "NotAuthorizedRetryLimit", authlimit ); bool warn = !pStatus.IsOK() && ( pStatus.code == errNotFound || pStatus.code == errRedirectLimit || ( pStatus.code == errAuthFailed && pNotAuthorizedCounter >= authlimit ) ); Log *log = DefaultEnv::GetLog(); if( warn ) log->Warning( XRootDMsg, sstrm.str().c_str() ); else log->Debug( XRootDMsg, sstrm.str().c_str() ); } // Read data from buffer //------------------------------------------------------------------------ template Status XRootDMsgHandler::ReadFromBuffer( char *&buffer, size_t &buflen, T& result ) { if( sizeof( T ) > buflen ) return Status( stError, errDataError ); memcpy(&result, buffer, sizeof(T)); buffer += sizeof( T ); buflen -= sizeof( T ); return Status(); } //------------------------------------------------------------------------ // Read a string from buffer //------------------------------------------------------------------------ Status XRootDMsgHandler::ReadFromBuffer( char *&buffer, size_t &buflen, std::string &result ) { Status status; char c = 0; while( true ) { if( !( status = ReadFromBuffer( buffer, buflen, c ) ).IsOK() ) return status; if( c == 0 ) break; result += c; } return status; } //------------------------------------------------------------------------ // Read a string from buffer //------------------------------------------------------------------------ Status XRootDMsgHandler::ReadFromBuffer( char *&buffer, size_t &buflen, size_t size, std::string &result ) { Status status; if( size > buflen ) return Status( stError, errDataError ); result.append( buffer, size ); buffer += size; buflen -= size; return status; } } xrootd-5.6.9/src/XrdCl/XrdClXRootDMsgHandler.hh000066400000000000000000000700261457266313600213010ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2014 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #ifndef __XRD_CL_XROOTD_MSG_HANDLER_HH__ #define __XRD_CL_XROOTD_MSG_HANDLER_HH__ #include "XrdCl/XrdClPostMasterInterfaces.hh" #include "XrdCl/XrdClXRootDResponses.hh" #include "XrdCl/XrdClDefaultEnv.hh" #include "XrdCl/XrdClMessage.hh" #include "XProtocol/XProtocol.hh" #include "XrdCl/XrdClLog.hh" #include "XrdCl/XrdClConstants.hh" #include "XrdCl/XrdClAsyncPageReader.hh" #include "XrdCl/XrdClAsyncVectorReader.hh" #include "XrdCl/XrdClAsyncRawReader.hh" #include "XrdCl/XrdClAsyncDiscardReader.hh" #include "XrdClAsyncRawReaderIntfc.hh" #include "XrdSys/XrdSysPthread.hh" #include "XrdSys/XrdSysPageSize.hh" #include "XrdSys/XrdSysKernelBuffer.hh" #include "XrdSys/XrdSysPlatform.hh" #include "XrdOuc/XrdOucPgrwUtils.hh" #include #include // for network unmarshaling stuff #include #include #include #include #include namespace XrdCl { class PostMaster; class SIDManager; class URL; class LocalFileHandler; class Socket; //---------------------------------------------------------------------------- // Single entry in the redirect-trace-back //---------------------------------------------------------------------------- struct RedirectEntry { enum Type { EntryRedirect, EntryRedirectOnWait, EntryRetry, EntryWait }; RedirectEntry( const URL &from, const URL &to, Type type ) : from( from ), to( to ), type( type ) { } URL from; URL to; Type type; XRootDStatus status; std::string ToString( bool prevok = true ) { const std::string tostr = to.GetLocation(); const std::string fromstr = from.GetLocation(); if( prevok ) { switch( type ) { case EntryRedirect: return "Redirected from: " + fromstr + " to: " + tostr; case EntryRedirectOnWait: return "Server responded with wait. " "Falling back to virtual redirector: " + tostr; case EntryRetry: return "Retrying: " + tostr; case EntryWait: return "Waited at server request. Resending: " + tostr; } } return "Failed at: " + fromstr + ", retrying at: " + tostr; } }; //---------------------------------------------------------------------------- //! Handle/Process/Forward XRootD messages //---------------------------------------------------------------------------- class XRootDMsgHandler: public MsgHandler { friend class HandleRspJob; public: //------------------------------------------------------------------------ //! Constructor //! //! @param msg message that has been sent out //! @param respHandler response handler to be called then the final //! final response arrives //! @param url the url the message has been sent to //! @param sidMgr the sid manager used to allocate SID for the initial //! message //------------------------------------------------------------------------ XRootDMsgHandler( Message *msg, ResponseHandler *respHandler, const URL *url, std::shared_ptr sidMgr, LocalFileHandler *lFileHandler): pRequest( msg ), pResponseHandler( respHandler ), pUrl( *url ), pEffectiveDataServerUrl( 0 ), pSidMgr( sidMgr ), pLFileHandler( lFileHandler ), pExpiration( 0 ), pRedirectAsAnswer( false ), pOksofarAsAnswer( false ), pHasLoadBalancer( false ), pHasSessionId( false ), pChunkList( 0 ), pKBuff( 0 ), pRedirectCounter( 0 ), pNotAuthorizedCounter( 0 ), pAsyncOffset( 0 ), pAsyncChunkIndex( 0 ), pPgWrtCksumBuff( 4 ), pPgWrtCurrentPageOffset( 0 ), pPgWrtCurrentPageNb( 0 ), pOtherRawStarted( false ), pFollowMetalink( false ), pStateful( false ), pAggregatedWaitTime( 0 ), pMsgInFly( false ), pTimeoutFence( false ), pDirListStarted( false ), pDirListWithStat( false ), pCV( 0 ), pSslErrCnt( 0 ) { pPostMaster = DefaultEnv::GetPostMaster(); if( msg->GetSessionId() ) pHasSessionId = true; Log *log = DefaultEnv::GetLog(); log->Debug( ExDbgMsg, "[%s] MsgHandler created: 0x%x (message: %s ).", pUrl.GetHostId().c_str(), this, pRequest->GetDescription().c_str() ); ClientRequestHdr *hdr = (ClientRequestHdr*)pRequest->GetBuffer(); if( ntohs( hdr->requestid ) == kXR_pgread ) { ClientPgReadRequest *pgrdreq = (ClientPgReadRequest*)pRequest->GetBuffer(); pCrc32cDigests.reserve( XrdOucPgrwUtils::csNum( ntohll( pgrdreq->offset ), ntohl( pgrdreq->rlen ) ) ); } if( ntohs( hdr->requestid ) == kXR_readv ) pBodyReader.reset( new AsyncVectorReader( *url, *pRequest ) ); else if( ntohs( hdr->requestid ) == kXR_read ) pBodyReader.reset( new AsyncRawReader( *url, *pRequest ) ); else pBodyReader.reset( new AsyncDiscardReader( *url, *pRequest ) ); } //------------------------------------------------------------------------ //! Destructor //------------------------------------------------------------------------ ~XRootDMsgHandler() { DumpRedirectTraceBack(); if( !pHasSessionId ) delete pRequest; delete pEffectiveDataServerUrl; pRequest = reinterpret_cast( 0xDEADBEEF ); pResponseHandler = reinterpret_cast( 0xDEADBEEF ); pPostMaster = reinterpret_cast( 0xDEADBEEF ); pLFileHandler = reinterpret_cast( 0xDEADBEEF ); pChunkList = reinterpret_cast( 0xDEADBEEF ); pEffectiveDataServerUrl = reinterpret_cast( 0xDEADBEEF ); Log *log = DefaultEnv::GetLog(); log->Debug( ExDbgMsg, "[%s] Destroying MsgHandler: 0x%x.", pUrl.GetHostId().c_str(), this ); } //------------------------------------------------------------------------ //! Examine an incoming message, and decide on the action to be taken //! //! @param msg the message, may be zero if receive failed //! @return action type that needs to be take wrt the message and //! the handler //------------------------------------------------------------------------ virtual uint16_t Examine( std::shared_ptr &msg ); //------------------------------------------------------------------------ //! Reexamine the incoming message, and decide on the action to be taken //! //! In case of kXR_status the message can be only fully examined after //! reading the whole body (without raw data). //! //! @param msg the message, may be zero if receive failed //! @return action type that needs to be take wrt the message and //! the handler //------------------------------------------------------------------------ virtual uint16_t InspectStatusRsp(); //------------------------------------------------------------------------ //! Get handler sid //! //! return sid of the corresponding request, otherwise 0 //------------------------------------------------------------------------ virtual uint16_t GetSid() const; //------------------------------------------------------------------------ //! Process the message if it was "taken" by the examine action //! //! @param msg the message to be processed //------------------------------------------------------------------------ virtual void Process(); //------------------------------------------------------------------------ //! Read message body directly from a socket - called if Examine returns //! Raw flag - only socket related errors may be returned here //! //! @param msg the corresponding message header //! @param socket the socket to read from //! @param bytesRead number of bytes read by the method //! @return stOK & suDone if the whole body has been processed //! stOK & suRetry if more data is needed //! stError on failure //------------------------------------------------------------------------ virtual XRootDStatus ReadMessageBody( Message *msg, Socket *socket, uint32_t &bytesRead ); //------------------------------------------------------------------------ //! Handle an event other that a message arrival //! //! @param event type of the event //! @param status status info //------------------------------------------------------------------------ virtual uint8_t OnStreamEvent( StreamEvent event, XRootDStatus status ); //------------------------------------------------------------------------ //! The requested action has been performed and the status is available //------------------------------------------------------------------------ virtual void OnStatusReady( const Message *message, XRootDStatus status ); //------------------------------------------------------------------------ //! Are we a raw writer or not? //------------------------------------------------------------------------ virtual bool IsRaw() const; //------------------------------------------------------------------------ //! Write message body directly to a socket - called if IsRaw returns //! true - only socket related errors may be returned here //! //! @param socket the socket to read from //! @param bytesWritten number of bytes written by the method //! @return stOK & suDone if the whole body has been processed //! stOK & suRetry if more data needs to be written //! stError on failure //------------------------------------------------------------------------ XRootDStatus WriteMessageBody( Socket *socket, uint32_t &bytesWritten ); //------------------------------------------------------------------------ //! Called after the wait time for kXR_wait has elapsed //! //! @param now current timestamp //------------------------------------------------------------------------ void WaitDone( time_t now ); //------------------------------------------------------------------------ //! Set a timestamp after which we give up //------------------------------------------------------------------------ void SetExpiration( time_t expiration ) { pExpiration = expiration; } //------------------------------------------------------------------------ //! Get a timestamp after which we give up //------------------------------------------------------------------------ time_t GetExpiration() { return pExpiration; } //------------------------------------------------------------------------ //! Treat the kXR_redirect response as a valid answer to the message //! and notify the handler with the URL as a response //------------------------------------------------------------------------ void SetRedirectAsAnswer( bool redirectAsAnswer ) { pRedirectAsAnswer = redirectAsAnswer; } //------------------------------------------------------------------------ //! Treat the kXR_oksofar response as a valid answer to the message //! and notify the handler with the URL as a response //------------------------------------------------------------------------ void SetOksofarAsAnswer( bool oksofarAsAnswer ) { pOksofarAsAnswer = oksofarAsAnswer; } //------------------------------------------------------------------------ //! Get the request pointer //------------------------------------------------------------------------ const Message *GetRequest() const { return pRequest; } //------------------------------------------------------------------------ //! Set the load balancer //------------------------------------------------------------------------ void SetLoadBalancer( const HostInfo &loadBalancer ) { if( !loadBalancer.url.IsValid() ) return; pLoadBalancer = loadBalancer; pHasLoadBalancer = true; } //------------------------------------------------------------------------ //! Set host list //------------------------------------------------------------------------ void SetHostList( HostList *hostList ) { pHosts.reset( hostList ); } //------------------------------------------------------------------------ //! Set the chunk list //------------------------------------------------------------------------ void SetChunkList( ChunkList *chunkList ) { pChunkList = chunkList; if( pBodyReader ) pBodyReader->SetChunkList( chunkList ); if( chunkList ) pChunkStatus.resize( chunkList->size() ); else pChunkStatus.clear(); } void SetCrc32cDigests( std::vector && crc32cDigests ) { pCrc32cDigests = std::move( crc32cDigests ); } //------------------------------------------------------------------------ //! Set the kernel buffer //------------------------------------------------------------------------ void SetKernelBuffer( XrdSys::KernelBuffer *kbuff ) { pKBuff = kbuff; } //------------------------------------------------------------------------ //! Set the redirect counter //------------------------------------------------------------------------ void SetRedirectCounter( uint16_t redirectCounter ) { pRedirectCounter = redirectCounter; } void SetFollowMetalink( bool followMetalink ) { pFollowMetalink = followMetalink; } void SetStateful( bool stateful ) { pStateful = stateful; } //------------------------------------------------------------------------ //! Bookkeeping after partial response has been received: //! - take down the timeout fence after oksofar response has been handled //! - reset status-response-body marshaled flag //------------------------------------------------------------------------ void PartialReceived(); private: //------------------------------------------------------------------------ //! Recover error //------------------------------------------------------------------------ void HandleError( XRootDStatus status ); //------------------------------------------------------------------------ //! Retry the request at another server //------------------------------------------------------------------------ Status RetryAtServer( const URL &url, RedirectEntry::Type entryType ); //------------------------------------------------------------------------ //! Unpack the message and call the response handler //------------------------------------------------------------------------ void HandleResponse(); //------------------------------------------------------------------------ //! Extract the status information from the stuff that we got //------------------------------------------------------------------------ XRootDStatus *ProcessStatus(); //------------------------------------------------------------------------ //! Parse the response and put it in an object that could be passed to //! the user //------------------------------------------------------------------------ Status ParseResponse( AnyObject *&response ); //------------------------------------------------------------------------ //! Parse the response to kXR_fattr request and put it in an object that //! could be passed to the user //------------------------------------------------------------------------ Status ParseXAttrResponse( char *data, size_t len, AnyObject *&response ); //------------------------------------------------------------------------ //! Perform the changes to the original request needed by the redirect //! procedure - allocate new streamid, append redirection data and such //------------------------------------------------------------------------ Status RewriteRequestRedirect( const URL &newUrl ); //------------------------------------------------------------------------ //! Some requests need to be rewritten also after getting kXR_wait - sigh //------------------------------------------------------------------------ Status RewriteRequestWait(); //------------------------------------------------------------------------ //! Update the "tried=" part of the CGI of the current message //------------------------------------------------------------------------ void UpdateTriedCGI(uint32_t errNo=0); //------------------------------------------------------------------------ //! Switch on the refresh flag for some requests //------------------------------------------------------------------------ void SwitchOnRefreshFlag(); //------------------------------------------------------------------------ //! If the current thread is a worker thread from our thread-pool //! handle the response, otherwise submit a new task to the thread-pool //------------------------------------------------------------------------ void HandleRspOrQueue(); //------------------------------------------------------------------------ //! Handle a redirect to a local file //------------------------------------------------------------------------ void HandleLocalRedirect( URL *url ); //------------------------------------------------------------------------ //! Check if it is OK to retry this request //! //! @param reuqest : the request in question //! @return : true if yes, false if no //------------------------------------------------------------------------ bool IsRetriable(); //------------------------------------------------------------------------ //! Check if for given request and Metalink redirector it is OK to omit //! the kXR_wait and proceed stright to the next entry in the Metalink file //! //! @param request : the request in question //! @param url : metalink URL //! @return : true if yes, false if no //------------------------------------------------------------------------ bool OmitWait( Message &request, const URL &url ); //------------------------------------------------------------------------ //! Checks if the given error returned by server is retriable. //! //! @param status : the status returned by the server //! @return : true if the load-balancer is a MetaManager and the //! error is retriable for MetaManagers //------------------------------------------------------------------------ bool RetriableErrorResponse( const Status &status ); //------------------------------------------------------------------------ //! Dump the redirect-trace-back into the log file //------------------------------------------------------------------------ void DumpRedirectTraceBack(); //! Read data from buffer //! //! @param buffer : the buffer with data //! @param buflen : the size of the buffer //! @param result : output parameter (data read) //! @return : status of the operation //------------------------------------------------------------------------ template Status ReadFromBuffer( char *&buffer, size_t &buflen, T& result ); //------------------------------------------------------------------------ //! Read a string from buffer //! //! @param buffer : the buffer with data //! @param buflen : the size of the buffer //! @param result : output parameter (data read) //! @return : status of the operation //------------------------------------------------------------------------ Status ReadFromBuffer( char *&buffer, size_t &buflen, std::string &result ); //------------------------------------------------------------------------ //! Read a string from buffer //! //! @param buffer : the buffer with data //! @param buflen : size of the buffer //! @param size : size of the data to read //! @param result : output parameter (data read) //! @return : status of the operation //------------------------------------------------------------------------ Status ReadFromBuffer( char *&buffer, size_t &buflen, size_t size, std::string &result ); //------------------------------------------------------------------------ // Helper struct for async reading of chunks //------------------------------------------------------------------------ struct ChunkStatus { ChunkStatus(): sizeError( false ), done( false ) {} bool sizeError; bool done; }; typedef std::list> RedirectTraceBack; static const size_t CksumSize = sizeof( uint32_t ); static const size_t PageWithCksum = XrdSys::PageSize + CksumSize; static const size_t MaxSslErrRetry = 3; inline static size_t NbPgPerRsp( uint64_t offset, uint32_t dlen ) { uint32_t pgcnt = 0; uint32_t remainder = offset % XrdSys::PageSize; if( remainder > 0 ) { // account for the first unaligned page ++pgcnt; // the size of the 1st unaligned page uint32_t _1stpg = XrdSys::PageSize - remainder; if( _1stpg + CksumSize > dlen ) _1stpg = dlen - CksumSize; dlen -= _1stpg + CksumSize; } pgcnt += dlen / PageWithCksum; if( dlen % PageWithCksum ) ++ pgcnt; return pgcnt; } Message *pRequest; std::shared_ptr pResponse; //< the ownership is shared with MsgReader std::vector> pPartialResps; //< the ownership is shared with MsgReader ResponseHandler *pResponseHandler; URL pUrl; URL *pEffectiveDataServerUrl; PostMaster *pPostMaster; std::shared_ptr pSidMgr; LocalFileHandler *pLFileHandler; XRootDStatus pStatus; Status pLastError; time_t pExpiration; bool pRedirectAsAnswer; bool pOksofarAsAnswer; std::unique_ptr pHosts; bool pHasLoadBalancer; HostInfo pLoadBalancer; bool pHasSessionId; std::string pRedirectUrl; ChunkList *pChunkList; std::vector pCrc32cDigests; XrdSys::KernelBuffer *pKBuff; std::vector pChunkStatus; uint16_t pRedirectCounter; uint16_t pNotAuthorizedCounter; uint32_t pAsyncOffset; uint32_t pAsyncChunkIndex; std::unique_ptr pPageReader; std::unique_ptr pBodyReader; Buffer pPgWrtCksumBuff; uint32_t pPgWrtCurrentPageOffset; uint32_t pPgWrtCurrentPageNb; bool pOtherRawStarted; bool pFollowMetalink; bool pStateful; int pAggregatedWaitTime; std::unique_ptr pRdirEntry; RedirectTraceBack pRedirectTraceBack; bool pMsgInFly; //------------------------------------------------------------------------ // true if MsgHandler is both in inQueue and installed in respective // Stream (this could happen if server gave oksofar response), otherwise // false //------------------------------------------------------------------------ std::atomic pTimeoutFence; //------------------------------------------------------------------------ // if we are serving chunked data to the user's handler in case of // kXR_dirlist we need to memorize if the response contains stat info or // not (the information is only encoded in the first chunk) //------------------------------------------------------------------------ bool pDirListStarted; bool pDirListWithStat; //------------------------------------------------------------------------ // synchronization is needed in case the MsgHandler has been configured // to serve kXR_oksofar as a response to the user's handler //------------------------------------------------------------------------ XrdSysCondVar pCV; //------------------------------------------------------------------------ // Count of consecutive `errTlsSslError` errors //------------------------------------------------------------------------ size_t pSslErrCnt; }; } #endif // __XRD_CL_XROOTD_MSG_HANDLER_HH__ xrootd-5.6.9/src/XrdCl/XrdClXRootDResponses.cc000066400000000000000000000641701457266313600212270ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2012 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . //------------------------------------------------------------------------------ #include "XrdCl/XrdClXRootDResponses.hh" #include "XrdCl/XrdClLog.hh" #include "XrdCl/XrdClDefaultEnv.hh" #include "XrdCl/XrdClConstants.hh" #include "XrdCl/XrdClUtils.hh" #include namespace XrdCl { //---------------------------------------------------------------------------- // LocationInfo constructor //---------------------------------------------------------------------------- LocationInfo::LocationInfo() { } //---------------------------------------------------------------------------- // Parse the server location response //---------------------------------------------------------------------------- bool LocationInfo::ParseServerResponse( const char *data ) { if( !data || strlen( data ) == 0 ) return false; std::vector locations; std::vector::iterator it; Utils::splitString( locations, data, " " ); for( it = locations.begin(); it != locations.end(); ++it ) if( ProcessLocation( *it ) == false ) return false; return true; } //---------------------------------------------------------------------------- // Process location //---------------------------------------------------------------------------- bool LocationInfo::ProcessLocation( std::string &location ) { if( location.length() < 5 ) return false; //-------------------------------------------------------------------------- // Decode location type //-------------------------------------------------------------------------- LocationInfo::LocationType type; switch( location[0] ) { case 'M': type = LocationInfo::ManagerOnline; break; case 'm': type = LocationInfo::ManagerPending; break; case 'S': type = LocationInfo::ServerOnline; break; case 's': type = LocationInfo::ServerPending; break; default: return false; } //-------------------------------------------------------------------------- // Decode access type //-------------------------------------------------------------------------- LocationInfo::AccessType access; switch( location[1] ) { case 'r': access = LocationInfo::Read; break; case 'w': access = LocationInfo::ReadWrite; break; default: return false; } //-------------------------------------------------------------------------- // Push the location info //-------------------------------------------------------------------------- pLocations.push_back( Location( location.substr(2), type, access ) ); return true; } //---------------------------------------------------------------------------- // StatInfo implementation //---------------------------------------------------------------------------- struct StatInfoImpl { StatInfoImpl() : pSize( 0 ), pFlags( 0 ), pModifyTime( 0 ), pChangeTime( 0 ), pAccessTime( 0 ), pExtended( false ), pHasCksum( false ) { } StatInfoImpl( const StatInfoImpl & pimpl ) : pId( pimpl.pId ), pSize( pimpl.pSize ), pFlags( pimpl.pFlags ), pModifyTime( pimpl.pModifyTime ), pChangeTime( pimpl.pChangeTime ), pAccessTime( pimpl.pAccessTime ), pMode( pimpl.pMode ), pOwner( pimpl.pOwner ), pGroup( pimpl.pGroup ), pExtended( pimpl.pExtended ), pHasCksum( pimpl.pHasCksum ) { } //------------------------------------------------------------------------ // Parse the stat info returned by the server //------------------------------------------------------------------------ bool ParseServerResponse( const char *data ) { if( !data || strlen( data ) == 0 ) return false; std::vector chunks; Utils::splitString( chunks, data, " " ); if( chunks.size() < 4 ) return false; pId = chunks[0]; char *result; pSize = ::strtoll( chunks[1].c_str(), &result, 0 ); if( *result != 0 ) { pSize = 0; return false; } pFlags = ::strtol( chunks[2].c_str(), &result, 0 ); if( *result != 0 ) { pFlags = 0; return false; } pModifyTime = ::strtoll( chunks[3].c_str(), &result, 0 ); if( *result != 0 ) { pModifyTime = 0; return false; } if( chunks.size() >= 9 ) { pChangeTime = ::strtoll( chunks[4].c_str(), &result, 0 ); if( *result != 0 ) { pChangeTime = 0; return false; } pAccessTime = ::strtoll( chunks[5].c_str(), &result, 0 ); if( *result != 0 ) { pAccessTime = 0; return false; } // we are expecting at least 4 characters, e.g.: 0644 if( chunks[6].size() < 4 ) return false; pMode = chunks[6]; pOwner = chunks[7]; pGroup = chunks[8]; pExtended = true; } // after the extended stat information, we might have the checksum if( chunks.size() >= 10 ) { if( ( chunks[9] == "[" ) && ( chunks[11] == "]" ) ) { pHasCksum = true; pCksum = chunks[10]; } } return true; } std::string pId; uint64_t pSize; uint32_t pFlags; uint64_t pModifyTime; uint64_t pChangeTime; uint64_t pAccessTime; std::string pMode; std::string pOwner; std::string pGroup; bool pExtended; bool pHasCksum; std::string pCksum; }; //---------------------------------------------------------------------------- // StatInfo constructor //---------------------------------------------------------------------------- StatInfo::StatInfo() : pImpl( new StatInfoImpl() ) { } //------------------------------------------------------------------------ // Constructor //------------------------------------------------------------------------ StatInfo::StatInfo( const std::string &id, uint64_t size, uint32_t flags, uint64_t modTime ) : pImpl( new StatInfoImpl() ) { pImpl->pId = id; pImpl->pSize = size; pImpl->pFlags = flags; pImpl->pModifyTime = modTime; } //------------------------------------------------------------------------ // Copy constructor //------------------------------------------------------------------------ StatInfo::StatInfo( const StatInfo &info ) : pImpl( new StatInfoImpl( *info.pImpl) ) { } //------------------------------------------------------------------------ // Destructor (it can be only defined after StatInfoImpl is defined!!!) //------------------------------------------------------------------------ StatInfo::~StatInfo() = default; //---------------------------------------------------------------------------- // Parse the stat info returned by the server //---------------------------------------------------------------------------- bool StatInfo::ParseServerResponse( const char *data ) { return pImpl->ParseServerResponse( data ); } //------------------------------------------------------------------------ //! Get id //------------------------------------------------------------------------ const std::string& StatInfo::GetId() const { return pImpl->pId; } //------------------------------------------------------------------------ //! Get size (in bytes) //------------------------------------------------------------------------ uint64_t StatInfo::GetSize() const { return pImpl->pSize; } //------------------------------------------------------------------------ //! Set size //------------------------------------------------------------------------ void StatInfo::SetSize( uint64_t size ) { pImpl->pSize = size; } //------------------------------------------------------------------------ //! Get flags //------------------------------------------------------------------------ uint32_t StatInfo::GetFlags() const { return pImpl->pFlags; } //------------------------------------------------------------------------ //! Set flags //------------------------------------------------------------------------ void StatInfo::SetFlags( uint32_t flags ) { pImpl->pFlags = flags; } //------------------------------------------------------------------------ //! Test flags //------------------------------------------------------------------------ bool StatInfo::TestFlags( uint32_t flags ) const { return pImpl->pFlags & flags; } //------------------------------------------------------------------------ //! Get modification time (in seconds since epoch) //------------------------------------------------------------------------ uint64_t StatInfo::GetModTime() const { return pImpl->pModifyTime; } //------------------------------------------------------------------------ //! Get modification time //------------------------------------------------------------------------ std::string StatInfo::GetModTimeAsString() const { return TimeToString( pImpl->pModifyTime ); } //------------------------------------------------------------------------ //! Get change time (in seconds since epoch) //------------------------------------------------------------------------ uint64_t StatInfo::GetChangeTime() const { return pImpl->pChangeTime; } //------------------------------------------------------------------------ //! Get change time //------------------------------------------------------------------------ std::string StatInfo::GetChangeTimeAsString() const { return TimeToString( pImpl->pChangeTime ); } //------------------------------------------------------------------------ //! Get change time (in seconds since epoch) //------------------------------------------------------------------------ uint64_t StatInfo::GetAccessTime() const { return pImpl->pAccessTime; } //------------------------------------------------------------------------ //! Get change time //------------------------------------------------------------------------ std::string StatInfo::GetAccessTimeAsString() const { return TimeToString( pImpl->pAccessTime ); } //------------------------------------------------------------------------ //! Get mode //------------------------------------------------------------------------ const std::string& StatInfo::GetModeAsString() const { return pImpl->pMode; } //------------------------------------------------------------------------ //! Get mode //------------------------------------------------------------------------ const std::string StatInfo::GetModeAsOctString() const { std::string ret; ret.reserve( 9 ); // we care about 3 last digits size_t size = pImpl->pMode.size(); uint8_t oct = pImpl->pMode[size - 3] - '0'; OctToString( oct, ret ); oct = pImpl->pMode[size - 2] - '0'; OctToString( oct, ret ); oct = pImpl->pMode[size - 1] - '0'; OctToString( oct, ret ); return ret; } //------------------------------------------------------------------------ //! Get owner //------------------------------------------------------------------------ const std::string& StatInfo::GetOwner() const { return pImpl->pOwner; } //------------------------------------------------------------------------ //! Get group //------------------------------------------------------------------------ const std::string& StatInfo::GetGroup() const { return pImpl->pGroup; } //------------------------------------------------------------------------ //! Get checksum //------------------------------------------------------------------------ const std::string& StatInfo::GetChecksum() const { return pImpl->pCksum; } //------------------------------------------------------------------------ //! Parse server response and fill up the object //------------------------------------------------------------------------ bool StatInfo::ExtendedFormat() const { return pImpl->pExtended; } //------------------------------------------------------------------------ //! Has checksum //------------------------------------------------------------------------ bool StatInfo::HasChecksum() const { return pImpl->pHasCksum; } //---------------------------------------------------------------------------- // StatInfo constructor //---------------------------------------------------------------------------- StatInfoVFS::StatInfoVFS(): pNodesRW( 0 ), pFreeRW( 0 ), pUtilizationRW( 0 ), pNodesStaging( 0 ), pFreeStaging( 0 ), pUtilizationStaging( 0 ) { } //---------------------------------------------------------------------------- // Parse the stat info returned by the server //---------------------------------------------------------------------------- bool StatInfoVFS::ParseServerResponse( const char *data ) { if( !data || strlen( data ) == 0 ) return false; std::vector chunks; Utils::splitString( chunks, data, " " ); if( chunks.size() < 6 ) return false; char *result; pNodesRW = ::strtoll( chunks[0].c_str(), &result, 0 ); if( *result != 0 ) { pNodesRW = 0; return false; } pFreeRW = ::strtoll( chunks[1].c_str(), &result, 0 ); if( *result != 0 ) { pFreeRW = 0; return false; } pUtilizationRW = ::strtol( chunks[2].c_str(), &result, 0 ); if( *result != 0 ) { pUtilizationRW = 0; return false; } pNodesStaging = ::strtoll( chunks[3].c_str(), &result, 0 ); if( *result != 0 ) { pNodesStaging = 0; return false; } pFreeStaging = ::strtoll( chunks[4].c_str(), &result, 0 ); if( *result != 0 ) { pFreeStaging = 0; return false; } pUtilizationStaging = ::strtol( chunks[5].c_str(), &result, 0 ); if( *result != 0 ) { pUtilizationStaging = 0; return false; } return true; } const std::string DirectoryList::dStatPrefix = ".\n0 0 0 0"; //---------------------------------------------------------------------------- // DirectoryList constructor //---------------------------------------------------------------------------- DirectoryList::DirectoryList() { } //---------------------------------------------------------------------------- // Destructor //---------------------------------------------------------------------------- DirectoryList::~DirectoryList() { for( Iterator it = pDirList.begin(); it != pDirList.end(); ++it ) delete *it; } //---------------------------------------------------------------------------- // Parse the directory list //---------------------------------------------------------------------------- bool DirectoryList::ParseServerResponse( const std::string &hostId, const char *data ) { if( !data ) return false; //-------------------------------------------------------------------------- // Check what kind of response we're dealing with //-------------------------------------------------------------------------- bool isDStat = HasStatInfo( data ); if( isDStat ) data += dStatPrefix.size(); return ParseServerResponse( hostId, data, isDStat ); } //------------------------------------------------------------------------ //! Parse chunked server response and fill up the object //------------------------------------------------------------------------ bool DirectoryList::ParseServerResponse( const std::string &hostId, const char *data, bool isDStat ) { if( !data ) return false; std::string dat = data; std::vector entries; std::vector::iterator it; Utils::splitString( entries, dat, "\n" ); //-------------------------------------------------------------------------- // Normal response //-------------------------------------------------------------------------- if( !isDStat ) { for( it = entries.begin(); it != entries.end(); ++it ) Add( new ListEntry( hostId, *it ) ); return true; } //-------------------------------------------------------------------------- // kXR_dstat //-------------------------------------------------------------------------- if( entries.size() % 2 ) return false; it = entries.begin(); //++it; ++it; for( ; it != entries.end(); ++it ) { ListEntry *entry = new ListEntry( hostId, *it ); Add( entry ); ++it; StatInfo *i = new StatInfo(); entry->SetStatInfo( i ); bool ok = i->ParseServerResponse( it->c_str() ); if( !ok ) return false; } return true; } //------------------------------------------------------------------------ // Returns true if data contain stat info //------------------------------------------------------------------------ bool DirectoryList::HasStatInfo( const char *data ) { std::string dat = data; return !dat.compare( 0, dStatPrefix.size(), dStatPrefix ); } struct PageInfoImpl { PageInfoImpl( uint64_t offset = 0, uint32_t length = 0, void *buffer = 0, std::vector &&cksums = std::vector() ) : offset( offset ), length( length ), buffer( buffer ), cksums( std::move( cksums ) ), nbrepair( 0 ) { } PageInfoImpl( PageInfoImpl &&pginf ) : offset( pginf.offset ), length( pginf.length ), buffer( pginf.buffer ), cksums( std::move( pginf.cksums ) ), nbrepair( pginf.nbrepair ) { } uint64_t offset; //> offset in the file uint32_t length; //> length of the data read void *buffer; //> buffer with the read data std::vector cksums; //> a vector of crc32c checksums size_t nbrepair; //> number of repaired pages }; //---------------------------------------------------------------------------- // Default constructor //---------------------------------------------------------------------------- PageInfo::PageInfo( uint64_t offset, uint32_t length, void *buffer, std::vector &&cksums ) : pImpl( new PageInfoImpl( offset, length, buffer, std::move( cksums ) ) ) { } //---------------------------------------------------------------------------- // Move constructor //---------------------------------------------------------------------------- PageInfo::PageInfo( PageInfo &&pginf ) : pImpl( std::move( pginf.pImpl ) ) { } //---------------------------------------------------------------------------- //! Move assigment operator //---------------------------------------------------------------------------- PageInfo& PageInfo::operator=( PageInfo &&pginf ) { pImpl.swap( pginf.pImpl ); return *this; } //---------------------------------------------------------------------------- // Destructor //---------------------------------------------------------------------------- PageInfo::~PageInfo() { } //---------------------------------------------------------------------------- // Get the offset //---------------------------------------------------------------------------- uint64_t PageInfo::GetOffset() const { return pImpl->offset; } //---------------------------------------------------------------------------- // Get the data length //---------------------------------------------------------------------------- uint32_t PageInfo::GetLength() const { return pImpl->length; } //---------------------------------------------------------------------------- // Get the buffer //---------------------------------------------------------------------------- void* PageInfo::GetBuffer() { return pImpl->buffer; } //---------------------------------------------------------------------------- // Get the buffer //---------------------------------------------------------------------------- std::vector& PageInfo::GetCksums() { return pImpl->cksums; } //---------------------------------------------------------------------------- // Set number of repaired pages //---------------------------------------------------------------------------- void PageInfo::SetNbRepair( size_t nbrepair ) { pImpl->nbrepair = nbrepair; } //---------------------------------------------------------------------------- // Get number of repaired pages //---------------------------------------------------------------------------- size_t PageInfo::GetNbRepair() { return pImpl->nbrepair; } struct RetryInfoImpl { RetryInfoImpl( std::vector> && retries ) : retries( std::move( retries ) ) { } std::vector> retries; }; //---------------------------------------------------------------------------- // Constructor //---------------------------------------------------------------------------- RetryInfo::RetryInfo( std::vector> && retries ) : pImpl( new RetryInfoImpl( std::move( retries ) ) ) { } //---------------------------------------------------------------------------- // Destructor //---------------------------------------------------------------------------- RetryInfo::~RetryInfo(){ } //---------------------------------------------------------------------------- // @return : true if some pages need retrying, false otherwise //---------------------------------------------------------------------------- bool RetryInfo::NeedRetry() { return !pImpl->retries.empty(); } //---------------------------------------------------------------------------- // @return number of pages that need to be retransmitted //---------------------------------------------------------------------------- size_t RetryInfo::Size() { return pImpl->retries.size(); } //---------------------------------------------------------------------------- // @return : offset and size of respective page that requires to be // retransmitted //---------------------------------------------------------------------------- std::tuple RetryInfo::At( size_t i ) { return pImpl->retries[i]; } //------------------------------------------------------------------------ // Factory function for generating handler objects from lambdas //------------------------------------------------------------------------ ResponseHandler* ResponseHandler::Wrap( std::function func ) { struct FuncHandler : public ResponseHandler { FuncHandler( std::function func ) : func( std::move( func ) ) { } void HandleResponse( XRootDStatus *status, AnyObject *response ) { // make sure the arguments will be released std::unique_ptr stptr( status ); std::unique_ptr rspptr( response ); // if there is no response provide a null reference placeholder static AnyObject nullref; if( response == nullptr ) response = &nullref; // call the user completion handler func( *status, *response ); // check if this is a final respons bool finalrsp = !( status->IsOK() && status->code == suContinue ); // deallocate the wrapper if final if( finalrsp ) delete this; } std::function func; }; return new FuncHandler( func ); } ResponseHandler* ResponseHandler::Wrap( std::function func ) { struct FuncHandler : public ResponseHandler { FuncHandler( std::function func ) : func( std::move( func ) ) { } void HandleResponse( XRootDStatus *status, AnyObject *response ) { // check if this is a final respons bool finalrsp = !( status->IsOK() && status->code == suContinue ); // call the user completion handler func( status, response ); // deallocate the wrapper if final if( finalrsp ) delete this; } std::function func; }; return new FuncHandler( func ); } } xrootd-5.6.9/src/XrdCl/XrdClXRootDResponses.hh000066400000000000000000001313131457266313600212330ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2012 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . //------------------------------------------------------------------------------ #ifndef __XRD_CL_XROOTD_RESPONSES_HH__ #define __XRD_CL_XROOTD_RESPONSES_HH__ #include "XrdCl/XrdClBuffer.hh" #include "XrdCl/XrdClStatus.hh" #include "XrdCl/XrdClURL.hh" #include "XrdCl/XrdClAnyObject.hh" #include "XProtocol/XProtocol.hh" #include #include #include #include #include #include #include #include namespace XrdCl { //---------------------------------------------------------------------------- //! Path location info //---------------------------------------------------------------------------- class LocationInfo { public: //------------------------------------------------------------------------ //! Describes the node type and file status for a given location //------------------------------------------------------------------------ enum LocationType { ManagerOnline, //!< manager node where the file is online ManagerPending, //!< manager node where the file is pending to be online ServerOnline, //!< server node where the file is online ServerPending //!< server node where the file is pending to be online }; //------------------------------------------------------------------------ //! Describes the allowed access type for the file at given location //------------------------------------------------------------------------ enum AccessType { Read, //!< read access is allowed ReadWrite //!< write access is allowed }; //------------------------------------------------------------------------ //! Location //------------------------------------------------------------------------ class Location { public: //-------------------------------------------------------------------- //! Constructor //-------------------------------------------------------------------- Location( const std::string &address, LocationType type, AccessType access ): pAddress( address ), pType( type ), pAccess( access ) {} //-------------------------------------------------------------------- //! Get address //-------------------------------------------------------------------- const std::string &GetAddress() const { return pAddress; } //-------------------------------------------------------------------- //! Get location type //-------------------------------------------------------------------- LocationType GetType() const { return pType; } //-------------------------------------------------------------------- //! Get access type //-------------------------------------------------------------------- AccessType GetAccessType() const { return pAccess; } //-------------------------------------------------------------------- //! Check whether the location is a server //-------------------------------------------------------------------- bool IsServer() const { return pType == ServerOnline || pType == ServerPending; } //-------------------------------------------------------------------- //! Check whether the location is a manager //-------------------------------------------------------------------- bool IsManager() const { return pType == ManagerOnline || pType == ManagerPending; } private: std::string pAddress; LocationType pType; AccessType pAccess; }; //------------------------------------------------------------------------ //! List of locations //------------------------------------------------------------------------ typedef std::vector LocationList; //------------------------------------------------------------------------ //! Iterator over locations //------------------------------------------------------------------------ typedef LocationList::iterator Iterator; //------------------------------------------------------------------------ //! Iterator over locations //------------------------------------------------------------------------ typedef LocationList::const_iterator ConstIterator; //------------------------------------------------------------------------ //! Constructor //------------------------------------------------------------------------ LocationInfo(); //------------------------------------------------------------------------ //! Get number of locations //------------------------------------------------------------------------ uint32_t GetSize() const { return pLocations.size(); } //------------------------------------------------------------------------ //! Get the location at index //------------------------------------------------------------------------ Location &At( uint32_t index ) { return pLocations[index]; } //------------------------------------------------------------------------ //! Get the location begin iterator //------------------------------------------------------------------------ Iterator Begin() { return pLocations.begin(); } //------------------------------------------------------------------------ //! Get the location begin iterator //------------------------------------------------------------------------ ConstIterator Begin() const { return pLocations.begin(); } //------------------------------------------------------------------------ //! Get the location end iterator //------------------------------------------------------------------------ Iterator End() { return pLocations.end(); } //------------------------------------------------------------------------ //! Get the location end iterator //------------------------------------------------------------------------ ConstIterator End() const { return pLocations.end(); } //------------------------------------------------------------------------ //! Add a location //------------------------------------------------------------------------ void Add( const Location &location ) { pLocations.push_back( location ); } //------------------------------------------------------------------------ //! Parse server response and fill up the object //------------------------------------------------------------------------ bool ParseServerResponse( const char *data ); private: bool ProcessLocation( std::string &location ); LocationList pLocations; }; //---------------------------------------------------------------------------- //! Request status //---------------------------------------------------------------------------- class XRootDStatus: public Status { public: //------------------------------------------------------------------------ //! Constructor //------------------------------------------------------------------------ XRootDStatus( uint16_t st = 0, uint16_t code = 0, uint32_t errN = 0, const std::string &message = "" ): Status( st, code, errN ), pMessage( message ) {} //------------------------------------------------------------------------ //! Constructor //------------------------------------------------------------------------ XRootDStatus( const Status &st, const std::string &message = "" ): Status( st ), pMessage( message ) {} //------------------------------------------------------------------------ //! Get error message //------------------------------------------------------------------------ const std::string &GetErrorMessage() const { return pMessage; } //------------------------------------------------------------------------ //! Set the error message //------------------------------------------------------------------------ void SetErrorMessage( const std::string &message ) { pMessage = message; } //------------------------------------------------------------------------ //! Convert to string //------------------------------------------------------------------------ std::string ToStr() const { if( code == errErrorResponse ) { std::ostringstream o; o << "[ERROR] Server responded with an error: [" << errNo << "] "; o << pMessage << std::endl; return o.str(); } std::string str = ToString(); if( !pMessage.empty() ) str += ": " + pMessage; return str; } private: std::string pMessage; }; //---------------------------------------------------------------------------- //! Tuple indexes of name and value fields in xattr_t //---------------------------------------------------------------------------- enum { xattr_name = 0, xattr_value = 1 }; //---------------------------------------------------------------------------- //! Extended attribute key - value pair //---------------------------------------------------------------------------- typedef std::tuple xattr_t; //---------------------------------------------------------------------------- //! Extended attribute operation status //---------------------------------------------------------------------------- struct XAttrStatus { friend class FileStateHandler; friend class FileSystem; XAttrStatus( const std::string &name, const XRootDStatus &status ) : name( name ), status( status ) { } std::string name; XRootDStatus status; }; //---------------------------------------------------------------------------- //! Extended attributes with status //---------------------------------------------------------------------------- struct XAttr : public XAttrStatus { friend class FileStateHandler; friend class FileSystem; XAttr( const std::string &name, const XRootDStatus &status ) : XAttrStatus( name, status ) { } XAttr( const std::string &name, const std::string &value = "", const XRootDStatus &status = XRootDStatus() ) : XAttrStatus( name, status ), value( value ) { } std::string value; }; //---------------------------------------------------------------------------- //! Binary buffer //---------------------------------------------------------------------------- typedef Buffer BinaryDataInfo; //---------------------------------------------------------------------------- //! Protocol response //---------------------------------------------------------------------------- class ProtocolInfo { public: //------------------------------------------------------------------------ //! Types of XRootD servers //------------------------------------------------------------------------ enum HostTypes { IsManager = kXR_isManager, //!< Manager IsServer = kXR_isServer, //!< Data server AttrMeta = kXR_attrMeta, //!< Meta attribute AttrProxy = kXR_attrProxy, //!< Proxy attribute AttrSuper = kXR_attrSuper //!< Supervisor attribute }; //------------------------------------------------------------------------ //! Constructor //------------------------------------------------------------------------ ProtocolInfo( uint32_t version, uint32_t hostInfo ): pVersion( version ), pHostInfo( hostInfo ) {} //------------------------------------------------------------------------ //! Get version info //------------------------------------------------------------------------ uint32_t GetVersion() const { return pVersion; } //------------------------------------------------------------------------ //! Get host info //------------------------------------------------------------------------ uint32_t GetHostInfo() const { return pHostInfo; } //------------------------------------------------------------------------ //! Test host info flags //------------------------------------------------------------------------ bool TestHostInfo( uint32_t flags ) { return pHostInfo & flags; } private: uint32_t pVersion; uint32_t pHostInfo; }; //---------------------------------------------------------------------------- //! Object stat info implementation forward declaration //---------------------------------------------------------------------------- struct StatInfoImpl; //---------------------------------------------------------------------------- //! Object stat info //---------------------------------------------------------------------------- class StatInfo { public: //------------------------------------------------------------------------ //! Flags //------------------------------------------------------------------------ enum Flags { XBitSet = kXR_xset, //!< Executable/searchable bit set IsDir = kXR_isDir, //!< This is a directory Other = kXR_other, //!< Neither a file nor a directory Offline = kXR_offline, //!< File is not online (ie. on disk) POSCPending = kXR_poscpend, //!< File opened with POST flag, not yet //!< successfully closed IsReadable = kXR_readable, //!< Read access is allowed IsWritable = kXR_writable, //!< Write access is allowed BackUpExists = kXR_bkpexist //!< Back up copy exists }; //------------------------------------------------------------------------ //! Constructor //------------------------------------------------------------------------ StatInfo(); //------------------------------------------------------------------------ //! Constructor //------------------------------------------------------------------------ StatInfo( const std::string &id, uint64_t size, uint32_t flags, uint64_t modTime ); //------------------------------------------------------------------------ //! Copy constructor //------------------------------------------------------------------------ StatInfo( const StatInfo &info ); //------------------------------------------------------------------------ //! Destructor //------------------------------------------------------------------------ ~StatInfo(); //------------------------------------------------------------------------ //! Get id //------------------------------------------------------------------------ const std::string& GetId() const; //------------------------------------------------------------------------ //! Get size (in bytes) //------------------------------------------------------------------------ uint64_t GetSize() const; //------------------------------------------------------------------------ //! Set size //------------------------------------------------------------------------ void SetSize( uint64_t size ); //------------------------------------------------------------------------ //! Get flags //------------------------------------------------------------------------ uint32_t GetFlags() const; //------------------------------------------------------------------------ //! Set flags //------------------------------------------------------------------------ void SetFlags( uint32_t flags ); //------------------------------------------------------------------------ //! Test flags //------------------------------------------------------------------------ bool TestFlags( uint32_t flags ) const; //------------------------------------------------------------------------ //! Get modification time (in seconds since epoch) //------------------------------------------------------------------------ uint64_t GetModTime() const; //------------------------------------------------------------------------ //! Get modification time //------------------------------------------------------------------------ std::string GetModTimeAsString() const; //------------------------------------------------------------------------ //! Get change time (in seconds since epoch) //------------------------------------------------------------------------ uint64_t GetChangeTime() const; //------------------------------------------------------------------------ //! Get change time //------------------------------------------------------------------------ std::string GetChangeTimeAsString() const; //------------------------------------------------------------------------ //! Get change time (in seconds since epoch) //------------------------------------------------------------------------ uint64_t GetAccessTime() const; //------------------------------------------------------------------------ //! Get change time //------------------------------------------------------------------------ std::string GetAccessTimeAsString() const; //------------------------------------------------------------------------ //! Get mode //------------------------------------------------------------------------ const std::string& GetModeAsString() const; //------------------------------------------------------------------------ //! Get mode //------------------------------------------------------------------------ const std::string GetModeAsOctString() const; //------------------------------------------------------------------------ //! Get owner //------------------------------------------------------------------------ const std::string& GetOwner() const; //------------------------------------------------------------------------ //! Get group //------------------------------------------------------------------------ const std::string& GetGroup() const; //------------------------------------------------------------------------ //! Get checksum //------------------------------------------------------------------------ const std::string& GetChecksum() const; //------------------------------------------------------------------------ //! Parse server response and fill up the object //------------------------------------------------------------------------ bool ParseServerResponse( const char *data ); //------------------------------------------------------------------------ //! Has extended stat information //------------------------------------------------------------------------ bool ExtendedFormat() const; //------------------------------------------------------------------------ //! Has checksum //------------------------------------------------------------------------ bool HasChecksum() const; private: static inline std::string TimeToString( uint64_t time ) { char ts[256]; time_t modTime = time; tm *t = gmtime( &modTime ); strftime( ts, 255, "%F %T", t ); return ts; } static inline void OctToString( uint8_t oct, std::string &str ) { static const uint8_t r_mask = 0x4; static const uint8_t w_mask = 0x2; static const uint8_t x_mask = 0x1; if( r_mask & oct ) str.push_back( 'r' ); else str.push_back( '-' ); if( w_mask & oct ) str.push_back( 'w' ); else str.push_back( '-' ); if( x_mask & oct ) str.push_back( 'x' ); else str.push_back( '-' ); } std::unique_ptr pImpl; }; //---------------------------------------------------------------------------- //! VFS stat info //---------------------------------------------------------------------------- class StatInfoVFS { public: //------------------------------------------------------------------------ //! Constructor //------------------------------------------------------------------------ StatInfoVFS(); //------------------------------------------------------------------------ //! Get number of nodes that can provide read/write space //------------------------------------------------------------------------ uint64_t GetNodesRW() const { return pNodesRW; } //------------------------------------------------------------------------ //! Get size of the largest contiguous area of free r/w space (in MB) //------------------------------------------------------------------------ uint64_t GetFreeRW() const { return pFreeRW; } //------------------------------------------------------------------------ //! Get percentage of the partition utilization represented by FreeRW //------------------------------------------------------------------------ uint8_t GetUtilizationRW() const { return pUtilizationRW; } //------------------------------------------------------------------------ //! Get number of nodes that can provide staging space //------------------------------------------------------------------------ uint64_t GetNodesStaging() const { return pNodesStaging; } //------------------------------------------------------------------------ //! Get size of the largest contiguous area of free staging space (in MB) //------------------------------------------------------------------------ uint64_t GetFreeStaging() const { return pFreeStaging; } //------------------------------------------------------------------------ //! Get percentage of the partition utilization represented by FreeStaging //------------------------------------------------------------------------ uint8_t GetUtilizationStaging() const { return pUtilizationStaging; } //------------------------------------------------------------------------ //! Parse server response and fill up the object //------------------------------------------------------------------------ bool ParseServerResponse( const char *data ); private: //------------------------------------------------------------------------ // kXR_vfs stat //------------------------------------------------------------------------ uint64_t pNodesRW; uint64_t pFreeRW; uint32_t pUtilizationRW; uint64_t pNodesStaging; uint64_t pFreeStaging; uint32_t pUtilizationStaging; }; //---------------------------------------------------------------------------- //! Directory list //---------------------------------------------------------------------------- class DirectoryList { public: //------------------------------------------------------------------------ //! Directory entry //------------------------------------------------------------------------ class ListEntry { public: //-------------------------------------------------------------------- //! Constructor //-------------------------------------------------------------------- ListEntry( const std::string &hostAddress, const std::string &name, StatInfo *statInfo = 0): pHostAddress( hostAddress ), pName( SanitizeName( name ) ), pStatInfo( statInfo ) {} //-------------------------------------------------------------------- //! Destructor //-------------------------------------------------------------------- ~ListEntry() { delete pStatInfo; } //-------------------------------------------------------------------- //! Get host address //-------------------------------------------------------------------- const std::string &GetHostAddress() const { return pHostAddress; } //-------------------------------------------------------------------- //! Get file name //-------------------------------------------------------------------- const std::string &GetName() const { return pName; } //-------------------------------------------------------------------- //! Get the stat info object //-------------------------------------------------------------------- StatInfo *GetStatInfo() { return pStatInfo; } //-------------------------------------------------------------------- //! Get the stat info object //-------------------------------------------------------------------- const StatInfo *GetStatInfo() const { return pStatInfo; } //-------------------------------------------------------------------- //! Set the stat info object (and transfer the ownership) //-------------------------------------------------------------------- void SetStatInfo( StatInfo *info ) { pStatInfo = info; } private: inline static std::string SanitizeName( const std::string &name ) { const char *cstr = name.c_str(); while( *cstr == '/' ) // the C string is guaranteed to end with '\0' ++cstr; return cstr; } std::string pHostAddress; std::string pName; StatInfo *pStatInfo; }; //------------------------------------------------------------------------ //! Constructor //------------------------------------------------------------------------ DirectoryList(); //------------------------------------------------------------------------ //! Destructor //------------------------------------------------------------------------ ~DirectoryList(); //------------------------------------------------------------------------ //! Directory listing //------------------------------------------------------------------------ typedef std::vector DirList; //------------------------------------------------------------------------ //! Directory listing iterator //------------------------------------------------------------------------ typedef DirList::iterator Iterator; //------------------------------------------------------------------------ //! Directory listing const iterator //------------------------------------------------------------------------ typedef DirList::const_iterator ConstIterator; //------------------------------------------------------------------------ //! Add an entry to the list - takes ownership //------------------------------------------------------------------------ void Add( ListEntry *entry ) { pDirList.push_back( entry ); } //------------------------------------------------------------------------ //! Get an entry at given index //------------------------------------------------------------------------ ListEntry *At( uint32_t index ) { return pDirList[index]; } //------------------------------------------------------------------------ //! Get the begin iterator //------------------------------------------------------------------------ Iterator Begin() { return pDirList.begin(); } //------------------------------------------------------------------------ //! Get the begin iterator //------------------------------------------------------------------------ ConstIterator Begin() const { return pDirList.begin(); } //------------------------------------------------------------------------ //! Get the end iterator //------------------------------------------------------------------------ Iterator End() { return pDirList.end(); } //------------------------------------------------------------------------ //! Get the end iterator //------------------------------------------------------------------------ ConstIterator End() const { return pDirList.end(); } //------------------------------------------------------------------------ //! Get the size of the listing //------------------------------------------------------------------------ uint32_t GetSize() const { return pDirList.size(); } //------------------------------------------------------------------------ //! Get parent directory name //------------------------------------------------------------------------ const std::string &GetParentName() const { return pParent; } //------------------------------------------------------------------------ //! Set name of the parent directory //------------------------------------------------------------------------ void SetParentName( const std::string &parent ) { size_t pos = parent.find( '?' ); pParent = pos == std::string::npos ? parent : parent.substr( 0, pos ); if( !pParent.empty() && pParent[pParent.length()-1] != '/' ) pParent += "/"; } //------------------------------------------------------------------------ //! Parse server response and fill up the object //------------------------------------------------------------------------ bool ParseServerResponse( const std::string &hostId, const char *data ); //------------------------------------------------------------------------ //! Parse chunked server response and fill up the object //------------------------------------------------------------------------ bool ParseServerResponse( const std::string &hostId, const char *data, bool isDStat ); //------------------------------------------------------------------------ //! Returns true if data contain stat info //------------------------------------------------------------------------ static bool HasStatInfo( const char *data ); private: DirList pDirList; std::string pParent; static const std::string dStatPrefix; }; //---------------------------------------------------------------------------- //! Information returned by file open operation //---------------------------------------------------------------------------- class OpenInfo { public: //------------------------------------------------------------------------ //! Constructor //------------------------------------------------------------------------ OpenInfo( const uint8_t *fileHandle, uint64_t sessionId, StatInfo *statInfo = 0 ): pSessionId(sessionId), pStatInfo( statInfo ) { memcpy( pFileHandle, fileHandle, 4 ); } //------------------------------------------------------------------------ //! Destructor //------------------------------------------------------------------------ ~OpenInfo() { delete pStatInfo; } //------------------------------------------------------------------------ //! Get the file handle (4bytes) //------------------------------------------------------------------------ void GetFileHandle( uint8_t *fileHandle ) const { memcpy( fileHandle, pFileHandle, 4 ); } //------------------------------------------------------------------------ //! Get the stat info //------------------------------------------------------------------------ const StatInfo *GetStatInfo() const { return pStatInfo; } //------------------------------------------------------------------------ // Get session ID //------------------------------------------------------------------------ uint64_t GetSessionId() const { return pSessionId; } private: uint8_t pFileHandle[4]; uint64_t pSessionId; StatInfo *pStatInfo; }; //---------------------------------------------------------------------------- //! Describe a data chunk for vector read //---------------------------------------------------------------------------- struct ChunkInfo { //-------------------------------------------------------------------------- //! Constructor //-------------------------------------------------------------------------- ChunkInfo( uint64_t off = 0, uint32_t len = 0, void *buff = 0 ): offset( off ), length( len ), buffer(buff) {} //---------------------------------------------------------------------------- //! Get the offset //---------------------------------------------------------------------------- inline uint64_t GetOffset() const { return offset; } //---------------------------------------------------------------------------- //! Get the data length //---------------------------------------------------------------------------- inline uint32_t GetLength() const { return length; } //---------------------------------------------------------------------------- //! Get the buffer //---------------------------------------------------------------------------- inline void* GetBuffer() { return buffer; } uint64_t offset; //! offset in the file uint32_t length; //! length of the chunk void *buffer; //! optional buffer pointer }; struct PageInfoImpl; struct PageInfo { //---------------------------------------------------------------------------- //! Default constructor //---------------------------------------------------------------------------- PageInfo( uint64_t offset = 0, uint32_t length = 0, void *buffer = 0, std::vector &&cksums = std::vector() ); //---------------------------------------------------------------------------- //! Move constructor //---------------------------------------------------------------------------- PageInfo( PageInfo &&pginf ); //---------------------------------------------------------------------------- //! Move assigment operator //---------------------------------------------------------------------------- PageInfo& operator=( PageInfo &&pginf ); //---------------------------------------------------------------------------- //! Destructor //---------------------------------------------------------------------------- ~PageInfo(); //---------------------------------------------------------------------------- //! Get the offset //---------------------------------------------------------------------------- uint64_t GetOffset() const; //---------------------------------------------------------------------------- //! Get the data length //---------------------------------------------------------------------------- uint32_t GetLength() const; //---------------------------------------------------------------------------- //! Get the buffer //---------------------------------------------------------------------------- void* GetBuffer(); //---------------------------------------------------------------------------- //! Get the checksums //---------------------------------------------------------------------------- std::vector& GetCksums(); //---------------------------------------------------------------------------- //! Get number of repaired pages //---------------------------------------------------------------------------- size_t GetNbRepair(); //---------------------------------------------------------------------------- //! Set number of repaired pages //---------------------------------------------------------------------------- void SetNbRepair( size_t nbrepair ); private: //-------------------------------------------------------------------------- //! pointer to implementation //-------------------------------------------------------------------------- std::unique_ptr pImpl; }; struct RetryInfoImpl; struct RetryInfo { //---------------------------------------------------------------------------- //! Constructor //---------------------------------------------------------------------------- RetryInfo( std::vector> && retries ); //---------------------------------------------------------------------------- //! Destructor //---------------------------------------------------------------------------- ~RetryInfo(); //---------------------------------------------------------------------------- //! @return : true if some pages need retrying, false otherwise //---------------------------------------------------------------------------- bool NeedRetry(); //---------------------------------------------------------------------------- //! @return number of pages that need to be retransmitted //---------------------------------------------------------------------------- size_t Size(); //---------------------------------------------------------------------------- //! @return : offset and size of respective page that requires to be // retransmitted //---------------------------------------------------------------------------- std::tuple At( size_t i ); private: //-------------------------------------------------------------------------- //! pointer to implementation //-------------------------------------------------------------------------- std::unique_ptr pImpl; }; //---------------------------------------------------------------------------- //! List of chunks //---------------------------------------------------------------------------- typedef std::vector ChunkList; //---------------------------------------------------------------------------- //! Vector read info //---------------------------------------------------------------------------- class VectorReadInfo { public: //------------------------------------------------------------------------ //! Constructor //------------------------------------------------------------------------ VectorReadInfo(): pSize( 0 ) {} //------------------------------------------------------------------------ //! Get Size //------------------------------------------------------------------------ uint32_t GetSize() const { return pSize; } //------------------------------------------------------------------------ //! Set size //------------------------------------------------------------------------ void SetSize( uint32_t size ) { pSize = size; } //------------------------------------------------------------------------ //! Get chunks //------------------------------------------------------------------------ ChunkList &GetChunks() { return pChunks; } //------------------------------------------------------------------------ //! Get chunks //------------------------------------------------------------------------ const ChunkList &GetChunks() const { return pChunks; } private: ChunkList pChunks; uint32_t pSize; }; //---------------------------------------------------------------------------- // List of URLs //---------------------------------------------------------------------------- struct HostInfo { HostInfo(): flags(0), protocol(0), loadBalancer(false) {} HostInfo( const URL &u, bool lb = false ): flags(0), protocol(0), loadBalancer(lb), url(u) {} uint32_t flags; //!< Host type uint32_t protocol; //!< Version of the protocol the host is speaking bool loadBalancer; //!< Was the host used as a load balancer URL url; //!< URL of the host }; typedef std::vector HostList; //---------------------------------------------------------------------------- //! Handle an async response //---------------------------------------------------------------------------- class ResponseHandler { public: virtual ~ResponseHandler() {} //------------------------------------------------------------------------ //! Called when a response to associated request arrives or an error //! occurs //! //! @param status status of the request //! @param response an object associated with the response //! (request dependent) //! @param hostList list of hosts the request was redirected to //------------------------------------------------------------------------ virtual void HandleResponseWithHosts( XRootDStatus *status, AnyObject *response, HostList *hostList ) { delete hostList; HandleResponse( status, response ); } //------------------------------------------------------------------------ //! Called when a response to associated request arrives or an error //! occurs //! //! @param status status of the request //! @param response an object associated with the response //! (request dependent) //------------------------------------------------------------------------ virtual void HandleResponse( XRootDStatus *status, AnyObject *response ) { (void)status; (void)response; } //------------------------------------------------------------------------ //! Factory function for generating handler objects from lambdas //! //! @param func : the callback, must not throw //! @return : ResponseHandler wrapper with the user callback //------------------------------------------------------------------------ static ResponseHandler* Wrap( std::function func ); //------------------------------------------------------------------------ //! Factory function for generating handler objects from lambdas //! //! @param func : the callback, must not throw //! @return : ResponseHandler wrapper with the user callback //------------------------------------------------------------------------ static ResponseHandler* Wrap( std::function func ); }; } #endif // __XRD_CL_XROOTD_RESPONSES_HH__ xrootd-5.6.9/src/XrdCl/XrdClXRootDTransport.cc000066400000000000000000003763511457266313600212510ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2014 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #include "XrdCl/XrdClXRootDTransport.hh" #include "XrdCl/XrdClConstants.hh" #include "XrdCl/XrdClLog.hh" #include "XrdCl/XrdClSocket.hh" #include "XrdCl/XrdClMessage.hh" #include "XrdCl/XrdClDefaultEnv.hh" #include "XrdCl/XrdClSIDManager.hh" #include "XrdCl/XrdClUtils.hh" #include "XrdCl/XrdClTransportManager.hh" #include "XrdCl/XrdClTls.hh" #include "XrdNet/XrdNetAddr.hh" #include "XrdNet/XrdNetUtils.hh" #include "XrdSys/XrdSysPlatform.hh" #include "XrdOuc/XrdOucErrInfo.hh" #include "XrdOuc/XrdOucUtils.hh" #include "XrdOuc/XrdOucCRC.hh" #include "XrdOuc/XrdOucTokenizer.hh" #include "XrdSys/XrdSysTimer.hh" #include "XrdSys/XrdSysAtomics.hh" #include "XrdSys/XrdSysPlugin.hh" #include "XrdSec/XrdSecLoadSecurity.hh" #include "XrdSec/XrdSecProtect.hh" #include "XrdSys/XrdSysE2T.hh" #include "XrdCl/XrdClTls.hh" #include "XrdCl/XrdClSocket.hh" #include "XProtocol/XProtocol.hh" #include "XrdVersion.hh" #include #include #include #include #include #include #include #include #include XrdVERSIONINFOREF( XrdCl ); namespace XrdCl { struct PluginUnloadHandler { PluginUnloadHandler() : unloaded( false ) { } static void UnloadHandler() { UnloadHandler( "root" ); UnloadHandler( "xroot" ); } static void UnloadHandler( const std::string &trProt ) { TransportManager *trManager = DefaultEnv::GetTransportManager(); TransportHandler *trHandler = trManager->GetHandler( trProt ); trHandler->WaitBeforeExit(); } void Register( const std::string &protocol ) { XrdSysRWLockHelper scope( lock, false ); // obtain write lock std::pair< std::set::iterator, bool > ret = protocols.insert( protocol ); // if that's the first time we are using the protocol, the sec lib // was just loaded so now's the time to register the atexit handler if( ret.second ) { atexit( UnloadHandler ); } } XrdSysRWLock lock; bool unloaded; std::set protocols; }; //---------------------------------------------------------------------------- //! Information holder for XRootDStreams //---------------------------------------------------------------------------- struct XRootDStreamInfo { //-------------------------------------------------------------------------- // Define the stream status for the link negotiation purposes //-------------------------------------------------------------------------- enum StreamStatus { Disconnected, Broken, HandShakeSent, HandShakeReceived, LoginSent, AuthSent, BindSent, EndSessionSent, Connected }; //-------------------------------------------------------------------------- // Constructor //-------------------------------------------------------------------------- XRootDStreamInfo(): status( Disconnected ), pathId( 0 ) { } StreamStatus status; uint8_t pathId; }; //---------------------------------------------------------------------------- //! Selects less loaded stream for read operation over multiple streams //---------------------------------------------------------------------------- struct StreamSelector { StreamSelector( uint16_t size ) { //---------------------------------------------------------------------- // Subtract one because we shouldn't take into account the control // stream. //---------------------------------------------------------------------- strmqueues.resize( size - 1, 0 ); } //------------------------------------------------------------------------ // @param size : number of streams //------------------------------------------------------------------------ void AdjustQueues( uint16_t size ) { strmqueues.resize( size - 1, 0); } //------------------------------------------------------------------------ // @param connected : bitarray stating if given sub-stream is connected // // @return : substream number //------------------------------------------------------------------------ uint16_t Select( const std::vector &connected ) { uint16_t ret = 0; size_t minval = std::numeric_limits::max(); for( uint16_t i = 0; i < connected.size() && i < strmqueues.size(); ++i ) { if( !connected[i] ) continue; if( strmqueues[i] < minval ) { ret = i; minval = strmqueues[i]; } } ++strmqueues[ret]; return ret + 1; } //-------------------------------------------------------------------------- // Update queue for given substream //-------------------------------------------------------------------------- void MsgReceived( uint16_t substrm ) { if( substrm > 0 ) --strmqueues[substrm - 1]; } private: std::vector strmqueues; }; struct BindPrefSelector { BindPrefSelector( std::vector && bindprefs ) : bindprefs( std::move( bindprefs ) ), next( 0 ) { } inline const std::string& Get() { std::string &ret = bindprefs[next]; ++next; if( next >= bindprefs.size() ) next = 0; return ret; } private: std::vector bindprefs; size_t next; }; //---------------------------------------------------------------------------- //! Information holder for xrootd channels //---------------------------------------------------------------------------- struct XRootDChannelInfo { //-------------------------------------------------------------------------- // Constructor //-------------------------------------------------------------------------- XRootDChannelInfo( const URL &url ): serverFlags(0), protocolVersion(0), firstLogIn(true), authBuffer(0), authProtocol(0), authParams(0), authEnv(0), finstcnt(0), openFiles(0), waitBarrier(0), protection(0), protRespBody(0), protRespSize(0), encrypted(false), istpc(false) { sidManager = SIDMgrPool::Instance().GetSIDMgr( url.GetChannelId() ); memset( sessionId, 0, 16 ); memset( oldSessionId, 0, 16 ); } //-------------------------------------------------------------------------- // Destructor //-------------------------------------------------------------------------- ~XRootDChannelInfo() { delete [] authBuffer; } typedef std::vector StreamInfoVector; //-------------------------------------------------------------------------- // Data //-------------------------------------------------------------------------- uint32_t serverFlags; uint32_t protocolVersion; uint8_t sessionId[16]; uint8_t oldSessionId[16]; bool firstLogIn; std::shared_ptr sidManager; char *authBuffer; XrdSecProtocol *authProtocol; XrdSecParameters *authParams; XrdOucEnv *authEnv; StreamInfoVector stream; std::string streamName; std::string authProtocolName; std::set sentOpens; std::set sentCloses; std::atomic finstcnt; // file instance count uint32_t openFiles; time_t waitBarrier; XrdSecProtect *protection; ServerResponseBody_Protocol *protRespBody; unsigned int protRespSize; std::unique_ptr strmSelector; bool encrypted; bool istpc; std::unique_ptr bindSelector; std::string logintoken; XrdSysMutex mutex; }; //---------------------------------------------------------------------------- // Constructor //---------------------------------------------------------------------------- XRootDTransport::XRootDTransport(): pSecUnloadHandler( new PluginUnloadHandler() ) { } //---------------------------------------------------------------------------- // Destructor //---------------------------------------------------------------------------- XRootDTransport::~XRootDTransport() { delete pSecUnloadHandler; pSecUnloadHandler = 0; } //---------------------------------------------------------------------------- // Read message header from socket //---------------------------------------------------------------------------- XRootDStatus XRootDTransport::GetHeader( Message &message, Socket *socket ) { //-------------------------------------------------------------------------- // A new message - allocate the space needed for the header //-------------------------------------------------------------------------- if( message.GetCursor() == 0 && message.GetSize() < 8 ) message.Allocate( 8 ); //-------------------------------------------------------------------------- // Read the message header //-------------------------------------------------------------------------- if( message.GetCursor() < 8 ) { size_t leftToBeRead = 8 - message.GetCursor(); while( leftToBeRead ) { int bytesRead = 0; XRootDStatus status = socket->Read( message.GetBufferAtCursor(), leftToBeRead, bytesRead ); if( !status.IsOK() || status.code == suRetry ) return status; leftToBeRead -= bytesRead; message.AdvanceCursor( bytesRead ); } UnMarshallHeader( message ); uint32_t bodySize = *(uint32_t*)(message.GetBuffer(4)); Log *log = DefaultEnv::GetLog(); log->Dump( XRootDTransportMsg, "[msg: 0x%x] Expecting %d bytes of message " "body", &message, bodySize ); return XRootDStatus( stOK, suDone ); } return XRootDStatus( stError, errInternal ); } //---------------------------------------------------------------------------- // Read message body from socket //---------------------------------------------------------------------------- XRootDStatus XRootDTransport::GetBody( Message &message, Socket *socket ) { //-------------------------------------------------------------------------- // Retrieve the body //-------------------------------------------------------------------------- size_t leftToBeRead = 0; uint32_t bodySize = 0; ServerResponseHeader* rsphdr = (ServerResponseHeader*)message.GetBuffer(); bodySize = rsphdr->dlen; if( message.GetSize() < bodySize + 8 ) message.ReAllocate( bodySize + 8 ); leftToBeRead = bodySize-(message.GetCursor()-8); while( leftToBeRead ) { int bytesRead = 0; XRootDStatus status = socket->Read( message.GetBufferAtCursor(), leftToBeRead, bytesRead ); if( !status.IsOK() || status.code == suRetry ) return status; leftToBeRead -= bytesRead; message.AdvanceCursor( bytesRead ); } return XRootDStatus( stOK, suDone ); } //---------------------------------------------------------------------------- // Read more of the message body from socket //---------------------------------------------------------------------------- XRootDStatus XRootDTransport::GetMore( Message &message, Socket *socket ) { ServerResponseHeader* rsphdr = (ServerResponseHeader*)message.GetBuffer(); if( rsphdr->status != kXR_status ) return XRootDStatus( stError, errInvalidOp ); //-------------------------------------------------------------------------- // In case of non kXR_status responses we read all the response, including // data. For kXR_status responses we first read only the remainder of the // header. The header must then be unmarshalled, and then a second call to // GetMore (repeated for suRetry as needed) will read the data. //-------------------------------------------------------------------------- uint32_t bodySize = rsphdr->dlen; if( bodySize+8 < sizeof( ServerResponseStatus ) ) return XRootDStatus( stError, errInvalidMessage, 0, "kXR_status: invalid message size." ); ServerResponseStatus *rspst = (ServerResponseStatus*)message.GetBuffer(); bodySize += rspst->bdy.dlen; if( message.GetSize() < bodySize + 8 ) message.ReAllocate( bodySize + 8 ); size_t leftToBeRead = bodySize-(message.GetCursor()-8); while( leftToBeRead ) { int bytesRead = 0; XRootDStatus status = socket->Read( message.GetBufferAtCursor(), leftToBeRead, bytesRead ); if( !status.IsOK() || status.code == suRetry ) return status; leftToBeRead -= bytesRead; message.AdvanceCursor( bytesRead ); } // Unmarchal to message body Log *log = DefaultEnv::GetLog(); XRootDStatus st = XRootDTransport::UnMarchalStatusMore( message ); if( !st.IsOK() && st.code == errDataError ) { log->Error( XRootDTransportMsg, "[msg: 0x%x] %s", &message, st.GetErrorMessage().c_str() ); return st; } if( !st.IsOK() ) { log->Error( XRootDTransportMsg, "[msg: 0x%x] Failed to unmarshall status body.", &message ); return st; } return XRootDStatus( stOK, suDone ); } //---------------------------------------------------------------------------- // Initialize channel //---------------------------------------------------------------------------- void XRootDTransport::InitializeChannel( const URL &url, AnyObject &channelData ) { XRootDChannelInfo *info = new XRootDChannelInfo( url ); XrdSysMutexHelper scopedLock( info->mutex ); channelData.Set( info ); Env *env = DefaultEnv::GetEnv(); int streams = DefaultSubStreamsPerChannel; env->GetInt( "SubStreamsPerChannel", streams ); if( streams < 1 ) streams = 1; info->stream.resize( streams ); info->strmSelector.reset( new StreamSelector( streams ) ); info->encrypted = url.IsSecure(); info->istpc = url.IsTPC(); info->logintoken = url.GetLoginToken(); } //---------------------------------------------------------------------------- // Finalize channel //---------------------------------------------------------------------------- void XRootDTransport::FinalizeChannel( AnyObject & ) { } //---------------------------------------------------------------------------- // HandShake //---------------------------------------------------------------------------- XRootDStatus XRootDTransport::HandShake( HandShakeData *handShakeData, AnyObject &channelData ) { XRootDChannelInfo *info = 0; channelData.Get( info ); XrdSysMutexHelper scopedLock( info->mutex ); if( info->stream.size() <= handShakeData->subStreamId ) { Log *log = DefaultEnv::GetLog(); log->Error( XRootDTransportMsg, "[%s] Internal error: not enough substreams", handShakeData->streamName.c_str() ); return XRootDStatus( stFatal, errInternal ); } if( handShakeData->subStreamId == 0 ) { info->streamName = handShakeData->streamName; return HandShakeMain( handShakeData, channelData ); } return HandShakeParallel( handShakeData, channelData ); } //---------------------------------------------------------------------------- // Hand shake the main stream //---------------------------------------------------------------------------- XRootDStatus XRootDTransport::HandShakeMain( HandShakeData *handShakeData, AnyObject &channelData ) { XRootDChannelInfo *info = 0; channelData.Get( info ); XRootDStreamInfo &sInfo = info->stream[handShakeData->subStreamId]; //-------------------------------------------------------------------------- // First step - we need to create and initial handshake and send it out //-------------------------------------------------------------------------- if( sInfo.status == XRootDStreamInfo::Disconnected || sInfo.status == XRootDStreamInfo::Broken ) { handShakeData->out = GenerateInitialHSProtocol( handShakeData, info, ClientProtocolRequest::kXR_ExpLogin ); sInfo.status = XRootDStreamInfo::HandShakeSent; return XRootDStatus( stOK, suContinue ); } //-------------------------------------------------------------------------- // Second step - we got the reply message to the initial handshake //-------------------------------------------------------------------------- if( sInfo.status == XRootDStreamInfo::HandShakeSent ) { XRootDStatus st = ProcessServerHS( handShakeData, info ); if( st.IsOK() ) sInfo.status = XRootDStreamInfo::HandShakeReceived; else sInfo.status = XRootDStreamInfo::Broken; return st; } //-------------------------------------------------------------------------- // Third step - we got the response to the protocol request, we need // to process it and send out a login request //-------------------------------------------------------------------------- if( sInfo.status == XRootDStreamInfo::HandShakeReceived ) { XRootDStatus st = ProcessProtocolResp( handShakeData, info ); if( !st.IsOK() ) { sInfo.status = XRootDStreamInfo::Broken; return st; } if( st.code == suRetry ) { handShakeData->out = GenerateProtocol( handShakeData, info, ClientProtocolRequest::kXR_ExpLogin ); sInfo.status = XRootDStreamInfo::HandShakeReceived; return XRootDStatus( stOK, suRetry ); } handShakeData->out = GenerateLogIn( handShakeData, info ); sInfo.status = XRootDStreamInfo::LoginSent; return XRootDStatus( stOK, suContinue ); } //-------------------------------------------------------------------------- // Fourth step - handle the log in response and proceed with the // authentication if required by the server //-------------------------------------------------------------------------- if( sInfo.status == XRootDStreamInfo::LoginSent ) { XRootDStatus st = ProcessLogInResp( handShakeData, info ); if( !st.IsOK() ) { sInfo.status = XRootDStreamInfo::Broken; return st; } if( st.IsOK() && st.code == suDone ) { //---------------------------------------------------------------------- // If it's not our first log in we need to end the previous session // to make sure that the server noticed our disconnection and closed // all the writable handles that we owned //---------------------------------------------------------------------- if( !info->firstLogIn ) { handShakeData->out = GenerateEndSession( handShakeData, info ); sInfo.status = XRootDStreamInfo::EndSessionSent; return XRootDStatus( stOK, suContinue ); } sInfo.status = XRootDStreamInfo::Connected; info->firstLogIn = false; return st; } st = DoAuthentication( handShakeData, info ); if( !st.IsOK() ) sInfo.status = XRootDStreamInfo::Broken; else sInfo.status = XRootDStreamInfo::AuthSent; return st; } //-------------------------------------------------------------------------- // Fifth step and later - proceed with the authentication //-------------------------------------------------------------------------- if( sInfo.status == XRootDStreamInfo::AuthSent ) { XRootDStatus st = DoAuthentication( handShakeData, info ); if( !st.IsOK() ) { sInfo.status = XRootDStreamInfo::Broken; return st; } if( st.IsOK() && st.code == suDone ) { //---------------------------------------------------------------------- // If it's not our first log in we need to end the previous session //---------------------------------------------------------------------- if( !info->firstLogIn ) { handShakeData->out = GenerateEndSession( handShakeData, info ); sInfo.status = XRootDStreamInfo::EndSessionSent; return XRootDStatus( stOK, suContinue ); } sInfo.status = XRootDStreamInfo::Connected; info->firstLogIn = false; return st; } return st; } //-------------------------------------------------------------------------- // The last step - kXR_endsess returned //-------------------------------------------------------------------------- if( sInfo.status == XRootDStreamInfo::EndSessionSent ) { XRootDStatus st = ProcessEndSessionResp( handShakeData, info ); if( st.IsOK() && st.code == suDone ) { sInfo.status = XRootDStreamInfo::Connected; } else if( !st.IsOK() ) { sInfo.status = XRootDStreamInfo::Broken; } return st; } return XRootDStatus( stOK, suDone ); } //---------------------------------------------------------------------------- // Hand shake parallel stream //---------------------------------------------------------------------------- XRootDStatus XRootDTransport::HandShakeParallel( HandShakeData *handShakeData, AnyObject &channelData ) { XRootDChannelInfo *info = 0; channelData.Get( info ); XRootDStreamInfo &sInfo = info->stream[handShakeData->subStreamId]; //-------------------------------------------------------------------------- // First step - we need to create and initial handshake and send it out //-------------------------------------------------------------------------- if( sInfo.status == XRootDStreamInfo::Disconnected || sInfo.status == XRootDStreamInfo::Broken ) { handShakeData->out = GenerateInitialHSProtocol( handShakeData, info, ClientProtocolRequest::kXR_ExpBind ); sInfo.status = XRootDStreamInfo::HandShakeSent; return XRootDStatus( stOK, suContinue ); } //-------------------------------------------------------------------------- // Second step - we got the reply message to the initial handshake, // if successful we need to send bind //-------------------------------------------------------------------------- if( sInfo.status == XRootDStreamInfo::HandShakeSent ) { XRootDStatus st = ProcessServerHS( handShakeData, info ); if( st.IsOK() ) sInfo.status = XRootDStreamInfo::HandShakeReceived; else sInfo.status = XRootDStreamInfo::Broken; return st; } //-------------------------------------------------------------------------- // Second step bis - we got the response to the protocol request, we need // to process it and send out a bind request //-------------------------------------------------------------------------- if( sInfo.status == XRootDStreamInfo::HandShakeReceived ) { XRootDStatus st = ProcessProtocolResp( handShakeData, info ); if( !st.IsOK() ) { sInfo.status = XRootDStreamInfo::Broken; return st; } handShakeData->out = GenerateBind( handShakeData, info ); sInfo.status = XRootDStreamInfo::BindSent; return XRootDStatus( stOK, suContinue ); } //-------------------------------------------------------------------------- // Third step - we got the response to the kXR_bind //-------------------------------------------------------------------------- if( sInfo.status == XRootDStreamInfo::BindSent ) { XRootDStatus st = ProcessBindResp( handShakeData, info ); if( !st.IsOK() ) { sInfo.status = XRootDStreamInfo::Broken; return st; } sInfo.status = XRootDStreamInfo::Connected; return XRootDStatus(); } return XRootDStatus(); } //------------------------------------------------------------------------ // @return true if handshake has been done and stream is connected, // false otherwise //------------------------------------------------------------------------ bool XRootDTransport::HandShakeDone( HandShakeData *handShakeData, AnyObject &channelData ) { XRootDChannelInfo *info = 0; channelData.Get( info ); XRootDStreamInfo &sInfo = info->stream[handShakeData->subStreamId]; return ( sInfo.status == XRootDStreamInfo::Connected ); } //---------------------------------------------------------------------------- // Check if the stream should be disconnected //---------------------------------------------------------------------------- bool XRootDTransport::IsStreamTTLElapsed( time_t inactiveTime, AnyObject &channelData ) { XRootDChannelInfo *info = 0; channelData.Get( info ); Env *env = DefaultEnv::GetEnv(); Log *log = DefaultEnv::GetLog(); //-------------------------------------------------------------------------- // Check the TTL settings for the current server //-------------------------------------------------------------------------- int ttl; if( info->serverFlags & kXR_isServer ) { ttl = DefaultDataServerTTL; env->GetInt( "DataServerTTL", ttl ); } else { ttl = DefaultLoadBalancerTTL; env->GetInt( "LoadBalancerTTL", ttl ); } //-------------------------------------------------------------------------- // See whether we can give a go-ahead for the disconnection //-------------------------------------------------------------------------- XrdSysMutexHelper scopedLock( info->mutex ); uint16_t allocatedSIDs = info->sidManager->GetNumberOfAllocatedSIDs(); log->Dump( XRootDTransportMsg, "[%s] Stream inactive since %d seconds, " "TTL: %d, allocated SIDs: %d, open files: %d, bound file objects: %d", info->streamName.c_str(), inactiveTime, ttl, allocatedSIDs, info->openFiles, info->finstcnt.load( std::memory_order_relaxed ) ); if( info->openFiles != 0 && info->finstcnt.load( std::memory_order_relaxed ) != 0 ) return false; if( !allocatedSIDs && inactiveTime > ttl ) return true; return false; } //---------------------------------------------------------------------------- // Check the stream is broken - ie. TCP connection got broken and // went undetected by the TCP stack //---------------------------------------------------------------------------- Status XRootDTransport::IsStreamBroken( time_t inactiveTime, AnyObject &channelData ) { XRootDChannelInfo *info = 0; channelData.Get( info ); Env *env = DefaultEnv::GetEnv(); Log *log = DefaultEnv::GetLog(); int streamTimeout = DefaultStreamTimeout; env->GetInt( "StreamTimeout", streamTimeout ); XrdSysMutexHelper scopedLock( info->mutex ); const time_t now = time(0); const bool anySID = info->sidManager->IsAnySIDOldAs( now - streamTimeout ); log->Dump( XRootDTransportMsg, "[%s] Stream inactive since %d seconds, " "stream timeout: %d, any SID: %d, wait barrier: %s", info->streamName.c_str(), inactiveTime, streamTimeout, anySID, Utils::TimeToString(info->waitBarrier).c_str() ); if( inactiveTime < streamTimeout ) return Status(); if( now < info->waitBarrier ) return Status(); if( !anySID ) return Status(); return Status( stError, errSocketTimeout ); } //---------------------------------------------------------------------------- // Multiplex //---------------------------------------------------------------------------- PathID XRootDTransport::Multiplex( Message *, AnyObject &, PathID * ) { return PathID( 0, 0 ); } //---------------------------------------------------------------------------- // Multiplex //---------------------------------------------------------------------------- PathID XRootDTransport::MultiplexSubStream( Message *msg, AnyObject &channelData, PathID *hint ) { XRootDChannelInfo *info = 0; channelData.Get( info ); XrdSysMutexHelper scopedLock( info->mutex ); //-------------------------------------------------------------------------- // If we're not connected to a data server or we don't know that yet // we stream through 0 //-------------------------------------------------------------------------- if( !(info->serverFlags & kXR_isServer) || info->stream.size() == 0 ) return PathID( 0, 0 ); //-------------------------------------------------------------------------- // Select the streams //-------------------------------------------------------------------------- Log *log = DefaultEnv::GetLog(); uint16_t upStream = 0; uint16_t downStream = 0; if( hint ) { upStream = hint->up; downStream = hint->down; } else { upStream = 0; std::vector connected; connected.reserve( info->stream.size() - 1 ); size_t nbConnected = 0; for( size_t i = 1; i < info->stream.size(); ++i ) if( info->stream[i].status == XRootDStreamInfo::Connected ) { connected.push_back( true ); ++nbConnected; } else connected.push_back( false ); if( nbConnected == 0 ) downStream = 0; else downStream = info->strmSelector->Select( connected ); } if( upStream >= info->stream.size() ) { log->Debug( XRootDTransportMsg, "[%s] Up link stream %d does not exist, using 0", info->streamName.c_str(), upStream ); upStream = 0; } if( downStream >= info->stream.size() ) { log->Debug( XRootDTransportMsg, "[%s] Down link stream %d does not exist, using 0", info->streamName.c_str(), downStream ); downStream = 0; } //-------------------------------------------------------------------------- // Modify the message //-------------------------------------------------------------------------- UnMarshallRequest( msg ); ClientRequestHdr *hdr = (ClientRequestHdr*)msg->GetBuffer(); switch( hdr->requestid ) { //------------------------------------------------------------------------ // Read - we update the path id to tell the server where we want to // get the response, but we still send the request through stream 0 // We need to allocate space for read_args if we don't have it // included yet //------------------------------------------------------------------------ case kXR_read: { if( msg->GetSize() < sizeof(ClientReadRequest) + 8 ) { msg->ReAllocate( sizeof(ClientReadRequest) + 8 ); void *newBuf = msg->GetBuffer(sizeof(ClientReadRequest)); memset( newBuf, 0, 8 ); ClientReadRequest *req = (ClientReadRequest*)msg->GetBuffer(); req->dlen += 8; } read_args *args = (read_args*)msg->GetBuffer(sizeof(ClientReadRequest)); args->pathid = info->stream[downStream].pathId; break; } //------------------------------------------------------------------------ // PgRead - we update the path id to tell the server where we want to // get the response, but we still send the request through stream 0 // We need to allocate space for ClientPgReadReqArgs if we don't have it // included yet //------------------------------------------------------------------------ case kXR_pgread: { if( msg->GetSize() < sizeof( ClientPgReadRequest ) + sizeof( ClientPgReadReqArgs ) ) { msg->ReAllocate( sizeof( ClientPgReadRequest ) + sizeof( ClientPgReadReqArgs ) ); void *newBuf = msg->GetBuffer( sizeof( ClientPgReadRequest ) ); memset( newBuf, 0, sizeof( ClientPgReadReqArgs ) ); ClientPgReadRequest *req = (ClientPgReadRequest*)msg->GetBuffer(); req->dlen += sizeof( ClientPgReadReqArgs ); } ClientPgReadReqArgs *args = reinterpret_cast( msg->GetBuffer( sizeof( ClientPgReadRequest ) ) ); args->pathid = info->stream[downStream].pathId; break; } //------------------------------------------------------------------------ // ReadV - the situation is identical to read but we don't need any // additional structures to specify the return path //------------------------------------------------------------------------ case kXR_readv: { ClientReadVRequest *req = (ClientReadVRequest*)msg->GetBuffer(); req->pathid = info->stream[downStream].pathId; break; } //------------------------------------------------------------------------ // Write - multiplexing writes doesn't work properly in the server //------------------------------------------------------------------------ case kXR_write: { // ClientWriteRequest *req = (ClientWriteRequest*)msg->GetBuffer(); // req->pathid = info->stream[downStream].pathId; break; } //------------------------------------------------------------------------ // WriteV - multiplexing writes doesn't work properly in the server //------------------------------------------------------------------------ case kXR_writev: { // ClientWriteVRequest *req = (ClientWriteVRequest*)msg->GetBuffer(); // req->pathid = info->stream[downStream].pathId; break; } //------------------------------------------------------------------------ // PgWrite - multiplexing writes doesn't work properly in the server //------------------------------------------------------------------------ case kXR_pgwrite: { // ClientWriteVRequest *req = (ClientWriteVRequest*)msg->GetBuffer(); // req->pathid = info->stream[downStream].pathId; break; } }; MarshallRequest( msg ); return PathID( upStream, downStream ); } //---------------------------------------------------------------------------- // Return a number of substreams per stream that should be created // This depends on the environment and whether we are connected to // a data server or not //---------------------------------------------------------------------------- uint16_t XRootDTransport::SubStreamNumber( AnyObject &channelData ) { XRootDChannelInfo *info = 0; channelData.Get( info ); XrdSysMutexHelper scopedLock( info->mutex ); //-------------------------------------------------------------------------- // If the connection has been opened in order to orchestrate a TPC or // the remote server is a Manager or Metamanager we will need only one // (control) stream. //-------------------------------------------------------------------------- if( info->istpc || !(info->serverFlags & kXR_isServer ) ) return 1; //-------------------------------------------------------------------------- // Number of streams requested by user //-------------------------------------------------------------------------- uint16_t ret = info->stream.size(); XrdCl::Env *env = XrdCl::DefaultEnv::GetEnv(); int nodata = DefaultTlsNoData; env->GetInt( "TlsNoData", nodata ); // Does the server require the stream 0 to be encrypted? bool srvTlsStrm0 = ( info->serverFlags & kXR_gotoTLS ) || ( info->serverFlags & kXR_tlsLogin ) || ( info->serverFlags & kXR_tlsSess ); // Does the server NOT require the data streams to be encrypted? bool srvNoTlsData = !( info->serverFlags & kXR_tlsData ); // Does the user require the stream 0 to be encrypted? bool usrTlsStrm0 = info->encrypted; // Does the user NOT require the data streams to be encrypted? bool usrNoTlsData = !info->encrypted || ( info->encrypted && nodata ); if( ( usrTlsStrm0 && usrNoTlsData && srvNoTlsData ) || ( srvTlsStrm0 && srvNoTlsData && usrNoTlsData ) ) { //------------------------------------------------------------------------ // The server or user asked us to encrypt stream 0, but to send the data // (read/write) using a plain TCP connection //------------------------------------------------------------------------ if( ret == 1 ) ++ret; } if( ret > info->stream.size() ) { info->stream.resize( ret ); info->strmSelector->AdjustQueues( ret ); } return ret; } //---------------------------------------------------------------------------- // Marshall //---------------------------------------------------------------------------- XRootDStatus XRootDTransport::MarshallRequest( char *msg ) { ClientRequest *req = (ClientRequest*)msg; switch( req->header.requestid ) { //------------------------------------------------------------------------ // kXR_protocol //------------------------------------------------------------------------ case kXR_protocol: req->protocol.clientpv = htonl( req->protocol.clientpv ); break; //------------------------------------------------------------------------ // kXR_login //------------------------------------------------------------------------ case kXR_login: req->login.pid = htonl( req->login.pid ); break; //------------------------------------------------------------------------ // kXR_locate //------------------------------------------------------------------------ case kXR_locate: req->locate.options = htons( req->locate.options ); break; //------------------------------------------------------------------------ // kXR_query //------------------------------------------------------------------------ case kXR_query: req->query.infotype = htons( req->query.infotype ); break; //------------------------------------------------------------------------ // kXR_truncate //------------------------------------------------------------------------ case kXR_truncate: req->truncate.offset = htonll( req->truncate.offset ); break; //------------------------------------------------------------------------ // kXR_mkdir //------------------------------------------------------------------------ case kXR_mkdir: req->mkdir.mode = htons( req->mkdir.mode ); break; //------------------------------------------------------------------------ // kXR_chmod //------------------------------------------------------------------------ case kXR_chmod: req->chmod.mode = htons( req->chmod.mode ); break; //------------------------------------------------------------------------ // kXR_open //------------------------------------------------------------------------ case kXR_open: req->open.mode = htons( req->open.mode ); req->open.options = htons( req->open.options ); break; //------------------------------------------------------------------------ // kXR_read //------------------------------------------------------------------------ case kXR_read: req->read.offset = htonll( req->read.offset ); req->read.rlen = htonl( req->read.rlen ); break; //------------------------------------------------------------------------ // kXR_write //------------------------------------------------------------------------ case kXR_write: req->write.offset = htonll( req->write.offset ); break; //------------------------------------------------------------------------ // kXR_mv //------------------------------------------------------------------------ case kXR_mv: req->mv.arg1len = htons( req->mv.arg1len ); break; //------------------------------------------------------------------------ // kXR_readv //------------------------------------------------------------------------ case kXR_readv: { uint16_t numChunks = (req->readv.dlen)/16; readahead_list *dataChunk = (readahead_list*)( msg + 24 ); for( size_t i = 0; i < numChunks; ++i ) { dataChunk[i].rlen = htonl( dataChunk[i].rlen ); dataChunk[i].offset = htonll( dataChunk[i].offset ); } break; } //------------------------------------------------------------------------ // kXR_writev //------------------------------------------------------------------------ case kXR_writev: { uint16_t numChunks = (req->writev.dlen)/16; XrdProto::write_list *wrtList = reinterpret_cast( msg + 24 ); for( size_t i = 0; i < numChunks; ++i ) { wrtList[i].wlen = htonl( wrtList[i].wlen ); wrtList[i].offset = htonll( wrtList[i].offset ); } break; } case kXR_pgread: { req->pgread.offset = htonll( req->pgread.offset ); req->pgread.rlen = htonl( req->pgread.rlen ); break; } case kXR_pgwrite: { req->pgwrite.offset = htonll( req->pgwrite.offset ); break; } //------------------------------------------------------------------------ // kXR_prepare //------------------------------------------------------------------------ case kXR_prepare: { req->prepare.optionX = htons( req->prepare.optionX ); req->prepare.port = htons( req->prepare.port ); break; } case kXR_chkpoint: { if( req->chkpoint.opcode == kXR_ckpXeq ) MarshallRequest( msg + 24 ); break; } }; req->header.requestid = htons( req->header.requestid ); req->header.dlen = htonl( req->header.dlen ); return XRootDStatus(); } //---------------------------------------------------------------------------- // Unmarshall the request - sometimes the requests need to be rewritten, // so we need to unmarshall them //---------------------------------------------------------------------------- XRootDStatus XRootDTransport::UnMarshallRequest( Message *msg ) { if( !msg->IsMarshalled() ) return XRootDStatus( stOK, suAlreadyDone ); // We rely on the marshaling process to be symmetric! // First we unmarshall the request ID and the length because // MarshallRequest() relies on these, and then we need to unmarshall these // two again, because they get marshalled in MarshallRequest(). // All this is pretty damn ugly and should be rewritten. ClientRequest *req = (ClientRequest*)msg->GetBuffer(); req->header.requestid = htons( req->header.requestid ); req->header.dlen = htonl( req->header.dlen ); XRootDStatus st = MarshallRequest( msg ); req->header.requestid = htons( req->header.requestid ); req->header.dlen = htonl( req->header.dlen ); msg->SetIsMarshalled( false ); return st; } //---------------------------------------------------------------------------- // Unmarshall the body of the incoming message //---------------------------------------------------------------------------- XRootDStatus XRootDTransport::UnMarshallBody( Message *msg, uint16_t reqType ) { ServerResponse *m = (ServerResponse *)msg->GetBuffer(); //-------------------------------------------------------------------------- // kXR_ok //-------------------------------------------------------------------------- if( m->hdr.status == kXR_ok ) { switch( reqType ) { //---------------------------------------------------------------------- // kXR_protocol //---------------------------------------------------------------------- case kXR_protocol: if( m->hdr.dlen < 8 ) return XRootDStatus( stError, errInvalidMessage, 0, "kXR_protocol: body too short." ); m->body.protocol.pval = ntohl( m->body.protocol.pval ); m->body.protocol.flags = ntohl( m->body.protocol.flags ); break; } } //-------------------------------------------------------------------------- // kXR_error //-------------------------------------------------------------------------- else if( m->hdr.status == kXR_error ) { if( m->hdr.dlen < 4 ) return XRootDStatus( stError, errInvalidMessage, 0, "kXR_error: body too short." ); m->body.error.errnum = ntohl( m->body.error.errnum ); } //-------------------------------------------------------------------------- // kXR_wait //-------------------------------------------------------------------------- else if( m->hdr.status == kXR_wait ) { if( m->hdr.dlen < 4 ) return XRootDStatus( stError, errInvalidMessage, 0, "kXR_wait: body too short." ); m->body.wait.seconds = htonl( m->body.wait.seconds ); } //-------------------------------------------------------------------------- // kXR_redirect //-------------------------------------------------------------------------- else if( m->hdr.status == kXR_redirect ) { if( m->hdr.dlen < 4 ) return XRootDStatus( stError, errInvalidMessage, 0, "kXR_redirect: body too short." ); m->body.redirect.port = htonl( m->body.redirect.port ); } //-------------------------------------------------------------------------- // kXR_waitresp //-------------------------------------------------------------------------- else if( m->hdr.status == kXR_waitresp ) { if( m->hdr.dlen < 4 ) return XRootDStatus( stError, errInvalidMessage, 0, "kXR_waitresp: body too short." ); m->body.waitresp.seconds = htonl( m->body.waitresp.seconds ); } //-------------------------------------------------------------------------- // kXR_attn //-------------------------------------------------------------------------- else if( m->hdr.status == kXR_attn ) { if( m->hdr.dlen < 4 ) return XRootDStatus( stError, errInvalidMessage, 0, "kXR_attn: body too short." ); m->body.attn.actnum = htonl( m->body.attn.actnum ); } return XRootDStatus(); } //------------------------------------------------------------------------ //! Unmarshall the body of the status response //------------------------------------------------------------------------ XRootDStatus XRootDTransport::UnMarshalStatusBody( Message &msg, uint16_t reqType ) { //-------------------------------------------------------------------------- // Calculate the crc32c before the unmarshaling the body! //-------------------------------------------------------------------------- ServerResponseStatus *rspst = (ServerResponseStatus*)msg.GetBuffer(); char *buffer = msg.GetBuffer( 8 + sizeof( rspst->bdy.crc32c ) ); size_t length = rspst->hdr.dlen - sizeof( rspst->bdy.crc32c ); uint32_t crcval = XrdOucCRC::Calc32C( buffer, length ); size_t stlen = sizeof( ServerResponseStatus ); switch( reqType ) { case kXR_pgread: { stlen += sizeof( ServerResponseBody_pgRead ); break; } case kXR_pgwrite: { stlen += sizeof( ServerResponseBody_pgWrite ); break; } } if( msg.GetSize() < stlen ) return XRootDStatus( stError, errInvalidMessage, 0, "kXR_status: invalid message size." ); rspst->bdy.crc32c = ntohl( rspst->bdy.crc32c ); rspst->bdy.dlen = ntohl( rspst->bdy.dlen ); switch( reqType ) { case kXR_pgread: { ServerResponseBody_pgRead *pgrdbdy = (ServerResponseBody_pgRead*)msg.GetBuffer( sizeof( ServerResponseStatus ) ); pgrdbdy->offset = ntohll( pgrdbdy->offset ); break; } case kXR_pgwrite: { ServerResponseBody_pgWrite *pgwrtbdy = (ServerResponseBody_pgWrite*)msg.GetBuffer( sizeof( ServerResponseStatus ) ); pgwrtbdy->offset = ntohll( pgwrtbdy->offset ); break; } } //-------------------------------------------------------------------------- // Do the integrity checks //-------------------------------------------------------------------------- if( crcval != rspst->bdy.crc32c ) { return XRootDStatus( stError, errDataError, 0, "kXR_status response header " "corrupted (crc32c integrity check failed)." ); } if( rspst->hdr.streamid[0] != rspst->bdy.streamID[0] || rspst->hdr.streamid[1] != rspst->bdy.streamID[1] ) { return XRootDStatus( stError, errDataError, 0, "response header corrupted " "(stream ID mismatch)." ); } if( rspst->bdy.requestid + kXR_1stRequest != reqType ) { return XRootDStatus( stError, errDataError, 0, "kXR_status response header corrupted " "(request ID mismatch)." ); } return XRootDStatus(); } XRootDStatus XRootDTransport::UnMarchalStatusMore( Message &msg ) { ServerResponseV2 *rsp = (ServerResponseV2*)msg.GetBuffer(); uint16_t reqType = rsp->status.bdy.requestid + kXR_1stRequest; switch( reqType ) { case kXR_pgwrite: { //-------------------------------------------------------------------------- // If there's no additional data there's nothing to unmarshal //-------------------------------------------------------------------------- if( rsp->status.bdy.dlen == 0 ) return XRootDStatus(); //-------------------------------------------------------------------------- // If there's not enough data to form correction-segment report an error //-------------------------------------------------------------------------- if( size_t( rsp->status.bdy.dlen ) < sizeof( ServerResponseBody_pgWrCSE ) ) return XRootDStatus( stError, errInvalidMessage, 0, "kXR_status: invalid message size." ); //-------------------------------------------------------------------------- // Calculate the crc32c for the additional data //-------------------------------------------------------------------------- ServerResponseBody_pgWrCSE *cse = (ServerResponseBody_pgWrCSE*)msg.GetBuffer( sizeof( ServerResponseV2 ) ); cse->cseCRC = ntohl( cse->cseCRC ); size_t length = rsp->status.bdy.dlen - sizeof( uint32_t ); void* buffer = msg.GetBuffer( sizeof( ServerResponseV2 ) + sizeof( uint32_t ) ); uint32_t crcval = XrdOucCRC::Calc32C( buffer, length ); //-------------------------------------------------------------------------- // Do the integrity checks //-------------------------------------------------------------------------- if( crcval != cse->cseCRC ) { return XRootDStatus( stError, errDataError, 0, "kXR_status response header " "corrupted (crc32c integrity check failed)." ); } cse->dlFirst = ntohs( cse->dlFirst ); cse->dlLast = ntohs( cse->dlLast ); size_t pgcnt = ( rsp->status.bdy.dlen - sizeof( ServerResponseBody_pgWrCSE ) ) / sizeof( kXR_int64 ); kXR_int64 *pgoffs = (kXR_int64*)msg.GetBuffer( sizeof( ServerResponseV2 ) + sizeof( ServerResponseBody_pgWrCSE ) ); for( size_t i = 0; i < pgcnt; ++i ) pgoffs[i] = ntohll( pgoffs[i] ); return XRootDStatus(); break; } default: break; } return XRootDStatus( stError, errNotSupported ); } //---------------------------------------------------------------------------- // Unmarshall the header of the incoming message //---------------------------------------------------------------------------- void XRootDTransport::UnMarshallHeader( Message &msg ) { ServerResponseHeader *header = (ServerResponseHeader *)msg.GetBuffer(); header->status = ntohs( header->status ); header->dlen = ntohl( header->dlen ); } //---------------------------------------------------------------------------- // Log server error response //---------------------------------------------------------------------------- void XRootDTransport::LogErrorResponse( const Message &msg ) { Log *log = DefaultEnv::GetLog(); ServerResponse *rsp = (ServerResponse *)msg.GetBuffer(); char *errmsg = new char[rsp->hdr.dlen-3]; errmsg[rsp->hdr.dlen-4] = 0; memcpy( errmsg, rsp->body.error.errmsg, rsp->hdr.dlen-4 ); log->Error( XRootDTransportMsg, "Server responded with an error [%d]: %s", rsp->body.error.errnum, errmsg ); delete [] errmsg; } //------------------------------------------------------------------------ // Number of currently connected data streams //------------------------------------------------------------------------ uint16_t XRootDTransport::NbConnectedStrm( AnyObject &channelData ) { XRootDChannelInfo *info = 0; channelData.Get( info ); XrdSysMutexHelper scopedLock( info->mutex ); uint16_t nbConnected = 0; for( size_t i = 1; i < info->stream.size(); ++i ) if( info->stream[i].status == XRootDStreamInfo::Connected ) ++nbConnected; return nbConnected; } //---------------------------------------------------------------------------- // The stream has been disconnected, do the cleanups //---------------------------------------------------------------------------- void XRootDTransport::Disconnect( AnyObject &channelData, uint16_t subStreamId ) { XRootDChannelInfo *info = 0; channelData.Get( info ); XrdSysMutexHelper scopedLock( info->mutex ); CleanUpProtection( info ); if( !info->stream.empty() ) { XRootDStreamInfo &sInfo = info->stream[subStreamId]; sInfo.status = XRootDStreamInfo::Disconnected; } if( subStreamId == 0 ) { info->sidManager->ReleaseAllTimedOut(); info->sentOpens.clear(); info->sentCloses.clear(); info->openFiles = 0; info->waitBarrier = 0; } } //------------------------------------------------------------------------ // Query the channel //------------------------------------------------------------------------ Status XRootDTransport::Query( uint16_t query, AnyObject &result, AnyObject &channelData ) { XRootDChannelInfo *info = 0; channelData.Get( info ); XrdSysMutexHelper scopedLock( info->mutex ); switch( query ) { //------------------------------------------------------------------------ // Protocol name //------------------------------------------------------------------------ case TransportQuery::Name: result.Set( (const char*)"XRootD", false ); return Status(); //------------------------------------------------------------------------ // Authentication //------------------------------------------------------------------------ case TransportQuery::Auth: result.Set( new std::string( info->authProtocolName ), false ); return Status(); //------------------------------------------------------------------------ // Server flags //------------------------------------------------------------------------ case XRootDQuery::ServerFlags: result.Set( new int( info->serverFlags ), false ); return Status(); //------------------------------------------------------------------------ // Protocol version //------------------------------------------------------------------------ case XRootDQuery::ProtocolVersion: result.Set( new int( info->protocolVersion ), false ); return Status(); case XRootDQuery::IsEncrypted: result.Set( new bool( info->encrypted ), false ); return Status(); }; return Status( stError, errQueryNotSupported ); } //---------------------------------------------------------------------------- // Check whether the transport can hijack the message //---------------------------------------------------------------------------- uint32_t XRootDTransport::MessageReceived( Message &msg, uint16_t subStream, AnyObject &channelData ) { XRootDChannelInfo *info = 0; channelData.Get( info ); XrdSysMutexHelper scopedLock( info->mutex ); Log *log = DefaultEnv::GetLog(); //-------------------------------------------------------------------------- // Update the substream queues //-------------------------------------------------------------------------- info->strmSelector->MsgReceived( subStream ); //-------------------------------------------------------------------------- // Check whether this message is a response to a request that has // timed out, and if so, drop it //-------------------------------------------------------------------------- ServerResponse *rsp = (ServerResponse*)msg.GetBuffer(); if( rsp->hdr.status == kXR_attn ) { return NoAction; } if( info->sidManager->IsTimedOut( rsp->hdr.streamid ) ) { log->Error( XRootDTransportMsg, "Message 0x%x, stream [%d, %d] is a " "response that we're no longer interested in (timed out)", &msg, rsp->hdr.streamid[0], rsp->hdr.streamid[1] ); //------------------------------------------------------------------------ // If it is kXR_waitresp there will be another one, // so we don't release the sid yet //------------------------------------------------------------------------ if( rsp->hdr.status != kXR_waitresp ) info->sidManager->ReleaseTimedOut( rsp->hdr.streamid ); //------------------------------------------------------------------------ // If it is a successful response to an open request // that timed out, we need to send a close //------------------------------------------------------------------------ uint16_t sid; memcpy( &sid, rsp->hdr.streamid, 2 ); std::set::iterator sidIt = info->sentOpens.find( sid ); if( sidIt != info->sentOpens.end() ) { info->sentOpens.erase( sidIt ); if( rsp->hdr.status == kXR_ok ) return RequestClose; } return DigestMsg; } //-------------------------------------------------------------------------- // If we have a wait or waitresp //-------------------------------------------------------------------------- uint32_t seconds = 0; if( rsp->hdr.status == kXR_wait ) seconds = ntohl( rsp->body.wait.seconds ) + 5; // we need extra time // to re-send the request else if( rsp->hdr.status == kXR_waitresp ) { seconds = ntohl( rsp->body.waitresp.seconds ); log->Dump( XRootDMsg, "[%s] Got kXR_waitresp response of %d seconds, " "setting up wait barrier.", info->streamName.c_str(), seconds ); } time_t barrier = time(0) + seconds; if( info->waitBarrier < barrier ) info->waitBarrier = barrier; //-------------------------------------------------------------------------- // If we got a response to an open request, we may need to bump the counter // of open files //-------------------------------------------------------------------------- uint16_t sid; memcpy( &sid, rsp->hdr.streamid, 2 ); std::set::iterator sidIt = info->sentOpens.find( sid ); if( sidIt != info->sentOpens.end() ) { if( rsp->hdr.status == kXR_waitresp ) return NoAction; info->sentOpens.erase( sidIt ); if( rsp->hdr.status == kXR_ok ) { ++info->openFiles; info->finstcnt.fetch_add( 1, std::memory_order_relaxed ); // another file File object instance has been bound with this connection } return NoAction; } //-------------------------------------------------------------------------- // If we got a response to a close, we may need to decrement the counter of // open files //-------------------------------------------------------------------------- sidIt = info->sentCloses.find( sid ); if( sidIt != info->sentCloses.end() ) { if( rsp->hdr.status == kXR_waitresp ) return NoAction; info->sentCloses.erase( sidIt ); --info->openFiles; return NoAction; } return NoAction; } //---------------------------------------------------------------------------- // Notify the transport about a message having been sent //---------------------------------------------------------------------------- void XRootDTransport::MessageSent( Message *msg, uint16_t subStream, uint32_t bytesSent, AnyObject &channelData ) { XRootDChannelInfo *info = 0; channelData.Get( info ); XrdSysMutexHelper scopedLock( info->mutex ); ClientRequest *req = (ClientRequest*)msg->GetBuffer(); uint16_t reqid = ntohs( req->header.requestid ); //-------------------------------------------------------------------------- // We need to track opens to know if we can close streams due to idleness //-------------------------------------------------------------------------- uint16_t sid; memcpy( &sid, req->header.streamid, 2 ); if( reqid == kXR_open ) info->sentOpens.insert( sid ); else if( reqid == kXR_close ) info->sentCloses.insert( sid ); } //---------------------------------------------------------------------------- // Get signature for given message //---------------------------------------------------------------------------- Status XRootDTransport::GetSignature( Message *toSign, Message *&sign, AnyObject &channelData ) { XRootDChannelInfo *info = 0; channelData.Get( info ); return GetSignature( toSign, sign, info ); } //------------------------------------------------------------------------ //! Get signature for given message //------------------------------------------------------------------------ Status XRootDTransport::GetSignature( Message *toSign, Message *&sign, XRootDChannelInfo *info ) { XrdSysRWLockHelper scope( pSecUnloadHandler->lock ); if( pSecUnloadHandler->unloaded ) return Status( stError, errInvalidOp ); ClientRequest *thereq = reinterpret_cast( toSign->GetBuffer() ); if( !info ) return Status( stError, errInternal ); if( info->protection ) { SecurityRequest *newreq = 0; // check if we have to secure the request in the first place if( !( NEED2SECURE ( info->protection )( *thereq ) ) ) return Status(); // secure (sign/encrypt) the request int rc = info->protection->Secure( newreq, *thereq, 0 ); // there was an error if( rc < 0 ) return Status( stError, errInternal, -rc ); sign = new Message(); sign->Grab( reinterpret_cast( newreq ), rc ); } return Status(); } //------------------------------------------------------------------------ //! Decrement file object instance count bound to this channel //------------------------------------------------------------------------ void XRootDTransport::DecFileInstCnt( AnyObject &channelData ) { XRootDChannelInfo *info = 0; channelData.Get( info ); if( info->finstcnt.load( std::memory_order_relaxed ) > 0 ) info->finstcnt.fetch_sub( 1, std::memory_order_relaxed ); } //---------------------------------------------------------------------------- // Wait before exit //---------------------------------------------------------------------------- void XRootDTransport::WaitBeforeExit() { XrdSysRWLockHelper scope( pSecUnloadHandler->lock, false ); // obtain write lock pSecUnloadHandler->unloaded = true; } //---------------------------------------------------------------------------- // @return : true if encryption should be turned on, false otherwise //---------------------------------------------------------------------------- bool XRootDTransport::NeedEncryption( HandShakeData *handShakeData, AnyObject &channelData ) { XRootDChannelInfo *info = 0; channelData.Get( info ); XrdCl::Env *env = XrdCl::DefaultEnv::GetEnv(); int notlsok = DefaultNoTlsOK; env->GetInt( "NoTlsOK", notlsok ); if( notlsok ) return info->encrypted; // Did the server instructed us to switch to TLS right away? if( info->serverFlags & kXR_gotoTLS ) { info->encrypted = true; return true ; } XRootDStreamInfo &sInfo = info->stream[handShakeData->subStreamId]; //-------------------------------------------------------------------------- // The control stream (sub-stream 0) might need to switch to TLS before // login or after login //-------------------------------------------------------------------------- if( handShakeData->subStreamId == 0 ) { //------------------------------------------------------------------------ // We are about to login and the server asked to start encrypting // before login //------------------------------------------------------------------------ if( ( sInfo.status == XRootDStreamInfo::LoginSent ) && ( info->serverFlags & kXR_tlsLogin ) ) { info->encrypted = true; return true; } //-------------------------------------------------------------------- // The hand-shake is done and the server requested to encrypt the session //-------------------------------------------------------------------- if( (sInfo.status == XRootDStreamInfo::Connected || //-------------------------------------------------------------------- // we really need to turn on TLS before we sent kXR_endsess and we // are about to do so (1st enable encryption, then send kXR_endsess) //-------------------------------------------------------------------- sInfo.status == XRootDStreamInfo::EndSessionSent ) && ( info->serverFlags & kXR_tlsSess ) ) { info->encrypted = true; return true; } } //-------------------------------------------------------------------------- // A data stream (sub-stream > 0) if need be will be switched to TLS before // bind. //-------------------------------------------------------------------------- else { //------------------------------------------------------------------------ // We are about to bind a data stream and the server asked to start // encrypting before bind //------------------------------------------------------------------------ if( ( sInfo.status == XRootDStreamInfo::BindSent ) && ( info->serverFlags & kXR_tlsData ) ) { info->encrypted = true; return true; } } return false; } //------------------------------------------------------------------------ // Get bind preference for the next data stream //------------------------------------------------------------------------ URL XRootDTransport::GetBindPreference( const URL &url, AnyObject &channelData ) { XRootDChannelInfo *info = 0; channelData.Get( info ); if( !bool( info->bindSelector ) ) return url; return URL( info->bindSelector->Get() ); } //---------------------------------------------------------------------------- // Generate the message to be sent as an initial handshake // (handshake+kXR_protocol) //---------------------------------------------------------------------------- Message *XRootDTransport::GenerateInitialHSProtocol( HandShakeData *hsData, XRootDChannelInfo *info, kXR_char expect ) { Log *log = DefaultEnv::GetLog(); log->Debug( XRootDTransportMsg, "[%s] Sending out the initial hand shake + kXR_protocol", hsData->streamName.c_str() ); Message *msg = new Message(); msg->Allocate( 20+sizeof(ClientProtocolRequest) ); msg->Zero(); ClientInitHandShake *init = (ClientInitHandShake *)msg->GetBuffer(); init->fourth = htonl(4); init->fifth = htonl(2012); ClientProtocolRequest *proto = (ClientProtocolRequest *)msg->GetBuffer(20); InitProtocolReq( proto, info, expect ); return msg; } //------------------------------------------------------------------------ // Generate the protocol message //------------------------------------------------------------------------ Message *XRootDTransport::GenerateProtocol( HandShakeData *hsData, XRootDChannelInfo *info, kXR_char expect ) { Log *log = DefaultEnv::GetLog(); log->Debug( XRootDTransportMsg, "[%s] Sending out the kXR_protocol", hsData->streamName.c_str() ); Message *msg = new Message(); msg->Allocate( sizeof(ClientProtocolRequest) ); msg->Zero(); ClientProtocolRequest *proto = (ClientProtocolRequest *)msg->GetBuffer(); InitProtocolReq( proto, info, expect ); return msg; } //------------------------------------------------------------------------ // Initialize protocol request //------------------------------------------------------------------------ void XRootDTransport::InitProtocolReq( ClientProtocolRequest *request, XRootDChannelInfo *info, kXR_char expect ) { request->requestid = htons(kXR_protocol); request->clientpv = htonl(kXR_PROTOCOLVERSION); request->flags = ClientProtocolRequest::kXR_secreqs | ClientProtocolRequest::kXR_bifreqs; int notlsok = DefaultNoTlsOK; int tlsnodata = DefaultTlsNoData; XrdCl::Env *env = XrdCl::DefaultEnv::GetEnv(); env->GetInt( "NoTlsOK", notlsok ); if (expect & ClientProtocolRequest::kXR_ExpBind) env->GetInt( "TlsNoData", tlsnodata ); if (info->encrypted || InitTLS()) request->flags |= ClientProtocolRequest::kXR_ableTLS; if (info->encrypted && !(notlsok || tlsnodata)) request->flags |= ClientProtocolRequest::kXR_wantTLS; request->expect = expect; //-------------------------------------------------------------------------- // If we are in the curse of establishing a connection in the context of // TPC update the expect! (this will be never followed be a bind) //-------------------------------------------------------------------------- if( info->istpc ) request->expect = ClientProtocolRequest::kXR_ExpTPC; } //---------------------------------------------------------------------------- // Process the server initial handshake response //---------------------------------------------------------------------------- XRootDStatus XRootDTransport::ProcessServerHS( HandShakeData *hsData, XRootDChannelInfo *info ) { Log *log = DefaultEnv::GetLog(); Message *msg = hsData->in; ServerResponseHeader *respHdr = (ServerResponseHeader *)msg->GetBuffer(); ServerInitHandShake *hs = (ServerInitHandShake *)msg->GetBuffer(4); if( respHdr->status != kXR_ok ) { log->Error( XRootDTransportMsg, "[%s] Invalid hand shake response", hsData->streamName.c_str() ); return XRootDStatus( stFatal, errHandShakeFailed, 0, "Invalid hand shake response." ); } info->protocolVersion = ntohl(hs->protover); info->serverFlags = ntohl(hs->msgval) == kXR_DataServer ? kXR_isServer: kXR_isManager; log->Debug( XRootDTransportMsg, "[%s] Got the server hand shake response (%s, protocol " "version %x)", hsData->streamName.c_str(), ServerFlagsToStr( info->serverFlags ).c_str(), info->protocolVersion ); return XRootDStatus( stOK, suContinue ); } //---------------------------------------------------------------------------- // Process the protocol response //---------------------------------------------------------------------------- XRootDStatus XRootDTransport::ProcessProtocolResp( HandShakeData *hsData, XRootDChannelInfo *info ) { Log *log = DefaultEnv::GetLog(); XRootDStatus st = UnMarshallBody( hsData->in, kXR_protocol ); if( !st.IsOK() ) return st; ServerResponse *rsp = (ServerResponse*)hsData->in->GetBuffer(); if( rsp->hdr.status != kXR_ok ) { log->Error( XRootDTransportMsg, "[%s] kXR_protocol request failed", hsData->streamName.c_str() ); return XRootDStatus( stFatal, errHandShakeFailed, 0, "kXR_protocol request failed" ); } XrdCl::Env *env = XrdCl::DefaultEnv::GetEnv(); int notlsok = DefaultNoTlsOK; env->GetInt( "NoTlsOK", notlsok ); if( rsp->body.protocol.pval < kXR_PROTTLSVERSION && info->encrypted ) { //------------------------------------------------------------------------ // User requested an encrypted connection but the server is to old to // support it! //------------------------------------------------------------------------ if( !notlsok ) return XRootDStatus( stFatal, errTlsError, ENOTSUP, "TLS not supported" ); //------------------------------------------------------------------------ // We are falling back to unencrypted data transmission, as configured // in XRD_NOTLSOK environment variable //------------------------------------------------------------------------ log->Info( XRootDTransportMsg, "[%s] Falling back to unencrypted transmission, server does " "not support TLS encryption.", hsData->streamName.c_str() ); info->encrypted = false; } if( rsp->body.protocol.pval >= 0x297 ) info->serverFlags = rsp->body.protocol.flags; if( rsp->hdr.dlen > 8 ) { info->protRespBody = new ServerResponseBody_Protocol(); info->protRespBody->flags = rsp->body.protocol.flags; info->protRespBody->pval = rsp->body.protocol.pval; char* bodybuff = reinterpret_cast( &rsp->body.protocol.secreq ); size_t bodysize = rsp->hdr.dlen - 8; XRootDStatus st = ProcessProtocolBody( bodybuff, bodysize, info ); if( !st.IsOK() ) return st; } log->Debug( XRootDTransportMsg, "[%s] kXR_protocol successful (%s, protocol version %x)", hsData->streamName.c_str(), ServerFlagsToStr( info->serverFlags ).c_str(), info->protocolVersion ); if( !( info->serverFlags & kXR_haveTLS ) && info->encrypted ) { //------------------------------------------------------------------------ // User requested an encrypted connection but the server was not configured // to support encryption! //------------------------------------------------------------------------ return XRootDStatus( stFatal, errTlsError, ECONNREFUSED, "Server was not configured to support encryption." ); } //-------------------------------------------------------------------------- // Now see if we have to enforce encryption in case the server does not // support PgRead/PgWrite //-------------------------------------------------------------------------- int tlsOnNoPgrw = DefaultWantTlsOnNoPgrw; env->GetInt( "WantTlsOnNoPgrw", tlsOnNoPgrw ); if( !( info->serverFlags & kXR_suppgrw ) && tlsOnNoPgrw ) { //------------------------------------------------------------------------ // If user requested encryption just make sure it is not switched off for // data //------------------------------------------------------------------------ if( info->encrypted ) { log->Debug( XRootDTransportMsg, "[%s] Server does not support PgRead/PgWrite and" " WantTlsOnNoPgrw is on; enforcing encryption for data.", hsData->streamName.c_str() ); env->PutInt( "TlsNoData", DefaultTlsNoData ); } //------------------------------------------------------------------------ // Otherwise, if server is not enforcing data encryption, we will need to // redo the protocol request with kXR_wantTLS set. //------------------------------------------------------------------------ else if( !( info->serverFlags & kXR_tlsData ) && ( info->serverFlags & kXR_haveTLS ) ) { info->encrypted = true; return XRootDStatus( stOK, suRetry ); } } return XRootDStatus( stOK, suContinue ); } XRootDStatus XRootDTransport::ProcessProtocolBody( char *bodybuff, size_t bodysize, XRootDChannelInfo *info ) { //-------------------------------------------------------------------------- // Parse bind preferences //-------------------------------------------------------------------------- XrdProto::bifReqs *bifreq = reinterpret_cast( bodybuff ); if( bodysize >= sizeof( XrdProto::bifReqs ) && bifreq->theTag == 'B' ) { bodybuff += sizeof( XrdProto::bifReqs ); bodysize -= sizeof( XrdProto::bifReqs ); if( bodysize < bifreq->bifILen ) return XRootDStatus( stError, errDataError, 0, "Received incomplete " "protocol response." ); std::string bindprefs_str( bodybuff, bifreq->bifILen ); std::vector bindprefs; Utils::splitString( bindprefs, bindprefs_str, "," ); info->bindSelector.reset( new BindPrefSelector( std::move( bindprefs ) ) ); bodybuff += bifreq->bifILen; bodysize -= bifreq->bifILen; } //-------------------------------------------------------------------------- // Parse security requirements //-------------------------------------------------------------------------- XrdProto::secReqs *secreq = reinterpret_cast( bodybuff ); if( bodysize >= 6 /*XrdProto::secReqs*/ && secreq->theTag == 'S' ) { memcpy( &info->protRespBody->secreq, secreq, bodysize ); info->protRespSize = bodysize + 8 /*pval & flags*/; } return XRootDStatus(); } //---------------------------------------------------------------------------- // Generate the bind message //---------------------------------------------------------------------------- Message *XRootDTransport::GenerateBind( HandShakeData *hsData, XRootDChannelInfo *info ) { Log *log = DefaultEnv::GetLog(); log->Debug( XRootDTransportMsg, "[%s] Sending out the bind request", hsData->streamName.c_str() ); Message *msg = new Message( sizeof( ClientBindRequest ) ); ClientBindRequest *bindReq = (ClientBindRequest *)msg->GetBuffer(); bindReq->requestid = kXR_bind; memcpy( bindReq->sessid, info->sessionId, 16 ); bindReq->dlen = 0; MarshallRequest( msg ); return msg; } //---------------------------------------------------------------------------- // Generate the bind message //---------------------------------------------------------------------------- XRootDStatus XRootDTransport::ProcessBindResp( HandShakeData *hsData, XRootDChannelInfo *info ) { Log *log = DefaultEnv::GetLog(); XRootDStatus st = UnMarshallBody( hsData->in, kXR_bind ); if( !st.IsOK() ) return st; ServerResponse *rsp = (ServerResponse*)hsData->in->GetBuffer(); if( rsp->hdr.status != kXR_ok ) { log->Error( XRootDTransportMsg, "[%s] kXR_bind request failed", hsData->streamName.c_str() ); return XRootDStatus( stFatal, errHandShakeFailed, 0, "kXR_bind request failed" ); } info->stream[hsData->subStreamId].pathId = rsp->body.bind.substreamid; log->Debug( XRootDTransportMsg, "[%s] kXR_bind successful", hsData->streamName.c_str() ); return XRootDStatus(); } //---------------------------------------------------------------------------- // Generate the login message //---------------------------------------------------------------------------- Message *XRootDTransport::GenerateLogIn( HandShakeData *hsData, XRootDChannelInfo *info ) { Log *log = DefaultEnv::GetLog(); Env *env = DefaultEnv::GetEnv(); //-------------------------------------------------------------------------- // Compute the login cgi //-------------------------------------------------------------------------- int timeZone = XrdSysTimer::TimeZone(); char *hostName = XrdNetUtils::MyHostName(); std::string countryCode = Utils::FQDNToCC( hostName ); char *cgiBuffer = new char[1024 + info->logintoken.size()]; std::string appName; std::string monInfo; env->GetString( "AppName", appName ); env->GetString( "MonInfo", monInfo ); if( info->logintoken.empty() ) { snprintf( cgiBuffer, 1024, "xrd.cc=%s&xrd.tz=%d&xrd.appname=%s&xrd.info=%s&" "xrd.hostname=%s&xrd.rn=%s", countryCode.c_str(), timeZone, appName.c_str(), monInfo.c_str(), hostName, XrdVERSION ); } else { snprintf( cgiBuffer, 1024, "xrd.cc=%s&xrd.tz=%d&xrd.appname=%s&xrd.info=%s&" "xrd.hostname=%s&xrd.rn=%s&%s", countryCode.c_str(), timeZone, appName.c_str(), monInfo.c_str(), hostName, XrdVERSION, info->logintoken.c_str() ); } uint16_t cgiLen = strlen( cgiBuffer ); free( hostName ); //-------------------------------------------------------------------------- // Generate the message //-------------------------------------------------------------------------- Message *msg = new Message( sizeof(ClientLoginRequest) + cgiLen ); ClientLoginRequest *loginReq = (ClientLoginRequest *)msg->GetBuffer(); loginReq->requestid = kXR_login; loginReq->pid = ::getpid(); loginReq->capver[0] = kXR_asyncap | kXR_ver005; loginReq->dlen = cgiLen; loginReq->ability = kXR_fullurl | kXR_readrdok | kXR_lclfile | kXR_redirflags; #ifdef WITH_XRDEC loginReq->ability2 = kXR_ecredir; #endif int multiProtocol = 0; env->GetInt( "MultiProtocol", multiProtocol ); if(multiProtocol) loginReq->ability |= kXR_multipr; //-------------------------------------------------------------------------- // Check the IP stacks //-------------------------------------------------------------------------- XrdNetUtils::NetProt stacks = XrdNetUtils::NetConfig(); bool dualStack = false; bool privateIPv6 = false; bool privateIPv4 = false; if( (stacks & XrdNetUtils::hasIP64) == XrdNetUtils::hasIP64 ) { dualStack = true; loginReq->ability |= kXR_hasipv64; } if( (stacks & XrdNetUtils::hasIPv6) && !(stacks & XrdNetUtils::hasPub6) ) { privateIPv6 = true; loginReq->ability |= kXR_onlyprv6; } if( (stacks & XrdNetUtils::hasIPv4) && !(stacks & XrdNetUtils::hasPub4) ) { privateIPv4 = true; loginReq->ability |= kXR_onlyprv4; } // The following code snippet tries to overcome the problem that this host // may still be dual-stacked but we don't know it because one of the // interfaces was not registered in DNS. // if( !dualStack && hsData->serverAddr ) {if ( ( ( stacks & XrdNetUtils::hasIPv4 ) && hsData->serverAddr->isIPType(XrdNetAddrInfo::IPv6)) || ( ( stacks & XrdNetUtils::hasIPv6 ) && hsData->serverAddr->isIPType(XrdNetAddrInfo::IPv4))) {dualStack = true; loginReq->ability |= kXR_hasipv64; } } //-------------------------------------------------------------------------- // Check the username //-------------------------------------------------------------------------- std::string buffer( 8, 0 ); if( hsData->url->GetUserName().length() ) buffer = hsData->url->GetUserName(); else { char *name = new char[1024]; if( !XrdOucUtils::UserName( geteuid(), name, 1024 ) ) buffer = name; else buffer = "_anon_"; delete [] name; } buffer.resize( 8, 0 ); std::copy( buffer.begin(), buffer.end(), (char*)loginReq->username ); msg->Append( cgiBuffer, cgiLen, 24 ); log->Debug( XRootDTransportMsg, "[%s] Sending out kXR_login request, " "username: %s, cgi: %s, dual-stack: %s, private IPv4: %s, " "private IPv6: %s", hsData->streamName.c_str(), loginReq->username, cgiBuffer, dualStack ? "true" : "false", privateIPv4 ? "true" : "false", privateIPv6 ? "true" : "false" ); delete [] cgiBuffer; MarshallRequest( msg ); return msg; } //---------------------------------------------------------------------------- // Process the protocol response //---------------------------------------------------------------------------- XRootDStatus XRootDTransport::ProcessLogInResp( HandShakeData *hsData, XRootDChannelInfo *info ) { Log *log = DefaultEnv::GetLog(); XRootDStatus st = UnMarshallBody( hsData->in, kXR_login ); if( !st.IsOK() ) return st; ServerResponse *rsp = (ServerResponse*)hsData->in->GetBuffer(); if( rsp->hdr.status != kXR_ok ) { log->Error( XRootDTransportMsg, "[%s] Got invalid login response", hsData->streamName.c_str() ); return XRootDStatus( stFatal, errLoginFailed, 0, "Got invalid login response." ); } if( !info->firstLogIn ) memcpy( info->oldSessionId, info->sessionId, 16 ); if( rsp->hdr.dlen == 0 && info->protocolVersion <= 0x289 ) { //-------------------------------------------------------------------------- // This if statement is there only to support dCache inaccurate // implementation of XRoot protocol, that in some cases returns // an empty login response for protocol version <= 2.8.9. //-------------------------------------------------------------------------- memset( info->sessionId, 0, 16 ); log->Warning( XRootDTransportMsg, "[%s] Logged in, accepting empty login response.", hsData->streamName.c_str() ); return XRootDStatus(); } if( rsp->hdr.dlen < 16 ) return XRootDStatus( stError, errDataError, 0, "Login response too short." ); memcpy( info->sessionId, rsp->body.login.sessid, 16 ); std::string sessId = Utils::Char2Hex( rsp->body.login.sessid, 16 ); log->Debug( XRootDTransportMsg, "[%s] Logged in, session: %s", hsData->streamName.c_str(), sessId.c_str() ); //-------------------------------------------------------------------------- // We have an authentication info to process //-------------------------------------------------------------------------- if( rsp->hdr.dlen > 16 ) { size_t len = rsp->hdr.dlen-16; info->authBuffer = new char[len+1]; info->authBuffer[len] = 0; memcpy( info->authBuffer, rsp->body.login.sec, len ); log->Debug( XRootDTransportMsg, "[%s] Authentication is required: %s", hsData->streamName.c_str(), info->authBuffer ); return XRootDStatus( stOK, suContinue ); } return XRootDStatus(); } //---------------------------------------------------------------------------- // Do the authentication //---------------------------------------------------------------------------- XRootDStatus XRootDTransport::DoAuthentication( HandShakeData *hsData, XRootDChannelInfo *info ) { //-------------------------------------------------------------------------- // Prepare //-------------------------------------------------------------------------- Log *log = DefaultEnv::GetLog(); XRootDStreamInfo &sInfo = info->stream[hsData->subStreamId]; XrdSecCredentials *credentials = 0; std::string protocolName; //-------------------------------------------------------------------------- // We're doing this for the first time //-------------------------------------------------------------------------- if( sInfo.status == XRootDStreamInfo::LoginSent ) { log->Debug( XRootDTransportMsg, "[%s] Sending authentication data", hsData->streamName.c_str() ); //------------------------------------------------------------------------ // Set up the authentication environment //------------------------------------------------------------------------ info->authEnv = new XrdOucEnv(); info->authEnv->Put( "sockname", hsData->clientName.c_str() ); info->authEnv->Put( "username", hsData->url->GetUserName().c_str() ); info->authEnv->Put( "password", hsData->url->GetPassword().c_str() ); const URL::ParamsMap &urlParams = hsData->url->GetParams(); URL::ParamsMap::const_iterator it; for( it = urlParams.begin(); it != urlParams.end(); ++it ) { if( it->first.compare( 0, 4, "xrd." ) == 0 || it->first.compare( 0, 6, "xrdcl." ) == 0 ) info->authEnv->Put( it->first.c_str(), it->second.c_str() ); } //------------------------------------------------------------------------ // Initialize some other structs //------------------------------------------------------------------------ size_t authBuffLen = strlen( info->authBuffer ); char *pars = (char *)malloc( authBuffLen + 1 ); memcpy( pars, info->authBuffer, authBuffLen ); info->authParams = new XrdSecParameters( pars, authBuffLen ); sInfo.status = XRootDStreamInfo::AuthSent; delete [] info->authBuffer; info->authBuffer = 0; //------------------------------------------------------------------------ // Find a protocol that gives us valid credentials //------------------------------------------------------------------------ XRootDStatus st = GetCredentials( credentials, hsData, info ); if( !st.IsOK() ) { CleanUpAuthentication( info ); return st; } protocolName = info->authProtocol->Entity.prot; } //-------------------------------------------------------------------------- // We've been here already //-------------------------------------------------------------------------- else { ServerResponse *rsp = (ServerResponse*)hsData->in->GetBuffer(); protocolName = info->authProtocol->Entity.prot; //------------------------------------------------------------------------ // We're required to send out more authentication data //------------------------------------------------------------------------ if( rsp->hdr.status == kXR_authmore ) { log->Debug( XRootDTransportMsg, "[%s] Sending more authentication data for %s", hsData->streamName.c_str(), protocolName.c_str() ); uint32_t len = rsp->hdr.dlen; char *secTokenData = (char*)malloc( len ); memcpy( secTokenData, rsp->body.authmore.data, len ); XrdSecParameters *secToken = new XrdSecParameters( secTokenData, len ); XrdOucErrInfo ei( "", info->authEnv); credentials = info->authProtocol->getCredentials( secToken, &ei ); delete secToken; //---------------------------------------------------------------------- // The protocol handler refuses to give us the data //---------------------------------------------------------------------- if( !credentials ) { log->Error( XRootDTransportMsg, "[%s] Auth protocol handler for %s refuses to give " "us more credentials %s", hsData->streamName.c_str(), protocolName.c_str(), ei.getErrText() ); CleanUpAuthentication( info ); return XRootDStatus( stFatal, errAuthFailed, 0, ei.getErrText() ); } } //------------------------------------------------------------------------ // We have succeeded //------------------------------------------------------------------------ else if( rsp->hdr.status == kXR_ok ) { info->authProtocolName = info->authProtocol->Entity.prot; //---------------------------------------------------------------------- // Do we need protection? //---------------------------------------------------------------------- if( info->protRespBody ) { int rc = XrdSecGetProtection( info->protection, *info->authProtocol, *info->protRespBody, info->protRespSize ); if( rc > 0 ) { log->Debug( XRootDTransportMsg, "[%s] XrdSecProtect loaded.", hsData->streamName.c_str() ); } else if( rc == 0 ) { log->Debug( XRootDTransportMsg, "[%s] XrdSecProtect: no protection needed.", hsData->streamName.c_str() ); } else { log->Debug( XRootDTransportMsg, "[%s] Failed to load XrdSecProtect: %s", hsData->streamName.c_str(), XrdSysE2T( -rc ) ); CleanUpAuthentication( info ); return XRootDStatus( stError, errAuthFailed, -rc, XrdSysE2T( -rc ) ); } } if( !info->protection ) CleanUpAuthentication( info ); else pSecUnloadHandler->Register( info->authProtocolName ); log->Debug( XRootDTransportMsg, "[%s] Authenticated with %s.", hsData->streamName.c_str(), protocolName.c_str() ); //-------------------------------------------------------------------- // Clear the SSL error queue of the calling thread, as there might be // some leftover from the authentication! //-------------------------------------------------------------------- Tls::ClearErrorQueue(); return XRootDStatus(); } //------------------------------------------------------------------------ // Failure //------------------------------------------------------------------------ else if( rsp->hdr.status == kXR_error ) { char *errmsg = new char[rsp->hdr.dlen-3]; errmsg[rsp->hdr.dlen-4] = 0; memcpy( errmsg, rsp->body.error.errmsg, rsp->hdr.dlen-4 ); log->Error( XRootDTransportMsg, "[%s] Authentication with %s failed: %s", hsData->streamName.c_str(), protocolName.c_str(), errmsg ); delete [] errmsg; info->authProtocol->Delete(); info->authProtocol = 0; //---------------------------------------------------------------------- // Find another protocol that gives us valid credentials //---------------------------------------------------------------------- XRootDStatus st = GetCredentials( credentials, hsData, info ); if( !st.IsOK() ) { CleanUpAuthentication( info ); return st; } protocolName = info->authProtocol->Entity.prot; } //------------------------------------------------------------------------ // God knows what //------------------------------------------------------------------------ else { info->authProtocolName = info->authProtocol->Entity.prot; CleanUpAuthentication( info ); log->Error( XRootDTransportMsg, "[%s] Authentication with %s failed: unexpected answer", hsData->streamName.c_str(), protocolName.c_str() ); return XRootDStatus( stFatal, errAuthFailed, 0, "Authentication failed: unexpected answer." ); } } //-------------------------------------------------------------------------- // Generate the client request //-------------------------------------------------------------------------- Message *msg = new Message( sizeof(ClientAuthRequest)+credentials->size ); msg->Zero(); ClientRequest *req = (ClientRequest*)msg->GetBuffer(); char *reqBuffer = msg->GetBuffer(sizeof(ClientAuthRequest)); req->header.requestid = kXR_auth; req->auth.dlen = credentials->size; memcpy( req->auth.credtype, protocolName.c_str(), protocolName.length() > 4 ? 4 : protocolName.length() ); memcpy( reqBuffer, credentials->buffer, credentials->size ); hsData->out = msg; MarshallRequest( msg ); delete credentials; //------------------------------------------------------------------------ // Clear the SSL error queue of the calling thread, as there might be // some leftover from the authentication! //------------------------------------------------------------------------ Tls::ClearErrorQueue(); return XRootDStatus( stOK, suContinue ); } //------------------------------------------------------------------------ // Get the initial credentials using one of the protocols //------------------------------------------------------------------------ XRootDStatus XRootDTransport::GetCredentials( XrdSecCredentials *&credentials, HandShakeData *hsData, XRootDChannelInfo *info ) { //-------------------------------------------------------------------------- // Set up the auth handler //-------------------------------------------------------------------------- Log *log = DefaultEnv::GetLog(); XrdOucErrInfo ei( "", info->authEnv); XrdSecGetProt_t authHandler = GetAuthHandler(); if( !authHandler ) return XRootDStatus( stFatal, errAuthFailed, 0, "Could not load authentication handler." ); //-------------------------------------------------------------------------- // Retrieve secuid and secgid, if available. These will override the fsuid // and fsgid of the current thread reading the credentials to prevent // security holes in case this process is running with elevated permissions. //-------------------------------------------------------------------------- char *secuidc = (ei.getEnv()) ? ei.getEnv()->Get("xrdcl.secuid") : 0; char *secgidc = (ei.getEnv()) ? ei.getEnv()->Get("xrdcl.secgid") : 0; int secuid = -1; int secgid = -1; if(secuidc) secuid = atoi(secuidc); if(secgidc) secgid = atoi(secgidc); #ifdef __linux__ ScopedFsUidSetter uidSetter(secuid, secgid, hsData->streamName); if(!uidSetter.IsOk()) { log->Error( XRootDTransportMsg, "[%s] Error while setting (fsuid, fsgid) to (%d, %d)", hsData->streamName.c_str(), secuid, secgid ); return XRootDStatus( stFatal, errAuthFailed, 0, "Error while setting (fsuid, fsgid)." ); } #else if(secuid >= 0 || secgid >= 0) { log->Error( XRootDTransportMsg, "[%s] xrdcl.secuid and xrdcl.secgid only supported on Linux.", hsData->streamName.c_str() ); return XRootDStatus( stFatal, errAuthFailed, 0, "xrdcl.secuid and xrdcl.secgid" " only supported on Linux" ); } #endif //-------------------------------------------------------------------------- // Loop over the possible protocols to find one that gives us valid // credentials //-------------------------------------------------------------------------- XrdNetAddr &srvAddrInfo = *const_cast(hsData->serverAddr); srvAddrInfo.SetTLS( info->encrypted ); while(1) { //------------------------------------------------------------------------ // Get the protocol //------------------------------------------------------------------------ info->authProtocol = (*authHandler)( hsData->url->GetHostName().c_str(), srvAddrInfo, *info->authParams, &ei ); if( !info->authProtocol ) { log->Error( XRootDTransportMsg, "[%s] No protocols left to try", hsData->streamName.c_str() ); return XRootDStatus( stFatal, errAuthFailed, 0, "No protocols left to try" ); } std::string protocolName = info->authProtocol->Entity.prot; log->Debug( XRootDTransportMsg, "[%s] Trying to authenticate using %s", hsData->streamName.c_str(), protocolName.c_str() ); //------------------------------------------------------------------------ // Get the credentials from the current protocol //------------------------------------------------------------------------ credentials = info->authProtocol->getCredentials( 0, &ei ); if( !credentials ) { log->Debug( XRootDTransportMsg, "[%s] Cannot get credentials for protocol %s: %s", hsData->streamName.c_str(), protocolName.c_str(), ei.getErrText() ); info->authProtocol->Delete(); continue; } return XRootDStatus( stOK, suContinue ); } } //------------------------------------------------------------------------ // Clean up the data structures created for the authentication process //------------------------------------------------------------------------ Status XRootDTransport::CleanUpAuthentication( XRootDChannelInfo *info ) { if( info->authProtocol ) info->authProtocol->Delete(); delete info->authParams; delete info->authEnv; info->authProtocol = 0; info->authParams = 0; info->authEnv = 0; Tls::ClearErrorQueue(); return Status(); } //------------------------------------------------------------------------ // Clean up the data structures created for the protection purposes //------------------------------------------------------------------------ Status XRootDTransport::CleanUpProtection( XRootDChannelInfo *info ) { XrdSysRWLockHelper scope( pSecUnloadHandler->lock ); if( pSecUnloadHandler->unloaded ) return Status( stError, errInvalidOp ); if( info->protection ) { info->protection->Delete(); info->protection = 0; CleanUpAuthentication( info ); } if( info->protRespBody ) { delete info->protRespBody; info->protRespBody = 0; info->protRespSize = 0; } return Status(); } //---------------------------------------------------------------------------- // Get the authentication function handle //---------------------------------------------------------------------------- XrdSecGetProt_t XRootDTransport::GetAuthHandler() { Log *log = DefaultEnv::GetLog(); char errorBuff[1024]; // the static constructor is invoked only once and it is guaranteed that this // is thread safe static std::atomic authHandler( XrdSecLoadSecFactory( errorBuff, 1024 ) ); auto ret = authHandler.load( std::memory_order_relaxed ); if( ret ) return ret; // if we are here it means we failed to load the security library for the // first time and we hope the environment changed // obtain a lock static XrdSysMutex mtx; XrdSysMutexHelper lck( mtx ); // check if in the meanwhile some else didn't load the library ret = authHandler.load( std::memory_order_relaxed ); if( ret ) return ret; // load the library ret = XrdSecLoadSecFactory( errorBuff, 1024 ); authHandler.store( ret, std::memory_order_relaxed ); // if we failed report an error if( !ret ) { log->Error( XRootDTransportMsg, "Unable to get the security framework: %s", errorBuff ); return 0; } return ret; } //---------------------------------------------------------------------------- // Generate the end session message //---------------------------------------------------------------------------- Message *XRootDTransport::GenerateEndSession( HandShakeData *hsData, XRootDChannelInfo *info ) { Log *log = DefaultEnv::GetLog(); //-------------------------------------------------------------------------- // Generate the message //-------------------------------------------------------------------------- Message *msg = new Message( sizeof(ClientEndsessRequest) ); ClientEndsessRequest *endsessReq = (ClientEndsessRequest *)msg->GetBuffer(); endsessReq->requestid = kXR_endsess; memcpy( endsessReq->sessid, info->oldSessionId, 16 ); std::string sessId = Utils::Char2Hex( endsessReq->sessid, 16 ); log->Debug( XRootDTransportMsg, "[%s] Sending out kXR_endsess for session:" " %s", hsData->streamName.c_str(), sessId.c_str() ); MarshallRequest( msg ); Message *sign = 0; GetSignature( msg, sign, info ); if( sign ) { //------------------------------------------------------------------------ // Now place both the signature and the request in a single buffer //------------------------------------------------------------------------ uint32_t size = sign->GetSize(); sign->ReAllocate( size + msg->GetSize() ); char* buffer = sign->GetBuffer( size ); memcpy( buffer, msg->GetBuffer(), msg->GetSize() ); msg->Grab( sign->GetBuffer(), sign->GetSize() ); } return msg; } //---------------------------------------------------------------------------- // Process the protocol response //---------------------------------------------------------------------------- Status XRootDTransport::ProcessEndSessionResp( HandShakeData *hsData, XRootDChannelInfo *info ) { Log *log = DefaultEnv::GetLog(); Status st = UnMarshallBody( hsData->in, kXR_endsess ); if( !st.IsOK() ) return st; ServerResponse *rsp = (ServerResponse*)hsData->in->GetBuffer(); // If we're good, we're good! if( rsp->hdr.status == kXR_ok ) return Status(); // we ignore not found errors as such an error means the connection // has been already terminated if( rsp->hdr.status == kXR_error && rsp->body.error.errnum == kXR_NotFound ) return Status(); // other errors if( rsp->hdr.status == kXR_error ) { std::string errorMsg( rsp->body.error.errmsg, rsp->hdr.dlen - 4 ); log->Error( XRootDTransportMsg, "[%s] Got error response to " "kXR_endsess: %s", hsData->streamName.c_str(), errorMsg.c_str() ); return Status( stFatal, errHandShakeFailed ); } // Wait Response. if( rsp->hdr.status == kXR_wait ) { std::string msg( rsp->body.wait.infomsg, rsp->hdr.dlen - 4 ); log->Info( XRootDTransportMsg, "[%s] Got wait response to " "kXR_endsess: %s", hsData->streamName.c_str(), msg.c_str() ); hsData->out = GenerateEndSession( hsData, info ); return Status( stOK, suRetry ); } // Any other response is protocol violation return Status( stError, errDataError ); } //---------------------------------------------------------------------------- // Get a string representation of the server flags //---------------------------------------------------------------------------- std::string XRootDTransport::ServerFlagsToStr( uint32_t flags ) { std::string repr = "type: "; if( flags & kXR_isManager ) repr += "manager "; else if( flags & kXR_isServer ) repr += "server "; repr += "["; if( flags & kXR_attrMeta ) repr += "meta "; else if( flags & kXR_attrProxy ) repr += "proxy "; else if( flags & kXR_attrSuper ) repr += "super "; else repr += " "; repr.erase( repr.length()-1, 1 ); repr += "]"; return repr; } } namespace { // Extract file name from a request //---------------------------------------------------------------------------- char *GetDataAsString( char *msg ) { ClientRequestHdr *req = (ClientRequestHdr*)msg; char *fn = new char[req->dlen+1]; memcpy( fn, msg + 24, req->dlen ); fn[req->dlen] = 0; return fn; } } namespace XrdCl { //---------------------------------------------------------------------------- // Get the description of a message //---------------------------------------------------------------------------- void XRootDTransport::GenerateDescription( char *msg, std::ostringstream &o ) { Log *log = DefaultEnv::GetLog(); if( log->GetLevel() < Log::ErrorMsg ) return; ClientRequestHdr *req = (ClientRequestHdr *)msg; switch( req->requestid ) { //------------------------------------------------------------------------ // kXR_open //------------------------------------------------------------------------ case kXR_open: { ClientOpenRequest *sreq = (ClientOpenRequest *)msg; o << "kXR_open ("; char *fn = GetDataAsString( msg ); o << "file: " << fn << ", "; delete [] fn; o << "mode: 0" << std::setbase(8) << sreq->mode << ", "; o << std::setbase(10); o << "flags: "; if( sreq->options == 0 ) o << "none"; else { if( sreq->options & kXR_delete ) o << "kXR_delete "; if( sreq->options & kXR_force ) o << "kXR_force "; if( sreq->options & kXR_mkpath ) o << "kXR_mkpath "; if( sreq->options & kXR_new ) o << "kXR_new "; if( sreq->options & kXR_nowait ) o << "kXR_delete "; if( sreq->options & kXR_open_apnd ) o << "kXR_open_apnd "; if( sreq->options & kXR_open_read ) o << "kXR_open_read "; if( sreq->options & kXR_open_updt ) o << "kXR_open_updt "; if( sreq->options & kXR_posc ) o << "kXR_posc "; if( sreq->options & kXR_refresh ) o << "kXR_refresh "; if( sreq->options & kXR_replica ) o << "kXR_replica "; if( sreq->options & kXR_seqio ) o << "kXR_seqio "; if( sreq->options & kXR_async ) o << "kXR_async "; if( sreq->options & kXR_retstat ) o << "kXR_retstat "; } o << ")"; break; } //------------------------------------------------------------------------ // kXR_close //------------------------------------------------------------------------ case kXR_close: { ClientCloseRequest *sreq = (ClientCloseRequest *)msg; o << "kXR_close ("; o << "handle: " << FileHandleToStr( sreq->fhandle ); o << ")"; break; } //------------------------------------------------------------------------ // kXR_stat //------------------------------------------------------------------------ case kXR_stat: { ClientStatRequest *sreq = (ClientStatRequest *)msg; o << "kXR_stat ("; if( sreq->dlen ) { char *fn = GetDataAsString( msg );; o << "path: " << fn << ", "; delete [] fn; } else { o << "handle: " << FileHandleToStr( sreq->fhandle ); o << ", "; } o << "flags: "; if( sreq->options == 0 ) o << "none"; else { if( sreq->options & kXR_vfs ) o << "kXR_vfs"; } o << ")"; break; } //------------------------------------------------------------------------ // kXR_read //------------------------------------------------------------------------ case kXR_read: { ClientReadRequest *sreq = (ClientReadRequest *)msg; o << "kXR_read ("; o << "handle: " << FileHandleToStr( sreq->fhandle ); o << std::setbase(10); o << ", "; o << "offset: " << sreq->offset << ", "; o << "size: " << sreq->rlen << ")"; break; } //------------------------------------------------------------------------ // kXR_pgread //------------------------------------------------------------------------ case kXR_pgread: { ClientPgReadRequest *sreq = (ClientPgReadRequest *)msg; o << "kXR_pgread ("; o << "handle: " << FileHandleToStr( sreq->fhandle ); o << std::setbase(10); o << ", "; o << "offset: " << sreq->offset << ", "; o << "size: " << sreq->rlen << ")"; break; } //------------------------------------------------------------------------ // kXR_write //------------------------------------------------------------------------ case kXR_write: { ClientWriteRequest *sreq = (ClientWriteRequest *)msg; o << "kXR_write ("; o << "handle: " << FileHandleToStr( sreq->fhandle ); o << std::setbase(10); o << ", "; o << "offset: " << sreq->offset << ", "; o << "size: " << sreq->dlen << ")"; break; } //------------------------------------------------------------------------ // kXR_pgwrite //------------------------------------------------------------------------ case kXR_pgwrite: { ClientPgWriteRequest *sreq = (ClientPgWriteRequest *)msg; o << "kXR_pgwrite ("; o << "handle: " << FileHandleToStr( sreq->fhandle ); o << std::setbase(10); o << ", "; o << "offset: " << sreq->offset << ", "; o << "size: " << sreq->dlen << ")"; break; } //------------------------------------------------------------------------ // kXR_sync //------------------------------------------------------------------------ case kXR_sync: { ClientSyncRequest *sreq = (ClientSyncRequest *)msg; o << "kXR_sync ("; o << "handle: " << FileHandleToStr( sreq->fhandle ); o << ")"; break; } //------------------------------------------------------------------------ // kXR_truncate //------------------------------------------------------------------------ case kXR_truncate: { ClientTruncateRequest *sreq = (ClientTruncateRequest *)msg; o << "kXR_truncate ("; if( !sreq->dlen ) o << "handle: " << FileHandleToStr( sreq->fhandle ); else { char *fn = GetDataAsString( msg ); o << "file: " << fn; delete [] fn; } o << std::setbase(10); o << ", "; o << "offset: " << sreq->offset; o << ")"; break; } //------------------------------------------------------------------------ // kXR_readv //------------------------------------------------------------------------ case kXR_readv: { unsigned char *fhandle = 0; o << "kXR_readv ("; o << "handle: "; readahead_list *dataChunk = (readahead_list*)(msg + 24 ); fhandle = dataChunk[0].fhandle; if( fhandle ) o << FileHandleToStr( fhandle ); else o << "unknown"; o << ", "; o << std::setbase(10); o << "chunks: ["; uint64_t size = 0; for( size_t i = 0; i < req->dlen/sizeof(readahead_list); ++i ) { size += dataChunk[i].rlen; o << "(offset: " << dataChunk[i].offset; o << ", size: " << dataChunk[i].rlen << "); "; } o << "], "; o << "total size: " << size << ")"; break; } //------------------------------------------------------------------------ // kXR_writev //------------------------------------------------------------------------ case kXR_writev: { unsigned char *fhandle = 0; o << "kXR_writev ("; XrdProto::write_list *wrtList = reinterpret_cast( msg + 24 ); uint64_t size = 0; uint32_t numChunks = 0; for( size_t i = 0; i < req->dlen/sizeof(XrdProto::write_list); ++i ) { fhandle = wrtList[i].fhandle; size += wrtList[i].wlen; ++numChunks; } o << "handle: "; if( fhandle ) o << FileHandleToStr( fhandle ); else o << "unknown"; o << ", "; o << std::setbase(10); o << "chunks: " << numChunks << ", "; o << "total size: " << size << ")"; break; } //------------------------------------------------------------------------ // kXR_locate //------------------------------------------------------------------------ case kXR_locate: { ClientLocateRequest *sreq = (ClientLocateRequest *)msg; char *fn = GetDataAsString( msg );; o << "kXR_locate ("; o << "path: " << fn << ", "; delete [] fn; o << "flags: "; if( sreq->options == 0 ) o << "none"; else { if( sreq->options & kXR_refresh ) o << "kXR_refresh "; if( sreq->options & kXR_prefname ) o << "kXR_prefname "; if( sreq->options & kXR_nowait ) o << "kXR_nowait "; if( sreq->options & kXR_force ) o << "kXR_force "; if( sreq->options & kXR_compress ) o << "kXR_compress "; } o << ")"; break; } //------------------------------------------------------------------------ // kXR_mv //------------------------------------------------------------------------ case kXR_mv: { ClientMvRequest *sreq = (ClientMvRequest *)msg; o << "kXR_mv ("; o << "source: "; o.write( msg + sizeof( ClientMvRequest ), sreq->arg1len ); o << ", "; o << "destination: "; o.write( msg + sizeof( ClientMvRequest ) + sreq->arg1len + 1, sreq->dlen - sreq->arg1len - 1 ); o << ")"; break; } //------------------------------------------------------------------------ // kXR_query //------------------------------------------------------------------------ case kXR_query: { ClientQueryRequest *sreq = (ClientQueryRequest *)msg; o << "kXR_query ("; o << "code: "; switch( sreq->infotype ) { case kXR_Qconfig: o << "kXR_Qconfig"; break; case kXR_Qckscan: o << "kXR_Qckscan"; break; case kXR_Qcksum: o << "kXR_Qcksum"; break; case kXR_Qopaque: o << "kXR_Qopaque"; break; case kXR_Qopaquf: o << "kXR_Qopaquf"; break; case kXR_Qopaqug: o << "kXR_Qopaqug"; break; case kXR_QPrep: o << "kXR_QPrep"; break; case kXR_Qspace: o << "kXR_Qspace"; break; case kXR_QStats: o << "kXR_QStats"; break; case kXR_Qvisa: o << "kXR_Qvisa"; break; case kXR_Qxattr: o << "kXR_Qxattr"; break; default: o << sreq->infotype; break; } o << ", "; if( sreq->infotype == kXR_Qopaqug || sreq->infotype == kXR_Qvisa ) { o << "handle: " << FileHandleToStr( sreq->fhandle ); o << ", "; } o << "arg length: " << sreq->dlen << ")"; break; } //------------------------------------------------------------------------ // kXR_rm //------------------------------------------------------------------------ case kXR_rm: { o << "kXR_rm ("; char *fn = GetDataAsString( msg );; o << "path: " << fn << ")"; delete [] fn; break; } //------------------------------------------------------------------------ // kXR_mkdir //------------------------------------------------------------------------ case kXR_mkdir: { ClientMkdirRequest *sreq = (ClientMkdirRequest *)msg; o << "kXR_mkdir ("; char *fn = GetDataAsString( msg ); o << "path: " << fn << ", "; delete [] fn; o << "mode: 0" << std::setbase(8) << sreq->mode << ", "; o << std::setbase(10); o << "flags: "; if( sreq->options[0] == 0 ) o << "none"; else { if( sreq->options[0] & kXR_mkdirpath ) o << "kXR_mkdirpath"; } o << ")"; break; } //------------------------------------------------------------------------ // kXR_rmdir //------------------------------------------------------------------------ case kXR_rmdir: { o << "kXR_rmdir ("; char *fn = GetDataAsString( msg ); o << "path: " << fn << ")"; delete [] fn; break; } //------------------------------------------------------------------------ // kXR_chmod //------------------------------------------------------------------------ case kXR_chmod: { ClientChmodRequest *sreq = (ClientChmodRequest *)msg; o << "kXR_chmod ("; char *fn = GetDataAsString( msg ); o << "path: " << fn << ", "; delete [] fn; o << "mode: 0" << std::setbase(8) << sreq->mode << ")"; break; } //------------------------------------------------------------------------ // kXR_ping //------------------------------------------------------------------------ case kXR_ping: { o << "kXR_ping ()"; break; } //------------------------------------------------------------------------ // kXR_protocol //------------------------------------------------------------------------ case kXR_protocol: { ClientProtocolRequest *sreq = (ClientProtocolRequest *)msg; o << "kXR_protocol ("; o << "clientpv: 0x" << std::setbase(16) << sreq->clientpv << ")"; break; } //------------------------------------------------------------------------ // kXR_dirlist //------------------------------------------------------------------------ case kXR_dirlist: { o << "kXR_dirlist ("; char *fn = GetDataAsString( msg );; o << "path: " << fn << ")"; delete [] fn; break; } //------------------------------------------------------------------------ // kXR_set //------------------------------------------------------------------------ case kXR_set: { o << "kXR_set ("; char *fn = GetDataAsString( msg );; o << "data: " << fn << ")"; delete [] fn; break; } //------------------------------------------------------------------------ // kXR_prepare //------------------------------------------------------------------------ case kXR_prepare: { ClientPrepareRequest *sreq = (ClientPrepareRequest *)msg; o << "kXR_prepare ("; o << "flags: "; if( sreq->options == 0 ) o << "none"; else { if( sreq->options & kXR_stage ) o << "kXR_stage "; if( sreq->options & kXR_wmode ) o << "kXR_wmode "; if( sreq->options & kXR_coloc ) o << "kXR_coloc "; if( sreq->options & kXR_fresh ) o << "kXR_fresh "; } o << ", priority: " << (int) sreq->prty << ", "; char *fn = GetDataAsString( msg ); char *cursor; for( cursor = fn; *cursor; ++cursor ) if( *cursor == '\n' ) *cursor = ' '; o << "paths: " << fn << ")"; delete [] fn; break; } case kXR_chkpoint: { ClientChkPointRequest *sreq = (ClientChkPointRequest*)msg; o << "kXR_chkpoint ("; o << "opcode: "; if( sreq->opcode == kXR_ckpBegin ) o << "kXR_ckpBegin)"; else if( sreq->opcode == kXR_ckpCommit ) o << "kXR_ckpCommit)"; else if( sreq->opcode == kXR_ckpQuery ) o << "kXR_ckpQuery)"; else if( sreq->opcode == kXR_ckpRollback ) o << "kXR_ckpRollback)"; else if( sreq->opcode == kXR_ckpXeq ) { o << "kXR_ckpXeq) "; // In this case our request body will be one of kXR_pgwrite, // kXR_truncate, kXR_write, or kXR_writev request. GenerateDescription( msg + sizeof( ClientChkPointRequest ), o ); } break; } //------------------------------------------------------------------------ // Default //------------------------------------------------------------------------ default: { o << "kXR_unknown (length: " << req->dlen << ")"; break; } }; } //---------------------------------------------------------------------------- // Get a string representation of file handle //---------------------------------------------------------------------------- std::string XRootDTransport::FileHandleToStr( const unsigned char handle[4] ) { std::ostringstream o; o << "0x"; for( uint8_t i = 0; i < 4; ++i ) { o << std::setbase(16) << std::setfill('0') << std::setw(2); o << (int)handle[i]; } return o.str(); } } xrootd-5.6.9/src/XrdCl/XrdClXRootDTransport.hh000066400000000000000000000542211457266313600212500ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2014 by European Organization for Nuclear Research (CERN) // Author: Lukasz Janyst //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #ifndef __XRD_CL_XROOTD_TRANSPORT_HH__ #define __XRD_CL_XROOTD_TRANSPORT_HH__ #include "XrdCl/XrdClPostMaster.hh" #include "XrdCl/XrdClMessage.hh" #include "XProtocol/XProtocol.hh" #include "XrdSec/XrdSecInterface.hh" #include "XrdOuc/XrdOucEnv.hh" class XrdSysPlugin; class XrdSecProtect; namespace XrdCl { class Tls; class Socket; struct XRootDChannelInfo; struct PluginUnloadHandler; //---------------------------------------------------------------------------- //! XRootD transport handler //---------------------------------------------------------------------------- class XRootDTransport: public TransportHandler { public: //------------------------------------------------------------------------ //! Constructor //------------------------------------------------------------------------ XRootDTransport(); //------------------------------------------------------------------------ //! Destructor //------------------------------------------------------------------------ ~XRootDTransport(); //------------------------------------------------------------------------ //! Read a message header from the socket, the socket is non-blocking, //! so if there is not enough data the function should return suRetry //! in which case it will be called again when more data arrives, with //! the data previously read stored in the message buffer //! //! @param message the message buffer //! @param socket the socket //! @return stOK & suDone if the whole message has been processed //! stOK & suRetry if more data is needed //! stError on failure //------------------------------------------------------------------------ virtual XRootDStatus GetHeader( Message &message, Socket *socket ); //------------------------------------------------------------------------ //! Read the message body from the socket, the socket is non-blocking, //! the method may be called multiple times - see GetHeader for details //! //! @param message the message buffer containing the header //! @param socket the socket //! @return stOK & suDone if the whole message has been processed //! stOK & suRetry if more data is needed //! stError on failure //------------------------------------------------------------------------ virtual XRootDStatus GetBody( Message &message, Socket *socket ); //------------------------------------------------------------------------ //! Read more of the message body from the socket, the socket is //! non-blocking the method may be called multiple times - see GetHeader //! for details //! //! @param message the message buffer containing the header //! @param socket the socket //! @return stOK & suDone if the whole message has been processed //! stOK & suRetry if more data is needed //! stError on failure //------------------------------------------------------------------------ virtual XRootDStatus GetMore( Message &message, Socket *socket ); //------------------------------------------------------------------------ //! Initialize channel //------------------------------------------------------------------------ virtual void InitializeChannel( const URL &url, AnyObject &channelData ); //------------------------------------------------------------------------ //! Finalize channel //------------------------------------------------------------------------ virtual void FinalizeChannel( AnyObject &channelData ); //------------------------------------------------------------------------ //! HandShake //------------------------------------------------------------------------ virtual XRootDStatus HandShake( HandShakeData *handShakeData, AnyObject &channelData ); //------------------------------------------------------------------------ // @return true if handshake has been done and stream is connected, // false otherwise //------------------------------------------------------------------------ virtual bool HandShakeDone( HandShakeData *handShakeData, AnyObject &channelData ); //------------------------------------------------------------------------ //! Check if the stream should be disconnected //------------------------------------------------------------------------ virtual bool IsStreamTTLElapsed( time_t time, AnyObject &channelData ); //------------------------------------------------------------------------ //! Check the stream is broken - ie. TCP connection got broken and //! went undetected by the TCP stack //------------------------------------------------------------------------ virtual Status IsStreamBroken( time_t inactiveTime, AnyObject &channelData ); //------------------------------------------------------------------------ //! Return the ID for the up stream this message should be sent by //! and the down stream which the answer should be expected at. //! Modify the message itself if necessary. //! If hint is non-zero then the message should be modified such that //! the answer will be returned via the hinted stream. //------------------------------------------------------------------------ virtual PathID Multiplex( Message *msg, AnyObject &channelData, PathID *hint = 0 ); //------------------------------------------------------------------------ //! Return the ID for the up substream this message should be sent by //! and the down substream which the answer should be expected at. //! Modify the message itself if necessary. //! If hint is non-zero then the message should be modified such that //! the answer will be returned via the hinted stream. //------------------------------------------------------------------------ virtual PathID MultiplexSubStream( Message *msg, AnyObject &channelData, PathID *hint = 0 ); //------------------------------------------------------------------------ //! Return a number of substreams per stream that should be created //------------------------------------------------------------------------ virtual uint16_t SubStreamNumber( AnyObject &channelData ); //------------------------------------------------------------------------ //! Return the information whether a control connection needs to be //! valid before establishing other connections //------------------------------------------------------------------------ virtual bool NeedControlConnection() { return true; } //------------------------------------------------------------------------ //! Marshal the outgoing message //------------------------------------------------------------------------ inline static XRootDStatus MarshallRequest( Message *msg ) { MarshallRequest( msg->GetBuffer() ); msg->SetIsMarshalled( true ); return XRootDStatus(); } //------------------------------------------------------------------------ //! Marshal the outgoing message //------------------------------------------------------------------------ static XRootDStatus MarshallRequest( char *msg ); //------------------------------------------------------------------------ //! Unmarshall the request - sometimes the requests need to be rewritten, //! so we need to unmarshall them //------------------------------------------------------------------------ static XRootDStatus UnMarshallRequest( Message *msg ); //------------------------------------------------------------------------ //! Unmarshall the body of the incoming message //------------------------------------------------------------------------ static XRootDStatus UnMarshallBody( Message *msg, uint16_t reqType ); //------------------------------------------------------------------------ //! Unmarshall the body of the status response //------------------------------------------------------------------------ static XRootDStatus UnMarshalStatusBody( Message &msg, uint16_t reqType ); //------------------------------------------------------------------------ //! Unmarshall the correction-segment of the status response for pgwrite //------------------------------------------------------------------------ static XRootDStatus UnMarchalStatusMore( Message &msg ); //------------------------------------------------------------------------ //! Unmarshall the header incoming message //------------------------------------------------------------------------ static void UnMarshallHeader( Message &msg ); //------------------------------------------------------------------------ //! Log server error response //------------------------------------------------------------------------ static void LogErrorResponse( const Message &msg ); //------------------------------------------------------------------------ //! Number of currently connected data streams //------------------------------------------------------------------------ static uint16_t NbConnectedStrm( AnyObject &channelData ); //------------------------------------------------------------------------ //! The stream has been disconnected, do the cleanups //------------------------------------------------------------------------ virtual void Disconnect( AnyObject &channelData, uint16_t subStreamId ); //------------------------------------------------------------------------ //! Query the channel //------------------------------------------------------------------------ virtual Status Query( uint16_t query, AnyObject &result, AnyObject &channelData ); //------------------------------------------------------------------------ //! Get the description of a message //------------------------------------------------------------------------ static void GenerateDescription( char *msg, std::ostringstream &o ); //------------------------------------------------------------------------ //! Get the description of a message //------------------------------------------------------------------------ inline static void SetDescription( Message *msg ) { std::ostringstream o; GenerateDescription( msg->GetBuffer(), o ); msg->SetDescription( o.str() ); } //------------------------------------------------------------------------ //! Check if the message invokes a stream action //------------------------------------------------------------------------ virtual uint32_t MessageReceived( Message &msg, uint16_t subStream, AnyObject &channelData ); //------------------------------------------------------------------------ //! Notify the transport about a message having been sent //------------------------------------------------------------------------ virtual void MessageSent( Message *msg, uint16_t subStream, uint32_t bytesSent, AnyObject &channelData ); //------------------------------------------------------------------------ //! Get signature for given message //------------------------------------------------------------------------ virtual Status GetSignature( Message *toSign, Message *&sign, AnyObject &channelData ); //------------------------------------------------------------------------ //! Get signature for given message //------------------------------------------------------------------------ virtual Status GetSignature( Message *toSign, Message *&sign, XRootDChannelInfo *info ); //------------------------------------------------------------------------ //! Decrement file object instance count bound to this channel //------------------------------------------------------------------------ virtual void DecFileInstCnt( AnyObject &channelData ); //------------------------------------------------------------------------ //! Wait until the program can safely exit //------------------------------------------------------------------------ virtual void WaitBeforeExit(); //------------------------------------------------------------------------ //! @return : true if encryption should be turned on, false otherwise //------------------------------------------------------------------------ virtual bool NeedEncryption( HandShakeData *handShakeData, AnyObject &channelData ); //------------------------------------------------------------------------ //! Get bind preference for the next data stream //------------------------------------------------------------------------ virtual URL GetBindPreference( const URL &url, AnyObject &channelData ); private: //------------------------------------------------------------------------ // Hand shake the main stream //------------------------------------------------------------------------ XRootDStatus HandShakeMain( HandShakeData *handShakeData, AnyObject &channelData ); //------------------------------------------------------------------------ // Hand shake a parallel stream //------------------------------------------------------------------------ XRootDStatus HandShakeParallel( HandShakeData *handShakeData, AnyObject &channelData ); //------------------------------------------------------------------------ // Generate the message to be sent as an initial handshake // (handshake + kXR_protocol) //------------------------------------------------------------------------ Message *GenerateInitialHSProtocol( HandShakeData *hsData, XRootDChannelInfo *info, kXR_char expect ); //------------------------------------------------------------------------ // Generate the protocol message //------------------------------------------------------------------------ Message *GenerateProtocol( HandShakeData *hsData, XRootDChannelInfo *info, kXR_char expect ); //------------------------------------------------------------------------ // Initialize protocol request //------------------------------------------------------------------------ void InitProtocolReq( ClientProtocolRequest *request, XRootDChannelInfo *info, kXR_char expect ); //------------------------------------------------------------------------ // Process the server initial handshake response //------------------------------------------------------------------------ XRootDStatus ProcessServerHS( HandShakeData *hsData, XRootDChannelInfo *info ); //------------------------------------------------------------------------ // Process the protocol response //------------------------------------------------------------------------ XRootDStatus ProcessProtocolResp( HandShakeData *hsData, XRootDChannelInfo *info ); //------------------------------------------------------------------------ // Process the protocol body: // * 'B' : bind preferences // * 'S' : security requirements //------------------------------------------------------------------------ XRootDStatus ProcessProtocolBody( char *bodybuff, size_t bodysize, XRootDChannelInfo *info ); //------------------------------------------------------------------------ // Generate the bind message //------------------------------------------------------------------------ Message *GenerateBind( HandShakeData *hsData, XRootDChannelInfo *info ); //------------------------------------------------------------------------ // Generate the bind message //------------------------------------------------------------------------ XRootDStatus ProcessBindResp( HandShakeData *hsData, XRootDChannelInfo *info ); //------------------------------------------------------------------------ // Generate the login message //------------------------------------------------------------------------ Message *GenerateLogIn( HandShakeData *hsData, XRootDChannelInfo *info ); //------------------------------------------------------------------------ // Process the login response //------------------------------------------------------------------------ XRootDStatus ProcessLogInResp( HandShakeData *hsData, XRootDChannelInfo *info ); //------------------------------------------------------------------------ // Do the authentication //------------------------------------------------------------------------ XRootDStatus DoAuthentication( HandShakeData *hsData, XRootDChannelInfo *info ); //------------------------------------------------------------------------ // Get the initial credentials using one of the protocols //------------------------------------------------------------------------ XRootDStatus GetCredentials( XrdSecCredentials *&credentials, HandShakeData *hsData, XRootDChannelInfo *info ); //------------------------------------------------------------------------ // Clean up the data structures created for the authentication process //------------------------------------------------------------------------ Status CleanUpAuthentication( XRootDChannelInfo *info ); //------------------------------------------------------------------------ // Clean up the data structures created for the protection purposes //------------------------------------------------------------------------ Status CleanUpProtection( XRootDChannelInfo *info ); //------------------------------------------------------------------------ // Get the authentication function handle //------------------------------------------------------------------------ XrdSecGetProt_t GetAuthHandler(); //------------------------------------------------------------------------ // Generate the end session message //------------------------------------------------------------------------ Message *GenerateEndSession( HandShakeData *hsData, XRootDChannelInfo *info ); //------------------------------------------------------------------------ // Process the end session response //------------------------------------------------------------------------ Status ProcessEndSessionResp( HandShakeData *hsData, XRootDChannelInfo *info ); //------------------------------------------------------------------------ // Get a string representation of the server flags //------------------------------------------------------------------------ static std::string ServerFlagsToStr( uint32_t flags ); //------------------------------------------------------------------------ // Get a string representation of file handle //------------------------------------------------------------------------ static std::string FileHandleToStr( const unsigned char handle[4] ); friend struct PluginUnloadHandler; PluginUnloadHandler *pSecUnloadHandler; }; } #endif // __XRD_CL_XROOTD_TRANSPORT_HANDLER_HH__ xrootd-5.6.9/src/XrdCl/XrdClZipArchive.cc000066400000000000000000001242521457266313600202100ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2014 by European Organization for Nuclear Research (CERN) // Author: Michal Simon //----------------------------------------------------------------------------- // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //----------------------------------------------------------------------------- #include "XrdCl/XrdClFileOperations.hh" #include "XrdCl/XrdClCheckpointOperation.hh" #include "XrdCl/XrdClZipArchive.hh" #include "XrdCl/XrdClLog.hh" #include "XrdCl/XrdClDefaultEnv.hh" #include "XrdCl/XrdClConstants.hh" #include "XrdCl/XrdClUtils.hh" #include "XrdZip/XrdZipZIP64EOCDL.hh" #include namespace XrdCl { using namespace XrdZip; //--------------------------------------------------------------------------- // Read data from a given file //--------------------------------------------------------------------------- template XRootDStatus ReadFromImpl( ZipArchive &me, const std::string &fn, uint64_t relativeOffset, uint32_t size, void *usrbuff, ResponseHandler *usrHandler, uint16_t timeout ) { if( me.openstage != ZipArchive::Done || !me.archive.IsOpen() ) return XRootDStatus( stError, errInvalidOp ); Log *log = DefaultEnv::GetLog(); auto cditr = me.cdmap.find( fn ); if( cditr == me.cdmap.end() ) return XRootDStatus( stError, errNotFound, errNotFound, "File not found." ); CDFH *cdfh = me.cdvec[cditr->second].get(); // check if the file is compressed, for now we only support uncompressed and inflate/deflate compression if( cdfh->compressionMethod != 0 && cdfh->compressionMethod != Z_DEFLATED ) return XRootDStatus( stError, errNotSupported, 0, "The compression algorithm is not supported!" ); // Now the problem is that at the beginning of our // file there is the Local-file-header, which size // is not known because of the variable size 'extra' // field, so we need to know the offset of the next // record and shift it by the file size. // The next record is either the next LFH (next file) // or the start of the Central-directory. uint64_t cdOffset = me.zip64eocd ? me.zip64eocd->cdOffset : me.eocd->cdOffset; uint64_t nextRecordOffset = ( cditr->second + 1 < me.cdvec.size() ) ? CDFH::GetOffset( *me.cdvec[cditr->second + 1] ) : cdOffset; uint64_t filesize = cdfh->compressedSize; if( filesize == std::numeric_limits::max() && cdfh->extra ) filesize = cdfh->extra->compressedSize; uint16_t descsize = cdfh->HasDataDescriptor() ? DataDescriptor::GetSize( cdfh->IsZIP64() ) : 0; uint64_t fileoff = nextRecordOffset - filesize - descsize; uint64_t offset = fileoff + relativeOffset; uint64_t uncompressedSize = cdfh->uncompressedSize; if( uncompressedSize == std::numeric_limits::max() && cdfh->extra ) uncompressedSize = cdfh->extra->uncompressedSize; uint64_t sizeTillEnd = relativeOffset > uncompressedSize ? 0 : uncompressedSize - relativeOffset; if( size > sizeTillEnd ) size = sizeTillEnd; // if it is a compressed file use ZIP cache to read from the file if( cdfh->compressionMethod == Z_DEFLATED ) { log->Dump( ZipMsg, "[0x%x] Reading compressed data.", &me ); // check if respective ZIP cache exists bool empty = me.zipcache.find( fn ) == me.zipcache.end(); // if the entry does not exist, it will be created using // default constructor ZipCache &cache = me.zipcache[fn]; if( relativeOffset > uncompressedSize ) { // we are reading past the end of file, // we can serve the request right away! RSP *r = new RSP( relativeOffset, 0, usrbuff ); AnyObject *rsp = new AnyObject(); rsp->Set( r ); usrHandler->HandleResponse( new XRootDStatus(), rsp ); return XRootDStatus(); } uint32_t sizereq = size; if( relativeOffset + size > uncompressedSize ) sizereq = uncompressedSize - relativeOffset; cache.QueueReq( relativeOffset, sizereq, usrbuff, usrHandler ); // if we have the whole ZIP archive we can populate the cache // straight away if( empty && me.buffer) { auto begin = me.buffer.get() + fileoff; auto end = begin + filesize ; buffer_t buff( begin, end ); cache.QueueRsp( XRootDStatus(), 0, std::move( buff ) ); return XRootDStatus(); } // if we don't have the data we need to issue a remote read if( !me.buffer ) { if( relativeOffset > filesize ) return XRootDStatus(); // there's nothing to do, // we already have all the data locally uint32_t rdsize = size; // check if this is the last read (we reached the end of // file from user perspective) if( relativeOffset + size >= uncompressedSize ) { // if yes, make sure we readout all the compressed data // Note: In a patological case the compressed size may // be greater than the uncompressed size rdsize = filesize > relativeOffset ? filesize - relativeOffset : 0; } // make sure we are not reading past the end of // compressed data if( relativeOffset + size > filesize ) rdsize = filesize - relativeOffset; // now read the data ... auto rdbuff = std::make_shared( rdsize ); Pipeline p = XrdCl::RdWithRsp( me.archive, offset, rdbuff->size(), rdbuff->data() ) >> [relativeOffset, rdbuff, &cache, &me]( XRootDStatus &st, RSP &rsp ) { Log *log = DefaultEnv::GetLog(); log->Dump( ZipMsg, "[0x%x] Read %d bytes of remote data at offset %d.", &me, rsp.GetLength(), rsp.GetOffset() ); cache.QueueRsp( st, relativeOffset, std::move( *rdbuff ) ); }; Async( std::move( p ), timeout ); } return XRootDStatus(); } // check if we have the whole file in our local buffer if( me.buffer || size == 0 ) { if( size ) { memcpy( usrbuff, me.buffer.get() + offset, size ); log->Dump( ZipMsg, "[0x%x] Serving read from local cache.", &me ); } if( usrHandler ) { XRootDStatus *st = ZipArchive::make_status(); RSP *rsp = new RSP( relativeOffset, size, usrbuff ); ZipArchive::Schedule( usrHandler, st, rsp ); } return XRootDStatus(); } Pipeline p = XrdCl::RdWithRsp( me.archive, offset, size, usrbuff ) >> [=, &me]( XRootDStatus &st, RSP &r ) { log->Dump( ZipMsg, "[0x%x] Read %d bytes of remote data at " "offset %d.", &me, r.GetLength(), r.GetOffset() ); if( usrHandler ) { XRootDStatus *status = ZipArchive::make_status( st ); RSP *rsp = nullptr; if( st.IsOK() ) rsp = new RSP( relativeOffset, r.GetLength(), r.GetBuffer() ); usrHandler->HandleResponse( status, ZipArchive::PkgRsp( rsp ) ); } }; Async( std::move( p ), timeout ); return XRootDStatus(); } //--------------------------------------------------------------------------- // Constructor //--------------------------------------------------------------------------- ZipArchive::ZipArchive( bool enablePlugIns) : archive( enablePlugIns ), archsize( 0 ), cdexists( false ), updated( false ), cdoff( 0 ), orgcdsz( 0 ), orgcdcnt( 0 ), openstage( None ), ckpinit( false ) { } //--------------------------------------------------------------------------- // Destructor //--------------------------------------------------------------------------- ZipArchive::~ZipArchive() { } //--------------------------------------------------------------------------- // Open the ZIP archive in read-only mode without parsing the central // directory. //--------------------------------------------------------------------------- XRootDStatus ZipArchive::OpenOnly( const std::string &url, bool update, ResponseHandler *handler, uint16_t timeout ) { OpenFlags::Flags flags = update ? OpenFlags::Update : OpenFlags::Read; Pipeline open_only = XrdCl::Open( archive, url, flags ) >> [=]( XRootDStatus &st, StatInfo &info ) { Log *log = DefaultEnv::GetLog(); // check the status is OK if( st.IsOK() ) { archsize = info.GetSize(); openstage = NotParsed; log->Debug( ZipMsg, "[0x%x] Opened (only) a ZIP archive (%s).", this, url.c_str() ); } else { log->Error( ZipMsg, "[0x%x] Failed to open-only a ZIP archive (%s): %s", this, url.c_str(), st.ToString().c_str() ); } if( handler ) handler->HandleResponse( make_status( st ), nullptr ); }; Async( std::move( open_only ), timeout ); return XRootDStatus(); } //--------------------------------------------------------------------------- // Open ZIP Archive (and parse the Central Directory) //--------------------------------------------------------------------------- XRootDStatus ZipArchive::OpenArchive( const std::string &url, OpenFlags::Flags flags, ResponseHandler *handler, uint16_t timeout ) { Log *log = DefaultEnv::GetLog(); Fwd rdsize; // number of bytes to be read Fwd rdoff; // offset for the read request Fwd rdbuff; // buffer for data to be read uint32_t maxrdsz = EOCD::maxCommentLength + EOCD::eocdBaseSize + ZIP64_EOCDL::zip64EocdlSize; Pipeline open_archive = // open the archive XrdCl::Open( archive, url, flags ) >> [=]( XRootDStatus &status, StatInfo &info ) mutable { // check the status is OK if( !status.IsOK() ) return; archsize = info.GetSize(); // if it is an empty file (possibly a new file) there's nothing more to do if( archsize == 0 ) { cdexists = false; openstage = Done; log->Dump( ZipMsg, "[0x%x] Opened a ZIP archive (file empty).", this ); Pipeline::Stop(); } // prepare the arguments for the subsequent read rdsize = ( archsize <= maxrdsz ? archsize : maxrdsz ); rdoff = archsize - *rdsize; buffer.reset( new char[*rdsize] ); rdbuff = buffer.get(); openstage = HaveEocdBlk; log->Dump( ZipMsg, "[0x%x] Opened a ZIP archive, reading " "Central Directory at offset: %d.", this, *rdoff ); } // read the Central Directory (in several stages if necessary) | XrdCl::Read( archive, rdoff, rdsize, rdbuff ) >> [=]( XRootDStatus &status, ChunkInfo &chunk ) mutable { // check the status is OK if( !status.IsOK() ) return; const char *buff = reinterpret_cast( chunk.buffer ); while( true ) { switch( openstage ) { case HaveEocdBlk: { // Parse the EOCD record const char *eocdBlock = EOCD::Find( buff, chunk.length ); if( !eocdBlock ) { XRootDStatus error( stError, errDataError, 0, "End-of-central-directory signature not found." ); Pipeline::Stop( error ); } try{ eocd.reset( new EOCD( eocdBlock, chunk.length - uint32_t(eocdBlock - buff) ) ); log->Dump( ZipMsg, "[0x%x] EOCD record parsed: %s", this, eocd->ToString().c_str() ); if(eocd->cdOffset > archsize || eocd->cdOffset + eocd->cdSize > archsize) throw bad_data(); } catch(const bad_data &ex){ XRootDStatus error( stError, errDataError, 0, "End-of-central-directory signature corrupted." ); Pipeline::Stop( error ); } // Do we have the whole archive? if( chunk.length == archsize ) { // If we managed to download the whole archive we don't need to // worry about zip64, it is so small that standard EOCD will do cdoff = eocd->cdOffset; orgcdsz = eocd->cdSize; orgcdcnt = eocd->nbCdRec; buff = buff + cdoff; openstage = HaveCdRecords; continue; } // Let's see if it is ZIP64 (if yes, the EOCD will be preceded with ZIP64 EOCD locator) const char *zip64EocdlBlock = eocdBlock - ZIP64_EOCDL::zip64EocdlSize; // make sure there is enough data to assume there's a ZIP64 EOCD locator if( zip64EocdlBlock > buffer.get() ) { uint32_t signature = to( zip64EocdlBlock ); if( signature == ZIP64_EOCDL::zip64EocdlSign ) { buff = zip64EocdlBlock; openstage = HaveZip64EocdlBlk; continue; } } // It's not ZIP64, we already know where the CD records are // we need to read more data cdoff = eocd->cdOffset; orgcdsz = eocd->cdSize; orgcdcnt = eocd->nbCdRec; rdoff = eocd->cdOffset; rdsize = eocd->cdSize; buffer.reset( new char[*rdsize] ); rdbuff = buffer.get(); openstage = HaveCdRecords; log->Dump( ZipMsg, "[0x%x] Reading additional data at offset: %d.", this, *rdoff ); Pipeline::Repeat(); break; // the break is really not needed ... } case HaveZip64EocdlBlk: { std::unique_ptr eocdl( new ZIP64_EOCDL( buff ) ); log->Dump( ZipMsg, "[0x%x] EOCDL record parsed: %s", this, eocdl->ToString().c_str() ); if( chunk.offset > eocdl->zip64EocdOffset ) { // we need to read more data, adjust the read arguments rdsize = archsize - eocdl->zip64EocdOffset; rdoff = eocdl->zip64EocdOffset; buffer.reset( new char[*rdsize] ); rdbuff = buffer.get(); openstage = HaveZip64EocdBlk; log->Dump( ZipMsg, "[0x%x] Reading additional data at offset: %d.", this, *rdoff ); Pipeline::Repeat(); } buff = buffer.get() + ( eocdl->zip64EocdOffset - chunk.offset ); openstage = HaveZip64EocdBlk; continue; } case HaveZip64EocdBlk: { uint32_t signature = to( buff ); if( signature != ZIP64_EOCD::zip64EocdSign ) { XRootDStatus error( stError, errDataError, 0, "ZIP64 End-of-central-directory signature not found." ); Pipeline::Stop( error ); } zip64eocd.reset( new ZIP64_EOCD( buff ) ); log->Dump( ZipMsg, "[0x%x] ZIP64EOCD record parsed: %s", this, zip64eocd->ToString().c_str() ); // now we can read the CD records, adjust the read arguments cdoff = zip64eocd->cdOffset; orgcdsz = zip64eocd->cdSize; orgcdcnt = zip64eocd->nbCdRec; rdoff = zip64eocd->cdOffset; rdsize = zip64eocd->cdSize; buffer.reset( new char[*rdsize] ); rdbuff = buffer.get(); openstage = HaveCdRecords; log->Dump( ZipMsg, "[0x%x] Reading additional data at offset: %d.", this, *rdoff ); Pipeline::Repeat(); break; // the break is really not needed ... } case HaveCdRecords: { // make a copy of the original CDFH records orgcdbuf.reserve( orgcdsz ); std::copy( buff, buff + orgcdsz, std::back_inserter( orgcdbuf ) ); try { if( zip64eocd ) std::tie( cdvec, cdmap ) = CDFH::Parse( buff, zip64eocd->cdSize, zip64eocd->nbCdRec ); else std::tie( cdvec, cdmap ) = CDFH::Parse( buff, eocd->cdSize, eocd->nbCdRec ); log->Dump( ZipMsg, "[0x%x] CD records parsed.", this ); uint64_t sumCompSize = 0; for (auto it = cdvec.begin(); it != cdvec.end(); it++) { sumCompSize += (*it)->IsZIP64() ? (*it)->extra->compressedSize : (*it)->compressedSize; if ((*it)->offset > archsize || (*it)->offset + (*it)->compressedSize > archsize) throw bad_data(); } if (sumCompSize > archsize) throw bad_data(); } catch( const bad_data &ex ) { XRootDStatus error( stError, errDataError, 0, "ZIP Central Directory corrupted." ); Pipeline::Stop( error ); } if( chunk.length != archsize ) buffer.reset(); openstage = Done; cdexists = true; break; } default: Pipeline::Stop( XRootDStatus( stError, errInvalidOp ) ); } break; } } | XrdCl::Final( [=]( const XRootDStatus &status ) { // finalize the pipeline by calling the user callback if( status.IsOK() ) log->Debug( ZipMsg, "[0x%x] Opened a ZIP archive (%s): %s", this, url.c_str(), status.ToString().c_str() ); else log->Error( ZipMsg, "[0x%x] Failed to open a ZIP archive (%s): %s", this, url.c_str(), status.ToString().c_str() ); if( handler ) handler->HandleResponse( make_status( status ), nullptr ); } ); Async( std::move( open_archive ), timeout ); return XRootDStatus(); } //--------------------------------------------------------------------------- // Open a file within the ZIP Archive //--------------------------------------------------------------------------- XRootDStatus ZipArchive::OpenFile( const std::string &fn, OpenFlags::Flags flags, uint64_t size, uint32_t crc32 ) { if( !openfn.empty() || openstage != Done || !archive.IsOpen() ) return XRootDStatus( stError, errInvalidOp ); Log *log = DefaultEnv::GetLog(); auto itr = cdmap.find( fn ); if( itr == cdmap.end() ) { // the file does not exist in the archive so it only makes sense // if our user is opening for append if( flags & OpenFlags::New ) { openfn = fn; lfh.reset( new LFH( fn, crc32, size, time( 0 ) ) ); log->Dump( ZipMsg, "[0x%x] File %s opened for append.", this, fn.c_str() ); return XRootDStatus(); } log->Dump( ZipMsg, "[0x%x] Open failed: %s not in the ZIP archive.", this, fn.c_str() ); return XRootDStatus( stError, errNotFound ); } // the file name exist in the archive but our user wants to append // a file with the same name if( flags & OpenFlags::New ) { log->Dump( ZipMsg, "[0x%x] Open failed: file exists %s, cannot append.", this, fn.c_str() ); return XRootDStatus( stError, errInvalidOp, EEXIST, "The file already exists in the ZIP archive." ); } openfn = fn; log->Dump( ZipMsg, "[0x%x] File %s opened for reading.", this, fn.c_str() ); return XRootDStatus(); } //--------------------------------------------------------------------------- // Get a buffer with central directory of the ZIP archive //--------------------------------------------------------------------------- buffer_t ZipArchive::GetCD() { uint32_t size = 0; uint32_t cdsize = CDFH::CalcSize( cdvec, orgcdsz, orgcdcnt ); // first create the EOCD record eocd.reset( new EOCD( cdoff, cdvec.size(), cdsize ) ); size += eocd->eocdSize ; size += eocd->cdSize; // then create zip64eocd & zip64eocdl if necessary std::unique_ptr zip64eocdl; if( eocd->useZip64 ) { zip64eocd.reset( new ZIP64_EOCD( cdoff, cdvec.size(), cdsize ) ); size += zip64eocd->zip64EocdTotalSize; zip64eocdl.reset( new ZIP64_EOCDL( *eocd, *zip64eocd ) ); size += ZIP64_EOCDL::zip64EocdlSize; } // Now serialize all records into a buffer buffer_t metadata; metadata.reserve( size ); CDFH::Serialize( orgcdcnt, orgcdbuf, cdvec, metadata ); if( zip64eocd ) zip64eocd->Serialize( metadata ); if( zip64eocdl ) zip64eocdl->Serialize( metadata ); eocd->Serialize( metadata ); return metadata; } //--------------------------------------------------------------------------- // Set central directory for the ZIP archive //--------------------------------------------------------------------------- void ZipArchive::SetCD( const buffer_t &buffer ) { if( openstage != NotParsed ) return; const char *buff = buffer.data(); size_t size = buffer.size(); // parse Central Directory records std::tie(cdvec, cdmap ) = CDFH::Parse( buff, size ); // make a copy of the original CDFH records orgcdsz = buff - buffer.data(); orgcdcnt = cdvec.size(); orgcdbuf.reserve( orgcdsz ); std::copy( buffer.data(), buff, std::back_inserter( orgcdbuf ) ); // parse ZIP64EOCD record if exists uint32_t signature = to( buff ); if( signature == ZIP64_EOCD::zip64EocdSign ) { zip64eocd.reset( new ZIP64_EOCD( buff ) ); buff += zip64eocd->zip64EocdTotalSize; // now shift the buffer by EOCDL size if necessary signature = to( buff ); if( signature == ZIP64_EOCDL::zip64EocdlSign ) buff += ZIP64_EOCDL::zip64EocdlSize; } // parse EOCD record eocd.reset( new EOCD( buff ) ); // update the state of the ZipArchive object openstage = XrdCl::ZipArchive::Done; cdexists = true; } //--------------------------------------------------------------------------- // Create the central directory at the end of ZIP archive and close it //--------------------------------------------------------------------------- XRootDStatus ZipArchive::CloseArchive( ResponseHandler *handler, uint16_t timeout ) { Log *log = DefaultEnv::GetLog(); //------------------------------------------------------------------------- // If the file was updated, we need to write the Central Directory before // closing the file. //------------------------------------------------------------------------- if( updated ) { ChunkList chunks; std::vector> wrtbufs; for( auto &p : newfiles ) { NewFile &nf = p.second; if( !nf.overwrt ) continue; uint32_t lfhlen = lfh->lfhSize; auto lfhbuf = std::make_shared(); lfhbuf->reserve( lfhlen ); nf.lfh->Serialize( *lfhbuf ); chunks.emplace_back( nf.offset, lfhbuf->size(), lfhbuf->data() ); wrtbufs.emplace_back( std::move( lfhbuf ) ); } auto wrtbuff = std::make_shared( GetCD() ); Pipeline p = XrdCl::Write( archive, cdoff, wrtbuff->size(), wrtbuff->data() ); wrtbufs.emplace_back( std::move( wrtbuff ) ); std::vector listsvec; XrdCl::Utils::SplitChunks( listsvec, chunks, 262144, 1024 ); for(auto itr = listsvec.rbegin(); itr != listsvec.rend(); ++itr) { p = XrdCl::VectorWrite( archive, *itr ) | p; } if( ckpinit ) p |= XrdCl::Checkpoint( archive, ChkPtCode::COMMIT ); p |= Close( archive ) >> [=]( XRootDStatus &st ) { if( st.IsOK() ) Clear(); else openstage = Error; } | XrdCl::Final( [=]( const XRootDStatus &st ) mutable { if( st.IsOK() ) log->Dump( ZipMsg, "[0x%x] Successfully closed ZIP archive " "(CD written).", this ); else log->Error( ZipMsg, "[0x%x] Failed to close ZIP archive: %s", this, st.ToString().c_str() ); wrtbufs.clear(); if( handler ) handler->HandleResponse( make_status( st ), nullptr ); } ); Async( std::move( p ), timeout ); return XRootDStatus(); } //------------------------------------------------------------------------- // Otherwise, just close the ZIP archive //------------------------------------------------------------------------- Pipeline p = Close( archive ) >> [=]( XRootDStatus &st ) { if( st.IsOK() ) { Clear(); log->Dump( ZipMsg, "[0x%x] Successfully closed " "ZIP archive.", this ); } else { openstage = Error; log->Error( ZipMsg, "[0x%x] Failed to close ZIP archive:" " %s", this, st.ToString().c_str() ); } if( handler ) handler->HandleResponse( make_status( st ), nullptr ); }; Async( std::move( p ), timeout ); return XRootDStatus(); } //--------------------------------------------------------------------------- // Read data from a given file //--------------------------------------------------------------------------- XRootDStatus ZipArchive::ReadFrom( const std::string &fn, uint64_t offset, uint32_t size, void *buffer, ResponseHandler *handler, uint16_t timeout ) { return ReadFromImpl( *this, fn, offset, size, buffer, handler, timeout ); } //--------------------------------------------------------------------------- // PgRead data from a given file //--------------------------------------------------------------------------- XRootDStatus ZipArchive::PgReadFrom( const std::string &fn, uint64_t offset, uint32_t size, void *buffer, ResponseHandler *handler, uint16_t timeout ) { return ReadFromImpl( *this, fn, offset, size, buffer, handler, timeout ); } //--------------------------------------------------------------------------- // List files in the ZIP archive //--------------------------------------------------------------------------- XRootDStatus ZipArchive::List( DirectoryList *&list ) { if( openstage != Done ) return XRootDStatus( stError, errInvalidOp, 0, "Archive not opened." ); std::string value; archive.GetProperty( "LastURL", value ); URL url( value ); StatInfo *infoptr = 0; XRootDStatus st = archive.Stat( false, infoptr ); std::unique_ptr info( infoptr ); list = new DirectoryList(); list->SetParentName( url.GetPath() ); auto itr = cdvec.begin(); for( ; itr != cdvec.end() ; ++itr ) { CDFH *cdfh = itr->get(); uint64_t uncompressedSize = cdfh->uncompressedSize; if( uncompressedSize == std::numeric_limits::max() && cdfh->extra ) uncompressedSize = cdfh->extra->uncompressedSize; StatInfo *entry_info = make_stat( *info, uncompressedSize ); DirectoryList::ListEntry *entry = new DirectoryList::ListEntry( url.GetHostId(), cdfh->filename, entry_info ); list->Add( entry ); } return XRootDStatus(); } //----------------------------------------------------------------------- // Append data to a new file, implementation //----------------------------------------------------------------------- XRootDStatus ZipArchive::WriteImpl( uint32_t size, const void *buffer, ResponseHandler *handler, uint16_t timeout ) { Log *log = DefaultEnv::GetLog(); std::vector iov( 2 ); //------------------------------------------------------------------------- // If there is a LFH we need to write it first ahead of the write-buffer // itself. //------------------------------------------------------------------------- std::shared_ptr lfhbuf; if( lfh ) { uint32_t lfhlen = lfh->lfhSize; lfhbuf = std::make_shared(); lfhbuf->reserve( lfhlen ); lfh->Serialize( *lfhbuf ); iov[0].iov_base = lfhbuf->data(); iov[0].iov_len = lfhlen; log->Dump( ZipMsg, "[0x%x] Will write LFH.", this ); } //------------------------------------------------------------------------- // If there is no LFH just make the first chunk empty. //------------------------------------------------------------------------- else { iov[0].iov_base = nullptr; iov[0].iov_len = 0; } //------------------------------------------------------------------------- // In the second chunk write the user data //------------------------------------------------------------------------- iov[1].iov_base = const_cast( buffer ); iov[1].iov_len = size; uint64_t wrtoff = cdoff; // we only support appending uint32_t wrtlen = iov[0].iov_len + iov[1].iov_len; Pipeline p; auto wrthandler = [=]( const XRootDStatus &st ) mutable { if( st.IsOK() ) updated = true; lfhbuf.reset(); if( handler ) handler->HandleResponse( make_status( st ), nullptr ); }; //------------------------------------------------------------------------- // If we are overwriting an existing CD we need to use checkpointed version // of WriteV. //------------------------------------------------------------------------- if( archsize > cdoff ) p = XrdCl::ChkptWrtV( archive, wrtoff, iov ) | XrdCl::Final( wrthandler ); //------------------------------------------------------------------------- // Otherwise use the ordinary WriteV. //------------------------------------------------------------------------- else p = XrdCl::WriteV( archive, wrtoff, iov ) | XrdCl::Final( wrthandler ); //----------------------------------------------------------------------- // If needed make sure the checkpoint is initialized //----------------------------------------------------------------------- if( archsize > cdoff && !ckpinit ) { p = XrdCl::Checkpoint( archive, ChkPtCode::BEGIN ) | p; ckpinit = true; } archsize += wrtlen; cdoff += wrtlen; //------------------------------------------------------------------------- // If we have written the LFH, add respective CDFH record //------------------------------------------------------------------------- if( lfh ) { mode_t mode = S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH; cdvec.emplace_back( new CDFH( lfh.get(), mode, wrtoff ) ); cdmap[openfn] = cdvec.size() - 1; // make sure we keep track of all appended files newfiles.emplace( std::piecewise_construct, std::forward_as_tuple( lfh->filename ), std::forward_as_tuple( wrtoff, std::move( lfh ) ) ); } Async( std::move( p ), timeout ); return XRootDStatus(); } //----------------------------------------------------------------------- // Update the metadata of the currently open file //----------------------------------------------------------------------- XRootDStatus ZipArchive::UpdateMetadata( uint32_t crc32 ) { if( openstage != Done || openfn.empty() ) return XRootDStatus( stError, errInvalidOp, 0, "Archive not opened." ); //--------------------------------------------------------------------- // Firstly, update the crc32 in the central directory //--------------------------------------------------------------------- auto itr = cdmap.find( openfn ); if( itr == cdmap.end() ) return XRootDStatus( stError, errInvalidOp ); cdvec[itr->second]->ZCRC32 = crc32; //--------------------------------------------------------------------- // Secondly, update the crc32 in the LFH and mark it as needing // overwriting //--------------------------------------------------------------------- auto itr2 = newfiles.find( openfn ); if( itr2 == newfiles.end() ) return XRootDStatus( stError, errInvalidOp ); itr2->second.lfh->ZCRC32 = crc32; return XRootDStatus(); } //----------------------------------------------------------------------- // Create a new file in the ZIP archive and append the data //----------------------------------------------------------------------- XRootDStatus ZipArchive::AppendFile( const std::string &fn, uint32_t crc32, uint32_t size, const void *buffer, ResponseHandler *handler, uint16_t timeout ) { Log *log = DefaultEnv::GetLog(); auto itr = cdmap.find( fn ); // check if the file already exists in the archive if( itr != cdmap.end() ) { log->Dump( ZipMsg, "[0x%x] Open failed: file exists %s, cannot append.", this, fn.c_str() ); return XRootDStatus( stError, errInvalidOp ); } log->Dump( ZipMsg, "[0x%x] Appending file: %s.", this, fn.c_str() ); //------------------------------------------------------------------------- // Create Local File Header record //------------------------------------------------------------------------- lfh.reset( new LFH( fn, crc32, size, time( 0 ) ) ); //------------------------------------------------------------------------- // And write it all //------------------------------------------------------------------------- return WriteImpl( size, buffer, handler, timeout ); } } /* namespace XrdZip */ xrootd-5.6.9/src/XrdCl/XrdClZipArchive.hh000066400000000000000000000673511457266313600202300ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2014 by European Organization for Nuclear Research (CERN) // Author: Michal Simon //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #ifndef SRC_XRDZIP_XRDZIPARCHIVE_HH_ #define SRC_XRDZIP_XRDZIPARCHIVE_HH_ #include "XrdCl/XrdClFile.hh" #include "XrdCl/XrdClResponseJob.hh" #include "XrdCl/XrdClJobManager.hh" #include "XrdCl/XrdClDefaultEnv.hh" #include "XrdCl/XrdClPostMaster.hh" #include "XrdZip/XrdZipEOCD.hh" #include "XrdZip/XrdZipCDFH.hh" #include "XrdZip/XrdZipZIP64EOCD.hh" #include "XrdZip/XrdZipLFH.hh" #include "XrdCl/XrdClZipCache.hh" #include #include //----------------------------------------------------------------------------- // Forward declaration needed for friendship //----------------------------------------------------------------------------- namespace XrdEc{ class StrmWriter; class Reader; template class OpenOnlyImpl; }; class MicroTest; class XrdEcTests; namespace XrdCl { using namespace XrdZip; //--------------------------------------------------------------------------- // ZipArchive provides following functionalities: // - parsing of existing ZIP archive // - reading data from existing ZIP archive // - appending data to existing ZIP archive // - querying stat info and checksum for given file in ZIP archive //--------------------------------------------------------------------------- class ZipArchive { friend class XrdEc::StrmWriter; friend class XrdEc::Reader; template friend class XrdEc::OpenOnlyImpl; friend class ::MicroTest; friend class ::XrdEcTests; template friend XRootDStatus ReadFromImpl( ZipArchive&, const std::string&, uint64_t, uint32_t, void*, ResponseHandler*, uint16_t ); public: //----------------------------------------------------------------------- //! Constructor //----------------------------------------------------------------------- ZipArchive( bool enablePlugIns = true ); //----------------------------------------------------------------------- //! Destructor //----------------------------------------------------------------------- virtual ~ZipArchive(); //----------------------------------------------------------------------- //! Open ZIP Archive (and parse the Central Directory) //! //! @param url : the URL of the ZIP archive //! @param flags : open flags to be used when openning the file //! @param handler : user callback //! @param timeout : operation timeout //! @return : the status of the operation //----------------------------------------------------------------------- XRootDStatus OpenArchive( const std::string &url, OpenFlags::Flags flags, ResponseHandler *handler, uint16_t timeout = 0 ); //----------------------------------------------------------------------- //! Open a file within the ZIP Archive //! //! @param fn : file name to be opened //! @param flags : open flags (either 'Read' or 'New | Write') //! @param size : file size (to be included in the LFH) //! @param crc32 : file crc32 (to be included in the LFH) //! @return : the status of the operation //----------------------------------------------------------------------- XRootDStatus OpenFile( const std::string &fn, OpenFlags::Flags flags = OpenFlags::None, uint64_t size = 0, uint32_t crc32 = 0 ); //----------------------------------------------------------------------- //! Read data from an open file //! //! @param offset : offset within the file to read at //! @param size : number of bytes to be read //! @param buffer : the buffer for the data //! @param handler : user callback //! @param timeout : operation timeout //! @return : the status of the operation //----------------------------------------------------------------------- inline XRootDStatus Read( uint64_t offset, uint32_t size, void *buffer, ResponseHandler *handler, uint16_t timeout = 0 ) { if( openfn.empty() ) return XRootDStatus( stError, errInvalidOp ); return ReadFrom( openfn, offset, size, buffer, handler, timeout ); } //----------------------------------------------------------------------- //! PgRead data from an open file //! //! @param offset : offset within the file to read at //! @param size : number of bytes to be read //! @param buffer : the buffer for the data //! @param handler : user callback //! @param timeout : operation timeout //! @return : the status of the operation //----------------------------------------------------------------------- inline XRootDStatus PgRead( uint64_t offset, uint32_t size, void *buffer, ResponseHandler *handler, uint16_t timeout = 0 ) { if( openfn.empty() ) return XRootDStatus( stError, errInvalidOp ); return PgReadFrom( openfn, offset, size, buffer, handler, timeout ); } //----------------------------------------------------------------------- //! Read data from a given file //! //! @param fn : the name of the file from which we are going to read //! @param offset : offset within the file to read at //! @param size : number of bytes to be read //! @param buffer : the buffer for the data //! @param handler : user callback //! @param timeout : operation timeout //! @return : the status of the operation //----------------------------------------------------------------------- XRootDStatus ReadFrom( const std::string &fn, uint64_t offset, uint32_t size, void *buffer, ResponseHandler *handler, uint16_t timeout = 0 ); //----------------------------------------------------------------------- //! PgRead data from a given file //! //! @param fn : the name of the file from which we are going to read //! @param offset : offset within the file to read at //! @param size : number of bytes to be read //! @param buffer : the buffer for the data //! @param handler : user callback //! @param timeout : operation timeout //! @return : the status of the operation //----------------------------------------------------------------------- XRootDStatus PgReadFrom( const std::string &fn, uint64_t offset, uint32_t size, void *buffer, ResponseHandler *handler, uint16_t timeout = 0 ); //----------------------------------------------------------------------- //! Append data to a new file //! //! @param size : number of bytes to be appended //! @param buffer : the buffer with the data to be appended //! @param handler : user callback //! @param timeout : operation timeout //! @return : the status of the operation //----------------------------------------------------------------------- inline XRootDStatus Write( uint32_t size, const void *buffer, ResponseHandler *handler, uint16_t timeout = 0 ) { if( openstage != Done || openfn.empty() ) return XRootDStatus( stError, errInvalidOp, 0, "Archive not opened." ); return WriteImpl( size, buffer, handler, timeout ); } //----------------------------------------------------------------------- //! Update the metadata of the currently open file //! //! @param crc32 : the crc32 checksum //! @return : the status of the operation //----------------------------------------------------------------------- XRootDStatus UpdateMetadata( uint32_t crc32 ); //----------------------------------------------------------------------- //! Create a new file in the ZIP archive and append the data //! //! @param fn : the name of the new file to be created //! @param crc32 : the crc32 of the file //! @param size : the size of the file //! @param buffer : the buffer with the data //! @param handler : user callback //! @param timeout : operation timeout //! @return : the status of the operation //----------------------------------------------------------------------- XRootDStatus AppendFile( const std::string &fn, uint32_t crc32, uint32_t size, const void *buffer, ResponseHandler *handler, uint16_t timeout = 0 ); //----------------------------------------------------------------------- //! Get stat info for given file //! //! @param fn : the name of the file //! @param info : output parameter //! @return : the status of the operation //----------------------------------------------------------------------- inline XRootDStatus Stat( const std::string &fn, StatInfo *&info ) { // make sure archive has been opened and CD has been parsed if( openstage != Done ) return XRootDStatus( stError, errInvalidOp ); // make sure the file is part of the archive auto cditr = cdmap.find( fn ); if( cditr == cdmap.end() ) return XRootDStatus( stError, errNotFound ); // create the result info = make_stat( fn ); if (info) return XRootDStatus(); else // have difficult to access the openned archive. return XRootDStatus( stError, errNotFound ); } //----------------------------------------------------------------------- //! Get stat info for an open file //! //! @param info : output parameter //! @return : the status of the operation //----------------------------------------------------------------------- inline XRootDStatus Stat( StatInfo *&info ) { if( openfn.empty() ) return XRootDStatus( stError, errInvalidOp ); return Stat( openfn, info ); } //----------------------------------------------------------------------- //! Get crc32 for a given file //! //! @param fn : file name //! @param cksum : output parameter //! @return : the status of the operation //----------------------------------------------------------------------- inline XRootDStatus GetCRC32( const std::string &fn, uint32_t &cksum ) { // make sure archive has been opened and CD has been parsed if( openstage != Done ) return XRootDStatus( stError, errInvalidOp ); // make sure the file is part of the archive auto cditr = cdmap.find( fn ); if( cditr == cdmap.end() ) return XRootDStatus( stError, errNotFound ); cksum = cdvec[cditr->second]->ZCRC32; return XRootDStatus(); } inline XRootDStatus GetOffset( const std::string &fn, uint64_t &offset){ if( openstage != XrdCl::ZipArchive::Done || !archive.IsOpen() ) return XrdCl::XRootDStatus( XrdCl::stError, XrdCl::errInvalidOp ); auto cditr = cdmap.find( fn ); if( cditr == cdmap.end() ) return XrdCl::XRootDStatus( XrdCl::stError, XrdCl::errNotFound, XrdCl::errNotFound, "File not found." ); XrdCl::CDFH *cdfh = cdvec[cditr->second].get(); // check if the file is compressed, for now we only support uncompressed and inflate/deflate compression if( cdfh->compressionMethod != 0 && cdfh->compressionMethod != Z_DEFLATED ) return XrdCl::XRootDStatus( XrdCl::stError, XrdCl::errNotSupported, 0, "The compression algorithm is not supported!" ); // Now the problem is that at the beginning of our // file there is the Local-file-header, which size // is not known because of the variable size 'extra' // field, so we need to know the offset of the next // record and shift it by the file size. // The next record is either the next LFH (next file) // or the start of the Central-directory. uint64_t cdOffset = zip64eocd ? zip64eocd->cdOffset : eocd->cdOffset; uint64_t nextRecordOffset = ( cditr->second + 1 < cdvec.size() ) ? XrdCl::CDFH::GetOffset( *cdvec[cditr->second + 1] ) : cdOffset; uint64_t filesize = cdfh->compressedSize; if( filesize == std::numeric_limits::max() && cdfh->extra ) filesize = cdfh->extra->compressedSize; uint16_t descsize = cdfh->HasDataDescriptor() ? XrdCl::DataDescriptor::GetSize( cdfh->IsZIP64() ) : 0; offset = nextRecordOffset - filesize - descsize; return XrdCl::XRootDStatus(); } //----------------------------------------------------------------------- //! Create the central directory at the end of ZIP archive and close it // //! @param handler : user callback //! @param timeout : operation timeout //! @return : the status of the operation //----------------------------------------------------------------------- XRootDStatus CloseArchive( ResponseHandler *handler, uint16_t timeout = 0 ); //----------------------------------------------------------------------- //! Close an open file within the ZIP archive //! @return : the status of the operation //----------------------------------------------------------------------- inline XRootDStatus CloseFile() { if( openstage != Done || openfn.empty() ) return XRootDStatus( stError, errInvalidOp, 0, "Archive not opened." ); openfn.clear(); lfh.reset(); return XRootDStatus(); } //----------------------------------------------------------------------- //! List files in the ZIP archive //! @return : the status of the operation //----------------------------------------------------------------------- XRootDStatus List( DirectoryList *&list ); //----------------------------------------------------------------------- //! @return : true if ZIP archive has been successfully opened //----------------------------------------------------------------------- inline bool IsOpen() { return openstage == Done; } //------------------------------------------------------------------------ //! Check if the underlying file is using an encrypted connection //------------------------------------------------------------------------ inline bool IsSecure() { return archive.IsSecure(); } //----------------------------------------------------------------------- //! Set property on the underlying File object //----------------------------------------------------------------------- inline bool SetProperty( const std::string &name, const std::string &value ) { return archive.SetProperty( name, value ); } //----------------------------------------------------------------------- //! Get property on the underlying File object //----------------------------------------------------------------------- inline bool GetProperty( const std::string &name, std::string &value ) { return archive.GetProperty( name, value ); } //----------------------------------------------------------------------- //! Get the underlying File object //----------------------------------------------------------------------- inline File& GetFile() { return archive; } private: //----------------------------------------------------------------------- //! Append data to a new file, implementation //! //! @param size : number of bytes to be appended //! @param buffer : the buffer with the data to be appended //! @param handler : user callback //! @param timeout : operation timeout //! @return : the status of the operation //----------------------------------------------------------------------- XRootDStatus WriteImpl( uint32_t size, const void *buffer, ResponseHandler *handler, uint16_t timeout ); //----------------------------------------------------------------------- //! Open the ZIP archive in read-only mode without parsing the central //! directory. //! //! @param url : url of the ZIP archive //! @param handler : user callback //! @param timeout : operation timeout //! @return : operation status //----------------------------------------------------------------------- XRootDStatus OpenOnly( const std::string &url, bool update, ResponseHandler *handler, uint16_t timeout = 0 ); //----------------------------------------------------------------------- //! Get a buffer with central directory of the ZIP archive //! //! @return : buffer with central directory //----------------------------------------------------------------------- buffer_t GetCD(); //----------------------------------------------------------------------- //! Set central directory for the ZIP archive //! //! @param buffer : a buffer with the central directory to be set //----------------------------------------------------------------------- void SetCD( const buffer_t &buffer ); //----------------------------------------------------------------------- //! Package a response into AnyObject (erase the type) //! //! @param rsp : the response to be packaged //! @return : AnyObject with the response //----------------------------------------------------------------------- template inline static AnyObject* PkgRsp( Response *rsp ) { if( !rsp ) return nullptr; AnyObject *pkg = new AnyObject(); pkg->Set( rsp ); return pkg; } //----------------------------------------------------------------------- //! Free status and response //----------------------------------------------------------------------- template inline static void Free( XRootDStatus *st, Response *rsp ) { delete st; delete rsp; } //----------------------------------------------------------------------- //! Schedule a user callback to be executed in the thread-pool with given //! status and response. //! //! @param handler : user callback to be scheduled //! @param st : status to be passed to the callback //! @param rsp : response to be passed to the callback //----------------------------------------------------------------------- template inline static void Schedule( ResponseHandler *handler, XRootDStatus *st, Response *rsp = nullptr ) { if( !handler ) return Free( st, rsp ); ResponseJob *job = new ResponseJob( handler, st, PkgRsp( rsp ), 0 ); DefaultEnv::GetPostMaster()->GetJobManager()->QueueJob( job ); } //----------------------------------------------------------------------- //! Create a StatInfo object from ZIP archive stat info and the file size. //! //! @param starch : ZIP archive stat info //! @param size : file size //! @return : StatInfo object //----------------------------------------------------------------------- inline static StatInfo* make_stat( const StatInfo &starch, uint64_t size ) { StatInfo *info = new StatInfo( starch ); uint32_t flags = info->GetFlags(); info->SetFlags( flags & ( ~StatInfo::IsWritable ) ); // make sure it is not listed as writable info->SetSize( size ); return info; } //----------------------------------------------------------------------- //! Create a StatInfo object for a given file within the ZIP archive. //! //! @param fn : file name //! @return : StatInfo object for the given file //----------------------------------------------------------------------- inline StatInfo* make_stat( const std::string &fn ) { StatInfo *infoptr = 0; XRootDStatus st = archive.Stat( false, infoptr ); if (!st.IsOK()) return nullptr; std::unique_ptr stinfo( infoptr ); auto itr = cdmap.find( fn ); if( itr == cdmap.end() ) return nullptr; size_t index = itr->second; uint64_t uncompressedSize = cdvec[index]->uncompressedSize; if( cdvec[index]->extra && uncompressedSize == std::numeric_limits::max() ) uncompressedSize = cdvec[index]->extra->uncompressedSize; return make_stat( *stinfo, uncompressedSize ); } //----------------------------------------------------------------------- //! Allocate new XRootDStatus object //----------------------------------------------------------------------- inline static XRootDStatus* make_status( const XRootDStatus &status = XRootDStatus() ) { return new XRootDStatus( status ); } //----------------------------------------------------------------------- //! Clear internal ZipArchive objects //----------------------------------------------------------------------- inline void Clear() { buffer.reset(); eocd.reset(); cdvec.clear(); cdmap.clear(); zip64eocd.reset(); openstage = None; } //----------------------------------------------------------------------- //! Stages of opening and parsing a ZIP archive //----------------------------------------------------------------------- enum OpenStages { None = 0, //< opening/parsing not started HaveEocdBlk, //< we have the End of Central Directory record HaveZip64EocdlBlk, //< we have the ZIP64 End of Central Directory locator record HaveZip64EocdBlk, //< we have the ZIP64 End of Central Directory record HaveCdRecords, //< we have Central Directory records Done, //< we are done parsing the Central Directory Error, //< opening/parsing failed NotParsed //< the ZIP archive has been opened but Central Directory is not parsed }; //----------------------------------------------------------------------- //! LFH of a newly appended file (in case it needs to be overwritten) //----------------------------------------------------------------------- struct NewFile { NewFile( uint64_t offset, std::unique_ptr lfh ) : offset( offset ), lfh( std::move( lfh ) ), overwrt( false ) { } NewFile( NewFile && nf ) : offset( nf.offset ), lfh( std::move( nf.lfh ) ), overwrt( nf.overwrt ) { } uint64_t offset; // the offset of the LFH of the file std::unique_ptr lfh; // LFH of the file bool overwrt; // if true the LFH needs to be overwritten on close }; //----------------------------------------------------------------------- //! Type that maps file name to its cache //----------------------------------------------------------------------- typedef std::unordered_map zipcache_t; typedef std::unordered_map new_files_t; File archive; //> File object for handling the ZIP archive uint64_t archsize; //> size of the ZIP archive bool cdexists; //> true if Central Directory exists, false otherwise bool updated; //> true if the ZIP archive has been updated, false otherwise std::unique_ptr buffer; //> buffer for keeping the data to be parsed or raw data std::unique_ptr eocd; //> End of Central Directory record cdvec_t cdvec; //> vector of Central Directory File Headers cdmap_t cdmap; //> mapping of file name to CDFH index uint64_t cdoff; //> Central Directory offset uint32_t orgcdsz; //> original CD size uint32_t orgcdcnt; //> original number CDFH records buffer_t orgcdbuf; //> buffer with the original CDFH records std::unique_ptr zip64eocd; //> ZIP64 End of Central Directory record OpenStages openstage; //> stage of opening / parsing a ZIP archive std::string openfn; //> file name of opened file zipcache_t zipcache; //> cache for inflating compressed data std::unique_ptr lfh; //> Local File Header record for the newly appended file bool ckpinit; //> a flag indicating whether a checkpoint has been initialized new_files_t newfiles; //> all newly appended files }; } /* namespace XrdZip */ #endif /* SRC_XRDZIP_XRDZIPARCHIVE_HH_ */ xrootd-5.6.9/src/XrdCl/XrdClZipCache.hh000066400000000000000000000206321457266313600176410ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2014 by European Organization for Nuclear Research (CERN) // Author: Michal Simon //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #ifndef SRC_XRDZIP_XRDZIPINFLCACHE_HH_ #define SRC_XRDZIP_XRDZIPINFLCACHE_HH_ #include "XrdCl/XrdClXRootDResponses.hh" #include #include #include #include #include #include #include namespace XrdCl { //--------------------------------------------------------------------------- //! An exception for carrying the XRootDStatus of InflCache //--------------------------------------------------------------------------- struct ZipError : public std::exception { ZipError( const XrdCl::XRootDStatus &status ) : status( status ) { } XrdCl::XRootDStatus status; }; //--------------------------------------------------------------------------- //! Utility class for inflating a compressed buffer //--------------------------------------------------------------------------- class ZipCache { public: typedef std::vector buffer_t; private: typedef std::tuple read_args_t; typedef std::tuple read_resp_t; struct greater_read_resp_t { inline bool operator() ( const read_resp_t &lhs, const read_resp_t &rhs ) const { return std::get<1>( lhs ) > std::get<1>( rhs ); } }; typedef std::priority_queue, greater_read_resp_t> resp_queue_t; public: ZipCache() : inabsoff( 0 ) { strm.zalloc = Z_NULL; strm.zfree = Z_NULL; strm.opaque = Z_NULL; strm.avail_in = 0; strm.next_in = Z_NULL; strm.avail_out = 0; strm.next_out = Z_NULL; // make sure zlib doesn't look for gzip headers, in order to do so // pass negative window bits !!! int rc = inflateInit2( &strm, -MAX_WBITS ); XrdCl::XRootDStatus st = ToXRootDStatus( rc, "inflateInit2" ); if( !st.IsOK() ) throw ZipError( st ); } ~ZipCache() { inflateEnd( &strm ); } inline void QueueReq( uint64_t offset, uint32_t length, void *buffer, ResponseHandler *handler ) { std::unique_lock lck( mtx ); rdreqs.emplace( offset, length, buffer, handler ); Decompress(); } inline void QueueRsp( const XRootDStatus &st, uint64_t offset, buffer_t &&buffer ) { std::unique_lock lck( mtx ); rdrsps.emplace( st, offset, std::move( buffer ) ); Decompress(); } private: inline bool HasInput() const { return strm.avail_in != 0; } inline bool HasOutput() const { return strm.avail_out != 0; } inline void Input( const read_resp_t &rdrsp ) { const buffer_t &buffer = std::get<2>( rdrsp ); strm.avail_in = buffer.size(); strm.next_in = (Bytef*)buffer.data(); } inline void Output( const read_args_t &rdreq ) { strm.avail_out = std::get<1>( rdreq ); strm.next_out = (Bytef*)std::get<2>( rdreq ); } inline bool Consecutive( const read_resp_t &resp ) const { return ( std::get<1>( resp ) == inabsoff ); } void Decompress() { while( HasInput() || HasOutput() || !rdreqs.empty() || !rdrsps.empty() ) { if( !HasOutput() && !rdreqs.empty() ) Output( rdreqs.front() ); if( !HasInput() && !rdrsps.empty() && Consecutive( rdrsps.top() ) ) // the response might come out of order so we need to check the offset Input( rdrsps.top() ); if( !HasInput() || !HasOutput() ) return; // check the response status XRootDStatus st = std::get<0>( rdrsps.top() ); if( !st.IsOK() ) return CallHandler( st ); // the available space in output buffer before inflating uInt avail_before = strm.avail_in; // decompress the data int rc = inflate( &strm, Z_SYNC_FLUSH ); st = ToXRootDStatus( rc, "inflate" ); if( !st.IsOK() ) return CallHandler( st ); // report error to user handler // update the absolute input offset by the number of bytes we consumed inabsoff += avail_before - strm.avail_in; if( !strm.avail_out ) // the output buffer is empty meaning a request has been fulfilled CallHandler( XRootDStatus() ); // the input buffer is empty meaning a response has been consumed // (we need to check if there are any elements in the responses // queue as the input buffer might have been set directly by the user) if( !strm.avail_in && !rdrsps.empty() ) rdrsps.pop(); } } static inline AnyObject* PkgRsp( ChunkInfo *chunk ) { if( !chunk ) return nullptr; AnyObject *rsp = new AnyObject(); rsp->Set( chunk ); return rsp; } inline void CallHandler( const XRootDStatus &st ) { if( rdreqs.empty() ) return; read_args_t args = std::move( rdreqs.front() ); rdreqs.pop(); ChunkInfo *chunk = nullptr; if( st.IsOK() ) chunk = new ChunkInfo( std::get<0>( args ), std::get<1>( args ), std::get<2>( args ) ); ResponseHandler *handler = std::get<3>( args ); handler->HandleResponse( new XRootDStatus( st ), PkgRsp( chunk ) ); } XrdCl::XRootDStatus ToXRootDStatus( int rc, const std::string &func ) { std::string msg = "[zlib] " + func + " : "; switch( rc ) { case Z_STREAM_END : case Z_OK : return XrdCl::XRootDStatus(); case Z_BUF_ERROR : return XrdCl::XRootDStatus( XrdCl::stOK, XrdCl::suContinue ); case Z_MEM_ERROR : return XrdCl::XRootDStatus( XrdCl::stError, XrdCl::errInternal, Z_MEM_ERROR, msg + "not enough memory." ); case Z_VERSION_ERROR : return XrdCl::XRootDStatus( XrdCl::stError, XrdCl::errInternal, Z_VERSION_ERROR, msg + "version mismatch." ); case Z_STREAM_ERROR : return XrdCl::XRootDStatus( XrdCl::stError, XrdCl::errInvalidArgs, Z_STREAM_ERROR, msg + "invalid argument." ); case Z_NEED_DICT : return XrdCl::XRootDStatus( XrdCl::stError, XrdCl::errDataError, Z_NEED_DICT, msg + "need dict."); case Z_DATA_ERROR : return XrdCl::XRootDStatus( XrdCl::stError, XrdCl::errDataError, Z_DATA_ERROR, msg + "corrupted data." ); default : return XrdCl::XRootDStatus( XrdCl::stError, XrdCl::errUnknown ); } } z_stream strm; // the zlib stream we will use for reading std::mutex mtx; uint64_t inabsoff; //< the absolute offset in the input file (compressed), ensures the user is actually streaming the data std::queue rdreqs; //< pending read requests (we only allow read requests to be submitted in order) resp_queue_t rdrsps; //< pending read responses (due to multiple-streams the read response may come out of order) }; } #endif /* SRC_XRDZIP_XRDZIPINFLCACHE_HH_ */ xrootd-5.6.9/src/XrdCl/XrdClZipListHandler.cc000066400000000000000000000077411457266313600210430ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2014 by European Organization for Nuclear Research (CERN) // Author: Michal Simon //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #include "XrdClZipListHandler.hh" namespace XrdCl { void ZipListHandler::HandleResponse( XrdCl::XRootDStatus *statusptr, XrdCl::AnyObject *responseptr ) { std::unique_ptr status( statusptr ); std::unique_ptr response( responseptr ); if( pStep == DONE ) { delete this; return; } if( !status->IsOK() ) { pHandler->HandleResponse( status.release(), response.release() ); delete this; return; } time_t took = time( 0 ) - pStartTime; if( took > pTimeout ) { *status = XRootDStatus( stError, errOperationExpired ); pHandler->HandleResponse( status.release(), 0 ); if( pZip.IsOpen() ) { DoZipClose( 1 ); pStep = DONE; } else delete this; return; } uint16_t left = pTimeout - took; switch( pStep ) { case STAT: { StatInfo *info = 0; response->Get( info ); if( info->TestFlags( StatInfo::IsDir ) ) DoDirList( left ); else DoZipOpen( left ); break; } case OPEN: { DirectoryList *list = 0; XRootDStatus st = pZip.List( list ); if( !st.IsOK() ) { pHandler->HandleResponse( new XRootDStatus( st ), 0 ); pStep = DONE; } else { pDirList.reset( list ); DoZipClose( left ); } break; } case CLOSE: { AnyObject *resp = new AnyObject(); resp->Set( pDirList.release() ); pHandler->HandleResponse( new XRootDStatus(), resp ); pStep = DONE; break; } } if( pStep == DONE ) delete this; } void ZipListHandler::DoDirList( time_t timeLeft ) { FileSystem fs( pUrl ); // remove the Zip flag so we don't enter an infinite loop pFlags &= ~DirListFlags::Zip; XRootDStatus st = fs.DirList( pUrl.GetPath(), pFlags, pHandler , timeLeft ); pStep = DONE; // no matter whether it works or not, either way we are done if( !st.IsOK() ) pHandler->HandleResponse( new XRootDStatus( st ), 0 ); } void ZipListHandler::DoZipOpen( time_t timeLeft ) { XRootDStatus st = pZip.OpenArchive( pUrl.GetURL(), OpenFlags::Read, this, timeLeft ); if( !st.IsOK() ) { pHandler->HandleResponse( new XRootDStatus( st ), 0 ); pStep = DONE; } else pStep = OPEN; } void ZipListHandler::DoZipClose( time_t timeLeft ) { XRootDStatus st = pZip.CloseArchive( this, timeLeft ); if( !st.IsOK() ) { pHandler->HandleResponse( new XRootDStatus( st ), 0 ); pStep = DONE; } else pStep = CLOSE; } } /* namespace XrdCl */ xrootd-5.6.9/src/XrdCl/XrdClZipListHandler.hh000066400000000000000000000121501457266313600210430ustar00rootroot00000000000000//------------------------------------------------------------------------------ // Copyright (c) 2011-2014 by European Organization for Nuclear Research (CERN) // Author: Michal Simon //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // // XRootD is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // XRootD is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with XRootD. If not, see . // // In applying this licence, CERN does not waive the privileges and immunities // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. //------------------------------------------------------------------------------ #ifndef SRC_XRDCL_XRDCLZIPLISTHANDLER_HH_ #define SRC_XRDCL_XRDCLZIPLISTHANDLER_HH_ #include "XrdCl/XrdClXRootDResponses.hh" #include "XrdCl/XrdClFileSystem.hh" #include "XrdCl/XrdClFile.hh" #include "XrdCl/XrdClZipArchive.hh" #include "XrdCl/XrdClConstants.hh" #include "XrdCl/XrdClDefaultEnv.hh" #include #include namespace XrdCl { //---------------------------------------------------------------------------- // DirList : Handle not a directory error //---------------------------------------------------------------------------- class ZipListHandler : public ResponseHandler { //------------------------------------------------------------------------ //! Possible steps in ZIP listing //! - STAT : stat the URL //! - OPEN : open the ZIP archive //! - CLOSE : close the ZIP archive //1 - DONE : we are done //------------------------------------------------------------------------ enum Steps { STAT = 0, OPEN = 1, CLOSE = 2, DONE = 4 }; public: //------------------------------------------------------------------------ //! Constructor //! //! @param url : endpoint URL //! @param path : path to the ZIP //! @param flags : listing flags //! @param handler : the original response handler //! @param timeout : operation timeout //------------------------------------------------------------------------ ZipListHandler( const URL &url, const std::string &path, DirListFlags::Flags flags, ResponseHandler *handler, uint16_t timeout = 0 ) : pUrl( url ), pFlags( flags ), pHandler( handler ), pTimeout( timeout ), pStartTime( time( 0 ) ), pStep( STAT ) { if( !pTimeout ) { int val = DefaultRequestTimeout; DefaultEnv::GetEnv()->GetInt( "RequestTimeout", val ); pTimeout = val; } pUrl.SetPath( path ); } //------------------------------------------------------------------------ //! Destructor //------------------------------------------------------------------------ ~ZipListHandler() { } //------------------------------------------------------------------------ //! Handle the server response //------------------------------------------------------------------------ virtual void HandleResponse( XrdCl::XRootDStatus *statusptr, XrdCl::AnyObject *responseptr ); private: //------------------------------------------------------------------------ //! Do normal listing if it is a directory (and not a ZIP archive) //------------------------------------------------------------------------ void DoDirList( time_t timeLeft ); //------------------------------------------------------------------------ //! Open the ZIP archive //------------------------------------------------------------------------ void DoZipOpen( time_t timeLeft ); //------------------------------------------------------------------------ //! Close the ZIP archive //------------------------------------------------------------------------ void DoZipClose( time_t timeLeft ); URL pUrl; DirListFlags::Flags pFlags; ResponseHandler *pHandler; uint16_t pTimeout; std::unique_ptr pDirList; time_t pStartTime; File pFile; ZipArchive pZip; int pStep; }; } /* namespace XrdCl */ #endif /* SRC_XRDCL_XRDCLZIPLISTHANDLER_HH_ */ xrootd-5.6.9/src/XrdCl/XrdClZipOperations.hh000066400000000000000000000660641457266313600207720ustar00rootroot00000000000000/* * XrdClZipOperations.hh * * Created on: 26 Nov 2020 * Author: simonm */ #ifndef SRC_XRDCL_XRDCLZIPOPERATIONS_HH_ #define SRC_XRDCL_XRDCLZIPOPERATIONS_HH_ #include "XrdCl/XrdClZipArchive.hh" #include "XrdCl/XrdClOperations.hh" #include "XrdCl/XrdClOperationHandlers.hh" #include "XrdCl/XrdClCtx.hh" namespace XrdCl { //---------------------------------------------------------------------------- //! Base class for all zip archive related operations //! //! @arg Derived : the class that derives from this template (CRTP) //! @arg HasHndl : true if operation has a handler, false otherwise //! @arg Args : operation arguments //---------------------------------------------------------------------------- template class Derived, bool HasHndl, typename Response, typename ... Arguments> class ZipOperation: public ConcreteOperation { template class, bool, typename, typename ...> friend class ZipOperation; public: //------------------------------------------------------------------------ //! Constructor //! //! @param zip : file on which the operation will be performed //! @param args : file operation arguments //------------------------------------------------------------------------ ZipOperation( Ctx zip, Arguments... args): ConcreteOperation( std::move( args )... ), zip( std::move( zip ) ) { } //------------------------------------------------------------------------ //! Move constructor from other states //! //! @arg from : state from which the object is being converted //! //! @param op : the object that is being converted //------------------------------------------------------------------------ template ZipOperation( ZipOperation && op ) : ConcreteOperation( std::move( op ) ), zip( op.zip ) { } //------------------------------------------------------------------------ //! Destructor //------------------------------------------------------------------------ virtual ~ZipOperation() { } protected: //------------------------------------------------------------------------ //! The file object itself //------------------------------------------------------------------------ Ctx zip; }; //---------------------------------------------------------------------------- //! OpenArchive operation (@see ZipOperation) //---------------------------------------------------------------------------- template class OpenArchiveImpl: public ZipOperation, Arg, Arg> { public: //------------------------------------------------------------------------ //! Inherit constructors from FileOperation (@see FileOperation) //------------------------------------------------------------------------ using ZipOperation, Arg, Arg>::ZipOperation; //------------------------------------------------------------------------ //! Argument indexes in the args tuple //------------------------------------------------------------------------ enum { UrlArg, FlagsArg }; //------------------------------------------------------------------------ //! @return : name of the operation (@see Operation) //------------------------------------------------------------------------ std::string ToString() { return "ZipOpen"; } protected: //------------------------------------------------------------------------ //! RunImpl operation (@see Operation) //! //! @param params : container with parameters forwarded from //! previous operation //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus RunImpl( PipelineHandler *handler, uint16_t pipelineTimeout ) { std::string &url = std::get( this->args ).Get(); OpenFlags::Flags flags = std::get( this->args ).Get(); uint16_t timeout = pipelineTimeout < this->timeout ? pipelineTimeout : this->timeout; return this->zip->OpenArchive( url, flags, handler, timeout ); } }; //---------------------------------------------------------------------------- //! Factory for creating OpenArchiveImpl objects //---------------------------------------------------------------------------- inline OpenArchiveImpl OpenArchive( Ctx zip, Arg fn, Arg flags, uint16_t timeout = 0 ) { return OpenArchiveImpl( std::move( zip ), std::move( fn ), std::move( flags ) ).Timeout( timeout ); } //---------------------------------------------------------------------------- //! OpenFile operation (@see ZipOperation) //---------------------------------------------------------------------------- template class OpenFileImpl: public ZipOperation, Arg, Arg, Arg, Arg> { public: //------------------------------------------------------------------------ //! Inherit constructors from FileOperation (@see FileOperation) //------------------------------------------------------------------------ using ZipOperation, Arg, Arg, Arg, Arg>::ZipOperation; //------------------------------------------------------------------------ //! Argument indexes in the args tuple //------------------------------------------------------------------------ enum { FnArg, FlagsArg, SizeArg, Crc32Arg }; //------------------------------------------------------------------------ //! @return : name of the operation (@see Operation) //------------------------------------------------------------------------ std::string ToString() { return "ZipOpenFile"; } protected: //------------------------------------------------------------------------ //! RunImpl operation (@see Operation) //! //! @param params : container with parameters forwarded from //! previous operation //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus RunImpl( PipelineHandler *handler, uint16_t pipelineTimeout ) { std::string &fn = std::get( this->args ).Get(); OpenFlags::Flags flags = std::get( this->args ).Get(); uint64_t size = std::get( this->args ).Get(); uint32_t crc32 = std::get( this->args ).Get(); XRootDStatus st = this->zip->OpenFile( fn, flags, size, crc32 ); if( !st.IsOK() ) return st; handler->HandleResponse( new XRootDStatus(), nullptr ); return XRootDStatus(); } }; //---------------------------------------------------------------------------- //! Factory for creating OpenFileImpl objects //---------------------------------------------------------------------------- inline OpenFileImpl OpenFile( Ctx zip, Arg fn, Arg flags = OpenFlags::None, Arg size = 0, Arg crc32 = 0, uint16_t timeout = 0 ) { return OpenFileImpl( std::move( zip ), std::move( fn ), std::move( flags ), std::move( size ), std::move( crc32 ) ).Timeout( timeout ); } //---------------------------------------------------------------------------- //! Read operation (@see ZipOperation) //---------------------------------------------------------------------------- template class ZipReadImpl: public ZipOperation, Arg, Arg, Arg> { public: //------------------------------------------------------------------------ //! Inherit constructors from FileOperation (@see FileOperation) //------------------------------------------------------------------------ using ZipOperation, Arg, Arg, Arg>::ZipOperation; //------------------------------------------------------------------------ //! Argument indexes in the args tuple //------------------------------------------------------------------------ enum { OffsetArg, SizeArg, BufferArg }; //------------------------------------------------------------------------ //! @return : name of the operation (@see Operation) //------------------------------------------------------------------------ std::string ToString() { return "ZipRead"; } protected: //------------------------------------------------------------------------ //! RunImpl operation (@see Operation) //! //! @param params : container with parameters forwarded from //! previous operation //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus RunImpl( PipelineHandler *handler, uint16_t pipelineTimeout ) { uint64_t offset = std::get( this->args ).Get(); uint32_t size = std::get( this->args ).Get(); void *buffer = std::get( this->args ).Get(); uint16_t timeout = pipelineTimeout < this->timeout ? pipelineTimeout : this->timeout; return this->zip->Read( offset, size, buffer, handler, timeout ); } }; //---------------------------------------------------------------------------- //! Factory for creating ArchiveReadImpl objects //---------------------------------------------------------------------------- inline ZipReadImpl Read( Ctx zip, Arg offset, Arg size, Arg buffer, uint16_t timeout = 0 ) { return ZipReadImpl( std::move( zip ), std::move( offset ), std::move( size ), std::move( buffer ) ).Timeout( timeout ); } //---------------------------------------------------------------------------- //! Read operation (@see ZipOperation) //---------------------------------------------------------------------------- template class ZipReadFromImpl: public ZipOperation, Arg, Arg, Arg, Arg> { public: //------------------------------------------------------------------------ //! Inherit constructors from FileOperation (@see FileOperation) //------------------------------------------------------------------------ using ZipOperation, Arg, Arg, Arg, Arg>::ZipOperation; //------------------------------------------------------------------------ //! Argument indexes in the args tuple //------------------------------------------------------------------------ enum { FileNameArg, OffsetArg, SizeArg, BufferArg }; //------------------------------------------------------------------------ //! @return : name of the operation (@see Operation) //------------------------------------------------------------------------ std::string ToString() { return "ZipReadFrom"; } protected: //------------------------------------------------------------------------ //! RunImpl operation (@see Operation) //! //! @param params : container with parameters forwarded from //! previous operation //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus RunImpl( PipelineHandler *handler, uint16_t pipelineTimeout ) { std::string &fn = std::get( this->args ).Get(); uint64_t offset = std::get( this->args ).Get(); uint32_t size = std::get( this->args ).Get(); void *buffer = std::get( this->args ).Get(); uint16_t timeout = pipelineTimeout < this->timeout ? pipelineTimeout : this->timeout; return this->zip->ReadFrom( fn, offset, size, buffer, handler, timeout ); } }; //---------------------------------------------------------------------------- //! Factory for creating ArchiveReadImpl objects //---------------------------------------------------------------------------- inline ZipReadFromImpl ReadFrom( Ctx zip, Arg fn, Arg offset, Arg size, Arg buffer, uint16_t timeout = 0 ) { return ZipReadFromImpl( std::move( zip ), std::move( fn ), std::move( offset ), std::move( size ), std::move( buffer ) ).Timeout( timeout ); } //---------------------------------------------------------------------------- //! Write operation (@see ZipOperation) //---------------------------------------------------------------------------- template class ZipWriteImpl: public ZipOperation, Arg, Arg> { public: //------------------------------------------------------------------------ //! Inherit constructors from FileOperation (@see FileOperation) //------------------------------------------------------------------------ using ZipOperation, Arg, Arg>::ZipOperation; //------------------------------------------------------------------------ //! Argument indexes in the args tuple //------------------------------------------------------------------------ enum { SizeArg, BufferArg }; //------------------------------------------------------------------------ //! @return : name of the operation (@see Operation) //------------------------------------------------------------------------ std::string ToString() { return "ZipWrite"; } protected: //------------------------------------------------------------------------ //! RunImpl operation (@see Operation) //! //! @param params : container with parameters forwarded from //! previous operation //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus RunImpl( PipelineHandler *handler, uint16_t pipelineTimeout ) { uint32_t size = std::get( this->args ).Get(); const void *buffer = std::get( this->args ).Get(); uint16_t timeout = pipelineTimeout < this->timeout ? pipelineTimeout : this->timeout; return this->zip->Write( size, buffer, handler, timeout ); } }; //---------------------------------------------------------------------------- //! Factory for creating ArchiveReadImpl objects //---------------------------------------------------------------------------- inline ZipWriteImpl Write( Ctx zip, Arg size, Arg buffer, uint16_t timeout = 0 ) { return ZipWriteImpl( std::move( zip ), std::move( size ), std::move( buffer ) ).Timeout( timeout ); } //---------------------------------------------------------------------------- //! AppendFile operation (@see ZipOperation) //---------------------------------------------------------------------------- template class AppendFileImpl: public ZipOperation, Arg, Arg, Arg, Arg> { public: //------------------------------------------------------------------------ //! Inherit constructors from FileOperation (@see FileOperation) //------------------------------------------------------------------------ using ZipOperation, Arg, Arg, Arg, Arg>::ZipOperation; //------------------------------------------------------------------------ //! Argument indexes in the args tuple //------------------------------------------------------------------------ enum { FnArg, CrcArg, SizeArg, BufferArg }; //------------------------------------------------------------------------ //! @return : name of the operation (@see Operation) //------------------------------------------------------------------------ std::string ToString() { return "AppendFile"; } protected: //------------------------------------------------------------------------ //! RunImpl operation (@see Operation) //! //! @param params : container with parameters forwarded from //! previous operation //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus RunImpl( PipelineHandler *handler, uint16_t pipelineTimeout ) { std::string &fn = std::get( this->args ).Get(); uint32_t crc32 = std::get( this->args ).Get(); uint32_t size = std::get( this->args ).Get(); const void *buffer = std::get( this->args ).Get(); uint16_t timeout = pipelineTimeout < this->timeout ? pipelineTimeout : this->timeout; return this->zip->AppendFile( fn, crc32, size, buffer, handler, timeout ); } }; //---------------------------------------------------------------------------- //! Factory for creating ArchiveReadImpl objects //---------------------------------------------------------------------------- inline AppendFileImpl AppendFile( Ctx zip, Arg fn, Arg crc32, Arg size, Arg buffer, uint16_t timeout = 0 ) { return AppendFileImpl( std::move( zip ), std::move( fn ), std::move( crc32 ), std::move( size ), std::move( buffer ) ).Timeout( timeout ); } //---------------------------------------------------------------------------- //! CloseFile operation (@see ZipOperation) //---------------------------------------------------------------------------- template class CloseFileImpl: public ZipOperation> { public: //------------------------------------------------------------------------ //! Inherit constructors from FileOperation (@see FileOperation) //------------------------------------------------------------------------ using ZipOperation>::ZipOperation; //------------------------------------------------------------------------ //! @return : name of the operation (@see Operation) //------------------------------------------------------------------------ std::string ToString() { return "ZipCloseFile"; } private: //------------------------------------------------------------------------ // this is not an async operation so we don't need a handler //------------------------------------------------------------------------ using ZipOperation>::operator>>; protected: //------------------------------------------------------------------------ //! RunImpl operation (@see Operation) //! //! @param params : container with parameters forwarded from //! previous operation //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus RunImpl( PipelineHandler *handler, uint16_t pipelineTimeout ) { XRootDStatus st = this->zip->CloseFile(); if( !st.IsOK() ) return st; handler->HandleResponse( new XRootDStatus(), nullptr ); return XRootDStatus(); } }; typedef CloseFileImpl CloseFile; //---------------------------------------------------------------------------- //! ZipStat operation (@see ZipOperation) //---------------------------------------------------------------------------- template class ZipStatImpl: public ZipOperation> { public: //------------------------------------------------------------------------ //! Inherit constructors from FileOperation (@see FileOperation) //------------------------------------------------------------------------ using ZipOperation>::ZipOperation; //------------------------------------------------------------------------ //! @return : name of the operation (@see Operation) //------------------------------------------------------------------------ std::string ToString() { return "ZipStat"; } protected: //------------------------------------------------------------------------ //! RunImpl operation (@see Operation) //! //! @param params : container with parameters forwarded from //! previous operation //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus RunImpl( PipelineHandler *handler, uint16_t pipelineTimeout ) { StatInfo *info = nullptr; XRootDStatus st = this->zip->Stat( info ); if( !st.IsOK() ) return st; AnyObject *rsp = new AnyObject(); rsp->Set( info ); handler->HandleResponse( new XRootDStatus(), rsp ); return XRootDStatus(); } }; //---------------------------------------------------------------------------- //! Factory for creating ZipStatImpl objects //---------------------------------------------------------------------------- inline ZipStatImpl Stat( Ctx zip ) { return ZipStatImpl( std::move( zip ) ); } //---------------------------------------------------------------------------- //! ZipList operation (@see ZipOperation) //---------------------------------------------------------------------------- template class ZipListImpl: public ZipOperation> { public: //------------------------------------------------------------------------ //! Inherit constructors from FileOperation (@see FileOperation) //------------------------------------------------------------------------ using ZipOperation>::ZipOperation; //------------------------------------------------------------------------ //! @return : name of the operation (@see Operation) //------------------------------------------------------------------------ std::string ToString() { return "ZipStat"; } protected: //------------------------------------------------------------------------ //! RunImpl operation (@see Operation) //! //! @param params : container with parameters forwarded from //! previous operation //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus RunImpl( PipelineHandler *handler, uint16_t pipelineTimeout ) { DirectoryList *list = nullptr; XRootDStatus st = this->zip->List( list ); if( !st.IsOK() ) return st; AnyObject *rsp = new AnyObject(); rsp->Set( list ); handler->HandleResponse( new XRootDStatus(), rsp ); return XRootDStatus(); } }; //---------------------------------------------------------------------------- //! Factory for creating ZipStatImpl objects //---------------------------------------------------------------------------- inline ZipListImpl List( Ctx zip ) { return ZipListImpl( std::move( zip ) ); } //---------------------------------------------------------------------------- //! CloseArchive operation (@see ZipOperation) //---------------------------------------------------------------------------- template class CloseArchiveImpl: public ZipOperation> { public: //------------------------------------------------------------------------ //! Inherit constructors from FileOperation (@see FileOperation) //------------------------------------------------------------------------ using ZipOperation>::ZipOperation; //------------------------------------------------------------------------ //! @return : name of the operation (@see Operation) //------------------------------------------------------------------------ std::string ToString() { return "ZipClose"; } protected: //------------------------------------------------------------------------ //! RunImpl operation (@see Operation) //! //! @param params : container with parameters forwarded from //! previous operation //! @return : status of the operation //------------------------------------------------------------------------ XRootDStatus RunImpl( PipelineHandler *handler, uint16_t pipelineTimeout ) { uint16_t timeout = pipelineTimeout < this->timeout ? pipelineTimeout : this->timeout; return this->zip->CloseArchive( handler, timeout ); } }; //---------------------------------------------------------------------------- //! Factory for creating CloseFileImpl objects //---------------------------------------------------------------------------- inline CloseArchiveImpl CloseArchive( Ctx zip, uint16_t timeout = 0 ) { return CloseArchiveImpl( std::move( zip ) ).Timeout( timeout ); } } #endif /* SRC_XRDCL_XRDCLZIPOPERATIONS_HH_ */ xrootd-5.6.9/src/XrdClHttp/000077500000000000000000000000001457266313600155325ustar00rootroot00000000000000xrootd-5.6.9/src/XrdClHttp/CMakeLists.txt000066400000000000000000000006371457266313600203000ustar00rootroot00000000000000set(libXrdClHttp_sources XrdClHttpPlugInFactory.cc XrdClHttpPlugInUtil.cc XrdClHttpFilePlugIn.cc XrdClHttpFileSystemPlugIn.cc XrdClHttpPosix.cc) set(PLUGIN_NAME "XrdClHttp-${PLUGIN_VERSION}") add_library(${PLUGIN_NAME} MODULE ${libXrdClHttp_sources}) target_link_libraries(${PLUGIN_NAME} PRIVATE Davix::Davix XrdCl XrdUtils) install(TARGETS ${PLUGIN_NAME} LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}) xrootd-5.6.9/src/XrdClHttp/XrdClHttpFilePlugIn.cc000066400000000000000000000344371457266313600216470ustar00rootroot00000000000000/** * This file is part of XrdClHttp */ #include "XrdClHttp/XrdClHttpFilePlugIn.hh" #include #include #include "XrdClHttp/XrdClHttpPlugInUtil.hh" #include "XrdClHttp/XrdClHttpPosix.hh" #include "XrdCl/XrdClDefaultEnv.hh" #include "XrdCl/XrdClLog.hh" #include "XrdCl/XrdClStatus.hh" #include "XrdOuc/XrdOucCRC.hh" namespace { int MakePosixOpenFlags(XrdCl::OpenFlags::Flags flags) { int posix_flags = 0; if (flags & XrdCl::OpenFlags::New) { posix_flags |= O_CREAT | O_EXCL; } if (flags & XrdCl::OpenFlags::Delete) { posix_flags |= O_CREAT | O_TRUNC; } if (flags & XrdCl::OpenFlags::Read) { posix_flags |= O_RDONLY; } if (flags & XrdCl::OpenFlags::Write) { posix_flags |= O_WRONLY; } if (flags & XrdCl::OpenFlags::Update) { posix_flags |= O_RDWR; } return posix_flags; } } // namespace namespace XrdCl { Davix::Context *root_davix_context_ = NULL; Davix::DavPosix *root_davix_client_file_ = NULL; HttpFilePlugIn::HttpFilePlugIn() : davix_fd_(nullptr), curr_offset(0), is_open_(false), filesize(0), url_(), properties_(), logger_(DefaultEnv::GetLog()) { SetUpLogging(logger_); logger_->Debug(kLogXrdClHttp, "HttpFilePlugin constructed."); std::string origin = getenv("XRDXROOTD_PROXY")? getenv("XRDXROOTD_PROXY") : ""; if ( origin.empty() || origin.find("=") == 0) { davix_context_ = new Davix::Context(); davix_client_ = new Davix::DavPosix(davix_context_); } else { if (root_davix_context_ == NULL) { root_davix_context_ = new Davix::Context(); root_davix_client_file_ = new Davix::DavPosix(root_davix_context_); } davix_context_ = root_davix_context_; davix_client_ = root_davix_client_file_; } } HttpFilePlugIn::~HttpFilePlugIn() noexcept { if (root_davix_context_ == NULL) { delete davix_client_; delete davix_context_; } } XRootDStatus HttpFilePlugIn::Open(const std::string &url, OpenFlags::Flags flags, Access::Mode /*mode*/, ResponseHandler *handler, uint16_t timeout) { if (is_open_) { logger_->Error(kLogXrdClHttp, "URL %s already open", url.c_str()); return XRootDStatus(stError, errInvalidOp); } if (XrdCl::URL(url).GetProtocol().find("https") == 0) isChannelEncrypted = true; else isChannelEncrypted = false; avoid_pread_ = false; if (getenv(HTTP_FILE_PLUG_IN_AVOIDRANGE_ENV) != NULL) avoid_pread_ = true; else { XrdCl::URL::ParamsMap CGIs = XrdCl::URL(url).GetParams(); auto search = CGIs.find(HTTP_FILE_PLUG_IN_AVOIDRANGE_CGI); if (search != CGIs.end()) avoid_pread_ = true; } Davix::RequestParams params; if (timeout != 0) { struct timespec ts = {timeout, 0}; params.setOperationTimeout(&ts); } if (flags & (OpenFlags::Write | OpenFlags::Update | OpenFlags::New)) { auto full_path = XrdCl::URL(url).GetLocation(); auto pos = full_path.find_last_of('/'); auto base_dir = pos != std::string::npos ? full_path.substr(0, pos) : full_path; auto mkdir_status = Posix::MkDir(*davix_client_, base_dir, XrdCl::MkDirFlags::MakePath, XrdCl::Access::None, timeout); if (mkdir_status.IsError()) { logger_->Error(kLogXrdClHttp, "Could not create parent directories when opening: %s", url.c_str()); return mkdir_status; } } if (((flags & OpenFlags::Write) || (flags & OpenFlags::Update)) && (flags & OpenFlags::Delete)) { auto stat_info = new StatInfo(); auto status = Posix::Stat(*davix_client_, url, timeout, stat_info); if (status.IsOK()) { auto unlink_status = Posix::Unlink(*davix_client_, url, timeout); if (unlink_status.IsError()) { logger_->Error( kLogXrdClHttp, "Could not delete existing destination file: %s. Error: %s", url.c_str(), unlink_status.GetErrorMessage().c_str()); return unlink_status; } } delete stat_info; } else if (flags & OpenFlags::Read) { auto stat_info = new StatInfo(); auto status = Posix::Stat(*davix_client_, url, timeout, stat_info); if (status.IsOK()) { filesize = stat_info->GetSize(); } delete stat_info; } auto posix_open_flags = MakePosixOpenFlags(flags); logger_->Debug(kLogXrdClHttp, "Open: URL: %s, XRootD flags: %d, POSIX flags: %d", url.c_str(), flags, posix_open_flags); // res == std::pair auto res = Posix::Open(*davix_client_, url, posix_open_flags, timeout); if (!res.first) { logger_->Error(kLogXrdClHttp, "Could not open: %s, error: %s", url.c_str(), res.second.ToStr().c_str()); return res.second; } davix_fd_ = res.first; logger_->Debug(kLogXrdClHttp, "Opened: %s", url.c_str()); is_open_ = true; url_ = url; auto status = new XRootDStatus(); handler->HandleResponse(status, nullptr); return XRootDStatus(); } XRootDStatus HttpFilePlugIn::Close(ResponseHandler *handler, uint16_t /*timeout*/) { if (!is_open_) { logger_->Error(kLogXrdClHttp, "Cannot close. URL hasn't been previously opened"); return XRootDStatus(stError, errInvalidOp); } logger_->Debug(kLogXrdClHttp, "Closing davix fd: %ld", davix_fd_); auto status = Posix::Close(*davix_client_, davix_fd_); if (status.IsError()) { logger_->Error(kLogXrdClHttp, "Could not close davix fd: %ld, error: %s", davix_fd_, status.ToStr().c_str()); return status; } is_open_ = false; url_.clear(); handler->HandleResponse(new XRootDStatus(), nullptr); return XRootDStatus(); } XRootDStatus HttpFilePlugIn::Stat(bool /*force*/, ResponseHandler *handler, uint16_t timeout) { if (!is_open_) { logger_->Error(kLogXrdClHttp, "Cannot stat. URL hasn't been previously opened"); return XRootDStatus(stError, errInvalidOp); } auto stat_info = new StatInfo(); auto status = Posix::Stat(*davix_client_, url_, timeout, stat_info); // A file that is_open_ = true should not retune 400/3011. the only time this // happen is a newly created file. Davix doesn't issue a http PUT so this file // won't show up for Stat(). Here we fake a response. if (status.IsError() && status.code == 400 && status.errNo == 3011) { std::ostringstream data; data << 140737018595560 << " " << filesize << " " << 33261 << " " << time(NULL); stat_info->ParseServerResponse(data.str().c_str()); } else if (status.IsError()) { logger_->Error(kLogXrdClHttp, "Stat failed: %s", status.ToStr().c_str()); return status; } logger_->Debug(kLogXrdClHttp, "Stat-ed URL: %s", url_.c_str()); auto obj = new AnyObject(); obj->Set(stat_info); handler->HandleResponse(new XRootDStatus(), obj); return XRootDStatus(); } XRootDStatus HttpFilePlugIn::Read(uint64_t offset, uint32_t size, void *buffer, ResponseHandler *handler, uint16_t /*timeout*/) { if (!is_open_) { logger_->Error(kLogXrdClHttp, "Cannot read. URL hasn't previously been opened"); return XRootDStatus(stError, errInvalidOp); } // DavPosix::pread will return -1 if the pread goes beyond the file size size = (offset + size > filesize)? filesize - offset : size; std::pair res; if (! avoid_pread_) { res = Posix::PRead(*davix_client_, davix_fd_, buffer, size, offset); } else { offset_locker.lock(); if (offset == curr_offset) { res = Posix::Read(*davix_client_, davix_fd_, buffer, size); } else { res = Posix::PRead(*davix_client_, davix_fd_, buffer, size, offset); } } if (res.second.IsError()) { logger_->Error(kLogXrdClHttp, "Could not read URL: %s, error: %s", url_.c_str(), res.second.ToStr().c_str()); if (avoid_pread_) offset_locker.unlock(); return res.second; } int num_bytes_read = res.first; curr_offset = offset + num_bytes_read; if (avoid_pread_) offset_locker.unlock(); logger_->Debug(kLogXrdClHttp, "Read %d bytes, at offset %d, from URL: %s", num_bytes_read, offset, url_.c_str()); auto status = new XRootDStatus(); auto chunk_info = new ChunkInfo(offset, num_bytes_read, buffer); auto obj = new AnyObject(); obj->Set(chunk_info); handler->HandleResponse(status, obj); return XRootDStatus(); } class PgReadSubstitutionHandler : public XrdCl::ResponseHandler { private: XrdCl::ResponseHandler *realHandler; bool isChannelEncrypted; public: // constructor PgReadSubstitutionHandler(XrdCl::ResponseHandler *a, bool isHttps) : realHandler(a), isChannelEncrypted(isHttps) {} // Response Handler void HandleResponse(XrdCl::XRootDStatus *status, XrdCl::AnyObject *rdresp) { if( !status->IsOK() ) { realHandler->HandleResponse( status, rdresp ); delete this; return; } //using namespace XrdCl; ChunkInfo *chunk = 0; rdresp->Get(chunk); std::vector cksums; if( isChannelEncrypted ) { size_t nbpages = chunk->length / XrdSys::PageSize; if( chunk->length % XrdSys::PageSize ) ++nbpages; cksums.reserve( nbpages ); size_t size = chunk->length; char *buffer = reinterpret_cast( chunk->buffer ); for( size_t pg = 0; pg < nbpages; ++pg ) { size_t pgsize = XrdSys::PageSize; if( pgsize > size ) pgsize = size; uint32_t crcval = XrdOucCRC::Calc32C( buffer, pgsize ); cksums.push_back( crcval ); buffer += pgsize; size -= pgsize; } } PageInfo *pages = new PageInfo(chunk->offset, chunk->length, chunk->buffer, std::move(cksums)); delete rdresp; AnyObject *response = new AnyObject(); response->Set( pages ); realHandler->HandleResponse( status, response ); delete this; } }; XRootDStatus HttpFilePlugIn::PgRead(uint64_t offset, uint32_t size, void *buffer, ResponseHandler *handler, uint16_t timeout) { ResponseHandler *substitHandler = new PgReadSubstitutionHandler( handler, isChannelEncrypted ); XRootDStatus st = Read(offset, size, buffer, substitHandler, timeout); return st; } XRootDStatus HttpFilePlugIn::Write(uint64_t offset, uint32_t size, const void *buffer, ResponseHandler *handler, uint16_t timeout) { if (!is_open_) { logger_->Error(kLogXrdClHttp, "Cannot write. URL hasn't previously been opened"); return XRootDStatus(stError, errInvalidOp); } // res == std::pair auto res = Posix::PWrite(*davix_client_, davix_fd_, offset, size, buffer, timeout); if (res.second.IsError()) { logger_->Error(kLogXrdClHttp, "Could not write URL: %s, error: %s", url_.c_str(), res.second.ToStr().c_str()); return res.second; } else filesize += res.first; logger_->Debug(kLogXrdClHttp, "Wrote %d bytes, at offset %d, to URL: %s", res.first, offset, url_.c_str()); handler->HandleResponse(new XRootDStatus(), nullptr); return XRootDStatus(); } //------------------------------------------------------------------------ //! @see XrdCl::File::PgWrite //------------------------------------------------------------------------ XRootDStatus HttpFilePlugIn::PgWrite( uint64_t offset, uint32_t size, const void *buffer, std::vector &cksums, ResponseHandler *handler, uint16_t timeout ) { (void)cksums; return Write(offset, size, buffer, handler, timeout); } XRootDStatus HttpFilePlugIn::Sync(ResponseHandler *handler, uint16_t timeout) { (void)handler; (void)timeout; logger_->Debug(kLogXrdClHttp, "Sync is a no-op for HTTP."); return XRootDStatus(); } XRootDStatus HttpFilePlugIn::VectorRead(const ChunkList &chunks, void *buffer, ResponseHandler *handler, uint16_t /*timeout*/) { if (!is_open_) { logger_->Error(kLogXrdClHttp, "Cannot read. URL hasn't previously been opened"); return XRootDStatus(stError, errInvalidOp); } const auto num_chunks = chunks.size(); std::vector input_vector(num_chunks); std::vector output_vector(num_chunks); for (size_t i = 0; i < num_chunks; ++i) { input_vector[i].diov_offset = chunks[i].offset; input_vector[i].diov_size = chunks[i].length; input_vector[i].diov_buffer = chunks[i].buffer; } // res == std::pair auto res = Posix::PReadVec(*davix_client_, davix_fd_, chunks, buffer); if (res.second.IsError()) { logger_->Error(kLogXrdClHttp, "Could not vectorRead URL: %s, error: %s", url_.c_str(), res.second.ToStr().c_str()); return res.second; } int num_bytes_read = res.first; logger_->Debug(kLogXrdClHttp, "VecRead %d bytes, from URL: %s", num_bytes_read, url_.c_str()); char *output = static_cast(buffer); for (size_t i = 0; i < num_chunks; ++i) { std::memcpy(output + input_vector[i].diov_offset, output_vector[i].diov_buffer, output_vector[i].diov_size); } auto status = new XRootDStatus(); auto read_info = new VectorReadInfo(); read_info->SetSize(num_bytes_read); read_info->GetChunks() = chunks; auto obj = new AnyObject(); obj->Set(read_info); handler->HandleResponse(status, obj); return XRootDStatus(); } bool HttpFilePlugIn::IsOpen() const { return is_open_; } bool HttpFilePlugIn::SetProperty(const std::string &name, const std::string &value) { properties_[name] = value; return true; } bool HttpFilePlugIn::GetProperty(const std::string &name, std::string &value) const { const auto p = properties_.find(name); if (p == std::end(properties_)) { return false; } value = p->second; return true; } } // namespace XrdCl xrootd-5.6.9/src/XrdClHttp/XrdClHttpFilePlugIn.hh000066400000000000000000000140521457266313600216500ustar00rootroot00000000000000/** * This file is part of XrdClHttp */ #ifndef __HTTP_FILE_PLUG_IN_ #define __HTTP_FILE_PLUG_IN_ #include "davix.hpp" #include "XrdCl/XrdClFile.hh" #include "XrdCl/XrdClFileSystem.hh" #include "XrdCl/XrdClPlugInInterface.hh" #include #include #include #include // Indicate desire to avoid http "Range: bytes=234-567" header // Some HTTP(s) data source does not honor Range request, and always start from // offset 0 when encounter a Range request, for example: // https://portal.nersc.gov/archive/home/projects/incite11/www/20C_Reanalysis_version_3/everymember_anal_netcdf/daily/WSPD10m/WSPD10m_1808_daily.tar // // 1. via Unix env via: this is global, avoid http ranger for all URLs #define HTTP_FILE_PLUG_IN_AVOIDRANGE_ENV "XRDCLHTTP_AVOIDRANGE" // 2. via CGI in URl, this only affect the associated URL #define HTTP_FILE_PLUG_IN_AVOIDRANGE_CGI "xrdclhttp_avoidrange" namespace XrdCl { class Log; class HttpFilePlugIn : public FilePlugIn { public: HttpFilePlugIn(); virtual ~HttpFilePlugIn() noexcept; //------------------------------------------------------------------------ //! @see XrdCl::File::Open //------------------------------------------------------------------------ virtual XRootDStatus Open( const std::string &url, OpenFlags::Flags flags, Access::Mode mode, ResponseHandler *handler, uint16_t timeout ) override; //------------------------------------------------------------------------ //! @see XrdCl::File::Close //------------------------------------------------------------------------ virtual XRootDStatus Close( ResponseHandler *handler, uint16_t timeout ) override; //------------------------------------------------------------------------ //! @see XrdCl::File::Stat //------------------------------------------------------------------------ virtual XRootDStatus Stat( bool force, ResponseHandler *handler, uint16_t timeout ) override; //------------------------------------------------------------------------ //! @see XrdCl::File::Read //------------------------------------------------------------------------ virtual XRootDStatus Read( uint64_t offset, uint32_t size, void *buffer, ResponseHandler *handler, uint16_t timeout ) override; //------------------------------------------------------------------------ //! @see XrdCl::File::PgRead - async //------------------------------------------------------------------------ virtual XRootDStatus PgRead( uint64_t offset, uint32_t size, void *buffer, ResponseHandler *handler, uint16_t timeout ) override; //------------------------------------------------------------------------ //! @see XrdCl::File::Write //------------------------------------------------------------------------ virtual XRootDStatus Write( uint64_t offset, uint32_t size, const void *buffer, ResponseHandler *handler, uint16_t timeout ) override; //------------------------------------------------------------------------ //! @see XrdCl::File::PgWrite - async //------------------------------------------------------------------------ virtual XRootDStatus PgWrite( uint64_t offset, uint32_t size, const void *buffer, std::vector &cksums, ResponseHandler *handler, uint16_t timeout ) override; //------------------------------------------------------------------------ //! @see XrdCl::File::Sync //------------------------------------------------------------------------ virtual XRootDStatus Sync( ResponseHandler *handler, uint16_t timeout ) override; //------------------------------------------------------------------------ //! @see XrdCl::File::VectorRead //------------------------------------------------------------------------ virtual XRootDStatus VectorRead( const ChunkList &chunks, void *buffer, XrdCl::ResponseHandler *handler, uint16_t timeout ) override; //------------------------------------------------------------------------ //! @see XrdCl::File::IsOpen //------------------------------------------------------------------------ virtual bool IsOpen() const override; //------------------------------------------------------------------------ //! @see XrdCl::File::SetProperty //------------------------------------------------------------------------ virtual bool SetProperty( const std::string &name, const std::string &value ) override; //------------------------------------------------------------------------ //! @see XrdCl::File::GetProperty //------------------------------------------------------------------------ virtual bool GetProperty( const std::string &name, std::string &value ) const override; private: Davix::Context *davix_context_; Davix::DavPosix *davix_client_; DAVIX_FD* davix_fd_; std::mutex offset_locker; uint64_t curr_offset; bool avoid_pread_; bool isChannelEncrypted; bool is_open_; uint64_t filesize; std::string url_; std::unordered_map properties_; Log* logger_; }; } #endif // __HTTP_FILE_PLUG_IN_ xrootd-5.6.9/src/XrdClHttp/XrdClHttpFileSystemPlugIn.cc000066400000000000000000000166021457266313600230460ustar00rootroot00000000000000/** * This file is part of XrdClHttp */ #include "XrdClHttp/XrdClHttpFileSystemPlugIn.hh" #include #include "davix.hpp" #include "XrdCl/XrdClDefaultEnv.hh" #include "XrdCl/XrdClLog.hh" #include "XrdCl/XrdClXRootDResponses.hh" #include "XrdClHttp/XrdClHttpFilePlugIn.hh" #include "XrdClHttp/XrdClHttpPlugInUtil.hh" #include "XrdClHttp/XrdClHttpPosix.hh" namespace XrdCl { Davix::Context *root_ctx_ = NULL; Davix::DavPosix *root_davix_client_ = NULL; HttpFileSystemPlugIn::HttpFileSystemPlugIn(const std::string &url) : url_(url), logger_(DefaultEnv::GetLog()) { SetUpLogging(logger_); logger_->Debug(kLogXrdClHttp, "HttpFileSystemPlugIn constructed with URL: %s.", url_.GetURL().c_str()); std::string origin = getenv("XRDXROOTD_PROXY")? getenv("XRDXROOTD_PROXY") : ""; if ( origin.empty() || origin.find("=") == 0) { ctx_ = new Davix::Context(); davix_client_ = new Davix::DavPosix(ctx_); } else { if (root_ctx_ == NULL) { root_ctx_ = new Davix::Context(); root_davix_client_ = new Davix::DavPosix(root_ctx_); } ctx_ = root_ctx_; davix_client_ = root_davix_client_; } } // destructor of davix_client_ or ctx_ will call something in ssl3 lib // which reset errno. We need to preserve errno so that XrdPssSys::Stat // will see it. HttpFileSystemPlugIn::~HttpFileSystemPlugIn() noexcept { int rc = errno; if (root_ctx_ == NULL) { delete davix_client_; delete ctx_; } errno = rc; } XRootDStatus HttpFileSystemPlugIn::Mv(const std::string &source, const std::string &dest, ResponseHandler *handler, uint16_t timeout) { //const auto full_source_path = url_.GetLocation() + source; //const auto full_dest_path = url_.GetLocation() + dest; const auto full_source_path = url_.GetProtocol() + "://" + url_.GetHostName() + ":" + std::to_string(url_.GetPort()) + source; const auto full_dest_path = url_.GetProtocol() + "://" + url_.GetHostName() + ":" + std::to_string(url_.GetPort()) + dest; logger_->Debug(kLogXrdClHttp, "HttpFileSystemPlugIn::Mv - src = %s, dest = %s, timeout = %d", full_source_path.c_str(), full_dest_path.c_str(), timeout); auto status = Posix::Rename(*davix_client_, full_source_path, full_dest_path, timeout); if (status.IsError()) { logger_->Error(kLogXrdClHttp, "Mv failed: %s", status.ToStr().c_str()); return status; } handler->HandleResponse(new XRootDStatus(status), nullptr); return XRootDStatus(); } XRootDStatus HttpFileSystemPlugIn::Rm(const std::string &path, ResponseHandler *handler, uint16_t timeout) { auto url = url_; url.SetPath(path); logger_->Debug(kLogXrdClHttp, "HttpFileSystemPlugIn::Rm - path = %s, timeout = %d", url.GetURL().c_str(), timeout); auto status = Posix::Unlink(*davix_client_, url.GetURL(), timeout); if (status.IsError()) { logger_->Error(kLogXrdClHttp, "Rm failed: %s", status.ToStr().c_str()); return status; } handler->HandleResponse(new XRootDStatus(status), nullptr); return XRootDStatus(); } XRootDStatus HttpFileSystemPlugIn::MkDir(const std::string &path, MkDirFlags::Flags flags, Access::Mode mode, ResponseHandler *handler, uint16_t timeout) { auto url = url_; url.SetPath(path); logger_->Debug( kLogXrdClHttp, "HttpFileSystemPlugIn::MkDir - path = %s, flags = %d, timeout = %d", url.GetURL().c_str(), flags, timeout); auto status = Posix::MkDir(*davix_client_, url.GetURL(), flags, mode, timeout); if (status.IsError()) { logger_->Error(kLogXrdClHttp, "MkDir failed: %s", status.ToStr().c_str()); return status; } handler->HandleResponse(new XRootDStatus(status), nullptr); return XRootDStatus(); } XRootDStatus HttpFileSystemPlugIn::RmDir(const std::string &path, ResponseHandler *handler, uint16_t timeout) { auto url = url_; url.SetPath(path); logger_->Debug(kLogXrdClHttp, "HttpFileSystemPlugIn::RmDir - path = %s, timeout = %d", url.GetURL().c_str(), timeout); auto status = Posix::RmDir(*davix_client_, url.GetURL(), timeout); if (status.IsError()) { logger_->Error(kLogXrdClHttp, "RmDir failed: %s", status.ToStr().c_str()); return status; } handler->HandleResponse(new XRootDStatus(status), nullptr); return XRootDStatus(); } XRootDStatus HttpFileSystemPlugIn::DirList(const std::string &path, DirListFlags::Flags flags, ResponseHandler *handler, uint16_t timeout) { auto url = url_; url.SetPath(path); const auto full_path = url.GetLocation(); logger_->Debug( kLogXrdClHttp, "HttpFileSystemPlugIn::DirList - path = %s, flags = %d, timeout = %d", full_path.c_str(), flags, timeout); const bool details = flags & DirListFlags::Stat; const bool recursive = flags & DirListFlags::Recursive; // res == std::pair auto res = Posix::DirList(*davix_client_, full_path, details, recursive, timeout); if (res.second.IsError()) { logger_->Error(kLogXrdClHttp, "Could not list dir: %s, error: %s", full_path.c_str(), res.second.ToStr().c_str()); return res.second; } auto obj = new AnyObject(); obj->Set(res.first); handler->HandleResponse(new XRootDStatus(), obj); return XRootDStatus(); } XRootDStatus HttpFileSystemPlugIn::Stat(const std::string &path, ResponseHandler *handler, uint16_t timeout) { //const auto full_path = url_.GetLocation() + path; const auto full_path = url_.GetProtocol() + "://" + url_.GetHostName() + ":" + std::to_string(url_.GetPort()) + "/" + path; logger_->Debug(kLogXrdClHttp, "HttpFileSystemPlugIn::Stat - path = %s, timeout = %d", full_path.c_str(), timeout); auto stat_info = new StatInfo(); //XRootDStatus status; auto status = Posix::Stat(*davix_client_, full_path, timeout, stat_info); if (status.IsError()) { logger_->Error(kLogXrdClHttp, "Stat failed: %s", status.ToStr().c_str()); return status; } auto obj = new AnyObject(); obj->Set(stat_info); handler->HandleResponse(new XRootDStatus(), obj); return XRootDStatus(); } bool HttpFileSystemPlugIn::SetProperty(const std::string &name, const std::string &value) { properties_[name] = value; return true; } bool HttpFileSystemPlugIn::GetProperty(const std::string &name, std::string &value) const { const auto p = properties_.find(name); if (p == std::end(properties_)) { return false; } value = p->second; return true; } } // namespace XrdCl xrootd-5.6.9/src/XrdClHttp/XrdClHttpFileSystemPlugIn.hh000066400000000000000000000035351457266313600230610ustar00rootroot00000000000000/** * This file is part of XrdClHttp */ #ifndef __HTTP_FILE_SYSTEM_PLUG_IN_ #define __HTTP_FILE_SYSTEM_PLUG_IN_ #include "davix.hpp" #include "XrdCl/XrdClPlugInInterface.hh" #include "XrdCl/XrdClURL.hh" #include namespace XrdCl { class Log; class HttpFileSystemPlugIn : public FileSystemPlugIn { public: HttpFileSystemPlugIn(const std::string &url); virtual ~HttpFileSystemPlugIn() noexcept; virtual XRootDStatus Mv(const std::string &source, const std::string &dest, ResponseHandler *handler, uint16_t timeout) override; virtual XRootDStatus Rm(const std::string &path, ResponseHandler *handler, uint16_t timeout) override; virtual XRootDStatus MkDir(const std::string &path, MkDirFlags::Flags flags, Access::Mode mode, ResponseHandler *handler, uint16_t timeout) override; virtual XRootDStatus RmDir(const std::string &path, ResponseHandler *handler, uint16_t timeout) override; virtual XRootDStatus DirList(const std::string &path, DirListFlags::Flags flags, ResponseHandler *handler, uint16_t timeout) override; virtual XRootDStatus Stat(const std::string &path, ResponseHandler *handler, uint16_t timeout) override; virtual bool SetProperty(const std::string &name, const std::string &value) override; virtual bool GetProperty(const std::string &name, std::string &value) const override; private: Davix::Context *ctx_; Davix::DavPosix *davix_client_; URL url_; std::unordered_map properties_; Log *logger_; }; } // namespace XrdCl #endif // __HTTP_FILE_SYSTEM_PLUG_IN_ xrootd-5.6.9/src/XrdClHttp/XrdClHttpPlugInFactory.cc000066400000000000000000000012641457266313600223670ustar00rootroot00000000000000/** * This file is part of XrdClHttp */ #include "XrdClHttp/XrdClHttpPlugInFactory.hh" #include "XrdVersion.hh" #include "XrdClHttp/XrdClHttpFilePlugIn.hh" #include "XrdClHttp/XrdClHttpFileSystemPlugIn.hh" XrdVERSIONINFO(XrdClGetPlugIn, XrdClGetPlugIn) extern "C" { void *XrdClGetPlugIn( const void* /*arg*/ ) { return static_cast( new HttpPlugInFactory()); } } HttpPlugInFactory::~HttpPlugInFactory() { } XrdCl::FilePlugIn* HttpPlugInFactory::CreateFile( const std::string &/*url*/ ) { return new XrdCl::HttpFilePlugIn(); } XrdCl::FileSystemPlugIn* HttpPlugInFactory::CreateFileSystem( const std::string& url ) { return new XrdCl::HttpFileSystemPlugIn(url); } xrootd-5.6.9/src/XrdClHttp/XrdClHttpPlugInFactory.hh000066400000000000000000000007411457266313600224000ustar00rootroot00000000000000/** * This file is part of XrdClHttp */ #ifndef __XRD_CL_HTTP__ #define __XRD_CL_HTTP__ #include extern "C" { void *XrdClGetPlugIn( const void* /*arg*/ ); } class HttpPlugInFactory : public XrdCl::PlugInFactory { virtual ~HttpPlugInFactory(); virtual XrdCl::FilePlugIn *CreateFile( const std::string &url ) override; virtual XrdCl::FileSystemPlugIn *CreateFileSystem( const std::string &url ) override; }; #endif // __XRD_CL_HTTP__ xrootd-5.6.9/src/XrdClHttp/XrdClHttpPlugInUtil.cc000066400000000000000000000006461457266313600217000ustar00rootroot00000000000000/** * This file is part of XrdClHttp */ #include "XrdClHttp/XrdClHttpPlugInUtil.hh" #include #include "XrdCl/XrdClLog.hh" static std::once_flag logging_topic_init; namespace XrdCl { void SetUpLogging(Log* logger) { // Assert that there is no existing topic std::call_once(logging_topic_init, [logger] { if (logger) { logger->SetTopicName(kLogXrdClHttp, "XrdClHttp"); } }); } } xrootd-5.6.9/src/XrdClHttp/XrdClHttpPlugInUtil.hh000066400000000000000000000005511457266313600217050ustar00rootroot00000000000000/** * This file is part of XrdClHttp */ #ifndef __HTTP_FILE_PLUG_IN_UTIL_ #define __HTTP_FILE_PLUG_IN_UTIL_ #include #include namespace XrdCl { class Log; // Topic id for the logger static const uint64_t kLogXrdClHttp = std::numeric_limits::max(); void SetUpLogging(Log* logger); } #endif // __HTTP_FILE_PLUG_IN_UTIL_xrootd-5.6.9/src/XrdClHttp/XrdClHttpPosix.cc000066400000000000000000000372331457266313600207500ustar00rootroot00000000000000/** * This file is part of XrdClHttp */ #include #include #include #include "XrdClHttp/XrdClHttpPosix.hh" #include "XProtocol/XProtocol.hh" #include "XrdCl/XrdClStatus.hh" #include "XrdCl/XrdClXRootDResponses.hh" #include "XrdCl/XrdClURL.hh" #include "auth/davixx509cred.hpp" #include "auth/davixauth.hpp" #include namespace { std::vector SplitString(const std::string& input, const std::string& delimiter) { size_t start = 0; size_t end = 0; size_t length = 0; auto result = std::vector{}; do { end = input.find(delimiter, start); if (end != std::string::npos) length = end - start; else length = input.length() - start; if (length) result.push_back(input.substr(start, length)); start = end + delimiter.size(); } while (end != std::string::npos); return result; } void SetTimeout(Davix::RequestParams& params, uint16_t timeout) { /* * At NERSC archive portal, we get error when setOperationTimeout() * if (timeout != 0) { struct timespec ts = {timeout, 0}; params.setOperationTimeout(&ts); } */ struct timespec ts = {0, 0}; ts.tv_sec = 30; params.setConnectionTimeout(&ts); params.setOperationRetry(0); params.setOperationRetryDelay(2); } XrdCl::XRootDStatus FillStatInfo(const struct stat& stats, XrdCl::StatInfo* stat_info) { std::ostringstream data; if (S_ISDIR(stats.st_mode)) { data << stats.st_dev << " " << stats.st_size << " " << (XrdCl::StatInfo::Flags::IsDir | XrdCl::StatInfo::Flags::IsReadable | XrdCl::StatInfo::Flags::IsWritable | XrdCl::StatInfo::Flags::XBitSet) << " " << stats.st_mtime; } else { if (getenv("AWS_ACCESS_KEY_ID")) { data << stats.st_dev << " " << stats.st_size << " " << XrdCl::StatInfo::Flags::IsReadable << " " << stats.st_mtime; } else { data << stats.st_dev << " " << stats.st_size << " " << stats.st_mode << " " << stats.st_mtime; } } if (!stat_info->ParseServerResponse(data.str().c_str())) { return XrdCl::XRootDStatus(XrdCl::stError, XrdCl::errDataError); } return XrdCl::XRootDStatus(); } // return NULL if no X509 proxy is found //Davix::X509Credential* LoadX509UserCredential() { // std::string myX509proxyFile; // if (getenv("X509_USER_PROXY") != NULL) // myX509proxyFile = getenv("X509_USER_PROXY"); // else // myX509proxyFile = "/tmp/x509up_u" + std::to_string(geteuid()); // // struct stat myX509proxyStat; // Davix::X509Credential* myX509proxy = NULL; // if (stat(myX509proxyFile.c_str(), &myX509proxyStat) == 0) { // myX509proxy = new Davix::X509Credential(); // myX509proxy->loadFromFilePEM(myX509proxyFile.c_str(), myX509proxyFile.c_str(), "", NULL); // } // return myX509proxy; //} // see auth/davixauth.hpp int LoadX509UserCredentialCallBack(void *userdata, const Davix::SessionInfo &info, Davix::X509Credential *cert, Davix::DavixError **err) { std::string myX509proxyFile; if (getenv("X509_USER_PROXY") != NULL) myX509proxyFile = getenv("X509_USER_PROXY"); else myX509proxyFile = "/tmp/x509up_u" + std::to_string(geteuid()); struct stat myX509proxyStat; if (stat(myX509proxyFile.c_str(), &myX509proxyStat) == 0) return cert->loadFromFilePEM(myX509proxyFile.c_str(), myX509proxyFile.c_str(), "", err); else return 1; } void SetX509(Davix::RequestParams& params) { params.setClientCertCallbackX509(&LoadX509UserCredentialCallBack, NULL); //Davix::X509Credential* myX509proxy = LoadX509UserCredential(); //if (myX509proxy != NULL) { // params.setClientCertX509(*myX509proxy); // delete myX509proxy; //} if (getenv("X509_CERT_DIR") != NULL) params.addCertificateAuthorityPath(getenv("X509_CERT_DIR")); else params.addCertificateAuthorityPath("/etc/grid-security/certificates"); } void SetAuthS3(Davix::RequestParams& params) { //Davix::setLogScope(DAVIX_LOG_SCOPE_ALL); //Davix::setLogScope(DAVIX_LOG_HEADER | DAVIX_LOG_S3); //Davix::setLogLevel(DAVIX_LOG_TRACE); params.setProtocol(Davix::RequestProtocol::AwsS3); params.setAwsAuthorizationKeys(getenv("AWS_SECRET_ACCESS_KEY"), getenv("AWS_ACCESS_KEY_ID")); params.setAwsAlternate(true); // if AWS region is not set, Davix will use the old AWS signature v2 if (getenv("AWS_REGION")) params.setAwsRegion(getenv("AWS_REGION")); else if (! getenv("AWS_SIGNATURE_V2")) params.setAwsRegion("mars"); } void SetAuthz(Davix::RequestParams& params) { if (getenv("AWS_ACCESS_KEY_ID") && getenv("AWS_SECRET_ACCESS_KEY")) SetAuthS3(params); else SetX509(params); } std::string SanitizedURL(const std::string& url) { XrdCl::URL xurl(url); std::string path = xurl.GetPath(); if (path.find("/") != 0) path = "/" + path; std::string returl = xurl.GetProtocol() + "://" + xurl.GetHostName() + ":" + std::to_string(xurl.GetPort()) + path; // for s3 storage using AWS_ACCESS_KEY_ID, filter out all CGIs // Known issues: // Google cloud storage does not like ?xrd.gsiusrpxy=/tmp/..., Will fail Stat() if (! getenv("AWS_ACCESS_KEY_ID") && ! xurl.GetParamsAsString().empty()) { returl = returl + xurl.GetParamsAsString(); } return returl; } // check davix/include/davix/status/davixstatusrequest.hpp and // XProtocol/XProtocol.hh (XErrorCode) for corresponding error codes. std::pair ErrCodeConvert(Davix::StatusCode::Code code) { if (code == Davix::StatusCode::FileNotFound) return std::make_pair(XrdCl::errErrorResponse, kXR_NotFound); else if (code == Davix::StatusCode::FileExist) return std::make_pair(XrdCl::errErrorResponse, kXR_ItExists); else if (code == Davix::StatusCode::PermissionRefused) return std::make_pair(XrdCl::errErrorResponse, kXR_NotAuthorized); else return std::make_pair(XrdCl::errErrorResponse, kXR_InvalidRequest); } } // namespace namespace Posix { using namespace XrdCl; std::pair Open(Davix::DavPosix& davix_client, const std::string& url, int flags, uint16_t timeout) { Davix::RequestParams params; SetTimeout(params, timeout); SetAuthz(params); Davix::DavixError* err = nullptr; DAVIX_FD* fd = davix_client.open(¶ms, SanitizedURL(url), flags, &err); XRootDStatus status; if (!fd) { auto res = ErrCodeConvert(err->getStatus()); status = XRootDStatus(stError, res.first, res.second, err->getErrMsg()); delete err; } else { status = XRootDStatus(); } return std::make_pair(fd, status); } XRootDStatus Close(Davix::DavPosix& davix_client, DAVIX_FD* fd) { Davix::DavixError* err = nullptr; if (davix_client.close(fd, &err)) { auto errStatus = XRootDStatus(stError, errInternal, err->getStatus(), err->getErrMsg()); delete err; return errStatus; } return XRootDStatus(); } XRootDStatus MkDir(Davix::DavPosix& davix_client, const std::string& path, XrdCl::MkDirFlags::Flags flags, XrdCl::Access::Mode /*mode*/, uint16_t timeout) { return XRootDStatus(); Davix::RequestParams params; SetTimeout(params, timeout); SetAuthz(params); auto DoMkDir = [&davix_client, ¶ms](const std::string& path) { Davix::DavixError* err = nullptr; if (davix_client.mkdir(¶ms, SanitizedURL(path), S_IRWXU, &err) && (err->getStatus() != Davix::StatusCode::FileExist)) { auto errStatus = XRootDStatus(stError, errInternal, err->getStatus(), err->getErrMsg()); delete err; return errStatus; } else { return XRootDStatus(); } }; auto url = XrdCl::URL(path); if (flags & XrdCl::MkDirFlags::MakePath) { // Also create intermediate directories auto dirs = SplitString(url.GetPath(), "/"); std::string dirs_cumul; for (const auto& d : dirs) { dirs_cumul += "/" + d; url.SetPath(dirs_cumul); auto status = DoMkDir(url.GetLocation()); if (status.IsError()) { return status; } } } else { // Only create final directory auto status = DoMkDir(url.GetURL()); if (status.IsError()) { return status; } } return XRootDStatus(); } XRootDStatus RmDir(Davix::DavPosix& davix_client, const std::string& path, uint16_t timeout) { Davix::RequestParams params; SetTimeout(params, timeout); SetAuthz(params); Davix::DavixError* err = nullptr; if (davix_client.rmdir(¶ms, path, &err)) { auto errStatus = XRootDStatus(stError, errInternal, err->getStatus(), err->getErrMsg()); delete err; return errStatus; } return XRootDStatus(); } std::pair DirList( Davix::DavPosix& davix_client, const std::string& path, bool details, bool /*recursive*/, uint16_t timeout) { Davix::RequestParams params; SetTimeout(params, timeout); SetAuthz(params); auto dir_list = new DirectoryList(); Davix::DavixError* err = nullptr; auto dir_fd = davix_client.opendirpp(¶ms, SanitizedURL(path), &err); if (!dir_fd) { auto errStatus = XRootDStatus(stError, errInternal, err->getStatus(), err->getErrMsg()); delete err; return std::make_pair(nullptr, errStatus); } struct stat info; while (auto entry = davix_client.readdirpp(dir_fd, &info, &err)) { if (err) { auto errStatus = XRootDStatus(stError, errInternal, err->getStatus(), err->getErrMsg()); delete err; return std::make_pair(nullptr, errStatus); } StatInfo* stat_info = nullptr; if (details) { stat_info = new StatInfo(); auto res = FillStatInfo(info, stat_info); if (res.IsError()) { delete entry; delete stat_info; return std::make_pair(nullptr, res); } } auto list_entry = new DirectoryList::ListEntry(path, entry->d_name, stat_info); dir_list->Add(list_entry); // do not delete "entry". davix_client.readdirpp() always return the same address // and will set it to NULL when there is no more directory entry to return //delete entry; } if (davix_client.closedirpp(dir_fd, &err)) { auto errStatus = XRootDStatus(stError, errInternal, err->getStatus(), err->getErrMsg()); delete err; return std::make_pair(nullptr, errStatus); } return std::make_pair(dir_list, XRootDStatus()); } XRootDStatus Rename(Davix::DavPosix& davix_client, const std::string& source, const std::string& dest, uint16_t timeout) { // most s3 storage systems either: // 1. do not support rename, especially for files that were uploaded using multi-part // 2. support by copy-n-delete. // we could implement copy-n-delete if necessary if (getenv("AWS_ACCESS_KEY_ID")) return XRootDStatus(stError, errErrorResponse, kXR_Unsupported); Davix::RequestParams params; SetTimeout(params, timeout); SetAuthz(params); Davix::DavixError* err = nullptr; if (davix_client.rename(¶ms, SanitizedURL(source), SanitizedURL(dest), &err)) { auto errStatus = XRootDStatus(stError, errInternal, err->getStatus(), err->getErrMsg()); delete err; return errStatus; } return XRootDStatus(); } XRootDStatus Stat(Davix::DavPosix& davix_client, const std::string& url, uint16_t timeout, StatInfo* stat_info) { Davix::RequestParams params; SetTimeout(params, timeout); SetAuthz(params); struct stat stats; Davix::DavixError* err = nullptr; if (davix_client.stat(¶ms, SanitizedURL(url), &stats, &err)) { auto res = ErrCodeConvert(err->getStatus()); auto errStatus = XRootDStatus(stError, res.first, res.second, err->getErrMsg()); delete err; return errStatus; } auto res = FillStatInfo(stats, stat_info); if (res.IsError()) { return res; } return XRootDStatus(); } XRootDStatus Unlink(Davix::DavPosix& davix_client, const std::string& url, uint16_t timeout) { Davix::RequestParams params; SetTimeout(params, timeout); SetAuthz(params); Davix::DavixError* err = nullptr; if (davix_client.unlink(¶ms, SanitizedURL(url), &err)) { auto errStatus = XRootDStatus(stError, errInternal, err->getStatus(), err->getErrMsg()); delete err; return errStatus; } return XRootDStatus(); } std::pair _PRead(Davix::DavPosix& davix_client, DAVIX_FD* fd, void* buffer, uint32_t size, uint64_t offset, bool no_pread = false) { Davix::DavixError* err = nullptr; int num_bytes_read; if (no_pread) { // continue reading from the current offset position num_bytes_read = davix_client.read(fd, buffer, size, &err); } else { num_bytes_read = davix_client.pread(fd, buffer, size, offset, &err); } if (num_bytes_read < 0) { auto errStatus = XRootDStatus(stError, errInternal, err->getStatus(), err->getErrMsg()); delete err; return std::make_pair(num_bytes_read, errStatus); } return std::make_pair(num_bytes_read, XRootDStatus()); } std::pair Read(Davix::DavPosix& davix_client, DAVIX_FD* fd, void* buffer, uint32_t size) { return _PRead(davix_client, fd, buffer, size, 0, true); } std::pair PRead(Davix::DavPosix& davix_client, DAVIX_FD* fd, void* buffer, uint32_t size, uint64_t offset) { return _PRead(davix_client, fd, buffer, size, offset, false); } std::pair PReadVec(Davix::DavPosix& davix_client, DAVIX_FD* fd, const XrdCl::ChunkList& chunks, void* buffer) { const auto num_chunks = chunks.size(); std::vector input_vector(num_chunks); std::vector output_vector(num_chunks); for (size_t i = 0; i < num_chunks; ++i) { input_vector[i].diov_offset = chunks[i].offset; input_vector[i].diov_size = chunks[i].length; input_vector[i].diov_buffer = chunks[i].buffer; } Davix::DavixError* err = nullptr; int num_bytes_read = davix_client.preadVec( fd, input_vector.data(), output_vector.data(), num_chunks, &err); if (num_bytes_read < 0) { auto errStatus = XRootDStatus(stError, errInternal, err->getStatus(), err->getErrMsg()); delete err; return std::make_pair(num_bytes_read, XRootDStatus(stError, errUnknown)); } return std::make_pair(num_bytes_read, XRootDStatus()); } std::pair PWrite(Davix::DavPosix& davix_client, DAVIX_FD* fd, uint64_t offset, uint32_t size, const void* buffer, uint16_t timeout) { Davix::DavixError* err = nullptr; off_t new_offset = davix_client.lseek(fd, offset, SEEK_SET, &err); if (uint64_t(new_offset) != offset) { auto errStatus = XRootDStatus(stError, errInternal, err->getStatus(), err->getErrMsg()); delete err; return std::make_pair(new_offset, errStatus); } int num_bytes_written = davix_client.write(fd, buffer, size, &err); if (num_bytes_written < 0) { auto errStatus = XRootDStatus(stError, errInternal, err->getStatus(), err->getErrMsg()); delete err; return std::make_pair(num_bytes_written, errStatus); } return std::make_pair(num_bytes_written, XRootDStatus()); } } // namespace Posix xrootd-5.6.9/src/XrdClHttp/XrdClHttpPosix.hh000066400000000000000000000050711457266313600207550ustar00rootroot00000000000000/** * This file is part of XrdClHttp */ #ifndef __HTTP_STAT_ #define __HTTP_STAT_ #include #include "XrdCl/XrdClFileSystem.hh" #include "XrdCl/XrdClXRootDResponses.hh" #include #include namespace XrdCl { class StatInfo; } namespace Posix { std::pair Open(Davix::DavPosix& davix_client, const std::string& url, int flags, uint16_t timeout); XrdCl::XRootDStatus Close(Davix::DavPosix& davix_client, DAVIX_FD* fd); XrdCl::XRootDStatus MkDir(Davix::DavPosix& davix_client, const std::string& path, XrdCl::MkDirFlags::Flags flags, XrdCl::Access::Mode mode, uint16_t timeout); XrdCl::XRootDStatus RmDir(Davix::DavPosix& davix_client, const std::string& path, uint16_t timeout); std::pair DirList( Davix::DavPosix& davix_client, const std::string& path, bool details, bool recursive, uint16_t timeout); XrdCl::XRootDStatus Rename(Davix::DavPosix& davix_client, const std::string& source, const std::string& dest, uint16_t timeout); XrdCl::XRootDStatus Stat(Davix::DavPosix& davix_client, const std::string& url, uint16_t timeout, XrdCl::StatInfo* stat_info); XrdCl::XRootDStatus Unlink(Davix::DavPosix& davix_client, const std::string& url, uint16_t timeout); std::pair Read(Davix::DavPosix& davix_client, DAVIX_FD* fd, void* buffer, uint32_t size); std::pair PRead(Davix::DavPosix& davix_client, DAVIX_FD* fd, void* buffer, uint32_t size, uint64_t offset); std::pair PReadVec(Davix::DavPosix& davix_client, DAVIX_FD* fd, const XrdCl::ChunkList& chunks, void* buffer); std::pair PWrite(Davix::DavPosix& davix_client, DAVIX_FD* fd, uint64_t offset, uint32_t size, const void* buffer, uint16_t timeout); } // namespace Posix #endif // __HTTP_STAT_ xrootd-5.6.9/src/XrdCms/000077500000000000000000000000001457266313600150565ustar00rootroot00000000000000xrootd-5.6.9/src/XrdCms/XrdCmsAdmin.cc000066400000000000000000000705521457266313600175470ustar00rootroot00000000000000/******************************************************************************/ /* */ /* X r d C m s A d m i n . c c */ /* */ /* (c) 2007 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include #include #include #include #include #include #include "XProtocol/XProtocol.hh" #include "XProtocol/YProtocol.hh" #include "XrdCms/XrdCmsAdmin.hh" #include "XrdCms/XrdCmsConfig.hh" #include "XrdCms/XrdCmsManager.hh" #include "XrdCms/XrdCmsMeter.hh" #include "XrdCms/XrdCmsPrepare.hh" #include "XrdCms/XrdCmsState.hh" #include "XrdCms/XrdCmsTrace.hh" #include "XrdNet/XrdNetSocket.hh" #include "XrdOuc/XrdOuca2x.hh" #include "XrdOuc/XrdOucName2Name.hh" #include "XrdOuc/XrdOucTList.hh" #include "XrdSys/XrdSysError.hh" #include "XrdSys/XrdSysPlatform.hh" #include "XrdSys/XrdSysTimer.hh" using namespace XrdCms; /******************************************************************************/ /* L o c a l C l a s s e s */ /******************************************************************************/ namespace XrdCms { class AdminReq { public: AdminReq *Next; const char *Req; const char *Path; CmsRRHdr Hdr; char *Data; int Dlen; static int numinQ; static const int maxinQ = 1024; static AdminReq *getReq() {AdminReq *arP; do {QPresent.Wait(); QMutex.Lock(); if ((arP = First)) {if (!(First = arP->Next)) Last = 0; numinQ--; } QMutex.UnLock(); } while (!arP); return arP; } void Requeue() {QMutex.Lock(); Next=First; First=this; QPresent.Post(); numinQ++; QMutex.UnLock(); } AdminReq(const char *req, XrdCmsRRData &RRD) : Next(0), Req(req), Path(RRD.Path ? RRD.Path : ""), Hdr(RRD.Request), Data(RRD.Buff), Dlen(RRD.Dlen) {RRD.Buff = 0; QMutex.Lock(); if (Last) {Last->Next = this; Last = this;} else First=Last = this; QPresent.Post(); numinQ++; QMutex.UnLock(); } ~AdminReq() {if (Data) free(Data);} private: static XrdSysSemaphore QPresent; static XrdSysMutex QMutex; static AdminReq *First; static AdminReq *Last; }; extern XrdCmsMeter Meter; }; /******************************************************************************/ /* G l o b a l s & S t a t i c s */ /******************************************************************************/ XrdSysSemaphore AdminReq::QPresent(0); XrdSysMutex AdminReq::QMutex; AdminReq *AdminReq::First = 0; AdminReq *AdminReq::Last = 0; int AdminReq::numinQ= 0; XrdOssStatInfo2_t XrdCmsAdmin::areFunc = 0; XrdOucTList *XrdCmsAdmin::areFirst = 0; XrdOucTList *XrdCmsAdmin::areLast = 0; XrdSysMutex XrdCmsAdmin::areMutex; XrdSysSemaphore XrdCmsAdmin::areSem(0); bool XrdCmsAdmin::arePost = false; XrdSysMutex XrdCmsAdmin::myMutex; XrdSysSemaphore *XrdCmsAdmin::SyncUp = 0; int XrdCmsAdmin::POnline= 0; /******************************************************************************/ /* E x t e r n a l T h r e a d I n t e r f a c e s */ /******************************************************************************/ namespace { void *AdminLogin(void *carg) {XrdCmsAdmin *Admin = new XrdCmsAdmin(); Admin->Login(*(int *)carg); delete Admin; return (void *)0; } void *AdminMonAds(void *carg) {XrdCmsAdmin *Admin = (XrdCmsAdmin *)carg; Admin->MonAds(); return (void *)0; } void *AdminMonARE(void *carg) {XrdCmsAdmin::RelayAREvent(); return (void *)0; } void *AdminSend(void *carg) {XrdCmsAdmin::Relay(0,0); return (void *)0; } } /******************************************************************************/ /* I n i t A R E v e n t s */ /******************************************************************************/ bool XrdCmsAdmin::InitAREvents(void *arFunc) { pthread_t tid; // Record the function we will be using // areFunc = (XrdOssStatInfo2_t)arFunc; // Start the event relay // if (XrdSysThread::Run(&tid, AdminMonARE, (void *)0)) {Say.Emsg("InitAREvents", errno, "start arevent relay"); return false; } // All done // return true; } /******************************************************************************/ /* L o g i n */ /******************************************************************************/ void XrdCmsAdmin::Login(int socknum) { const char *epname = "Admin_Login"; const char *sMsg[2] = {"temporary suspend requested by", "long-term suspend requested by"}; char *request, *tp; int sPerm; // Attach the socket FD to a stream // Stream.Attach(socknum); // The first request better be "login" // if ((request = Stream.GetLine())) {DEBUG("initial request: '" < 0); } while(rc < 0 && errno == EINTR); if (rc < 0) Say.Emsg(epname, errno, "maintain contact with", pname); else Say.Emsg(epname,"Lost contact with", pname); CmsState.Update(XrdCmsState::FrontEnd, 0, -1); close(sFD); XrdSysTimer::Snooze(15); } while(1); } /******************************************************************************/ /* N o t e s */ /******************************************************************************/ void *XrdCmsAdmin::Notes(XrdNetSocket *AnoteSock) { const char *epname = "Notes"; char *request, *tp; int rc; // Bind the udp socket to a stream // Stream.Attach(AnoteSock->Detach()); Sname = strdup("anon"); // Accept notifications in an endless loop // do {while((request = Stream.GetLine())) {DEBUG("received notification: '" <= 0) close(curSock); else if (newSock >= 0) SReady.Post(); if (newSock < 0) curSock = -1; else {curSock = dup(newSock); XrdNetSocket::setOpts(curSock, 0);} SMutex.UnLock(); return; } // This is just an endless loop // do {while(mySock < 0) {SMutex.Lock(); if (curSock < 0) {SMutex.UnLock(); SReady.Wait(); SMutex.Lock();} mySock = curSock; curSock = -1; SMutex.UnLock(); } do {arP = AdminReq::getReq(); if ((retc = write(mySock, &arP->Hdr, HdrSz)) != HdrSz || (retc = write(mySock, arP->Data, arP->Dlen)) != arP->Dlen) retc = (retc < 0 ? errno : ECANCELED); else {DEBUG("sent " <Req <<' ' <Path); delete arP; retc = 0; } } while(retc == 0); if (retc) Say.Emsg("AdminRelay", retc, "relay", arP->Req); arP->Requeue(); close(mySock); mySock = -1; } while(1); } /******************************************************************************/ /* R e l a y A R E v e n t */ /******************************************************************************/ void XrdCmsAdmin::RelayAREvent() { EPNAME("RelayAREvent"); const char *evWhat; XrdOucTList *evP; int evType, mod; // Endless loop relaying events // do{areMutex.Lock(); while((evP = areFirst)) {if (evP == areLast) areFirst = areLast = 0; else areFirst = evP->next; areMutex.UnLock(); XrdCms::CmsReqCode reqCode = static_cast(evP->ival[0]); mod = evP->ival[1]; if (reqCode == kYR_have) {if (mod & CmsHaveRequest::Pending) {evType = XrdOssStatEvent::PendAdded; evWhat = "pend "; } else { evType = XrdOssStatEvent::FileAdded; evWhat = "have "; } } else { evType = XrdOssStatEvent::FileRemoved; evWhat = "gone "; } (*areFunc)(evP->text, 0, evType, 0, evP->text); DEBUG("sending managers " <text); XrdCmsManager::Inform(reqCode, mod, evP->text, strlen(evP->text)+1); delete evP; areMutex.Lock(); } arePost = true; areMutex.UnLock(); areSem.Wait(); } while(true); } /******************************************************************************/ /* S e n d */ /******************************************************************************/ void XrdCmsAdmin::Send(const char *Req, XrdCmsRRData &Data) { // AdminReq *arP; if (AdminReq::numinQ < AdminReq::maxinQ) new AdminReq(Req, Data); else Say.Emsg("Send", "Queue full; ignoring", Req, Data.Path); } /******************************************************************************/ /* S t a r t */ /******************************************************************************/ void *XrdCmsAdmin::Start(XrdNetSocket *AdminSock) { const char *epname = "Start"; int InSock; pthread_t tid; // Start the relay thread // if (XrdSysThread::Run(&tid, AdminSend, (void *)0)) Say.Emsg(epname, errno, "start admin relay"); // If we are in independent mode then let the caller continue // if (Config.doWait) {if (Config.adsPort) BegAds(); {char dest[512]; AdminSock->SockName(dest, sizeof(dest)); Say.Emsg(epname, "Waiting for primary server to login via", dest); } } else if (SyncUp) {SyncUp->Post(); SyncUp = 0;} // Accept connections in an endless loop // while(1) if ((InSock = AdminSock->Accept()) >= 0) {XrdNetSocket::setOpts(InSock, 0); if (XrdSysThread::Run(&tid, AdminLogin, (void *)&InSock)) {Say.Emsg(epname, errno, "start admin"); close(InSock); } } else Say.Emsg(epname, errno, "accept connection"); return (void *)0; } /******************************************************************************/ /* P r i v a t e M e t h o d s */ /******************************************************************************/ /******************************************************************************/ /* A d d E v e n t */ /******************************************************************************/ void XrdCmsAdmin::AddEvent(const char *path, XrdCms::CmsReqCode req, int mods) { int info[2] = {(int)req, mods}; XrdOucTList *evP = new XrdOucTList(path, info); // Add the event to he queue // areMutex.Lock(); if (areLast) areLast->next = evP; else areFirst = evP; areLast = evP; if (arePost) {areSem.Post(); arePost = false;} areMutex.UnLock(); } /******************************************************************************/ /* B e g A d s */ /******************************************************************************/ void XrdCmsAdmin::BegAds() { const char *epname = "BegAds"; pthread_t tid; // If we don't need to monitor he alternate data server then we are all set // if (!Config.adsMon) {Say.Emsg(epname, "Assuming alternate data server is functional."); CmsState.Update(XrdCmsState::FrontEnd, 1, Config.adsPort); if (SyncUp) {SyncUp->Post(); SyncUp = 0;} return; } // Start the connection/ping thread for the alternate data server // if (XrdSysThread::Run(&tid, AdminMonAds, (void *)this)) Say.Emsg(epname, errno, "start alternate data server monitor"); } /******************************************************************************/ /* C h e c k V N i d */ /******************************************************************************/ bool XrdCmsAdmin::CheckVNid(const char *xNid) { // Check if we have a vnid but the server is supplying one or is not the same // if (Config.myVNID) {if (!xNid) {Say.Emsg("do_Login", "Warning! No xrootd vnid specified; " "proceeding only with cmsd vnid."); return true; } if (!strcmp(xNid, Config.myVNID)) return true; std::string msg("xrootd vnid '"); msg += xNid; msg += "' does not match cmsd vnid '"; msg += Config.myVNID; msg += "'."; Say.Emsg("do_Login", msg.c_str()); return false; } // We don't have a vnid, check if one is present // if (xNid) Say.Emsg("do_Login", "Warning! xrootd has a vnid but cmsd does " "not; proceeding without a vnid!"); return true; } /******************************************************************************/ /* C o n 2 A d s */ /******************************************************************************/ int XrdCmsAdmin::Con2Ads(const char *pname) { const char *epname = "Con2Ads"; static ClientInitHandShake hsVal = {0, 0, 0, (int)htonl(4), (int)htonl(2012)}; static ClientLoginRequest loginReq = {{0, 0}, (kXR_unt16)htons(kXR_login), (kXR_int32)htonl(getpid()), {'c', 'm', 's', 'd', 0, 0, 0, 0}, 0, 0, {0}, 0, 0}; struct {kXR_int32 siHS[4];} hsRsp; XrdNetSocket adsSocket; int ecode, snum; char ecnt = 10; // Create a socket and to connect to the alternate data server // do{while((snum = adsSocket.Open("localhost", Config.adsPort)) < 0) {if (ecnt >= 10) {ecode = adsSocket.LastError(); Say.Emsg(epname, ecode, "connect to", pname); ecnt = 1; } else ecnt++; XrdSysTimer::Snooze(3); } // Write the handshake to make sure the connection went fine // if (write(snum, &hsVal, sizeof(hsVal)) < 0) {Say.Emsg(epname, errno, "send handshake to", pname); close(snum); continue; } // Read the mandatory response // if (recv(snum, &hsRsp, sizeof(hsRsp), MSG_WAITALL) < 0) {Say.Emsg(epname, errno, "recv handshake from", pname); close(snum); continue; } // Now we need to send the login request // if (write(snum, &loginReq, sizeof(loginReq)) < 0) {Say.Emsg(epname, errno, "send login to", pname); close(snum); continue; } else break; } while(1); // Indicate what we just did // Say.Emsg(epname, "Logged into", pname); // We connected, so we indicate that the alternate is ok // myMutex.Lock(); CmsState.Update(XrdCmsState::FrontEnd, 1, Config.adsPort); if (SyncUp) {SyncUp->Post(); SyncUp = 0;} myMutex.UnLock(); // All done // return adsSocket.Detach(); } /******************************************************************************/ /* d o _ L o g i n */ /******************************************************************************/ int XrdCmsAdmin::do_Login() { std::string vnidVal; const char *emsg; char buff[64], *tp, Ltype = 0; int Port = 0; // Process: login {p | P | s | u} [port ] [nid ] // if (!(tp = Stream.GetToken())) {Say.Emsg("do_Login", "login type not specified"); return 0; } Ltype = *tp; if (*(tp+1) == '\0') switch (*tp) {case 'p': Stype = "Primary server"; break; case 'P': Stype = "Proxy server"; break; case 's': Stype = "Server"; break; case 'u': Stype = "Admin"; break; default: Ltype = 0; break; } if (!Ltype) {Say.Emsg("do_Login", "Invalid login type,", tp); return 0; } else Ltype = *tp; if (Config.adsPort && Ltype != 'u') {Say.Emsg("do_login", Stype, " login rejected; configured for an " "alternate data server."); return 0; } if (!(tp = Stream.GetToken())) {Say.Emsg("do_Login", "login name not specified"); return 0; } else Sname = strdup(tp); // Get any additional options // while((tp = Stream.GetToken())) { if (!strcmp(tp, "port")) {if (!(tp = Stream.GetToken())) {Say.Emsg("do_Login", "login port not specified"); return 0; } if (XrdOuca2x::a2i(Say,"login port",tp,&Port,0)) return 0; } else if (!strcmp(tp, "vnid")) {if (!(tp = Stream.GetToken())) {Say.Emsg("do_Login", "vnid value not specified"); return 0; } vnidVal = tp; } else {Say.Emsg("do_Login", "invalid login option -", tp); return 0; } } // If this is not a primary, we are done. Otherwise there is much more. We // must make sure we are compatible with the login. Note that for alternate // data servers we already screened out primary logins, so we will return. // if (Ltype != 'p' && Ltype != 'P') return 1; if (Ltype == 'p' && Config.asProxy()) emsg = "only accepts proxies"; else if (Ltype == 'P' && !Config.asProxy()) emsg = "does not accept proxies"; else emsg = 0; if (emsg) {Say.Emsg("do_login", "Server login rejected; configured role", emsg); return 0; } // Verify virtual networking // if ((vnidVal.length() || Config.myVNID) && !CheckVNid(vnidVal.length() ? vnidVal.c_str() : 0)) {Say.Emsg("do_login", "Server login rejected; virtual networking error."); return 0; } // Discard login if this is a duplicate primary server // myMutex.Lock(); if (POnline) {myMutex.UnLock(); Say.Emsg("do_Login", "Primary server already logged in; login of", tp, "rejected."); return 0; } // Indicate we have a primary // Primary = 1; POnline = 1; Relay(1, Stream.FDNum()); CmsState.Update(XrdCmsState::FrontEnd, 1, Port); // Check if this is the first primary login and resume if we must // if (SyncUp) {SyncUp->Post(); SyncUp = 0;} myMutex.UnLock(); // Document the login // sprintf(buff, "logged in; data port is %d", Port); Say.Emsg("do_Login:", Stype, Sname, buff); return 1; } /******************************************************************************/ /* d o _ P e r f */ /******************************************************************************/ void XrdCmsAdmin::do_Perf(bool alert) { const char *epname = "do_Perf"; char buff[256]; if (!Stream.GetRest(buff, sizeof(buff))) Say.Emsg(epname,"performance data is too long."); else if (!Meter.Update(buff, alert)) Say.Emsg(epname,"performance data is invalid."); } /******************************************************************************/ /* d o _ R m D i d */ /******************************************************************************/ void XrdCmsAdmin::do_RmDid(int isPfn) { const char *epname = "do_RmDid"; char *tp, *thePath, apath[XrdCmsMAX_PATH_LEN]; int rc; if (!(tp = Stream.GetToken())) {Say.Emsg(epname,"removed path not specified by",Stype,Sname); return; } // Handle prepare queue removal // if (PrepQ.isOK()) {if (!isPfn && Config.lcl_N2N) if ((rc = Config.lcl_N2N->lfn2pfn(tp, apath, sizeof(apath)))) {Say.Emsg(epname, rc, "determine pfn for removed path", tp); thePath = 0; } else thePath = apath; else thePath = tp; if (thePath) PrepQ.Gone(thePath); } // If we have a pfn then we must get the lfn to inform our manager about the file // if (isPfn && Config.lcl_N2N) {if ((rc = Config.lcl_N2N->pfn2lfn(tp, apath, sizeof(apath)))) {Say.Emsg(epname, rc, "determine lfn for removed path", tp); return; } else tp = apath; } // Check if we are relaying remove events and, if so, vector through that. // if (areFunc) AddEvent(tp, kYR_gone, kYR_raw); else {DEBUG("sending managers gone " <pfn2lfn(tp, apath, sizeof(apath)))) {Say.Emsg(epname, rc, "determine lfn for added path", tp); return; } else tp = apath; } // Check if we are relaying remove events and, if so, vector through that. // if (areFunc) AddEvent(tp, kYR_have, Mods); else {DEBUG("sending managers have online " <. */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include #include "XrdCms/XrdCmsProtocol.hh" #include "XrdCms/XrdCmsRRData.hh" #include "XrdOss/XrdOssStatInfo.hh" #include "XrdOuc/XrdOucStream.hh" #include "XrdSys/XrdSysPthread.hh" class XrdNetSocket; class XrdOucTList; class XrdCmsAdmin { public: static bool InitAREvents(void *arFunc); void Login(int socknum); void MonAds(); static void setSync(XrdSysSemaphore *sync) {SyncUp = sync;} void *Notes(XrdNetSocket *AdminSock); static void Relay(int setSock, int newSock); static void RelayAREvent(); void Send(const char *Req, XrdCmsRRData &Data); void *Start(XrdNetSocket *AdminSock); XrdCmsAdmin() {Sname = 0; Stype = "Server"; Primary = 0;} ~XrdCmsAdmin() {if (Sname) free(Sname);} private: static void AddEvent(const char *path, XrdCms::CmsReqCode req, int mods); void BegAds(); bool CheckVNid(const char *xNid); int Con2Ads(const char *pname); int do_Login(); void do_Perf(bool alert=false); void do_RmDid(int dotrim=0); void do_RmDud(int dotrim=0); static XrdOssStatInfo2_t areFunc; static XrdOucTList *areFirst; static XrdOucTList *areLast; static XrdSysMutex areMutex; static XrdSysSemaphore areSem; static bool arePost; static XrdSysMutex myMutex; static XrdSysSemaphore *SyncUp; static int POnline; XrdOucStream Stream; const char *Stype; char *Sname; int Primary; }; #endif xrootd-5.6.9/src/XrdCms/XrdCmsBaseFS.cc000066400000000000000000000365741457266313600176300ustar00rootroot00000000000000/******************************************************************************/ /* */ /* X r d C m s B a s e F S . c c */ /* */ /* (c) 2011 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include #include #include #include #include "XProtocol/YProtocol.hh" #include "XProtocol/XPtypes.hh" #include "XrdCms/XrdCmsBaseFS.hh" #include "XrdCms/XrdCmsConfig.hh" #include "XrdCms/XrdCmsPrepare.hh" #include "XrdCms/XrdCmsTrace.hh" #include "XrdOss/XrdOss.hh" #include "XrdSfs/XrdSfsFlags.hh" #include "XrdSys/XrdSysError.hh" #include "XrdSys/XrdSysTimer.hh" using namespace XrdCms; /******************************************************************************/ /* E x t e r n a l T h r e a d I n t e r f a c e s */ /******************************************************************************/ void *XrdCmsBasePacer(void *carg) {((XrdCmsBaseFS *)carg)->Pacer(); return (void *)0; } void *XrdCmsBaseRunner(void *carg) {((XrdCmsBaseFS *)carg)->Runner(); return (void *)0; } /******************************************************************************/ /* Private: B y p a s s */ /******************************************************************************/ int XrdCmsBaseFS::Bypass() { static XrdSysTimer Window; // If we are not timing requests, we can bypass (typically checked beforehand) // if (!theQ.rLimit) return 1; // If this is a fixed rate queue then we cannot bypass // if (Fixed) return 0; // Check if we can reset the number of requests that can be issued inline. We // do this to bypass the queue unless until we get flooded by requests. // theQ.Mutex.Lock(); if (!theQ.rLeft && !theQ.pqFirst) {unsigned long Interval = 0; Window.Report(Interval); if (Interval >= 450) {theQ.rLeft = theQ.rAgain; Window.Reset(); std::cerr <<"BYPASS " <= 0 && Path[fnPos] != '/'; fnPos--) {} if (fnPos > 0 && !hasDir(Path, fnPos)) return -1; } // Issue stat() via oss plugin. If it succeeds, return result. // if (!(fRC = Config.ossFS->Stat(Path, &buf, Opts))) {if ((buf.st_mode & S_IFMT) == S_IFREG) return (buf.st_mode & XRDSFS_POSCPEND ? CmsHaveRequest::Pending : CmsHaveRequest::Online); return (buf.st_mode & S_IFMT) == S_IFDIR ? CmsHaveRequest::Online : -1; } // The entry does not exist but if we are a staging server then it may be in // the prepare queue in which case we must say that it is pending arrival. // if (Config.DiskSS && PrepQ.Exists(Path)) return CmsHaveRequest::Pending; // The entry does not exist. Check if the directory exists and if not, put it // in our directory missing table so we don't keep hitting this directory. // This is disabled by default and enabled by the cms.dfs directive. // if (fnPos > 0 && dmLife) {struct dMoP *xVal = &dirMiss; int xLife = dmLife; Path[fnPos] = '\0'; if (!(dRC = Config.ossFS->Stat(Path, &buf, XRDOSS_resonly))) {xLife = dpLife; xVal = &dirPres;} if (dRC && dRC != -ENOENT) {fsMutex.Lock(); eCnt = badDStat++; fsMutex.UnLock(); if (!(eCnt & 0xff)) {char buff[80]; snprintf(buff, sizeof(buff), "to stat dir (events=%d)", eCnt+1); Say.Emsg("Exists", dRC, buff, Path); Path[fnPos] = '/'; } } else { fsMutex.Lock(); fsDirMP.Rep(Path, xVal, xLife, Hash_keepdata); fsMutex.UnLock(); DEBUG("add " <Present ? " okdir ":" nodir ") <Present : 1); fsMutex.UnLock(); Path[fnPos] = '/'; return Have; } /******************************************************************************/ /* I n i t */ /******************************************************************************/ void XrdCmsBaseFS::Init(int Opts, int DMLife, int DPLife) { // Set values. // dmLife = DMLife; dpLife = DPLife ? DPLife : DMLife * 10; Server = (Opts & Servr) != 0; lclStat = (Opts & Cntrl) != 0 || Server; preSel = (Opts & Immed) == 0; dfsSys = (Opts & DFSys) != 0; } /******************************************************************************/ /* L i m i t */ /******************************************************************************/ void XrdCmsBaseFS::Limit(int rLim, int Qmax) { // Establish the limits // if (rLim < 0) {theQ.rAgain=theQ.rLeft = -1; rLim = -rLim; Fixed = 1;} else {theQ.rAgain = theQ.rLeft = (rLim > 1 ? rLim/2 : 1); Fixed = 0;} theQ.rLimit = (rLim <= 1000 ? rLim : 0); if (Qmax > 0) theQ.qMax = Qmax; else if (!(theQ.qMax = theQ.rLimit*2 + theQ.rLimit/2)) theQ.qMax = 1; } /******************************************************************************/ /* P a c e r */ /******************************************************************************/ void XrdCmsBaseFS::Pacer() { XrdCmsBaseFR *rP; int inQ, rqRate = 1000/theQ.rLimit; // Process requests at the given rate // do{theQ.pqAvail.Wait(); theQ.Mutex.Lock(); inQ = 1; while((rP = theQ.pqFirst)) {if (!(theQ.pqFirst = rP->Next)) {theQ.pqLast = 0; inQ = 0;} theQ.Mutex.UnLock(); if (rP->PDirLen > 0 && !hasDir(rP->Path, rP->PDirLen)) {delete rP; continue;} theQ.Mutex.Lock(); if (theQ.rqFirst) {theQ.rqLast->Next = rP; theQ.rqLast = rP;} else {theQ.rqFirst = theQ.rqLast = rP; theQ.rqAvail.Post();} theQ.Mutex.UnLock(); XrdSysTimer::Wait(rqRate); if (!inQ) break; theQ.Mutex.Lock(); } if (inQ) theQ.Mutex.UnLock(); } while(1); } /******************************************************************************/ /* Q u e u e */ /******************************************************************************/ void XrdCmsBaseFS::Queue(XrdCmsRRData &Arg, XrdCmsPInfo &Who, int fnpos, int Force) { EPNAME("Queue"); static int noMsg = 1; XrdCmsBaseFR *rP; int Msg, n, prevHWM; // If we can bypass the queue and execute this now. Avoid the grabbing the buff. // if (!Force) {XrdCmsBaseFR myReq(&Arg, Who, fnpos); Xeq(&myReq); return; } // Queue this request for callback after an appropriate time. // We will also steal the underlying data buffer from the Arg. // DEBUG("inq " < prevHWM))) theQ.qHWM = n; if (theQ.pqFirst) {theQ.pqLast->Next = rP; theQ.pqLast = rP;} else {theQ.pqFirst = theQ.pqLast = rP; theQ.pqAvail.Post();} theQ.Mutex.UnLock(); // Issue a warning message if we have an excessive number of requests queued // if (n > theQ.qMax && Msg && (n-prevHWM > 3 || noMsg)) {int Pct = n/theQ.qMax; char Buff[80]; noMsg = 0; sprintf(Buff, "Queue overrun %d%%; %d requests now queued.", Pct, n); Say.Emsg("Pacer", Buff); } } /******************************************************************************/ /* R u n n e r */ /******************************************************************************/ void XrdCmsBaseFS::Runner() { XrdCmsBaseFR *rP; int inQ; // Process requests at the given rate // do{theQ.rqAvail.Wait(); theQ.Mutex.Lock(); inQ = 1; while((rP = theQ.rqFirst)) {if (!(theQ.rqFirst = rP->Next)) {theQ.rqLast = 0; inQ = 0;} theQ.qNum--; theQ.Mutex.UnLock(); Xeq(rP); delete rP; if (!inQ) break; theQ.Mutex.Lock(); } if (inQ) theQ.Mutex.UnLock(); } while(1); } /******************************************************************************/ /* S t a r t */ /******************************************************************************/ void XrdCmsBaseFS::Start() { EPNAME("Start"); void *Me = (void *)this; pthread_t tid; // Issue some debugging here so we know how we are starting up // DEBUG("Srv=" <PDirLen > 0 && !hasDir(rP->Path, rP->PDirLen)) {if (cBack) (*cBack)(rP, -1); return; } // If we have exceeded the queue limit and this is a meta-manager request // then just deep-six it. Local requests must complete // if (theQ.qNum > theQ.qMax) {Say.Emsg("Xeq", "Queue limit exceeded; ignoring lkup for", rP->Path); return; } // Perform a local stat() and if we don't have the file // rc = Exists(rP->Path, rP->PDirLen); if (cBack) (*cBack)(rP, rc); } xrootd-5.6.9/src/XrdCms/XrdCmsBaseFS.hh000066400000000000000000000214461457266313600176320ustar00rootroot00000000000000#ifndef __CMS_BASEFS_H__ #define __CMS_BASEFS_H__ /******************************************************************************/ /* */ /* X r d C m s B a s e F S . h h */ /* */ /* (c) 2011 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include #include "XrdCms/XrdCmsPList.hh" #include "XrdCms/XrdCmsRRData.hh" #include "XrdCms/XrdCmsTypes.hh" #include "XrdOuc/XrdOucHash.hh" #include "XrdSys/XrdSysPthread.hh" /******************************************************************************/ /* C l a s s X r d C m s B a s e F R */ /******************************************************************************/ class XrdCmsPInfo; class XrdCmsBaseFR { public: SMask_t Route; SMask_t RouteW; XrdCmsBaseFR *Next; char *Buff; char *Path; short PathLen; short PDirLen; kXR_unt32 Sid; kXR_char Mod; XrdCmsBaseFR(XrdCmsRRData &Arg, XrdCmsPInfo &Who, int Dln) : Route(Who.rovec), RouteW(Who.rwvec), Next(0), PathLen(Arg.PathLen), PDirLen(Dln), Sid(Arg.Request.streamid), Mod(Arg.Request.modifier) {if (Arg.Buff) {Path=Arg.Path; Buff=Arg.Buff; Arg.Buff=0;} else Buff = Path = strdup(Arg.Path); } XrdCmsBaseFR(XrdCmsRRData *aP, XrdCmsPInfo &Who, int Dln) : Route(Who.rovec), RouteW(Who.rwvec), Next(0), Buff(0), Path(aP->Path), PathLen(aP->PathLen), PDirLen(Dln), Sid(aP->Request.streamid), Mod(aP->Request.modifier) {} ~XrdCmsBaseFR() {if (Buff) free(Buff); Buff = 0;} }; /******************************************************************************/ /* C l a s s X r d C m s B a s e F S */ /******************************************************************************/ class XrdCmsBaseFS { public: int dfsTries() {return dfsMaxTries;} // Exists() returns a tri-logic state: // CmsHaveRequest::Online -> File is known to exist and is available // CmsHaveRequest::Pending -> File is known to exist but is not available // 0 -> File state unknown, result will be provided later // -1 -> File is known not to exist // int Exists(XrdCmsRRData &Arg,XrdCmsPInfo &Who,int noLim=0); // The following exists works as above but limits are never enforced and it // never returns 0. Additionally, the fnpos parameter works as follows: // // > 0 -> Offset into path to the last slash before the filename. // = 0 -> A valid offset has not been calculated nor should it be calculated. // < 0 -> A valid offset has not been calculated, fnpos = -(length of Path). int Exists(char *Path, int fnPos, int UpAT=0); // Valid Opts for Init() // static const int Cntrl = 0x0001; // Centralize stat() o/w distribute it static const int DFSys = 0x0002; // Distributed filesystem o/w jbods static const int Immed = 0x0004; // Redirect immediately o/w preselect static const int Servr = 0x0100; // This is a pure server node void Init(int Opts, int DMlife, int DPLife); inline int isDFS() {return dfsSys;} inline int Limit() {return theQ.rLimit;} void Limit(int rLim, int qMax); inline int Local() {return lclStat;} void Pacer(); void Runner(); static const int dfltDfsTries = 2; static const int dfltStgTries = 3; void SetTries(bool xdfs, int tcnt) {if (xdfs) dfsMaxTries = (tcnt < 0 ? dfltDfsTries : tcnt); else stgMaxTries = (tcnt < 1 ? dfltStgTries : tcnt); } void Start(); int stgTries() {return stgMaxTries;} inline int Trim() {return preSel;} inline int Traverse() {return Punt;} XrdCmsBaseFS(void (*theCB)(XrdCmsBaseFR *, int)) : cBack(theCB), dfsMaxTries(dfltDfsTries), stgMaxTries(dfltStgTries), dmLife(0), dpLife(0), lclStat(0), preSel(1), dfsSys(0), Server(0), Fixed(0), Punt(0) {} ~XrdCmsBaseFS() {} private: struct dMoP {int Present;}; int Bypass(); int FStat( char *Path, int fnPos, int upat=0); int hasDir(char *Path, int fnPos); void Queue(XrdCmsRRData &Arg, XrdCmsPInfo &Who, int dln, int Frc=0); void Xeq(XrdCmsBaseFR *rP); XrdSysMutex fsMutex; XrdOucHash fsDirMP; void (*cBack)(XrdCmsBaseFR *, int); struct RequestQ {XrdSysMutex Mutex; XrdSysSemaphore pqAvail; XrdSysSemaphore rqAvail; XrdCmsBaseFR *pqFirst; XrdCmsBaseFR *pqLast; XrdCmsBaseFR *rqFirst; XrdCmsBaseFR *rqLast; int rLimit; // Maximum number of requests per second int qHWM; // Queue high watermark int qMax; // Maximum elements to be queued int qNum; // Total number of queued elements (pq + rq) int rLeft; // Number of non-queue requests allowed int rAgain; // Value to reinitialize rLeft RequestQ() : pqAvail(0), rqAvail(0), pqFirst(0), pqLast(0), rqFirst(0), rqLast(0), rLimit(0), qHWM(0), qMax(1), qNum(0), rLeft(0), rAgain(0) {} ~RequestQ() {} } theQ; int dfsMaxTries; int stgMaxTries; int dmLife; int dpLife; char lclStat; // 1-> Local stat() calls wanted char preSel; // 1-> Preselect before redirect char dfsSys; // 1-> Distributed Filesystem char Server; // 1-> This is a data server char Fixed; // 1-> Use fixed rate processing char Punt; // 1-> Pass through any forwarding }; namespace XrdCms { extern XrdCmsBaseFS baseFS; } #endif xrootd-5.6.9/src/XrdCms/XrdCmsBlackList.cc000066400000000000000000000456611457266313600203720ustar00rootroot00000000000000/******************************************************************************/ /* */ /* X r d C m s B l a c k L i s t . c c */ /* */ /* (c) 2014 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include #include #include #include "Xrd/XrdScheduler.hh" #include "XrdCms/XrdCmsBlackList.hh" #include "XrdCms/XrdCmsCluster.hh" #include "XrdCms/XrdCmsTrace.hh" #include "XrdCms/XrdCmsUtils.hh" #include "XrdNet/XrdNetAddr.hh" #include "XrdOuc/XrdOucEnv.hh" #include "XrdOuc/XrdOucTList.hh" #include "XrdOuc/XrdOucStream.hh" #include "XrdOuc/XrdOucTokenizer.hh" #include "XrdSys/XrdSysError.hh" #include "XrdSys/XrdSysLogger.hh" #include "XrdSys/XrdSysPthread.hh" /******************************************************************************/ /* L o c a l C l a s s e s */ /******************************************************************************/ class BL_Grip {public: void Add(XrdOucTList *tP) {if (last) last->next = tP; else first = tP; last = tP; } XrdOucTList **Array(int &anum) {XrdOucTList *tP = first; anum = Count(); if (!anum) return 0; XrdOucTList **vec = new XrdOucTList *[anum]; for (int i = 0; i < anum; i++) {vec[i] = tP; tP = tP->next;} first = last = 0; return vec; } int Count() {XrdOucTList *tP = first; int n = 0; while(tP) {tP=tP->next; n++;} return n; } XrdOucTList *Export() {XrdOucTList *tP = first; first = last = 0; return tP; } bool Include(const char *item, int &i) {XrdOucTList *tP = first; i = 0; while(tP && strcmp(item,tP->text)) {tP=tP->next; i++;} if (!tP) {Add(new XrdOucTList(item)); return false;} return true; } BL_Grip() : first(0), last(0) {} ~BL_Grip() {XrdOucTList *tP; while((tP = first)) {first = tP->next; delete tP;} last = 0; } private: XrdOucTList *first; XrdOucTList *last; }; union BL_Info {long long info; struct {short flags; short pfxLen; short sfxLen; short totLen; } v; enum {exact = 0x8000, redir = 0x4000, rmask = 0x00ff }; }; /******************************************************************************/ /* G l o b a l O b j e c t s */ /******************************************************************************/ namespace XrdCms { XrdScheduler *blSched = 0; XrdCmsCluster *blCluster; class MidNightTask : public XrdSysLogger::Task {public: void Ring(); MidNightTask() {} ~MidNightTask() {} }; MidNightTask blMN; XrdCmsBlackList BlackList; XrdSysMutex blMutex; XrdOucTList *blReal = 0; XrdOucTList **blRedr = 0; char *blFN; time_t blTime = 0; int blChk; int blRcnt = 0; bool isWList=false; extern XrdSysError Say; }; using namespace XrdCms; /******************************************************************************/ /* Private: A d d B L */ /******************************************************************************/ bool XrdCmsBlackList::AddBL(BL_Grip &bAnchor, char *hSpec, BL_Grip *rAnchor, char *rSpec) { const char *bwTag = (isWList ? "whitelist '" : "blacklist '"); XrdNetAddr blAddr; XrdOucTList *bP; BL_Info Hdr; const char *eText; char *Ast, blBuff[512]; // Initialize the header // Hdr.info = 0; // Check if we are processing a redirect // if (rSpec) {int i = AddRD(rAnchor, rSpec, hSpec); if (i < 0) return false; Hdr.v.flags |= BL_Info::redir | i; } // Get the full name if this is not generic // if (!(Ast = index(hSpec, '*'))) {if ((eText = blAddr.Set(hSpec,0))) {snprintf(blBuff, sizeof(blBuff), "'; %s", eText); Say.Say("Config ", "Unable to ", bwTag, hSpec, blBuff); return false; } blAddr.Format(blBuff, sizeof(blBuff), XrdNetAddrInfo::fmtName, XrdNetAddrInfo::noPort); hSpec = blBuff; Hdr.v.flags |= BL_Info::exact; } else { Hdr.v.pfxLen = Ast - hSpec; Hdr.v.sfxLen = strlen(hSpec + Hdr.v.pfxLen + 1); Hdr.v.totLen = Hdr.v.pfxLen + Hdr.v.sfxLen; } // Add specification to the list // bP = new XrdOucTList(hSpec, &Hdr.info); bAnchor.Add(bP); return true; } /******************************************************************************/ /* A d d R D */ /******************************************************************************/ int XrdCmsBlackList::AddRD(BL_Grip *rAnchor, char *rSpec, char *hSpec) { XrdOucTList *rP, *rList = 0; char *rTarg; int ival; bool aOK = true; // First see if we have this entry already // if (rAnchor[0].Include(rSpec, ival)) return ival; // Make sure we did not exceed the maximum number of redirect entries // if (ival > BL_Info::rmask) {Say.Say("Config ", "Too many different redirects at ", hSpec, "redirect", rSpec); return -1; } // We now ned to tokenize the specification // XrdOucTokenizer rToks(rSpec); rToks.GetLine(); // Process each item // while((rTarg = rToks.GetToken()) && *rTarg) aOK &= AddRD(&rList,rTarg,hSpec); if (!aOK) return -1; // Flatten the list and add it to out list of redirect targets // rP = Flatten(rList, rList->val); rAnchor[1].Add(rP); // Delete the rlist // while((rP = rList)) {rList = rList->next; delete rP;} // All done // return ival; } /******************************************************************************/ bool XrdCmsBlackList::AddRD(XrdOucTList **rList, char *rSpec, char *hSpec) { char *rPort; // Screen out IPV6 specifications // if (*rSpec == '[') {if (!(rPort = index(rSpec, ']'))) {Say.Say("Config ","Invalid ",hSpec," redirect specification - ",rSpec); return -1; } } else rPort = rSpec; // Grab the port number // if ((rPort = index(rPort, ':'))) {if (!(*(rPort+1))) rPort = 0; else *rPort++ = '\0'; } // We should have a port specification now // if (!rPort) {Say.Say("Config ", "redirect port not specified for ", hSpec); return -1; } // Convert this to a list of redirect targets // return XrdCmsUtils::ParseMan(&Say, rList, rSpec, rPort, 0, true); } /******************************************************************************/ /* D o I t */ /******************************************************************************/ void XrdCmsBlackList::DoIt() { struct stat Stat; XrdOucTList **blOldRedr = 0, **blNewRedr = 0, *blNewReal = 0, *tP = 0, *nP; int rc, blOldRcnt = 0, blNewRcnt; bool doUpdt = false, doPrt = false; // Check if the black list file was modified // rc = stat(blFN, &Stat); if ((!rc && blTime != Stat.st_mtime) || (rc && blTime && errno == ENOENT)) {blTime = (rc ? 0 : Stat.st_mtime); if (GetBL(blNewReal, blNewRedr, blNewRcnt)) {blMutex.Lock(); tP = blReal; blReal = blNewReal; blOldRedr = blRedr; blRedr = blNewRedr; blOldRcnt = blRcnt; blRcnt = blNewRcnt; blMutex.UnLock(); if (!blReal && tP) doPrt = !isWList; else blMN.Ring(); doUpdt = true; } } // Delete any list we need to release // while(tP) {if (doPrt) Say.Say("Config ", tP->text, " removed from blacklist."); nP = tP->next; delete tP; tP = nP; } // Delete the old redirect array // if (blOldRedr) {for (int i = 0; i < blOldRcnt; i++) delete blOldRedr[i]; delete [] blOldRedr; } // Do real-time update if need be // if (doUpdt) blCluster->BlackList(blReal); // Reschedule this to check any modifications // blSched->Schedule((XrdJob *)&BlackList, time(0) + blChk); } /******************************************************************************/ /* Private: F l a t t e n */ /******************************************************************************/ XrdOucTList *XrdCmsBlackList::Flatten(XrdOucTList *tList, int tPort) { XrdOucTList *tP = tList; char buff[4096], bPort[8], *bP = buff; int n, pLen, bleft = sizeof(buff); short xdata[4] = {0}; // Convert port to a suffix // pLen = sprintf(bPort, ":%d", tPort); *buff = 0; // Fill the buffer and truncate as necessary // while(tP) {n = strlen(tP->text)+pLen+2; if (n >= bleft) break; n = sprintf(bP, " %s%s", tP->text, bPort); bP += n; bleft -= n; tP = tP->next; } // Get actual length including null byte // xdata[0] = strlen(buff+1) + 1; xdata[1] = xdata[0] + sizeof(short); xdata[2] = htons(xdata[0]); // Create a new tlist item // tP = new XrdOucTList(buff+1, xdata); return tP; } /******************************************************************************/ /* Private: G e t B L */ /******************************************************************************/ bool XrdCmsBlackList::GetBL(XrdOucTList *&bList, XrdOucTList **&rList, int &rcnt, bool isInit) { static int msgCnt = 0; XrdOucEnv myEnv; XrdOucStream blFile(&Say, getenv("XRDINSTANCE"), &myEnv, "=====> "); BL_Grip bAnchor, rAnchor[2]; const char *fType, *oEmsg, *rEmsg; char *hsp, *rsp, hspBuff[512], rSpec[4096]; int blFD, retc; bool aOK = true; // Setup message plugins // if (isWList) {oEmsg = "open whitelist file"; rEmsg = "read whitelist file"; fType = "whitelist"; } else { oEmsg = "open blacklist file"; rEmsg = "read blacklist file"; fType = "blacklist"; } // Open the config file // if ( (blFD = open(blFN, O_RDONLY, 0)) < 0) {if (errno == ENOENT) return true; if (!(msgCnt & 0x03)) Say.Emsg("GetBL", errno, oEmsg, blFN); return false; } blFile.Attach(blFD, 4096); // Trace this now // Say.Say("Config processing ", fType, " file ", blFN); // Start reading the black list // while((hsp = blFile.GetMyFirstWord())) {if (strlen(hsp) >= sizeof(hspBuff)) {Say.Say("Config ", hsp, " is too long."); aOK = false; continue;} strcpy(hspBuff, hsp); hsp = hspBuff; if ( (rsp = blFile.GetWord()) && *rsp) {if (strcmp("redirect", rsp)) {Say.Say("Config ", rsp, " is an invalid modifier for ", hsp); aOK = false; continue; } *rSpec = 0; rsp = rSpec; if (!blFile.GetRest(rSpec, sizeof(rSpec))) {Say.Say("Config ", "redirect target too long ", hsp); aOK = false; continue; } if (!(*rSpec)) {Say.Say("Config ", "redirect target missing for ", hsp); aOK = false; continue; } } else rsp = 0; blFile.noEcho(); if (!AddBL(bAnchor, hsp, rAnchor, rsp)) aOK = false; } // Now check if any errors occurred during file i/o // if ((retc = blFile.LastError())) {Say.Emsg("GetBL", retc, rEmsg, blFN); aOK = false;} else if (!aOK) Say.Emsg("GetBL", "Error(s) encountered in",fType,"file!"); // Return ending status // blFile.Close(); bList = ((aOK || isInit) ? bAnchor.Export() : nullptr); rList = rAnchor[1].Array(rcnt); return aOK; } /******************************************************************************/ /* I n i t */ /******************************************************************************/ void XrdCmsBlackList::Init(XrdScheduler *sP, XrdCmsCluster *cP, const char *blfn, int chkt) { struct stat Stat; const char *cfn; // Copy out the scheduler and cluster pointers // blSched = sP; blCluster = cP; // Determine if this is a black or white list // if (chkt < 0) {isWList = true; chkt = -chkt;} // Copy the file path (this is a one time call during config) // if (blfn) blFN = strdup(blfn); else if (!(cfn = getenv("XRDCONFIGFN"))) return; else {char pBuff[2048], *Slash; strcpy(pBuff, cfn); if (!(Slash = rindex(pBuff, '/'))) return; strcpy(Slash+1, (isWList ? "cms.whitelist" : "cms.blacklist")); blFN = strdup(pBuff); } // Check if the black list file exists, it might not. If it does, process it // if (!stat(blFN, &Stat)) {blTime = Stat.st_mtime; GetBL(blReal, blRedr, blRcnt, /* isInit = */ true); if (blReal) blMN.Ring(); } // Schedule this to recheck any modifications // blChk = chkt; blSched->Schedule((XrdJob *)&BlackList, time(0) + chkt); // Add ourselves to the midnight run list // Say.logger()->AtMidnight(&blMN); } /******************************************************************************/ /* P r e s e n t */ /******************************************************************************/ int XrdCmsBlackList::Present(const char *hName, XrdOucTList *bList, char *rBuff, int rBLen) { BL_Info Hdr; int hLen, retval; bool doUnLk; // Check if we really have a name here // if (!hName || !blSched) return 0; // Check if we need to supply our list // if (bList) doUnLk = false; else {doUnLk = true; blMutex.Lock(); bList = blReal; } // By definition, if there is no list at all then everybody is allowed // if (!bList) {if (doUnLk) blMutex.UnLock(); return 0; } // Run through the list and try to compare // hLen = strlen(hName); while(bList) {Hdr.info = bList->dval; if (Hdr.v.flags & BL_Info::exact) {if (!strcmp(hName, bList->text)) break;} else if (hLen >= Hdr.v.totLen) {if (!Hdr.v.pfxLen || !strncmp(bList->text, hName, Hdr.v.pfxLen)) {if (!Hdr.v.sfxLen || !strncmp(bList->text+Hdr.v.pfxLen+1, hName + (hLen - Hdr.v.sfxLen), Hdr.v.sfxLen)) break; } } bList = bList->next; } // If we have a black list check if we should redirect // if (bList) {if (!(Hdr.v.flags & BL_Info::redir)) retval = (isWList ? 0 : -1); else {XrdOucTList *rP = blRedr[Hdr.v.flags & BL_Info::rmask]; if (rP) {retval = rP->sval[1]; if (!rBuff || retval > rBLen) retval = -retval; else {memcpy(rBuff, &(rP->sval[2]), sizeof(short)); memcpy(rBuff+sizeof(short), rP->text, rP->sval[0]); } } else retval = -1; } } else retval = (isWList ? -1 : 0); // Unlock ourselves if need be and return result // if (doUnLk) blMutex.UnLock(); return retval; } /******************************************************************************/ /* R i n g */ /******************************************************************************/ void MidNightTask::Ring() { BL_Info Hdr; XrdOucTList *tP; const char *bwTag = (isWList ? "Whitelisting " : "Blacklisting "); // Get the list lock // blMutex.Lock(); tP = blReal; // Print the list // while(tP) {Hdr.info = tP->dval; if (!(Hdr.v.flags & BL_Info::redir)) Say.Say("Config ", bwTag, tP->text); else {XrdOucTList *rP = blRedr[Hdr.v.flags & BL_Info::rmask]; Say.Say("Config Blacklisting ",tP->text," redirect ",rP->text); } tP = tP->next; } // All done // blMutex.UnLock(); } xrootd-5.6.9/src/XrdCms/XrdCmsBlackList.hh000066400000000000000000000124311457266313600203710ustar00rootroot00000000000000#ifndef __XRDCMSBLACKLIST_HH__ #define __XRDCMSBLACKLIST_HH__ /******************************************************************************/ /* */ /* X r d C m s B l a c k L i s t . h h */ /* */ /* (c) 2014 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include "Xrd/XrdJob.hh" class XrdCmsCluster; class XrdOucTList; class XrdScheduler; class BL_Grip; class XrdCmsBlackList : public XrdJob { public: //------------------------------------------------------------------------------ //! Time driven method for checking black list file. //------------------------------------------------------------------------------ void DoIt(); //------------------------------------------------------------------------------ //! Initialize the black list //! //! @param sP Pointer to the scheduler object. //! @param cP Pointer to the cluster object. //! @param blfn The path to the black list file or null. //! @param chkt Seconds between checks for blacklist changes. If the value //! is negative, the blacklist is treated as a whitelist. //------------------------------------------------------------------------------ static void Init(XrdScheduler *sP, XrdCmsCluster *cP, const char *blfn, int chkt=600); //------------------------------------------------------------------------------ //! Check if host is in the black list and how it should be managed. //! //! @param hName Pointer to the host name or address. //! @param bList Optional pointer to a private black list. //! @param rbuff Pointer to the buffer to contain the redirect response. If //! nil, the host is not redirected. //! @param rblen The size of rbuff. If zero or insufficiently large the host //! is not redirected. //! //! @return < -1 Host is in the black list and would be redirected; //! but either rbuff was nil or the buffer was too small. The //! abs(returned value) is the size the buffer should have been. //! @return = -1 Host is in the black list and should not be redirected. //! @return = 0 Host not in the black list. //! @return > 0 Host is in the black list and should be redirected. //! The return value is the size of the redirect response placed //! in the supplied buffer. //------------------------------------------------------------------------------ static int Present(const char *hName, XrdOucTList *bList=0, char *rbuff=0, int rblen=0); //------------------------------------------------------------------------------ //! Constructor and Destructor //------------------------------------------------------------------------------ XrdCmsBlackList() : XrdJob("Black List Check") {} ~XrdCmsBlackList() {} private: static bool AddBL(BL_Grip &bAnchor, char *hSpec, BL_Grip *rAnchor, char *rSpec); static int AddRD(BL_Grip *rAnchor, char *rSpec, char *hSpec); static bool AddRD(XrdOucTList **rList, char *rSpec, char *hSpec); static XrdOucTList *Flatten(XrdOucTList *tList, int tPort); static bool GetBL(XrdOucTList *&bList, XrdOucTList **&rList, int &rcnt, bool isInit = false); }; #endif xrootd-5.6.9/src/XrdCms/XrdCmsCache.cc000066400000000000000000000512311457266313600175130ustar00rootroot00000000000000/******************************************************************************/ /* */ /* X r d C m s C a c h e . c c */ /* */ /* (c) 2007 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include #include "XrdCms/XrdCmsCache.hh" #include "XrdCms/XrdCmsRRQ.hh" #include "XrdCms/XrdCmsTrace.hh" #include "XrdSys/XrdSysTimer.hh" #include "Xrd/XrdJob.hh" #include "Xrd/XrdScheduler.hh" namespace XrdCms { extern XrdScheduler *Sched; } using namespace XrdCms; /******************************************************************************/ /* G l o b a l s */ /******************************************************************************/ XrdCmsCache XrdCms::Cache; /******************************************************************************/ /* L o c a l C l a s s e s */ /******************************************************************************/ class XrdCmsCacheJob : XrdJob { public: void DoIt() {Cache.Recycle(myList); delete this;} XrdCmsCacheJob(XrdCmsKeyItem *List) : XrdJob("cache scrubber"), myList(List) {} ~XrdCmsCacheJob() {} private: XrdCmsKeyItem *myList; }; /******************************************************************************/ /* E x t e r n a l T h r e a d I n t e r f a c e s */ /******************************************************************************/ void *XrdCmsStartTickTock(void *carg) {XrdCmsCache *myCache = (XrdCmsCache *)carg; return myCache->TickTock(); } /******************************************************************************/ /* P u b l i c C a c h e M a n i p u l a t i o n M e t h o d s */ /******************************************************************************/ /******************************************************************************/ /* Public A d d F i l e */ /******************************************************************************/ // This method insert or updates information about a path in the cache. // Key was found: Location information is updated depending on mask // mask == 0 Indicates that the information is being refreshed. // Location information is nullified. The update deadline is set // QDelay seconds in the future. The entry window is set to the // current window to be held for a full fxhold period. // mask != 0 Indicates that some location information is now known. // Location information is updated according to the mask. // For a r/w location, the deadline is satisfied and all // callbacks are dispatched. For an r/o location the deadline // is satisfied if no r/w callback is pending. Any r/o // callback is dispatched. The Info object is ignored. // Key not found: A selective addition occurs, depending on Sel.Opts // Opts !Advisory: The entry is added to the cache with location information // set as passed (usually 0). The update deadline is us set to // DLTtime seconds in the future. The entry window is set // to the current window. // Opts Advisory: The call is ignored since we do not keep information about // paths that were never asked for. // Returns True If this is the first time location information was added // to the entry. // Returns False Otherwise. int XrdCmsCache::AddFile(XrdCmsSelect &Sel, SMask_t mask) { XrdCmsKeyItem *iP; SMask_t xmask; int isrw = (Sel.Opts & XrdCmsSelect::Write), isnew = 0; // Serialize processing // myMutex.Lock(); // Check for fast path processing // if ( !(iP = Sel.Path.TODRef) || !(iP->Key.Equiv(Sel.Path))) if ((iP = Sel.Path.TODRef = CTable.Find(Sel.Path))) Sel.Path.Ref = iP->Key.Ref; // Add/Modify the entry // if (iP) {if (!mask) {iP->Loc.deadline = QDelay + time(0); iP->Loc.lifeline = nilTMO + iP->Loc.deadline; iP->Loc.hfvec = 0; iP->Loc.pfvec = 0; iP->Loc.qfvec = 0; iP->Loc.TOD_B = BClock; iP->Key.TOD = Tock; } else { xmask = iP->Loc.pfvec; if (Sel.Opts & XrdCmsSelect::Pending) iP->Loc.pfvec |= mask; else iP->Loc.pfvec &= ~mask; isnew = (iP->Loc.hfvec == 0) || (iP->Loc.pfvec != xmask); iP->Loc.hfvec |= mask; iP->Loc.qfvec &= ~mask; if (isrw) {iP->Loc.deadline = 0; if (iP->Loc.roPend || iP->Loc.rwPend) Dispatch(Sel, iP, iP->Loc.roPend, iP->Loc.rwPend); } else {if (!iP->Loc.rwPend) iP->Loc.deadline = 0; if (iP->Loc.roPend) Dispatch(Sel, iP, iP->Loc.roPend, 0); } } } else if (!(Sel.Opts & XrdCmsSelect::Advisory)) {Sel.Path.TOD = Tock; if ((iP = CTable.Add(Sel.Path))) {iP->Loc.pfvec = (Sel.Opts&XrdCmsSelect::Pending?mask:0); iP->Loc.hfvec = mask; iP->Loc.TOD_B = BClock; iP->Loc.qfvec = 0; iP->Loc.deadline = QDelay + time(0); iP->Loc.lifeline = nilTMO + iP->Loc.deadline; Sel.Path.Ref = iP->Key.Ref; Sel.Path.TODRef = iP; isnew = 1; } } // All done // myMutex.UnLock(); return isnew; } /******************************************************************************/ /* Public D e l F i l e */ /******************************************************************************/ // This method removes location information from existing valid entries. If an // existing valid entry is found, based on Sel.Opts the following occurs: // Opts Advisory only locate information is removed. // Opts !Advisory if the entry has no location information it is removed from // the cache, if possible. // TRUE is returned if the entry was valid but location information was cleared. // Otherwise, FALSE is returned. int XrdCmsCache::DelFile(XrdCmsSelect &Sel, SMask_t mask) { XrdCmsKeyItem *iP; int gone4good; // Lock the hash table // myMutex.Lock(); // Look up the entry and remove server // if ((iP = CTable.Find(Sel.Path))) {iP->Loc.hfvec &= ~mask; iP->Loc.pfvec &= ~mask; if ((gone4good = (iP->Loc.hfvec == 0))) {if (nilTMO) iP->Loc.lifeline = nilTMO + time(0); if (!(Sel.Opts & XrdCmsSelect::Advisory) && XrdCmsKeyItem::Unload(iP) && !CTable.Recycle(iP)) Say.Emsg("DelFile", "Delete failed for", iP->Key.Val); } } else gone4good = 0; // All done // myMutex.UnLock(); return gone4good; } /******************************************************************************/ /* Public G e t F i l e */ /******************************************************************************/ // This method looks up entries in the cache. An "entry not found" condition // holds is the entry was found but is marked as deleted. // Entry was found: Location information is passed bask. If the update deadline // has passed, it is nullified and 1 is returned. Otherwise, // -1 is returned indicating a query is in progress. // Entry not found: FALSE is returned. int XrdCmsCache::GetFile(XrdCmsSelect &Sel, SMask_t mask) { XrdCmsKeyItem *iP; SMask_t bVec; int retc; // Lock the hash table // myMutex.Lock(); // Look up the entry and return location information // if ((iP = CTable.Find(Sel.Path))) {if ((bVec = (iP->Loc.TOD_B < BClock ? getBVec(iP->Key.TOD, iP->Loc.TOD_B) & mask : 0))) {iP->Loc.hfvec &= ~bVec; iP->Loc.pfvec &= ~bVec; iP->Loc.qfvec &= ~mask; iP->Loc.deadline = QDelay + time(0); iP->Loc.lifeline = nilTMO + iP->Loc.deadline; retc = -1; } else if (iP->Loc.deadline) if (iP->Loc.deadline > time(0)) retc = -1; else {iP->Loc.deadline = 0; retc = 1;} else retc = 1; if (nilTMO && retc == 1 && iP->Loc.hfvec == 0 && iP->Loc.lifeline <= time(0)) retc = 0; Sel.Vec.hf = okVec & iP->Loc.hfvec; Sel.Vec.pf = okVec & iP->Loc.pfvec; Sel.Vec.bf = okVec & (bVec | iP->Loc.qfvec); iP->Loc.qfvec = 0; Sel.Path.Ref = iP->Key.Ref; } else retc = 0; // All done // myMutex.UnLock(); Sel.Path.TODRef = iP; return retc; } /******************************************************************************/ /* Public U n k F i l e */ /******************************************************************************/ int XrdCmsCache::UnkFile(XrdCmsSelect &Sel, SMask_t mask) { EPNAME("UnkFile"); XrdCmsKeyItem *iP; // Make sure we have the proper information. If so, lock the hash table // myMutex.Lock(); // Look up the entry and if valid update the unqueried vector. Note that // this method may only be called after GetFile() or AddFile() for a new entry // if ((iP = Sel.Path.TODRef)) {if (iP->Key.Equiv(Sel.Path)) iP->Loc.qfvec = mask; else iP = 0; } // Return result // myMutex.UnLock(); DEBUG("rc=" <<(iP ? 1 : 0) <<" path=" <Key.Equiv(Sel.Path))) retc = DLTime; else if (iP->Loc.hfvec != mask) retc = 1; else {Now = time(0); retc = 0; if (iP->Loc.deadline && iP->Loc.deadline <= Now) iP->Loc.deadline = DLTime + Now; Add2Q(Sel.InfoP, iP, Sel.Opts); } // Return result // myMutex.UnLock(); DEBUG("rc=" <Recycle(); // All done // return 1; } /******************************************************************************/ /* public T i c k T o c k */ /******************************************************************************/ void *XrdCmsCache::TickTock() { XrdCmsKeyItem *iP; // Simply adjust the clock and trim old entries // do {XrdSysTimer::Snooze(Tick); myMutex.Lock(); Tock = (Tock+1) & XrdCmsKeyItem::TickMask; Bhistory[Tock].Start = Bhistory[Tock].End = 0; iP = XrdCmsKeyItem::Unload(Tock); myMutex.UnLock(); if (iP) Sched->Schedule((XrdJob *)new XrdCmsCacheJob(iP)); } while(1); // Keep compiler happy // return (void *)0; } /******************************************************************************/ /* P r i v a t e M e t h o d s */ /******************************************************************************/ /******************************************************************************/ /* A d d 2 Q */ /******************************************************************************/ void XrdCmsCache::Add2Q(XrdCmsRRQInfo *Info, XrdCmsKeyItem *iP, int selOpts) { bool isrw = (selOpts & XrdCmsSelect::Write) != 0; short Slot = (isrw ? iP->Loc.rwPend : iP->Loc.roPend); // Add the request to the appropriate pending queue // Info->Key = iP; Info->isRW= isrw; Info->ifOP= (selOpts & XrdCmsSelect::ifWant); if (!(Slot = RRQ.Add(Slot, Info))) Info->Key = 0; else if (isrw) iP->Loc.rwPend = Slot; else iP->Loc.roPend = Slot; } /******************************************************************************/ /* D i s p a t c h */ /******************************************************************************/ void XrdCmsCache::Dispatch(XrdCmsSelect &Sel, XrdCmsKeyItem *iP, short roQ, short rwQ) { // Dispatching shared-everything nodes is very different from shared-nothing // since one ready node means all are ready and we can use any one of them. // The current list of nodes must is provided by the caller adding the entry. // Note that the minimum number of nodes will always be set to 0 via config. // if (isDFS) {if (roQ && RRQ.Ready(roQ, iP, Sel.Vec.hf, Sel.Vec.pf & Sel.Vec.hf)) iP->Loc.roPend = 0; if((rwQ && Sel.Vec.wf) && RRQ.Ready(rwQ, iP, Sel.Vec.wf, Sel.Vec.pf & Sel.Vec.wf)) iP->Loc.rwPend = 0; return; } // Disptaching shared-nothing nodes is a one-shot affair. Only one node becomes // ready at a time and we can immediately disptach that node unless we need to // wait for more nodes to respond. // if (roQ && RRQ.Ready(roQ, iP, iP->Loc.hfvec, iP->Loc.pfvec)) iP->Loc.roPend = 0; if (rwQ && RRQ.Ready(rwQ, iP, iP->Loc.hfvec, iP->Loc.pfvec)) iP->Loc.rwPend = 0; } /******************************************************************************/ /* g e t B V e c */ /******************************************************************************/ SMask_t XrdCmsCache::getBVec(unsigned int TODa, unsigned int &TODb) { EPNAME("getBVec"); SMask_t BVec(0); long long i; // See if we can use a previously calculated bVec // if (Bhistory[TODa].End == BClock && Bhistory[TODa].Start <= TODb) {Bhits++; TODb = BClock; return Bhistory[TODa].Vec;} // Calculate the new vector // for (i = 0; i <= vecHi; i++) if (TODb < Bounced[i]) BVec |= 1ULL << i; Bhistory[TODa].Vec = BVec; Bhistory[TODa].Start = TODb; Bhistory[TODa].End = BClock; TODb = BClock; Bmiss++; if (!(Bmiss & 0xff)) DEBUG("hits=" <. */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include "Xrd/XrdJob.hh" #include "Xrd/XrdScheduler.hh" #include "XrdCms/XrdCmsKey.hh" #include "XrdCms/XrdCmsNash.hh" #include "XrdCms/XrdCmsPList.hh" #include "XrdSys/XrdSysPthread.hh" #include "XrdCms/XrdCmsSelect.hh" #include "XrdCms/XrdCmsTypes.hh" class XrdCmsCache { public: friend class XrdCmsCacheJob; XrdCmsPList_Anchor Paths; // AddFile() returns true if this is the first addition, false otherwise. See // method for detailed information on processing. // int AddFile(XrdCmsSelect &Sel, SMask_t mask); // DelFile() returns true if this is the last deletion, false otherwise // int DelFile(XrdCmsSelect &Sel, SMask_t mask); // GetFile() returns true if we actually found the file // int GetFile(XrdCmsSelect &Sel, SMask_t mask); // UnkFile() updates the unqueried vector and returns 1 upon success, 0 o/w. // int UnkFile(XrdCmsSelect &Sel, SMask_t mask); // WT4File() adds a request to the callback queue and returns a 0 if added // of a wait time to be returned to the client. // int WT4File(XrdCmsSelect &Sel, SMask_t mask); void Bounce(SMask_t smask, int SNum); void Drop(SMask_t mask, int SNum, int xHi); int Init(int fxHold, int fxDelay, int fxQuery, int seFS, int nxHold); void *TickTock(); static const int min_nxTime = 60; XrdCmsCache() : okVec(0), Tick(8*60*60), Tock(0), BClock(0), nilTMO(0), DLTime(5), QDelay(5), Bhits(0), Bmiss(0), vecHi(-1), isDFS(0) {memset(Bounced, 0, sizeof(Bounced)); memset(Bhistory, 0, sizeof(Bhistory)); } ~XrdCmsCache() {} // Never gets deleted private: void Add2Q(XrdCmsRRQInfo *Info, XrdCmsKeyItem *cp, int selOpts); void Dispatch(XrdCmsSelect &Sel, XrdCmsKeyItem *cinfo, short roQ, short rwQ); SMask_t getBVec(unsigned int todA, unsigned int &todB); void Recycle(XrdCmsKeyItem *theList); struct {SMask_t Vec; unsigned int Start; unsigned int End; } Bhistory[XrdCmsKeyItem::TickRate]; XrdSysMutex myMutex; XrdCmsNash CTable; unsigned int Bounced[STMax]; SMask_t okVec; unsigned int Tick; unsigned int Tock; unsigned int BClock; int nilTMO; int DLTime; int QDelay; int Bhits; int Bmiss; int vecHi; int isDFS; }; namespace XrdCms { extern XrdCmsCache Cache; } #endif xrootd-5.6.9/src/XrdCms/XrdCmsClient.cc000066400000000000000000000062471457266313600177350ustar00rootroot00000000000000/******************************************************************************/ /* */ /* X r d C m s C l i e n t . c c */ /* */ /* (c) 2012 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include "XrdCms/XrdCmsClient.hh" #include "XrdCms/XrdCmsFinder.hh" /******************************************************************************/ /* G e t D e f a u l t C l i e n t */ /******************************************************************************/ namespace XrdCms { XrdCmsClient *GetDefaultClient(XrdSysLogger *Logger, int opMode, int myPort ) { // Determine which client to generate for the caller. This function is // provided as an ABI-compatible interface to obtaining a default client. // if (opMode & IsRedir) return (XrdCmsClient *)new XrdCmsFinderRMT(Logger, opMode, myPort); if (opMode & IsTarget) return (XrdCmsClient *)new XrdCmsFinderTRG(Logger, opMode, myPort); return 0; } }; xrootd-5.6.9/src/XrdCms/XrdCmsClient.hh000066400000000000000000000536511457266313600177500ustar00rootroot00000000000000#ifndef __CMS_CLIENT__ #define __CMS_CLIENT__ /******************************************************************************/ /* */ /* X r d C m s C l i e n t . h h */ /* */ /* (c) 2007 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ class XrdOucEnv; class XrdOucErrInfo; class XrdOucLogger; class XrdOucTList; struct XrdSfsPrep; class XrdSysLogger; /******************************************************************************/ /* R e t u r n C o n v e n t i o n s */ /******************************************************************************/ /* The following return conventions are use by Forward(), Locate(), & Prepare() Return Val Resp.errcode Resp.errtext --------- ------------------- -------- SFS_DATA Length of data. Data to be returned to caller. Action: Caller is provided data as successful response. SFS_ERROR errno Error message text. Action: Caller given error response. SFS_REDIRECT port (0 for default) Host name Action: Caller is redirected to : SFS_STARTED Expected seconds n/a Action: Caller is told to wait for the "expected seconds" for a callback with the result. A callback must follow. See how to do callbacks below. > 0 Wait time (= retval) Reason for wait Action: Caller told to wait retval seconds and retry request. < 0 Error number Error message Action: Same as SFS_ERROR. You should *always* use SFS_ERROR. = 0 Not applicable Not applicable (see below) Action: Forward() -> Return success; request forwarded. Locate() -> Redirection does not apply, operation should be done against local file system. Prepare() -> Return success, request submitted. */ /******************************************************************************/ /* C a l l b a c k C o n v e n t i o n s */ /******************************************************************************/ /* Most operations allow you to return SFS_STARTED to setup a callback. Callback information is contained in the XrdOucErrInfo object passed to Forward(), Locate() and Prepare(); the only methods that can apply callbacks. Use a callback when the operation will take at least several seconds so as to not occupy the calling thread for an excessive amount of time. The actual mechanics of a callback are rather complicated because callbacks are subject to non-causaility if not correctly handled. In order to avoid such issues, you should use the XrdOucCallBack object (see XrdOucCallBack.hh) to test for applicability, setup, and effect a callback. When calling back, you return the same information you would have returned had the execution path been synchronous. From that standpoint callbacks are relatively easy to understand. All you are doing is defering the return of information without occupying a thread while waiting to do so. A typical scenario, using Resp and the original ErrInfo object, would be.... XrdOucCallBack cbObject; // Must be persistent for the callback duration if (XrdOucCallBack::Allowed(Resp)) {cbObject.Init(Resp); Resp.setErrCode(); return SFS_STARTED; // Effect callback response! } Once the thread doing the work has a result, send it via a callback as if the work was done in a synchronous fashion. cbObject->Reply(retValue, ErrCodeValue, ErrTextValue); */ /******************************************************************************/ /* C l a s s X r d C m s C l i e n t */ /******************************************************************************/ class XrdCmsClient { public: //------------------------------------------------------------------------------ //! Notify the cms of a newly added file or a file whose state has changed on //! a data server node. //! //! @param path The logical file name. //! @param Pend When true, the file is scheduled to be present in the future //! (e.g. copied in). //------------------------------------------------------------------------------ virtual void Added(const char *path, int Pend=0) { (void)path; (void)Pend; } //------------------------------------------------------------------------------ //! Configure the client object. //! //! @param cfn The configuration file name. //! @param Parms Any parameters specified in the cmslib directive. If none, //! the pointer may be null. //! @param EnvInfo Environmental information of the caller. //! //! @return Success !0 //! Failure =0 //------------------------------------------------------------------------------ virtual int Configure(const char *cfn, char *Parms, XrdOucEnv *EnvInfo) = 0; //------------------------------------------------------------------------------ //! Relay a meta-operation to all nodes in the cluster. //! //! This method is only used on manager nodes and is enabled by the ofs.forward //! directive. //! //! @param Resp Object where messages are to be returned. //! @param cmd The operation being performed (see table below). //! If it starts with a '+' then a response (2way) is needed. //! Otherwise, a best-effort is all that is all that is required //! and success can always be returned. //! @param arg1 1st argument to cmd. //! @param arg2 2nd argument to cmd, which may be null if none exists. //! @param Env1 Associated environmental information for arg1 (e.g., cgi info //! which can be retrieved by Env1->Env()). //! @param Env2 Associated environmental information for arg2 (e.g., cgi info //! which can be retrieved by Env1->Env()). //! //! //! cmd arg1 arg2 cmd arg1 arg2 //! -------- ------ ------ -------- ------ ------ //! [+]chmod [+]rmdir 0 //! [+]mkdir [+]mv //! [+]mkpath [+]trunc //! [+]rm 0 //! //! @return As explained under "return conventions". //------------------------------------------------------------------------------ virtual int Forward(XrdOucErrInfo &Resp, const char *cmd, const char *arg1=0, const char *arg2=0, XrdOucEnv *Env1=0, XrdOucEnv *Env2=0) { (void)Resp; (void)cmd; (void)arg1; (void)arg2; (void)Env1; (void)Env2; return 0; } //------------------------------------------------------------------------------ //! Check if this client is configured for a manager node. //! //! @return !0 Yes, configured as a manager. //! =0 No. //------------------------------------------------------------------------------ virtual int isRemote() {return myPersona == XrdCmsClient::amRemote;} //------------------------------------------------------------------------------ //! Retrieve file location information. //! //! @param Resp Object where message or response is to be returned. //! @param path The logical path whise location is wanted. //! @param flags One or more of the following: //! //! SFS_O_LOCATE - return the list of servers that have the file. //! Otherwise, redirect to the best server for the file. //! SFS_O_NOWAIT - w/ SFS_O_LOCATE return readily available info. //! Otherwise, select online files only. //! SFS_O_CREAT - file will be created. //! SFS_O_NOWAIT - select server if file is online. //! SFS_O_REPLICA - a replica of the file will be made. //! SFS_O_STAT - only stat() information wanted. //! SFS_O_TRUNC - file will be truncated. //! //! For any the the above, additional flags are passed: //! SFS_O_META - data will not change (inode operation only) //! SFS_O_RESET - reset cached info and recaculate the location(s). //! SFS_O_WRONLY - file will be only written (o/w RDWR or RDONLY). //! SFS_O_RDWR - file may be read and written (o/w WRONLY or RDONLY). //! //! @param Info Associated environmental information for arg2 (e.g., cgi info //! which can be retrieved by Env1->Env()). //! //! @return As explained under "return conventions". //------------------------------------------------------------------------------ virtual int Locate(XrdOucErrInfo &Resp, const char *path, int flags, XrdOucEnv *Info=0) = 0; //------------------------------------------------------------------------------ //! Obtain the list of cmsd's being used by a manager node along with their //! associated index numbers, origin 1. //! //! @return The list of cmsd's being used. The list is considered permanent //! and is not deleted. // Return: A list of managers or null if none exist. //------------------------------------------------------------------------------ virtual XrdOucTList *Managers() {return 0;} //------------------------------------------------------------------------------ //! Start the preparation of a file for future processing. //! //! @param Resp Object where message or response is to be returned. //! @param pargs Information on which and how to prepare the file. //! @param Info Associated environmental information. //! //! @return As explained under "return conventions". //------------------------------------------------------------------------------ virtual int Prepare(XrdOucErrInfo &Resp, XrdSfsPrep &pargs, XrdOucEnv *Info=0) { (void)Resp; (void)pargs; (void)Info; return 0; } //------------------------------------------------------------------------------ //! Notify the cmsd that a file or directory has been deleted. It is only called //! called on a data server node. //! //! @param path The logical file name that was removed. //------------------------------------------------------------------------------ virtual void Removed(const char *path) { (void)path; } //------------------------------------------------------------------------------ //! Resume service after a suspension. //! //! @param Perm When true the resume persist across server restarts. Otherwise, //! it is treated as a temporary request. //------------------------------------------------------------------------------ virtual void Resume (int Perm=1) { (void)Perm; } //------------------------------------------------------------------------------ //! Suspend service. //! //! @param Perm When true the suspend persist across server restarts. //! Otherwise, it is treated as a temporary request. //------------------------------------------------------------------------------ virtual void Suspend(int Perm=1) { (void)Perm; } // The following set of functions can be used to control whether or not clients // are dispatched to this data server based on a virtual resource. The default // implementations do nothing. // //------------------------------------------------------------------------------ //! Enables the Reserve() & Release() methods. //! //! @param n a positive integer that specifies the amount of resource units //! that are available. It may be reset at any time. //! //! @return The previous resource value. This first call returns 0. //------------------------------------------------------------------------------ virtual int Resource(int n) { (void)n; return 0;} //------------------------------------------------------------------------------ //! Decreases the amount of resources available. When the available resources //! becomes non-positive, perform a temporary suspend to prevent additional //! clients from being dispatched to this data server. //! //! @param n The value by which resources are decreased (default 1). //! //! @return The amount of resource left. //------------------------------------------------------------------------------ virtual int Reserve (int n=1) { (void)n; return 0;} //------------------------------------------------------------------------------ //! Increases the amount of resource available. When transitioning from a //! a non-positive to a positive resource amount, perform a resume so that //! additional clients may be dispatched to this data server. //! //! @param n The value to add to the resources available (default 1). The //! total amount is capped by the amount specified by Resource(). //! //! @return The amount of resource left. //------------------------------------------------------------------------------ virtual int Release (int n=1) { (void)n; return 0;} //------------------------------------------------------------------------------ //! Obtain the overall space usage of a cluster. Called only on manager nodes. //! //! @param Resp Object to hold response or error message. //! @param path Associated logical path for the space request. //! @param Info Associated cgi information for path. //! //! @return Space information as defined by the response to kYR_statfs. For a //! typical implementation see XrdCmsNode::do_StatFS(). //------------------------------------------------------------------------------ virtual int Space(XrdOucErrInfo &Resp, const char *path, XrdOucEnv *Info=0) = 0; //------------------------------------------------------------------------------ //! Report utilization of this server. This may be used in lieu of general //! performance metric reporting. For consistent results use only one method. //! //! @param util A value from 0 to 100 representing utilization. Values //! greater than 100 are set to be 100. //! @param alert When true the utilization is forcibly report to the //! cluster managers. Otherwise, reporting is done only when //! it will significantly change server selection. //------------------------------------------------------------------------------ virtual void Utilization(unsigned int util, bool alert=false) {(void)util; (void)alert;} //------------------------------------------------------------------------------ //! Constructor //! //! @param acting The type of function this object is performing. //------------------------------------------------------------------------------ enum Persona {amLocal, //!< Not affiliated with a cluster amRemote, //!< Am a manager and issue redirects amTarget //!< Am a server and field redirects }; XrdCmsClient(Persona acting) : myPersona(acting) {} //------------------------------------------------------------------------------ //! Destructor //------------------------------------------------------------------------------ virtual ~XrdCmsClient() {} protected: Persona myPersona; }; /******************************************************************************/ /* I n s t a n t i a t i o n M o d e F l a g s */ /******************************************************************************/ /*! The following instantiation mode flags are passed to the instantiator (see comments that follow). They may be or'd together, depending on which mode the cms client should operate. They are defined as follows: */ namespace XrdCms { enum {IsProxy = 1, //!< The role is proxy {plus one or more of the below} IsRedir = 2, //!< The role is manager and will redirect users IsTarget = 4, //!< The role is server and will be a redirection target IsMeta = 8 //!< The role is meta {plus one or more of the above} }; } /******************************************************************************/ /* C M S C l i e n t I n s t a n t i a t o r */ /******************************************************************************/ //------------------------------------------------------------------------------ //! Obtain an instance of a configured XrdCmsClient. //! //! The following extern "C" function is called to obtain an instance of the //! XrdCmsClient object. This is only used if the client is an actual plug-in //! as identified by the ofs.cmslib directive. Once the XrdCmsClient object //! is obtained, its Configure() method is called to initialize the object. //! //! @param logger -> XrdSysLogger to be tied to an XrdSysError object for //! any messages. //! @param opMode -> The operational mode as defined by the enum above. There //! are two general types of clients, IsRedir and IsTarget. //! The IsProxy and IsMeta modes are specialization of these //! two basic types. The plug-in must provide an instance of //! the one asked for whether or not they actually do anything. //! //! IsRedir clients are anything other than a data provider //! (i.e., data servers). These clients are expected //! to locate files and redirect a requestor to an //! actual data server. //! //! IsTarget clients are typically data providers (i.e., data //! servers) but may actually do other functions are //! are allowed to redirect as well. //! //! @param myPort -> The server's port number. //! @param theSS -> The object that implements he underlying storage system. //! This object may be passed for historic reasons. //! //! @return Success: a pointer to the appropriate object (IsRedir or IsTarget). //! //! Failure: a null pointer which causes initialization to fail. //------------------------------------------------------------------------------ class XrdOss; typedef XrdCmsClient *(*XrdCmsClient_t)(XrdSysLogger *, int, int, XrdOss *); /*! extern "C" XrdCmsClient *XrdCmsGetClient(XrdSysLogger *Logger, int opMode, int myPort XrdOss *theSS); */ //------------------------------------------------------------------------------ //! Obtain an instance of a default unconfigured XrdCmsClient. //! //! The following function may be called to obtain an instance of the default //! XrdCmsClient object. The Configure() method is *not* called before the //! object is returned. The parameters are the same as those for the function //! XrdCmsGetClient(), above. Note that you need not supply a pointer to the //! underlying storage system, as this is historic in nature. //! //! @return Success: a pointer to the appropriate object (IsRedir or IsTarget). //! //! Failure: a null pointer, neither ISRedir nor IsTarget has been //! specified or there is insufficient memory. //------------------------------------------------------------------------------ namespace XrdCms { XrdCmsClient *GetDefaultClient(XrdSysLogger *Logger, int opMode, int myPort ); } //------------------------------------------------------------------------------ //! Declare compilation version. //! //! Additionally, you *should* declare the xrootd version you used to compile //! your plug-in. //------------------------------------------------------------------------------ /*! #include "XrdVersion.hh" XrdVERSIONINFO(XrdCmsGetClient,); */ #endif xrootd-5.6.9/src/XrdCms/XrdCmsClientConfig.cc000066400000000000000000000617151457266313600210640ustar00rootroot00000000000000/******************************************************************************/ /* */ /* X r d C m s C l i e n t C o n f i g . c c */ /* */ /* (c) 2011 by the Board of Trustees of the Leland Stanford, Jr., University */ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ /* This file is part of the XRootD software suite. */ /* */ /* XRootD is free software: you can redistribute it and/or modify it under */ /* the terms of the GNU Lesser General Public License as published by the */ /* Free Software Foundation, either version 3 of the License, or (at your */ /* option) any later version. */ /* */ /* XRootD is distributed in the hope that it will be useful, but WITHOUT */ /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ /* License for more details. */ /* */ /* You should have received a copy of the GNU Lesser General Public License */ /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ /* COPYING (GPL license). If not, see . */ /* */ /* The copyright holder's institutional names and contributor's names may not */ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ #include #include #include #include #include #include #include #include "XrdCms/XrdCmsClientConfig.hh" #include "XrdCms/XrdCmsClientMsg.hh" #include "XrdCms/XrdCmsPerfMon.hh" #include "XrdCms/XrdCmsSecurity.hh" #include "XrdCms/XrdCmsTrace.hh" #include "XrdCms/XrdCmsUtils.hh" #include "XrdOuc/XrdOuca2x.hh" #include "XrdOuc/XrdOucEnv.hh" #include "XrdOuc/XrdOucStream.hh" #include "XrdOuc/XrdOucTList.hh" #include "XrdOuc/XrdOucUtils.hh" #include "XrdSys/XrdSysHeaders.hh" /******************************************************************************/ /* G l o b a l s */ /******************************************************************************/ namespace XrdCms { extern XrdVersionInfo myVersion; } using namespace XrdCms; /******************************************************************************/ /* d e f i n e s */ /******************************************************************************/ #define TS_Xeq(x,m) if (!strcmp(x,var)) return m(Config); /******************************************************************************/ /* D e s t r u c t o r */ /******************************************************************************/ XrdCmsClientConfig::~XrdCmsClientConfig() { XrdOucTList *tp, *tpp; tpp = ManList; while((tp = tpp)) {tpp = tp->next; delete tp;} tpp = PanList; while((tp = tpp)) {tpp = tp->next; delete tp;} if (VNID_Lib) free(VNID_Lib); if (VNID_Parms) free(VNID_Parms); if (prfLib) free(prfLib); if (prfParms) free(prfParms); } /******************************************************************************/ /* C o n f i g u r e */ /******************************************************************************/ int XrdCmsClientConfig::Configure(const char *cfn, configWhat What, configHow How) { /* Function: Establish configuration at start up time. Input: None. Output: 0 upon success or !0 otherwise. */ XrdOucTList *tpe, *tpl; int i, NoGo = 0; const char *eText = 0; char buff[256], *slash, *temp, *bP, sfx; // Preset some values // myHost = getenv("XRDHOST"); if (!myHost) myHost = "localhost"; myName = XrdOucUtils::InstName(1); CMSPath = getenv("XRDADMINPATH"); if (CMSPath && *CMSPath) CMSPath = strdup(CMSPath); else CMSPath = XrdOucUtils::genPath("/tmp/",XrdOucUtils::InstName(myName,0)); isMeta = (How & configMeta) != 0; isMan = (What& configMan) != 0; isServer = What == configServer; // Process the configuration file // if (!(NoGo = ConfigProc(cfn)) && isMan) {if (How & configProxy) eText = (PanList ? 0 : "Proxy manager"); else if (!ManList) eText = (How & configMeta ? "Meta manager" : "Manager"); if (eText) {Say.Emsg("Config", eText, "not specified."); NoGo=1;} } // Reset tracing options // if (getenv("XRDDEBUG")) Trace.What = TRACE_ALL; // Set proper local socket path // temp = XrdOucUtils::genPath(CMSPath, (const char *)0, ".olb"); free(CMSPath); CMSPath = temp; XrdOucEnv::Export("XRDCMSPATH", temp); XrdOucEnv::Export("XRDOLBPATH", temp); //Compatibility // Determine what type of role we are playing // if (What & configServer) sfx = 's'; else if (What & configSuper) sfx = 'u'; else sfx = 'm'; // Determine which manager list we will be using // if (How & configProxy) {sfx = toupper(sfx); tpl = PanList; } else tpl = ManList; // Generate the system ID for this configuration. // if (!ConfigSID(cfn, tpl, sfx)) NoGo = 1; // Export the manager list // if (tpl) {i = 0; tpe = tpl; while(tpe) {i += strlen(tpe->text) + 9; tpe = tpe->next;} bP = temp = (char *)malloc(i); while(tpl) {bP += sprintf(bP, "%s:%d ", tpl->text, tpl->val); tpl = tpl->next; } *(bP-1) = '\0'; XrdOucEnv::Export("XRDCMSMAN", temp); free(temp); } // Construct proper communications path for a supervisor node // i = strlen(CMSPath); if (What & configSuper) {while((tpl = ManList)) {ManList = tpl->next; delete tpl;} slash = (CMSPath[i-1] == '/' ? (char *)"" : (char *)"/"); sprintf(buff, "%s%solbd.super", CMSPath, slash); ManList = new XrdOucTList(buff, -1, 0); SMode = SModeP = FailOver; } // Construct proper old communication path for a target node // temp = (What & (configMan|configSuper) ? (char *)"nimda" : (char *)"admin"); slash = (CMSPath[i-1] == '/' ? (char *)"" : (char *)"/"); sprintf(buff, "%s%solbd.%s", CMSPath, slash, temp); free(CMSPath); CMSPath = strdup(buff); RepWaitMS = RepWait * 1000; // Initialize the msg queue // if (XrdCmsClientMsg::Init()) {Say.Emsg("Config", ENOMEM, "allocate initial msg objects"); NoGo = 1; } // Load the performance monitor (server pre-screened) if specified. // if (prfLib && cmsMon) {perfMon = XrdCmsUtils::loadPerfMon(&Say, prfLib, XrdCms::myVersion); if (!perfMon || !perfMon->Configure(cfn, prfParms, *Say.logger(), *cmsMon, 0, false)) {Say.Emsg("Config","Unable to configure performance monitor plugin."); NoGo = 1; } } return NoGo; } /******************************************************************************/ /* P r i v a t e F u n c t i o n s */ /******************************************************************************/ /******************************************************************************/ /* C o n f i g P r o c */ /******************************************************************************/ int XrdCmsClientConfig::ConfigProc(const char *ConfigFN) { static int DoneOnce = 0; char *var; int cfgFD, retc, NoGo = 0; XrdOucEnv myEnv; XrdOucStream Config((DoneOnce ? 0 : &Say), getenv("XRDINSTANCE"), &myEnv, "=====> "); // Make sure we have a config file // if (!ConfigFN || !*ConfigFN) {Say.Emsg("Config", "cms configuration file not specified."); return 1; } // Try to open the configuration file. // if ( (cfgFD = open(ConfigFN, O_RDONLY, 0)) < 0) {Say.Emsg("Config", errno, "open config file", ConfigFN); return 1; } Config.Attach(cfgFD); static const char *cvec[] = { "*** cms plugin config:", 0 }; Config.Capture(cvec); // Now start reading records until eof. // while((var = Config.GetMyFirstWord())) {if (!strncmp(var, "cms.", 4) || !strncmp(var, "odc.", 4) // Compatibility || !strcmp(var, "all.manager") || !strcmp(var, "all.adminpath") || !strcmp(var, "olb.adminpath")) if (ConfigXeq(var+4, Config)) {Config.Echo(); NoGo = 1;} } // Now check if any errors occurred during file i/o // if ((retc = Config.LastError())) NoGo = Say.Emsg("Config", retc, "read config file", ConfigFN); Config.Close(); // Return final return code // DoneOnce = 1; return NoGo; } /******************************************************************************/ /* C o n f i g S i d */ /******************************************************************************/ bool XrdCmsClientConfig::ConfigSID(const char *cFN, XrdOucTList *tpl, char sfx) { char *sidVal; // Get the node ID if we need to // if (VNID_Lib) {myVNID = XrdCmsSecurity::getVnId(Say, cFN, VNID_Lib, VNID_Parms, sfx); if (!myVNID) return false; } // Generate the system ID and set the cluster ID // sidVal = XrdCmsSecurity::setSystemID(tpl, myVNID, cidTag, sfx); if (!sidVal || *sidVal == '!') {const char *msg; if (!sidVal) msg = "too many managers."; else msg = sidVal+1; Say.Emsg("Config ","Unable to generate system ID; ", msg); return false; } else if (QTRACE(Debug)) Say.Say("Config ", "Global System Identification: ", sidVal); return true; } /******************************************************************************/ /* C o n f i g X e q */ /******************************************************************************/ int XrdCmsClientConfig::ConfigXeq(char *var, XrdOucStream &Config) { // Process items. for either a local or a remote configuration // TS_Xeq("adminpath", xapath); TS_Xeq("cidtag", xcidt); TS_Xeq("conwait", xconw); TS_Xeq("manager", xmang); TS_Xeq("perf", xperf); TS_Xeq("request", xreqs); TS_Xeq("trace", xtrac); TS_Xeq("vnid", xvnid); return 0; } /******************************************************************************/ /* x a p a t h */ /******************************************************************************/ /* Function: xapath Purpose: To parse the directive: adminpath [ group ] the path of the named socket to use for admin requests. Only the path may be specified, not the filename. group allow group access to the path. Type: Manager only, non-dynamic. Output: 0 upon success or !0 upon failure. */ int XrdCmsClientConfig::xapath(XrdOucStream &Config) { char *pval; // Get the path // pval = Config.GetWord(); if (!pval || !pval[0]) {Say.Emsg("Config", "cms admin path not specified"); return 1;} // Make sure it's an absolute path // if (*pval != '/') {Say.Emsg("Config", "cms admin path not absolute"); return 1;} // Record the path // if (CMSPath) free(CMSPath); CMSPath = XrdOucUtils::genPath(pval,XrdOucUtils::InstName(myName,0)); return 0; } /******************************************************************************/ /* x c i d t */ /******************************************************************************/ /* Function: xcidt Purpose: To parse the directive: cidtag a 1- to 16-character cluster ID tag. Output: 0 upon success or !0 upon failure. */ int XrdCmsClientConfig::xcidt(XrdOucStream &Config) { char *val; // Get the path // if (!(val = Config.GetWord()) || !val[0]) {Say.Emsg("Config", "tag not specified"); return 1;} // Make sure it is not too long // if ((int)strlen(val) > 16) {Say.Emsg("Config", "tag is > 16 characters"); return 1;} // Record the tag // if (cidTag) free(cidTag); cidTag = strdup(val); return 0; } /******************************************************************************/ /* x c o n w */ /******************************************************************************/ /* Function: xconw Purpose: To parse the directive: conwait number of seconds to wait for a manager connection Type: Remote server only, dynamic. Output: 0 upon success or !0 upon failure. */ int XrdCmsClientConfig::xconw(XrdOucStream &Config) { char *val; int cw; if (!(val = Config.GetWord())) {Say.Emsg("Config", "conwait value not specified."); return 1;} if (XrdOuca2x::a2tm(Say,"conwait value",val,&cw,1)) return 1; ConWait = cw; return 0; } /******************************************************************************/ /* x m a n g */ /******************************************************************************/ /* Function: xmang Purpose: Parse: manager [meta | peer | proxy] [all|any] [+][:|] [if ...] meta For cmsd: Specifies the manager when running as a manager For xrootd: Specifies the manager when running as a meta peer For cmsd: Specifies the manager when running as a peer For xrootd: The directive is ignored. proxy For cmsd: This directive is ignored. For xrootd: Specifies the cms-proxy service manager all Distribute requests across all managers. any Choose different manager only when necessary (default). The dns name of the host that is the cache manager. If the host name ends with a plus, all addresses that are associated with the host are treated as managers. The port number to use for this host. if Apply the manager directive if "if" is true. See XrdOucUtils:doIf() for "if" syntax. Notes: Any number of manager directives can be given. When niether peer nor proxy is specified, then regardless of role the following occurs: cmsd: Subscribes to each manager whens role is not peer. xrootd: Logins in as a redirector to each manager when role is not proxy or server. Type: Remote server only, non-dynamic. Output: 0 upon success or !0 upon failure. */ int XrdCmsClientConfig::xmang(XrdOucStream &Config) { class StorageHelper {public: StorageHelper(char **v1, char **v2) : val1(v1), val2(v2) {} ~StorageHelper() {if (*val1) free(*val1); if (*val2) free(*val2); } char **val1, **val2; }; XrdOucTList **theList; char *val, *hSpec = 0, *hPort = 0; StorageHelper SHelp(&hSpec, &hPort); int rc, xMeta = 0, xProxy = 0, smode = FailOver; // Process the optional "peer" or "proxy" // if ((val = Config.GetWord())) {if (!strcmp("peer", val)) return Config.noEcho(); if ((xProxy = !strcmp("proxy", val))) val = Config.GetWord(); else if ((xMeta = !strcmp("meta", val))) if (isMeta || isMan) val = Config.GetWord(); else return Config.noEcho(); else if (isMeta) return Config.noEcho(); } // We can accept this manager. Skip the optional "all" or "any" // if (val) { if (!strcmp("any", val)) smode = FailOver; else if (!strcmp("all", val)) smode = RoundRob; else smode = 0; if (smode) {if (xProxy) SModeP = smode; else SMode = smode; val = Config.GetWord(); } } // Get the actual manager // if (!val) {Say.Emsg("Config","manager host name not specified"); return 1;} else hSpec = strdup(val); // Grab the port number (either in hostname or following token) // if (!(hPort = XrdCmsUtils::ParseManPort(&Say, Config, hSpec))) return 1; // Process any "if" clause now // if ((val = Config.GetWord()) && !strcmp("if", val)) if ((rc = XrdOucUtils::doIf(&Say,Config,"manager directive", myHost, myName, getenv("XRDPROG"))) <= 0) {if (!rc) Config.noEcho(); return (rc < 0);} // If we are a manager and found a meta-manager indidicate it and bail. // if (xMeta && !isMeta) {haveMeta = 1; return 0;} theList = (xProxy ? &PanList : &ManList); // Parse the manager list and return the result // return (XrdCmsUtils::ParseMan(&Say, theList, hSpec, hPort, 0) ? 0 : 1); } /******************************************************************************/ /* x p e r f */ /******************************************************************************/ /* Function: xperf Purpose: To parse the directive: perf [xrootd] [int ] [lib [] | pgm ] int