pax_global_header00006660000000000000000000000064145360414030014512gustar00rootroot0000000000000052 comment=97f05430f0767c1e056fa8949c309996775431d2 gfal2-util-v1.8.1/000077500000000000000000000000001453604140300136555ustar00rootroot00000000000000gfal2-util-v1.8.1/.gitignore000066400000000000000000000000551453604140300156450ustar00rootroot00000000000000*.pyc .idea *.rpm *.tar.gz *.log *.gz *.dsc gfal2-util-v1.8.1/.gitlab-ci.yml000066400000000000000000000070231453604140300163130ustar00rootroot00000000000000stages: - build - test - publish #-------------------------- # Build templates #-------------------------- .build-template: &build-template_definition stage: build script: - ci/fedora-packages.sh - ci/common-rpm-build.sh - mkdir ${CI_JOB_NAME} - cp -rv build/RPMS build/SRPMS ${CI_JOB_NAME} - tree ${CI_JOB_NAME} variables: BRANCH: ${CI_COMMIT_REF_NAME} artifacts: paths: - "$CI_JOB_NAME" #-------------------------- # Build jobs #-------------------------- cc7: image: gitlab-registry.cern.ch/linuxsupport/cc7-base <<: *build-template_definition alma8: image: gitlab-registry.cern.ch/linuxsupport/alma8-base <<: *build-template_definition alma9: image: gitlab-registry.cern.ch/linuxsupport/alma9-base <<: *build-template_definition fedora-37: image: fedora:37 <<: *build-template_definition fedora-38: image: fedora:38 <<: *build-template_definition fedora-rawhide: image: fedora:rawhide <<: *build-template_definition allow_failure: true when: manual #-------------------------- # Test templates #-------------------------- .test-template: &test-template_definition stage: test script: - dnf install -y git - PLATFORM=${CI_JOB_NAME%-*} - ci/write-repo-file.sh - dnf install -y ${PLATFORM}/RPMS/*/*.rpm - shopt -s expand_aliases - python --version &>/dev/null || alias python=python3 - python test/functional/test_all.py variables: BRANCH: ${CI_COMMIT_REF_NAME} #-------------------------- # Test jobs #-------------------------- cc7-test: image: gitlab-registry.cern.ch/linuxsupport/cc7-base needs: - job: cc7 before_script: - yum install -y dnf epel-release <<: *test-template_definition alma8-test: image: gitlab-registry.cern.ch/linuxsupport/alma8-base needs: - job: alma8 before_script: - dnf install -y epel-release <<: *test-template_definition alma9-test: image: gitlab-registry.cern.ch/linuxsupport/alma9-base needs: - job: alma9 before_script: - dnf install -y epel-release <<: *test-template_definition fedora-37-test: image: fedora:37 needs: - job: fedora-37 <<: *test-template_definition fedora-38-test: image: fedora:38 needs: - job: fedora-38 <<: *test-template_definition #-------------------------- # Publish templates #-------------------------- .publish-template: &publish-template_definition stage: publish image: gitlab-registry.cern.ch/eos/gitlab-eos/alma9:latest dependencies: - cc7 - alma8 - alma9 - fedora-37 - fedora-38 - fedora-rawhide script: - automount - cat "$repo_passwd" | kinit "$repo_user" - | for platform in cc7 alma8 alma9 fedora-37 fedora-38 fedora-rawhide; do if [[ "${platform}" == "fedora-rawhide" ]] && [[ ! -d ${platform} ]] ; then echo "Skipping ${platform} packages..." ; continue ; fi packaging/gfal2-repo-manager.py --action add --base /eos/project-d/dmc/www/repos/ --arch-dir x86_64 --ref ${CI_COMMIT_REF_NAME} --packages ${platform}/RPMS/*/*.rpm ${platform}/SRPMS/* done tags: - docker-privileged-xl retry: 2 #-------------------------- # Publish jobs #-------------------------- rpms: <<: *publish-template_definition rules: - if: '$CI_PROJECT_NAMESPACE != "dmc"' when: never - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' when: never - if: '$CI_COMMIT_REF_NAME == "develop" || $CI_COMMIT_TAG != null' - if: '$CI_COMMIT_REF_NAME != "develop" && $CI_COMMIT_TAG == null' when: manual gfal2-util-v1.8.1/.gitmodules000066400000000000000000000000001453604140300160200ustar00rootroot00000000000000gfal2-util-v1.8.1/LICENSE000066400000000000000000000236761453604140300147000ustar00rootroot00000000000000 Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS gfal2-util-v1.8.1/MANIFEST000066400000000000000000000007551453604140300150150ustar00rootroot00000000000000setup.py LICENSE MANIFEST RELEASE-NOTES VERSION doc/gfal-cat.1 doc/gfal-copy.1 doc/gfal-ls.1 doc/gfal-mkdir.1 doc/gfal-rm.1 doc/gfal-save.1 doc/gfal-sum.1 doc/gfal-xattr.1 src/gfal-cat src/gfal-copy src/gfal-ls src/gfal-mkdir src/gfal-rm src/gfal-save src/gfal-sum src/gfal-xattr src/gfal2_util/__init__.py src/gfal2_util/base.py src/gfal2_util/commands.py src/gfal2_util/gfal2_utils_parameters.py src/gfal2_util/ls.py src/gfal2_util/progress.py src/gfal2_util/shell.py src/gfal2_util/utils.pygfal2-util-v1.8.1/README.md000066400000000000000000000076331453604140300151450ustar00rootroot00000000000000gfal2-util ========== GFAL 2 utils are a group of command line tools for file manipulations with any protocol managed by gfal2 ## Installation ```bash yum install gfal2-util gfal2-all ``` Mind that the protocol support of gfal2-util depends on the set of plugins installed in your machine. Installing gfal2-all will trigger an installation of the plugins for file, SRM, GridFTP, HTTP/DAV, LFC, RFIO and DCAP. If you need XROOTD support, you will need to install additionally gfal2-plugin-xrootd. ## List of commands We provide here a brief overview of the command line tools we provide as part of this package. To see detailed usage of each one, you can invoke them with --help to show all the possible options. ### Standard commands #### gfal-bringonline Perform a staging operation on the given SURL. #### gfal-archivepoll Perform an archive polling operation on a given SURL. #### gfal-cat Equivalent to cat. Print into the standard output the content of the requested file #### gfal-chmod Equivalent to chmod. Changes the access permissions of the requested file #### gfal-copy Equivalent to cp but more powerful. Copy a file, or set of files, between different storages and/or local. Recursive mode is supported. Try --dry-run to validate what will happen. #### gfal-evict Requests a Tape storage to release the given file's disk replica. - For SRM storage, this is the equivalent of an unpin operation. The file is marked as eligible for garbage collection - For CTA storage, this is the equivalent of an evict operation. The file's evict counter is decreased. When the counter reaches zero, the file's disk replica is deleted immediately #### gfal-ls Equivalent to ls. List the content of a directory. Use -l to generate a detailed list. #### gfal-mkdir Equivalent to mkdir. Create a directory #### gfal-rename Equivalent to rename command. Moves the file from one location to another one within the same storage #### gfal-rm Equivalent to rm. Remove a file, or a directory if -r is specified. Try --dry-run if you want to validate what will happen without doing anything. #### gfal-save Reads from stdin and writes into a local or remote file. #### gfal-stat Equivalent to stat. Print information about the file or directory. #### gfal-sum Query or calculate the checksum of the given file. Normally recognised checksums are adler32, md5 and sha1, but it depends on the protocol and/or storage software. #### gfal-token Requests a Storage Endpoint (SE) to issue a token for the given SURL (SE-issued token). The token should be considered opaque, but most often used Grid implementations are JWTs or macaroons. These tokens have great applicability in HTTP-TPC as they avoid the need to perform proxy certificate delegation. #### gfal-xattr Equivalent to attr. Query or set an extended attribute from a file or directory. Which extended attributes are supported depend on the protocol. For instance user.replicas or user.comment can be used for SRM or LFC. ### Legacy commands This commands exists purely to provide ways to access functionality previously provided by lcg-util. We recommend avoiding the usage of these tools if possible. #### gfal-legacy-register Register a new replica in a catalog. We recommend using instead ```bash gfal-xattr [surl] user.replicas=+new_replica ``` #### gfal-legacy-replicas List replicas from a catalog or SRM. We recommend using instead ```bash gfal-xattr [surl] user.replicas ``` #### gfal-legacy-unregister Remove an existing replica from a catalog. We recommend using instead ```bash gfal-xattr [surl] user.replicas=-replica ``` ## License This software is licensed under the [Apache 2 License](http://www.apache.org/licenses/LICENSE-2.0.html) Copyright (c) 2013-2022 CERN Copyright (c) 2012-2013 Members of the EMI Collaboration See http://www.eu-emi.eu/partners for details on the copyright holders. ## Contact You can notify bugs or ask for feature requests via dmc-support@cern.ch or dmc-devel@cern.ch gfal2-util-v1.8.1/RELEASE-NOTES000066400000000000000000000200101453604140300155370ustar00rootroot00000000000000gfal2-util release notes ======================== ## Tue Dec 12 2023 Mihai Patrascoiu - 1.8.1 ### Bug - [DMC-1395] - Gfal2 CLI return code is not within bash return code range - [gfal2-util/issues/1] - gfal-copy fails to complete the data transfer but returns exit code 0 - [cern-fts/gfal2-util/pull/5] - CommandRm should call CommandBase constructor (solve issue #4) ### New Feature - [DMC-1344] - Propagate Scitags during TPC transfers ### Task - [DMC-1327] - DMC packages for Centos Stream 9 - [DMC-1359] - Data Management Clients for Fedora 37 - [DMC-1379] - Move DMC projects to Alma8 and Alma9 - [DMC-1387] - Data Management Clients for Fedora 38 - [DMC-1389] - Port Gfal2 Python3 projects away from "distutils" ## Fri Sep 02 2022 Joao Lopes - 1.8.0 ### Bug - [DMC-1328] - Exception with gfal-copy progress bar when source size is known and transfer rate is not reported yet ### New Feature - [DMC-1319] - Renew gfal-legacy-bringonline command - [DMC-1332] - Create gfal-archivepoll command ### Task - [DMC-1336] - Data Management Clients for Fedora 36 - [DMC-1341] - Change gfal2-util license to Apache 2.0 ### Improvement - [DMC-1208] - Gfal2-util bringonline with "--from-file" option - [gfal2-util/pull/7] - Add missing man page section for "gfal-legacy-register" ### Sub-task - [DMC-1333] - Create a single file for tape commands in gfal-util - [DMC-1342] - gfal-archivepoll with "--from-file" option - [DMC-1343] - Create gfal-evict command ## Mon Mar 07 2022 Mihai Patrascoiu - 1.7.1 ### Bug - [DMC-1285] - Stuck transfers to XRoot sites with enabled TLS - [DMC-1288] - Python3 "gfal-cat" fails on binary files - [gfal2-util/pull/5] - DMC-1285: Transfers gets stuck when calling subprocess ### Task - [DMC-1299] - Update man pages for gfal2-util ### Improvement - [DMC-1308] - Move Data Management Clients to Fedora 34 & 35 ## Thu Sep 23 2021 Joao Lopes - 1.7.0 ### Epic - [DMC-1249] - Migrate CI infrastructure to gitlab-CI ### New Feature - [DMC-1265] - Export SE token retrieval in Gfal2 Python binding ### Task - [DMC-1250] - Gfal2 packages for Centos8 - [DMC-1264] - Gfal2 packages for Fedora >= 33 ## Thu Nov 19 2020 Petr Vokac - 1.6.0 ### Improvement - [DMC-1223] make sources compatible with python 2.4 - 3.9 - [DMC-1223] automatically choose available python interpreter the one from GFAL_PYTHONBIN env is most preferred with fallback to python3, python2, python - [DMC-1223] drop separate python3 code - [DMC-1229] Introduce Gfal2 transfer parameter to enable/disable proxy delegation - Fix the parsing of passed-in definitions to allow complex groups ## Mon Sep 14 2020 Mihai Patrascoiu - 1.5.4 ### Bug - [DMC-1212] - Prevent gfal2-copy output crashing when filesize is reported 0 - [DMC-1222] - gfal-rm reports return code 0 on MISSING file URLs ### Improvement - [DMC-1225] - Allow gfal2 CLI clients to separate output generated by the clients from gfal2 library log output - Change order of uid/gid to match POSIX 'ls -l' ## Thu Jul 11 2019 Andrea Manzi - 1.5.3 ### Bug - [DMC-1155] - gfal-rm with signed https URL gives 401 error ### Task - [DMC-1150] - Set XrdSecGSIDELEGPROXY=1 in gfal-copy to enable XRD TPC with delegation ## Thu Feb 21 2019 Andrea Manzi - 1.5.2 ### Bug - [DMC-1070] - gfal-ls cannot list WebDAV directories - [DMC-1142] - Shebang on gfal-utils commands should specify python2 instead of python ### Improvement - [DMC-1124] - Add --copy-mode to gfal-copy - [DMC-1077] - gfal-copy --just-copy stats dest ## Wed Jun 21 2017 Alejandro Alvarez - 1.5.1 ### Bug - [DMC-933] - gfal-copy does stat/propfind even if --just-copy is given ## Mon Feb 06 2017 Alejandro Alvarez - 1.5.0 ### Bug - [DMC-883] - gfal-ls fails when trying to evaluate the LS_COLORS envvar - [DMC-899] - gfal2-util must use /usr/bin/env python as shebang ### New Feature - [DMC-892] - Generalize checksum methods ### Improvement - [DMC-880] - If the option -t is passed, make sure that Davix knows it ## Mon Aug 01 2016 Alejandro Alvarez - 1.4.0 ### New Feature - [DMC-862] - /some/path: Automatically prefix things with file:// - [DMC-865] - If destination file is special type (i.e. stdout, null), do not fail ### Improvement - [DMC-836] - Add bulk deletion option ## Mon Mar 07 2016 Alejandro Alvarez - 1.3.2 ### Bug - [DMC-807] - gfal2-util: -4 and -6 flags do nothing ### Improvement - [DMC-810] - Better document timeout vs transfer timeout ## Mon Nov 09 2015 Alejandro Alvarez - 1.3.1 ### Bug - [DMC-767] - Register callbacks in gfal-copy - [DMC-768] - Fix progress bar for gfal-copy ## Mon Oct 12 2015 Alejandro Alvarez - 1.3.0 ### Bug - [DMC-723] - gfal2-copy does not allow adding new replicas to an LFC entry ### Epic - [DMC-698] - Port DMC clients to MacOSX ### Improvement - [DMC-672] - When listing extended attributes, do not stop on failure - [DMC-674] - Validate version as part of the packaging - [DMC-709] - In gfal-ls, show size before date, as with system's ls - [DMC-710] - Add partial support for LS_COLORS - [DMC-721] - Man pages for gfal-copy missing information on how to register into the LFC ### New Feature - [DMC-706] - Add --full-time and --time-style flags to gfal-ls - [DMC-730] - Add a gfal-chmod command ## Fri Apr 10 2015 Alejandro Alvarez - 1.2.1 ### Bug - [DMC-599] - Transfer timeout not respected - [DMC-628] - Stat surl before deciding if doing unlink or rmdir ### Improvement - [DMC-649] - Core: Improve logger system, deprecate old methods ### New Feature - [DMC-532] - gfal-copy -r (recursive) - [DMC-545] - Provide --dry-run for gfal-rm - [DMC-585] - Add a gfal-rename command - [DMC-667] - Make version (--version/-V) flag more verbose ## Fri Nov 07 2014 Alejandro Alvarez - 1.1.0 - [DMC-517] - Update gfal-copy manpage - [DMC-522] - is_alive is not available in Python 2.4 (EL5) - [DMC-442] - gfal-copy should accept a checksum parameter - [DMC-467] - Create a cli around xattr+replicas to ease its use - [DMC-481] - Expose 'strict mode' parameter via Python bindings and gfal2-util - [DMC-513] - New command gfal-stat - [DMC-405] - support input from file in gfal-copy and gfal-del ## Wed Jul 02 2014 Alejandro Alvarez - 1.0.0 - [LCGUTIL-313] - gfal-copy force not working - [LCGUTIL-401] - Pass overwrite (--force) parameter to gfal2 - [LCGUTIL-415] - gfal-ls barfs with head (sometimes) - [LCGUTIL-308] - gfal-cp to support chained copies? - [LCGUTIL-314] - gfal2 default timeout - [LCGUTIL-385] - GridFTP listing improvement (RELNOTE: session reuse enabled by default!) - [LCGUTIL-387] - Add buffer size parameter to gfal-cp - [LCGUTIL-392] - gfal-copy does not give the option to create parent directory - [LCGUTIL-283] - add an option to specify credentials to gfal-utils tools - [LCGUTIL-431] - Allow to do an ls with additional metadata - [LCGUTIL-340] - gfal2 - utils : exception triggered on gfal2-ls -l - [LCGUTIL-404] - gfal2-util broken on SL5 - [LCGUTIL-413] - gfal-ls -l should indicate file type ## Version 0.2.0 : - [LCGUTIL-75] - gfal-tools : implement basic cancel logic in order to interrupt I/O operation - [LCGUTIL-76] - gfal-utils : implement advanced cancel logic in order to interrupt I/O operation - [LCGUTIL-208] - gfal-utils : gfal-ls should be able to stat() file and do long style listing of directory like "ls -l" - [LCGUTIL-209] - gfal-utils : gfal-copy should have a progress bar to allow to follow the status of a transfer - [LCGUTIL-252] - Add a gfal-gettturls (or similar) - [LCGUTIL-247] - GFAL2-UTILS : gfal-utils interrupt cmd line are not triggered properly ## Version 0.0.14-0.1.alpha - initial release of gfal-copy logic with basic third-party and copy support - initial release of gfal-sum, the checksum calculation tool - initial release of gfal-rm, remove file tools - add support for spacetokens inside gfal-copy - add fast cancel operation support inside gfal2-utils - preview and demonstration purpose only gfal2-util-v1.8.1/VERSION000066400000000000000000000000061453604140300147210ustar00rootroot000000000000001.8.1 gfal2-util-v1.8.1/ci/000077500000000000000000000000001453604140300142505ustar00rootroot00000000000000gfal2-util-v1.8.1/ci/common-rpm-build.sh000077500000000000000000000030431453604140300177700ustar00rootroot00000000000000#!/usr/bin/env bash set -e function print_info { printf "======================\n" printf "%-17s%s\n" "Distribution:" "${DIST}" printf "%-17s%s\n" "Dist name:" "${DISTNAME}" printf "%-17s%s\n" "Build type:" "${BUILD}" printf "%-17s%s\n" "Branch:" "${BRANCH}" printf "%-17s%s\n" "Release:" "${RELEASE}" printf "%-17s%s\n" "DMC Repository:" "${REPO_FILE}" printf "%-17s%s\n" "RPM build flags:" "${RPMBUILD_FLAGS}" printf "======================\n" } TIMESTAMP=`date +%y%m%d%H%M` GITREF=`git rev-parse --short HEAD` RELEASE=r${TIMESTAMP}git${GITREF} BUILD="devel" if [[ -z ${BRANCH} ]]; then BRANCH=`git name-rev $GITREF --name-only` else printf "Using environment set variable BRANCH=%s\n" "${BRANCH}" fi if [[ $BRANCH =~ ^(tags/)?(v)[.0-9]+(-(rc)?([0-9]+))?$ ]]; then RELEASE="${BASH_REMATCH[4]}${BASH_REMATCH[5]}" BUILD="rc" fi DIST=$(rpm --eval "%{dist}" | cut -d. -f2) DISTNAME=${DIST} # Special handling of FC rawhide [[ "${DISTNAME}" == "fc39" ]] && DISTNAME="fc-rawhide" [[ "${DISTNAME}" == "fc40" ]] && DISTNAME="fc-rawhide" # Write repository files to /etc/yum.repos.d/ based on the branch name REPO_FILE=$(./ci/write-repo-file.sh) print_info RPMBUILD=${PWD}/build SRPMS=${RPMBUILD}/SRPMS cd packaging/ make srpm RELEASE=${RELEASE} RPMBUILD=${RPMBUILD} SRPMS=${SRPMS} RPMBUILD_SRC_EXTRA_FLAGS="${RPMBUILD_FLAGS}" if [[ -f /usr/bin/dnf ]]; then dnf install -y epel-release || true dnf builddep -y ${SRPMS}/* else yum-builddep -y ${SRPMS}/* fi rpmbuild --rebuild --define="_topdir ${RPMBUILD}" ${RPMBUILD_FLAGS} ${SRPMS}/* gfal2-util-v1.8.1/ci/fedora-packages.sh000077500000000000000000000004321453604140300176220ustar00rootroot00000000000000#!/usr/bin/env bash set -e if [[ -f /usr/bin/dnf ]]; then dnf install -y dnf-plugins-core git rpm-build tree which \ cmake make gcc gcc-c++ else yum install -y yum-utils git rpm-build make tree which python2 \ cmake cmake3 make gcc gcc-c++ fi gfal2-util-v1.8.1/ci/write-repo-file.sh000077500000000000000000000017521453604140300176260ustar00rootroot00000000000000#!/usr/bin/env bash set -e GITREF=`git rev-parse --short HEAD` if [[ -z ${BRANCH} ]]; then BRANCH=`git name-rev $GITREF --name-only` fi if [[ $BRANCH =~ ^(tags/)?(v)[.0-9]+(-(rc)?([0-9]+))?$ ]]; then BUILD="rc" elif [[ ! -z ${DMC_REPO_BRANCH} ]]; then BUILD="${DMC_REPO_BRANCH}" else BUILD="develop" fi DIST=$(rpm --eval "%{dist}" | cut -d. -f2) DISTNAME=${DIST} # Special handling of FC rawhide [[ "${DISTNAME}" == "fc39" ]] && DISTNAME="fc-rawhide" [[ "${DISTNAME}" == "fc40" ]] && DISTNAME="fc-rawhide" if [[ ${BUILD} == "rc" ]]; then REPO_PATH="${BUILD}/${DISTNAME}/\$basearch" elif [[ ${BUILD} == "develop" ]]; then REPO_PATH="testing/${DISTNAME}/\$basearch" else REPO_PATH="testing/${BUILD}/${DISTNAME}/\$basearch" fi cat <<- EOF > "/etc/yum.repos.d/dmc-${BUILD}-${DISTNAME}.repo" [dmc-${BUILD}-${DISTNAME}] name=DMC Repository baseurl=http://dmc-repo.web.cern.ch/dmc-repo/${REPO_PATH} gpgcheck=0 enabled=1 protect=0 priority=3 EOF echo "dmc-${BUILD}-${DISTNAME}.repo" gfal2-util-v1.8.1/doc/000077500000000000000000000000001453604140300144225ustar00rootroot00000000000000gfal2-util-v1.8.1/doc/man/000077500000000000000000000000001453604140300151755ustar00rootroot00000000000000gfal2-util-v1.8.1/doc/man/gfal-archivepoll.1000066400000000000000000000052601453604140300205010ustar00rootroot00000000000000.\" Manpage for gfal-archivepoll .\" .TH GFAL-ARCHIVEPOLL 1 "Set 2022" "v1.8.0" .SH NAME gfal-archivepoll \- Perform an archive polling operation on the given SURL .SH SYNOPSIS .B gfal-archivepoll [ .I "OPTION" ]... .I FILE .SH DESCRIPTION .B gfal-archivepoll This command performs an archive polling operation on a given SURL. If a polling timeout greater than 0 is provided the command blocks for polling the status of the archiving request and only returns when the file is on tape or the timeout expires. .SH OPTIONS .TP .B "-h, --help" show help about the command. .TP .B "-V, --version" output version information and exit. .TP .B "-v, --verbose" verbose mode. Repeat up to 3 times for the most verbose mode. .TP .BI "-D, --definition " DEFINITION define properties for gfal2. Definitions have the following syntax: "CONFIG_GROUP:KEY=VALUE" .TP .BI "-t, --timeout " TIMEOUT global timeout for the execution of the command. Command is interrupted if time expires before it finishes. Note: A grace period of 30 seconds is given to the global timeout. .TP .BI "-E, --cert " CERT user certificate. (This option takes effect only after X509_USER_PROXY environment variable and default /tmp/x509up_u`id -u` location cannot provide a certificate) .TP .BI "--key " KEY user private key. .TP .B "-4" forces Gfal2 to use IPv4 addresses only. (Option valid only for GridFTP protocol) .TP .B "-6" forces Gfal2 to use IPv6 addresses only. (Option valid only for GridFTP protocol) .TP .BI "-C, --client-info " CLIENT_INFO provide custom client-side information. .TP .BI "--log-file " LOG_FILE write Gfal2 library logs to the given file location. .TP .BI "--from-file " FROM_FILE read sources from a file. .TP .B "--polling-timeout" The timeout for the polling operation. If a polling timeout greater than 0 is provided the command blocks for polling the status of the archiving request and only returns when the file is on tape or the timeout expires. .SH EXAMPLES .TP .B gfal-archivepoll http://endpoint.cern.ch/path/file --polling-timeout=3600 .SH FILES .I /etc/gfal2.d/* .RS All configuration files in this directory affect the behavior of the underlying gfal2. .SH AUTHORS Duarte Meneses .br Adrien Devresse .br Alejandro Alvarez Ayllon .br Mihai Patrascoiu .br Joao Lopes .SH "SEE ALSO" .BR gfal-cat (1), .BR gfal-chmod (1), .BR gfal-copy (1), .BR gfal-ls (1), .BR gfal-mkdir (1), .BR gfal-rename (1), .BR gfal-rm (1), .BR gfal-save (1), .BR gfal-stat (1), .BR gfal-sum (1), .BR gfal-token (1), .BR gfal-xattr (1), .BR gfal-evict (1), .BR gfal-bringonline (1), .BR gfal-archivepoll (1) gfal2-util-v1.8.1/doc/man/gfal-bringonline.1000066400000000000000000000056301453604140300205000ustar00rootroot00000000000000.\" Manpage for gfal-bringonline .\" .TH GFAL-BRINGONLINE 1 "May 2022" "v1.8.0" .SH NAME gfal-bringonline \- Perform a staging operation on the given SURL .SH SYNOPSIS .B gfal-bringonline [ .I "OPTION" ]... .I FILE .SH DESCRIPTION .B gfal-bringonline This command performs a staging operation on the given SURL. A bringonline token is returned if the staging request is successful. In addition, if the polling timeout is greater than 0 the command blocks for polling the status of the staging request and only returns when the file is online or the timeout expires. .SH OPTIONS .TP .B "-h, --help" show help about the command. .TP .B "-V, --version" output version information and exit. .TP .B "-v, --verbose" verbose mode. Repeat up to 3 times for the most verbose mode. .TP .BI "-D, --definition " DEFINITION define properties for gfal2. Definitions have the following syntax: "CONFIG_GROUP:KEY=VALUE" .TP .BI "-t, --timeout " TIMEOUT global timeout for the execution of the command. Command is interrupted if time expires before it finishes. Note: A grace period of 30 seconds is given to the global timeout. .TP .BI "-E, --cert " CERT user certificate. (This option takes effect only after X509_USER_PROXY environment variable and default /tmp/x509up_u`id -u` location cannot provide a certificate) .TP .BI "--key " KEY user private key. .TP .B "-4" forces Gfal2 to use IPv4 addresses only. (Option valid only for GridFTP protocol) .TP .B "-6" forces Gfal2 to use IPv6 addresses only. (Option valid only for GridFTP protocol) .TP .BI "-C, --client-info " CLIENT_INFO provide custom client-side information. .TP .BI "--log-file " LOG_FILE write Gfal2 library logs to the given file location. .TP .TP .B "--pin-lifetime" Desired pin lifetime .TP .B "--desired-request-time" Desired total request time .TP .B "--staging-metadata" Metadata for the bringonline operation .BI "--from-file " FROM_FILE read list of surls from a file. .TP .B "--polling-timeout" The timeout for the polling operation. If a polling timeout greater than 0 is provided the command blocks for polling the status of the staging request and only returns when the file is on disk or the timeout expires. .SH EXAMPLES .TP .B gfal-bringonline http://endpoint.cern.ch/path/file --polling-timeout=3600 .SH FILES .I /etc/gfal2.d/* .RS All configuration files in this directory affect the behavior of the underlying gfal2. .SH AUTHORS Duarte Meneses .br Adrien Devresse .br Alejandro Alvarez Ayllon .br Mihai Patrascoiu .br Joao Lopes .SH "SEE ALSO" .BR gfal-cat (1), .BR gfal-chmod (1), .BR gfal-copy (1), .BR gfal-ls (1), .BR gfal-mkdir (1), .BR gfal-rename (1), .BR gfal-rm (1), .BR gfal-save (1), .BR gfal-stat (1), .BR gfal-sum (1), .BR gfal-token (1), .BR gfal-xattr (1), .BR gfal-evict (1), .BR gfal-bringonline (1), .BR gfal-archivepoll (1) gfal2-util-v1.8.1/doc/man/gfal-cat.1000066400000000000000000000044321453604140300167400ustar00rootroot00000000000000.\" Manpage for gfal-cat .\" .TH GFAL-CAT 1 "March 2022" "v1.7.1" .SH NAME gfal-cat \- concatenate a file and print on the standard output .SH SYNOPSIS .B gfal-cat [ .I "OPTION" ]... .I FILE .SH DESCRIPTION .B gfal-cat Concatenate file to standard output. It simply opens the specified file and prints all its contents to stdout. .SH OPTIONS .TP .B "-h, --help" show help about the command. .TP .B "-V, --version" output version information and exit. .TP .B "-v, --verbose" verbose mode. Repeat up to 3 times for the most verbose mode. .TP .BI "-D, --definition " DEFINITION define properties for gfal2. Definitions have the following syntax: "CONFIG_GROUP:KEY=VALUE" .TP .BI "-t, --timeout " TIMEOUT global timeout for the execution of the command. Command is interrupted if time expires before it finishes. Note: A grace period of 30 seconds is given to the global timeout. .TP .BI "-E, --cert " CERT user certificate. (This option takes effect only after X509_USER_PROXY environment variable and default /tmp/x509up_u`id -u` location cannot provide a certificate) .TP .BI "--key " KEY user private key. .TP .B "-4" forces Gfal2 to use IPv4 addresses only. (Option valid only for GridFTP protocol) .TP .B "-6" forces Gfal2 to use IPv6 addresses only. (Option valid only for GridFTP protocol) .TP .BI "-C, --client-info " CLIENT_INFO provide custom client-side information. .TP .BI "--log-file " LOG_FILE write Gfal2 library logs to the given file location. .TP .B "-b, --bytes" handle file contents as bytes. (only in Python3) .SH EXAMPLES .TP .B gfal-cat srm://endpoint.cern.ch/path/file .SH FILES .I /etc/gfal2.d/* .RS All configuration files in this directory affect the behavior of the underlying gfal2. .SH AUTHORS Duarte Meneses .br Adrien Devresse .br Alejandro Alvarez Ayllon .br Andrea Manzi .br Mihai Patrascoiu .br Joao Lopes .SH "SEE ALSO" .BR gfal-cat (1), .BR gfal-chmod (1), .BR gfal-copy (1), .BR gfal-ls (1), .BR gfal-mkdir (1), .BR gfal-rename (1), .BR gfal-rm (1), .BR gfal-save (1), .BR gfal-stat (1), .BR gfal-sum (1), .BR gfal-token (1), .BR gfal-xattr (1), .BR gfal-evict (1), .BR gfal-bringonline (1), .BR gfal-archivepoll (1) gfal2-util-v1.8.1/doc/man/gfal-chmod.1000066400000000000000000000041741453604140300172660ustar00rootroot00000000000000.\" Manpage for gfal-chmod .\" .TH GFAL-CHMOD 1 "March 2022" "v1.7.1" .SH NAME gfal-chmod \- Change the permissions of a file .SH SYNOPSIS .B gfal-chmod [ .I "OPTION" ]... .I MODE .I URL .SH DESCRIPTION .B gfal-chmod Change the permissions of a file. .SH OPTIONS .TP .B "-h, --help" show help about the command. .TP .B "-V, --version" output version information and exit. .TP .B "-v, --verbose" verbose mode. Repeat up to 3 times for the most verbose mode. .TP .BI "-D, --definition " DEFINITION define properties for gfal2. Definitions have the following syntax: "CONFIG_GROUP:KEY=VALUE" .TP .BI "-t, --timeout " TIMEOUT global timeout for the execution of the command. Command is interrupted if time expires before it finishes. Note: A grace period of 30 seconds is given to the global timeout. .TP .BI "-E, --cert " CERT user certificate. (This option takes effect only after X509_USER_PROXY environment variable and default /tmp/x509up_u`id -u` location cannot provide a certificate) .TP .BI "--key " KEY user private key. .TP .B "-4" forces Gfal2 to use IPv4 addresses only. (Option valid only for GridFTP protocol) .TP .B "-6" forces Gfal2 to use IPv6 addresses only. (Option valid only for GridFTP protocol) .TP .BI "-C, --client-info " CLIENT_INFO provide custom client-side information. .TP .BI "--log-file " LOG_FILE write Gfal2 library logs to the given file location. .SH EXAMPLES .B gfal-chmod 0755 file:///tmp/original .SH FILES .I /etc/gfal2.d/* .RS All configuration files in this directory affect the behavior of the underlying gfal2. .SH AUTHORS Duarte Meneses .br Adrien Devresse .br Alejandro Alvarez Ayllon .br Andrea Manzi .br Mihai Patrascoiu .br Joao Lopes .SH "SEE ALSO" .BR gfal-cat (1), .BR gfal-chmod (1), .BR gfal-copy (1), .BR gfal-ls (1), .BR gfal-mkdir (1), .BR gfal-rename (1), .BR gfal-rm (1), .BR gfal-save (1), .BR gfal-stat (1), .BR gfal-sum (1), .BR gfal-token (1), .BR gfal-xattr (1), .BR gfal-evict (1), .BR gfal-bringonline (1), .BR gfal-archivepoll (1) gfal2-util-v1.8.1/doc/man/gfal-copy.1000066400000000000000000000115411453604140300171420ustar00rootroot00000000000000.\" Manpage for gfal-copy .\" .TH GFAL-COPY 1 "March 2022" "v1.7.1" .SH NAME gfal-copy \- Copy files .SH SYNOPSIS .B gfal-copy [ .I OPTION ]... .I SOURCE .I DEST1 [ .I DEST2 ]... .SH DESCRIPTION .B gfal-copy Copies files from source to destination(s). If several destinations are specified, the file will be copy in a chain: src -> dst1, dst1 -> dst2, ... If used in an interactive console, transfer information will be giving during the copy, which varies depending on the available information. This command can be used to upload local files to remote file systems and to register files in the file catalog (see examples). .SH OPTIONS .TP .B "-h, --help" show help about the command. .TP .B "-V, --version" output version information and exit. .TP .B "-v, --verbose" verbose mode. Repeat up to 3 times for the most verbose mode. .TP .BI "-D, --definition " DEFINITION define properties for gfal2. Definitions have the following syntax: "CONFIG_GROUP:KEY=VALUE" .TP .BI "-t, --timeout " TIMEOUT global timeout for the execution of the command. Command is interrupted if time expires before it finishes. Note: A grace period of 30 seconds is given to the global timeout. .TP .BI "-E, --cert " CERT user certificate. (This option takes effect only after X509_USER_PROXY environment variable and default /tmp/x509up_u`id -u` location cannot provide a certificate) .TP .BI "--key " KEY user private key. .TP .B "-4" forces Gfal2 to use IPv4 addresses only. (Option valid only for GridFTP protocol) .TP .B "-6" forces Gfal2 to use IPv6 addresses only. (Option valid only for GridFTP protocol) .TP .BI "-C, --client-info " CLIENT_INFO provide custom client-side information. .TP .BI "--log-file " LOG_FILE write Gfal2 library logs to the given file location. .TP .B "-f, --force" if destination file(s) cannot be overwritten, delete it and try again. .TP .B "-p, --parent" if the destination directory does not exist, create it. .TP .BI "-n, --nbstreams " NBSTREAMS specify the maximum number of parallel streams to use for the copy. .TP .BI "--tcp-buffersize " TCP_BUFFERSIZE specify the TCP buffersize. .TP .BI "-s, --src-spacetoken " SRC_SPACETOKEN source spacetoken to use for the transfer. .TP .BI "-S, --dst-spacetoken " DST_SPACETOKEN destination spacetoken to use for the transfer. .TP .BI "-T, --transfer-timeout " TRANSFER_TIMEOUT timeout for the transfer operation. This timeout is enforced by each plugin, so it applies to the transfer operation only. Ensure it is equal or smaller than "--timeout" setting. .TP .BI "-K, --checksum " CHECKSUM checksum algorithm to use, or algorithm:value. .TP .BI "--checksum-mode " CHECKSUM_MODE checksum validation mode: 'source', 'target' or 'both'. .TP .BI "--from-file " FROM_FILE read sources from a file. .TP .BI "--copy-mode " COPY_MODE select the TPC copy mode: 'pull', 'push' or 'streamed'. (Option valid only for HTTP protocol) .TP .B "--just-copy" just do the copy and skip any preparation (i.e. checksum, overwrite, stats etc.). Warning: use of this switch is incompatible with the presence of directories in the argument list. .TP .B "--no-delegation" disable TPC with proxy delegation. .TP .B "--evict" evict source file from disk buffer when the transfer is finished. .TP .B "--scitag" SCITAG SciTag transfer flow identifier (number in [65-65535] range) (available only for HTTP-TPC) .TP .B "-r, --recursive" copy directories recursively. .TP .B "--abort-on-failure" abort the whole copy as soon as one failure is encountered. .TP .B "--dry-run" print what would have happened without any actual change. .SH EXAMPLES .TP Copy a file to srm://endpoint.cern.ch/path/group, preferring a protocol as specified in the configuration file: .B gfal-copy file:///etc/group srm://endpoint.cern.ch/path/ .PP .TP You can specify directly a protocol, if you know that it is supported by the end point: .B gfal-copy file:///tmp/test gsiftp://endpoint.cern.ch/path/test .PP .TP .TP Do a chained copy: .B gfal-copy file:///tmp/test gsiftp://endpoint.cern.ch/path/test file:///tmp/test2 .PP .TP Registration of a file in the LFC: .B gfal-copy file:///tmp/source gsiftp://endpoint.cern.ch/path/test lfc://lfc-host.cern.ch/lfn .SH FILES .I /etc/gfal2.d/* .RS All configuration files in this directory affect the behavior of the underlying gfal2, in particular, the default choice of transfer protocols. .SH AUTHORS Duarte Meneses .br Adrien Devresse .br Alejandro Alvarez Ayllon .br Andrea Manzi .br Mihai Patrascoiu .br Joao Lopes .SH "SEE ALSO" .BR gfal-cat (1), .BR gfal-chmod (1), .BR gfal-copy (1), .BR gfal-ls (1), .BR gfal-mkdir (1), .BR gfal-rename (1), .BR gfal-rm (1), .BR gfal-save (1), .BR gfal-stat (1), .BR gfal-sum (1), .BR gfal-token (1), .BR gfal-xattr (1), .BR gfal-evict (1), .BR gfal-bringonline (1), .BR gfal-archivepoll (1) gfal2-util-v1.8.1/doc/man/gfal-evict.1000066400000000000000000000032521453604140300173020ustar00rootroot00000000000000.\" Manpage for gfal-evict .\" .TH GFAL-EVICT 1 "April 2021" "v1.8.0" .SH NAME gfal-evict \- Evict file from a disk buffer .SH SYNOPSIS .B gfal-evict [ .I "OPTION" ]... .I FILE [ .I TOKEN ] .SH DESCRIPTION .B gfal-evict This command evicts a file from a disk buffer .SH OPTIONS .TP .B "-h, --help" show help about the command. .TP .B "-V, --version" output version information and exit. .TP .B "-v, --verbose" verbose mode. Repeat up to 3 times for the most verbose mode. .TP .B "-D, --definition" define properties for gfal2. .TP .B "-C, --client-info" provide custom client-side information .TP .B "-t, --timeout" global timeout for the execution of the command. Command is interrupted if time expires before it finishes. .TP .B "-E, --cert" user certificate .TP .B "--key" user private key .TP .B FILE URI to the file to be evicted .TP .B TOKEN The token from the bring online request. Optional argument for some protocols (For example: xrootd) .TP .SH EXAMPLES .TP Evict a file from a disk buffer .B gfal-evict https://endpoint.cern.ch/path/file .SH FILES .I /etc/gfal2.d/* .RS All configuration files in this directory affect the behavior of the underlying gfal2. .SH AUTHORS Duarte Meneses .br Adrien Devresse .br Alejandro Alvarez Ayllon .br Mihai Patrascoiu .br Joao Lopes .SH "SEE ALSO" .BR gfal-cat (1), .BR gfal-chmod (1), .BR gfal-copy (1), .BR gfal-ls (1), .BR gfal-mkdir (1), .BR gfal-rename (1), .BR gfal-rm (1), .BR gfal-save (1), .BR gfal-stat (1), .BR gfal-sum (1), .BR gfal-xattr (1) .BR gfal-bringonline (1) .BR gfal-archivepoll (1) gfal2-util-v1.8.1/doc/man/gfal-legacy-register.1000066400000000000000000000024741453604140300212630ustar00rootroot00000000000000.\" Manpage for gfal-save .\" .TH GFAL-LEGACY-REGISTER 1 "May 2017" "v1.5.0" .SH NAME gfal-legacy-register \- register a replica .SH SYNOPSIS .B gfal-legacy-register [ .B "-h" ] [ .B -V ] [ .B -v ] [ .B -D .I DEFINITION ] [ .B -C .I CLIENT-INFO ] [ .B -t .I TIMEOUT ] \fI lfc surl\fR .SH DESCRIPTION .B gfal-legacy-register Register a new replica into a catalog .SH OPTIONS .TP .B "-h, --help" show help about the command. .TP .B "-V, --version" output version information and exit. .TP .B "-v, --verbose" verbose mode. Repeat up to 3 times for the most verbose mode. .TP .B "-D, --definition" define properties for gfal2. .TP .B "-C, --client-info" provide custom client-side information .TP .B "-t, --timeout" global timeout for the execution of the command. Command is interrupted if time expires before it finishes. .TP .B lfc LFC entry (lfc:// or guid:) .TP .B surl Site URL .SH EXAMPLES .TP .B gfal-legacy-register guid:12345-678-abcdef srm://endpoint.cern.ch/path/group .SH FILES .I /etc/gfal2.d/* .RS All configuration files in this directory affect the behavior of the underlying gfal2. .SH AUTHORS Duarte Meneses .br Adrien Devresse .br Alejandro Alvarez Ayllon .SH "SEE ALSO" .BR gfal-legacy-replicas (1), .BR gfal-legacy-unregister (1) gfal2-util-v1.8.1/doc/man/gfal-legacy-replicas.1000066400000000000000000000024371453604140300212400ustar00rootroot00000000000000.\" Manpage for gfal-save .\" .TH GFAL-LEGACY-REPLICAS 1 "May 2017" "v1.5.0" .SH NAME gfal-legacy-replicas \- list replicas .SH SYNOPSIS .B gfal-legacy-replicas [ .B "-h" ] [ .B -V ] [ .B -v ] [ .B -D .I DEFINITION ] [ .B -C .I CLIENT-INFO ] [ .B -t .I TIMEOUT ] \fI lfc .SH DESCRIPTION .B gfal-legacy-replicas List the replicas of an entry in a catalog. .SH OPTIONS .TP .B "-h, --help" show help about the command. .TP .B "-V, --version" output version information and exit. .TP .B "-v, --verbose" verbose mode. Repeat up to 3 times for the most verbose mode. .TP .B "-D, --definition" define properties for gfal2. .TP .B "-C, --client-info" provide custom client-side information .TP .B "-t, --timeout" global timeout for the execution of the command. Command is interrupted if time expires before it finishes. .TP .B lfc LFC entry (lfc:// or guid:) .SH EXAMPLES .B gfal-legacy-replicas guid:12345-678-abcdef .TP srm://endpoint.cern.ch/path/group .SH FILES .I /etc/gfal2.d/* .RS All configuration files in this directory affect the behavior of the underlying gfal2. .SH AUTHORS Duarte Meneses .br Adrien Devresse .br Alejandro Alvarez Ayllon .SH "SEE ALSO" .BR gfal-legacy-register (1), .BR gfal-legacy-unregister (1)gfal2-util-v1.8.1/doc/man/gfal-legacy-unregister.1000066400000000000000000000024771453604140300216310ustar00rootroot00000000000000.\" Manpage for gfal-save .\" .TH GFAL-LEGACY-UNREGISTER 1 "May 2017" "v1.5.0" .SH NAME gfal-legacy-unregister \- register a replica .SH SYNOPSIS .B gfal-legacy-unregister [ .B "-h" ] [ .B -V ] [ .B -v ] [ .B -D .I DEFINITION ] [ .B -C .I CLIENT-INFO ] [ .B -t .I TIMEOUT ] \fI lfc surl\fR .SH DESCRIPTION .B gfal-legacy-unregister Removes a replica from a catalog .SH OPTIONS .TP .B "-h, --help" show help about the command. .TP .B "-V, --version" output version information and exit. .TP .B "-v, --verbose" verbose mode. Repeat up to 3 times for the most verbose mode. .TP .B "-D, --definition" define properties for gfal2. .TP .B "-C, --client-info" provide custom client-side information .TP .B "-t, --timeout" global timeout for the execution of the command. Command is interrupted if time expires before it finishes. .TP .B lfc LFC entry (lfc:// or guid:) .TP .B surl Site URL .SH EXAMPLES .TP .B gfal-legacy-unregister guid:12345-678-abcdef srm://endpoint.cern.ch/path/group .SH FILES .I /etc/gfal2.d/* .RS All configuration files in this directory affect the behavior of the underlying gfal2. .SH AUTHORS Duarte Meneses .br Adrien Devresse .br Alejandro Alvarez Ayllon .SH "SEE ALSO" .BR gfal-legacy-replicas (1), .BR gfal-legacy-register (1) gfal2-util-v1.8.1/doc/man/gfal-ls.1000066400000000000000000000064021453604140300166060ustar00rootroot00000000000000.\" Manpage for gfal-ls .\" .TH GFAL-LS 1 "March 2022" "v1.7.1" .SH NAME gfal-ls \- list directory contents or file information .SH SYNOPSIS .B gfal-ls [ .I "OPTION" ]... .I FILE .SH DESCRIPTION .B gfal-ls List information about the file. Depending on the implementation, some information from the long listing, such as group and user ids, might not be available and will be displayed as 0. When listing several files or directories, if one returns an error, the execution is canceled. .SH OPTIONS .TP .B "-h, --help" show help about the command. .TP .B "-V, --version" output version information and exit. .TP .B "-v, --verbose" verbose mode. Repeat up to 3 times for the most verbose mode. .TP .BI "-D, --definition " DEFINITION define properties for gfal2. Definitions have the following syntax: "CONFIG_GROUP:KEY=VALUE" .TP .BI "-t, --timeout " TIMEOUT global timeout for the execution of the command. Command is interrupted if time expires before it finishes. Note: A grace period of 30 seconds is given to the global timeout. .TP .BI "-E, --cert " CERT user certificate. (This option takes effect only after X509_USER_PROXY environment variable and default /tmp/x509up_u`id -u` location cannot provide a certificate) .TP .BI "--key " KEY user private key. .TP .B "-4" forces Gfal2 to use IPv4 addresses only. (Option valid only for GridFTP protocol) .TP .B "-6" forces Gfal2 to use IPv6 addresses only. (Option valid only for GridFTP protocol) .TP .BI "-C, --client-info " CLIENT_INFO provide custom client-side information. .TP .BI "--log-file " LOG_FILE write Gfal2 library logs to the given file location. .TP .B "-a, --all" don't hide files with name starting in '.'. .TP .B "-l, --long" long listing format. Following columns are displayed for each entry: mode, number of links, group id, user id, last modification time, size and name. .TP .B "-d, --directory" list directory entries instead of its contents. .TP .B "-H, --human-readable" with -l, print sizes in human readable format (e.g.: 1K, 234M, 2G). .TP .BI "--xattr " XATTR with -l, query additional attributes. Can be specified multiple times. .TP .BI "--time-style " STYLE specify the time style. Supports the following values: locale, iso, long-iso or full-iso (partially). .TP .B "--full-time" equivalent to --time-style=full-iso. .TP .BI "--color " MODE with -l, enable or disable color output. Supports the following values: always, never, auto. .SH EXAMPLES .TP Long listing of all files: .B gfal-ls -laH srm://endpoint.cern.ch/path/file .PP .TP List a directory instead of its contents: .B gfal-ls -d file:///tmp .SH FILES .I /etc/gfal2.d/* .RS All configuration files in this directory affect the behavior of the underlying gfal2. .SH AUTHORS Duarte Meneses .br Adrien Devresse .br Alejandro Alvarez Ayllon .br Andrea Manzi .br Mihai Patrascoiu .br Joao Lopes .SH "SEE ALSO" .BR gfal-cat (1), .BR gfal-chmod (1), .BR gfal-copy (1), .BR gfal-ls (1), .BR gfal-mkdir (1), .BR gfal-rename (1), .BR gfal-rm (1), .BR gfal-save (1), .BR gfal-stat (1), .BR gfal-sum (1), .BR gfal-token (1), .BR gfal-xattr (1), .BR gfal-evict (1), .BR gfal-bringonline (1), .BR gfal-archivepoll (1) gfal2-util-v1.8.1/doc/man/gfal-mkdir.1000066400000000000000000000047771453604140300173130ustar00rootroot00000000000000.\" Manpage for gfal-mkdir .\" .TH GFAL-MKDIR 1 "March 2022" "v1.7.1" .SH NAME gfal-mkdir \- make directories .SH SYNOPSIS .B gfal-mkdir [ .I "OPTION" ]... \fIDIRECTORY\fR... .SH DESCRIPTION .B gfal-mkdir Create the DIRECTORY(ies), if they do not already exist. By default, it creates them with mode 0755. .SH OPTIONS .TP .B "-h, --help" show help about the command. .TP .B "-V, --version" output version information and exit. .TP .B "-v, --verbose" verbose mode. Repeat up to 3 times for the most verbose mode. .TP .BI "-D, --definition " DEFINITION define properties for gfal2. Definitions have the following syntax: "CONFIG_GROUP:KEY=VALUE" .TP .BI "-t, --timeout " TIMEOUT global timeout for the execution of the command. Command is interrupted if time expires before it finishes. Note: A grace period of 30 seconds is given to the global timeout. .TP .BI "-E, --cert " CERT user certificate. (This option takes effect only after X509_USER_PROXY environment variable and default /tmp/x509up_u`id -u` location cannot provide a certificate) .TP .BI "--key " KEY user private key. .TP .B "-4" forces Gfal2 to use IPv4 addresses only. (Option valid only for GridFTP protocol) .TP .B "-6" forces Gfal2 to use IPv6 addresses only. (Option valid only for GridFTP protocol) .TP .BI "-C, --client-info " CLIENT_INFO provide custom client-side information. .TP .BI "--log-file " LOG_FILE write Gfal2 library logs to the given file location. .TP .BI "-m, --mode " MODE set file mode (default is 0755). Depending on the storage implementation, it might not be used. .TP .B "-p, --parents" make parent directories as needed. Reports no error if directory exists. .SH EXAMPLES .TP .B gfal-mkdir -p -m 0700 file:///tmp/path1/path2 .PP .TP .B gfal-mkdir srm://endpoint.cern.ch/path/directory1 srm://endpoint.cern.ch/path/directory2 .SH FILES .I /etc/gfal2.d/* .RS All configuration files in this directory affect the behavior of the underlying gfal2. .SH AUTHORS Duarte Meneses .br Adrien Devresse .br Alejandro Alvarez Ayllon .br Andrea Manzi .br Mihai Patrascoiu .br Joao Lopes .SH "SEE ALSO" .BR gfal-cat (1), .BR gfal-chmod (1), .BR gfal-copy (1), .BR gfal-ls (1), .BR gfal-mkdir (1), .BR gfal-rename (1), .BR gfal-rm (1), .BR gfal-save (1), .BR gfal-stat (1), .BR gfal-sum (1), .BR gfal-token (1), .BR gfal-xattr (1), .BR gfal-evict (1), .BR gfal-bringonline (1), .BR gfal-archivepoll (1) gfal2-util-v1.8.1/doc/man/gfal-rename.1000066400000000000000000000042301453604140300174340ustar00rootroot00000000000000.\" Manpage for gfal-rename .\" .TH GFAL-RENAME 1 "March 2022" "v1.7.1" .SH NAME gfal-rename \- Renames files or directories .SH SYNOPSIS .B gfal-rename [ .I "OPTION" ]... .I SOURCE .I DESTINATION .SH DESCRIPTION .B gfal-rename Renames SOURCE to DESTINATION. .SH OPTIONS .TP .B "-h, --help" show help about the command. .TP .B "-V, --version" output version information and exit. .TP .B "-v, --verbose" verbose mode. Repeat up to 3 times for the most verbose mode. .TP .BI "-D, --definition " DEFINITION define properties for gfal2. Definitions have the following syntax: "CONFIG_GROUP:KEY=VALUE" .TP .BI "-t, --timeout " TIMEOUT global timeout for the execution of the command. Command is interrupted if time expires before it finishes. Note: A grace period of 30 seconds is given to the global timeout. .TP .BI "-E, --cert " CERT user certificate. (This option takes effect only after X509_USER_PROXY environment variable and default /tmp/x509up_u`id -u` location cannot provide a certificate) .TP .BI "--key " KEY user private key. .TP .B "-4" forces Gfal2 to use IPv4 addresses only. (Option valid only for GridFTP protocol) .TP .B "-6" forces Gfal2 to use IPv6 addresses only. (Option valid only for GridFTP protocol) .TP .BI "-C, --client-info " CLIENT_INFO provide custom client-side information. .TP .BI "--log-file " LOG_FILE write Gfal2 library logs to the given file location. .SH EXAMPLES .TP .B gfal-rename file:///tmp/original file:///tmp/newname .SH FILES .I /etc/gfal2.d/* .RS All configuration files in this directory affect the behavior of the underlying gfal2. .SH AUTHORS Duarte Meneses .br Adrien Devresse .br Alejandro Alvarez Ayllon .br Andrea Manzi .br Mihai Patrascoiu .br Joao Lopes .SH "SEE ALSO" .BR gfal-cat (1), .BR gfal-chmod (1), .BR gfal-copy (1), .BR gfal-ls (1), .BR gfal-mkdir (1), .BR gfal-rename (1), .BR gfal-rm (1), .BR gfal-save (1), .BR gfal-stat (1), .BR gfal-sum (1), .BR gfal-token (1), .BR gfal-xattr (1), .BR gfal-evict (1), .BR gfal-bringonline (1), .BR gfal-archivepoll (1) gfal2-util-v1.8.1/doc/man/gfal-rm.1000066400000000000000000000053051453604140300166070ustar00rootroot00000000000000.\" Manpage for gfal-rm .\" .TH GFAL-RM 1 "March 2022" "v1.7.1" .SH NAME gfal-rm \- Remove files or directories .SH SYNOPSIS .B gfal-rm [ .I "OPTION" ]... \fIFILE\fR... .SH DESCRIPTION .B gfal-rm Removes each specified file or directory. By default, it does not remove directories. To delete a directory, --recursiv needs to be given, in which case all given directories are removed too along with all their contents. .SH OPTIONS .TP .B "-h, --help" show help about the command. .TP .B "-V, --version" output version information and exit. .TP .B "-v, --verbose" verbose mode. Repeat up to 3 times for the most verbose mode. .TP .BI "-D, --definition " DEFINITION define properties for gfal2. Definitions have the following syntax: "CONFIG_GROUP:KEY=VALUE" .TP .BI "-t, --timeout " TIMEOUT global timeout for the execution of the command. Command is interrupted if time expires before it finishes. Note: A grace period of 30 seconds is given to the global timeout. .TP .BI "-E, --cert " CERT user certificate. (This option takes effect only after X509_USER_PROXY environment variable and default /tmp/x509up_u`id -u` location cannot provide a certificate) .TP .BI "--key " KEY user private key. .TP .B "-4" forces Gfal2 to use IPv4 addresses only. (Option valid only for GridFTP protocol) .TP .B "-6" forces Gfal2 to use IPv6 addresses only. (Option valid only for GridFTP protocol) .TP .BI "-C, --client-info " CLIENT_INFO provide custom client-side information. .TP .BI "--log-file " LOG_FILE write Gfal2 library logs to the given file location. .TP .B "-r, -R, --recursive" remove directories and their contents recursively. .TP .B "--dry-run" print what would have happened without any actual change. .TP .B "--just-delete" do not perform any check on the file prior to delete. (Needed to delete HTTP signed URLs) .TP .BI "--from-file " FROM_FILE read SURLs from a file. .TP .B "--bulk" perform bulk deletion. .SH EXAMPLES .TP .B gfal-rm file:///tmp/test .PP .TP .B gfal-rm -r srm://endpoint.cern.ch/path/directory .SH FILES .I /etc/gfal2.d/* .RS All configuration files in this directory affect the behavior of the underlying gfal2. .SH AUTHORS Duarte Meneses .br Adrien Devresse .br Alejandro Alvarez Ayllon .br Andrea Manzi .br Mihai Patrascoiu .br Joao Lopes .SH "SEE ALSO" .BR gfal-cat (1), .BR gfal-chmod (1), .BR gfal-copy (1), .BR gfal-ls (1), .BR gfal-mkdir (1), .BR gfal-rename (1), .BR gfal-rm (1), .BR gfal-save (1), .BR gfal-stat (1), .BR gfal-sum (1), .BR gfal-token (1), .BR gfal-xattr (1), .BR gfal-evict (1), .BR gfal-bringonline (1), .BR gfal-archivepoll (1) gfal2-util-v1.8.1/doc/man/gfal-save.1000066400000000000000000000043421453604140300171270ustar00rootroot00000000000000.\" Manpage for gfal-save .\" .TH GFAL-SAVE 1 "March 2022" "v1.7.1" .SH NAME gfal-save \- read from standard input and write it to a file .SH SYNOPSIS .B gfal-save [ .I "OPTION" ]... .I FILE .SH DESCRIPTION .B gfal-save Reads from stdin and writes to a file until it finds EOF. If the file exists, it will be overwritten. .SH OPTIONS .TP .B "-h, --help" show help about the command. .TP .B "-V, --version" output version information and exit. .TP .B "-v, --verbose" verbose mode. Repeat up to 3 times for the most verbose mode. .TP .BI "-D, --definition " DEFINITION define properties for gfal2. Definitions have the following syntax: "CONFIG_GROUP:KEY=VALUE" .TP .BI "-t, --timeout " TIMEOUT global timeout for the execution of the command. Command is interrupted if time expires before it finishes. Note: A grace period of 30 seconds is given to the global timeout. .TP .BI "-E, --cert " CERT user certificate. (This option takes effect only after X509_USER_PROXY environment variable and default /tmp/x509up_u`id -u` location cannot provide a certificate) .TP .BI "--key " KEY user private key. .TP .B "-4" forces Gfal2 to use IPv4 addresses only. (Option valid only for GridFTP protocol) .TP .B "-6" forces Gfal2 to use IPv6 addresses only. (Option valid only for GridFTP protocol) .TP .BI "-C, --client-info " CLIENT_INFO provide custom client-side information. .TP .BI "--log-file " LOG_FILE write Gfal2 library logs to the given file location. .SH EXAMPLES .TP .B cat /etc/group | .B gfal-save srm://endpoint.cern.ch/path/group .SH FILES .I /etc/gfal2.d/* .RS All configuration files in this directory affect the behavior of the underlying gfal2. .SH AUTHORS Duarte Meneses .br Adrien Devresse .br Alejandro Alvarez Ayllon .br Andrea Manzi .br Mihai Patrascoiu .br Joao Lopes .SH "SEE ALSO" .BR gfal-cat (1), .BR gfal-chmod (1), .BR gfal-copy (1), .BR gfal-ls (1), .BR gfal-mkdir (1), .BR gfal-rename (1), .BR gfal-rm (1), .BR gfal-save (1), .BR gfal-stat (1), .BR gfal-sum (1), .BR gfal-token (1), .BR gfal-xattr (1), .BR gfal-evict (1), .BR gfal-bringonline (1), .BR gfal-archivepoll (1) gfal2-util-v1.8.1/doc/man/gfal-stat.1000066400000000000000000000052341453604140300171450ustar00rootroot00000000000000.\" Manpage for gfal-stat .\" .TH GFAL-STAT 1 "March 2022" "v1.7.1" .SH NAME gfal-stat \- display extended information about a file or directory .SH SYNOPSIS .B gfal-stat [ .I "OPTION" ]... .I URL .SH DESCRIPTION .B gfal-stat Display extended information about a file or directory. .SH OPTIONS .TP .B "-h, --help" show help about the command. .TP .B "-V, --version" output version information and exit. .TP .B "-v, --verbose" verbose mode. Repeat up to 3 times for the most verbose mode. .TP .BI "-D, --definition " DEFINITION define properties for gfal2. Definitions have the following syntax: "CONFIG_GROUP:KEY=VALUE" .TP .BI "-t, --timeout " TIMEOUT global timeout for the execution of the command. Command is interrupted if time expires before it finishes. Note: A grace period of 30 seconds is given to the global timeout. .TP .BI "-E, --cert " CERT user certificate. (This option takes effect only after X509_USER_PROXY environment variable and default /tmp/x509up_u`id -u` location cannot provide a certificate) .TP .BI "--key " KEY user private key. .TP .B "-4" forces Gfal2 to use IPv4 addresses only. (Option valid only for GridFTP protocol) .TP .B "-6" forces Gfal2 to use IPv6 addresses only. (Option valid only for GridFTP protocol) .TP .BI "-C, --client-info " CLIENT_INFO provide custom client-side information. .TP .BI "--log-file " LOG_FILE write Gfal2 library logs to the given file location. .SH EXAMPLES .B gfal-stat file:///tmp/path1/path2 File: 'file:///tmp/path1/path2' Size: 1986 directory Access: (1777/drwxrwxrwx) Uid: 0 Gid: 0 Access: 2017-05-10 13:15:35.000000 Modify: 2017-05-10 13:24:04.000000 Change: 2017-05-10 13:24:04.000000 .PP .B gfal-stat srm://endpoint.cern.ch/path/directory1 File: 'srm://endpoint.cern.ch/path/directory1' Size: 1986 directory Access: (1777/drwxrwxrwx) Uid: 0 Gid: 0 Access: 2017-05-10 13:15:35.000000 Modify: 2017-05-10 13:24:04.000000 Change: 2017-05-10 13:24:04.000000 .SH FILES .I /etc/gfal2.d/* .RS All configuration files in this directory affect the behavior of the underlying gfal2. .SH AUTHORS Duarte Meneses .br Adrien Devresse .br Alejandro Alvarez Ayllon .br Andrea Manzi .br Mihai Patrascoiu .br Joao Lopes .SH "SEE ALSO" .BR gfal-cat (1), .BR gfal-chmod (1), .BR gfal-copy (1), .BR gfal-ls (1), .BR gfal-mkdir (1), .BR gfal-rename (1), .BR gfal-rm (1), .BR gfal-save (1), .BR gfal-stat (1), .BR gfal-sum (1), .BR gfal-token (1), .BR gfal-xattr (1), .BR gfal-evict (1), .BR gfal-bringonline (1), .BR gfal-archivepoll (1) gfal2-util-v1.8.1/doc/man/gfal-sum.1000066400000000000000000000044501453604140300167750ustar00rootroot00000000000000.\" Manpage for gfal-sum .\" .TH GFAL-SUM 1 "March 2022" "v1.7.1" .SH NAME gfal-sum \- Calculate checksum of a file .SH SYNOPSIS .B gfal-sum [ .I "OPTION" ] .I "FILE CHECKSUM_TYPE" .SH DESCRIPTION .B gfal-sum Calculates the checksum of the specified file, using a specified checksum algorithm. The supported algorithm depend on the protocol and endpoint. Some examples are .I ADLER32 , .I CRC32 and .I MD5 .SH OPTIONS .TP .B "-h, --help" show help about the command. .TP .B "-V, --version" output version information and exit. .TP .B "-v, --verbose" verbose mode. Repeat up to 3 times for the most verbose mode. .TP .BI "-D, --definition " DEFINITION define properties for gfal2. Definitions have the following syntax: "CONFIG_GROUP:KEY=VALUE" .TP .BI "-t, --timeout " TIMEOUT global timeout for the execution of the command. Command is interrupted if time expires before it finishes. Note: A grace period of 30 seconds is given to the global timeout. .TP .BI "-E, --cert " CERT user certificate. (This option takes effect only after X509_USER_PROXY environment variable and default /tmp/x509up_u`id -u` location cannot provide a certificate) .TP .BI "--key " KEY user private key. .TP .B "-4" forces Gfal2 to use IPv4 addresses only. (Option valid only for GridFTP protocol) .TP .B "-6" forces Gfal2 to use IPv6 addresses only. (Option valid only for GridFTP protocol) .TP .BI "-C, --client-info " CLIENT_INFO provide custom client-side information. .TP .BI "--log-file " LOG_FILE write Gfal2 library logs to the given file location. .SH EXAMPLES .TP .B gfal-sum srm://endpoint.cern.ch/path/file ADLER32 .SH FILES .I /etc/gfal2.d/* .RS All configuration files in this directory affect the behavior of the underlying gfal2. .SH AUTHORS Duarte Meneses .br Adrien Devresse .br Alejandro Alvarez Ayllon .br Andrea Manzi .br Mihai Patrascoiu .br Joao Lopes .SH "SEE ALSO" .BR gfal-cat (1), .BR gfal-chmod (1), .BR gfal-copy (1), .BR gfal-ls (1), .BR gfal-mkdir (1), .BR gfal-rename (1), .BR gfal-rm (1), .BR gfal-save (1), .BR gfal-stat (1), .BR gfal-sum (1), .BR gfal-token (1), .BR gfal-xattr (1), .BR gfal-evict (1), .BR gfal-bringonline (1), .BR gfal-archivepoll (1) gfal2-util-v1.8.1/doc/man/gfal-token.1000066400000000000000000000063441453604140300173150ustar00rootroot00000000000000.\" Manpage for gfal-token .\" .TH GFAL-TOKEN 1 "March 2022" "v1.7.1" .SH NAME gfal-token \- retrieve SE-issued token for a given path .SH SYNOPSIS .B gfal-token [ .I "OPTION" ]... .I PATH [ .I ACTIVITY ]... .SH DESCRIPTION .B gfal-token This command retrieves a Storage Element issued token for a given path. Additionally, a token issuer URL may be passed. The command can take either a list of user-defined activities or use predefined activities for the read/write access flag. Behind the scenes, Gfal2 will try to first retrieve a SciTokens-style token, followed by a macaroon-style token. If at any point, a token is successfully retrieved, operation stops and the token is printed. .SH OPTIONS .TP .B "-h, --help" show help about the command. .TP .B "-V, --version" output version information and exit. .TP .B "-v, --verbose" verbose mode. Repeat up to 3 times for the most verbose mode. .TP .BI "-D, --definition " DEFINITION define properties for gfal2. Definitions have the following syntax: "CONFIG_GROUP:KEY=VALUE" .TP .BI "-t, --timeout " TIMEOUT global timeout for the execution of the command. Command is interrupted if time expires before it finishes. Note: A grace period of 30 seconds is given to the global timeout. .TP .BI "-E, --cert " CERT user certificate. (This option takes effect only after X509_USER_PROXY environment variable and default /tmp/x509up_u`id -u` location cannot provide a certificate) .TP .BI "--key " KEY user private key. .TP .B "-4" forces Gfal2 to use IPv4 addresses only. (Option valid only for GridFTP protocol) .TP .B "-6" forces Gfal2 to use IPv6 addresses only. (Option valid only for GridFTP protocol) .TP .BI "-C, --client-info " CLIENT_INFO provide custom client-side information. .TP .BI "--log-file " LOG_FILE write Gfal2 library logs to the given file location. .TP .BI "--issuer " ISSUER token issuer endpoint. .TP .BI "--validity " VALIDITY token validity expressed in minutes. .TP .B "-w, --write" token read/write access flag. Default activities associated with these flags: read = [LIST,DOWNLOAD], write = [LIST,DOWNLOAD,MANAGE,UPLOAD,DELETE] .TP .B PATH resource URI for which token is requested. .TP .B ACTIVITY optional argument, user-specified activity list. This list takes precedence over the read/write access flag default activities. .SH EXAMPLES .TP Retrieve a SE-issued token using default write activities .B gfal-token --write https://endpoint.cern.ch/path/file .PP .TP Retrieve a SE-issued token using custom activities .B gfal-token https://endpoint.cern.ch/path/file LIST MANAGE UPLOAD .SH FILES .I /etc/gfal2.d/* .RS All configuration files in this directory affect the behavior of the underlying gfal2. .SH AUTHORS Duarte Meneses .br Adrien Devresse .br Alejandro Alvarez Ayllon .br Andrea Manzi .br Mihai Patrascoiu .br Joao Lopes .SH "SEE ALSO" .BR gfal-cat (1), .BR gfal-chmod (1), .BR gfal-copy (1), .BR gfal-ls (1), .BR gfal-mkdir (1), .BR gfal-rename (1), .BR gfal-rm (1), .BR gfal-save (1), .BR gfal-stat (1), .BR gfal-sum (1), .BR gfal-token (1), .BR gfal-xattr (1), .BR gfal-evict (1), .BR gfal-bringonline (1), .BR gfal-archivepoll (1) gfal2-util-v1.8.1/doc/man/gfal-xattr.1000066400000000000000000000060671453604140300173410ustar00rootroot00000000000000.\" Manpage for gfal-xattr .\" .TH GFAL-XATTR 1 "March 2022" "v1.7.1" .SH NAME gfal-xattr \- show or set attributes of a file .SH SYNOPSIS .B gfal-xattr [ .I "OPTION" ]... .I FILE [ .I ATTRIBUTE [ = .I VALUE ]] .SH DESCRIPTION .B gfal-xattr Display attributes of a file or set them to a new value. This command comes in three forms, making it possible, for a specified file, to list all attribute and corresponding values, display the value of a particular attribute or assign a new value to a particular attribute - see examples. .SH OPTIONS .TP .B "-h, --help" show help about the command. .TP .B "-V, --version" output version information and exit. .TP .B "-v, --verbose" verbose mode. Repeat up to 3 times for the most verbose mode. .TP .BI "-D, --definition " DEFINITION define properties for gfal2. Definitions have the following syntax: "CONFIG_GROUP:KEY=VALUE" .TP .BI "-t, --timeout " TIMEOUT global timeout for the execution of the command. Command is interrupted if time expires before it finishes. Note: A grace period of 30 seconds is given to the global timeout. .TP .BI "-E, --cert " CERT user certificate. (This option takes effect only after X509_USER_PROXY environment variable and default /tmp/x509up_u`id -u` location cannot provide a certificate) .TP .BI "--key " KEY user private key. .TP .B "-4" forces Gfal2 to use IPv4 addresses only. (Option valid only for GridFTP protocol) .TP .B "-6" forces Gfal2 to use IPv6 addresses only. (Option valid only for GridFTP protocol) .TP .BI "-C, --client-info " CLIENT_INFO provide custom client-side information. .TP .BI "--log-file " LOG_FILE write Gfal2 library logs to the given file location. .TP .B FILE file URI .TP .B ATTRIBUTE optional argument, specifying an attribute to get or set. If no attribute is given, all attributes and corresponding values are displayed. .TP .B VALUE if an attribute is given, a new value can optionally be assigned to it. .SH EXAMPLES .TP Show all attributes and values of a file: .B gfal-xattr srm://endpoint.cern.ch/path/file .PP .TP Show the value of a particular attribute: .B gfal-xattr srm://endpoint.cern.ch/path/file user.status .PP .TP Set a new value to an attribute: .B gfal-xattr srm://endpoint.cern.ch/path/file user.status=new_value .PP .TP Typically, you can get the transport urls (turls) with the "user.replicas" attribute: srm://endpoint.cern.ch/path/file user.replicas .SH FILES .I /etc/gfal2.d/* .RS All configuration files in this directory affect the behavior of the underlying gfal2. .SH AUTHORS Duarte Meneses .br Adrien Devresse .br Alejandro Alvarez Ayllon .br Andrea Manzi .br Mihai Patrascoiu .br Joao Lopes .SH "SEE ALSO" .BR gfal-cat (1), .BR gfal-chmod (1), .BR gfal-copy (1), .BR gfal-ls (1), .BR gfal-mkdir (1), .BR gfal-rename (1), .BR gfal-rm (1), .BR gfal-save (1), .BR gfal-stat (1), .BR gfal-sum (1), .BR gfal-token (1), .BR gfal-xattr (1), .BR gfal-evict (1), .BR gfal-bringonline (1), .BR gfal-archivepoll (1) gfal2-util-v1.8.1/packaging/000077500000000000000000000000001453604140300156015ustar00rootroot00000000000000gfal2-util-v1.8.1/packaging/Dockerfile000066400000000000000000000013701453604140300175740ustar00rootroot00000000000000# # Copyright (c) 2016-2022 CERN # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # gfal2-util image FROM gitlab-registry.cern.ch/dmc/gfal2 RUN yum install -y lcg-CA voms-clients voms-config-all gfal2-util fts-client # Leave yum cache clean RUN yum clean all gfal2-util-v1.8.1/packaging/Makefile000066400000000000000000000043441453604140300172460ustar00rootroot00000000000000NAME=gfal2-util SPEC=rpm/$(NAME).spec VERSION=${shell grep '^Version:' $(SPEC) | awk '{print $$2}' } # Leave blank. To be overriden by CI tools. RELEASE= CWD=${shell pwd} RPMBUILD=/tmp/rpmbuild SRPMS=$(CWD) RPMS=$(CWD)/out MOCK_CHROOT=epel-6-cernonly-x86_64 MOCK_FLAGS=--verbose RPMDEFINES_SRC=--define='_topdir $(RPMBUILD)' \ --define='_sourcedir $(CWD)' \ --define='_builddir %{_topdir}/BUILD' \ --define='_srcrpmdir $(SRPMS)' \ --define='_rpmdir $(RPMS)' \ --define "_source_filedigest_algorithm md5" \ $(RPMBUILD_SRC_EXTRA_FLAGS) RPMDEFINES_BIN=--define='_topdir $(RPMBUILD)' \ --define='_sourcedir %{_topdir}/SOURCES' \ --define='_builddir %{_topdir}/BUILD' \ --define='_srcrpmdir $(SRPMS).' \ --define='_rpmdir $(RPMS)' \ --define "_binary_filedigest_algorithm md5" \ $(RPMBUILD_BIN_EXTRA_FLAGS) PBUILDER_FLAGS= PBUILDER_TMP="/tmp/" all: srpm clean: rm -fv *.tar rm -fv *.tar.gz rm -fv *.rpm rm -fv *.log rm -rfv out rm -fv *.deb rm -fv *.gz rm -fv *.dsc rm -fv *.changes rm -rfv "$(RPMBUILD)" dist: clean tar vczf "$(NAME)-$(VERSION).tar.gz" --exclude="packaging" --exclude="ci/repo" --exclude=".git*" --exclude="*.pyc" --transform="s,^,$(NAME)-$(VERSION)/," .. $(RPMBUILD): mkdir -p "$(RPMBUILD)" override_release: $(SPEC) $(if $(RELEASE), sed -i "s/Release:.*/Release: $(RELEASE)%{?dist}/g" "$(SPEC)") srpm: dist $(SPEC) $(RPMBUILD) override_release /usr/bin/rpmbuild --nodeps -bs $(RPMDEFINES_SRC) $(SPEC) rpm: srpm /usr/bin/rpmbuild --rebuild $(RPMDEFINES_BIN) $(NAME)-$(VERSION)-*.src.rpm mock: srpm /usr/bin/mock $(MOCK_FLAGS) -r $(MOCK_CHROOT) $(NAME)-$(VERSION)-*.src.rpm deb-src: dist rm -rf "$(PBUILDER_TMP)/$(NAME)-$(VERSION)" tar xzf "$(NAME)-$(VERSION).tar.gz" -C "$(PBUILDER_TMP)" cp -rv debian "$(PBUILDER_TMP)/$(NAME)-$(VERSION)" $(if $(RELEASE), sed -ri "s/($(NAME) )\((([0-9]+\.)+[0-9]+)-[0-9]+\)/\\1(\\2-$(RELEASE))/g" "$(PBUILDER_TMP)/$(NAME)-$(VERSION)/debian/changelog") cp -f "$(NAME)-$(VERSION).tar.gz" "$(PBUILDER_TMP)/$(NAME)_$(VERSION).orig.tar.gz" cd "$(PBUILDER_TMP)/$(NAME)-$(VERSION)"; \ debuild -us -uc -S mv $(PBUILDER_TMP)/$(NAME)_$(VERSION)*.gz . mv $(PBUILDER_TMP)/$(NAME)_$(VERSION)*.dsc . deb: deb-src pbuilder build $(PBUILDER_FLAGS) $(NAME)_$(VERSION)*.dsc gfal2-util-v1.8.1/packaging/debian/000077500000000000000000000000001453604140300170235ustar00rootroot00000000000000gfal2-util-v1.8.1/packaging/debian/changelog000066400000000000000000000034361453604140300207030ustar00rootroot00000000000000gfal2-util (1.8.1-1) unstable; urgency=low * New release -- DMC Devel Tue, 12 Dec 2023 12:00:00 +0100 gfal2-util (1.8.0-1) unstable; urgency=low * New release -- DMC Devel Fri, 02 Sep 2022 12:00:00 +0100 gfal2-util (1.7.1-1) unstable; urgency=low * New release -- DMC Devel Mon, 07 Mar 2022 12:00:00 +0100 gfal2-util (1.7.0-1) unstable; urgency=low * New release -- DMC Devel Thu, 23 Sep 2021 12:00:00 +0100 gfal2-util (1.6.0-1) unstable; urgency=low * New release -- DMC Devel Thu, 19 Nov 2020 16:00:00 +0100 gfal2-util (1.5.4-1) unstable; urgency=low * New release -- DMC Devel Mon, 14 Sep 2020 14:45:00 +0200 gfal2-util (1.5.3-1) unstable; urgency=medium * New release -- DMC Devel Wed, 5 Apr 2019 09:40:00 +0200 gfal2-util (1.5.2-1) unstable; urgency=medium * New release -- DMC Devel Wed, 29 Aug 2018 09:40:00 +0200 gfal2-util (1.5.1-1) unstable; urgency=medium * New release -- DMC Devel Wed, 21 Jun 2017 13:41:15 +0200 gfal2-util (1.5.0-1) unstable; urgency=low * New release -- DMC Devel Mon, 12 Sep 2016 09:40:00 +0100 gfal2-util (1.4.0-1) unstable; urgency=low * New release -- DMC Devel Tue, 17 May 2016 15:26:00 +0100 gfal2-util (1.3.2-1) unstable; urgency=low * New release -- DMC Devel Mon, 07 Mar 2016 16:00:00 +0100 gfal2-util (1.3.1-1) unstable; urgency=low * New release -- DMC Devel Mon, 09 Nov 2015 10:55:00 +0100 gfal2-util (1.3.0-1) unstable; urgency=low * Deb packaging on CI -- DMC Devel Thu, 15 Jul 2015 10:00:00 +0200 gfal2-util-v1.8.1/packaging/debian/compat000066400000000000000000000000021453604140300202210ustar00rootroot000000000000008 gfal2-util-v1.8.1/packaging/debian/control000066400000000000000000000012661453604140300204330ustar00rootroot00000000000000Source: gfal2-util Priority: optional Maintainer: DMC Devel Build-Depends: debhelper (>= 8.0.0), python (>= 2.7), gfal2-python (>= 1.8.0), gfal2-plugin-file Standards-Version: 3.9.5 Section: net Homepage: http://dmc.web.cern.ch/ Package: gfal2-util Architecture: any Depends: gfal2-python (>= 1.8.0) Recommends: gfal2-plugin-file, gfal2-plugin-lfc, gfal2-plugin-dcap, gfal2-plugin-srm, gfal2-plugin-gridftp, gfal2-plugin-http Description: GFAL2 utility tools gfal2-util is a set of basic utility tools for file interactions and file copy based on the GFAL 2.0 toolkit. gfal2-util supports the protocols of GFAL 2.0 : WebDav(s), gridFTP, http(s), SRM, xrootd, etc... gfal2-util-v1.8.1/packaging/debian/copyright000066400000000000000000000022021453604140300207520ustar00rootroot00000000000000Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ Upstream-Name: gfal2-util Source: https://gitlab.cern.ch/dmc/gfal2-util/repository/archive.tar.gz Files: * Copyright: Copyright (c) Members of the EGEE Collaboration. 2004-2013. See http://www.eu-egee.org/partners/ for details on the copyright holders. Copyright (c) CERN 2013-2012 License: Apache-2.0 Files: debian/* Copyright: 2015, DMC Devel / CERN License: Apache-2.0 License: Apache-2.0 Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at . http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. . On Debian systems the full text of the Apache-2.0 license can be found in the /usr/share/common-licenses/Apache-2.0 file. gfal2-util-v1.8.1/packaging/debian/gfal2-util.docs000066400000000000000000000000371453604140300216430ustar00rootroot00000000000000RELEASE-NOTES VERSION LICENSE gfal2-util-v1.8.1/packaging/debian/gfal2-util.install000066400000000000000000000001221453604140300223540ustar00rootroot00000000000000usr/bin/gfal-* usr/share/man/man1/* usr/lib/python2.7/dist-packages/gfal2_util/* gfal2-util-v1.8.1/packaging/debian/rules000077500000000000000000000006241453604140300201050ustar00rootroot00000000000000#!/usr/bin/make -f # -*- makefile -*- -include /usr/share/dpkg/buildflags.mk CFLAGS += $(CPPFLAGS) CXXFLAGS += $(CPPFLAGS) export CFLAGS CXXFLAGS LDFLAGS %: dh $@ override_dh_auto_configure: dh_auto_configure override_dh_auto_build: python setup.py build override_dh_install: dh_auto_install override_dh_auto_clean: dh_auto_clean override_dh_auto_test: python test/functional/test_all.py gfal2-util-v1.8.1/packaging/gfal2-repo-manager.py000077500000000000000000000172641453604140300215360ustar00rootroot00000000000000#!/usr/bin/env python3 # Author: Georgios Bitzes import os, subprocess, sys, inspect, argparse, re, shutil, errno DRY_RUN = False NO_CREATE_REPO = False RAWHIDE_VERSIONS = ["fc39", "fc40 "] def sh(cmd): # poor man's subprocess.check_output, not supported on SL6 process = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True, stderr=subprocess.STDOUT) output, unused_err = process.communicate() retcode = process.poll() if retcode: raise Exception("Command {0} exited with code {1}. Output: {2}".format(cmd, retcode, output)) return output def ensure_valid_choice(parser, choice, text, available): if choice: if type(choice) == str: choice = [choice] for item in choice: if item not in available: parser.error("unrecognized {0}: '{1}'. Available choices: {2}".format(text, item, available)) def add_dependency(parser, when_present, dependency): if hasattr(parser, when_present): if not hasattr(parser, dependency): parser.error("argument --{0} is required when --{1} is present".format(dependency, when_present)) def declare_required(parser, args, choice): choice = choice.replace("-", "_") if not hasattr(args, choice): parser.error("argument --{0} is required".format(choice)) def bailout(msg): raise ValueError(msg) # poor man's enum class PackageType: Binary, NoArch, Source = range(1, 4) class Package(object): def __init__(self, path): self.path = path if not os.path.isfile(self.path): bailout("Not a file: {0}".format(self.path)) self.filename = os.path.basename(self.path) if self.filename.endswith(".src.rpm"): self.type = PackageType.Source tmp = self.filename[0:-8] elif self.filename.endswith(".noarch.rpm"): self.type = PackageType.NoArch tmp = self.filename[0:-11] elif self.filename.endswith(".rpm"): self.type = PackageType.Binary tmp = self.filename[0:-4] else: bailout("Unable to parse RPM type for {0}".format(self.path)) if self.type == PackageType.Source or self.type == PackageType.NoArch: self.arch = None elif tmp.endswith(".x86_64"): tmp = tmp[0:-7] self.arch = "x86_64" elif tmp.endswith(".i386"): tmp = tmp[0:-5] self.arch = "i386" else: bailout("Unable to determine architecture for {0}".format(self.path)) self.platform = tmp.split(".")[-1] tmp = tmp[0:-len(self.platform)-1] if self.platform == "cern": self.platform = tmp.split(".")[-1] tmp = tmp[0:-len(self.platform)-1] if self.platform in RAWHIDE_VERSIONS: self.platform = "fc-rawhide" self.packagename = tmp def construct_location(platform, arch, filename): return "{0}/{1}/{2}".format(platform, arch, filename) def is_tag(ref): return (re.compile("""^(v)(\d+)\.(\d+)\.(\d+)(-(rc)?(\d+))?$""").match(ref) != None or re.compile("""^(v)(\d+)\.(\d+)$""").match(ref) != None) def mkdir_p(path): try: os.makedirs(path) except OSError as exc: # Python >2.5 if exc.errno == errno.EEXIST and os.path.isdir(path): pass else: raise def createrepo(repo): print("-- Running createrepo on {0}".format(repo)) if NO_CREATE_REPO: return sh("rm -rf {0}".format(repo + "/.olddata")) sh("createrepo -q {0}".format(repo)) def copy_to_repo(source, repo): print("-- Copying {0} to {1}".format(source, repo)) if DRY_RUN: return mkdir_p(repo) shutil.copyfile(source, "{0}/{1}".format(repo, os.path.basename(source))) class Repository(object): def __init__(self, base): self.base = base if not os.path.isdir(self.base): bailout("Not a directory: {0}".format(self.base)) def store(self, ref, packages, arch_dir): platforms = set([x.platform for x in packages]) if len(platforms) != 1: raise ValueError("Cannot mix packages of different platforms in the same invocation: {0}".format(list(platforms))) archs = set([x.arch for x in packages]) archs.add(arch_dir) archs.discard(None) if len(archs) > 1: raise ValueError("Cannot mix packages of different architectures in the same invocation: {0}".format(list(archs))) elif len(archs) == 0: raise ValueError("Only noarch packages: Provide a arch directory to deploy the packages.") tag = is_tag(ref) if tag: base = "{0}/rc".format(self.base) else: base = "{0}/testing".format(self.base) if ref != "develop": base = "{0}/testing/{1}".format(self.base, ref) base += "/" + list(platforms)[0] reposToCreate = set() for package in packages: repo = "{0}/{1}".format(base, list(archs)[0]) copy_to_repo(package.path, repo) reposToCreate.add(repo) for repo in reposToCreate: createrepo(repo) def declare_incompatible_options(parser, option, group): if option not in sys.argv: return for item in group: if item in sys.argv: parser.error("argument {0} is incompatible with argument {1}".format(option, item)) def parseargs(): parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter, description="An opinionated yum repository manager.\n") parser.add_argument('--base', type=str, required=True, help="The base directory for your project.") parser.add_argument('--action', type=str, required=True, help="The action to perform. Choices: ['add', 'cleanup']") parser.add_argument('--dry-run', action="store_true", help="If set, don't actually change any files, just show what would happen if ran.") parser.add_argument('--no-create-repo', action="store_true", help="If set, don't run createrepo at the end.") parser.set_defaults(dry_run=False, no_create_repo=False) group = parser.add_argument_group('add options') group.add_argument('--ref', type=str, help="The branch or tag that is being built. Tag names must match 'x.y' or 'x.y.z' (may be prepended by 'v')") group.add_argument('--packages', type=str, nargs='+', help="The list of packages to add") group.add_argument('--arch-dir', type=str, help="In case of noarch packages specify in which arch directory the packages are deployed.") group = parser.add_argument_group('cleanup options') group.add_argument('--keep-last-days', type=int, help="How many days worth of RPMs to keep. (only affects branches)") args = parser.parse_args() ensure_valid_choice(parser, args.arch_dir, "arch_dir", ["x86_64", "i386"]) ensure_valid_choice(parser, args.action, "action", ["add", "cleanup"]) declare_incompatible_options(parser, "--no-create-repo", ["--dry-run"]) if args.action == "add": declare_required(parser, args, "ref") declare_required(parser, args, "packages") if args.action == "cleanup": declare_required(parser, args, "keep-last-days") bailout("NYI") if args.ref == "tags" or args.ref == "tag": bailout("A branch named '{0}'? Really?".format(args.ref)) global DRY_RUN global NO_CREATE_REPO if args.dry_run: DRY_RUN = True NO_CREATE_REPO = True if args.no_create_repo: NO_CREATE_REPO = True return args def main(): args = parseargs() repository = Repository(args.base) packages = [Package(x) for x in args.packages] repository.store(args.ref, packages, args.arch_dir) if __name__ == '__main__': main() gfal2-util-v1.8.1/packaging/rpm/000077500000000000000000000000001453604140300163775ustar00rootroot00000000000000gfal2-util-v1.8.1/packaging/rpm/gfal2-util.spec000066400000000000000000000166031453604140300212270ustar00rootroot00000000000000#------------------------------------------------------------------------------- # Configure python2/3 according to platform and passed-in parameter #------------------------------------------------------------------------------- # Require --without=python3 in order to disable python3 build package %bcond_without python3 # Require --without=python2 in order to disable python2 build package on RHEL7 %if 0%{?rhel} == 7 %bcond_without python2 %endif %if 0%{with python2} %{!?python2_sitelib: %global python2_sitelib %(%{__python2} -c "from sysconfig import get_path; print get_path('purelib')")} %endif %if 0%{with python3} %{!?python3_sitelib: %global python3_sitelib %(%{__python3} -c "from sysconfig import get_path; print(get_path('purelib'))")} %endif Name: gfal2-util Version: 1.8.1 Release: 1%{?dist} Summary: GFAL2 utility tools Group: Applications/Internet License: ASL 2.0 URL: http://dmc.web.cern.ch/ # git clone https://gitlab.cern.ch/dmc/gfal2-util.git gfal2-util-1.8.1 --depth=1 # pushd gfal2-util-1.8.1 # git checkout v1.8.1 # popd # tar czf gfal2-util-1.8.1.tar.gz --exclude-vcs gfal2-util-1.8.1 Source0: %{name}-%{version}.tar.gz BuildRoot: %(mktemp -ud %{_tmppath}/%{name}-%{version}-%{release}-XXXXXX) BuildArch: noarch BuildRequires: gfal2-core BuildRequires: gfal2-plugin-file %global _description \ gfal2-util is a set of basic utility tools for file \ interactions and file copy based on the GFAL 2.0 toolkit. \ gfal2-util supports the protocols of GFAL 2.0 : WebDav(s), \ gridFTP, http(s), SRM, xrootd, etc... %description %_description %prep %if 0%{?without python2} && 0%{?without python3} echo "Must either remove --without=python2 or provide --with=python3 switch" exit 1 %endif %setup -q %build # Validate the version gfal2_util_ver=`sed -n "s/VERSION = '\(.*\)'/\1/p" src/gfal2_util/base.py` gfal2_util_spec_ver=`expr "%{version}" : '\([0-9]*\\.[0-9]*\\.[0-9]*\)'` if [ "$gfal2_util_ver" != "$gfal2_util_spec_ver" ]; then echo "The version in the spec file does not match the base.py version!" echo "%{version} != $gfal2_util_ver" exit 1 fi %if 0%{with python2} python2 setup.py build %endif %if 0%{with python3} python3 setup.py build %endif %install rm -rf %{buildroot} %if 0%{with python2} python2 setup.py install --root=%{buildroot} %endif %if 0%{with python3} python3 setup.py install --root=%{buildroot} %endif %clean rm -rf %{buildroot} #------------------------------------------------------------------------------- # Gfal2-util-scripts package #------------------------------------------------------------------------------- %package scripts Summary: gfal2 command line scripts %description scripts Provides a set of command line scripts to call gfal2-util python functions. %files scripts %defattr (-,root,root) %{_bindir}/gfal-* %{_mandir}/man1/* #------------------------------------------------------------------------------- # Gfal2-util package for Python2 #------------------------------------------------------------------------------- %if 0%{with python2} %package -n python2-gfal2-util Summary: gfal2 clients for python2 BuildRequires: python2-gfal2 >= 1.12.0 BuildRequires: python2 BuildRequires: python2-rpm-macros BuildRequires: python2-setuptools BuildRequires: python2-future Requires: python2-gfal2 >= 1.12.0 Requires: gfal2-util-scripts = %{version}-%{release} Requires: gfal2-plugin-file Requires: python2 Requires: python2-future # Introduced in v1.6.0 / Remove around FC36 Provides: gfal2-util = %{version}-%{release} Obsoletes: gfal2-util < %{version}-%{release} %description -n python2-gfal2-util %_description %files -n python2-gfal2-util %defattr (-,root,root) %{python2_sitelib}/gfal2_util* %doc RELEASE-NOTES VERSION LICENSE readme.html %endif #------------------------------------------------------------------------------- # Gfal2-util package for Python3 #------------------------------------------------------------------------------- %if 0%{with python3} %package -n python3-gfal2-util Summary: gfal2 clients for python3 BuildRequires: python3-gfal2 >= 1.12.0 BuildRequires: python3 BuildRequires: python3-rpm-macros BuildRequires: python3-setuptools Requires: python3-gfal2 >= 1.12.0 Requires: gfal2-util-scripts = %{version}-%{release} Requires: gfal2-plugin-file Requires: python3 # EL7 upgrade path is for python2-gfal2-util %if 0%{?rhel} != 7 Provides: gfal2-util = %{version}-%{release} %endif # Introduced in v1.6.0 / Remove around FC36 Obsoletes: gfal2-util < %{version}-%{release} %description -n python3-gfal2-util %_description %files -n python3-gfal2-util %defattr (-,root,root) %{python3_sitelib}/gfal2_util* %doc RELEASE-NOTES VERSION LICENSE readme.html %endif %changelog * Tue Dec 12 2023 Mihai Patrascoiu - 1.8.1-1 - New upstream release * Fri Sep 02 2022 Joao Lopes - 1.8.0-1 - New upstream release - Renew gfal-legacy-bringonline command - Introduces gfal-evict and gfal-archivepoll commands * Mon Mar 07 2022 Mihai Patrascoiu - 1.7.1-1 - New upstream release * Thu Oct 07 2021 Mihai Patrascoiu - 1.7.0-2 - New upstream release - Python3 package with Provides and Obsoletes capabilities * Thu Sep 23 2021 Joao Lopes - 1.7.0-1 - New upstream release - Introduces SE-Token retrieval * Thu Nov 19 2020 Petr Vokac - 1.6.0-1 - New upstream release - Provide distinct packages for Python2 and Python3 * Mon Sep 14 2020 Mihai Patrascoiu - 1.5.4-1 - New upstream release * Fri Mar 29 2019 Andrea Manzi - 1.5.3-1 - New upstream release * Mon Feb 20 2017 Alejandro Alvarez - 1.5.0-1 - New upstream release * Fri Feb 10 2017 Fedora Release Engineering - 1.4.0-2 - Rebuilt for https://fedoraproject.org/wiki/Fedora_26_Mass_Rebuild * Tue Sep 27 2016 Alejandro Alvarez - 1.4.0-1 - New upstream release - python-argparse is part of python's stdlib * Tue Jul 19 2016 Fedora Release Engineering - 1.3.2-2 - https://fedoraproject.org/wiki/Changes/Automatic_Provides_for_Python_RPM_Packages * Tue Mar 08 2016 Alejandro Alvarez - 1.3.2-1 - Update for new upstream 1.3.2 release * Wed Feb 03 2016 Fedora Release Engineering - 1.3.1-2 - Rebuilt for https://fedoraproject.org/wiki/Fedora_24_Mass_Rebuild * Mon Nov 09 2015 Alejandro Alvarez - 1.3.1-1 - Update for new upstream 1.3.1 release * Wed Jun 17 2015 Fedora Release Engineering - 1.2.1-2 - Rebuilt for https://fedoraproject.org/wiki/Fedora_23_Mass_Rebuild * Fri Apr 17 2015 Alejandro Alvarez - 1.2.1-1 - Update for new upstream 1.2.1 release * Fri Nov 07 2014 Alejandro Alvarez - 1.1.0-1 - Update for new upstream 1.1.0 release * Wed Jul 02 2014 Alejandro Alvarez - 1.0.0-1 - Update for new upstream 1.0.0 release - Installation done with distutils - Run tests on check stage * Sat Jun 07 2014 Fedora Release Engineering - 0.2.1-2 - Rebuilt for https://fedoraproject.org/wiki/Fedora_21_Mass_Rebuild * Mon Nov 04 2013 Adrien Devresse - 0.2.1-1 - Initial EPEL compatible version gfal2-util-v1.8.1/readme.html000066400000000000000000000331071453604140300160040ustar00rootroot00000000000000 gfal2-util

gfal2-util

Description

GFAL 2 utils are a group of command line tools for file manipulations with any protocol managed by gfal2

Installation

yum install gfal2-util gfal2-all

List of command line tools

gfal-cat
Concatenate a list of files into the standard output.
gfal-chmod
Change permissions for a file or directory.
gfal-copy
Copy files from source to destination(s).
gfal-bringonline
Perform a staging operation on the given SURL.
gfal-archivepoll
Perform an archive polling operation on a given SURL.
gfal-legacy-register
Register a new replica into a catalog.
gfal-legacy-replicas
List the replicas of an entry in a catalog.
gfal-legacy-unregister
Remove a replica from a catalog.
gfal-ls
Equivalent to the system `ls` command.
gfal-mkdir
Create a directory.
gfal-rename
Rename a file or directory.
gfal-rm
Remove a file or directory (with -r).
gfal-save
Write standard input to a file.
gfal-stat
Show extended information about a file or directory.
gfal-sum
Retrieve the file checksum.
gfal-xattr
List or display a file's extended attributes.

Third party copy

To understand some parts of this documentation, it is useful to know the difference between a third party copy and a regular one.

Third party copy: When doing a copy between two remote endpoints, the data is sent directly between the two participating storages.
Non-third party copy: The data goes through the client doing the copy.

Gfal2 is capable of doing both, depending on the protocol combination. Some protocols do support third party copy (see below), some others do not. In any case, when copying between two different protocols, the copy will not be third party copy (e.g from GridFTP to DAV).
SRM is an exception to this.

Protocol support

Gfal2 relies on plugins to add support for different protocols. Depending on your installation, some may not be available.
To use a given protocol, the general syntax is protocol://host[:port]/path. For instance:

  • gsiftp://dpm.cern.ch/dpm/cern.ch/home/dteam
  • davs://dpm.cern.ch/dpm/cern.ch/home/dteam
  • davs://dpm.cern.ch:443/dpm/cern.ch/home/dteam

SRM is an exception to this.

You can check what is installed using rpm -qa | grep gfal2-plugin. The normally available protocols are:

file://
Local file access. It can be omitted (i.e. gfal-ls /).
gsiftp://
GridFTP protocol. It has third-party copy support.
srm://
SRM protocol. Its support for third-party copies depends on the transfer URL negotiated with the endpoint, and the protocol used for the other peer.
gfal2 is capable of resolving the full SURL using the information system (e.g srm://host/path), but otherwise, you can also use the full SURL (e.g srm://host:8446/srm/managerv2?SFN=/path)
root://
XRootD protocol. Some storages support third party copies, others do not.
dav(s)://
WebDAV protocol. Some storages support third party copies, but to enable it, you need to use davs+3rd://
https://
Plain HTTPS protocol. Some storages support third party copies, but to enable it, you need to use https+3rd://
s3://
S3 protocol. Does not support third party copy, but it could still be achieved if the peer is a davs+3rd endpoint
lfc://
LFC endpoints. Requires the endpoint to be part of the url (lfc://lfc.cern.ch/path)
guid:
LFC endpoints, by GUID. It uses the LFC_HOST configured under /etc/gfal2.d/lfc_plugin.conf
rfio://
RFIO protocol.
dcap://
DCAP protocol, only supported by dCache endpoints.

These plugins may not be normally installed.

sftp://
Experimental. SFTP support. Works with an SSH private key, instead of a X509 proxy.
mock://
Useful for testing. Allows to force a given reply depending on the URL used. See its documentation.

Configuration

All plugin configuration files are stored under /etc/gfal2.d/. You can navigate them and have a look at the different configuration options on the gfal2 repository.

In any case, for all commands, any value can be overridden by using the flag -D"GROUP:PARAM=VALUE"

For example, let's say we want to do a listing of an SRM endpoint, but using a different infosys endpoint. 

gfal-ls -D"BDII:LCG_GFAL_INFOSYS=myinfosys.cern.ch:2170" srm://endpoint.cern.ch/dpm/cern.ch/home/dteam/

Examples

Uploading a file to the Grid

gfal-copy -fpK /home/doe/file1 srm://dpmhead-rc.cern.ch:8446/srm/managerv2?SFN=/dpm/cern.ch/home/dteam/dir/nested/file1

In this example, the flag -f tells gfal-copy to overwrite the destination if it already exists, -p enables automatic directory creation, and -K enables checksum validation.

Copying files out of the Grid

gfal-copy can be used for uploading, downloading, and also third party copies. Same syntax applies. 

gfal-copy -fpK srm://dpmhead-rc.cern.ch:8446/srm/managerv2?SFN=/dpm/cern.ch/home/dteam/dir/nested/file1 /home/doe/file1

Listing files and directories

gfal-ls -l srm://dpmhead-rc.cern.ch:8446/srm/managerv2?SFN=/dpm/cern.ch/home/dteam/
-rw-rw-r--   1 46    45          192 Jun 21  2016 853.test
-rw-rw-r--   1 46    45       641064 Jun 16  2016 aaaabbaaa
-rw-rw-r--   1 46    48            0 Jun  6  2016 build.out
-rw-rw-r--   1 46    49        49185 Nov 18 16:04 dmc-901.test
-rw-rw-r--   1 46    49        49185 Nov 18 16:05 dmc-902.test
-rw-rw-r--   1 46    50      5279801 Apr 28 16:20 test

In this example, -l enables long listing, which displays then mode, owner, size and mtime next to each entry. Without -l, the short version is displayed instead: only the names.

gfal-ls uses the environment variable LS_COLORS when -l is passed, if the terminal supports it.

Get information about a file or directory

gfal-stat srm://dpmhead-rc.cern.ch:8446/srm/managerv2?SFN=/dpm/cern.ch/
  File: 'srm://dpmhead-trunk.cern.ch:8446/srm/managerv2?SFN=/dpm/cern.ch'
  Size: 0	directory
Access: (0755/drwxr-xr-x)	Uid: 45	Gid: 45
Access: 1970-01-01 01:00:00.000000
Modify: 2015-06-10 18:26:00.000000
Change: 2017-05-09 11:42:52.000000

Get the checksum of a file

gfal-sum srm://dpmhead-trunk.cern.ch:8446/srm/managerv2?SFN=/dpm/cern.ch/home/dteam/aalvarez/aaaabbaaa adler32
srm://dpmhead-trunk.cern.ch:8446/srm/managerv2?SFN=/dpm/cern.ch/home/dteam/aalvarez/aaaabbaaa c88e70d6
gfal-sum srm://dpmhead-trunk.cern.ch:8446/srm/managerv2?SFN=/dpm/cern.ch/home/dteam/aalvarez/aaaabbaaa md5
srm://dpmhead-trunk.cern.ch:8446/srm/managerv2?SFN=/dpm/cern.ch/home/dteam/aalvarez/aaaabbaaa 64fd36ae477e970b06949fe90d0c8c20

Get information about space tokens

gfal-xattr srm://dpmhead-trunk.cern.ch:8446/srm/managerv2?SFN=/dpm/cern.ch/home/dteam/ spacetoken
["65e3ad04-ada1-11e6-b531-c860001bd938","bd1dae50-108c-11e7-82fa-c860001bd938","77056126-194f-11e7-82fa-c860001bd938","4ad03b22-2a92-11e7-82fa-c860001bd938","45e35c78-2b52-11e7-82fa-c860001bd938"]
gfal-xattr srm://dpmhead-trunk.cern.ch:8446/srm/managerv2?SFN=/dpm/cern.ch/home/dteam/ spacetoken.token?65e3ad04-ada1-11e6-b531-c860001bd938
{ "spacetoken": "65e3ad04-ada1-11e6-b531-c860001bd938", "owner": "root", "totalsize": 14937625, "unusedsize": 112193261767, "usedsize": 0, "guaranteedsize": 107374182400, "lifetimeleft": -1, "retention": "CUSTODIAL", "accesslatency": "NEARLINE" }

Apendix: lcg-util equivalence table

Description lcg-util gfal-util
Add an alias for a given GUID lcg-aa Deprecated
Bring SURLs online lcg-bringonline gfal-bringonline
Poll archiving status of SURLs gfal-archivepoll
Copy files with no catalog involved lcg-cp gfal-copy
Copy and register a file lcg-cr gfal-copy
Delete a file / directory lcg-del gfal-rm
Get the file checksum lcg-get-checksum gfal-sum
Get the TURLs for given SURLs lcg-getturls gfal-xattr "user.replicas"
Get the TURL for a given SURL lcg-gt gfal-xattr "user.replicas"
List aliases for a given LFN/GUID lcg-la Deprecated
Get the GUID for a given LFN lcg-lg gfal-xattr "user.guid"
Lists the replicas for a given LFN lcg-lr gfal-xattr "user.replicas"
List information / directory lcg-ls gfal-ls
Remove an alias lcg-ra Deprecated
Copy between Ses using the catalog lcg-rep gfal-copy
Register a file in a catalog lcg-rf gfal-copy
Set a file status to done lcg-sd Deprecated*
Get space tokens associated to a description lcg-stmd gfal-xattr "spacetoken"/"spacetoken.token?$tok"/"spacetoken.description?$desc"
Unregister a file lcg-uf gfal-legacy-unregister
Create a directory gfal-mkdir
Dump the standard input into a file gfal-save
Dump a file into the standard output gfal-cat

(*) These actions are actually performed internally by the SRM plugin when needed.

License

This software is licensed under the Apache 2 License. See LICENSE for details.

Copyright (c) 2013-2022 CERN Copyright (c) 2012-2013 Members of the EMI Collaboration See http://www.eu-emi.eu/partners for details on the copyright holders.

Release Notes

See RELEASE-NOTES for a detailed changelog.

Contact

You can notify bugs or ask for feature requests via

Tracker
https://its.cern.ch/jira/browse/DMC/component/12918
Mailing list
dmc-support@cern.ch
gfal2-util-v1.8.1/setup.py000066400000000000000000000017571453604140300154010ustar00rootroot00000000000000#!/usr/bin/env python from setuptools import setup from glob import glob doc_files = ['RELEASE-NOTES', 'VERSION', 'LICENSE', 'readme.html'] bin_files = glob('src/gfal-*') module_name = 'gfal2_util' data_files = [] man_root = 'share/man/man1' man_files = glob('doc/man/gfal*.1') data_files.append((man_root, man_files)) setup(name=module_name, version='1.8.1', license='Apache License 2.0', description='GFAL2 utility tools', long_description='''gfal2-util is a set of basic utility tools for file interactions and file copy based on the GFAL 2.0 toolkit. gfal2-util supports the protocols of GFAL 2.0 : WebDav(s), gridFTP, http(s), SRM, xrootd, etc...''', author='Duarte Meneses, Adrien Devresse, Andrea Manzi, Mihai Patrascoiu', author_email='dmc-devel@cern.ch', url='http://dmc.web.cern.ch/projects/gfal2-utils', packages=[module_name], package_dir={module_name: 'src/' + module_name}, scripts=bin_files, data_files=data_files, ) gfal2-util-v1.8.1/src/000077500000000000000000000000001453604140300144445ustar00rootroot00000000000000gfal2-util-v1.8.1/src/gfal-archivepoll000066400000000000000000000026711453604140300176140ustar00rootroot00000000000000#!/bin/sh # -*- coding: utf-8 -*- # # Copyright (c) 2022 CERN # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Execute script content with first python interpreter found: # * GFAL_PYTHONBIN environment variable # * python on the PATH if import gfal2, gfal2_util succeeds # * python3 on the PATH if import gfal2, gfal2_util succeeds # * python2 on the PATH if import gfal2, gfal2_util succeeds # * /usr/bin/python "exec" "$( check_interpreter() { unalias $1 2> /dev/null; unset $1; GFAL_PYTHONBIN=$(command -v $1); [ $GFAL_PYTHONBIN ] && $GFAL_PYTHONBIN -c 'import gfal2, gfal2_util' > /dev/null 2>&1 && { echo $GFAL_PYTHONBIN; unset GFAL_PYTHONBIN; }; }; [ $GFAL_PYTHONBIN ] && echo $GFAL_PYTHONBIN || check_interpreter python || check_interpreter python3 || check_interpreter python2 || echo /usr/bin/python )" "-u" "-Wignore" "$0" "$@" from gfal2_util.shell import Gfal2Shell import sys if __name__ == "__main__": sys.exit(Gfal2Shell().main(sys.argv)) gfal2-util-v1.8.1/src/gfal-bringonline000066400000000000000000000031151453604140300176040ustar00rootroot00000000000000#!/bin/sh # -*- coding: utf-8 -*- # # Copyright (c) 2013-2022 CERN # # Copyright (c) 2012-2013 Members of the EMI Collaboration # See http://www.eu-emi.eu/partners for details on the copyright # holders. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Execute script content with first python interpreter found: # * GFAL_PYTHONBIN environment variable # * python on the PATH if import gfal2, gfal2_util succeeds # * python3 on the PATH if import gfal2, gfal2_util succeeds # * python2 on the PATH if import gfal2, gfal2_util succeeds # * /usr/bin/python "exec" "$( check_interpreter() { unalias $1 2> /dev/null; unset $1; GFAL_PYTHONBIN=$(command -v $1); [ $GFAL_PYTHONBIN ] && $GFAL_PYTHONBIN -c 'import gfal2, gfal2_util' > /dev/null 2>&1 && { echo $GFAL_PYTHONBIN; unset GFAL_PYTHONBIN; }; }; [ $GFAL_PYTHONBIN ] && echo $GFAL_PYTHONBIN || check_interpreter python || check_interpreter python3 || check_interpreter python2 || echo /usr/bin/python )" "-u" "-Wignore" "$0" "$@" from gfal2_util.shell import Gfal2Shell import sys if __name__ == "__main__": sys.exit(Gfal2Shell().main(sys.argv)) gfal2-util-v1.8.1/src/gfal-cat000077500000000000000000000031151453604140300160500ustar00rootroot00000000000000#!/bin/sh # -*- coding: utf-8 -*- # # Copyright (c) 2013-2022 CERN # # Copyright (c) 2012-2013 Members of the EMI Collaboration # See http://www.eu-emi.eu/partners for details on the copyright # holders. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Execute script content with first python interpreter found: # * GFAL_PYTHONBIN environment variable # * python on the PATH if import gfal2, gfal2_util succeeds # * python3 on the PATH if import gfal2, gfal2_util succeeds # * python2 on the PATH if import gfal2, gfal2_util succeeds # * /usr/bin/python "exec" "$( check_interpreter() { unalias $1 2> /dev/null; unset $1; GFAL_PYTHONBIN=$(command -v $1); [ $GFAL_PYTHONBIN ] && $GFAL_PYTHONBIN -c 'import gfal2, gfal2_util' > /dev/null 2>&1 && { echo $GFAL_PYTHONBIN; unset GFAL_PYTHONBIN; }; }; [ $GFAL_PYTHONBIN ] && echo $GFAL_PYTHONBIN || check_interpreter python || check_interpreter python3 || check_interpreter python2 || echo /usr/bin/python )" "-u" "-Wignore" "$0" "$@" from gfal2_util.shell import Gfal2Shell import sys if __name__ == "__main__": sys.exit(Gfal2Shell().main(sys.argv)) gfal2-util-v1.8.1/src/gfal-chmod000077500000000000000000000031151453604140300163730ustar00rootroot00000000000000#!/bin/sh # -*- coding: utf-8 -*- # # Copyright (c) 2013-2022 CERN # # Copyright (c) 2012-2013 Members of the EMI Collaboration # See http://www.eu-emi.eu/partners for details on the copyright # holders. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Execute script content with first python interpreter found: # * GFAL_PYTHONBIN environment variable # * python on the PATH if import gfal2, gfal2_util succeeds # * python3 on the PATH if import gfal2, gfal2_util succeeds # * python2 on the PATH if import gfal2, gfal2_util succeeds # * /usr/bin/python "exec" "$( check_interpreter() { unalias $1 2> /dev/null; unset $1; GFAL_PYTHONBIN=$(command -v $1); [ $GFAL_PYTHONBIN ] && $GFAL_PYTHONBIN -c 'import gfal2, gfal2_util' > /dev/null 2>&1 && { echo $GFAL_PYTHONBIN; unset GFAL_PYTHONBIN; }; }; [ $GFAL_PYTHONBIN ] && echo $GFAL_PYTHONBIN || check_interpreter python || check_interpreter python3 || check_interpreter python2 || echo /usr/bin/python )" "-u" "-Wignore" "$0" "$@" from gfal2_util.shell import Gfal2Shell import sys if __name__ == "__main__": sys.exit(Gfal2Shell().main(sys.argv)) gfal2-util-v1.8.1/src/gfal-copy000077500000000000000000000032071453604140300162550ustar00rootroot00000000000000#!/bin/sh # -*- coding: utf-8 -*- # # Copyright (c) 2013-2022 CERN # # Copyright (c) 2012-2013 Members of the EMI Collaboration # See http://www.eu-emi.eu/partners for details on the copyright # holders. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Execute script content with first python interpreter found: # * GFAL_PYTHONBIN environment variable # * python on the PATH if import gfal2, gfal2_util succeeds # * python3 on the PATH if import gfal2, gfal2_util succeeds # * python2 on the PATH if import gfal2, gfal2_util succeeds # * /usr/bin/python "exec" "$( check_interpreter() { unalias $1 2> /dev/null; unset $1; GFAL_PYTHONBIN=$(command -v $1); [ $GFAL_PYTHONBIN ] && $GFAL_PYTHONBIN -c 'import gfal2, gfal2_util' > /dev/null 2>&1 && { echo $GFAL_PYTHONBIN; unset GFAL_PYTHONBIN; }; }; [ $GFAL_PYTHONBIN ] && echo $GFAL_PYTHONBIN || check_interpreter python || check_interpreter python3 || check_interpreter python2 || echo /usr/bin/python )" "-u" "-Wignore" "$0" "$@" from gfal2_util.shell import Gfal2Shell import sys import os if __name__ == "__main__": os.environ["XrdSecGSIDELEGPROXY"] = "1" sys.exit(Gfal2Shell().main(sys.argv)) gfal2-util-v1.8.1/src/gfal-evict000077500000000000000000000026761453604140300164260ustar00rootroot00000000000000#!/bin/sh # -*- coding: utf-8 -*- # # Copyright (c) 2021-2022 CERN # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Execute script content with first python interpreter found: # * GFAL_PYTHONBIN environment variable # * python on the PATH if import gfal2, gfal2_util succeeds # * python3 on the PATH if import gfal2, gfal2_util succeeds # * python2 on the PATH if import gfal2, gfal2_util succeeds # * /usr/bin/python "exec" "$( check_interpreter() { unalias $1 2> /dev/null; unset $1; GFAL_PYTHONBIN=$(command -v $1); [ $GFAL_PYTHONBIN ] && $GFAL_PYTHONBIN -c 'import gfal2, gfal2_util' > /dev/null 2>&1 && { echo $GFAL_PYTHONBIN; unset GFAL_PYTHONBIN; }; }; [ $GFAL_PYTHONBIN ] && echo $GFAL_PYTHONBIN || check_interpreter python || check_interpreter python3 || check_interpreter python2 || echo /usr/bin/python )" "-u" "-Wignore" "$0" "$@" from gfal2_util.shell import Gfal2Shell import sys if __name__ == "__main__": sys.exit(Gfal2Shell().main(sys.argv)) gfal2-util-v1.8.1/src/gfal-legacy-bringonline000066400000000000000000000032371453604140300210530ustar00rootroot00000000000000#!/bin/sh # -*- coding: utf-8 -*- # # Copyright (c) 2013-2022 CERN # # Copyright (c) 2012-2013 Members of the EMI Collaboration # See http://www.eu-emi.eu/partners for details on the copyright # holders. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Execute script content with first python interpreter found: # * GFAL_PYTHONBIN environment variable # * python on the PATH if import gfal2, gfal2_util succeeds # * python3 on the PATH if import gfal2, gfal2_util succeeds # * python2 on the PATH if import gfal2, gfal2_util succeeds # * /usr/bin/python "exec" "$( check_interpreter() { unalias $1 2> /dev/null; unset $1; GFAL_PYTHONBIN=$(command -v $1); [ $GFAL_PYTHONBIN ] && $GFAL_PYTHONBIN -c 'import gfal2, gfal2_util' > /dev/null 2>&1 && { echo $GFAL_PYTHONBIN; unset GFAL_PYTHONBIN; }; }; [ $GFAL_PYTHONBIN ] && echo $GFAL_PYTHONBIN || check_interpreter python || check_interpreter python3 || check_interpreter python2 || echo /usr/bin/python )" "-u" "-Wignore" "$0" "$@" from gfal2_util.shell import Gfal2Shell import sys if __name__ == "__main__": print("This command is deprecated. Please use gfal-bringonline instead.") sys.exit(Gfal2Shell().main(sys.argv)) gfal2-util-v1.8.1/src/gfal-legacy-register000077500000000000000000000031151453604140300203670ustar00rootroot00000000000000#!/bin/sh # -*- coding: utf-8 -*- # # Copyright (c) 2013-2022 CERN # # Copyright (c) 2012-2013 Members of the EMI Collaboration # See http://www.eu-emi.eu/partners for details on the copyright # holders. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Execute script content with first python interpreter found: # * GFAL_PYTHONBIN environment variable # * python on the PATH if import gfal2, gfal2_util succeeds # * python3 on the PATH if import gfal2, gfal2_util succeeds # * python2 on the PATH if import gfal2, gfal2_util succeeds # * /usr/bin/python "exec" "$( check_interpreter() { unalias $1 2> /dev/null; unset $1; GFAL_PYTHONBIN=$(command -v $1); [ $GFAL_PYTHONBIN ] && $GFAL_PYTHONBIN -c 'import gfal2, gfal2_util' > /dev/null 2>&1 && { echo $GFAL_PYTHONBIN; unset GFAL_PYTHONBIN; }; }; [ $GFAL_PYTHONBIN ] && echo $GFAL_PYTHONBIN || check_interpreter python || check_interpreter python3 || check_interpreter python2 || echo /usr/bin/python )" "-u" "-Wignore" "$0" "$@" from gfal2_util.shell import Gfal2Shell import sys if __name__ == "__main__": sys.exit(Gfal2Shell().main(sys.argv)) gfal2-util-v1.8.1/src/gfal-legacy-replicas000077500000000000000000000031151453604140300203450ustar00rootroot00000000000000#!/bin/sh # -*- coding: utf-8 -*- # # Copyright (c) 2013-2022 CERN # # Copyright (c) 2012-2013 Members of the EMI Collaboration # See http://www.eu-emi.eu/partners for details on the copyright # holders. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Execute script content with first python interpreter found: # * GFAL_PYTHONBIN environment variable # * python on the PATH if import gfal2, gfal2_util succeeds # * python3 on the PATH if import gfal2, gfal2_util succeeds # * python2 on the PATH if import gfal2, gfal2_util succeeds # * /usr/bin/python "exec" "$( check_interpreter() { unalias $1 2> /dev/null; unset $1; GFAL_PYTHONBIN=$(command -v $1); [ $GFAL_PYTHONBIN ] && $GFAL_PYTHONBIN -c 'import gfal2, gfal2_util' > /dev/null 2>&1 && { echo $GFAL_PYTHONBIN; unset GFAL_PYTHONBIN; }; }; [ $GFAL_PYTHONBIN ] && echo $GFAL_PYTHONBIN || check_interpreter python || check_interpreter python3 || check_interpreter python2 || echo /usr/bin/python )" "-u" "-Wignore" "$0" "$@" from gfal2_util.shell import Gfal2Shell import sys if __name__ == "__main__": sys.exit(Gfal2Shell().main(sys.argv)) gfal2-util-v1.8.1/src/gfal-legacy-unregister000077500000000000000000000031151453604140300207320ustar00rootroot00000000000000#!/bin/sh # -*- coding: utf-8 -*- # # Copyright (c) 2013-2022 CERN # # Copyright (c) 2012-2013 Members of the EMI Collaboration # See http://www.eu-emi.eu/partners for details on the copyright # holders. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Execute script content with first python interpreter found: # * GFAL_PYTHONBIN environment variable # * python on the PATH if import gfal2, gfal2_util succeeds # * python3 on the PATH if import gfal2, gfal2_util succeeds # * python2 on the PATH if import gfal2, gfal2_util succeeds # * /usr/bin/python "exec" "$( check_interpreter() { unalias $1 2> /dev/null; unset $1; GFAL_PYTHONBIN=$(command -v $1); [ $GFAL_PYTHONBIN ] && $GFAL_PYTHONBIN -c 'import gfal2, gfal2_util' > /dev/null 2>&1 && { echo $GFAL_PYTHONBIN; unset GFAL_PYTHONBIN; }; }; [ $GFAL_PYTHONBIN ] && echo $GFAL_PYTHONBIN || check_interpreter python || check_interpreter python3 || check_interpreter python2 || echo /usr/bin/python )" "-u" "-Wignore" "$0" "$@" from gfal2_util.shell import Gfal2Shell import sys if __name__ == "__main__": sys.exit(Gfal2Shell().main(sys.argv)) gfal2-util-v1.8.1/src/gfal-ls000077500000000000000000000031151453604140300157170ustar00rootroot00000000000000#!/bin/sh # -*- coding: utf-8 -*- # # Copyright (c) 2013-2022 CERN # # Copyright (c) 2012-2013 Members of the EMI Collaboration # See http://www.eu-emi.eu/partners for details on the copyright # holders. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Execute script content with first python interpreter found: # * GFAL_PYTHONBIN environment variable # * python on the PATH if import gfal2, gfal2_util succeeds # * python3 on the PATH if import gfal2, gfal2_util succeeds # * python2 on the PATH if import gfal2, gfal2_util succeeds # * /usr/bin/python "exec" "$( check_interpreter() { unalias $1 2> /dev/null; unset $1; GFAL_PYTHONBIN=$(command -v $1); [ $GFAL_PYTHONBIN ] && $GFAL_PYTHONBIN -c 'import gfal2, gfal2_util' > /dev/null 2>&1 && { echo $GFAL_PYTHONBIN; unset GFAL_PYTHONBIN; }; }; [ $GFAL_PYTHONBIN ] && echo $GFAL_PYTHONBIN || check_interpreter python || check_interpreter python3 || check_interpreter python2 || echo /usr/bin/python )" "-u" "-Wignore" "$0" "$@" from gfal2_util.shell import Gfal2Shell import sys if __name__ == "__main__": sys.exit(Gfal2Shell().main(sys.argv)) gfal2-util-v1.8.1/src/gfal-mkdir000077500000000000000000000031151453604140300164070ustar00rootroot00000000000000#!/bin/sh # -*- coding: utf-8 -*- # # Copyright (c) 2013-2022 CERN # # Copyright (c) 2012-2013 Members of the EMI Collaboration # See http://www.eu-emi.eu/partners for details on the copyright # holders. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Execute script content with first python interpreter found: # * GFAL_PYTHONBIN environment variable # * python on the PATH if import gfal2, gfal2_util succeeds # * python3 on the PATH if import gfal2, gfal2_util succeeds # * python2 on the PATH if import gfal2, gfal2_util succeeds # * /usr/bin/python "exec" "$( check_interpreter() { unalias $1 2> /dev/null; unset $1; GFAL_PYTHONBIN=$(command -v $1); [ $GFAL_PYTHONBIN ] && $GFAL_PYTHONBIN -c 'import gfal2, gfal2_util' > /dev/null 2>&1 && { echo $GFAL_PYTHONBIN; unset GFAL_PYTHONBIN; }; }; [ $GFAL_PYTHONBIN ] && echo $GFAL_PYTHONBIN || check_interpreter python || check_interpreter python3 || check_interpreter python2 || echo /usr/bin/python )" "-u" "-Wignore" "$0" "$@" from gfal2_util.shell import Gfal2Shell import sys if __name__ == "__main__": sys.exit(Gfal2Shell().main(sys.argv)) gfal2-util-v1.8.1/src/gfal-rename000077500000000000000000000031151453604140300165500ustar00rootroot00000000000000#!/bin/sh # -*- coding: utf-8 -*- # # Copyright (c) 2013-2022 CERN # # Copyright (c) 2012-2013 Members of the EMI Collaboration # See http://www.eu-emi.eu/partners for details on the copyright # holders. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Execute script content with first python interpreter found: # * GFAL_PYTHONBIN environment variable # * python on the PATH if import gfal2, gfal2_util succeeds # * python3 on the PATH if import gfal2, gfal2_util succeeds # * python2 on the PATH if import gfal2, gfal2_util succeeds # * /usr/bin/python "exec" "$( check_interpreter() { unalias $1 2> /dev/null; unset $1; GFAL_PYTHONBIN=$(command -v $1); [ $GFAL_PYTHONBIN ] && $GFAL_PYTHONBIN -c 'import gfal2, gfal2_util' > /dev/null 2>&1 && { echo $GFAL_PYTHONBIN; unset GFAL_PYTHONBIN; }; }; [ $GFAL_PYTHONBIN ] && echo $GFAL_PYTHONBIN || check_interpreter python || check_interpreter python3 || check_interpreter python2 || echo /usr/bin/python )" "-u" "-Wignore" "$0" "$@" from gfal2_util.shell import Gfal2Shell import sys if __name__ == "__main__": sys.exit(Gfal2Shell().main(sys.argv)) gfal2-util-v1.8.1/src/gfal-rm000077500000000000000000000031151453604140300157170ustar00rootroot00000000000000#!/bin/sh # -*- coding: utf-8 -*- # # Copyright (c) 2013-2022 CERN # # Copyright (c) 2012-2013 Members of the EMI Collaboration # See http://www.eu-emi.eu/partners for details on the copyright # holders. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Execute script content with first python interpreter found: # * GFAL_PYTHONBIN environment variable # * python on the PATH if import gfal2, gfal2_util succeeds # * python3 on the PATH if import gfal2, gfal2_util succeeds # * python2 on the PATH if import gfal2, gfal2_util succeeds # * /usr/bin/python "exec" "$( check_interpreter() { unalias $1 2> /dev/null; unset $1; GFAL_PYTHONBIN=$(command -v $1); [ $GFAL_PYTHONBIN ] && $GFAL_PYTHONBIN -c 'import gfal2, gfal2_util' > /dev/null 2>&1 && { echo $GFAL_PYTHONBIN; unset GFAL_PYTHONBIN; }; }; [ $GFAL_PYTHONBIN ] && echo $GFAL_PYTHONBIN || check_interpreter python || check_interpreter python3 || check_interpreter python2 || echo /usr/bin/python )" "-u" "-Wignore" "$0" "$@" from gfal2_util.shell import Gfal2Shell import sys if __name__ == "__main__": sys.exit(Gfal2Shell().main(sys.argv)) gfal2-util-v1.8.1/src/gfal-save000077500000000000000000000031151453604140300162370ustar00rootroot00000000000000#!/bin/sh # -*- coding: utf-8 -*- # # Copyright (c) 2013-2022 CERN # # Copyright (c) 2012-2013 Members of the EMI Collaboration # See http://www.eu-emi.eu/partners for details on the copyright # holders. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Execute script content with first python interpreter found: # * GFAL_PYTHONBIN environment variable # * python on the PATH if import gfal2, gfal2_util succeeds # * python3 on the PATH if import gfal2, gfal2_util succeeds # * python2 on the PATH if import gfal2, gfal2_util succeeds # * /usr/bin/python "exec" "$( check_interpreter() { unalias $1 2> /dev/null; unset $1; GFAL_PYTHONBIN=$(command -v $1); [ $GFAL_PYTHONBIN ] && $GFAL_PYTHONBIN -c 'import gfal2, gfal2_util' > /dev/null 2>&1 && { echo $GFAL_PYTHONBIN; unset GFAL_PYTHONBIN; }; }; [ $GFAL_PYTHONBIN ] && echo $GFAL_PYTHONBIN || check_interpreter python || check_interpreter python3 || check_interpreter python2 || echo /usr/bin/python )" "-u" "-Wignore" "$0" "$@" from gfal2_util.shell import Gfal2Shell import sys if __name__ == "__main__": sys.exit(Gfal2Shell().main(sys.argv)) gfal2-util-v1.8.1/src/gfal-stat000077500000000000000000000031151453604140300162540ustar00rootroot00000000000000#!/bin/sh # -*- coding: utf-8 -*- # # Copyright (c) 2013-2022 CERN # # Copyright (c) 2012-2013 Members of the EMI Collaboration # See http://www.eu-emi.eu/partners for details on the copyright # holders. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Execute script content with first python interpreter found: # * GFAL_PYTHONBIN environment variable # * python on the PATH if import gfal2, gfal2_util succeeds # * python3 on the PATH if import gfal2, gfal2_util succeeds # * python2 on the PATH if import gfal2, gfal2_util succeeds # * /usr/bin/python "exec" "$( check_interpreter() { unalias $1 2> /dev/null; unset $1; GFAL_PYTHONBIN=$(command -v $1); [ $GFAL_PYTHONBIN ] && $GFAL_PYTHONBIN -c 'import gfal2, gfal2_util' > /dev/null 2>&1 && { echo $GFAL_PYTHONBIN; unset GFAL_PYTHONBIN; }; }; [ $GFAL_PYTHONBIN ] && echo $GFAL_PYTHONBIN || check_interpreter python || check_interpreter python3 || check_interpreter python2 || echo /usr/bin/python )" "-u" "-Wignore" "$0" "$@" from gfal2_util.shell import Gfal2Shell import sys if __name__ == "__main__": sys.exit(Gfal2Shell().main(sys.argv)) gfal2-util-v1.8.1/src/gfal-sum000077500000000000000000000031151453604140300161050ustar00rootroot00000000000000#!/bin/sh # -*- coding: utf-8 -*- # # Copyright (c) 2013-2022 CERN # # Copyright (c) 2012-2013 Members of the EMI Collaboration # See http://www.eu-emi.eu/partners for details on the copyright # holders. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Execute script content with first python interpreter found: # * GFAL_PYTHONBIN environment variable # * python on the PATH if import gfal2, gfal2_util succeeds # * python3 on the PATH if import gfal2, gfal2_util succeeds # * python2 on the PATH if import gfal2, gfal2_util succeeds # * /usr/bin/python "exec" "$( check_interpreter() { unalias $1 2> /dev/null; unset $1; GFAL_PYTHONBIN=$(command -v $1); [ $GFAL_PYTHONBIN ] && $GFAL_PYTHONBIN -c 'import gfal2, gfal2_util' > /dev/null 2>&1 && { echo $GFAL_PYTHONBIN; unset GFAL_PYTHONBIN; }; }; [ $GFAL_PYTHONBIN ] && echo $GFAL_PYTHONBIN || check_interpreter python || check_interpreter python3 || check_interpreter python2 || echo /usr/bin/python )" "-u" "-Wignore" "$0" "$@" from gfal2_util.shell import Gfal2Shell import sys if __name__ == "__main__": sys.exit(Gfal2Shell().main(sys.argv)) gfal2-util-v1.8.1/src/gfal-token000066400000000000000000000026761453604140300164310ustar00rootroot00000000000000#!/bin/sh # -*- coding: utf-8 -*- # # Copyright (c) 2021-2022 CERN # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Execute script content with first python interpreter found: # * GFAL_PYTHONBIN environment variable # * python on the PATH if import gfal2, gfal2_util succeeds # * python3 on the PATH if import gfal2, gfal2_util succeeds # * python2 on the PATH if import gfal2, gfal2_util succeeds # * /usr/bin/python "exec" "$( check_interpreter() { unalias $1 2> /dev/null; unset $1; GFAL_PYTHONBIN=$(command -v $1); [ $GFAL_PYTHONBIN ] && $GFAL_PYTHONBIN -c 'import gfal2, gfal2_util' > /dev/null 2>&1 && { echo $GFAL_PYTHONBIN; unset GFAL_PYTHONBIN; }; }; [ $GFAL_PYTHONBIN ] && echo $GFAL_PYTHONBIN || check_interpreter python || check_interpreter python3 || check_interpreter python2 || echo /usr/bin/python )" "-u" "-Wignore" "$0" "$@" from gfal2_util.shell import Gfal2Shell import sys if __name__ == "__main__": sys.exit(Gfal2Shell().main(sys.argv)) gfal2-util-v1.8.1/src/gfal-xattr000077500000000000000000000031151453604140300164430ustar00rootroot00000000000000#!/bin/sh # -*- coding: utf-8 -*- # # Copyright (c) 2013-2022 CERN # # Copyright (c) 2012-2013 Members of the EMI Collaboration # See http://www.eu-emi.eu/partners for details on the copyright # holders. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Execute script content with first python interpreter found: # * GFAL_PYTHONBIN environment variable # * python on the PATH if import gfal2, gfal2_util succeeds # * python3 on the PATH if import gfal2, gfal2_util succeeds # * python2 on the PATH if import gfal2, gfal2_util succeeds # * /usr/bin/python "exec" "$( check_interpreter() { unalias $1 2> /dev/null; unset $1; GFAL_PYTHONBIN=$(command -v $1); [ $GFAL_PYTHONBIN ] && $GFAL_PYTHONBIN -c 'import gfal2, gfal2_util' > /dev/null 2>&1 && { echo $GFAL_PYTHONBIN; unset GFAL_PYTHONBIN; }; }; [ $GFAL_PYTHONBIN ] && echo $GFAL_PYTHONBIN || check_interpreter python || check_interpreter python3 || check_interpreter python2 || echo /usr/bin/python )" "-u" "-Wignore" "$0" "$@" from gfal2_util.shell import Gfal2Shell import sys if __name__ == "__main__": sys.exit(Gfal2Shell().main(sys.argv)) gfal2-util-v1.8.1/src/gfal2_util/000077500000000000000000000000001453604140300164745ustar00rootroot00000000000000gfal2-util-v1.8.1/src/gfal2_util/__init__.py000066400000000000000000000013221453604140300206030ustar00rootroot00000000000000# # Copyright (c) 2013-2022 CERN # # Copyright (c) 2012-2013 Members of the EMI Collaboration # See http://www.eu-emi.eu/partners for details on the copyright # holders. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. gfal2-util-v1.8.1/src/gfal2_util/base.py000066400000000000000000000223331453604140300177630ustar00rootroot00000000000000# # Copyright (c) 2013-2022 CERN # # Copyright (c) 2012-2013 Members of the EMI Collaboration # See http://www.eu-emi.eu/partners for details on the copyright # holders. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. #from __future__ import absolute_import # not available in python 2.4 from __future__ import division import argparse import logging import signal try: from urllib.parse import urlparse, urlunparse except ImportError: from urlparse import urlparse, urlunparse from threading import Thread import sys import errno import os import gfal2 from gfal2_util.gfal2_utils_parameters import apply_option VERSION = '1.8.1' def arg(*args, **kwargs): """Decorator for CLI args.""" def _decorator(func): __add_arg(func, *args, **kwargs) return func return _decorator def __add_arg(f, *args, **kwargs): """Bind CLI arguments to a shell.py `do_foo` function.""" if not hasattr(f, 'arguments'): f.arguments = [] if (args, kwargs) not in f.arguments: f.arguments.insert(0, (args, kwargs)) class Gfal2VersionAction(argparse.Action): """ Custom Version action, so we can insert new lines and so on """ def __init__(self, *args, **kwargs): kwargs['nargs'] = 0 super(Gfal2VersionAction, self).__init__(*args, **kwargs) def __call__(self, *args, **kwargs): """ Pretty print of gfal2-util version """ version_str = "gfal2-util version %s (gfal2 %s)" % (VERSION, gfal2.get_version()) for plugin in sorted(gfal2.creat_context().get_plugin_names()): version_str += '\n\t' + plugin print(version_str) sys.exit(0) class CommandBase(object): def __init__(self): self.context = None self.progress_bar = None self.running = False self.return_code = -1 @staticmethod def get_subclasses(): return CommandBase.__subclasses__() @staticmethod def __setup_logger(level, log_file): # Handle logging level level = max(0, level) level = min(3, level) log_level_value = logging.ERROR - (level * 10) if level < 3: gfal2.set_verbose(gfal2.verbose_level.verbose) else: gfal2.set_verbose(gfal2.verbose_level.debug) # Handle log file log_stream = sys.stdout if log_file: log_stream = open(log_file, 'w+') root_logger = logging.getLogger() root_logger.setLevel(log_level_value) handler = logging.StreamHandler(log_stream) handler.setLevel(log_level_value) handler.setFormatter(logging.Formatter('%(levelname)s %(message)s')) if log_stream.isatty(): logging.addLevelName(logging.DEBUG, "\033[1;2m%-8s\033[1;m" % logging.getLevelName(logging.DEBUG)) logging.addLevelName(logging.INFO, "\033[1;34m%-8s\033[1;m" % logging.getLevelName(logging.INFO)) logging.addLevelName(logging.ERROR, "\033[1;31m%-8s\033[1;m" % logging.getLevelName(logging.ERROR)) logging.addLevelName(logging.WARNING, "\033[1;33m%-8s\033[1;m" % logging.getLevelName(logging.WARNING)) root_logger.addHandler(handler) #wrap method to catch exceptions in thread's stack def executor(self, func): try: self.return_code = func(self) except IOError: e = sys.exc_info()[1] if e.errno != errno.EPIPE: raise except gfal2.GError: e = sys.exc_info()[1] sys.stderr.write("%s error: %d (%s) - %s\n" % (self.progr, e.code, os.strerror(e.code), e.message)) self.return_code = e.code if 0 <= e.code <= 255 else 255 def execute(self, func): def cancel(): self.context.cancel() # Set X509_ environment if --cert is used if self.params.cert: if not self.params.key: self.params.key = self.params.cert os.environ['X509_USER_CERT'] = self.params.cert os.environ['X509_USER_KEY'] = self.params.key if 'X509_USER_PROXY' in os.environ: del os.environ['X509_USER_PROXY'] # Setup log verbosity and destination self.__setup_logger(self.params.verbose, self.params.log_file) self.context = gfal2.creat_context() apply_option(self.context, self.params) self.context.set_user_agent("gfal2-util", VERSION) t_main = Thread(target=self.executor, args=[func]) t_main.daemon = True if not hasattr(t_main, 'is_alive'): # is_alive was added in python 2.6 and isAlive deprecated in python 3.8 t_main.is_alive = t_main.isAlive try: #run in another thread to be able to catch signals while C functions don't return # See rule #3 in http://docs.python.org/2/library/signal.html t_main.start() if self.params.timeout > 0: # Increment the timeout a bit so plugins have a chance to timeout themselves t_main.join(self.params.timeout + 30) else: #if join(None) is used, it doesn't catch signals while t_main.is_alive(): t_main.join(3600) #self._enable_output() if t_main.is_alive(): if self.progress_bar is not None: self.progress_bar.stop(False) sys.stderr.write('Command timed out after %d seconds!\n' % self.params.timeout) return errno.ETIMEDOUT return self.return_code except KeyboardInterrupt: sys.stderr.write("Caught keyboard interrupt. Canceling...") #ignore any other interrupt signal signal.signal(signal.SIGINT, signal.SIG_IGN) #cancel in another thread to avoid blocking us t_cancel = Thread(target=cancel) t_cancel.daemon = True # in no case hog the entire program if not hasattr(t_cancel, 'is_alive'): # is_alive was added in python 2.6 and isAlive deprecated in python 3.8 t_cancel.is_alive = t_cancel.isAlive t_cancel.start() t_cancel.join(4) if t_cancel.is_alive(): sys.stderr.write("failed to cancel after waiting some time\n") return errno.EINTR def parse(self, func, a): #Collect some info about the function command = func.__name__[8:] desc = func.__doc__ or '' doc = desc.strip().split('\n')[0] description = 'Gfal util ' + command.upper() + ' command. ' + doc if description[-1] != '.': description += '.' arguments = getattr(func, 'arguments', []) #Create parser and parse arguments self.parser = argparse.ArgumentParser(prog=os.path.basename(a[0]), description=description, add_help=True) self.parser.add_argument('-V', '--version', action=Gfal2VersionAction, help="output version information and exit") self.parser.add_argument('-v', '--verbose', action='count', default=0, help="enable the verbose mode, -v for warning, -vv for info, -vvv for debug") self.parser.add_argument('-D', '--definition', nargs=1, type=str, help="override a gfal parameter", action='append') self.parser.add_argument('-t', '--timeout', type=int, default=1800, help="maximum time for the operation to terminate - default is 1800 seconds") self.parser.add_argument('-E', '--cert', type=str, default=None, help="user certificate") self.parser.add_argument('--key', type=str, default=None, help="user private key") self.parser.add_argument('-4', dest='ipv4', action='store_true', help="forces gfal2-util to use IPv4 addresses only. N.B. this is valid only for gridftp") self.parser.add_argument('-6', dest='ipv6', action='store_true', help="forces gfal2-util to use IPv6 addresses only. N.B. this is valid only for gridftp") self.parser.add_argument('-C', '--client-info', type=str, help="provide custom client-side information", action='append') self.parser.add_argument('--log-file', type=str, default=None, help="write Gfal2 library logs to the given file location") for (args, kwargs) in arguments: self.parser.add_argument(*args, **kwargs) self.params = self.parser.parse_args(a[1:]) self.progr = os.path.basename(a[0]) def surl(value): """ Special "type" for surls. It will convert, for instance, paths of the form "/path" to "file:///path" """ if value == '-': return value parsed = urlparse(value) if not parsed[0]: return urlunparse(('file', None, os.path.abspath(parsed[2]), None, None, None)) return value gfal2-util-v1.8.1/src/gfal2_util/commands.py000066400000000000000000000202771453604140300206570ustar00rootroot00000000000000# # Copyright (c) 2013-2022 CERN # # Copyright (c) 2012-2013 Members of the EMI Collaboration # See http://www.eu-emi.eu/partners for details on the copyright # holders. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. #from __future__ import absolute_import # not available in python 2.4 from __future__ import division import gfal2 import sys import stat from datetime import datetime from gfal2_util import base from gfal2_util.utils import file_type_str, file_mode_str class GfalCommands(base.CommandBase): @base.arg('-m', '--mode', action='store', type=int, default=755, help="display hidden files") @base.arg('-p', '--parents', action='store_true', help="no error if existing, make parent directories as needed") @base.arg('directory', action='store', nargs='+', type=base.surl, help="Directory's uri") def execute_mkdir(self): """ Makes directories. By default, it sets file mode 0755. """ if self.params.mode: try: mode = int(str(self.params.mode), 8) except ValueError: mode = int('755', 8) # python < 2.6 doesn't support 0o755 else: mode = int('755', 8) # python < 2.6 doesn't support 0o755 for d in self.params.directory: if self.params.parents: self.context.mkdir_rec(d, mode) else: self.context.mkdir(d, mode) @base.arg('file', action='store', type=base.surl, help="uri of the file to be written") def execute_save(self): """ Reads from stdin and writes to a file. If the file exists, it will be overwritten """ f = self.context.open(self.params.file, 'w') while True: b = sys.stdin.read(65000) if b: f.write(b) else: break @base.arg('-b', '--bytes', action='store_true', help="handle file contents as bytes (only in Python3)") @base.arg('file', action='store', nargs='+', type=base.surl, help="uri of the file to be displayed") def execute_cat(self): """ Sends to stdout the contents of files """ for fname in self.params.file: f = self.context.open(fname, 'r') while True: if self.params.bytes and sys.version_info >= (3, 0): b = f.read_bytes(65000) sys.stdout.buffer.write(b) else: b = f.read(65000) sys.stdout.write(b) if not b: break @base.arg('file', action='store', type=base.surl, help="file uri") @base.arg('attribute', nargs='?', type=str, help="attribute to retrieve or set. To set, use key=value") def execute_xattr(self): """ Gets or set the extended attributes of files and directories """ if self.params.attribute is not None: if '=' in self.params.attribute: i = self.params.attribute.find("=") attr = self.params.attribute[:i] v = self.params.attribute[(i+1):] if len(attr) > 0 and len(v) > 0: self.context.setxattr(self.params.file, attr, v, 0) else: v = self.context.getxattr(self.params.file, self.params.attribute) sys.stdout.write(v + '\n') else: l = self.context.listxattr(self.params.file) for attr in l: try: v = self.context.getxattr(self.params.file, attr) sys.stdout.write(attr + ' = ' + v + '\n') except gfal2.GError: e = sys.exc_info()[1] sys.stdout.write(attr + ' FAILED: ' + str(e) + '\n') @base.arg('file', action='store', type=base.surl, help="file uri to use for checksum calculation") @base.arg('checksum_type', action='store', type=str, help="checksum algorithm to use. For example: ADLER32, CRC32, MD5") def execute_sum(self): """ Calculates the checksum of a file """ checksum = self.context.checksum(self.params.file, self.params.checksum_type) sys.stdout.write(self.params.file + ' ' + checksum + '\n') @base.arg('file', action='store', type=base.surl, help="uri of the file to be stat") def execute_stat(self): """ Stats a file """ st = self.context.stat(self.params.file) print(" File: '%s'" % self.params.file) print(" Size: %d\t%s" % (st.st_size, file_type_str(stat.S_IFMT(st.st_mode)))) print("Access: (%04o/%s)\tUid: %d\tGid: %d\t" % (stat.S_IMODE(st.st_mode), file_mode_str(st.st_mode), st.st_uid, st.st_gid)) if sys.version_info >= (2, 6): print("Access: %s" % datetime.fromtimestamp(st.st_atime).strftime("%Y-%m-%d %H:%M:%S.%f")) print("Modify: %s" % datetime.fromtimestamp(st.st_mtime).strftime("%Y-%m-%d %H:%M:%S.%f")) print("Change: %s" % datetime.fromtimestamp(st.st_ctime).strftime("%Y-%m-%d %H:%M:%S.%f")) else: # python 2.4 and 2.5 doesn't support %f in strftime print("Access: %s" % datetime.fromtimestamp(st.st_atime).strftime("%Y-%m-%d %H:%M:%S.000000")) print("Modify: %s" % datetime.fromtimestamp(st.st_mtime).strftime("%Y-%m-%d %H:%M:%S.000000")) print("Change: %s" % datetime.fromtimestamp(st.st_ctime).strftime("%Y-%m-%d %H:%M:%S.000000")) @base.arg('source', action='store', type=base.surl, help="original file name") @base.arg('destination', action='store', type=base.surl, help="new file name") def execute_rename(self): """ Renames files or directories """ self.context.rename(self.params.source, self.params.destination) @base.arg('mode', action='store', type=str, help="new mode, in octal") @base.arg('file', action='store', type=base.surl, help="uri of the file to change permissions") def execute_chmod(self): """ Change the permissions of a file """ try: mode = int(self.params.mode, base=8) except ValueError: self.parser.error('Mode must be an octal number (i.e. 0755)') return self.context.chmod(self.params.file, mode) @base.arg('--issuer', action='store', type=str, help="token issuer URL") @base.arg('--validity', action='store', type=int, default=60, help="token validity in minutes") @base.arg('-w', '--write', dest='write_access', action='store_true', help="flag to request write access token") @base.arg('path', action='store', type=base.surl, help="URI to request token for") @base.arg('activities', action='store', nargs='*', type=str, help="activities for macaroon request") def execute_token(self): """ Retrieve a SE-issued token """ if self.params.validity < 0: sys.stderr.write("Validity must be a number >= 0\n") return 1 if self.params.verbose: if len(self.params.activities) > 0: print("Will use user-provided activities") else: print("Will use default activities for {} access" .format(['read', 'write'][self.params.write_access])) issuer = self.params.issuer if self.params.issuer is not None else '' if len(self.params.activities) > 0: token = self.context.token_retrieve(self.params.path, issuer, self.params.validity, self.params.activities) else: token = self.context.token_retrieve(self.params.path, issuer, self.params.validity, self.params.write_access) sys.stdout.write(token + '\n') gfal2-util-v1.8.1/src/gfal2_util/copy.py000066400000000000000000000337471453604140300200360ustar00rootroot00000000000000# # Copyright (c) 2013-2022 CERN # # Copyright (c) 2012-2013 Members of the EMI Collaboration # See http://www.eu-emi.eu/partners for details on the copyright # holders. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. #from __future__ import absolute_import # not available in python 2.4 from __future__ import division import logging import sys import os import stat import errno import gfal2 from gfal2_util import base from gfal2_util.progress import Progress log = logging.getLogger(__name__) def _is_special_file(fstat): """ Returns true if the file is a special one (i.e. stdout, stderr, null....) """ return stat.S_ISFIFO(fstat.st_mode) or stat.S_ISCHR(fstat.st_mode) or stat.S_ISSOCK(fstat.st_mode) class CommandCopy(base.CommandBase): @base.arg('-f', "--force", action='store_true', help="if destination file(s) cannot be overwritten, delete it and try again") @base.arg('-p', "--parent", action='store_true', help="if the destination directory does not exist, create it") @base.arg('-n', "--nbstreams", type=int, default=None, help="specify the maximum number of parallel streams to use for the copy") @base.arg("--tcp-buffersize", type=int, default=None, help="specify the TCP buffersize") @base.arg('-s', "--src-spacetoken", type=str, default="", help="source spacetoken to use for the transfer") @base.arg('-S', "--dst-spacetoken", type=str, default="", help="destination spacetoken to use for the transfer") @base.arg('-T', "--transfer-timeout", type=int, default=None, help="global timeout for the transfer operation") @base.arg('-K', "--checksum", type=str, default=None, help='checksum algorithm to use, or algorithm:value') @base.arg("--checksum-mode", type=str, default='both', choices=['source', 'target', 'both'], help='checksum validation mode') @base.arg('--from-file', type=str, default=None, help="read sources from a file") @base.arg("--copy-mode", type=str, default='', choices=['pull', 'push', 'streamed'], help='copy mode. N.B. supported only for HTTP/DAV to HTTP/DAV transfers, if not specified the pull mode will be executed first with fallbacks to other modes in case of errors') @base.arg('--just-copy', action='store_true', help="just do the copy and skip any preparation (i.e. checksum, overwrite, etc.)") @base.arg('--no-delegation', action='store_true', help="disable TPC with proxy delegation") @base.arg('--evict', action='store_true', help="evict source file from disk buffer when the transfer is finished") @base.arg('--scitag', type=int, default=None, help="SciTag transfer flow identifier (number in [65-65535] range) (available only for HTTP-TPC)") @base.arg('-r', '--recursive', action='store_true', help="copy directories recursively") @base.arg('--abort-on-failure', action='store_true', help="abort the whole copy as soon as one failure is encountered") @base.arg('--dry-run', action='store_true', help="do not perform any action, just print what would be done") @base.arg('src', type=base.surl, nargs='?', help="source file") @base.arg('dst', action='store', nargs='+', type=base.surl, help="destination file(s). If more than one is given, they will be chained copy: src -> dst1, dst1->dst2, ...") def execute_copy(self): """ Copy a file or set of files """ if self.params.from_file and self.params.src: sys.stderr.write("Cannot combine '--from-file' with a source in the positional arguments\n") return 1 copy_jobs = list() if self.params.from_file: src_file = open(self.params.from_file) dst = self.params.dst[0] for src in map(str.strip, src_file.readlines()): if src: copy_jobs.append((src, dst)) src_file.close() elif self.params.src: s = self.params.src for dst in self.params.dst: copy_jobs.append((s, dst)) # Next hop # If dst happens to be a dir, append the file name is_dst_dir = False if not self.params.just_copy: try: is_dst_dir = stat.S_ISDIR(self.context.stat(dst).st_mode) except: pass if is_dst_dir: s = dst + '/' + os.path.basename(s) else: s = dst else: sys.stderr.write("Missing source\n") return 1 # Do the actual work for (source, destination) in copy_jobs: if destination == '-': destination = 'file:///dev/stdout' if self.params.just_copy: self._do_file_copy(source, destination, 0) else: self._do_copy(source, destination) return 0 def _failure(self, msg, errno): if self.params.abort_on_failure or not self.params.recursive: raise gfal2.GError(msg, errno) print("ERROR (%d): %s" % (errno, msg)) return False def _do_copy(self, source, destination): # Check what are we dealing with try: source_stat = self.context.stat(source) source_isdir = stat.S_ISDIR(source_stat.st_mode) except gfal2.GError: e = sys.exc_info()[1] return self._failure("Could not stat the source: %s" % e.message, e.code) dest_isdir = False dest_exists = False dest_special = False try: dest_stat = self.context.stat(destination) dest_isdir = stat.S_ISDIR(dest_stat.st_mode) dest_exists = True if destination.startswith('file:'): dest_special = _is_special_file(dest_stat) except: pass # Perform some checks before continuing is_lfc = destination.startswith('lfc://') or destination.startswith('lfn://') or destination.startswith('guid://') if dest_exists and not dest_isdir and not dest_special and not self.params.force: if is_lfc: log.warning("Destination exists, but it is an LFC, so try to add a new replica") else: return self._failure("Destination %s exists and overwrite is not set" % destination, errno.EEXIST) if dest_exists and not dest_isdir and source_isdir: return self._failure("Can not copy a directory over a file", errno.EISDIR) if source_isdir and not dest_exists: try: self._mkdir(destination) except gfal2.GError: e = sys.exc_info()[1] return self._failure("Could not create the directory: %s" % e.message, e.code) return self._recursive_copy(source, destination) elif dest_isdir and source_isdir: if self.params.recursive: return self._recursive_copy(source, destination) else: print("Skipping %s" % source) return True elif dest_isdir: if destination[-1] != '/': destination += '/' destination += os.path.basename(source) return self._do_file_copy(source, destination, source_stat.st_size) def _mkdir(self, surl): print("Mkdir %s" % surl) if not self.params.dry_run: self.context.mkdir_rec(surl, int('755', 8)) # python < 2.6 doesn't support 0o755 def _recursive_copy(self, source, destination): all_sources = self.context.listdir(source) src_base_dir = source dst_base_dir = destination if src_base_dir[-1] != '/': src_base_dir += '/' if dst_base_dir[-1] != '/': dst_base_dir += '/' for entry in all_sources: if entry not in ['.', '..']: new_source = src_base_dir + entry new_destination = dst_base_dir + entry self._do_copy(new_source, new_destination) def _setup_transfer_params(self, t, event_callback, monitor_callback): if self.params.nbstreams: t.nbstreams = self.params.nbstreams if self.params.transfer_timeout: t.timeout = self.params.transfer_timeout if self.params.src_spacetoken: t.src_spacetoken = self.params.src_spacetoken if self.params.dst_spacetoken: t.dst_spacetoken = self.params.dst_spacetoken if self.params.parent: t.create_parent = self.params.parent if self.params.tcp_buffersize: t.tcp_buffersize = self.params.tcp_buffersize if self.params.force: t.overwrite = self.params.force if self.params.just_copy: t.strict_copy = True if self.params.no_delegation: t.proxy_delegation = False if self.params.evict: # available since gfal2-python 1.12.0 if hasattr(t, 'evict'): t.evict = True else: sys.stderr.write("[warn] '--evict' flag requires python{}-gfal2 >= 1.12.0\n".format(sys.version_info.major)) if self.params.scitag is not None: if hasattr(t, 'scitag'): # available since gfal2-python 1.12.2 t.scitag = self.params.scitag else: sys.stderr.write("[warn] '--scitag' flag requires python{}-gfal2 >= 1.12.2\n".format(sys.version_info.major)) if self.params.checksum: chk_args = self.params.checksum.split(':') if len(chk_args) == 1: chk_args.append('') if hasattr(t, 'set_checksum'): # available since gfal2-python 1.9.0 mode = dict( source=gfal2.checksum_mode.source, target=gfal2.checksum_mode.target, both=gfal2.checksum_mode.both )[self.params.checksum_mode] t.set_checksum(mode, chk_args[0], chk_args[1]) else: t.checksum_check = True t.set_user_defined_checksum(chk_args[0], chk_args[1]) if self.params.copy_mode: if self.params.copy_mode == 'pull': self.context.set_opt_boolean("HTTP PLUGIN", "ENABLE_REMOTE_COPY", True) self.context.set_opt_boolean("HTTP PLUGIN", "ENABLE_FALLBACK_TPC_COPY", False) self.context.set_opt_string("HTTP PLUGIN", "DEFAULT_COPY_MODE", "3rd pull") elif self.params.copy_mode == 'push': self.context.set_opt_boolean("HTTP PLUGIN", "ENABLE_REMOTE_COPY", True) self.context.set_opt_boolean("HTTP PLUGIN", "ENABLE_FALLBACK_TPC_COPY", False) self.context.set_opt_string("HTTP PLUGIN", "DEFAULT_COPY_MODE", "3rd push") elif self.params.copy_mode == 'streamed': self.context.set_opt_boolean("HTTP PLUGIN", "ENABLE_REMOTE_COPY", False) self.context.set_opt_string("HTTP PLUGIN", "DEFAULT_COPY_MODE", "streamed") else: self.context.set_opt_boolean("HTTP PLUGIN", "ENABLE_REMOTE_COPY", True) self.context.set_opt_boolean("HTTP PLUGIN", "ENABLE_FALLBACK_TPC_COPY", True) self.context.set_opt_string("HTTP PLUGIN", "DEFAULT_COPY_MODE", "3rd pull") if event_callback: t.event_callback = event_callback if monitor_callback: t.monitor_callback = monitor_callback def _do_file_copy(self, source, destination, source_size): def event_callback(event): if self.params.verbose: print("event: %s" % str(event)) def monitor_callback(src, dst, avg, inst, trans, elapsed): if self.params.verbose: print("monitor: %s %s %s %s %s %s" % ( str(src), str(dst), str(avg), str(inst), str(trans), str(elapsed) )) if self.progress_bar: self.progress_bar.update(trans, source_size, avg, elapsed) t = self.context.transfer_parameters() self._setup_transfer_params(t, event_callback, monitor_callback) self.progress_bar = None if not self.params.dry_run and not self.params.verbose and sys.stdout.isatty(): self.progress_bar = Progress("Copying %s" % source) self.progress_bar.update(total_size=source_size) self.progress_bar.start() else: print("Copying %d bytes %s => %s" % (source_size, source, destination)) try: if not self.params.dry_run: ret = self.context.filecopy(t, source, destination) if self.progress_bar: self.progress_bar.stop(True) print("") except gfal2.GError: e = sys.exc_info()[1] if self.progress_bar: self.progress_bar.stop(False) print("") if e.code == errno.EEXIST and self.params.force: self.context.unlink(destination) return self._do_file_copy(source, destination, source_size) return self._failure(e.message, e.code) except SystemError: #e = sys.exc_info()[1] #etype, value = sys.exc_info()[:2] etype, value, tb = sys.exc_info() print("ERROR: %s (%s)" % (str(etype), str(value))) import traceback traceback.print_exception(etype, value, tb) gfal2-util-v1.8.1/src/gfal2_util/gfal2_utils_parameters.py000066400000000000000000000064631453604140300235150ustar00rootroot00000000000000# # Copyright (c) 2013-2022 CERN # # Copyright (c) 2012-2013 Members of the EMI Collaboration # See http://www.eu-emi.eu/partners for details on the copyright # holders. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import division def get_parameter_from_str_list(str_value_list): str_value_tab = str_value_list.split(",") return [get_parameter_from_str(i) for i in str_value_tab] def get_parameter_from_str(str_value): def str2bool(v): s = v.lower() if s in ["true", "yes", "y", "1"]: return True if s in ["false", "no", "n", "0"]: return False raise ValueError("is not a boolean") for i in [int, str2bool, str]: try: return i(str_value) except ValueError: pass raise ValueError("not a valid parameter type") def parse_parameter(str_params): value_sep = str_params.find('=') if value_sep == -1: raise ValueError("parameter '%s' doesn't include value, use 'group:option=value'" % str_params) group_sep = str_params[:value_sep].rfind(':') if group_sep == -1: raise ValueError("parameter '%s' doesn't include group name, use 'group:option=value'" % str_params) group = str_params[:group_sep] option = str_params[group_sep + 1:value_sep] value = str_params[value_sep + 1:] return (group, option, get_parameter_from_str_list(value)) def set_gfal_tool_parameter(context, param_struct): def set_params_struct(p, f): f(p[0], p[1], p[2][0]) if len(param_struct[2]) > 1: context.set_opt_string_list(param_struct[0], param_struct[1], param_struct[2]) elif int == type(param_struct[2][0]): set_params_struct(param_struct, context.set_opt_integer) elif bool == type(param_struct[2][0]): set_params_struct(param_struct, context.set_opt_boolean) elif str == type(param_struct[2][0]): set_params_struct(param_struct, context.set_opt_string) else: raise ValueError("not a valid parameter type") def apply_option(context, params): if params.definition: p_list = [parse_parameter(str_param[0]) for str_param in params.definition] [set_gfal_tool_parameter(context, tuple_param) for tuple_param in p_list] if params.client_info: for cinfo in params.client_info: try: key, val = cinfo.split('=', 2) context.add_client_info(key, val) except: context.add_client_info(cinfo, '') if params.ipv6: context.set_opt_boolean("GRIDFTP PLUGIN", "IPV6", True) elif params.ipv4: context.set_opt_boolean("GRIDFTP PLUGIN", "IPV6", False) if params.timeout: context.set_opt_integer("CORE", "NAMESPACE_TIMEOUT", params.timeout) context.set_opt_integer("CORE", "CHECKSUM_TIMEOUT", params.timeout) gfal2-util-v1.8.1/src/gfal2_util/legacy.py000066400000000000000000000040021453604140300203060ustar00rootroot00000000000000# # Copyright (c) 2013-2022 CERN # # Copyright (c) 2012-2013 Members of the EMI Collaboration # See http://www.eu-emi.eu/partners for details on the copyright # holders. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. #from __future__ import absolute_import # not available in python 2.4 from __future__ import division from gfal2_util import base class CommandLegacy(base.CommandBase): """ Implement some legacy support around gfal2 """ @base.arg('lfc', action='store', type=base.surl, help="LFC entry (lfc:// or guid:)") @base.arg('surl', action='store', type=base.surl, help="Site URL to be unregistered") def execute_unregister(self): """ Unregister a replica. """ value = '-' + self.params.surl self.context.setxattr(self.params.lfc, 'user.replicas', value, len(value)) @base.arg('lfc', action='store', type=base.surl, help="LFC entry (lfc:// or guid:)") @base.arg('surl', action='store', type=base.surl, help="Site URL to be unregistered") def execute_register(self): """ Register a replica. """ value = '+' + self.params.surl self.context.setxattr(self.params.lfc, 'user.replicas', value, len(value)) @base.arg('lfc', action='store', type=base.surl, help="LFC entry (lfc:// or guid:)") def execute_replicas(self): """ List replicas. """ replicas = self.context.getxattr(self.params.lfc, 'user.replicas').split('\n') for replica in replicas: print(replica) gfal2-util-v1.8.1/src/gfal2_util/ls.py000066400000000000000000000163611453604140300174730ustar00rootroot00000000000000# # Copyright (c) 2013-2022 CERN # # Copyright (c) 2012-2013 Members of the EMI Collaboration # See http://www.eu-emi.eu/partners for details on the copyright # holders. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. #from __future__ import absolute_import # not available in python 2.4 from __future__ import division import logging import math from datetime import datetime import os import sys import stat from gfal2_util import base from gfal2_util.utils import file_mode_str def full_iso(date): dt = datetime.fromtimestamp(date) if sys.version_info >= (2, 6): return dt.strftime("%Y-%m-%d %H:%M:%S.%f +0000") else: # python 2.4 and 2.5 doesn't support %f in strftime return dt.strftime("%Y-%m-%d %H:%M:%S.000000 +0000") def long_iso(date): dt = datetime.fromtimestamp(date) return dt.strftime("%Y-%m-%d %H:%M") def time_iso(date): dt = datetime.fromtimestamp(date) dt_now = datetime.now() diff_months = (dt_now - dt).days / 30 # approximate... if diff_months < 6: return dt.strftime("%m-%d %H:%M") else: return dt.strftime("%Y-%m-%d") def time_locale(date): dt = datetime.fromtimestamp(date) dt_now = datetime.now() diff_months = (dt_now - dt).days / 30 # approximate... day = dt.strftime("%d").lstrip("0").rjust(2) if diff_months < 6: return dt.strftime("%b " + day + " %H:%M") else: return dt.strftime("%b " + day + " %Y") time_formats = { 'full-iso': full_iso, 'long-iso': long_iso, 'iso': time_iso, 'locale': time_locale } color_dict = dict() color_env = os.environ.get('LS_COLORS', None) if color_env: for entry in [entry for entry in color_env.split(':') if '=' in entry]: try: typ, color = entry.split('=') color_dict[typ] = color except Exception: e = sys.exc_info()[1] sys.stderr.write("unparsable value for LS_COLORS environment variable: %s\n" % entry) class CommandLs(base.CommandBase): @base.arg('-a', '--all', action="store_true", help="display hidden files") @base.arg('-l', '--long', action="store_true", help='long listing format') @base.arg('-d', '--directory', action="store_true", help='list directory entries instead of contents') @base.arg('-H', '--human-readable', action="store_true", help='with -l, prints size in human readable format (e.g., 1K 234M 2G') @base.arg('--xattr', type=str, action='append', default=[], help="query additional attributes. Can be specified multiple times. Only works for --long output") @base.arg('--time-style', type=str, default='locale', choices=list(time_formats.keys()), help="time style") @base.arg('--full-time', action="store_true", help="same as --time-style=full-iso") @base.arg('--color', type=str, choices=['always', 'never', 'auto'], default='auto', help='print colored entries with -l') @base.arg('file', type=base.surl, help="file's uri") def execute_ls(self): """List directory's contents""" if self.params.full_time: self.params.time_style = 'long-iso' st = self.context.stat(self.params.file) if stat.S_ISDIR(st.st_mode) and not self.params.directory: directory = self.context.opendir(self.params.file) st = None while True: if self.params.long: (dirent, st) = directory.readpp() else: dirent = directory.read() if dirent is None or dirent.d_name is None or len(dirent.d_name) == 0: break if not self.params.all and dirent.d_name[0] == '.': continue extra = list() if self.params.long: for xattr in self.params.xattr: surl = os.path.join(self.params.file, dirent.d_name) extra.append(self.context.getxattr(surl, xattr)) self._print_ls_entry(dirent.d_name, st, extra) else: extra = list() if self.params.long: for xattr in self.params.xattr: extra.append(self.context.getxattr(self.params.file, xattr)) self._print_ls_entry(self.params.file, st, extra) return 0 def _print_ls_entry(self, name, stat, extra=None): space = { 'st_mode': 5, 'st_nlink': 3, 'st_gid': 5, 'st_uid': 5, 'st_mtime': 11, 'size': 9, 'size_human': 4 } #if long, print some stuff from stat. Try to align as best as possible without buffering if self.params.long: size = stat.st_size size_sp = space['size'] if self.params.human_readable: size = self._size_to_human(size) size_sp = space['size_human'] date = time_formats[self.params.time_style](stat.st_mtime) extra_str = '' if extra: extra_str = '\t'.join(extra) sys.stdout.write( "%s %s %s %s %s %s %s\t%s\n" % ( file_mode_str(stat.st_mode), str(stat.st_nlink).rjust(space['st_nlink']), str(stat.st_uid).ljust(space['st_uid']), str(stat.st_gid).ljust(space['st_gid']), str(size).rjust(size_sp), str(date).ljust(space['st_mtime']), self.color(name, stat.st_mode), extra_str ) ) else: sys.stdout.write("%s\n" % self.color(name, None)) @staticmethod def _size_to_human(size): degree_symbols = ['', 'K', 'M', 'G', 'T', 'P'] degree = 0 while float(size) >= 1024.0 and degree < len(degree_symbols)-1: size = float(size) / 1024.0 degree += 1 #round up the result if size < 10.0: return "%0.1f%s" % (math.ceil(size*10.0)/10.0, degree_symbols[degree]) else: return "%0.0f%s" % (math.ceil(size), degree_symbols[degree]) def color(self, name, mode): apply_color = False if self.params.color == 'always': apply_color = True elif self.params.color == 'auto': apply_color = sys.stdout.isatty() if not apply_color: return name color = '037' if mode is None: color = color_dict.get('no', color) elif stat.S_ISDIR(mode): color = color_dict.get('di', color) elif stat.S_ISLNK(mode): color = color_dict.get('ln', color) elif mode & (stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH): color = color_dict.get('ex', color) return '\033[%sm%s\033[0m' % (color, name) gfal2-util-v1.8.1/src/gfal2_util/progress.py000066400000000000000000000202351453604140300207140ustar00rootroot00000000000000# # Copyright (c) 2013-2022 CERN # # Copyright (c) 2012-2013 Members of the EMI Collaboration # See http://www.eu-emi.eu/partners for details on the copyright # holders. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import division import sys import datetime import math import time import threading import fcntl import termios import struct class Progress(object): def __init__(self, label): self.started = False self.stopped = False self.status = None self.label = label self.lock = threading.Lock() def start(self): #don't let it start twice, thread safe self.lock.acquire() if self.started or self.stopped: self.lock.release() raise self.lock.release() self.started = True self.start_time = datetime.datetime.now() self.dots = 0 self.t_main = threading.Thread(target=self._run) self.t_main.daemon = True if not hasattr(self.t_main, 'is_alive'): # is_alive was added in python 2.6 and isAlive deprecated in python 3.8 self.t_main.is_alive = self.t_main.isAlive self.t_main.start() @staticmethod def _total_seconds(td): #for compatibility with python < 2.7 return (td.microseconds + (td.seconds + td.days * 24 * 3600) * 10**6) / 10**6 def _run(self): #use lock to avoid main thread assume we are done while we are printing while True: self.lock.acquire() if self.stopped or not self.started: break try: self._update() finally: self.lock.release() time.sleep(0.5) self.lock.release() def _update(self): """ Prints label, time elapsed, percentage, progress bar, size transferred, transfer rate. Some elements are optional, depending on the available info """ time_elapsed = datetime.datetime.now() - self.start_time self._clean() #write label str_label = self.label + ('.' * self.dots).ljust(3) sys.stdout.write(str_label) #write total time elapsed str_time = " %ds " % self._total_seconds(time_elapsed) sys.stdout.write(str_time) #we hold the lock, so no concurrency probs while accessing status if self.status: total_width = self._get_width() #we have everything if self.status.get('percentage'): str_percentage = str(int(round(self.status['percentage']))) + '% ' str_rate = self._get_rate_str(self.status['rate']) str_curr_size = ' ' + self._get_size_str(self.status['curr_size']) + ' ' used_width = len(str_label) + len(str_time) + len(str_percentage) + len(str_curr_size) + len(str_rate) unused_width = total_width - used_width if unused_width >= 7: bar_progr_width = unused_width-2 else: bar_progr_width = 5 bars_len = int(round((float(self.status['percentage'])*bar_progr_width)/100.0)) if bars_len == 0: bars_len = 1 space_len = unused_width-bars_len-2 progress_str = '[' + '='*(bars_len-1) + '>' + ' '*space_len + ']' sys.stdout.write(str_percentage) sys.stdout.write(progress_str) #write total data size sys.stdout.write(str_curr_size) #write average transfer rate sys.stdout.write(str_rate) elif self.status.get('total_size'): str_filesize = " File size: %s" % \ self._get_size_str(self.status['total_size']) #write file size sys.stdout.write(str_filesize) used_width = len(str_label) + len(str_time) + len(str_filesize) unused_width = total_width - used_width #white space sys.stdout.write(' ' * unused_width) elif self.status.get('curr_size'): str_rate = '' if self.status.get('rate'): str_rate = self._get_rate_str(self.status['rate']) str_curr_size = self._get_size_str(self.status['curr_size']) + ' ' used_width = len(str_label) + len(str_time) + len(str_curr_size) + len(str_rate) unused_width = total_width - used_width #white space sys.stdout.write(' ' * unused_width) #write transferred data size sys.stdout.write(str_curr_size) #write average transfer rate sys.stdout.write(str_rate) else: used_width = len(str_label) + len(str_time) unused_width = total_width - used_width #white space sys.stdout.write(' ' * unused_width) sys.stdout.flush() self.dots += 1 if self.dots == 4: self.dots = 0 def update(self, curr_size=None, total_size=None, rate=None, time_elapsed=None): self.lock.acquire() self.status = {} if curr_size: self.status['curr_size'] = curr_size if total_size: self.status['total_size'] = total_size if curr_size and time_elapsed and total_size: self.status['rate'] = float(curr_size)/float(time_elapsed) self.status['percentage'] = (float(curr_size) / float(total_size))*100.0 elif rate: self.status['rate'] = rate self.lock.release() def stop(self, success): if not self.started: return if self.t_main.is_alive(): self.lock.acquire() if self.stopped: self.lock.release() return self.stopped = True self.lock.release() #thread might have died without releasing the lock else: if self.stopped: return self.stopped = True self._clean() time_elapsed = datetime.datetime.now() - self.start_time if success: outcome = 'DONE' else: outcome = 'FAILED' str_msg = self.label + ' [' + outcome + '] after %ds' % self._total_seconds(time_elapsed) spaces = self._get_width() - len(str_msg) sys.stdout.write(str_msg) sys.stdout.write(' ' * spaces) sys.stdout.flush() @staticmethod def _get_width(): try: data = fcntl.ioctl(sys.stdin.fileno(), termios.TIOCGWINSZ, struct.pack('HHHH', 0, 0, 0, 0)) h, w, hp, wp = struct.unpack('HHHH', data) return w except: # in case of exception just return default value pass return 80 @staticmethod def _clean(): sys.stdout.write('\r') @staticmethod def _get_size_str(size): size_str = Progress._get_rate_str(size)[:-2] if size_str[-1] != 'B': size_str = (size_str + "B") return size_str @staticmethod def _get_rate_str(rate): degree_symbols = ['B', 'K', 'M', 'G', 'T', 'P'] degree = 0 while float(rate) >= 1024.0 and degree < len(degree_symbols)-1: rate = float(rate) / 1024.0 degree += 1 digits = len(str(math.floor(rate))) str_units = degree_symbols[degree] + "/s" if digits < 3 and degree != 0: precision = 3 - digits else: precision = 0 fmt = "%0." + str(precision) + "f" str_rate = fmt % round(rate, precision) return str_rate + str_units gfal2-util-v1.8.1/src/gfal2_util/rm.py000066400000000000000000000126701453604140300174720ustar00rootroot00000000000000# # Copyright (c) 2013-2022 CERN # # Copyright (c) 2012-2013 Members of the EMI Collaboration # See http://www.eu-emi.eu/partners for details on the copyright # holders. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. #from __future__ import absolute_import # not available in python 2.4 from __future__ import division import sys import stat import errno import gfal2 from gfal2_util import base class CommandRm(base.CommandBase): def __init__(self): super(CommandRm, self).__init__() self.return_code = 0 @base.arg("-r", "-R", "--recursive", action='store_true', help="remove directories and their contents recursively") @base.arg("--dry-run", action='store_true', help="do not perform any actual change, just print what would happen") @base.arg("--just-delete", action='store_true', help="do not perform any check on the file, this is needed for HTTP signed URLs") @base.arg("--from-file", type=str, default=None, help="read surls from a file") @base.arg("--bulk", action='store_true', default=False, help="use bulk deletion") @base.arg("file", action='store', nargs='*', type=base.surl, help="uri(s) of the file(s) to be deleted") def execute_rm(self): """ Removes files or directories """ if self.params.from_file and self.params.file: sys.stderr.write("--from-file and positional arguments can not be used at the same time\n") return errno.EINVAL if self.params.bulk and self.params.recursive: sys.stderr.write("--bulk and --recursive can not be used at the same time\n") return errno.EINVAL if self.params.file: files = self.params.file elif self.params.from_file: files = [line.strip() for line in open(self.params.from_file, 'r').readlines()] files = [f for f in files if len(f) > 0] else: sys.stderr.write("Missing surl\n") return errno.EINVAL if self.params.bulk: self._do_bulk(files) else: for f in files: self._do_rm(f) return self.return_code def _do_rm(self, surl): """ Perform the action, either removing or just informing """ if not self.params.just_delete: try: st = self.context.stat(surl) except gfal2.GError: e = sys.exc_info()[1] self._propagate_error_code(e.code) if e.code == errno.ENOENT: print("%s\tMISSING" % surl) return else: print("%s\tFAILED" % surl) raise if stat.S_ISDIR(st.st_mode): self._do_rmdir(surl) return if self.params.dry_run: print("%s\tSKIP" % surl) return try: self.context.unlink(surl) print("%s\tDELETED" % surl) except gfal2.GError: e = sys.exc_info()[1] self._propagate_error_code(e.code) if e.code == errno.ENOENT: print("%s\tMISSING" % surl) return else: print("%s\tFAILED" % surl) raise def _do_rmdir(self, surl): """ Remove a directory recursively """ if not self.params.recursive: raise gfal2.GError("Can not remove %s, is a directory" % surl, errno.EISDIR) # Content base_dir = surl if base_dir[-1] != '/': base_dir += '/' contents = self.context.listdir(surl) for c in contents: if c in ['.', '..']: continue self._do_rm(base_dir + c) # Rmdir self if self.params.dry_run: print("%s\tSKIP DIR" % surl) else: try: self.context.rmdir(surl) print("%s\tRMDIR" % surl) except gfal2.GError: e = sys.exc_info()[1] self._propagate_error_code(e.code) if e.code == errno.ENOENT: print("%s\tMISSING" % surl) else: print("%s\tFAILED" % surl) raise def _do_bulk(self, surls): """ Do a bulk deletion """ if self.params.dry_run: print("\tBULK DELETION") return errors = self.context.unlink(surls) for error, surl in zip(errors, surls): if error is None: print("%s\tDELETED" % surl) else: print("%s\tFAILED: %s" % (surl, error)) self._propagate_error_code(error.code) def _propagate_error_code(self, error_code): """ Propagate the error code if the return code is empty """ if self.return_code == 0: self.return_code = error_code gfal2-util-v1.8.1/src/gfal2_util/shell.py000066400000000000000000000043301453604140300201550ustar00rootroot00000000000000# # Copyright (c) 2013-2022 CERN # # Copyright (c) 2012-2013 Members of the EMI Collaboration # See http://www.eu-emi.eu/partners for details on the copyright # holders. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. #from __future__ import absolute_import # not available in python 2.4 from __future__ import division from gfal2_util import base from gfal2_util import commands # @UnusedImport from gfal2_util import ls # @UnusedImport from gfal2_util import legacy # @UnusedImport from gfal2_util import copy # @UnusedImport from gfal2_util import rm # @UnusedImport from gfal2_util import tape class CommandFactory(object): @staticmethod def __find_classes(): return base.CommandBase().get_subclasses() @staticmethod def __find_command(clasz, cmd): for name in (a for a in dir(clasz) if a == ('execute_' + cmd)): return getattr(clasz, name) return None @staticmethod def get_command(cmd): classes = CommandFactory.__find_classes() for c in classes: func = CommandFactory.__find_command(c, cmd) if func: return c, func raise ValueError("Invalid command") class Gfal2Shell(object): @staticmethod def __get_command_string(progr): return progr.rsplit('-', 1)[1].lower() def main(self, args): """Entry point""" #get class/function corresponding to the command issued cmd = self.__get_command_string(args[0]) (command_class, command_func) = CommandFactory.get_command(cmd) #instantiate class and let it know what is the func called and the args given inst = command_class() inst.parse(command_func, args) return inst.execute(command_func) gfal2-util-v1.8.1/src/gfal2_util/tape.py000066400000000000000000000124321453604140300200010ustar00rootroot00000000000000# # Copyright (c) 2022 CERN # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import time import sys import errno from gfal2_util import base def _evaluate_errors(errors, surls, polling): n_terminal = 0 for surl, error in zip(surls, errors): if error: if error.code != errno.EAGAIN: print("%s => FAILED: %s" % (surl, error.message)) n_terminal += 1 else: print("%s QUEUED" % surl) elif not polling: print("%s QUEUED" % surl) else: n_terminal += 1 print("%s READY" % surl) return n_terminal class CommandTape(base.CommandBase): """ Implement tape operations support via Gfal2 library """ @base.arg('--pin-lifetime', action='store', type=int, default=0, help='Desired pin lifetime') @base.arg('--desired-request-time', action='store', type=int, default=28800, help='Desired total request time') @base.arg('--staging-metadata', action='store', type=str, default="", help='Metadata for the bringonline operation') @base.arg('--polling-timeout', action='store', type=int, default=0, help='Timeout for the polling operation') @base.arg('--from-file', type=str, default=None, help="read surls from a file") @base.arg('surl', action='store', type=base.surl, nargs='?', help='Site URL') def execute_bringonline(self): """ Execute bring online """ if self.params.from_file and self.params.surl: sys.stderr.write('Could not combine --from-file with a surl in the positional arguments\n' ) return 1 surls = list() if self.params.from_file: src_file = open(self.params.from_file) for src in map(str.strip, src_file.readlines()): if src: surls.append(src) src_file.close() elif self.params.surl: surls.append(self.params.surl) else: sys.stderr.write('Missing surl\n') return 1 nbfiles = len(surls) metadata_list = [self.params.staging_metadata] * nbfiles # Create bringonline request (errors, token) = self.context.bring_online(surls, metadata_list, self.params.pin_lifetime, self.params.desired_request_time, True) n_terminal = _evaluate_errors(errors, surls, polling=False) # Start the polling wait = self.params.polling_timeout sleep = 1 while n_terminal != nbfiles and wait > 0: print("Request queued, sleep %d seconds..." % sleep) wait -= sleep time.sleep(sleep) errors = self.context.bring_online_poll(surls, token) n_terminal = _evaluate_errors(errors, surls, polling=True) sleep *= 2 sleep = min(sleep, 300) @base.arg('--polling-timeout', action='store', type=int, default=0, help='Timeout for the polling operation') @base.arg('--from-file', type=str, default=None, help="read surls from a file") @base.arg('surl', action='store', type=base.surl, nargs='?', help='Site URL') def execute_archivepoll(self): """ Execute bring online """ if self.params.from_file and self.params.surl: sys.stderr.write('Could not combine --from-file with a surl in the positional arguments\n' ) return 1 surls = list() if self.params.from_file: src_file = open(self.params.from_file) for src in map(str.strip, src_file.readlines()): if src: surls.append(src) src_file.close() elif self.params.surl: surls.append(self.params.surl) else: sys.stderr.write('Missing surl\n') return 1 nbfiles = len(surls) wait = self.params.polling_timeout sleep = 1 errors = self.context.archive_poll(surls) n_terminal = _evaluate_errors(errors, surls, polling=True) while n_terminal != nbfiles and wait > 0: print("Archiving ongoing, sleep %d seconds..." % sleep) wait -= sleep time.sleep(sleep) errors = self.context.archive_poll(surls) n_terminal = _evaluate_errors(errors, surls, polling=True) sleep *= 2 sleep = min(sleep, 300) @base.arg('file', action='store', type=base.surl, help="URI to the file to be evicted") @base.arg('token', type=str, nargs='?', default="", help="The token from the bring online request") def execute_evict(self): """ Evict file from a disk buffer """ st = self.context.release(self.params.file, self.params.token) gfal2-util-v1.8.1/src/gfal2_util/utils.py000066400000000000000000000043411453604140300202100ustar00rootroot00000000000000# # Copyright (c) 2013-2022 CERN # # Copyright (c) 2012-2013 Members of the EMI Collaboration # See http://www.eu-emi.eu/partners for details on the copyright # holders. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import division import signal import stat class Timeout: """Timeout class using ALARM signal.""" class Timeout(Exception): pass def __init__(self, sec): self.sec = sec def __enter__(self): signal.signal(signal.SIGALRM, self.raise_timeout) signal.alarm(self.sec) def __exit__(self, *args): signal.alarm(0) # disable alarm def raise_timeout(self, *args): raise Timeout.Timeout() def file_type_str(type): ftype_str = { stat.S_IFBLK: 'block device', stat.S_IFCHR: 'character device', stat.S_IFDIR: 'directory', stat.S_IFIFO: 'fifo', stat.S_IFLNK: 'symbolic link', stat.S_IFREG: 'regular file', stat.S_IFSOCK: 'socket' } repr = ftype_str.get(type, 'unknown') return repr def _mode_str_triplet(mode): mode_str = ['-'] * 3 if mode & stat.S_IROTH: mode_str[0] = 'r' if mode & stat.S_IWOTH: mode_str[1] = 'w' if mode & stat.S_IXOTH: mode_str[2] = 'x' return ''.join(mode_str) def file_mode_str(mode): mode_str = '' if stat.S_ISDIR(mode): mode_str += 'd' elif stat.S_ISBLK(mode): mode_str +='b' elif stat.S_ISCHR(mode): mode_str += 'c' elif stat.S_ISFIFO(mode): mode_str += 'f' elif stat.S_ISSOCK(mode): mode_str += 's' else: mode_str += '-' mode_str += _mode_str_triplet(mode >> 6) + _mode_str_triplet(mode >> 3) + _mode_str_triplet(mode) return mode_str gfal2-util-v1.8.1/test/000077500000000000000000000000001453604140300146345ustar00rootroot00000000000000gfal2-util-v1.8.1/test/functional/000077500000000000000000000000001453604140300167765ustar00rootroot00000000000000gfal2-util-v1.8.1/test/functional/__init__.py000066400000000000000000000000001453604140300210750ustar00rootroot00000000000000gfal2-util-v1.8.1/test/functional/base.py000066400000000000000000000016111453604140300202610ustar00rootroot00000000000000import unittest import shutil import os import utils class TestBase(unittest.TestCase): def setUp(self): self.dirname = '/tmp/test_' + utils.create_random_suffix() os.mkdir(self.dirname) self.subdir = self.dirname + '/test_' + utils.create_random_suffix() os.mkdir(self.subdir) self.fname1 = 'f1_' + utils.create_random_suffix() self.fname2 = 'f2_' + utils.create_random_suffix() self.ffname1 = self.dirname + '/' + self.fname1 self.ffname2 = self.dirname + '/' + self.fname2 utils.create_file(self.ffname1, 1025) utils.create_file(self.ffname2, 1025) def tearDown(self): if os.path.isdir(self.dirname): shutil.rmtree(self.dirname) if(os.path.isfile(self.ffname2)): os.remove(self.ffname2) if(os.path.isfile(self.ffname1)): os.remove(self.ffname1) gfal2-util-v1.8.1/test/functional/test_all.py000066400000000000000000000006201453604140300211550ustar00rootroot00000000000000import sys if __name__ == '__main__' and __package__: import os sys.path.append(os.path.dirname(__file__)) from test_ls import * from test_copy import * from test_general import * from test_rm import * from test_mkdir import * from xmlrunner import XMLTestRunner if __name__ == '__main__': s=open('test_results.xml', 'w') unittest.main(testRunner=XMLTestRunner(stream=sys.stdout)) gfal2-util-v1.8.1/test/functional/test_copy.py000066400000000000000000000104011453604140300213550ustar00rootroot00000000000000import unittest import os from base import TestBase import utils class UtilCopyTest(TestBase): def test_copy(self): ffname3 = self.ffname1 + "_copy" self.assertFalse(os.path.isfile(ffname3)) (ret, out, err) = utils.run_command('gfal-copy', \ 'file://' + self.ffname1 + ' file://' + ffname3) self.assertTrue(os.path.isfile(ffname3)) self.assertEqual(ret, 0) if os.path.isfile(ffname3): os.remove(ffname3) def test_copy_no_basename(self): self.assertTrue(os.path.isfile(self.ffname1)) dst_path = '/tmp/' dst = dst_path + os.path.basename(self.ffname1) self.assertFalse(os.path.isfile(dst)) (ret, out, err) = utils.run_command('gfal-copy', \ 'file://' + self.ffname1 + ' file://' + dst_path) self.assertTrue(os.path.isfile(dst)) self.assertEqual(ret, 0) if os.path.isfile(dst): os.remove(dst) def test_chain_copy(self): self.assertTrue(os.path.isfile(self.ffname1)) d1 = self.ffname1 + '_cp1' d2 = self.ffname1 + '_cp2' d3 = '/tmp' d4 = self.ffname1 + '_cp3' args = 'file://' + self.ffname1 + ' file://' + d1 + ' file://' + d2 \ + ' file://' + d3 + ' file://' + d4 (ret, out, err) = utils.run_command('gfal-copy', args) self.assertTrue(os.path.isfile(d1)) self.assertTrue(os.path.isfile(d2)) self.assertTrue(os.path.isfile(d3 + '/' + os.path.basename(d2))) self.assertTrue(os.path.isfile(d4)) if os.path.isfile(d1): os.remove(d1) if os.path.isfile(d2): os.remove(d2) if os.path.isfile(d3): os.remove(d3) def test_copy_dst_dir(self): self.assertTrue(os.path.isfile(self.ffname1)) dst_path = '/tmp' dst = dst_path + '/' + os.path.basename(self.ffname1) self.assertFalse(os.path.isfile(dst)) (ret, out, err) = utils.run_command('gfal-copy', \ 'file://' + self.ffname1 + ' file://' + dst_path) self.assertTrue(os.path.isfile(dst)) self.assertEqual(ret, 0) if os.path.isfile(dst): os.remove(dst) def test_copy_dir(self): self.assertTrue(os.path.isfile(self.ffname1)) src = os.path.dirname(self.ffname1) self.assertTrue(os.path.isdir(src)) dst = '/tmp/' (ret, out, err) = utils.run_command('gfal-copy', \ 'file://' + src + ' file://' + dst) self.assertFalse(os.path.isfile(dst + os.path.basename(self.ffname1))) self.assertTrue(ret >= 0) def test_copy_parent_enoent(self): self.assertTrue(os.path.isfile(self.ffname1)) dst_path = '/tmp/make/parent' self.assertFalse(os.path.isfile(dst_path)) (ret, out, err) = utils.run_command('gfal-copy', 'file://' + self.ffname1 + ' file://' + dst_path) self.assertFalse(os.path.isfile(dst_path)) self.assertTrue(ret >= 0) def test_copy_parent_mkdir(self): self.assertTrue(os.path.isfile(self.ffname1)) dst_path = '/tmp/make/parent' self.assertFalse(os.path.isfile(dst_path)) (ret, out, err) = utils.run_command('gfal-copy', '-p file://' + self.ffname1 + ' file://' + dst_path) self.assertTrue(os.path.isfile(dst_path)) self.assertTrue(ret >= 0) os.unlink(dst_path) os.rmdir('/tmp/make/') def test_copy_pseudotty(self): """ Regression test for DMC-522 Trick gfal-copy into thinking it is inside a tty so we trigger some logic that would not be executed otherwise """ ffname3 = self.ffname1 + "_copy" self.assertFalse(os.path.isfile(ffname3)) (ret, out, err) = utils.run_command_pty('gfal-copy', \ 'file://' + self.ffname1 + ' file://' + ffname3) self.assertTrue(os.path.isfile(ffname3)) self.assertNotEqual(len(out), 0) # this makes sure the interactive mode works! self.assertEqual(ret, 0) if os.path.isfile(ffname3): os.remove(ffname3) if __name__ == '__main__': unittest.main() gfal2-util-v1.8.1/test/functional/test_general.py000066400000000000000000000016441453604140300220310ustar00rootroot00000000000000from builtins import bytes import unittest import shutil import os from base import TestBase import utils class UtilGeneralTest(TestBase): def test_protocol_not_supported(self): (ret, out, err) = utils.run_command('gfal-ls', 'xyzf://fakepath') self.assertEqual(len(out), 0) self.assertTrue(bytes('Protocol not supported', 'utf-8') in err) self.assertEqual(ret, 93) def _test_full_path(self): (ret, out, err) = utils.run_command('/usr/local/bin/gfal-ls', 'file://' + self.dirname) self.assertEqual(len(err), 0) self.assertEqual(ret, 0) def _test_non_verbose_error(self): (ret, out, err) = utils.run_command('/usr/local/bin/gfal-ls', self.dirname) self.assertEqual(len(out), 0) self.assertEqual(len(err.splitlines()), 1) self.assertTrue(err.startswith('gfal-ls:')) if __name__ == '__main__': unittest.main() gfal2-util-v1.8.1/test/functional/test_ls.py000066400000000000000000000035131453604140300210270ustar00rootroot00000000000000from builtins import bytes import unittest import os from base import TestBase import utils class UtilLsTest(TestBase): def test_size(self): (ret, out, err) = utils.run_command('gfal-ls', '-lH ' + ' file://' + self.ffname1) self.assertTrue(bytes(' 1.1K ', 'utf-8') in out) self.assertEqual(ret, 0) (ret, out, err) = utils.run_command('gfal-ls', '-l' + ' file://' + self.ffname1) self.assertTrue(bytes(' 1025 ', 'utf-8') in out) self.assertEqual(ret, 0) def test_invalid(self): inv_name = self.ffname1 + "INVALID" (ret, out, err) = utils.run_command('gfal-ls', '-lH ' + ' file://' + inv_name) self.assertEqual(ret, 2) self.assertTrue(bytes('No such file or directory', 'utf-8') in err) self.assertEqual(len(out), 0) def test_basic(self): (ret, out, err) = utils.run_command('gfal-ls', 'file://' + self.dirname) self.assertEqual(len(out.splitlines()), utils.num_entries(self.dirname)) self.assertTrue(bytes(self.fname1, 'utf-8') in out) self.assertTrue(bytes(self.fname2, 'utf-8') in out) self.assertEqual(ret, 0) def test_directory(self): (ret, out, err) = utils.run_command('gfal-ls', '-ld' + ' file://' + self.ffname1) self.assertEqual(len(out.splitlines()), 1) self.assertEqual(ret, 0) def test_name(self): (ret, out, err) = utils.run_command('gfal-ls', '-l' + ' file://' + self.ffname1) self.assertTrue(bytes(' file://' + self.ffname1, 'utf-8') in out) self.assertEqual(ret, 0) (ret, out, err) = utils.run_command('gfal-ls', '-dl' + ' file://' + self.ffname1) self.assertTrue(bytes(' file://' + self.ffname1, 'utf-8') in out) self.assertEqual(ret, 0) if __name__ == '__main__': unittest.main() gfal2-util-v1.8.1/test/functional/test_mkdir.py000066400000000000000000000047711453604140300215260ustar00rootroot00000000000000from builtins import bytes import unittest import os import shutil from base import TestBase import utils class UtilMkdirTest(TestBase): def test_mkdir(self): shutil.rmtree(self.dirname) self.assertFalse(os.path.isdir(self.dirname)) (ret, out, err) = utils.run_command('gfal-mkdir', 'file://' + self.dirname) self.assertTrue(os.path.isdir(self.dirname)) self.assertEqual(utils.get_permissions(self.dirname), '755') self.assertEqual(len(out), 0) self.assertEqual(ret, 0) def test_mkdir_mode(self): shutil.rmtree(self.dirname) self.assertFalse(os.path.isdir(self.dirname)) (ret, out, err) = utils.run_command('gfal-mkdir', '-m 0700 file://' + self.dirname) self.assertTrue(os.path.isdir(self.dirname)) self.assertEqual(utils.get_permissions(self.dirname), '700') self.assertEqual(len(out), 0) self.assertEqual(ret, 0) def test_mkdir_recursive(self): d = self.dirname + '/subdir/subdir' shutil.rmtree(self.dirname) self.assertFalse(os.path.isdir(d)) (ret, out, err) = utils.run_command('gfal-mkdir', '-p file://' + d) self.assertTrue(os.path.isdir(d)) self.assertEqual(utils.get_permissions(d), '755') self.assertEqual(len(out), 0) self.assertEqual(ret, 0) def test_already_exists(self): self.assertTrue(os.path.isdir(self.dirname)) (ret, out, err) = utils.run_command('gfal-mkdir', 'file://' + self.dirname) self.assertTrue(os.path.isdir(self.dirname)) self.assertEqual(len(out), 0) self.assertTrue(len(err) > 0) self.assertTrue(bytes('File exists', 'utf-8') in err) self.assertEqual(ret, 17) def test_already_exists_p(self): self.assertTrue(os.path.isdir(self.dirname)) (ret, out, err) = utils.run_command('gfal-mkdir', '-p file://' + self.dirname) self.assertTrue(os.path.isdir(self.dirname)) self.assertEqual(len(out), 0) self.assertEqual(ret, 0) def test_invalid_mode(self): shutil.rmtree(self.dirname) self.assertFalse(os.path.isdir(self.dirname)) (ret, out, err) = utils.run_command('gfal-mkdir', '-m A file://' + self.dirname) self.assertFalse(os.path.isdir(self.dirname)) self.assertFalse(ret == 0) if __name__ == '__main__': unittest.main() gfal2-util-v1.8.1/test/functional/test_results.xml000066400000000000000000000040461453604140300222640ustar00rootroot00000000000000 gfal2-util-v1.8.1/test/functional/test_rm.py000066400000000000000000000022141453604140300210240ustar00rootroot00000000000000from builtins import bytes import unittest import os from base import TestBase import utils class UtilRmTest(TestBase): def test_recursive(self): self.assertTrue(os.path.exists(self.dirname)) (ret, out, err) = utils.run_command('gfal-rm', '-r ' + ' file://' + self.dirname) self.assertFalse(os.path.exists(self.dirname)) self.assertEqual(ret, 0) def test_dir_non_rec(self): self.assertTrue(os.path.exists(self.dirname)) (ret, out, err) = utils.run_command('gfal-rm', 'file://' + self.dirname) self.assertTrue(os.path.exists(self.dirname)) self.assertTrue(bytes("directory", 'utf-8') in err) self.assertEqual(ret, 21) def test_multiple(self): self.assertTrue(os.path.exists(self.ffname1)) self.assertTrue(os.path.exists(self.ffname2)) (ret, out, err) = utils.run_command('gfal-rm',\ 'file://' + self.ffname1 + ' file://' + self.ffname2) self.assertFalse(os.path.exists(self.ffname1)) self.assertFalse(os.path.exists(self.ffname2)) self.assertEqual(ret, 0) if __name__ == '__main__': unittest.main() gfal2-util-v1.8.1/test/functional/utils.py000066400000000000000000000030221453604140300205050ustar00rootroot00000000000000import subprocess import datetime import os import pty import select import stat import inspect def create_file(path, size): fout = open(path, 'wb') fout.write(bytearray(os.urandom(size))) fout.close() def create_random_suffix(): return datetime.datetime.now().strftime("%y%m%d_%H%M%S") def remove_file(path): os.remove(path) def run_command(cmd, args): script_path = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) cmd = script_path + '/../../src/' + cmd p = subprocess.Popen([cmd] + args.split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = p.communicate() return (p.returncode, out, err) def run_command_pty(cmd, args): script_path = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) cmd = script_path + '/../../src/' + cmd master, slave = pty.openpty() p = subprocess.Popen([cmd] + args.split(), stdout=slave, stderr=slave, close_fds=True) os.close(slave) output = '' while True: ready, _, _ = select.select([master], [], [], 1) if ready: try: data = os.read(ready[0], 512) except: data = None if data: output += str(data) else: break os.close(master) rstatus = p.wait() return (rstatus, output, None) def num_entries(directory): return len([name for name in os.listdir(directory)]) def get_permissions(file): return oct(os.stat(file)[stat.ST_MODE])[-3:] gfal2-util-v1.8.1/test/functional/xmlrunner.py000066400000000000000000000307271453604140300214130ustar00rootroot00000000000000# Written by Sebastian Rittau and placed in # the Public Domain. With contributions by Paolo Borelli and others. from __future__ import absolute_import """ XML Test Runner for PyUnit """ __version__ = "0.1" import os.path import re import sys import time import traceback import unittest from xml.sax.saxutils import escape try: from StringIO import StringIO except ImportError: from io import StringIO class _TestInfo(object): """Information about a particular test. Used by _XMLTestResult. """ def __init__(self, test, time): (self._class, self._method) = test.id().rsplit(".", 1) self._time = time self._error = None self._failure = None @staticmethod def create_success(test, time): """Create a _TestInfo instance for a successful test.""" return _TestInfo(test, time) @staticmethod def create_failure(test, time, failure): """Create a _TestInfo instance for a failed test.""" info = _TestInfo(test, time) info._failure = failure return info @staticmethod def create_error(test, time, error): """Create a _TestInfo instance for an erroneous test.""" info = _TestInfo(test, time) info._error = error return info def print_report(self, stream): """Print information about this test case in XML format to the supplied stream. """ stream.write(' ' % \ { "class": self._class, "method": self._method, "time": self._time, }) if self._failure is not None: self._print_error(stream, 'failure', self._failure) if self._error is not None: self._print_error(stream, 'error', self._error) stream.write('\n') def _print_error(self, stream, tagname, error): """Print information from a failure or error to the supplied stream.""" text = escape(str(error[1])) stream.write('\n') stream.write(' <%s type="%s">%s\n' \ % (tagname, _clsname(error[0]), text)) tb_stream = StringIO() traceback.print_tb(error[2], None, tb_stream) stream.write(escape(tb_stream.getvalue())) stream.write(' \n' % tagname) stream.write(' ') def _clsname(cls): return cls.__module__ + "." + cls.__name__ class _XMLTestResult(unittest.TestResult): """A test result class that stores result as XML. Used by XMLTestRunner. """ def __init__(self, classname): unittest.TestResult.__init__(self) self._test_name = classname self._start_time = None self._tests = [] self._error = None self._failure = None def startTest(self, test): unittest.TestResult.startTest(self, test) self._error = None self._failure = None self._start_time = time.time() def stopTest(self, test): time_taken = time.time() - self._start_time unittest.TestResult.stopTest(self, test) if self._error: info = _TestInfo.create_error(test, time_taken, self._error) elif self._failure: info = _TestInfo.create_failure(test, time_taken, self._failure) else: info = _TestInfo.create_success(test, time_taken) self._tests.append(info) def addError(self, test, err): unittest.TestResult.addError(self, test, err) self._error = err def addFailure(self, test, err): unittest.TestResult.addFailure(self, test, err) self._failure = err def print_report(self, stream, time_taken, out, err): """Prints the XML report to the supplied stream. The time the tests took to perform as well as the captured standard output and standard error streams must be passed in.a """ stream.write('\n' % \ { "n": self._test_name, "t": self.testsRun, "time": time_taken, }) for info in self._tests: info.print_report(stream) stream.write(' \n' % out) stream.write(' \n' % err) stream.write('\n') class XMLTestRunner(object): """A test runner that stores results in XML format compatible with JUnit. XMLTestRunner(stream=None) -> XML test runner The XML file is written to the supplied stream. If stream is None, the results are stored in a file called TEST-..xml in the current working directory (if not overridden with the path property), where and are the module and class name of the test class. """ def __init__(self, stream=None): self._stream = stream self._path = "." def run(self, test): """Run the given test case or test suite.""" class_ = test.__class__ classname = class_.__module__ + "." + class_.__name__ if self._stream == None: filename = "TEST-%s.xml" % classname stream = file(os.path.join(self._path, filename), "w") stream.write('\n') else: stream = self._stream result = _XMLTestResult(classname) start_time = time.time() self._orig_stdout = sys.stdout self._orig_stderr = sys.stderr sys.stdout = StringIO() sys.stderr = StringIO() test(result) try: out_s = sys.stdout.getvalue() except AttributeError: out_s = "" try: err_s = sys.stderr.getvalue() except AttributeError: err_s = "" sys.stdout = self._orig_stdout sys.stderr = self._orig_stderr time_taken = time.time() - start_time result.print_report(stream, time_taken, out_s, err_s) if self._stream is None: stream.close() return result def _set_path(self, path): self._path = path path = property(lambda self: self._path, _set_path, None, """The path where the XML files are stored. This property is ignored when the XML file is written to a file stream.""") class _fake_std_streams(object): def __enter__(self): self._orig_stdout = sys.stdout self._orig_stderr = sys.stderr sys.stdout = StringIO() sys.stderr = StringIO() def __exit__(self, exc_type, exc_val, exc_tb): sys.stdout = self._orig_stdout sys.stderr = self._orig_stderr class XMLTestRunnerTest(unittest.TestCase): def setUp(self): self._stream = StringIO() def _try_test_run(self, test_class, expected): """Run the test suite against the supplied test class and compare the XML result against the expected XML string. Fail if the expected string doesn't match the actual string. All time attributes in the expected string should have the value "0.000". All error and failure messages are reduced to "Foobar". """ runner = XMLTestRunner(self._stream) runner.run(unittest.makeSuite(test_class)) got = self._stream.getvalue() # Replace all time="X.YYY" attributes by time="0.000" to enable a # simple string comparison. got = re.sub(r'time="\d+\.\d+"', 'time="0.000"', got) # Likewise, replace all failure and error messages by a simple "Foobar" # string. got = re.sub(r'(?s).*?', r'Foobar', got) got = re.sub(r'(?s).*?', r'Foobar', got) # And finally Python 3 compatibility. got = got.replace('type="builtins.', 'type="exceptions.') self.assertEqual(expected, got) def test_no_tests(self): """Regression test: Check whether a test run without any tests matches a previous run. """ class TestTest(unittest.TestCase): pass self._try_test_run(TestTest, """ """) def test_success(self): """Regression test: Check whether a test run with a successful test matches a previous run. """ class TestTest(unittest.TestCase): def test_foo(self): pass self._try_test_run(TestTest, """ """) def test_failure(self): """Regression test: Check whether a test run with a failing test matches a previous run. """ class TestTest(unittest.TestCase): def test_foo(self): self.assert_(False) self._try_test_run(TestTest, """ Foobar """) def test_error(self): """Regression test: Check whether a test run with a erroneous test matches a previous run. """ class TestTest(unittest.TestCase): def test_foo(self): raise IndexError() self._try_test_run(TestTest, """ Foobar """) def test_stdout_capture(self): """Regression test: Check whether a test run with output to stdout matches a previous run. """ class TestTest(unittest.TestCase): def test_foo(self): sys.stdout.write("Test\n") self._try_test_run(TestTest, """ """) def test_stderr_capture(self): """Regression test: Check whether a test run with output to stderr matches a previous run. """ class TestTest(unittest.TestCase): def test_foo(self): sys.stderr.write("Test\n") self._try_test_run(TestTest, """ """) class NullStream(object): """A file-like object that discards everything written to it.""" def write(self, buffer): pass def test_unittests_changing_stdout(self): """Check whether the XMLTestRunner recovers gracefully from unit tests that change stdout, but don't change it back properly. """ class TestTest(unittest.TestCase): def test_foo(self): sys.stdout = XMLTestRunnerTest.NullStream() runner = XMLTestRunner(self._stream) runner.run(unittest.makeSuite(TestTest)) def test_unittests_changing_stderr(self): """Check whether the XMLTestRunner recovers gracefully from unit tests that change stderr, but don't change it back properly. """ class TestTest(unittest.TestCase): def test_foo(self): sys.stderr = XMLTestRunnerTest.NullStream() runner = XMLTestRunner(self._stream) runner.run(unittest.makeSuite(TestTest)) if __name__ == "__main__": unittest.main()