pax_global_header00006660000000000000000000000064146524001450014513gustar00rootroot0000000000000052 comment=205bc9307331d3d8da210a678d60bacaa7846525 gfal2-v2.23.0/000077500000000000000000000000001465240014500127605ustar00rootroot00000000000000gfal2-v2.23.0/.gitignore000066400000000000000000000003051465240014500147460ustar00rootroot00000000000000.idea .project .cproject .settings *.rpm *.tar.gz *.gz *.dsc *.o *.a *.so *.pc CMakeCache.txt **/Makefile **/CMakeFiles DartConfiguration.tcl doc/Doxyfile install_manifest.txt #build folder build gfal2-v2.23.0/.gitlab-ci.yml000066400000000000000000000057161465240014500154250ustar00rootroot00000000000000stages: - build - test - publish #-------------------------- # Build templates #-------------------------- .build-template: &build-template_definition stage: build script: - ci/fedora-packages.sh - ci/common-rpm-build.sh - mkdir ${CI_JOB_NAME} - cp -rv build/RPMS build/SRPMS ${CI_JOB_NAME} - tree ${CI_JOB_NAME} variables: BRANCH: ${CI_COMMIT_REF_NAME} artifacts: paths: - "$CI_JOB_NAME" #-------------------------- # Build jobs #-------------------------- alma8: image: gitlab-registry.cern.ch/linuxsupport/alma8-base <<: *build-template_definition alma9: image: gitlab-registry.cern.ch/linuxsupport/alma9-base <<: *build-template_definition fedora-39: image: fedora:39 <<: *build-template_definition fedora-40: image: fedora:40 <<: *build-template_definition fedora-rawhide: image: fedora:rawhide <<: *build-template_definition allow_failure: true #-------------------------- # Test templates #-------------------------- .test-template: &test-template_definition stage: test script: - dnf install -y git - PLATFORM=${CI_JOB_NAME%-*} - ci/write-repo-file.sh - dnf install -y ${PLATFORM}/RPMS/*/*.rpm - gfal2-unit-tests variables: BRANCH: ${CI_COMMIT_REF_NAME} #-------------------------- # Test jobs #-------------------------- alma8-test: image: gitlab-registry.cern.ch/linuxsupport/alma8-base needs: - job: alma8 before_script: - dnf install -y epel-release <<: *test-template_definition alma9-test: image: gitlab-registry.cern.ch/linuxsupport/alma9-base needs: - job: alma9 before_script: - dnf install -y epel-release <<: *test-template_definition fedora-39-test: image: fedora:39 needs: - job: fedora-39 <<: *test-template_definition fedora-40-test: image: fedora:40 needs: - job: fedora-40 <<: *test-template_definition #-------------------------- # Publish jobs #-------------------------- rpms: stage: publish image: gitlab-registry.cern.ch/eos/gitlab-eos/alma9:latest dependencies: - alma8 - alma9 - fedora-39 - fedora-40 - fedora-rawhide script: - automount - cat "${repo_passwd}" | kinit "${repo_user}" - | for platform in alma8 alma9 fedora-39 fedora-40 fedora-rawhide; do if [[ "${platform}" == "fedora-rawhide" ]] && [[ ! -d ${platform} ]] ; then echo "Skipping ${platform} packages..." ; continue ; fi packaging/gfal2-repo-manager.py --action add --base /eos/project-d/dmc/www/repos/ --ref ${CI_COMMIT_REF_NAME} --packages ${platform}/RPMS/*/*.rpm ${platform}/SRPMS/* done - sleep 30 rules: - if: '$CI_PROJECT_NAMESPACE != "dmc"' when: never - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' when: never - if: '$CI_COMMIT_REF_NAME == "develop" || $CI_COMMIT_TAG != null' - if: '$CI_COMMIT_REF_NAME != "develop" && $CI_COMMIT_TAG == null' when: manual tags: - docker-privileged-xl retry: 2 gfal2-v2.23.0/.gitmodules000066400000000000000000000000001465240014500151230ustar00rootroot00000000000000gfal2-v2.23.0/CMakeLists.txt000066400000000000000000000067231465240014500155300ustar00rootroot00000000000000## cmake build script for srm-ifce project (gfal2) cmake_minimum_required (VERSION 2.6) message ("cmake source dir : ${CMAKE_SOURCE_DIR}") # default set of options set (MAIN_CORE TRUE CACHE STRING "enable compilation of the main library") set (MAIN_TRANSFER TRUE CACHE STRING "enable compilation of the transfer library") set (PLUGIN_SFTP TRUE CACHE STRING "enable compilation of the sftp plugin") set (PLUGIN_SRM TRUE CACHE STRING "enable compilation of the srm plugin") set (PLUGIN_DCAP TRUE CACHE STRING "enable compilation of the DCAP plugin") set (PLUGIN_FILE TRUE CACHE STRING "enable compilation of the local File plugin") set (PLUGIN_GRIDFTP TRUE CACHE STRING "enable compilation of the GRIDFTP plugin") set (PLUGIN_HTTP TRUE CACHE STRING "enable compilation of the HTTP plugin") set (PLUGIN_XROOTD TRUE CACHE STRING "enable compilation of the XROOTD plugin") set (PLUGIN_MOCK FALSE CACHE STRING "enable compilation of the MOCK plugin") # Deprecated / unsupported protocols after CC7 set (PLUGIN_LFC FALSE CACHE STRING "enable compilation of the lfc plugin") set (PLUGIN_RFIO FALSE CACHE STRING "enable compilation of the RFIO plugin") # build type set (CMAKE_BUILD_TYPE "RelWithDebInfo" CACHE STRING "type of build") set (CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/modules/") # load modules include(DefineInstallationPaths REQUIRED) include(MacroCopyFile REQUIRED) include(ReleaseDebugAutoFlags REQUIRED) include(CMakeGeneratePkgConfig REQUIRED) include(MacroAddDoxygen REQUIRED) include(CMakeCXX11Support REQUIRED) # Enable C++11 support set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CXX11_FLAG_ENABLE}") # enable testing include (CTest) #define PROJECT vars set (PROJECT_NAME_MAIN "gfal2") set (OUTPUT_NAME_MAIN "gfal2") set (PROJECT_NAME_TRANSFER "gfal_transfer") set (OUTPUT_NAME_TRANSFER "gfal_transfer") set (VERSION_MAJOR 2) set (VERSION_MINOR 23) set (VERSION_PATCH 0) set (VERSION_STRING ${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_PATCH}) add_definitions(-DVERSION="${VERSION_STRING}") # out of project test compilation set (ONLY_TESTS FALSE CACHE STRING "build only the tests") set (SKIP_TESTS FALSE CACHE STRING "skip the tests") # libs checks find_package (GLIB2 REQUIRED) find_package (GTHREAD2 REQUIRED) find_package (JSONC REQUIRED) # include directories include_directories (${GLIB2_INCLUDE_DIRS}) include_directories (${GTHREAD2_INCLUDE_DIRS}) include_directories (${CMAKE_SOURCE_DIR}/src/core) include_directories (${CMAKE_SOURCE_DIR}/src/utils) # Core and plugins if (NOT ONLY_TESTS) # general parameters for configuration add_definitions ( -DGFAL_PLUGIN_DIR_DEFAULT="${PLUGIN_INSTALL_DIR}" ) add_definitions ( -DGFAL_CONFIG_DIR_DEFAULT="${SYSCONF_INSTALL_DIR}" ) # add file offset 64 for 32 bits add_definitions ( -D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE -D_LARGEFILE64_SOURCE -D_GNU_SOURCE) #install doc license install (FILES "LICENSE" "README.md" "RELEASE-NOTES" "DESCRIPTION" "readme.html" DESTINATION ${DOC_INSTALL_DIR}) # install global configuration files list (APPEND lst_conf_files "${CMAKE_SOURCE_DIR}/dist/etc/gfal2.d/gfal2_core.conf" "${CMAKE_SOURCE_DIR}/dist/etc/gfal2.d/bdii.conf" "${CMAKE_SOURCE_DIR}/dist/etc/gfal2.d/x509.conf") install (FILES ${lst_conf_files} DESTINATION ${SYSCONF_INSTALL_DIR}/gfal2.d/) add_subdirectory (doc) add_subdirectory (src) endif (NOT ONLY_TESTS) if (NOT SKIP_TESTS) add_subdirectory (test) endif (NOT SKIP_TESTS) gfal2-v2.23.0/DESCRIPTION000066400000000000000000000010311465240014500144610ustar00rootroot00000000000000The Grid File Access Library 2.0 that offers a POSIX interface to the distributed files system supported by WLCG. Official documentation : http://dmc.web.cern.ch/ Differents file systems are supported by a collections of GFAL 2.0's plugins, this is a non complete list : - LFC ( Logical File catalog) with lfn:// url - SRM ( Storage Resource Manager) with srm:// - RFIO ( remote file I/O) with rfio:// - DCAP ( remote ftp for Dcache ) with gsidcap:// - HTTP/Webdav with cluster, third party copy and client side credential support. gfal2-v2.23.0/HOWTO-gfalPlugin000066400000000000000000000036661465240014500157040ustar00rootroot00000000000000How to create a new method on a gfal plugin. The example used here will be with adding a QoS method on the http plugin You can checkout the commit doing so here: https://gitlab.cern.ch/dmc/gfal2/commit/87ca30684d1e85c59edafa7935e830b9006f054f 1) Choose the plugin you want to extend and create a new file that will hold the methods to be created ex: a) Create the src/plugins/http/gfal_http_qos.cpp file b) Create the gfal_http_check_classes method 2) The new method created has to be declared on the src/plugins/http/gfal_http_plugin.h file 3) Since this method will be a plugin method, it needs to be added to the interface of the plugins here: src/core/common/gfal_plugin_interface.h A new mode has to be created for it as well (check example at gfal_plugin_interface.h) In our case, GFAL_PLUGIN_QOS_CHECK_CLASSES will have to be added A new method to generically access this plugin function has to be declared on this file: gfal_plugin_qos_check_classes in our case. 4) The aforementioned method has to be implemented on src/core/common/gfal_plugin.c 5) Since the methods before implement the functionality of distinguishing between plugins, another method has to be declared and used for generic usage of what we want to do. This is done at src/core/file/gfal_file_api.h and src/core/file/gfal2_standard_file_operations.c 6) We finally need to define the actual plugin behavior when this method is requested. This is done at: src/plugins/http/gfal_http_plugin.cpp There, is also defined if this plugin supports this specific behavior (GFAL_PLUGIN_QOS_CHECK_CLASSES in our case). 7) A functional test needs to be created in order to test these: a) Create the test at: test/functional/gfal_test_qos.cpp b) Declare the test at: test/functional/CMakeLists.txt and then test/functional/functional-test-parameters.cmake Complete example at https://gitlab.cern.ch/dmc/gfal2/commit/87ca30684d1e85c59edafa7935e830b9006f054fgfal2-v2.23.0/LICENSE000066400000000000000000000051121465240014500137640ustar00rootroot00000000000000Copyright (c) CERN 2013-2017 Copyright (c) Members of the EMI Collaboration. 2010-2013 Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. Members of the EMI Collaboration -------------------------------- European Organization for Nuclear Research (CERN) Switzerland Centro de Supercomputación de Galicia (CESGA) Spain CESNET, zajmove sdruzeni pravnickych osob Czech Republic CINECA - Consorzio Interuniversitario Italy Agencia Estatal Consejo Superior de Investigaciones Cientificas (CSIC) Spain Deutsches Elektronen-Synchrotron (DESY) Germany Fundamenteel Onderzoek der Materie (FOM) The Netherlands Forschungszentrum Jülich GmbH (FZJ) Germany Greek Research and Technology Network SA (GRNET) Greece Istituto Nazionale di Fisica Nucleare (INFN) Italy Lunds Universitet (LU) Sweden National Information Infrastructure Development Institute (NIIF) Hungary Science and Technology Facilities Council (STFC) United Kingdom Teleinformatikdienste fuer Lehre und Forschung (SWITCH) Switzerland Trinity College Dublin (TCD) Ireland Technische Universität Dresden (TUD) Germany University of Copenhagen (UCPH) Denmark University of Helsinki – Helsinki institute of Physics (UH.HIP) Finland Universitetet i Oslo (UiO) Norway Univerzita Pavla Jozefa Šafarika V Kosiciach (UPJS) Slovak Republic Uppsala Universitet (UU) Sweden Uniwersytet Warszawski (UWAR) Poland Korea Institute of Science and Technology Information (KISTI) South Korea Academia Sinica Grid Computing (ASGC) Taiwan gfal2-v2.23.0/README.md000066400000000000000000000025711465240014500142440ustar00rootroot00000000000000What is GFAL2 ? =============== GFAL2 offers an a single and simple toolkit for the file operations in grids and cloud environments. The set of supported protocols depends on the installed plugins. Supported protocols : * Local File * SRM * GSIFTP * HTTP(S) , WebDav(s) * XROOTD * DCAP/GSIDCAP/KDCAP * RFIO * LFC License ======= GFAL2 is under the Apache License 2.0 Documentation ============= See https://dmc-docs.web.cern.ch/dmc-docs/gfal2/gfal2.html for more details Build ===== ## Install dependencies On a clean SLCX/fedora/EL system the following 'extra' packages are needed to be installed: ```bash yum install cmake doxygen glib2-devel libattr-devel openldap-devel zlib-devel lfc-devel dpm-devel srm-ifce-devel dcap-devel globus-gass-copy-devel davix-devel xrootd-client-devel libssh2-devel gtest-devel ``` Additionally, `e2fsprogs-devel` on SLC5, or `libuuid-devel` on higher versions. ## Compile ```bash git clone https://gitlab.cern.ch/dmc/gfal2.git cd gfal2 mkdir build cd build cmake .. ccmake .. # configure the plugins that you need and enable the test if wished make ``` ## Installation ```bash make install ``` ## Tests ```bash make test ``` ## Contributions Any contributions, patch or external plugins is welcome. # Contact dmc-devel@cern.ch or dmc-support@cern.ch gfal2-v2.23.0/RELEASE-NOTES000066400000000000000000001310121465240014500146470ustar00rootroot00000000000000GFAL2 RELEASE-NOTES =================== ## Wed Jul 31 2024 Mihai Patrascoiu - 2.23.0 ###Task - [DMC-1412] - Gfal2 support for archive metadata - [DMC-1413] - Data Management Clients for Fedora 39 and Fedora 40 - [DMC-1414] - Stop building Data Management Clients for CC7 ###Improvement - [DMC-1416] - Disable the automatic clean-up when a copy fails - [gfal2/pull/61] - Fixes for 64 bit time_t on 32 bit systems ###Change - [DMC-1409] - Rename FTS archive-metadata from "TransferMetadata" to "ArchiveMetadata" header in Gfal2 - [DMC-1415] - Modify HTTP "user.status" extended attribute to return file locality even in the presence of the "/archiveinfo" error field ## Tue Mar 19 2024 Mihai Patrascoiu - 2.22.2 ###Bug - [DMC-1398] - Gfal2 does not handle endpoints without reverse DNS entry - [DMC-1399] - DNS resolution breaks config and credential look-up - [DMC-1402] - Gfal2 XRootd bringonline does not pass query arguments - [DMC-1407] - Incorrect IPv4 reported for HTTP-TPC transfers that don't send "RemoteConnections" marker ###New Feature - [DMC-1400] - Make Gfal2 HTTP "RETRIEVE_BEARER_TOKEN" option configurable per Storage Endpoint ###Improvement - [DMC-1377] - Configure "RETRIEVE_BEARER_TOKEN" per Storage Endpoint - [DMC-1393] - Preserve SE-specific settings when DNS host resolution is applied - [DMC-1408] - Gfal2 should allow logging the HTTP body of requests/responses ###Question - [DMC-1397] - Tape REST API endpoints and trailing slash ## Mon Dec 11 2023 Joao Lopes - 2.22.1 ###Improvement - [DMC-1396] - Get storage-issued tokens with OAuth2 flow ## Wed Oct 18 2023 Joao Lopes - 2.22.0 ###New Feature - [DMC-1344] - Propagate Scitags during TPC transfers ###Task - [DMC-1379] - Move DMC projects to Alma8 and Alma9 - [DMC-1387] - Data Management Clients for Fedora 38 ## Tue Jul 25 2023 Mihai Patrascoiu - 2.21.5 ###Bug - [DMC-1380] - Return an error code when checksum request fails over HTTP ###New Feature - [DMC-1382] - Gfal2 should propagate the CLEANUP event ###Improvement - [DMC-1378] - Archive and stage poll should show response from XrootD server when response is an invalid JSON - [DMC-1381] - Configure DEFAULT_COPY_MODE per storage endpoint ## Mon Apr 03 2023 Joao Lopes - 2.21.4 ###New Feature - [DMC-1367] - Gfal2 support for transfer metadata ###Improvement - [DMC-1374] - Propagate all error messages reported by the HTTP Tape REST API ###Sub-task - [DMC-1348] - Extend DNS host resolution for HTTP protocol - [DMC-1368] - Pass transfer metadata in the HTTP TPC command ## Tue Feb 07 2023 Joao Lopes - 2.21.3 ###New Feature - [DMC-1290] - Gfal2 Mock Plugin support for Archive operation - [DMC-1357] - Report source and destination hosts involved in HTTP Third Party Copy - [DMC-1369] - Make Gfal2 HTTP "HEADERS" option configurable per Storage Endpoint ###Task - [DMC-1359] - Data Management Clients for Fedora 37 ###Improvement - [DMC-1362] - Gfal2 HTTP logging should respect the "LOG_LEVEL" configuration - [DMC-1365] - Protect gfal_xrootd_bring_online_list() from reading un-allocated memory - [DMC-1366] - Two GFAL2 XRootD bring on-line error messages have the same message ###Bug - [cern-fts/gfal2/pull/14] - Don't downgrade the C++ version (bugzilla#2163832) ## Thu Dec 01 2022 Mihai Patrascoiu - 2.21.2 ###New Feature - [DMC-1353] - Always return Adler32 checksum as 8-byte string - [DMC-1354] - Make Gfal2 HTTP "ENABLE_REMOTE_COPY" option configurable per Storage Endpoint - [gfal2/pull/53] - Allow the use of roots:// and xroots:// schema for XRootd + TLS ###Task - [DMC-1351] - Pass bringonline token to evict call inside HTTP copy ###Improvement - [DMC-1350] - Reduce metalink requests in Gfal2 HTTP plugin - [DMC-1355] - Disable metalink support by default for Gfal2 transfers ## Thu Sep 29 2022 Mihai Patrascoiu - 2.21.1 ###Bug - [DMC-1334] - Gfal2 HTTP bring online polling missing files ###New Feature - [DMC-1347] - Allow GFAL2 ENABLE_FALLBACK_TPC_COPY to be overridden per SE - [DMC-1349] - Report IPv4 transfer event ###Task - [DMC-1327] - DMC packages for Centos Stream 9 - [DMC-1329] - Create "taperestapi.uri" extended attribute in Gfal2 HTTP plugin - [DMC-1331] - Create "taperestapi.sitename" extended attribute in Gfal2 HTTP plugin - [DMC-1336] - Data Management Clients for Fedora 36 ###Improvement - [DMC-1339] - HTTP streamed copy should apply the transfer timeout to both source and destination operations - [DMC-1340] - Better reporting of errors during HTTP streamed transfers ## Fri Jul 08 2022 Joao Lopes - 2.21.0 ###Epic - [DMC-1301] - Gfal2 Support for HTTP Tape Operations ###New Feature - [DMC-1302] - Implement HTTP bringonline operation - [DMC-1303] - Implement HTTP bringonline polling operation - [DMC-1304] - Implement HTTP archive polling operation - [DMC-1305] - Implement HTTP release-file operation - [DMC-1306] - Implement HTTP eviction on successful copy operation - [DMC-1307] - Implement HTTP Tape Endpoint discovery - [DMC-1311] - Report IP stack used during HTTP-TPC in Gfal2 - [DMC-1314] - Implement HTTP bringonline cancel operation - [DMC-1316] - Allow configuration of HTTP "ENABLE_STREAM_COPY" parameter by Storage Endpoint - [DMC-1321] - HTTP tape polling operations must not assume order is preserved in server response - [DMC-1323] - Gfal2 should propagate the EVICT event - [DMC-1324] - Gfal2 support for extended attributes via HTTP ###Improvement - [DMC-1260] - Expose Gfal2 XRootD eviction via the file API - [DMC-1317] - Prevent HTTP Copy fallback for non-retryable errors - [DMC-1318] - Refactor the HTTP Copy error reporting ## Fri Mar 04 2022 Mihai Patrascoiu - 2.20.5 ###Bug - [gfal2/pull/42] - Check _DARWIN_FEATURE_ONLY_64_BIT_INODE to avoid errors from stat64 being undefined - [gfal2/pull/43] - Fix building against OpenLDAP 2.5+ ###Improvement - [DMC-1308] - Move Data Management Clients to Fedora 34 & 35 - [cern-fts/gfal2/pull/9] - Remove trailing white spaces ## Tue Feb 22 2022 Mihai Patrascoiu - 2.20.4 ###Bug - [DMC-1297] - HTTP MOVE operation + SETokens needs to request a token for common base directory - [DMC-1298] - GridFTP unsupported extended attribute error message should be clearer - [DMC-1300] - Memory leak in Macaroon retrieval - [gfal2/pull/38] - Do not hardcode error numbers - [gfal2/pull/40] - Define ENODATA if not defined - [cern-fts/gfal2/pull/10] - [CMake] Explicitly state the dependency relation of the XRootD plugin and the uuid ###Request - [DMC-1296] - Improve GridFTP DNS gateway resolution mechanism ## Wed Jan 19 2022 Joao Lopes - 2.20.3 ###Improvement - [gfal2/pull/41] - Simplified tokens management for CS3 storages ## Mon Dec 06 2021 Mihai Patrascoiu - 2.20.2 ###Bug - [DMC-1149] - Misleading error message from mkdir -p over GridFTP when x509 credential can not be found - [DMC-1282] - Gfal2 MKCOL with SE-tokens incompatible with dCache storage - [DMC-1283] - HTTP Rmdir + SETokens fails on DPM when path missing trailing '/' - [DMC-1284] - Do not use Authorization Header when requesting SEToken ## Mon Oct 25 2021 Mihai Patrascoiu - 2.20.1 ###Bug - [DMC-1272] - Segfault when macaroon request exceeds maximum response size - [DMC-1273] - Gfal2 HTTP plugin does not clear SE-issued tokens from the Gfal2 core credential map - [DMC-1275] - Gfal2 credential map should return partial prefix matches only in the case of directories - [DMC-1276] - Gfal2 HTTP SE-issued write token should also include all read permissions - [gfal2/pull/37] - Define ECOMM and ECHRNG if required (e.g. macOS) ###Improvement - [DMC-1274] - Provide credential delete method in the Gfal2 credential API ## Mon Sep 20 2021 Mihai Patrascoiu - 2.20.0 ###Epic - [DMC-1249] - Migrate CI infrastructure to gitlab-CI ###Bug - [DMC-1262] - Gfal2 list polling functions return error array with incomplete first element when plugin not found - [DMC-1266] - Missing null byte in readlink Gfal2 file plugin ###New Feature - [DMC-1228] - Implement SE token retrieval in Gfal2 - [DMC-1247] - Add SWIFT support in Gfal2 - [DMC-1269] - Add CS3API support in Gfal2 ###Task - [DMC-1250] - Gfal2 packages for Centos8 - [DMC-1257] - Treat "davs" and "https" protocols as the same in SRM TURL selection - [DMC-1264] - Gfal2 packages for Fedora >= 33 ###Improvement - [DMC-1246] - Avoid exposing sensitive information in Gfal2 HTTP debug output - [DMC-1261] - Improve XRootd Status code to errno mapping - [DMC-1270] - Support for checksum and getxattr calls in in Gfal2 mock plugin ## Wed May 26 2021 Mihai Patrascoiu - 2.19.2 ###Bug - [DMC-1248] - Misleading error message when XRootD bringonline bulk request fails - [DMC-1252] - Memory leak in XRootd bringonline/archive polling ## Thu Mar 25 2021 Mihai Patrascoiu - 2.19.1 ###Bug - [DMC-1237] - gfal-rm -r fails on HTTP endpoints - [DMC-1244] - XRootd copy failing when checksum value has uppercase characters ###Improvement - [DMC-1194] - gfal-xattr should avoid triggering an srmPrepareToGet in case the file is not ONLINE - [DMC-1233] - UTF8 escape XRootd error messages ## Fri Dec 04 2020 Mihai Patrascoiu - 2.19.0 ###Bug - [DMC-1227] - gfal-ls with https/davs - [DMC-1231] - Do not use BEARER tokens when dealing with S3/GCloud endpoints ###Task - [DMC-1217] - Implement gfal2 archived poll API using Xrdfs query prepare ###Improvement - [DMC-1224] - Bringonline polling should always propagate the error_text field, if it exists - [DMC-1229] - Introduce Gfal2 transfer parameter to enable/disable proxy delegation - Refactor the way S3 request parameters are obtained from the loaded configuration - Non-UTF8 characters in Davix error and log messages are correctly handled in the HTTP plugin (attributed to Petr Vokac) ## Mon Jul 27 2020 Mihai Patrascoiu - 2.18.2 ###Bug - [DMC-1220] - Filesize of GridFTP incomplete files reported as -1 ###Task - [DMC-1216] - Modify gfal2 Xrdfs query prepare to recognize both path_exists and exists fields ## Wed Jun 17 2020 Mihai Patrascoiu - 2.18.1 ###Bug - (DMC-1110) Set RequireChecksumVerification header to false also for checksum mode "both" - Fix JSON headers for HTTP plugin ## Fri Jun 05 2020 Mihai Patrascoiu - 2.18.0 ###Bug - [DMC-1201] - gfal2-plugin-xrootd incompatible with the xrootd master (v5) ###Task - [DMC-1059] - Extend the gfal2 HTTP plugin to implement CDMI QoS operations ## Thu May 07 2020 Mihai Patrascoiu - 2.17.3 ###Bug - [DMC-1203] - gfal attempts delegation with S3 / gcloud during third party copy - [DMC-1210] - gfal2 recursive mkdir fails with file exist - Fix incorrect parenthesis in GridFTP checksum validation - Token credentials support for same-host HTTP TPC transfers * Thank you to Petr Vokac for same-host HTTP TPC support using token credentials ## Tue Jan 28 2020 Michal Simon - 2.17.2 ###Bug - [DMC-1198] - bring_online_poll_list: collapse redundant slashes in the path. ## Thu Nov 28 2019 Andrea Manzi - 2.17.1 ###Task - [DMC-1192] - Rewrite gfal_xrootd_bring_online_poll_list to use the Xrdfs Query Prepare implementation - [DMC-1196] - Set IPV6=false by default for gridftp plugin ###Improvement - [DMC-1187] - Add COPY_FAIL_NEARLINE option for SRM copy_file ## Tue Oct 22 2019 Andrea Manzi - 2.17.0 ###Sub-task - [DMC-1168] - add root and https to TURL_3RD_PARTY_PROTOCOLS ###Bug - [DMC-1185] - When an Xrootd staging request part of a bulk fails, all staging requests are also set as failed ###Task - [DMC-1151] - Set IPV6=true by default for gridftp plugin - [DMC-1167] - Make sure SRM<->http and SRM<->root TPC is supported - [DMC-1176] - Stop building lfc and rfio plugins on Rawhide/F31/EPEL8 ###Improvement - [DMC-1061] - xrootd plugin: checksum logic broken - [DMC-1182] - Add multistream support in Xrootd plugin ## Wed Sep 18 2019 Andrea Manzi - 2.16.4 ###Bug - [DMC-1173] - The xrootd plugin ignores the make parent option ###Task - [DMC-1159] - Implement Xrootd prepare evict on sources when Xrootd TPC transfers succeed - [DMC-1160] - Remove conditional compilation code to still support xrootd v3 on xrootd plugin - [DMC-1166] - Add support for canceling HTTP TPC transfers ## Fri May 17 2019 Andrea Manzi - 2.16.3 ###Bug - [DMC-1026] - gfal-copy segmentation fault on MacOSx - [DMC-1137] - gfal2.Gfal2Context.cancel issue for Xrootd transfers - [DMC-1147] - The autoCleanFileCopy function in gridftp plugin never executes the file unlink - [DMC-1155] - gfal-rm with signed https URL gives 401 error ###Task - [DMC-1152] - Map stat file flags for Xrootd to SRM user.status xattr - [DMC-1156] - Sanitize gridftp checksum returned by globus - [DMC-1157] - Integrate new API for staging canceling in Xrootd ## Thu Feb 21 2019 Andrea Manzi - 2.16.2 ###Bug - [DMC-1131] - Transfers gridftp->ftp fail as GLOBUS_FTP_CONTROL_MODE_EXTENDED_BLOCK is not supported on FTP ###Task - [DMC-1123] - Add ENABLE_FALLBACK_COPY variable to http plugin - [DMC-1125] - Add X509 delegation to Xrootd TPC job properties - [DMC-1129] - Add a query parameter to HTTP urls for the configuration of the copy mode ## Wed Oct 31 2018 Andrea Manzi - 2.16.1 ###Bug - [DMC-1099] - Checksum comparison in Xrootd is wrongly enabled for FTS transfers - [DMC-1106] - HTTP to HTTP streamed transfers timeout after 300 secs - [DMC-1107] - The instant throughut for HTTP Streamed transfers is wrongly calculated - [DMC-1109] - HTTP TPC failures are not logged at INFO level - [DMC-1110] - RequireChecksumVerification set to false only for HTTP TPC PULL - [DMC-1112] - Setting BEARER_TOKEN breaks pre-signed URLa ###Task - [DMC-1095] - Refactor http streamed copy to support s3 multipart upload via Davix ## Mon Seo 28 2018 Andrea Manzi - 2.16.0 ### Bug - [DMC-1048] - Xrootd plugin: the query for the EOS-CTA extended attribute "sys.retrieve.error" should be asynchronous - [DMC-1055] - The Xrootd checksum preset should not contain leading 0s - [DMC-1057] - xattr.h is in glibc-headers >= 2.27 - [DMC-1066] - gfal-sum does not honour HTTP 202 responses - [DMC-1067] - Xrootd plugin does not pass the checksum type - [DMC-1068] - Remove file after error in HTTP COPY - [DMC-1069] - Work with endpoints that support HTTP macaroons but not for COPY - [DMC-1084] - TPC with destination/source http are not working with dCache ### Task - [DMC-1046] - add support for gCloud in gfal2 http plugin - [DMC-1060] - xrootd plugin should clean destination on failure - [DMC-1064] - Add possibility to pass a BEARER token via ENV var ### Improvement - [DMC-1028] - RD_NB_STREAM in /etc/gfal2.d/gsiftp_plugin.conf has no effect - [DMC-1074] - Add DEFAULT_COPY_MODE to gfal2 http conf ## Mon May 28 2018 Andrea Manzi - 2.15.5 ### Bug - [DMC-1023] - Xrootd plugin:The checksum string provided by the user is not correctly parsed - [DMC-1041] - Incorrect GridFTP listing output ### Improvement - [DMC-1036] - gfal_http_get_token should read the credentials associated to the host ### New Feature - [DMC-1038] - Implement abort files for Xrootd bringonline - [DMC-1040] - Xrootd plugin: implement the query for the EOS-CTA extended attribute 'sys.retrieve.error' ## Thu Apr 05 2018 Andrea Manzi - 2.15.4 ### Bug - [DMC-1027] - DAVIX operation timeout incorrect for HTTP-based transfers - [DMC-1031] - gfal_http_get_token does not free the gchar* token - [DMC-1034] - When ENABLE_STREAM_COPY is disabled the http copy process crashes ## Mon March 12 2018 Andrea Manzi - 2.15.3 ### Bug - [DMC-1017] - gfal2 examples don't compile ### Improvement - [DMS-1014] - change the default copy method to PULL for HTTP - [DMC-1015] - gfal_http_copy should try all available COPY modes regardless of the error ## Mon Feb 12 2018 Alejandro Alvarez - 2.15.2 ### Bug - [DMC-1009] - Segfault on gfal_load_configuration_to_conf_manager when there is an error loading the configuration file - [DMC-1010] - HTTP Plugin: Normalisation of destination url breaks S3 support for push third party copy - [DMC-1011] - HTTP transfers wrongly reported with success ## Thu Feb 01 2018 Alejandro Alvarez - 2.15.1 ### Bug - [DMC-998] - XRootD plugin: xrootd is not recognised as a valid prefix for copies - [DMC-999] - GridFTP Plugin: Third party copy does not work for ftp => gridftp - [DMC-1000] - GridFTP Plugin: Disable multistreams for plain FTP - [DMC-1003] - Fix typo UNKNOW => UNKNOWN for replica locality - [DMC-1004] - SRM Plugin: Map "None" locality - [DMC-1006] - if GFAL_CRED_X509_CERT is not defined, the XROOTD PLUGIN options are skipped - [DMC-1007] - Wrong default values in configuration files (TRUE instead of true) ## Thu Nov 09 2017 Alejandro Alvarez - 2.15.0 ### Bug - [DMC-977] - Missing closing bracket on the space reporting output - [DMC-979] - Total size reported incorrectly to be -1 - [DMC-991] - GridFTP Plugin: Reset credentials when reusing handle with a different endpoint ### New Feature - [DMC-989] - HTTP Plugin: If checksum is MD5, send Content-MD5 header ### Improvement - [DMC-954] - Drop davs+3rd, use only davs - [DMC-965] - When copying into/from srm, if the other pair is *not* srm, it should appear first on the list of protocols - [DMC-970] - Add Castor to the test endpoints - [DMC-975] - XRootD Plugin: Make normalization configurable (disable by default?) - [DMC-986] - Performance markers timeout should return ETIMEDOUT instead of ECANCEL ## Wed Jul 26 2017 Alejandro Alvarez - 2.14.2 ### Bug - [DMC-976] - DMC-878 breaks GridFTP towards Castor ## Fri Jul 14 2017 Alejandro Alvarez - 2.14.1 ### Bug - [DMC-973] - SRM Plugin: Revert multiple slash addition ## Wed Jun 21 2017 Alejandro Alvarez - 2.14.0 ### Bug - [DMC-918] - LFC Plugin: Doesn't work with [X509] CERT and KEY configuration - [DMC-921] - GridFTP plugin: When asking for server, there is a missing error check - [DMC-923] - GridFTP Plugin: PASV plugin leaks - [DMC-949] - SRM Plugin: Use double slash for absolute paths - [DMC-955] - HTTP Plugin: For third party copies, fallback should be done for EPERM ### New Feature - [DMC-878] - Allow to use different credentials for different storages/paths - [DMC-931] - GridFTP Plugin: Striped Passive when IPv6 is used - [DMC-932] - XRootD Plugin: Prototype staging implementation using XRootD - [DMC-948] - LFC Plugin: Add checksum support - [DMC-953] - Copy event should specify the copy mode - [DMC-959] - Mock Plugin: Add signals on creation time, allow to disable by configuration ### Task - [DMC-929] - XRootD Plugin: Allow to specify custom xrd.* flags ### Improvement - [DMC-915] - Reorganize core - [DMC-928] - Clean up gfal2-util timeout info - [DMC-962] - Add a method to remove a configuration key ### Question - [DMC-936] - Copyright clarification ## Mon May 29 2017 Alejandro Alvarez - 2.13.4 ### Bug - [DMC-951] - Default checksum for local copy was removed ## Fri Mar 17 2017 Alejandro Alvarez - 2.13.3 ### Bug - [DMC-924] - gfal2 breaks backwards compatibility for checksums ## Mon Feb 20 2017 Alejandro Alvarez - 2.13.1 ### Bug - [DMC-919] - Remove calls to globus_module_deactivate ## Mon Feb 06 2017 Alejandro Alvarez - 2.13.0 ### Bug - [DMC-895] - Recursive directory creation: EEXIST shouldn't cause a failure - [DMC-901] - Xrootd succeeds to copy even if one of the storages does not support checksums - [DMC-909] - SRM Plugin: When BDII is disabled, SRM endpoint is built incorrectly - [DMC-910] - Leaks in BDII bindings and GridFTP plugin ### New Feature - [DMC-885] - XrootD Plugin: Add space reporting to the API - [DMC-886] - Consolidate space reporting between plugins - [DMC-892] - Generalize checksum methods - [DMC-896] - SFTP plugin ### Task - [DMC-902] - Delete old deprecated methods - [DMC-905] - Expose Low level parameters for local -> remote copies ### Improvement - [DMC-880] - If the option -t is passed, make sure that Davix knows it - [DMC-884] - XRootD Plugin: Pass desired checksum type to the storage - [DMC-900] - Apply QA tools feedback - [DMC-913] - Log a warning and keep going if loading a plugin fails ## Thu Oct 27 2016 Alejandro Alvarez - 2.12.3 ### Bug - [DMC-891] - GridFTP Plugin: IPv6 detection misfire in some cases ## Tue Sep 06 2016 Alejandro Alvarez - 2.12.2 ### Bug - [DMC-879] - XRootD Plugin: Segfault on 3rd party copy ## Tue Aug 16 2016 Alejandro Alvarez - 2.12.1 ### Bug - [DMC-872] - SRM Plugin: DMC-860 broke directory listing when endpoint can not be resolved from BDII ### New Feature - [DMC-870] - Add support for s3 alternate syntax URLs to gfal ## Mon Aug 01 2016 Alejandro Alvarez - 2.12.0 ### Bug - [DMC-833] - gfal2_add_client_info should overwrite existing key - [DMC-841] - Make scheme handling consistent across gfal2 - [DMC-846] - File Plugin: Only strict URIs must be accepted - [DMC-848] - GridFTP Plugin: PASV plugin parses incorrectly some IPv6 - [DMC-853] - [Known Issue] Connection issues when a PUT and a GET are done through the same GridFTP connection - [DMC-856] - Set user and password when accessing plain FTP - [DMC-858] - XRootD: Urldecode path before passing to xrootd libraries - [DMC-860] - SRM Plugin: Urldecode path before passing to srm-ifce ### New Feature - [DMC-827] - GridFTP Plugin: Allow to configure blocksize - [DMC-849] - GridFTP Plugin: Fire an event if IPv6 is used - [DMC-866] - Expand '-' to stdout when used as destination for gfal-copy - [DMC-868] - GridFTP Plugin: Add spacetoken xattr ### Task - [DMC-852] - GridFTP: Workaround for Globus' higher priority to /etc/grid-security/hostcert.pem ### Improvement - [DMC-837] - When an operation is not supported, include the url on the error - [DMC-851] - Improve URI parser (breaks internal API) - [DMC-855] - Accept xroot:// as well as root:// - [DMC-857] - GridFTP Plugin: Set ALLO to remote storages - [DMC-864] - Can gfal2 indicate the origin of errors (source/destination) when using GridFTP? ## Thu Apr 14 2016 Alejandro Alvarez - 2.11.1 ### Bug - [DMC-813] - XRootD plugin: gfal2_set_error called without format string - [DMC-818] - SRM plugin not invoked when using a relative path to the source file ## Mon Mar 07 2016 Alejandro Alvarez - 2.11.0 ### Bug - [DMC-758] - Add a Requires on glib2 >= 2.28 - [DMC-769] - GridFTP Plugin: Missing throw inside copy - [DMC-783] - SRM endpoint resolution fails due to an off-by-one introduced when switched to g_strlcpy - [DMC-798] - FIle Plugin: Readlink implemented but not exposed - [DMC-803] - GridFTP Plugin: PASV plugin fails to extract IP - [DMC-808] - gfal2_parse_uri does not support IPv6 URLS - [DMC-811] - gfal2 error GLib-WARNING **: GError set over the top of a previous GError or uninitialized memory ### Improvement - [DMC-793] - SRM Plugin: Set "GRIDFTP PLUGIN:STAT_ON_OPEN" to false when the endpoint is Castor - [DMC-795] - XRootD Plugin: Generate better error messages - [DMC-809] - XRootD Plugin: Copy from/to XRootD using the Client (as with 3rd party copy) - [DMC-812] - Add open/read/write/close to the mock plugin ### New Feature - [DMC-756] - Mock plugin: Add directory listing and signal raising - [DMC-763] - Add plain FTP support (anonymous only) - [DMC-781] - Add support for short-term S3 credentials - [DMC-797] - HTTP Plugin: Add rename support - [DMC-801] - dCache davs pull copy use checksum verification by default, should be disabled if checksums are not asked for - [DMC-804] - Allow to pass arbitrary headers to davix ## Wed Dec 09 2015 Alejandro Alvarez - 2.10.3 ### Bug - [DMC-783] - SRM endpoint resolution fails due to an off-by-one introduced when switched to g_strlcpy ## Fri Nov 06 2015 Alejandro Alvarez - 2.10.2 ### Bug - [DMC-769] - GridFTP Plugin: Missing throw inside copy ## Tue Oct 20 2015 Alejandro Alvarez - 2.10.1 ### Bug - [DMC-750] - Respect GLOBUS_THREAD_MODEL environment ### Improvement - [DMC-738] - HTTP Plugin: Skip source validation checksum if the algorithm is not supported ## Mon Oct 12 2015 Alejandro Alvarez - 2.10.0 ### Bug - [DMC-676] - SRM Plugin: Response for bringonline may be shorter than the request - [DMC-682] - SURLS in responses may not contain full endpoint - [DMC-692] - gfal_rw_seq_SRM_DPM hangs somewhere when the proxy is not initialized - [DMC-694] - GridFTP Plugin: Malformed IPv6 printed on the logs - [DMC-717] - Remove Boost dependency from gfal2 - [DMC-720] - SRM Plugin should handle transfers also for file:/path - [DMC-722] - LFC Plugin registers the TURL rather than the SURL into the catalog - [DMC-724] - XROOTD plugin: Case-insensitive checksum type comparison - [DMC-731] - Segfault on xrootd 3rd party copies - [DMC-741] - Replace G_LOG_LEVEL_INFO with other levels - [DMC-742] - SRM Plugin: Convert UTC to local timestamp ### Improvement - [DMC-674] - Validate version as part of the packaging - [DMC-679] - SRM Plugin: Reduce log level for the unlinking messages - [DMC-685] - All: Mark in a common way when a checksum comparison fails - [DMC-698] - Port DMC clients to MacOSX - [DMC-737] - File plugin: Avoid large allocation on the stack - [DMC-744] - GridFTP Plugin: Parse uid and gid if available ### New Feature - [DMC-669] - GridFTP Plugin: Hook a plugin to capture final endpoints on transfer - [DMC-673] - SRM Plugin: Add srm.type extended attribute - [DMC-697] - HTTP Plugin: Add support for pull copies ### Task - [DMC-680] - Integrate gfal2-plugin-xrootd into mainline gfal2 - [DMC-712] - Improve mock plugin ## Tue Jun 08 2015 Alejandro Alvarez - 2.9.3 ### Bug - [DMC-682] - SRM Plugin: SURLS in responses may not contain full endpoint - [DMC-676] - SRM Plugin: Response for poll may be shorter too ## Mon Jun 01 2015 Alejandro Alvarez - 2.9.2 ### Bug - [DMC-676] - SRM Plugin: Response for bringonline may be shorter than the request ## Fri Apr 10 2015 Alejandro Alvarez - 2.9.1 ### Bug - [DMC-620] - HTTP Plugin: Normalize S3:host to S3:HOST on all cases - [DMC-622] - Core: gfal2 crash when aborting staging operation - [DMC-626] - LFC Plugin: Off by one bug when checking for parent directory - [DMC-627] - LFC Plugin: When lfn:// is used, lfc_host was being left to NULL - [DMC-632] - GridFTP Plugin: When the timeout expires, wait for the callback after cancellation - [DMC-638] - GridFTP Plugin: Performance marker timeout causes abort - [DMC-639] - SRM Plugin: Fix build in Rawhide - [DMC-645] - GridFTP Plugin: Always register performance marker callback - [DMC-647] - GridFTP Plugin: In some machines, the threading model is event, and writing to castor do not work - [DMC-653] - SRM Plugin: Closing when reading must issue a release - [DMC-661] - GridFTP Plugin: Error messages generated during transfer seem to be truncated - [DMC-662] - GridFTP Plugin: Map correctly "Disk quota" errors ### Improvement - [DMC-613] - HTTP Plugin: Davix already stats before deletion, so remove the check - [DMC-617] - HTTP Plugin: Set Davix retrials to 0 - [DMC-619] - HTTP Plugin: Add KEEP_ALIVE option - [DMC-640] - Conditionally enable C++11 - [DMC-648] - Core: Provide gfal2_ prefixed methods for uri methods - [DMC-649] - Core: Improve logger system, deprecate old methods - [DMC-650] - SRM Plugin: Log the request tokens - [DMC-651] - Core and plugins: Allow to send custom key/value to the servers ### New Feature - [DMC-641] - Core: Allow to register multiple event listeners - [DMC-642] - Core: Give plugins a chance to register their own event listeners - [DMC-643] - Core: Add an event for copies that is triggered at the beginning - [DMC-657] - SRM Plugin: On prepare to get and put, get the desired request lifetime from the configuration file - [DMC-663] - Add functionality to allow querying the versions of the plugins loaded ## Fri Feb 27 2015 Alejandro Alvarez - 2.8.4 - [DMC-638] - GridFTP Plugin: Performance marker timeout causes abort ## Tue Feb 10 2015 Alejandro Alvarez - 2.8.3 - [DMC-632] - GridFTP Plugin: When the timeout expires, wait for the callback after cancellation ## Thu Jan 29 2014 Alejandro Alvarez - 2.8.2 - [DMC-622] - Core: gfal2 crash when aborting staging operation ## Tue Jan 06 2015 Alejandro Alvarez - 2.8.1 - [DMC-529] - [EPEL7 Only] GridFTP fails to load the configured credentials - [DMC-531] - GridFTP Plugin: Partial read and write missing EOF's - [DMC-541] - GridFTP and SRM plugins: Transferring from Castor SRM to non-SRM fail with "Connection refused" - [DMC-544] - GridFTP Plugin: Map "System error in name: Is a directory" to EISDIR - [DMC-553] - SRM Plugin: TURLS retrieved from a SRM endpoint should be of one of the requested protocols - [DMC-559] - SRM Plugin: gfal_srm_readdir_pipeline ignore errors - [DMC-563] - SRM Plugin: Typo in TURL resolution when copying - [DMC-566] - Mock plugin: Only size should be enough to specify the file size - [DMC-570] - GridFTP Plugin: Establishing an lfc connection before loading GridFTP makes the GridFTP plugin freeze in most (all?) operations - [DMC-580] - Core: Do not try to create parent on "transfer only" copies - [DMC-582] - SRM Plugin: Missing parameter in call to gfal2_set_error when the protocol is not expected - [DMC-584] - SRM Plugin: Cache does not invalidate entries when removing a directory - [DMC-587] - File: CRC32 should be represented in decimal - [DMC-596] - SRM Plugin: Getting file:// is a legitimate use case - [DMC-597] - GridFTP Plugin: Sessions are not reused - [DMC-598] - GridFTP Plugin: Transfer timeout is not taken into account - [DMC-601] - Core: Streamed copy buffer should be moved to the heap - [DMC-565] - Mock plugin: Reduce default sleep time for transferring to ~5 secs - [DMC-530] - SRM plugin doesn't seem to handle urls without path - [DMC-533] - GridFTP Plugin: Partial read and write, avoid seeking if not strictly required - [DMC-540] - HTTP Plugin: Assume DAV when http/https urls are used - [DMC-556] - SRM Plugin: On copies, an error on RELEASE should not be considered an error - [DMC-558] - Core: Log when events are triggered - [DMC-560] - Core: Notify events also in streamed copies - [DMC-568] - GridFTP Plugin: Enforce thread model to pthread - [DMC-574] - File Plugin: Package separately - [DMC-575] - SRM Plugin: Try listing in chunks when the storage replies "too many results" - [DMC-589] - HTTP Plugin: Reduce the verbosity level configured for Davix - [DMC-592] - GridFTP Plugin: In bulk copies, avoid redundant parent creation - [DMC-562] - SRM Plugin: Trigger events for TURL resolution when copying - [DMC-583] - Core: Add call to recover list of installed plugins - [DMC-487] - GridFTP Plugin: Refactoring - [DMC-536] - SRM Plugin: Protect the srm context with recursive mutexes - [DMC-550] - HTTP Plugin: Add option to configure davix log level for debugging - [DMC-576] - Merge gfal2, gfal2-core and gfal2-transfer in one single package - [DMC-603] - Core: Transfer logic from c++ to c - [DMC-604] - Remove dependency on glibmm24 * Mon Nov 17 2014 Alejandro Alvarez - 2.7.8 - [DMC-553] - TURLS retrieved from a SRM endpoint should be of one of the requested protocols ## Mon Nov 10 2014 Alejandro Alvarez - 2.7.7 - [DMC-539] - GridFTP Plugin: Transfers from gridftp endpoints hang when a read operation fails - [DMC-541] - GridFTP and SRM plugins: Transferring from Castor SRM to non-SRM fail with "Connection refused" ## Fri Nov 07 2014 Alejandro Alvarez - 2.7.6 - [DMC-469] - LFC Plugin: Calling gfal2_set_error with wrong parameters - [DMC-473] - HTTP Plugin: 3rd party copy should translate non canonical urls (davs+3rd...) to https - [DMC-485] - Core: Recursive mkdir() fail when because of a race an intermediate directory is created - [DMC-502] - GridFTP Plugin: if ucert == ukey, do not concatenate them - [DMC-504] - SRM Plugin: mkdir-ing an existing directory must nor fail - [DMC-509] - Core: Cannot use unix special files as a destination for file transfer - [DMC-510] - srm-ifce is not mapping correctly status codes (Was Revert DMC-433) - [DMC-515] - bring online timeout is not set correctly in gfal_srmv2_bring_online_internal - [DMC-516] - SRM Plugin: Bringonline internals only checks for the status of the first file - [DMC-523] - Core: Do not try to create parent directory if it exists already (local copy) - [DMC-531] - GridFTP Plugin: Partial read and write missing EOF's - [DMC-453] - Deprecate gfal_handle and gfal_context_t inside gfal2 - [DMC-463] - GridFTP Plugin: Log storage IPv6 - [DMC-465] - HTTP Plugin: Make possible dissabling certificate verification - [DMC-507] - Core: Expose URI API publicly - [DMC-508] - Deprecate direct inclusion of header files - [DMC-533] - GridFTP Plugin: Partial read and write, avoid seeking if not strictly required - [DMC-445] - HTTP Plugin: 3rd party copies explicit (davs+3rd://) - [DMC-448] - SRM Plugin: Disable GridFTP session reuse if a castor srm endpoint is involved - [DMC-452] - Core: For convenience, xattr user.checksum directly mapped to gfal2_checksum - [DMC-459] - Core: New method to load configuration from a file (gfal2_load_opts_from_file) - [DMC-466] - LFC Plugin: Handle replica deletion via extended attributes - [DMC-490] - HTTP Plugin: Add support for AWS credentials - [DMC-493] - Core: Add gfal2_get_opt_string_with_default - [DMC-460] - GridFTP Plugin: Trimming bug in gridftp_readdir_parser - [DMC-462] - GridFTP Plugin: Pipeline copy support ## Mon Jul 28 2014 Alejandro Alvarez - 2.6.8 - [LCGUTIL-169] - GFAL 2.0 : Enable tests for xrootd and http supports - [LCGUTIL-345] - GFAL 2.0: add support for dCache test bed for functional tests - [LCGUTIL-45] - GFAL 2.0 : non third party transfer should respect all transfers options - [LCGUTIL-166] - GFAL 2 : Update gfal 2.0 functional tests - [LCGUTIL-348] - gfal-copy produces misleading error - [LCGUTIL-355] - HTTP plugin does not do 3rd party copy when dav/davs is used as scheme - [LCGUTIL-359] - GFAL 2.0 : GFAL2 copy does not manage properly spacetoken in file <-> SRM transfer - [LCGUTIL-361] - Some transfers finish with GridFTP transferring 0 bytes, and keep going - [LCGUTIL-363] - Some tests do not clean after themselves - [LCGUTIL-365] - SRM Checksum fallback fails for IPv6 - [LCGUTIL-393] - Spacetoken operations do not handle null values - [LCGUTIL-420] - Unhandled exception causes abort - [LCGUTIL-422] - Memory leak in gfal_srm_cache_stat_add - [LCGUTIL-428] - Bringonline does not support destination space token - [LCGUTIL-429] - thread-safety issue with SSL - [LCGUTIL-432] - gfal_srmv2_bring_online returns > 0 if the file is already pinned - [LCGUTIL-433] - gfal2-srm: If the file is pinned (22), gfal_srmv2_bring_online_internal must return 1 too - [LCGUTIL-358] - When rolling back a SRM PUT, it may be necessary to explicitly remove the file - [LCGUTIL-362] - Clarify error messages coming from gfal2 on copies - [LCGUTIL-385] - GridFTP listing improvement (RELNOTE: session reuse enabled by default!) - [LCGUTIL-386] - Improvements to GridFTP 3rd party copy to achieve session reuse - [LCGUTIL-395] - Do not fail on replica registration if the same replica is already there - [LCGUTIL-412] - Delegate the grid authentication of GFAL 2.0 http plugin to davix - [LCGUTIL-435] - SRM: Cache file locality - [LCGUTIL-55] - GFAL 2.0 : set a clean API to set/get at runtime the different possible credential - [LCGUTIL-306] - lcg-stmd equivalent - [LCGUTIL-352] - support GridFTP UDT in gfal2 - [LCGUTIL-356] - Add an option to disable http 3rd party copies - [LCGUTIL-360] - Copy to http(s) does not work if the file is bigger than the buffer - [LCGUTIL-423] - Bulk bring online - [LCGUTIL-424] - Bulk deletions - [LCGUTIL-425] - srm-ifce KeepAlive is not honored - [LCGUTIL-439] - Missing abort function for gfal2 - [LCGUTIL-440] - stop perf markers callback when transferred_bytes == filesize - [LCGUTIL-326] - GFAL 2.0: Map Davix Checksum support to GFAL 2.0 - [LCGUTIL-347] - gfla2 lfc plugin: set serrno to 0 before readdir calls - [LCGUTIL-353] - HTTP plugin does not clean up destination on failure - [LCGUTIL-354] - GFAL2.0 : srm segfault problem related to GFAL 2.0 - [LCGUTIL-403] - gfal2 segfault when reading dir via srm - [LCGUTIL-406] - symbol lookup error: /usr/lib64/gfal2-plugins//libgfal_plugin_xrootd.so: undefined symbol: _ZN14XrdPosixXrootd8setDebugEi - [LCGUTIL-417] - gfal-copy return success when there is some error in the plugin - [LCGUTIL-460] - Issue with gfalFS and gridftp endpoints ## Wed Feb 26 2014 Adrien Devresse - 2.5.5-1 - [LCGUTIL-40] - GFAL 2.0 : implement lfc logic that allows third party copy logic with SRM - [LCGUTIL-268] - gfal2 returns no error when proxy is invalid - [LCGUTIL-290] - Problems with dates in gfalFS - [LCGUTIL-295] - GridFTP plugin returns the full path when listing an empty directory - [LCGUTIL-284] - LFC registration (as 3rd party copy) should perform some validations - [LCGUTIL-286] - GFAL2 doesn't support multiple BDII - [LCGUTIL-287] - If LFC_HOST is not specified, it should be looked in the BDII - [LCGUTIL-288] - Implement readdirpp directly in plugins that support it - [LCGUTIL-46] - GFAL 2.0 : find a workaround for non posix file information that are not mapped to xattr - [LCGUTIL-267] - gfal2 checksum doesn't work for local files - [LCGUTIL-285] - Add gsiftp meta-data cache ## Fri Dec 06 2013 Alejandro Alvarez - 2.4.8-0 - [LCGUTIL-216] - Checksum comparison must ignore heading '0' - [LCGUTIL-220] - gfal2 crashes with SIGSEGV sometimes on bringonline - [LCGUTIL-222] - Timeout doesn't seem to be honored - [LCGUTIL-236] - Segfault inside GridFTP_Request_state::~GridFTP_Request_state - [LCGUTIL-248] - GFAL 2.0 : Recent platform with recent glib2 version triggers "GLib-WARNING" in some case - [LCGUTIL-255] - unlink on a dir returns ERR 22 (Invalid arg) - [LCGUTIL-278] - seg fault - setxattr() in local files - [LCGUTIL-297] - gfal2 perf markers 0 must be treated as no markers and respect timeout - [LCGUTIL-160] - gfal2 needs to search the cache first (then BDII) if submitter not providing full srm url - [LCGUTIL-217] - SRM plugin should allow empty checksums on source - [LCGUTIL-303] - EOS GridFTP implementation does not return 'EEXIST' error properly - [LCGUTIL-226] - Partial listing of directory to avoid loading too much the server - [LCGUTIL-233] - GridFTP plugin should allow to skip the source checksum ## Tue Jul 02 2013 Adrien Devresse - 2.3.0-0 - [LCGUTIL-99] gfal2_cancel crashes gfal2 if called twice - [LCGUTIL-135] GFAL 2.0 : Migrate the http plugin support for PEM from the old manual one to the new Davix one - [LCGUTIL-143] GFAL 2.0 : Regression bug of the stat() call against the dcache instance - [LCGUTIL-145] GridFTP plugin tries to delete the destination when getting the source checksum fails (!) - [LCGUTIL-147] GFAL 2.0 : bug on the session re-use mechanism - [LCGUTIL-165] GFAL 2.0 : enable HTTP/Webdav support by default, packaged on EPEL with davix - [LCGUTIL-179] manpage-section-mismatch - [LCGUTIL-141] GFAL 2 : add a flag in the gsiftp standard option in order to allow the gsiftp redirection, this is needed for the DPM gsiftp redirection mechanism - [LCGUTIL-51] GFAL 2.0 : implement the last missing operation : srm_mv logic for file renaming - [LCGUTIL-144] GFAL 2.0 : Update all functional test for a post-EMI usage - First version fully compatible with a standard FTS 3.0 release - Minor error correction from coverity report - bug fixes related to gridFTP thread safety for meta-data operations. ## Thu Jul 02 2013 Michail Salichos - 2.2.2-11 - pass gfal2 to coverity and fix minor issues reported ## Mon Jun 10 2013 Michail Salichos - 2.2.2-10 - fixed memory leaks and variables initialization - moved event exit message outside try-catch block - fix potential crash when can't resolve IP from hostname - LCGUTIL-142: gfal2_context_t wrapped in shared_ptr to avoid early destruction ## Thu Jun 06 2013 Michail Salichos - 2.2.2-6 - log IP and port of gridftp server in IPv4/IPv6 compatible way ## Thu May 27 2013 Michail Salichos - 2.2.2-5 - log IP and port of gridftp server ## Thu May 16 2013 Michail Salichos - 2.2.2-1 - replace gridftp exists with mlst ## Wed Mar 20 2013 Adrien Devresse - 2.2.0-0 - fix thread safety issue with gsiftp plugin - add the bring online API - support for the http plugin by default - remove executable stack need - remove openMP dependency - add synchronous cancellation API - add gsiftp performance marker timeout - support for srm session reuse - reduce memory footprint ## Thu Jan 10 2013 Adrien Devresse - 2.1.1-0 - fix a minor memory issue with the gfal_transfer stack - fix a wrong error report problem with srm third party copy ## Wed Dec 05 2012 Adrien Devresse - 2.1.0-2 - fix an issue this surl to turl resolution for SRM third party copy ## Fri Nov 30 2012 Adrien Devresse - 2.1.0-0 - One-globus session system for gsiftp plugin ( FTS 3.0 need ) - correct a major issue with the gass attribute system in gsiftp plugin - change the lfc set/get env var for a one compatible with set/get opt - add set/nb streams option for gsiftp - add the mkdir rec function for SRM transfer - correct an issue with opendir and srm_ls ( ENOTDIR error silent ) - correct a memory leak in the cache system - correct timeout support for gsiftp transfer - implement tcp buffer size support for gsiftp layer - apply a correction on the SRM over-write logic, linked to a BeStMan errcode problem on File Not Found with srmRm ( EOS ) - apply a fix on the transfer gsiftp timeout ( protection against multiple cancel ) - fix for SRM filesize problem ( defined to 0, workaround ) related to globus 426 error bad filesize - secure the callback system for globus gass timeout - base implementation of the http plugin - improve reliability of the bdii resolution - add a fallback mechanism in case of bdii bad resolution - correct several race conditions in the bdii layer - add thread safe support for set/get variables in liblfc - correct a deadlock problem with globus and gisftp plugin - implement the mkdir_rec logic for general purpose - implement the parent folder creation logic with gridftp - add support for lfc://host/path URL style for the lfc plugin - switch off_t to 64bits size by default ( _FILE_OFFSET_BITS=64) - provide a "nobdii" like option - provide the choice of turl protocol resolution for srm plugin ## Fri Jul 20 2012 Adrien Devresse - 2.0.0-1 - Official initial release candidate of gfal 2.0 - Transfer API is official - gridftp support for performance marker, checksum - gridftp support for gridftpv2, dcau param - SRM support for spacetoken in transfer - SRM abort auto-management - parallel operations in transfers - file protocol dedicated in a plugin - configuration file support - srm timeout support - general purpose checksum operation support - POSIX operation support for gridftp - cleaner plugin API - new documentation - I hope that you will enjoy gfal 2.0 :) ## 2.0.0-0.10 - effective set/get opt support for lfc/srm/gsiftp - checksum calculation management - auto-check functionality on transfer - support for user-defined checksum - performance marker implementation for SRM/GSIFTP ## 2.0.0-0.9 - Without environment variable design implemented - add set/get option calls - add SRM src/dst spacetoken management for gfal transfer - re-factor properly gfal-transfer and gfal2_context_t - generate a new clean documentation with log/transfer/context support - allow direct modification of the internal posix context - fix memory leak with gsiftp error report ## 2.0.0-0.8 - big improvement on the gridftp support - split the headers files properly - remove the carriage return from the gridftp error messages - add plugin dedicaded API. - refactor the parameter management for the transfer library - add the "transfer replace" logic for the transfer library ( FTS 3.0 requirement ) - add logger system based on the glib log system ( ARC requirement ) - correct memory leaks and a session-reuse memory corruption for gridftp - improve gridFTP global speed with request optimisations. - remove the old config/parameters system for the posix library. - add several functional tests for the new features. - add SRM to X and X to SRM modes to the SRM plugin ( FTS 3.0 requirement ) ## 2.0.0-0.7 - major improvement for third party transfer - initial gridftp support - begin to switch to configuration without env var ## 2.0.0-0.6 - initial tagged release gfal2-v2.23.0/ci/000077500000000000000000000000001465240014500133535ustar00rootroot00000000000000gfal2-v2.23.0/ci/common-rpm-build.sh000077500000000000000000000024651465240014500171020ustar00rootroot00000000000000#!/usr/bin/env bash set -e function print_info { printf "======================\n" printf "%-16s%s\n" "Distribution:" "${DIST}" printf "%-16s%s\n" "Dist name:" "${DISTNAME}" printf "%-16s%s\n" "Build type:" "${BUILD}" printf "%-16s%s\n" "Branch:" "${BRANCH}" printf "%-16s%s\n" "Release:" "${RELEASE}" printf "%-16s%s\n" "DMC Repository:" "${REPO_FILE}" printf "======================\n" } TIMESTAMP=$(git log -1 --format="%at" | xargs -I{} date -d @{} +%y%m%d%H%M) GITREF=`git rev-parse --short=7 HEAD` RELEASE=r${TIMESTAMP}git${GITREF} BUILD="devel" if [[ -z ${BRANCH} ]]; then BRANCH=`git name-rev $GITREF --name-only` else printf "Using environment set variable BRANCH=%s\n" "${BRANCH}" fi if [[ $BRANCH =~ ^(tags/)?(v)[.0-9]+(-(rc)?([0-9]+))?$ ]]; then RELEASE="${BASH_REMATCH[4]}${BASH_REMATCH[5]}" BUILD="rc" fi DIST=$(rpm --eval "%{dist}" | cut -d. -f2) DISTNAME=${DIST} # Special handling of FC rawhide [[ "${DISTNAME}" == "fc41" ]] && DISTNAME="fc-rawhide" # Write repository files to /etc/yum.repos.d/ based on the branch name REPO_FILE=$(./ci/write-repo-file.sh) print_info RPMBUILD=${PWD}/build SRPMS=${RPMBUILD}/SRPMS cd packaging/ make srpm RELEASE=${RELEASE} RPMBUILD=${RPMBUILD} SRPMS=${SRPMS} dnf builddep -y ${SRPMS}/* rpmbuild --rebuild --define="_topdir ${RPMBUILD}" ${SRPMS}/* gfal2-v2.23.0/ci/fedora-packages.sh000077500000000000000000000004711465240014500167300ustar00rootroot00000000000000#!/usr/bin/env bash set -e # Ensure "epel-release" package is installed if ! rpm -q --quiet epel-release ; then dnf install -y epel-release || true fi # Fedora rawhide (FC41) dnf install -y dnf5-plugins || true dnf install -y dnf-plugins-core git rpm-build tree which \ cmake make gcc gcc-c++ gfal2-v2.23.0/ci/write-repo-file.sh000077500000000000000000000016631465240014500167320ustar00rootroot00000000000000#!/usr/bin/env bash set -e GITREF=`git rev-parse --short HEAD` if [[ -z ${BRANCH} ]]; then BRANCH=`git name-rev $GITREF --name-only` fi if [[ $BRANCH =~ ^(tags/)?(v)[.0-9]+(-(rc)?([0-9]+))?$ ]]; then BUILD="rc" elif [[ ! -z ${DMC_REPO_BRANCH} ]]; then BUILD="${DMC_REPO_BRANCH}" else BUILD="develop" fi DIST=$(rpm --eval "%{dist}" | cut -d. -f2) DISTNAME=${DIST} # Special handling of FC rawhide [[ "${DISTNAME}" == "fc41" ]] && DISTNAME="fc-rawhide" if [[ ${BUILD} == "rc" ]]; then REPO_PATH="${BUILD}/${DISTNAME}/\$basearch" elif [[ ${BUILD} == "develop" ]]; then REPO_PATH="testing/${DISTNAME}/\$basearch" else REPO_PATH="testing/${BUILD}/${DISTNAME}/\$basearch" fi cat <<- EOF > "/etc/yum.repos.d/dmc-${BUILD}-${DISTNAME}.repo" [dmc-${BUILD}-${DISTNAME}] name=DMC Repository baseurl=http://dmc-repo.web.cern.ch/dmc-repo/${REPO_PATH} gpgcheck=0 enabled=1 protect=0 priority=3 EOF echo "dmc-${BUILD}-${DISTNAME}.repo" gfal2-v2.23.0/cmake/000077500000000000000000000000001465240014500140405ustar00rootroot00000000000000gfal2-v2.23.0/cmake/modules/000077500000000000000000000000001465240014500155105ustar00rootroot00000000000000gfal2-v2.23.0/cmake/modules/CMakeCXX11Support.cmake000066400000000000000000000024161465240014500216170ustar00rootroot00000000000000include(CheckCXXSourceCompiles REQUIRED) if(CMAKE_COMPILER_IS_GNUCXX) execute_process(COMMAND ${CMAKE_C_COMPILER} -dumpversion OUTPUT_VARIABLE GCC_VERSION) if(GCC_VERSION VERSION_GREATER 6.1 OR GCC_VERSION VERSION_EQUAL 6.1) SET(HAVE_CXX011_FULL_SUPPORT TRUE) SET(HAVE_CXX011_PARTIAL_SUPPORT TRUE) elseif(GCC_VERSION VERSION_GREATER 4.7 OR GCC_VERSION VERSION_EQUAL 4.7) SET(HAVE_CXX011_FULL_SUPPORT TRUE) SET(HAVE_CXX011_PARTIAL_SUPPORT TRUE) SET(CXX11_FLAG_ENABLE "-std=c++11") elseif(GCC_VERSION VERSION_GREATER 4.3 OR GCC_VERSION VERSION_EQUAL 4.3) message(STATUS "C++11 partial support") SET(HAVE_CXX011_PARTIAL_SUPPORT TRUE) SET(CXX11_FLAG_ENABLE "-std=c++0x") else () message(STATUS "C++11 no support ") SET(CXX11_FLAG_ENABLE "") endif() else(CMAKE_COMPILER_IS_GNUCXX) message(STATUS "C++11 activated full") SET(HAVE_CXX011_FULL_SUPPORT TRUE) SET(HAVE_CXX011_PARTIAL_SUPPORT TRUE) SET(CXX11_FLAG_ENABLE "-std=c++0x") endif(CMAKE_COMPILER_IS_GNUCXX) ## Check TR1 CHECK_CXX_SOURCE_COMPILES(" #include int main() { return 0; }" HAVE_TR1_SUPPORT) if(HAVE_TR1_SUPPORT) message(STATUS "TR1 support detected") else(HAVE_TR1_SUPPORT) message(STATUS "no TR1 support") endif(HAVE_TR1_SUPPORT) gfal2-v2.23.0/cmake/modules/CMakeGeneratePkgConfig.cmake000066400000000000000000000076131465240014500227440ustar00rootroot00000000000000# @title cmake macro for pkgconfig files generation # @brief generate a .pc package config file with a given name # @author Adrien Devresse include(DefineInstallationPaths REQUIRED) include(CMakeMacroParseArguments REQUIRED) include(CMakeStringHelpers REQUIRED) SET(CMAKE_PKGCONFIG_TEMPLATE "prefix=@PREFIX@ exec_prefix=@PREFIX@ libdir=@LIBDIR_VAR includedir=@INCLUDE_VAR@ Name: @NAME_PROJECT@ Description: @DESCRIPTION_PROJECT@ Version: @VERSION_PROJECT@ URL: @URL_PROJECT@ Requires: @REQUIRES_PROJECT@ Conflicts: @CONFLICTS_PROJECT@ Libs: @LIBS_PROJECT@ Libs.private: @LIBS_PRIVATE_PROJECT@ Cflags: @CFLAGS_PROJECT@ ") SET(CMAKE_PKGCONFIG_TEMPLATE_BASE " prefix=@PREFIX@ exec_prefix= \\\${prefix} libdir= @LIBDIR_VAR@ includedir=@INCLUDE_VAR@ Name: @NAME_PROJECT@ Description: @DESCRIPTION_PROJECT@ Version: @VERSION_PROJECT@ Requires: @REQUIRES_PROJECT@ Libs: @LIBS_PROJECT@ Cflags: @CFLAGS_PROJECT@ " ) LIST(APPEND CMAKE_PKGCONFIG_TEMPLATE_BASE_PATTERN "@PREFIX@" "@LIBDIR_VAR@" "@INCLUDE_VAR@" "@NAME_PROJECT@" "@DESCRIPTION_PROJECT@" "@VERSION_PROJECT@" "@REQUIRES_PROJECT@" "@LIBS_PROJECT@" "@CFLAGS_PROJECT@") # main function to use # FORMAT : add_PkgConfigFile_for_Library("string_filename.pc" target_library # [DESCRIPTION] "description of the pkgconfig files" # [HEADER_DIRS] dir1, dir2 # [REQUIRES] req1 req 2 ) # list of dir to include in $prefix/include/, ex : $prefix/include/dir1 # the pc file is produced in the ${CMAKE_CURRENT_BINARY_DIR} directory function(add_PkgConfigFile_for_Library) PARSE_ARGUMENTS(PKGCONFIGFILE "HEADER_DIRS;DESCRIPTION;REQUIRES;CFLAGS" "" ${ARGN} ) LIST(GET PKGCONFIGFILE_DEFAULT_ARGS 0 pkgconfig_filename) LIST(GET PKGCONFIGFILE_DEFAULT_ARGS 1 lib_target) LIST(GET PKGCONFIGFILE_DESCRIPTION 0 description) get_target_property(library_name ${lib_target} OUTPUT_NAME) get_target_property(library_version ${lib_target} VERSION) set(pkgconfig_prefix "${CMAKE_INSTALL_PREFIX}") set(pkgconfig_libdir_var "\\\${prefix}/lib${LIB_SUFFIX}") set(pkgconfig_include_var "\\\${prefix}/include") set(pkgconfig_linkflags "-l${library_name} -L\\\${libdir}") set(pkgconfig_name "${pkgconfig_filename}") set(pkgconfig_version "${library_version}") set(pkgconfig_description "pkgconfig file for ${library_name}") set(pkgconfig_requires " ") set(pkgconfig_cflags "") IF(PKGCONFIGFILE_REQUIRES) FOREACH(req ${PKGCONFIGFILE_REQUIRES}) set(pkgconfig_requires "${pkgconfig_requires} ${req}") ENDFOREACH(req PKGCONFIGFILE_REQUIRES) ENDIF(PKGCONFIGFILE_REQUIRES) IF(PKGCONFIGFILE_CFLAGS) FOREACH(req ${PKGCONFIGFILE_CFLAGS}) set(pkgconfig_cflags "${pkgconfig_cflags} ${req}") ENDFOREACH(req PKGCONFIGFILE_CFLAGS) ENDIF(PKGCONFIGFILE_CFLAGS) IF(PKGCONFIGFILE_HEADER_DIRS) FOREACH(dir ${PKGCONFIGFILE_HEADER_DIRS}) set(pkgconfig_includedir "${pkgconfig_includedir} -I\\\${includedir}/${dir}") ENDFOREACH(dir PKGCONFIGFILE_HEADER_DIRS) ELSE(PKGCONFIGFILE_HEADER_DIRS) set(pkgconfig_includedir " -I\\\${includedir}") ENDIF(PKGCONFIGFILE_HEADER_DIRS) IF(description) set(pkgconfig_description "${description}") ENDIF(description) set(pkgconfig_cflags "${pkgconfig_cflags} ${pkgconfig_includedir} ") LIST(APPEND pkgconfig_list_var ${pkgconfig_prefix} ${pkgconfig_libdir_var} ${pkgconfig_include_var} ${pkgconfig_name} ${pkgconfig_description} ${pkgconfig_version} ${pkgconfig_requires} ${pkgconfig_linkflags} ${pkgconfig_cflags}) replace_all_occurence(pc_file_content ${CMAKE_PKGCONFIG_TEMPLATE_BASE} LIST_PATTERN ${CMAKE_PKGCONFIG_TEMPLATE_BASE_PATTERN} LIST_REPLACER ${pkgconfig_list_var}) SET(filename "${CMAKE_CURRENT_BINARY_DIR}/${pkgconfig_filename}") FILE(WRITE ${filename} "${pc_file_content}" ) message(STATUS "generate pkgconfig file for ${lib_target} under ${filename}") endfunction(add_PkgConfigFile_for_Library) gfal2-v2.23.0/cmake/modules/CMakeMacroParseArguments.cmake000066400000000000000000000017761465240014500233500ustar00rootroot00000000000000 MACRO(PARSE_ARGUMENTS prefix arg_names option_names) SET(DEFAULT_ARGS) FOREACH(arg_name ${arg_names}) SET(${prefix}_${arg_name}) ENDFOREACH(arg_name) FOREACH(option ${option_names}) SET(${prefix}_${option} FALSE) ENDFOREACH(option) SET(current_arg_name DEFAULT_ARGS) SET(current_arg_list) FOREACH(arg ${ARGN}) SET(larg_names ${arg_names}) LIST(FIND larg_names "${arg}" is_arg_name) IF (is_arg_name GREATER -1) SET(${prefix}_${current_arg_name} ${current_arg_list}) SET(current_arg_name ${arg}) SET(current_arg_list) ELSE (is_arg_name GREATER -1) SET(loption_names ${option_names}) LIST(FIND loption_names "${arg}" is_option) IF (is_option GREATER -1) SET(${prefix}_${arg} TRUE) ELSE (is_option GREATER -1) SET(current_arg_list ${current_arg_list} ${arg}) ENDIF (is_option GREATER -1) ENDIF (is_arg_name GREATER -1) ENDFOREACH(arg) SET(${prefix}_${current_arg_name} ${current_arg_list}) ENDMACRO(PARSE_ARGUMENTS) gfal2-v2.23.0/cmake/modules/CMakeStringHelpers.cmake000066400000000000000000000032331465240014500222050ustar00rootroot00000000000000##convenience function for string manipulation function(replace_occurence output input pattern replacer) string(REGEX REPLACE ${pattern} ${replacer} tmp_str ${input}) set(${output} ${tmp_str} PARENT_SCOPE) endfunction(replace_occurence output input pattern replacer) function(replace_all_occurence) PARSE_ARGUMENTS(REPLACE_ALL "LIST_PATTERN;LIST_REPLACER" "" ${ARGN} ) LIST(APPEND list_pattern ${REPLACE_ALL_LIST_PATTERN}) LIST(APPEND list_replacer ${REPLACE_ALL_LIST_REPLACER}) LIST(LENGTH list_pattern list_size ) LIST(LENGTH list_replacer list2_size ) math(EXPR list_size ${list_size}-1) LIST(GET REPLACE_ALL_DEFAULT_ARGS 0 output_var) LIST(GET REPLACE_ALL_DEFAULT_ARGS 1 intput_content ) SET(tmp_str ${intput_content}) SET(tmp_str2 "") foreach(i RANGE ${list_size}) list(GET list_pattern ${i} current_pattern ) list(GET list_replacer ${i} current_replacer ) replace_occurence(tmp_str2 ${tmp_str} ${current_pattern} ${current_replacer} ) SET(tmp_str ${tmp_str2}) endforeach(i RANGE ${list_size}) SET(${output_var} ${tmp_str} PARENT_SCOPE) endfunction(replace_all_occurence) function(STRING_APPEND var_name content) SET(${var_name} "${${var_name}}${content}" PARENT_SCOPE) endfunction(STRING_APPEND var_name content) function(parse_lib_path lib_link lib_directory lib_path) string(REGEX REPLACE "^.*/(lib)?(.*)\\.(so|dll)" "\\2" my_lib_link ${lib_path}) string(REGEX REPLACE "^(.*)/(lib)?(.*)\\.(so|dll)" "\\1" my_lib_dir ${lib_path}) SET(${lib_link} ${my_lib_link} PARENT_SCOPE) SET(${lib_directory} ${my_lib_dir} PARENT_SCOPE) endfunction(parse_lib_path lib_link lib_directory lib_path) gfal2-v2.23.0/cmake/modules/DefineInstallationPaths.cmake000066400000000000000000000105551465240014500232740ustar00rootroot00000000000000if (UNIX) IF (NOT APPLICATION_NAME) MESSAGE(STATUS "${PROJECT_NAME} is used as APPLICATION_NAME") SET(APPLICATION_NAME ${PROJECT_NAME}) ENDIF (NOT APPLICATION_NAME) # Suffix for Linux IF (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") SET(LIB_SUFFIX "" CACHE STRING "Suffix of the lib") ELSE () IF (CMAKE_SIZEOF_VOID_P EQUAL 4) SET(LIB_SUFFIX "" CACHE STRING "Suffix of the lib") SET (PKG_ARCH "i386") ELSE (CMAKE_SIZEOF_VOID_P EQUAL 4) SET(LIB_SUFFIX "64" CACHE STRING "Suffix of the lib") SET (PKG_ARCH "x86_64") ENDIF (CMAKE_SIZEOF_VOID_P EQUAL 4) ENDIF () SET(EXEC_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}" CACHE PATH "Base directory for executables and libraries" ) SET(SHARE_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}/share" CACHE PATH "Base directory for files which go to share/" ) SET(DATA_INSTALL_PREFIX "${SHARE_INSTALL_PREFIX}/${APPLICATION_NAME}" CACHE PATH "The parent directory where applications can install their data") # The following are directories where stuff will be installed to SET(BIN_INSTALL_DIR "${EXEC_INSTALL_PREFIX}/bin" CACHE PATH "The ${APPLICATION_NAME} binary install dir (default prefix/bin)" ) SET(SBIN_INSTALL_DIR "${EXEC_INSTALL_PREFIX}/sbin" CACHE PATH "The ${APPLICATION_NAME} sbin install dir (default prefix/sbin)" ) SET(LIB_INSTALL_DIR "${EXEC_INSTALL_PREFIX}/lib${LIB_SUFFIX}" CACHE PATH "The subdirectory relative to the install prefix where libraries will be installed (default is prefix/lib)" ) SET(LIBEXEC_INSTALL_DIR "${EXEC_INSTALL_PREFIX}/libexec" CACHE PATH "The subdirectory relative to the install prefix where libraries will be installed (default is prefix/libexec)" ) SET(PKGCONFIG_FILES_DIR "${LIB_INSTALL_DIR}/pkgconfig/" CACHE PATH "subdirectory relative to the install prefix where pkgconfig files (.pc) will be installed" ) SET(PLUGIN_INSTALL_DIR "${LIB_INSTALL_DIR}/${APPLICATION_NAME}-plugins/" CACHE PATH "The subdirectory relative to the install prefix where plugins will be installed (default is prefix/lib/${APPLICATION_NAME})" ) SET(INCLUDE_INSTALL_DIR "${CMAKE_INSTALL_PREFIX}/include" CACHE PATH "The subdirectory to the header prefix (default prefix/include)" ) SET(DATA_INSTALL_DIR "${DATA_INSTALL_PREFIX}" CACHE PATH "The parent directory where applications can install their data (default prefix/share/${APPLICATION_NAME})" ) SET(DOC_INSTALL_DIR "${SHARE_INSTALL_PREFIX}/doc/${APPLICATION_NAME}" CACHE PATH "The parent directory where applications can install their documentation (default prefix/share/doc/${APPLICATION_NAME})" ) SET(HTML_INSTALL_DIR "${DATA_INSTALL_PREFIX}/doc/HTML" CACHE PATH "The HTML install dir for documentation (default data/doc/html)" ) SET(ICON_INSTALL_DIR "${DATA_INSTALL_PREFIX}/icons" CACHE PATH "The icon install dir (default data/icons/)" ) SET(SOUND_INSTALL_DIR "${DATA_INSTALL_PREFIX}/sounds" CACHE PATH "The install dir for sound files (default data/sounds)" ) SET(LOCALE_INSTALL_DIR "${SHARE_INSTALL_PREFIX}/locale" CACHE PATH "The install dir for translations (default prefix/share/locale)" ) SET(XDG_APPS_DIR "${SHARE_INSTALL_PREFIX}/applications/" CACHE PATH "The XDG apps dir" ) SET(XDG_DIRECTORY_DIR "${SHARE_INSTALL_PREFIX}/desktop-directories" CACHE PATH "The XDG directory" ) SET(SYSCONF_INSTALL_DIR "${EXEC_INSTALL_PREFIX}/etc" CACHE PATH "The ${APPLICATION_NAME} sysconfig install dir (default prefix/etc)" ) SET(MAN_INSTALL_DIR "${SHARE_INSTALL_PREFIX}/man" CACHE PATH "The ${APPLICATION_NAME} man install dir (default prefix/man)" ) SET(INFO_INSTALL_DIR "${SHARE_INSTALL_PREFIX}/info" CACHE PATH "The ${APPLICATION_NAME} info install dir (default prefix/info)" ) endif (UNIX) if (WIN32) # Same same set(BIN_INSTALL_DIR "." CACHE PATH "-") set(SBIN_INSTALL_DIR "." CACHE PATH "-") set(LIB_INSTALL_DIR "lib" CACHE PATH "-") set(INCLUDE_INSTALL_DIR "include" CACHE PATH "-") set(PLUGIN_INSTALL_DIR "plugins" CACHE PATH "-") set(HTML_INSTALL_DIR "doc/HTML" CACHE PATH "-") set(ICON_INSTALL_DIR "." CACHE PATH "-") set(SOUND_INSTALL_DIR "." CACHE PATH "-") set(LOCALE_INSTALL_DIR "lang" CACHE PATH "-") endif (WIN32) gfal2-v2.23.0/cmake/modules/FindCGSI_GSOAP.cmake000066400000000000000000000040741465240014500207760ustar00rootroot00000000000000# # This module detects if CGSI_GSOAP is installed and determines where the # include files and libraries are. # # This code sets the following variables: # # CGSI_GSOAP_LIBRARIES = full path to the CGSI_GSOAP libraries # CGSI_GSOAP_INCLUDE_DIRS = include dir to be used when using the CGSI_GSOAP library # CGSI_GSOAP_FOUND = set to true if CGSI_GSOAP was found successfully # # CGSI_GSOAP_LOCATION # setting this enables search for CGSI_GSOAP libraries / headers in this location include(CMakeStringHelpers) # ----------------------------------------------------- # CGSI_GSOAP Libraries # ----------------------------------------------------- find_library(CGSI_GSOAP_LIBRARIES NAMES cgsi_plugin HINTS ${CGSI_GSOAP_LOCATION} ${STAGE_DIR} ${CMAKE_INSTALL_PREFIX}/cgsigsoap/*/${PLATFORM}/lib ${CMAKE_INSTALL_PREFIX}/cgsigsoap/*/${PLATFORM}/lib64 ${CMAKE_INSTALL_PREFIX}/opt/cgsi-gsoap/lib ${CMAKE_INSTALL_PREFIX}/opt/cgsi-gsoap/lib64 DOC "The main CGSI_GSOAP library" ) # ----------------------------------------------------- # CGSI_GSOAP Include Directories # ----------------------------------------------------- find_path(CGSI_GSOAP_INCLUDE_DIRS NAMES cgsi_plugin.h HINTS ${CGSI_GSOAP_LOCATION} ${STAGE_DIR}/include ${CMAKE_INSTALL_PREFIX}/cgsigsoap/*/${PLATFORM} ${CMAKE_INSTALL_PREFIX}/cgsigsoap/*/${PLATFORM}/include DOC "The CGSI_GSOAP include directory" ) if(CGSI_GSOAP_INCLUDE_DIRS) message(STATUS "CGSI_GSOAP includes found in ${CGSI_GSOAP_INCLUDE_DIRS}") endif() if(CGSI_GSOAP_LIBRARIES) message(STATUS "CGSI_GSOAP libraries found in ${CGSI_GSOAP_LIBRARIES}") endif() # ----------------------------------------------------- # handle the QUIETLY and REQUIRED arguments and set CGSI_GSOAP_FOUND to TRUE if # all listed variables are TRUE # ----------------------------------------------------- include(FindPackageHandleStandardArgs) find_package_handle_standard_args(CGSI_GSOAP DEFAULT_MSG CGSI_GSOAP_LIBRARIES) mark_as_advanced(CGSI_GSOAP_INCLUDE_DIRS CGSI_GSOAP_LIBRARIES) gfal2-v2.23.0/cmake/modules/FindCryptopp.cmake000066400000000000000000000036271465240014500211430ustar00rootroot00000000000000# # Copyright (c) CERN 2023 # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # This code sets the following variables: # # CRYPTOPP_LIBRARIES = full path to the cryptopp libraries # CRYPTOPP_INCLUDE_DIRS = include dir to be used when using the cryptopp library # CRYPTOPP_FOUND = set to true if cryptopp was found successfully # ----------------------------------------------------- # Cryptopp Libraries # ----------------------------------------------------- find_library(CRYPTOPP_LIBRARIES NAMES cryptopp HINTS ${CRYPTOPP_LOCATION} /lib /lib64 /usr/lib /usr/lib64 DOC "The cryptopp library" ) # ----------------------------------------------------- # Cryptopp Include Directories # ----------------------------------------------------- find_path(CRYPTOPP_INCLUDE_DIRS NAMES base64.h # header for base64 encoding, not processor architecture HINTS ${CRYPTOPP_LOCATION} /usr/include/cryptopp DOC "The cryptopp headers" ) # ------------------------------------------------------------------- # Handle the QUIETLY and REQUIRED arguments # and set CRYPTOPP to TRUE if all listed variables are TRUE # ------------------------------------------------------------------- include(FindPackageHandleStandardArgs) find_package_handle_standard_args(cryptopp DEFAULT_MSG CRYPTOPP_LIBRARIES CRYPTOPP_INCLUDE_DIRS) mark_as_advanced(CRYPTOPP_LIBRARIES CRYPTOPP_INCLUDE_DIRS) gfal2-v2.23.0/cmake/modules/FindDCAP.cmake000066400000000000000000000036111465240014500200230ustar00rootroot00000000000000# # This module detects if DCAP is installed and determines where the # include files and libraries are. # # This code sets the following variables: # # DCAP_LIBRARIES = full path to the DCAP libraries # DCAP_INCLUDE_DIR = include dir to be used when using the DCAP library # DCAP_FOUND = set to true if DCAP was found successfully # # DCAP_LOCATION # setting this enables search for DCAP libraries / headers in this location # ----------------------------------------------------- # DCAP Libraries # ----------------------------------------------------- find_library(DCAP_LIBRARIES NAMES dcap HINTS ${DCAP_LOCATION} ${STAGE_DIR} ${CMAKE_INSTALL_PREFIX}/dcap/*/${PLATFORM}/lib ${CMAKE_INSTALL_PREFIX}/dcap/*/${PLATFORM}/lib64 ${CMAKE_INSTALL_PREFIX}/Grid/dcap/*/${PLATFORM}/lib ${CMAKE_INSTALL_PREFIX}/Grid/dcap/*/${PLATFORM}/lib64 DOC "The main DCAP library" ) # ----------------------------------------------------- # DCAP Include Directories # ----------------------------------------------------- find_path(DCAP_INCLUDE_DIR NAMES dcap.h HINTS ${DCAP_LOCATION} ${STAGE_DIR} ${CMAKE_INSTALL_PREFIX}/dcap/*/${PLATFORM}/ ${CMAKE_INSTALL_PREFIX}/Grid/dcap/*/${PLATFORM}/include DOC "The DCAP include directory" ) if (DCAP_LIBRARIES) message (STATUS "DCAP libraries found in ${DCAP_LIBRARIES}") endif (DCAP_LIBRARIES) if(DCAP_INCLUDE_DIR) message(STATUS "DCAP includes found in ${DCAP_INCLUDE_DIR}") endif() # ----------------------------------------------------- # handle the QUIETLY and REQUIRED arguments and set DCAP_FOUND to TRUE if # all listed variables are TRUE # ----------------------------------------------------- include(FindPackageHandleStandardArgs) find_package_handle_standard_args(DCAP DEFAULT_MSG DCAP_LIBRARIES DCAP_INCLUDE_DIR) mark_as_advanced(DCAP_INCLUDE_DIR DCAP_LIBRARIES) gfal2-v2.23.0/cmake/modules/FindDPM.cmake000066400000000000000000000051451465240014500177400ustar00rootroot00000000000000# # This module detects if dpm is installed and determines where the # include files and libraries are. # # This code sets the following variables: # # DPM_LIBRARIES = full path to the dpm libraries # DPM_INCLUDE_DIR = include dir to be used when using the dpm library # DPM_FOUND = set to true if dpm was found successfully # # DPM_LOCATION # setting this enables search for dpm libraries / headers in this location # ----------------------------------------------------- # DPM Libraries # ----------------------------------------------------- find_library(DPM_LIBRARIES NAMES dpm HINTS ${DPM_LOCATION} ${STAGE_DIR} ${CMAKE_INSTALL_PREFIX}/dcap/*/${PLATFORM}/ ${CMAKE_INSTALL_PREFIX}/Grid/dcap/*/${PLATFORM}/ DOC "The main dpm library" ) # ----------------------------------------------------- # LCGDM Libraries # ----------------------------------------------------- find_library(LCGDM_LIBRARIES NAMES lcgdm HINTS ${DPM_LOCATION} ${STAGE_DIR} ${CMAKE_INSTALL_PREFIX}/dcap/*/${PLATFORM}/ ${CMAKE_INSTALL_PREFIX}/Grid/dcap/*/${PLATFORM}/ DOC "The main lcgdm library" ) # ----------------------------------------------------- # DPM Include Directories # ----------------------------------------------------- find_path(DPM_INCLUDE_DIR NAMES dpm/dpm_api.h HINTS ${DPM_LOCATION} ${STAGE_DIR} ${CMAKE_INSTALL_PREFIX}/dcap/*/${PLATFORM}/ ${CMAKE_INSTALL_PREFIX}/Grid/dcap/*/${PLATFORM}/ DOC "The dpm include directory" ) if(DPM_INCLUDE_DIR) message(STATUS "dpm includes found in ${DPM_INCLUDE_DIR}") endif() # ----------------------------------------------------- # LCGDM Include Directories # ----------------------------------------------------- find_path(LCGDM_INCLUDE_DIR NAMES Cinit.h HINTS ${LCGDM_LOCATION} ${STAGE_DIR} ${CMAKE_INSTALL_PREFIX}/dcap/*/${PLATFORM}/ ${CMAKE_INSTALL_PREFIX}/Grid/dcap/*/${PLATFORM}/ DOC "The LCGDM include directory" ) if(LCGDM_INCLUDE_DIR) message(STATUS "lcgdm includes found in ${LCGDM_INCLUDE_DIR}") endif() # ----------------------------------------------------- # handle the QUIETLY and REQUIRED arguments and set DPM_FOUND to TRUE if # all listed variables are TRUE # ----------------------------------------------------- include(FindPackageHandleStandardArgs) find_package_handle_standard_args(dpm DEFAULT_MSG DPM_LIBRARIES DPM_INCLUDE_DIR) find_package_handle_standard_args(lcgdm DEFAULT_MSG LCGDM_LIBRARIES LCGDM_INCLUDE_DIR) mark_as_advanced(DPM_INCLUDE_DIR DPM_LIBRARIES) mark_as_advanced(LCGDM_INCLUDE_DIR LCGDM_LIBRARIES) gfal2-v2.23.0/cmake/modules/FindDavix.cmake000066400000000000000000000055641465240014500204000ustar00rootroot00000000000000# # This module detects if Davix is installed and determines where the # include files and libraries are. # # This code sets the following variables: # # DAVIX_LIBRARIES = full path to the Davix libraries # DAVIX_INCLUDE_DIR = include dir to be used when using the Davix library # DAVIX_FOUND = set to true if Davix was found successfully # # DAVIX_LOCATION # setting this enables search for Davix libraries / headers in this location # ----------------------------------------------------- # Try with pkgconfig first # ----------------------------------------------------- pkg_check_modules(DAVIX_PKG davix>=0.7.6) pkg_check_modules(DAVIX_COPY_PKG davix_copy>=0.7.6) if (DAVIX_PKG_FOUND AND DAVIX_COPY_PKG_FOUND) set (DAVIX_INCLUDE_DIR "${DAVIX_PKG_INCLUDE_DIRS}" "${DAVIX_COPY_PKG_INCLUDE_DIRS}") set (DAVIX_LIBRARIES "${DAVIX_PKG_LIBRARIES}" "${DAVIX_COPY_PKG_LIBRARIES}") set (DAVIX_CFLAGS "${DAVIX_PKG_CFLAGS} ${DAVIX_COPY_PKG_FLAGS}") else () # Davix Libraries find_library(DAVIX_MAIN_LIBRARY NAMES davix HINTS ${DAVIX_LOCATION}/lib ${DAVIX_LOCATION}/lib64 ${DAVIX_LOCATION}/lib32 ${STAGE_DIR}/lib ${STAGE_DIR}/lib64 ${CMAKE_INSTALL_PREFIX}/Davix/*/${PLATFORM}/lib ${CMAKE_INSTALL_PREFIX}/Davix/*/${PLATFORM}/lib64 ${CMAKE_INSTALL_PREFIX}/lib DOC "The main davix library" ) find_library(DAVIX_COPY_LIBRARY NAMES davix_copy HINTS ${DAVIX_LOCATION}/lib ${DAVIX_LOCATION}/lib64 ${DAVIX_LOCATION}/lib32 ${STAGE_DIR}/lib ${STAGE_DIR}/lib64 ${CMAKE_INSTALL_PREFIX}/Davix/*/${PLATFORM}/lib ${CMAKE_INSTALL_PREFIX}/Davix/*/${PLATFORM}/lib64 ${CMAKE_INSTALL_PREFIX}/lib DOC "The davix copy library" ) set (DAVIX_LIBRARIES ${DAVIX_MAIN_LIBRARY} ${DAVIX_COPY_LIBRARY}) # Davix Include Directories find_path(DAVIX_INCLUDE_DIR NAMES davix.hpp HINTS ${DAVIX_LOCATION} ${DAVIX_LOCATION}/include ${DAVIX_LOCATION}/include/* ${STAGE_DIR}/include ${STAGE_DIR}/include ${CMAKE_INSTALL_PREFIX}/Davix/*/${PLATFORM}/include/* ${CMAKE_INSTALL_PREFIX}/include/davix DOC "Davix include directory" ) set (DAVIX_CFLAGS "") endif() if (DAVIX_LIBRARIES) message (STATUS "DAVIX libraries: ${DAVIX_LIBRARIES}") endif (DAVIX_LIBRARIES) if(DAVIX_INCLUDE_DIR) message(STATUS "DAVIX includes found in ${DAVIX_INCLUDE_DIR}") endif() # ----------------------------------------------------- # handle the QUIETLY and REQUIRED arguments and set DAVIX_FOUND to TRUE if # all listed variables are TRUE # ----------------------------------------------------- include(FindPackageHandleStandardArgs) find_package_handle_standard_args(DAVIX DEFAULT_MSG DAVIX_LIBRARIES DAVIX_INCLUDE_DIR) mark_as_advanced(DAVIX_LIBRARIES DAVIX_INCLUDE_DIR DAVIX_CFLAGS) gfal2-v2.23.0/cmake/modules/FindGFAL2.cmake000066400000000000000000000050051465240014500201060ustar00rootroot00000000000000# # This module detects if gfal2 is installed and determines where the # include files and libraries are. # # This code sets the following variables: # # GFAL2_LIBRARIES = full path to the gfal2 libraries # GFAL2_INCLUDE_DIR = include dir to be used when using the gfal2 library # GFAL2_FOUND = set to true if gfal2 was found successfully # # GFAL2_LOCATION # setting this enables search for gfal2 libraries / headers in this location find_package (PkgConfig) pkg_check_modules(GFAL2_PKG gfal2) pkg_check_modules(GFAL2_TRANSFER_PKG gfal_transfer) if (GFAL2_PKG_FOUND AND GFAL2_TRANSFER_PKG_FOUND) set (GFAL2_LIBRARIES ${GFAL2_PKG_LIBRARIES} ${GFAL2_TRANSFER_PKG_LIBRARIES}) set (GFAL2_INCLUDE_DIRS ${GFAL2_PKG_INCLUDE_DIRS} ${GFAL2_TRANSFER_PKG_INCLUDE_DIRS}) set (GFAL2_DEFINITIONS "${GFAL2_PKG_CFLAGS} ${GFAL2_TRANSFER_PKG_CFLAGS}") set (GFAL2_LIBRARY_DIRS ${GFAL2_PKG_LIBRARY_DIRS}) else (GFAL2_PKG_FOUND AND GFAL2_TRANSFER_PKG_FOUND) find_library(GFAL2_CORE_LIBRARIES NAMES gfal2 HINTS ${GFAL2_LOCATION} ${CMAKE_INSTALL_PREFIX}/Grid/gfal2/*/${PLATFORM}/lib ${CMAKE_INSTALL_PREFIX}/Grid/gfal2/*/${PLATFORM}/lib64 DOC "The main gfal2 library" ) find_library(GFAL2_TRANSFER_LIBRARIES NAMES gfal_transfer HINTS ${GFAL2_LOCATION} ${CMAKE_INSTALL_PREFIX}/Grid/gfal2/*/${PLATFORM}/lib ${CMAKE_INSTALL_PREFIX}/Grid/gfal2/*/${PLATFORM}/lib64 DOC "The transfer gfal2 library" ) set (GFAL2_LIBRARIES ${GFAL2_CORE_LIBRARIES} ${GFAL2_TRANSFER_LIBRARIES}) find_path(GFAL2_INCLUDE_DIRS NAMES gfal_api.h HINTS ${GFAL2_LOCATION}/include/* ${CMAKE_INSTALL_PREFIX}/Grid/gfal2/*/${PLATFORM}/include/* DOC "The gfal2 include directory" ) set (GFAL2_DEFINITIONS "") endif (GFAL2_PKG_FOUND AND GFAL2_TRANSFER_PKG_FOUND) if (GFAL2_LIBRARIES) message (STATUS "GFAL2 libraries: ${GFAL2_LIBRARIES}") endif (GFAL2_LIBRARIES) if (GFAL2_INCLUDE_DIRS) message (STATUS "GFAL2 include dir: ${GFAL2_INCLUDE_DIRS}") endif (GFAL2_INCLUDE_DIRS) # ----------------------------------------------------- # handle the QUIETLY and REQUIRED arguments and set GFAL2_FOUND to TRUE if # all listed variables are TRUE # ----------------------------------------------------- include(FindPackageHandleStandardArgs) find_package_handle_standard_args (GFAL2 DEFAULT_MSG GFAL2_LIBRARIES GFAL2_INCLUDE_DIRS ) mark_as_advanced(GFAL2_INCLUDE_DIRS GFAL2_LIBRARIES GFAL2_LIBRARY_DIRS) gfal2-v2.23.0/cmake/modules/FindGLIB2.cmake000066400000000000000000000037451465240014500201230ustar00rootroot00000000000000# # This module detects if glib2 is installed and determines where the # include files and libraries are. # # This code sets the following variables: # # GLIB2_LIBRARIES = full path to the glib2 libraries # GLIB2_INCLUDE_DIR = include dir to be used when using the glib2 library # GLIB2_FOUND = set to true if glib2 was found successfully # # GLIB2_LOCATION # setting this enables search for glib2 libraries / headers in this location find_package (PkgConfig) pkg_check_modules (GLIB2_PKG glib-2.0) if (GLIB2_PKG_FOUND) set (GLIB2_LIBRARIES ${GLIB2_PKG_LIBRARIES}) set (GLIB2_INCLUDE_DIRS ${GLIB2_PKG_INCLUDE_DIRS}) set (GLIB2_DEFINITIONS "${GLIB2_PKG_CFLAGS} ${GLIB2_PKG_CFLAGS_OTHER}") set (GLIB2_LIBRARY_DIRS ${GLIB2_PKG_LIBRARY_DIRS}) else (GLIB2_PKG_FOUND) find_library(GLIB2_LIBRARIES NAMES libglib-2.0 HINTS ${GLIB2_LOCATION} ${CMAKE_INSTALL_PREFIX}/glib2/*/${PLATFORM}/ DOC "The main glib2 library" ) find_path(GLIB2_INCLUDE_DIRS NAMES glib.h HINTS ${GLIB2_LOCATION}/include/* ${CMAKE_INSTALL_PREFIX}/glib2/*/${PLATFORM}/ DOC "The glib2 include directory" ) set (GLIB2_DEFINITIONS "") endif (GLIB2_PKG_FOUND) if (GLIB2_LIBRARIES) message (STATUS "GLIB2 libraries: ${GLIB2_LIBRARIES}") endif (GLIB2_LIBRARIES) if (GLIB2_INCLUDE_DIRS) message (STATUS "GLIB2 include dir: ${GLIB2_INCLUDE_DIRS}") endif (GLIB2_INCLUDE_DIRS) if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") link_directories ("${CMAKE_INSTALL_PREFIX}/opt/gettext/lib") endif () # ----------------------------------------------------- # handle the QUIETLY and REQUIRED arguments and set GLIB2_FOUND to TRUE if # all listed variables are TRUE # ----------------------------------------------------- include(FindPackageHandleStandardArgs) find_package_handle_standard_args (GLIB2 DEFAULT_MSG GLIB2_LIBRARIES GLIB2_INCLUDE_DIRS ) mark_as_advanced(GLIB2_INCLUDE_DIRS GLIB2_LIBRARIES GLIB2_LIBRARY_DIRS) gfal2-v2.23.0/cmake/modules/FindGRIDFTP_IFCE.cmake000066400000000000000000000033051465240014500212010ustar00rootroot00000000000000# # This module detects if GRIDFTP_IFCE is installed and determines where the # include files and libraries are. # # This code sets the following variables: # # GRIDFTP_IFCE_LIBRARIES = full path to the dpm libraries # GRIDFTP_IFCE_INCLUDE_DIR = include dir to be used when using the dpm library # GRIDFTP_IFCE_FOUND = set to true if dpm was found successfully # # GRIDFTP_IFCE_LOCATION # setting this enables search for dpm libraries / headers in this location # ----------------------------------------------------- # DPM Libraries # ----------------------------------------------------- find_library(GRIDFTP_IFCE_LIBRARIES NAMES gridftp_ifce HINTS ${GRIDFTP_IFCE_LOCATION}/lib ${GRIDFTP_IFCE_LOCATION}/lib64 ${GRIDFTP_IFCE_LOCATION}/lib32 DOC "The main gridftp_ifce library" ) # ----------------------------------------------------- # GRIDFTP_IFCE Include Directories # ----------------------------------------------------- find_path(GRIDFTP_IFCE_INCLUDE_DIR NAMES gridftp-ifce.h HINTS ${GRIDFTP_IFCE_LOCATION} ${GRIDFTP_IFCE_LOCATION}/include ${GRIDFTP_IFCE_LOCATION}/include/* DOC "The gridftp-ifce.h include directory" ) if(GRIDFTP_IFCE_INCLUDE_DIR) message(STATUS "gridftp_ifce includes found in ${GRIDFTP_IFCE_INCLUDE_DIR}") endif() # ----------------------------------------------------- # handle the QUIETLY and REQUIRED arguments and set GRIDFTP_IFCE_FOUND to TRUE if # all listed variables are TRUE # ----------------------------------------------------- include(FindPackageHandleStandardArgs) find_package_handle_standard_args(gridftp_ifce DEFAULT_MSG GRIDFTP_IFCE_LIBRARIES GRIDFTP_IFCE_INCLUDE_DIR) mark_as_advanced(GRIDFTP_IFCE_INCLUDE_DIR GRIDFTP_IFCE_LIBRARIES) gfal2-v2.23.0/cmake/modules/FindGTEST.cmake000066400000000000000000000037631465240014500202120ustar00rootroot00000000000000# # This module detects if GTEST is installed and determines where the # include files and libraries are. # # This code sets the following variables: # # GTEST_LIBRARIES = full path to the GTEST libraries # GTEST_SSL_LIBRARIES = full path to the GTEST ssl libraries # GTEST_INCLUDE_DIR = include dir to be used when using the GTEST library # GTEST_WSDL2H = wsdl2h binary # GTEST_SOAPCPP2 = soapcpp2 binary # GTEST_FOUND = set to true if GTEST was found successfully # # GTEST_LOCATION # setting this enables search for GTEST libraries / headers in this location # ----------------------------------------------------- # GTEST Libraries # ----------------------------------------------------- find_library(GTEST_LIBRARIES NAMES gtest HINTS ${GTEST_LOCATION}/lib ${GTEST_LOCATION}/lib64 ${GTEST_LOCATION}/lib32 DOC "The main GTEST library" ) # ----------------------------------------------------- # GTEST Libraries # ----------------------------------------------------- find_library(GTEST_MAIN_LIBRARIES NAMES gtest_main HINTS ${GTEST_LOCATION}/lib ${GTEST_LOCATION}/lib64 ${GTEST_LOCATION}/lib32 DOC "The main GTEST main library" ) # ----------------------------------------------------- # GTEST Include Directories # ----------------------------------------------------- find_path(GTEST_INCLUDE_DIR NAMES gtest.h HINTS ${GTEST_LOCATION} ${GTEST_LOCATION}/include ${GTEST_LOCATION}/include/gtest ${GTEST_LOCATION}/include/* /usr/include/gtest DOC "The GTEST include directory" ) SET(GTEST_DEFINITIONS "") # ----------------------------------------------------- # handle the QUIETLY and REQUIRED arguments and set GTEST_FOUND to TRUE if # all listed variables are TRUE # ----------------------------------------------------- include(FindPackageHandleStandardArgs) find_package_handle_standard_args(GTEST DEFAULT_MSG GTEST_LIBRARIES GTEST_MAIN_LIBRARIES GTEST_INCLUDE_DIR) mark_as_advanced(GTEST_INCLUDE_DIR GTEST_LIBRARIES ) gfal2-v2.23.0/cmake/modules/FindGTHREAD2.cmake000066400000000000000000000036431465240014500204610ustar00rootroot00000000000000# # This module detects if gthread2 is installed and determines where the # include files and libraries are. # # This code sets the following variables: # # GTHREAD2_LIBRARIES = full path to the glib2 libraries # GTHREAD2_INCLUDE_DIR = include dir to be used when using the glib2 library # GTHREAD2_FOUND = set to true if glib2 was found successfully # # GTHREAD2_LOCATION # setting this enables search for gthread2 libraries / headers in this location find_package (PkgConfig) pkg_check_modules(GTHREAD2_PKG gthread-2.0) if (GTHREAD2_PKG_FOUND) set (GTHREAD2_LIBRARIES ${GTHREAD2_PKG_LIBRARIES}) set (GTHREAD2_INCLUDE_DIRS ${GTHREAD2_PKG_INCLUDE_DIRS}) set (GTHREAD2_DEFINITIONS "${GTHREAD2_PKG_CFLAGS} ${GTHREAD2_PKG_CFLAGS_OTHER}") else (GTHREAD2_PKG_FOUND) find_library(GTHREAD2_LIBRARIES NAMES libgthread-2.0.so.0 HINTS ${GTHREAD2_LOCATION} ${CMAKE_INSTALL_PREFIX}/glib2/*/${PLATFORM}/ DOC "The main gthread2 library" ) find_path(GTHREAD2_INCLUDE_DIRS NAMES gthread.h HINTS ${GTHREAD2_LOCATION}/include/* ${CMAKE_INSTALL_PREFIX}/glib2/*/${PLATFORM}/ DOC "The gthread2 include directory" ) set (GTHREAD2_DEFINITIONS "") endif (GTHREAD2_PKG_FOUND) if (GTHREAD2_LIBRARIES) message (STATUS "GTHREAD2 libraries: ${GTHREAD2_LIBRARIES}") endif (GTHREAD2_LIBRARIES) if (GTHREAD2_INCLUDE_DIRS) message (STATUS "GTHREAD2 include dir: ${GTHREAD2_INCLUDE_DIRS}") endif (GTHREAD2_INCLUDE_DIRS) # ----------------------------------------------------- # handle the QUIETLY and REQUIRED arguments and set GTHREAD2_FOUND to TRUE if # all listed variables are TRUE # ----------------------------------------------------- include(FindPackageHandleStandardArgs) find_package_handle_standard_args (GTHREAD2 DEFAULT_MSG GTHREAD2_LIBRARIES GTHREAD2_INCLUDE_DIRS ) mark_as_advanced(GTHREAD2_INCLUDE_DIRS GTHREAD2_LIBRARIES) gfal2-v2.23.0/cmake/modules/FindGlobus_COMMON.cmake000066400000000000000000000050701465240014500216200ustar00rootroot00000000000000# # This module detects if globus-common is installed and determines where the # include files and libraries are. # # This code sets the following variables: # # GLOBUS_COMMON_LIBRARIES = full path to the globus-common libraries # GLOBUS_COMMON_INCLUDE_DIR = include dir to be used when using the globus-common library # GLOBUS_COMMON_FOUND = set to true if globus-common was found successfully # # GLOBUS_COMMON_LOCATION # setting this enables search for globus-common libraries / headers in this location find_package (PkgConfig) pkg_check_modules(GLOBUS_COMMON_PKG globus-common) if (GLOBUS_COMMON_PKG_FOUND) set (GLOBUS_COMMON_LIBRARIES ${GLOBUS_COMMON_PKG_LIBRARIES}) set (GLOBUS_COMMON_INCLUDE_DIRS ${GLOBUS_COMMON_PKG_INCLUDE_DIRS}) set (GLOBUS_COMMON_DEFINITIONS "${GLOBUS_COMMON_PKG_CFLAGS}") else (GLOBUS_COMMON_PKG_FOUND) set (CMAKE_FIND_FRAMEWORK NEVER) find_library(GLOBUS_COMMON_LIBRARIES NAMES globus_common HINTS ${GLOBUS_COMMON_LOCATION} ${CMAKE_INSTALL_PREFIX}/globus/*/${PLATFORM}/ ${CMAKE_INSTALL_PREFIX}/Grid/epel/*/${PLATFORM}/lib ${CMAKE_INSTALL_PREFIX}/Grid/epel/*/${PLATFORM}/lib64 ${CMAKE_INSTALL_PREFIX}/opt/globus-toolkit/libexec/lib ${GLOBUS_PREFIX}/libexec/lib DOC "The main globus-common library" ) find_path(GLOBUS_COMMON_INCLUDE_DIRS NAMES globus_common.h HINTS ${GLOBUS_COMMON_LOCATION}/include/* ${CMAKE_INSTALL_PREFIX}/globus/*/${PLATFORM}/include ${CMAKE_INSTALL_PREFIX}/Grid/epel/*/${PLATFORM}/include ${CMAKE_INSTALL_PREFIX}/globus-toolkit/libexec/include ${GLOBUS_PREFIX}/libexec/include DOC "The globus-common include directory" ) set (GLOBUS_COMMON_DEFINITIONS "") endif (GLOBUS_COMMON_PKG_FOUND) if (GLOBUS_COMMON_LIBRARIES) message (STATUS "GLOBUS_COMMON libraries: ${GLOBUS_COMMON_LIBRARIES}") endif (GLOBUS_COMMON_LIBRARIES) if (GLOBUS_COMMON_INCLUDE_DIRS) message (STATUS "GLOBUS_COMMON include dir: ${GLOBUS_COMMON_INCLUDE_DIRS}") endif (GLOBUS_COMMON_INCLUDE_DIRS) # ----------------------------------------------------- # handle the QUIETLY and REQUIRED arguments and set GLOBUS_COMMON_FOUND to TRUE if # all listed variables are TRUE # ----------------------------------------------------- include(FindPackageHandleStandardArgs) find_package_handle_standard_args (GLOBUS_COMMON DEFAULT_MSG GLOBUS_COMMON_LIBRARIES GLOBUS_COMMON_INCLUDE_DIRS ) mark_as_advanced(GLOBUS_COMMON_INCLUDE_DIRS GLOBUS_COMMON_LIBRARIES) gfal2-v2.23.0/cmake/modules/FindGlobus_FTP_CLIENT.cmake000066400000000000000000000053501465240014500223200ustar00rootroot00000000000000# # This module detects if globus-ftp-client is installed and determines where the # include files and libraries are. # # This code sets the following variables: # # GLOBUS_FTP_CLIENT_LIBRARIES = full path to the globus-ftp-client libraries # GLOBUS_FTP_CLIENT_INCLUDE_DIR = include dir to be used when using the globus-ftp-client library # GLOBUS_FTP_CLIENT_FOUND = set to true if globus-ftp-client was found successfully # # GLOBUS_FTP_CLIENT_LOCATION # setting this enables search for globus-ftp-client libraries / headers in this location find_package (PkgConfig) pkg_check_modules(GLOBUS_FTP_CLIENT_PKG globus-ftp-client) if (GLOBUS_FTP_CLIENT_PKG_FOUND) set (GLOBUS_FTP_CLIENT_LIBRARIES ${GLOBUS_FTP_CLIENT_PKG_LIBRARIES}) set (GLOBUS_FTP_CLIENT_INCLUDE_DIRS ${GLOBUS_FTP_CLIENT_PKG_INCLUDE_DIRS}) set (GLOBUS_FTP_CLIENT_DEFINITIONS "${GLOBUS_FTP_CLIENT_PKG_CFLAGS}") else (GLOBUS_FTP_CLIENT_PKG_FOUND) set (CMAKE_FIND_FRAMEWORK NEVER) find_library(GLOBUS_FTP_CLIENT_LIBRARIES NAMES globus_ftp_client HINTS ${GLOBUS_FTP_CLIENT_LOCATION} ${CMAKE_INSTALL_PREFIX}/globus/*/${PLATFORM}/ ${CMAKE_INSTALL_PREFIX}/Grid/epel/*/${PLATFORM}/lib ${CMAKE_INSTALL_PREFIX}/Grid/epel/*/${PLATFORM}/lib64 ${CMAKE_INSTALL_PREFIX}/opt/globus-toolkit/libexec/lib ${GLOBUS_PREFIX}/libexec/lib DOC "The main globus-ftp-client library" ) find_path(GLOBUS_FTP_CLIENT_INCLUDE_DIRS NAMES globus_ftp_client.h HINTS ${GLOBUS_FTP_CLIENT_LOCATION}/include/* ${CMAKE_INSTALL_PREFIX}/globus/*/${PLATFORM}/include ${CMAKE_INSTALL_PREFIX}/Grid/epel/*/${PLATFORM}/include ${CMAKE_INSTALL_PREFIX}/opt/globus-toolkit/libexec/include ${GLOBUS_PREFIX}/libexec/include DOC "The globus-ftp-client include directory" ) set (GLOBUS_FTP_CLIENT_DEFINITIONS "") endif (GLOBUS_FTP_CLIENT_PKG_FOUND) if (GLOBUS_FTP_CLIENT_LIBRARIES) message (STATUS "GLOBUS_FTP_CLIENT libraries: ${GLOBUS_FTP_CLIENT_LIBRARIES}") endif (GLOBUS_FTP_CLIENT_LIBRARIES) if (GLOBUS_FTP_CLIENT_INCLUDE_DIRS) message (STATUS "GLOBUS_FTP_CLIENT include dir: ${GLOBUS_FTP_CLIENT_INCLUDE_DIRS}") endif (GLOBUS_FTP_CLIENT_INCLUDE_DIRS) # ----------------------------------------------------- # handle the QUIETLY and REQUIRED arguments and set GLOBUS_FTP_CLIENT_FOUND to TRUE if # all listed variables are TRUE # ----------------------------------------------------- include(FindPackageHandleStandardArgs) find_package_handle_standard_args (GLOBUS_FTP_CLIENT DEFAULT_MSG GLOBUS_FTP_CLIENT_LIBRARIES GLOBUS_FTP_CLIENT_INCLUDE_DIRS ) mark_as_advanced(GLOBUS_FTP_CLIENT_INCLUDE_DIRS GLOBUS_FTP_CLIENT_LIBRARIES) gfal2-v2.23.0/cmake/modules/FindGlobus_FTP_CONTROL.cmake000066400000000000000000000054231465240014500224630ustar00rootroot00000000000000# # This module detects if globus-ftp-control is installed and determines where the # include files and libraries are. # # This code sets the following variables: # # GLOBUS_FTP_CONTROL_LIBRARIES = full path to the globus-ftp-control libraries # GLOBUS_FTP_CONTROL_INCLUDE_DIR = include dir to be used when using the globus-ftp-control library # GLOBUS_FTP_CONTROL_FOUND = set to true if globus-ftp-control was found successfully # # GLOBUS_FTP_CONTROL_LOCATION # setting this enables search for globus-ftp-control libraries / headers in this location find_package (PkgConfig) pkg_check_modules(GLOBUS_FTP_CONTROL_PKG globus-ftp-control) if (GLOBUS_FTP_CONTROL_PKG_FOUND) set (GLOBUS_FTP_CONTROL_LIBRARIES ${GLOBUS_FTP_CONTROL_PKG_LIBRARIES}) set (GLOBUS_FTP_CONTROL_INCLUDE_DIRS ${GLOBUS_FTP_CONTROL_PKG_INCLUDE_DIRS}) set (GLOBUS_FTP_CONTROL_DEFINITIONS "${GLOBUS_FTP_CONTROL_PKG_CFLAGS}") else (GLOBUS_FTP_CONTROL_PKG_FOUND) set (CMAKE_FIND_FRAMEWORK NEVER) find_library(GLOBUS_FTP_CONTROL_LIBRARIES NAMES globus_ftp_control HINTS ${GLOBUS_FTP_CONTROL_LOCATION} ${CMAKE_INSTALL_PREFIX}/globus/*/${PLATFORM}/ ${CMAKE_INSTALL_PREFIX}/Grid/epel/*/${PLATFORM}/lib ${CMAKE_INSTALL_PREFIX}/Grid/epel/*/${PLATFORM}/lib64 ${CMAKE_INSTALL_PREFIX}/opt/globus-toolkit/libexec/lib ${GLOBUS_PREFIX}/libexec/lib DOC "The main globus-ftp-control library" ) find_path(GLOBUS_FTP_CONTROL_INCLUDE_DIRS NAMES globus_ftp_control.h HINTS ${GLOBUS_FTP_CONTROL_LOCATION}/include/* ${CMAKE_INSTALL_PREFIX}/globus/*/${PLATFORM}/include ${CMAKE_INSTALL_PREFIX}/Grid/epel/*/${PLATFORM}/include ${CMAKE_INSTALL_PREFIX}/opt/globus-toolkit/libexec/include ${GLOBUS_PREFIX}/libexec/include DOC "The globus-ftp-control include directory" ) set (GLOBUS_FTP_CONTROL_DEFINITIONS "") endif (GLOBUS_FTP_CONTROL_PKG_FOUND) if (GLOBUS_FTP_CONTROL_LIBRARIES) message (STATUS "GLOBUS_FTP_CONTROL libraries: ${GLOBUS_FTP_CONTROL_LIBRARIES}") endif (GLOBUS_FTP_CONTROL_LIBRARIES) if (GLOBUS_FTP_CONTROL_INCLUDE_DIRS) message (STATUS "GLOBUS_FTP_CONTROL include dir: ${GLOBUS_FTP_CONTROL_INCLUDE_DIRS}") endif (GLOBUS_FTP_CONTROL_INCLUDE_DIRS) # ----------------------------------------------------- # handle the QUIETLY and REQUIRED arguments and set GLOBUS_FTP_CONTROL_FOUND to TRUE if # all listed variables are TRUE # ----------------------------------------------------- include(FindPackageHandleStandardArgs) find_package_handle_standard_args (GLOBUS_FTP_CONTROL DEFAULT_MSG GLOBUS_FTP_CONTROL_LIBRARIES GLOBUS_FTP_CONTROL_INCLUDE_DIRS ) mark_as_advanced(GLOBUS_FTP_CONTROL_INCLUDE_DIRS GLOBUS_FTP_CONTROL_LIBRARIES) gfal2-v2.23.0/cmake/modules/FindGlobus_GASS_COPY.cmake000066400000000000000000000052141465240014500222170ustar00rootroot00000000000000# # This module detects if globus-gass-copy is installed and determines where the # include files and libraries are. # # This code sets the following variables: # # GLOBUS_GASS_COPY_LIBRARIES = full path to the globus-gass-copy libraries # GLOBUS_GASS_COPY_INCLUDE_DIR = include dir to be used when using the globus-gass-copy library # GLOBUS_GASS_COPY_FOUND = set to true if globus-gass-copy was found successfully # # GLOBUS_GASS_COPY_LOCATION # setting this enables search for globus-gass-copy libraries / headers in this location find_package (PkgConfig) pkg_check_modules(GLOBUS_GASS_COPY_PKG globus-gass-copy) if (GLOBUS_GASS_COPY_PKG_FOUND) set (GLOBUS_GASS_COPY_LIBRARIES ${GLOBUS_GASS_COPY_PKG_LIBRARIES}) set (GLOBUS_GASS_COPY_INCLUDE_DIRS ${GLOBUS_GASS_COPY_PKG_INCLUDE_DIRS}) set (GLOBUS_GASS_COPY_DEFINITIONS "${GLOBUS_GASS_COPY_PKG_CFLAGS}") else (GLOBUS_GASS_COPY_PKG_FOUND) find_library(GLOBUS_GASS_COPY_LIBRARIES NAMES globus_gass_copy HINTS ${GLOBUS_GASS_COPY_LOCATION} ${CMAKE_INSTALL_PREFIX}/globus/*/${PLATFORM}/ ${CMAKE_INSTALL_PREFIX}/Grid/epel/*/${PLATFORM}/lib ${CMAKE_INSTALL_PREFIX}/Grid/epel/*/${PLATFORM}/lib64 ${CMAKE_INSTALL_PREFIX}/opt/globus-toolkit/libexec/lib ${GLOBUS_PREFIX}/libexec/lib DOC "The main globus-gass-copy library" ) find_path(GLOBUS_GASS_COPY_INCLUDE_DIRS NAMES globus_gass_copy.h HINTS ${GLOBUS_GASS_COPY_LOCATION}/include/* ${CMAKE_INSTALL_PREFIX}/globus/*/${PLATFORM}/ ${CMAKE_INSTALL_PREFIX}/Grid/epel/*/${PLATFORM}/ ${CMAKE_INSTALL_PREFIX}/opt/globus-toolkit/libexec/include ${GLOBUS_PREFIX}/libexec/include DOC "The globus-gass-copy include directory" ) set (GLOBUS_GASS_COPY_DEFINITIONS "") endif (GLOBUS_GASS_COPY_PKG_FOUND) if (GLOBUS_GASS_COPY_LIBRARIES) message (STATUS "GLOBUS_GSSAPI_GSI libraries: ${GLOBUS_GASS_COPY_LIBRARIES}") endif (GLOBUS_GASS_COPY_LIBRARIES) if (GLOBUS_GASS_COPY_INCLUDE_DIRS) message (STATUS "GLOBUS_GSSAPI_GSI include dir: ${GLOBUS_GASS_COPY_INCLUDE_DIRS}") endif (GLOBUS_GASS_COPY_INCLUDE_DIRS) # ----------------------------------------------------- # handle the QUIETLY and REQUIRED arguments and set GLOBUS_GASS_COPY_FOUND to TRUE if # all listed variables are TRUE # ----------------------------------------------------- include(FindPackageHandleStandardArgs) find_package_handle_standard_args (GLOBUS_GSSAPI_GSI DEFAULT_MSG GLOBUS_GASS_COPY_LIBRARIES GLOBUS_GASS_COPY_INCLUDE_DIRS ) mark_as_advanced(GLOBUS_GASS_COPY_INCLUDE_DIRS GLOBUS_GASS_COPY_LIBRARIES) gfal2-v2.23.0/cmake/modules/FindGlobus_GSI_CERT_UTILS.cmake000066400000000000000000000056241465240014500230540ustar00rootroot00000000000000# # This module detects if globus-gsi-cert-utils is installed and determines where the # include files and libraries are. # # This code sets the following variables: # # GLOBUS_GSI_CERT_UTILS_LIBRARIES = full path to the globus-gsi-cert-utils libraries # GLOBUS_GSI_CERT_UTILS_INCLUDE_DIR = include dir to be used when using the globus-gsi-cert-utils library # GLOBUS_GSI_CERT_UTILS_FOUND = set to true if globus-gsi-cert-utils was found successfully # # GLOBUS_GSI_CERT_UTILS_LOCATION # setting this enables search for globus-gsi-cert-utils libraries / headers in this location find_package (PkgConfig) pkg_check_modules(GLOBUS_GSI_CERT_UTILS_PKG globus-gsi-cert-utils) if (GLOBUS_GSI_CERT_UTILS_PKG_FOUND) set (GLOBUS_GSI_CERT_UTILS_LIBRARIES ${GLOBUS_GSI_CERT_UTILS_PKG_LIBRARIES}) set (GLOBUS_GSI_CERT_UTILS_INCLUDE_DIRS ${GLOBUS_GSI_CERT_UTILS_PKG_INCLUDE_DIRS}) set (GLOBUS_GSI_CERT_UTILS_DEFINITIONS "${GLOBUS_GSI_CERT_UTILS_PKG_CFLAGS}") else (GLOBUS_GSI_CERT_UTILS_PKG_FOUND) set (CMAKE_FIND_FRAMEWORK NEVER) find_library(GLOBUS_GSI_CERT_UTILS_LIBRARIES NAMES globus_gsi_cert_utils HINTS ${GLOBUS_GSI_CERT_UTILS_LOCATION} ${CMAKE_INSTALL_PREFIX}/globus/*/${PLATFORM}/ ${CMAKE_INSTALL_PREFIX}/Grid/epel/*/${PLATFORM}/lib ${CMAKE_INSTALL_PREFIX}/Grid/epel/*/${PLATFORM}/lib64 ${CMAKE_INSTALL_PREFIX}/opt/globus-toolkit/libexec/lib ${GLOBUS_PREFIX}/libexec/lib DOC "The main globus-gsi-cert-utils library" ) find_path(GLOBUS_GSI_CERT_UTILS_INCLUDE_DIRS NAMES globus_gsi_cert_utils.h HINTS ${GLOBUS_GSI_CERT_UTILS_LOCATION}/include/* ${CMAKE_INSTALL_PREFIX}/globus/*/${PLATFORM}/include ${CMAKE_INSTALL_PREFIX}/Grid/epel/*/${PLATFORM}/include ${CMAKE_INSTALL_PREFIX}/opt/globus-toolkit/libexec/include ${GLOBUS_PREFIX}/libexec/include DOC "The globus-gsi-cert-utils include directory" ) set (GLOBUS_GSI_CERT_UTILS_DEFINITIONS "") endif (GLOBUS_GSI_CERT_UTILS_PKG_FOUND) if (GLOBUS_GSI_CERT_UTILS_LIBRARIES) message (STATUS "GLOBUS_GSI_CERT_UTILS libraries: ${GLOBUS_GSI_CERT_UTILS_LIBRARIES}") endif (GLOBUS_GSI_CERT_UTILS_LIBRARIES) if (GLOBUS_GSI_CERT_UTILS_INCLUDE_DIRS) message (STATUS "GLOBUS_GSI_CERT_UTILS include dir: ${GLOBUS_GSI_CERT_UTILS_INCLUDE_DIRS}") endif (GLOBUS_GSI_CERT_UTILS_INCLUDE_DIRS) # ----------------------------------------------------- # handle the QUIETLY and REQUIRED arguments and set GLOBUS_GSI_CERT_UTILS_FOUND to TRUE if # all listed variables are TRUE # ----------------------------------------------------- include(FindPackageHandleStandardArgs) find_package_handle_standard_args (GLOBUS_GSI_CERT_UTILS DEFAULT_MSG GLOBUS_GSI_CERT_UTILS_LIBRARIES GLOBUS_GSI_CERT_UTILS_INCLUDE_DIRS ) mark_as_advanced(GLOBUS_GSI_CERT_UTILS_INCLUDE_DIRS GLOBUS_GSI_CERT_UTILS_LIBRARIES) gfal2-v2.23.0/cmake/modules/FindGlobus_GSI_CREDENTIAL.cmake000066400000000000000000000056241465240014500227510ustar00rootroot00000000000000# # This module detects if globus-gsi-credential is installed and determines where the # include files and libraries are. # # This code sets the following variables: # # GLOBUS_GSI_CREDENTIAL_LIBRARIES = full path to the globus-gsi-credential libraries # GLOBUS_GSI_CREDENTIAL_INCLUDE_DIR = include dir to be used when using the globus-gsi-credential library # GLOBUS_GSI_CREDENTIAL_FOUND = set to true if globus-gsi-credential was found successfully # # GLOBUS_GSI_CREDENTIAL_LOCATION # setting this enables search for globus-gsi-credential libraries / headers in this location find_package (PkgConfig) pkg_check_modules(GLOBUS_GSI_CREDENTIAL_PKG globus-gsi-credential) if (GLOBUS_GSI_CREDENTIAL_PKG_FOUND) set (GLOBUS_GSI_CREDENTIAL_LIBRARIES ${GLOBUS_GSI_CREDENTIAL_PKG_LIBRARIES}) set (GLOBUS_GSI_CREDENTIAL_INCLUDE_DIRS ${GLOBUS_GSI_CREDENTIAL_PKG_INCLUDE_DIRS}) set (GLOBUS_GSI_CREDENTIAL_DEFINITIONS "${GLOBUS_GSI_CREDENTIAL_PKG_CFLAGS}") else (GLOBUS_GSI_CREDENTIAL_PKG_FOUND) set (CMAKE_FIND_FRAMEWORK NEVER) find_library(GLOBUS_GSI_CREDENTIAL_LIBRARIES NAMES globus_gsi_credential HINTS ${GLOBUS_GSI_CREDENTIAL_LOCATION} ${CMAKE_INSTALL_PREFIX}/globus/*/${PLATFORM}/ ${CMAKE_INSTALL_PREFIX}/Grid/epel/*/${PLATFORM}/lib ${CMAKE_INSTALL_PREFIX}/Grid/epel/*/${PLATFORM}/lib64 ${CMAKE_INSTALL_PREFIX}/opt/globus-toolkit/libexec/lib ${GLOBUS_PREFIX}/libexec/lib DOC "The main globus-gsi-credential library" ) find_path(GLOBUS_GSI_CREDENTIAL_INCLUDE_DIRS NAMES globus_gsi_credential.h HINTS ${GLOBUS_GSI_CREDENTIAL_LOCATION}/include/* ${CMAKE_INSTALL_PREFIX}/globus/*/${PLATFORM}/include ${CMAKE_INSTALL_PREFIX}/Grid/epel/*/${PLATFORM}/include ${CMAKE_INSTALL_PREFIX}/opt/globus-toolkit/libexec/include ${GLOBUS_PREFIX}/libexec/include DOC "The globus-gsi-credential include directory" ) set (GLOBUS_GSI_CREDENTIAL_DEFINITIONS "") endif (GLOBUS_GSI_CREDENTIAL_PKG_FOUND) if (GLOBUS_GSI_CREDENTIAL_LIBRARIES) message (STATUS "GLOBUS_GSI_CREDENTIAL libraries: ${GLOBUS_GSI_CREDENTIAL_LIBRARIES}") endif (GLOBUS_GSI_CREDENTIAL_LIBRARIES) if (GLOBUS_GSI_CREDENTIAL_INCLUDE_DIRS) message (STATUS "GLOBUS_GSI_CREDENTIAL include dir: ${GLOBUS_GSI_CREDENTIAL_INCLUDE_DIRS}") endif (GLOBUS_GSI_CREDENTIAL_INCLUDE_DIRS) # ----------------------------------------------------- # handle the QUIETLY and REQUIRED arguments and set GLOBUS_GSI_CREDENTIAL_FOUND to TRUE if # all listed variables are TRUE # ----------------------------------------------------- include(FindPackageHandleStandardArgs) find_package_handle_standard_args (GLOBUS_GSI_CREDENTIAL DEFAULT_MSG GLOBUS_GSI_CREDENTIAL_LIBRARIES GLOBUS_GSI_CREDENTIAL_INCLUDE_DIRS ) mark_as_advanced(GLOBUS_GSI_CREDENTIAL_INCLUDE_DIRS GLOBUS_GSI_CREDENTIAL_LIBRARIES) gfal2-v2.23.0/cmake/modules/FindGlobus_GSSAPI_GSI.cmake000066400000000000000000000053351465240014500223240ustar00rootroot00000000000000# # This module detects if globus-gssapi-gsi is installed and determines where the # include files and libraries are. # # This code sets the following variables: # # GLOBUS_GSSAPI_GSI_LIBRARIES = full path to the globus-gssapi-gsi libraries # GLOBUS_GSSAPI_GSI_INCLUDE_DIR = include dir to be used when using the globus-gssapi-gsi library # GLOBUS_GSSAPI_GSI_FOUND = set to true if globus-gssapi-gsi was found successfully # # GLOBUS_GSSAPI_GSI_LOCATION # setting this enables search for globus-gssapi-gsi libraries / headers in this location find_package (PkgConfig) pkg_check_modules(GLOBUS_GSSAPI_GSI_PKG globus-gssapi-gsi) if (GLOBUS_GSSAPI_GSI_PKG_FOUND) set (GLOBUS_GSSAPI_GSI_LIBRARIES ${GLOBUS_GSSAPI_GSI_PKG_LIBRARIES}) set (GLOBUS_GSSAPI_GSI_INCLUDE_DIRS ${GLOBUS_GSSAPI_GSI_PKG_INCLUDE_DIRS}) set (GLOBUS_GSSAPI_GSI_DEFINITIONS "${GLOBUS_GSSAPI_GSI_PKG_CFLAGS}") else (GLOBUS_GSSAPI_GSI_PKG_FOUND) set (CMAKE_FIND_FRAMEWORK NEVER) find_library(GLOBUS_GSSAPI_GSI_LIBRARIES NAMES globus_gssapi_gsi HINTS ${GLOBUS_GSSAPI_GSI_LOCATION} ${CMAKE_INSTALL_PREFIX}/globus/*/${PLATFORM}/ ${CMAKE_INSTALL_PREFIX}/Grid/epel/*/${PLATFORM}/lib ${CMAKE_INSTALL_PREFIX}/Grid/epel/*/${PLATFORM}/lib64 ${CMAKE_INSTALL_PREFIX}/opt/globus-toolkit/libexec/lib ${GLOBUS_PREFIX}/libexec/lib DOC "The main globus-gssapi-gsi library" ) find_path(GLOBUS_GSSAPI_GSI_INCLUDE_DIRS NAMES gssapi.h HINTS ${GLOBUS_GSSAPI_GSI_LOCATION}/include/* ${CMAKE_INSTALL_PREFIX}/globus/*/${PLATFORM}/include ${CMAKE_INSTALL_PREFIX}/Grid/epel/*/${PLATFORM}/include ${CMAKE_INSTALL_PREFIX}/opt/globus-toolkit/libexec/include ${GLOBUS_PREFIX}/libexec/include DOC "The globus-gssapi-gsi include directory" ) set (GLOBUS_GSSAPI_GSI_DEFINITIONS "") endif (GLOBUS_GSSAPI_GSI_PKG_FOUND) if (GLOBUS_GSSAPI_GSI_LIBRARIES) message (STATUS "GLOBUS_GSSAPI_GSI libraries: ${GLOBUS_GSSAPI_GSI_LIBRARIES}") endif (GLOBUS_GSSAPI_GSI_LIBRARIES) if (GLOBUS_GSSAPI_GSI_INCLUDE_DIRS) message (STATUS "GLOBUS_GSSAPI_GSI include dir: ${GLOBUS_GSSAPI_GSI_INCLUDE_DIRS}") endif (GLOBUS_GSSAPI_GSI_INCLUDE_DIRS) # ----------------------------------------------------- # handle the QUIETLY and REQUIRED arguments and set GLOBUS_GSSAPI_GSI_FOUND to TRUE if # all listed variables are TRUE # ----------------------------------------------------- include(FindPackageHandleStandardArgs) find_package_handle_standard_args (GLOBUS_GSSAPI_GSI DEFAULT_MSG GLOBUS_GSSAPI_GSI_LIBRARIES GLOBUS_GSSAPI_GSI_INCLUDE_DIRS ) mark_as_advanced(GLOBUS_GSSAPI_GSI_INCLUDE_DIRS GLOBUS_GSSAPI_GSI_LIBRARIES) gfal2-v2.23.0/cmake/modules/FindGlobus_GSS_ASSIST.cmake000066400000000000000000000052761465240014500223620ustar00rootroot00000000000000# # This module detects if globus-gssapi-gsi is installed and determines where the # include files and libraries are. # # This code sets the following variables: # # GLOBUS_GSS_ASSIST_LIBRARIES = full path to the globus-gssapi-gsi libraries # GLOBUS_GSS_ASSIST_INCLUDE_DIR = include dir to be used when using the globus-gssapi-gsi library # GLOBUS_GSS_ASSIST_FOUND = set to true if globus-gssapi-gsi was found successfully # # GLOBUS_GSS_ASSIST_LOCATION # setting this enables search for globus-gssapi-gsi libraries / headers in this location find_package (PkgConfig) pkg_check_modules(GLOBUS_GSS_ASSIST_PKG globus-gss-assist) if (GLOBUS_GSS_ASSIST_PKG_FOUND) set (GLOBUS_GSS_ASSIST_LIBRARIES ${GLOBUS_GSS_ASSIST_PKG_LIBRARIES}) set (GLOBUS_GSS_ASSIST_INCLUDE_DIRS ${GLOBUS_GSS_ASSIST_PKG_INCLUDE_DIRS}) set (GLOBUS_GSS_ASSIST_DEFINITIONS "${GLOBUS_GSS_ASSIST_PKG_CFLAGS}") else (GLOBUS_GSS_ASSIST_PKG_FOUND) find_library(GLOBUS_GSS_ASSIST_LIBRARIES NAMES globus_gss_assist HINTS ${GLOBUS_GSS_ASSIST_LOCATION} ${CMAKE_INSTALL_PREFIX}/globus/*/${PLATFORM}/ ${CMAKE_INSTALL_PREFIX}/Grid/epel/*/${PLATFORM}/lib ${CMAKE_INSTALL_PREFIX}/Grid/epel/*/${PLATFORM}/lib64 ${CMAKE_INSTALL_PREFIX}/opt/globus-toolkit/libexec/lib ${GLOBUS_PREFIX}/libexec/lib DOC "The main globus-gssapi-gsi library" ) find_path(GLOBUS_GSS_ASSIST_INCLUDE_DIRS NAMES globus_gss_assist.h HINTS ${GLOBUS_GSS_ASSIST_LOCATION}/include/* ${CMAKE_INSTALL_PREFIX}/globus/*/${PLATFORM}/include ${CMAKE_INSTALL_PREFIX}/Grid/epel/*/${PLATFORM}/include ${CMAKE_INSTALL_PREFIX}/opt/globus-toolkit/libexec/include ${GLOBUS_PREFIX}/libexec/include DOC "The globus-gss-assist include directory" ) set (GLOBUS_GSS_ASSIST_DEFINITIONS "") endif (GLOBUS_GSS_ASSIST_PKG_FOUND) if (GLOBUS_GSS_ASSIST_LIBRARIES) message (STATUS "GLOBUS GSS ASSIST libraries: ${GLOBUS_GSS_ASSIST_LIBRARIES}") endif (GLOBUS_GSS_ASSIST_LIBRARIES) if (GLOBUS_GSS_ASSIST_INCLUDE_DIRS) message (STATUS "GLOBUS GSS ASSIST include dir: ${GLOBUS_GSS_ASSIST_INCLUDE_DIRS}") endif (GLOBUS_GSS_ASSIST_INCLUDE_DIRS) # ----------------------------------------------------- # handle the QUIETLY and REQUIRED arguments and set GLOBUS_GSS_ASSIST_FOUND to TRUE if # all listed variables are TRUE # ----------------------------------------------------- include(FindPackageHandleStandardArgs) find_package_handle_standard_args (GLOBUS_ASSIST DEFAULT_MSG GLOBUS_GSS_ASSIST_LIBRARIES GLOBUS_GSS_ASSIST_INCLUDE_DIRS ) mark_as_advanced(GLOBUS_GSS_ASSIST_INCLUDE_DIRS GLOBUS_GSS_ASSIST_LIBRARIES) gfal2-v2.23.0/cmake/modules/FindGlobus_OPENSSL.cmake000066400000000000000000000051561465240014500217600ustar00rootroot00000000000000# # This module detects if globus-openssl is installed and determines where the # include files and libraries are. # # This code sets the following variables: # # GLOBUS_OPENSSL_LIBRARIES = full path to the globus-openssl libraries # GLOBUS_OPENSSL_INCLUDE_DIR = include dir to be used when using the globus-openssl library # GLOBUS_OPENSSL_FOUND = set to true if globus-openssl was found successfully # # GLOBUS_OPENSSL_LOCATION # setting this enables search for globus-openssl libraries / headers in this location find_package (PkgConfig) pkg_check_modules(GLOBUS_OPENSSL_PKG globus-openssl-module) if (GLOBUS_OPENSSL_PKG_FOUND) set (GLOBUS_OPENSSL_LIBRARIES ${GLOBUS_OPENSSL_PKG_LIBRARIES}) set (GLOBUS_OPENSSL_INCLUDE_DIRS ${GLOBUS_OPENSSL_PKG_INCLUDE_DIRS}) set (GLOBUS_OPENSSL_DEFINITIONS "${GLOBUS_OPENSSL_PKG_CFLAGS}") else (GLOBUS_OPENSSL_PKG_FOUND) set (CMAKE_FIND_FRAMEWORK NEVER) find_library(GLOBUS_OPENSSL_LIBRARIES NAMES globus_openssl HINTS ${GLOBUS_OPENSSL_LOCATION} ${CMAKE_INSTALL_PREFIX}/globus/*/${PLATFORM}/ ${CMAKE_INSTALL_PREFIX}/Grid/epel/*/${PLATFORM}/lib ${CMAKE_INSTALL_PREFIX}/Grid/epel/*/${PLATFORM}/lib64 ${CMAKE_INSTALL_PREFIX}/opt/globus-toolkit/libexec/lib ${GLOBUS_PREFIX}/libexec/lib DOC "The main globus-openssl library" ) find_path(GLOBUS_OPENSSL_INCLUDE_DIRS NAMES globus_openssl.h HINTS ${GLOBUS_OPENSSL_LOCATION}/include/* ${CMAKE_INSTALL_PREFIX}/globus/*/${PLATFORM}/include ${CMAKE_INSTALL_PREFIX}/Grid/epel/*/${PLATFORM}/include ${CMAKE_INSTALL_PREFIX}/opt/globus-toolkit/libexec/include ${GLOBUS_PREFIX}/libexec/include DOC "The globus-openssl include directory" ) set (GLOBUS_OPENSSL_DEFINITIONS "") endif (GLOBUS_OPENSSL_PKG_FOUND) if (GLOBUS_OPENSSL_LIBRARIES) message (STATUS "GLOBUS_OPENSSL libraries: ${GLOBUS_OPENSSL_LIBRARIES}") endif (GLOBUS_OPENSSL_LIBRARIES) if (GLOBUS_OPENSSL_INCLUDE_DIRS) message (STATUS "GLOBUS_OPENSSL include dir: ${GLOBUS_OPENSSL_INCLUDE_DIRS}") endif (GLOBUS_OPENSSL_INCLUDE_DIRS) # ----------------------------------------------------- # handle the QUIETLY and REQUIRED arguments and set GLOBUS_OPENSSL_FOUND to TRUE if # all listed variables are TRUE # ----------------------------------------------------- include(FindPackageHandleStandardArgs) find_package_handle_standard_args (GLOBUS_OPENSSL DEFAULT_MSG GLOBUS_OPENSSL_LIBRARIES GLOBUS_OPENSSL_INCLUDE_DIRS ) mark_as_advanced(GLOBUS_OPENSSL_INCLUDE_DIRS GLOBUS_OPENSSL_LIBRARIES) gfal2-v2.23.0/cmake/modules/FindJSONC.cmake000066400000000000000000000013101465240014500201620ustar00rootroot00000000000000# Find json-c find_package (PkgConfig) pkg_check_modules(JSONC_PKG json-c) if (JSONC_PKG_FOUND) set (JSONC_LIBRARIES ${JSONC_PKG_LIBRARIES}) set (JSONC_INCLUDE_DIRS ${JSONC_PKG_INCLUDE_DIRS}) else (JSONC_PKG) find_library(JSONC_LIBRARIES NAMES json-c json HINTS /lib /lib64 /usr/lib /usr/lib64 DOC "json-c library" ) find_path(JSONC_INCLUDE_DIRS NAMES json.h HINTS /usr/include/json /usr/include/json-c DOC "json-c headers" ) endif (JSONC_PKG_FOUND) include(FindPackageHandleStandardArgs) find_package_handle_standard_args(json-c DEFAULT_MSG JSONC_LIBRARIES JSONC_INCLUDE_DIRS ) mark_as_advanced(JSONC_INCLUDE_DIRS JSONC_LIBRARIES) gfal2-v2.23.0/cmake/modules/FindLFC.cmake000066400000000000000000000040461465240014500177230ustar00rootroot00000000000000# # This module detects if LFC is installed and determines where the # include files and libraries are. # # This code sets the following variables: # # LFC_LIBRARIES = full path to the LFC libraries # LFC_INCLUDE_DIR = include dir to be used when using the LFC library # LFC_FOUND = set to true if LFC was found successfully # # LFC_LOCATION # setting this enables search for LFC libraries / headers in this location # ----------------------------------------------------- # LFC Libraries # ----------------------------------------------------- find_library(LFC_LIBRARIES NAMES lfc lcgdm HINTS ${LFC_LOCATION} ${STAGE_DIR} ${CMAKE_INSTALL_PREFIX}/lfc/*/${PLATFORM}/ ${CMAKE_INSTALL_PREFIX}/Grid/lfc/*/${PLATFORM}/ DOC "The main LFC library" ) # ----------------------------------------------------- # LFC Include Directories # ----------------------------------------------------- find_path(LFC_INCLUDE_DIR NAMES lfc/lfc_api.h HINTS ${LFC_LOCATION} ${STAGE_DIR} ${CMAKE_INSTALL_PREFIX}/lfc/*/${PLATFORM}/ ${CMAKE_INSTALL_PREFIX}/Grid/lfc/*/${PLATFORM}/ DOC "The LFC include directory" ) if(LFC_INCLUDE_DIR) message(STATUS "LFC includes found in ${LFC_INCLUDE_DIR}") endif() # ----------------------------------------------------- # LCGDM Include Directories # ----------------------------------------------------- find_path(LCGDM_INCLUDE_DIR NAMES Cinit.h HINTS ${LCGDM_LOCATION} ${LCGDM_LOCATION}/include ${LCGDM_LOCATION}/include/lcgdm /usr/include/lcgdm ${LCGDM_LOCATION}/include/dpm /usr/include/dpm /usr/include/lfc DOC "The LCGDM include directory" ) # ----------------------------------------------------- # handle the QUIETLY and REQUIRED arguments and set LFC_FOUND to TRUE if # all listed variables are TRUE # ----------------------------------------------------- include(FindPackageHandleStandardArgs) find_package_handle_standard_args(LFC DEFAULT_MSG LFC_LIBRARIES LFC_INCLUDE_DIR) mark_as_advanced(LFC_INCLUDE_DIR LFC_LIBRARIES) gfal2-v2.23.0/cmake/modules/FindLIBSSH2.cmake000066400000000000000000000036771465240014500203760ustar00rootroot00000000000000# # This module detects if libssh2 is installed and determines where the # include files and libraries are. # # This code sets the following variables: # # LIBSSH2_LIBRARIES = full path to the libssh2 libraries # LIBSSH2_INCLUDE_DIR = include dir to be used when using the libssh2 library # LIBSSH2_FOUND = set to true if libssh2 was found successfully # # LIBSSH2_LOCATION # setting this enables search for libssh2 libraries / headers in this location find_package (PkgConfig) pkg_check_modules(LIBSSH2_PKG libssh2) if (LIBSSH2_PKG_FOUND) set (LIBSSH2_LIBRARIES ${LIBSSH2_PKG_LIBRARIES}) set (LIBSSH2_INCLUDE_DIRS ${LIBSSH2_PKG_INCLUDE_DIRS}) set (LIBSSH2_DEFINITIONS "${LIBSSH2_PKG_CFLAGS}") else (LIBSSH2_PKG_FOUND) set (CMAKE_FIND_FRAMEWORK NEVER) find_library(LIBSSH2_LIBRARIES NAMES ssh2 HINTS ${LIBSSH2_LOCATION} ${CMAKE_INSTALL_PREFIX}/Grid/epel/*/${PLATFORM}/lib ${CMAKE_INSTALL_PREFIX}/Grid/epel/*/${PLATFORM}/lib64 DOC "The main libssh2 library" ) find_path(LIBSSH2_INCLUDE_DIRS NAMES libssh2_sftp.h HINTS ${LIBSSH2_LOCATION}/include/* ${CMAKE_INSTALL_PREFIX}/Grid/epel/*/${PLATFORM}/include DOC "The libssh2 include directory" ) set (LIBSSH2_DEFINITIONS "") endif (LIBSSH2_PKG_FOUND) if (LIBSSH2_LIBRARIES) message (STATUS "LIBSSH2 libraries: ${LIBSSH2_LIBRARIES}") endif (LIBSSH2_LIBRARIES) if (LIBSSH2_INCLUDE_DIRS) message (STATUS "LIBSSH2 include dir: ${LIBSSH2_INCLUDE_DIRS}") endif (LIBSSH2_INCLUDE_DIRS) # ----------------------------------------------------- # handle the QUIETLY and REQUIRED arguments and set LIBSSH2_FOUND to TRUE if # all listed variables are TRUE # ----------------------------------------------------- include(FindPackageHandleStandardArgs) find_package_handle_standard_args (SSH2 DEFAULT_MSG LIBSSH2_LIBRARIES LIBSSH2_INCLUDE_DIRS ) mark_as_advanced(LIBSSH2_INCLUDE_DIRS LIBSSH2_LIBRARIES) gfal2-v2.23.0/cmake/modules/FindPugiXML.cmake000066400000000000000000000031251465240014500206010ustar00rootroot00000000000000# # This module detects if pugixml is installed and determines where the # include files and libraries are. # # This code sets the following variables: # # PUGIXML_LIBRARIES = full path to the pugixml libraries # PUGIXML_INCLUDE_DIR = include dir to be used when using the pugixml library # PUGIXML_FOUND = set to true if pugixml was found successfully # # PUGIXML_LOCATION # setting this enables search for pugixml libraries / headers in this location # ----------------------------------------------------- # PUGIXML Libraries # ----------------------------------------------------- find_library(PUGIXML_LIBRARIES NAMES pugixml HINTS ${PUGIXML_LOCATION}/lib ${PUGIXML_LOCATION}/lib64 ${PUGIXML_LOCATION}/lib32 DOC "The main pugixml library" ) # ----------------------------------------------------- # PUGIXML Include Directories # ----------------------------------------------------- find_path(PUGIXML_INCLUDE_DIR NAMES pugixml.hpp HINTS ${PUGIXML_LOCATION} ${PUGIXML_LOCATION}/include ${PUGIXML_LOCATION}/include/* /usr/include DOC "The pugixml include directory" ) if(PUGIXML_INCLUDE_DIR) message(STATUS "pugixml includes found in ${PUGIXML_INCLUDE_DIR}") endif() # ----------------------------------------------------- # handle the QUIETLY and REQUIRED arguments and set PUGIXML_FOUND to TRUE if # all listed variables are TRUE # ----------------------------------------------------- include(FindPackageHandleStandardArgs) find_package_handle_standard_args(pugixml DEFAULT_MSG PUGIXML_LIBRARIES PUGIXML_INCLUDE_DIR) mark_as_advanced(PUGIXML_INCLUDE_DIR PUGIXML_LIBRARIES) gfal2-v2.23.0/cmake/modules/FindPythonEasy.cmake000066400000000000000000000120451465240014500214200ustar00rootroot00000000000000# - Find python libraries # This module find the current version of python on your installation in a easy way # # PYTHON_LIBRARIES = path to the python library # PYTHON_LIBRARIES_${VERSION} = path to the python library for the distribution version # PYTHON_SITE_PACKAGES_${_VERSION} = path to the python modules dir # PYTHON_LIBRARIES = path to the python modules dir for the distribution version # PYTHON_INCLUDE_PATH = path to where Python.h is found # PYTHON_INCLUDE_PATH_${VERSION} = path to where Python.h for the distribution version # PYTHON_EXECUTABLE = python interpreter for the distribution version # PYTHON_EXECUTABLE_${VERSION} = available python version # PYTHON_AVAILABLE_VERSIONS = list all the version available on the system # -- LIST(APPEND L_PYTHON_VERSIONS "1.5" "1.6" "2.0" "2.1" "2.2" "2.4" "2.5" "2.6" "2.7" "2.8" "3" "3.0" "3.1" "3.2" "3.3" "3.4" "3.5" ) INCLUDE(FindPackageHandleStandardArgs) # determine the std version # main version executable FIND_PROGRAM(PYTHON_EXECUTABLE NAMES python HINTS ${ALT_PYTHON_LOCATION}/bin PATHS [HKEY_LOCAL_MACHINE\\SOFTWARE\\Python\\PythonCore\\3.4\\InstallPath] [HKEY_LOCAL_MACHINE\\SOFTWARE\\Python\\PythonCore\\3.3\\InstallPath] [HKEY_LOCAL_MACHINE\\SOFTWARE\\Python\\PythonCore\\3.2\\InstallPath] [HKEY_LOCAL_MACHINE\\SOFTWARE\\Python\\PythonCore\\3.1\\InstallPath] [HKEY_LOCAL_MACHINE\\SOFTWARE\\Python\\PythonCore\\3.0\\InstallPath] [HKEY_LOCAL_MACHINE\\SOFTWARE\\Python\\PythonCore\\2.8\\InstallPath] [HKEY_LOCAL_MACHINE\\SOFTWARE\\Python\\PythonCore\\2.7\\InstallPath] [HKEY_LOCAL_MACHINE\\SOFTWARE\\Python\\PythonCore\\2.6\\InstallPath] [HKEY_LOCAL_MACHINE\\SOFTWARE\\Python\\PythonCore\\2.5\\InstallPath] [HKEY_LOCAL_MACHINE\\SOFTWARE\\Python\\PythonCore\\2.4\\InstallPath] [HKEY_LOCAL_MACHINE\\SOFTWARE\\Python\\PythonCore\\2.3\\InstallPath] [HKEY_LOCAL_MACHINE\\SOFTWARE\\Python\\PythonCore\\2.2\\InstallPath] [HKEY_LOCAL_MACHINE\\SOFTWARE\\Python\\PythonCore\\2.1\\InstallPath] [HKEY_LOCAL_MACHINE\\SOFTWARE\\Python\\PythonCore\\2.0\\InstallPath] [HKEY_LOCAL_MACHINE\\SOFTWARE\\Python\\PythonCore\\1.6\\InstallPath] [HKEY_LOCAL_MACHINE\\SOFTWARE\\Python\\PythonCore\\1.5\\InstallPath] ) EXECUTE_PROCESS( COMMAND ${PYTHON_EXECUTABLE} -c "import sys; print('%s.%s' % sys.version_info[:2])" OUTPUT_VARIABLE PYTHON_CURRENT_VERSION OUTPUT_STRIP_TRAILING_WHITESPACE) ## tests for all versions of the packages FOREACH(_VERSION ${L_PYTHON_VERSIONS}) STRING(REPLACE "." "" _VERSION_NO_DOTS ${_VERSION}) FIND_PROGRAM(PYTHON_EXECUTABLE_${_VERSION} NAMES python${_VERSION} HINTS ${ALT_PYTHON_LOCATION}/bin/ ) IF(PYTHON_EXECUTABLE_${_VERSION}) LIST(APPEND PYTHON_AVAILABLE_VERSIONS ${_VERSION}) EXECUTE_PROCESS(COMMAND ${PYTHON_EXECUTABLE_${_VERSION}} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib(True))" OUTPUT_VARIABLE PYTHON_SITE_PACKAGES_${_VERSION} OUTPUT_STRIP_TRAILING_WHITESPACE) # find libs EXECUTE_PROCESS(COMMAND python${_VERSION}-config --libs OUTPUT_VARIABLE PYTHON_LIBRARY_${_VERSION} OUTPUT_STRIP_TRAILING_WHITESPACE ) SET(PYTHON_LIBRARIES_${_VERSION} ${PYTHON_LIBRARY_${_VERSION}}) # find include EXECUTE_PROCESS(COMMAND ${PYTHON_EXECUTABLE_${_VERSION}} -c "from distutils import sysconfig; print(sysconfig.get_python_inc())" OUTPUT_VARIABLE PYTHON_INCLUDE_PATH_${_VERSION} OUTPUT_STRIP_TRAILING_WHITESPACE ) FIND_PACKAGE_HANDLE_STANDARD_ARGS(Python${_VERSION} DEFAULT_MSG PYTHON_EXECUTABLE_${_VERSION}) MARK_AS_ADVANCED(PYTHON_EXECUTABLE_${_VERSION}) MARK_AS_ADVANCED(PYTHON_SITE_PACKAGES_${_VERSION}) MARK_AS_ADVANCED(PYTHON_LIBRARIES_${_VERSION}) MARK_AS_ADVANCED(PYTHON_INCLUDE_PATH_${_VERSION}) ENDIF(PYTHON_EXECUTABLE_${_VERSION}) ENDFOREACH(_VERSION ${L_PYTHON_VERSIONS}) SET(PYTHON_SITE_PACKAGES ${PYTHON_SITE_PACKAGES_${PYTHON_CURRENT_VERSION}} CACHE PATH "path to the python modules dir for the distribution version") SET(PYTHON_LIBRARIES ${PYTHON_LIBRARIES_${PYTHON_CURRENT_VERSION}} CACHE PATH "path to the python modules dir for the distribution version") SET(PYTHON_INCLUDE_PATH ${PYTHON_INCLUDE_PATH_${PYTHON_CURRENT_VERSION}} CACHE PATH "path to the python include dir for the distribution version") FIND_PACKAGE_HANDLE_STANDARD_ARGS(PythonCurrentVersion DEFAULT_MSG PYTHON_CURRENT_VERSION ) FIND_PACKAGE_HANDLE_STANDARD_ARGS(PythonCurrentInclude DEFAULT_MSG PYTHON_INCLUDE_PATH ) FIND_PACKAGE_HANDLE_STANDARD_ARGS(PythonCurrentLibs DEFAULT_MSG PYTHON_LIBRARIES ) FIND_PACKAGE_HANDLE_STANDARD_ARGS(PythonCurrentModsDir DEFAULT_MSG PYTHON_SITE_PACKAGES ) IF(PYTHON_EXECUTABLE_3) EXECUTE_PROCESS(COMMAND ${PYTHON_EXECUTABLE_3} -c "import sys; print('%s.%s' % sys.version_info[:2])" OUTPUT_VARIABLE PYTHON3_CURRENT_VERSION OUTPUT_STRIP_TRAILING_WHITESPACE) ENDIF() MARK_AS_ADVANCED(PYTHON_EXECUTABLE) MARK_AS_ADVANCED(PYTHON_SITE_PACKAGES) MARK_AS_ADVANCED(PYTHON_LIBRARIES) MARK_AS_ADVANCED(PYTHON_INCLUDE_PATH) gfal2-v2.23.0/cmake/modules/FindSRM_IFCE.cmake000066400000000000000000000045371465240014500205530ustar00rootroot00000000000000# # This module detects if SRM-IFCE is installed and determines where the # include files and libraries are. # # This code sets the following variables: # # SRM_IFCE_LIBRARIES = full path to the SRM-IFCE libraries # SRM_IFCE_INCLUDE_DIR = include dir to be used when using the SRM-IFCE library # SRM_IFCE_FOUND = set to true if SRM-IFCE was found successfully # # SRM_IFCE_LOCATION # setting this enables search for SRM-IFCE libraries / headers in this location # ----------------------------------------------------- # Try with pkgconfig first # ----------------------------------------------------- find_package(PkgConfig) pkg_check_modules(SRM_IFCE_PKG REQUIRED srm-ifce>=1.15.0) if (SRM_IFCE_PKG_FOUND) set (SRM_IFCE_LIBRARIES "${SRM_IFCE_PKG_LIBRARIES}") set (SRM_IFCE_CFLAGS "${SRM_IFCE_PKG_CFLAGS}") if (SRM_IFCE_PKG_INCLUDE_DIRS) set (SRM_IFCE_INCLUDE_DIR "${SRM_IFCE_PKG_INCLUDE_DIRS}") else () set (SRM_IFCE_INCLUDE_DIR "/usr/include") endif () else () # SRM-IFCE Libraries find_library(SRM_IFCE_LIBRARIES NAMES gfal_srm_ifce HINTS ${SRM_IFCE_LOCATION}/lib ${SRM_IFCE_LOCATION}/lib64 ${SRM_IFCE_LOCATION}/lib32 ${STAGE_DIR}/lib ${STAGE_DIR}/lib64 ${CMAKE_INSTALL_PREFIX}/Grid/srm-ifce/*/${PLATFORM}/lib ${CMAKE_INSTALL_PREFIX}/Grid/srm-ifce/*/${PLATFORM}/lib64 DOC "The main srm-ifce library" ) # SRM-IFCE Include Directories find_path(SRM_IFCE_INCLUDE_DIR NAMES gfal_srm_ifce.h HINTS ${SRM_IFCE_LOCATION} ${SRM_IFCE_LOCATION}/include ${SRM_IFCE_LOCATION}/include/* ${STAGE_DIR}/include ${STAGE_DIR}/include ${CMAKE_INSTALL_PREFIX}/Grid/srm-ifce/*/${PLATFORM}/include DOC "The srm-ifce include directory" ) set (SRM_IFCE_CFLAGS "") endif() if(SRM_IFCE_INCLUDE_DIR) message(STATUS "SRM-IFCE includes found in ${SRM_IFCE_INCLUDE_DIR}") endif() # ----------------------------------------------------- # handle the QUIETLY and REQUIRED arguments and set SRM_IFCE_FOUND to TRUE if # all listed variables are TRUE # ----------------------------------------------------- include(FindPackageHandleStandardArgs) find_package_handle_standard_args(SRM_IFCE DEFAULT_MSG SRM_IFCE_LIBRARIES SRM_IFCE_INCLUDE_DIR) mark_as_advanced(SRM_IFCE_LIBRARIES SRM_IFCE_INCLUDE_DIR SRM_IFCE_CFLAGS) gfal2-v2.23.0/cmake/modules/FindUUID.cmake000066400000000000000000000035301465240014500200620ustar00rootroot00000000000000# # This module detects if uuid is installed and determines where the # include files and libraries are. # # This code sets the following variables: # # UUID_LIBRARIES = full path to the uuid libraries # UUID_INCLUDE_DIR = include dir to be used when using the uuid library # UUID_FOUND = set to true if uuid was found successfully # # UUID_LOCATION # setting this enables search for uuid libraries / headers in this location find_package (PkgConfig) pkg_check_modules(UUID_PKG uuid) if (UUID_PKG_FOUND) set (UUID_LIBRARIES ${UUID_PKG_LIBRARIES}) set (UUID_INCLUDE_DIRS ${UUID_PKG_INCLUDE_DIRS}) if (NOT UUID_INCLUDE_DIRS) set (UUID_INCLUDE_DIRS "/usr/include") endif (NOT UUID_INCLUDE_DIRS) set (UUID_DEFINITIONS "${UUID_PKG_CFLAGS} ${UUID_PKG_CFLAGS_OTHER}") else (UUID_PKG_FOUND) find_library(UUID_LIBRARIES NAMES uuid HINTS ${UUID_LOCATION} ${CMAKE_INSTALL_PREFIX}/uuid/*/${PLATFORM}/ DOC "The uuid library" ) find_path(UUID_INCLUDE_DIRS NAMES uuid.h HINTS ${UUID_LOCATION}/include/* ${CMAKE_INSTALL_PREFIX}/uuid/*/${PLATFORM}/ DOC "The uuid include directory" ) set (UUID_DEFINITIONS "") endif (UUID_PKG_FOUND) if (UUID_LIBRARIES) message (STATUS "UUID libraries: ${UUID_LIBRARIES}") endif (UUID_LIBRARIES) if (UUID_INCLUDE_DIRS) message (STATUS "UUID include dir: ${UUID_INCLUDE_DIRS}") endif (UUID_INCLUDE_DIRS) # ----------------------------------------------------- # handle the QUIETLY and REQUIRED arguments and set UUID_FOUND to TRUE if # all listed variables are TRUE # ----------------------------------------------------- include(FindPackageHandleStandardArgs) find_package_handle_standard_args (UUID DEFAULT_MSG UUID_LIBRARIES UUID_INCLUDE_DIRS ) mark_as_advanced(UUID_INCLUDE_DIRS UUID_LIBRARIES) gfal2-v2.23.0/cmake/modules/FindVOMS.cmake000066400000000000000000000041121465240014500200750ustar00rootroot00000000000000# # This module detects if voms is installed and determines where the # include files and libraries are. # # This code sets the following variables: # # VOMS_LIBRARIES = full path to the voms libraries # VOMS_INCLUDE_DIR = include dir to be used when using the voms library # VOMS_DEFINITIONS = compiler flags # VOMS_FOUND = set to true if voms was found successfully # # VOMS_LOCATION # setting this enables search for voms libraries / headers in this location find_package (PkgConfig) pkg_search_module(VOMS_PKG voms-2.0 voms) if (VOMS_PKG_FOUND) set (VOMS_LIBRARIES ${VOMS_PKG_LIBRARIES}) set (VOMS_INCLUDE_DIRS ${VOMS_PKG_INCLUDE_DIRS}) if (NOT VOMS_INCLUDE_DIRS) set (VOMS_INCLUDE_DIRS "/usr/include") endif (NOT VOMS_INCLUDE_DIRS) set (VOMS_DEFINITIONS "${VOMS_PKG_CFLAGS} ${VOMS_PKG_CFLAGS_OTHER}") else (VOMS_PKG_FOUND) message("SEARCH FOR ${CMAKE_INSTALL_PREFIX}/Grid/voms/*/${PLATFORM}/lib64") find_library(VOMS_LIBRARIES NAMES vomsapi HINTS ${VOMS_LOCATION} ${CMAKE_INSTALL_PREFIX}/Grid/voms/*/${PLATFORM}/lib ${CMAKE_INSTALL_PREFIX}/Grid/voms/*/${PLATFORM}/lib64 DOC "The voms library" ) find_path(VOMS_INCLUDE_DIRS NAMES voms/voms_api.h HINTS ${VOMS_LOCATION}/include/* ${CMAKE_INSTALL_PREFIX}/Grid/voms/*/${PLATFORM}/include DOC "The voms include directory" ) set (VOMS_CFLAGS "") endif (VOMS_PKG_FOUND) if (VOMS_LIBRARIES) message (STATUS "VOMS libraries: ${VOMS_LIBRARIES}") endif (VOMS_LIBRARIES) if (VOMS_INCLUDE_DIRS) message (STATUS "VOMS include dir: ${VOMS_INCLUDE_DIRS}") endif (VOMS_INCLUDE_DIRS) # ----------------------------------------------------- # handle the QUIETLY and REQUIRED arguments and set VOMS_FOUND to TRUE if # all listed variables are TRUE # ----------------------------------------------------- include(FindPackageHandleStandardArgs) find_package_handle_standard_args (VOMS DEFAULT_MSG VOMS_LIBRARIES VOMS_INCLUDE_DIRS ) mark_as_advanced(VOMS_LIBRARIES VOMS_INCLUDE_DIRS VOMS_DEFINITIONS) gfal2-v2.23.0/cmake/modules/FindXROOTD.cmake000066400000000000000000000036671465240014500203460ustar00rootroot00000000000000# - Try to find XROOTD libraries # # XROOTD_FOUND - System has XROOTD # XROOTD_INCLUDE_DIR - The XROOTD include directory # XROOTD_LIBRARIES - The libraries needed to use XROOTD # # XROOTD_LOCATION # setting this enables search for xrootd libraries / headers in this location # ----------------------------------------------------- # XROOTD Libraries # ----------------------------------------------------- find_library(XROOTD_CL NAMES XrdCl HINTS ${XROOTD_LOCATION}/lib ${XROOTD_LOCATION}/lib64 ${XROOTD_LOCATION}/lib32 DOC "xrootd cl" ) find_library(XROOTD_POSIX NAMES XrdPosix HINTS ${XROOTD_LOCATION}/lib ${XROOTD_LOCATION}/lib64 ${XROOTD_LOCATION}/lib32 DOC "xrootd posix libraries" ) find_library(XROOTD_UTIL NAMES XrdUtils HINTS ${XROOTD_LOCATION}/lib ${XROOTD_LOCATION}/lib64 ${XROOTD_LOCATION}/lib32 DOC "xrootd util" ) set(XROOTD_LIBRARIES ${XROOTD_CL} ${XROOTD_POSIX} ${XROOTD_UTIL} ) if(XROOTD_LIBRARIES) message(STATUS "xrootd library found in ${XROOTD_LIBRARIES}") endif() # ----------------------------------------------------- # XROOTD Include Directories # ----------------------------------------------------- find_path(XROOTD_INCLUDE_DIR NAMES XrdVersion.hh HINTS ${XROOTD_LOCATION} ${XROOTD_LOCATION}/include ${XROOTD_LOCATION}/include/* ${XROOTD_LOCATION}/src/ /usr/include/xrootd ${CMAKE_INSTALL_PREFIX}/include/xrootd DOC "The xrootd include directory" ) if(XROOTD_INCLUDE_DIR) message(STATUS "xrootd includes found in ${XROOTD_INCLUDE_DIR}") endif() # ----------------------------------------------------- # handle the QUIETLY and REQUIRED arguments and set XROOTD_FOUND to TRUE if # all listed variables are TRUE # ----------------------------------------------------- include(FindPackageHandleStandardArgs) find_package_handle_standard_args(xrootd DEFAULT_MSG XROOTD_LIBRARIES XROOTD_INCLUDE_DIR) mark_as_advanced(XROOTD_INCLUDE_DIR XROOTD_LIBRARIES) gfal2-v2.23.0/cmake/modules/FindZLIB.cmake000066400000000000000000000035401465240014500200550ustar00rootroot00000000000000# # This module detects if zlib is installed and determines where the # include files and libraries are. # # This code sets the following variables: # # ZLIB_LIBRARIES = full path to the zlib libraries # ZLIB_INCLUDE_DIR = include dir to be used when using the glib2 library # ZLIB_FOUND = set to true if zlib was found successfully # # ZLIB_LOCATION # setting this enables search for zlib libraries / headers in this location find_package (PkgConfig) pkg_check_modules(ZLIB_PKG zlib) if (ZLIB_PKG_FOUND) set (ZLIB_LIBRARIES ${ZLIB_PKG_LIBRARIES}) set (ZLIB_INCLUDE_DIRS ${ZLIB_PKG_INCLUDE_DIRS}) if (NOT ZLIB_INCLUDE_DIRS) set (ZLIB_INCLUDE_DIRS "/usr/include") endif (NOT ZLIB_INCLUDE_DIRS) set (ZLIB_DEFINITIONS "${ZLIB_PKG_CFLAGS} ${ZLIB_PKG_CFLAGS_OTHER}") else (ZLIB_PKG_FOUND) find_library(ZLIB_LIBRARIES NAMES zlib HINTS ${ZLIB_LOCATION} ${CMAKE_INSTALL_PREFIX}/zlib/*/${PLATFORM}/ DOC "The main glib2 library" ) find_path(ZLIB_INCLUDE_DIRS NAMES zlib.h HINTS ${ZLIB_LOCATION}/include/* ${CMAKE_INSTALL_PREFIX}/zlib/*/${PLATFORM}/ DOC "The glib2 include directory" ) set (ZLIB_DEFINITIONS "") endif (ZLIB_PKG_FOUND) if (ZLIB_LIBRARIES) message (STATUS "ZLIB libraries: ${ZLIB_LIBRARIES}") endif (ZLIB_LIBRARIES) if (ZLIB_INCLUDE_DIRS) message (STATUS "ZLIB include dir: ${ZLIB_INCLUDE_DIRS}") endif (ZLIB_INCLUDE_DIRS) # ----------------------------------------------------- # handle the QUIETLY and REQUIRED arguments and set ZLIB_FOUND to TRUE if # all listed variables are TRUE # ----------------------------------------------------- include(FindPackageHandleStandardArgs) find_package_handle_standard_args (ZLIB DEFAULT_MSG ZLIB_LIBRARIES ZLIB_INCLUDE_DIRS ) mark_as_advanced(ZLIB_INCLUDE_DIRS ZLIB_LIBRARIES) gfal2-v2.23.0/cmake/modules/FindgSOAP.cmake000066400000000000000000000105221465240014500202240ustar00rootroot00000000000000# # This module detects if gsoap is installed and determines where the # include files and libraries are. # # This code sets the following variables: # # GSOAP_LIBRARIES = full path to the gsoap libraries # GSOAP_SSL_LIBRARIES = full path to the gsoap ssl libraries # GSOAP_INCLUDE_DIR = include dir to be used when using the gsoap library # GSOAP_WSDL2H = wsdl2h binary # GSOAP_SOAPCPP2 = soapcpp2 binary # GSOAP_FOUND = set to true if gsoap was found successfully # # GSOAP_LOCATION # setting this enables search for gsoap libraries / headers in this location # ------------------------------------------------------ # try pkg config search # # ----------------------------------------------------- find_package(PkgConfig) pkg_check_modules(PC_GSOAP gsoap) IF(PC_GSOAP_FOUND) SET(GSOAP_LIBRARIES ${PC_GSOAP_LIBRARIES}) SET(GSOAP_INCLUDE_DIR ${PC_GSOAP_INCLUDE_DIRS}) if (NOT GSOAP_INCLUDE_DIR) set (GSOAP_INCLUDE_DIR "/usr/include") endif (NOT GSOAP_INCLUDE_DIR) SET(GSOAP_DEFINITIONS "${PC_GSOAP_CFLAGS} ${PC_GSOAP_CFLAGS_OTHER}") ELSE(PC_GSOAP_FOUND) # ----------------------------------------------------- # GSOAP Libraries # ----------------------------------------------------- find_library(GSOAP_LIBRARIES NAMES gsoap HINTS ${GSOAP_LOCATION} ${CMAKE_INSTALL_PREFIX}/gsoap/*/${PLATFORM}/ DOC "The main gsoap library" ) # ----------------------------------------------------- # GSOAP Include Directories # ----------------------------------------------------- find_path(GSOAP_INCLUDE_DIR NAMES stdsoap2.h HINTS ${GSOAP_LOCATION} ${CMAKE_INSTALL_PREFIX}/gsoap/*/${PLATFORM}/ DOC "The gsoap include directory" ) SET(GSOAP_DEFINITIONS "") ENDIF(PC_GSOAP_FOUND) # ----------------------------------------------------- # GSOAP ssl Libraries # ----------------------------------------------------- find_library(GSOAP_SSL_LIBRARIES NAMES gsoapssl HINTS ${GSOAP_LOCATION} ${CMAKE_INSTALL_PREFIX}/gsoap/*/${PLATFORM}/ DOC "The ssl gsoap library" ) # ----------------------------------------------------- # GSOAP Binaries # ----------------------------------------------------- find_program(GSOAP_WSDL2H NAMES wsdl2h HINTS ${GSOAP_LOCATION}/bin ${CMAKE_INSTALL_PREFIX}/gsoap/*/${PLATFORM}/bin/ DOC "The gsoap bin directory" ) find_program(GSOAP_SOAPCPP2 NAMES soapcpp2 HINTS ${GSOAP_LOCATION}/bin ${CMAKE_INSTALL_PREFIX}/gsoap/*/${PLATFORM}/bin/ DOC "The gsoap bin directory" ) # ----------------------------------------------------- # GSOAP_276_COMPAT_FLAGS and GSOAPVERSION # try to determine the flagfor the 2.7.6 compatiblity, break with 2.7.13 and re-break with 2.7.16 # ---------------------------------------------------- message(STATUS " - wsdlh : ${GSOAP_WSDL2H}") message(STATUS " - SOAPCPP2 : ${GSOAP_SOAPCPP2}") # some versions of soapcpp2 interpret "-v" as verbose, and hang while waiting for input # try "-help" first, and if it fails, do "-v" execute_process(COMMAND ${GSOAP_SOAPCPP2} "-help" OUTPUT_VARIABLE GSOAP_STRING_VERSION ERROR_VARIABLE GSOAP_STRING_VERSION ) string(REGEX MATCH "[0-9]*\\.[0-9]*\\.[0-9]*" GSOAP_VERSION ${GSOAP_STRING_VERSION}) if( "${GSOAP_VERSION}" STREQUAL "..") execute_process(COMMAND ${GSOAP_SOAPCPP2} "-v" OUTPUT_VARIABLE GSOAP_STRING_VERSION ERROR_VARIABLE GSOAP_STRING_VERSION ) string(REGEX MATCH "[0-9]*\\.[0-9]*\\.[0-9]*" GSOAP_VERSION ${GSOAP_STRING_VERSION}) endif() message(STATUS " - GSOAP VERSION : ${GSOAP_VERSION}") if( "${GSOAP_VERSION}" VERSION_LESS "2.7.6") set(GSOAP_276_COMPAT_FLAGS "") elseif ( "${GSOAP_VERSION}" VERSION_LESS "2.7.14") set(GSOAP_276_COMPAT_FLAGS "-z") else ( "${GSOAP_VERSION}" VERSION_LESS "2.7.14") set(GSOAP_276_COMPAT_FLAGS "-z1 -z2") endif ( "${GSOAP_VERSION}" VERSION_LESS "2.7.6") # ----------------------------------------------------- # handle the QUIETLY and REQUIRED arguments and set GSOAP_FOUND to TRUE if # all listed variables are TRUE # ----------------------------------------------------- include(FindPackageHandleStandardArgs) find_package_handle_standard_args(gsoap DEFAULT_MSG GSOAP_LIBRARIES GSOAP_INCLUDE_DIR GSOAP_WSDL2H GSOAP_SOAPCPP2) mark_as_advanced(GSOAP_INCLUDE_DIR GSOAP_LIBRARIES GSOAP_DEFINITIONS GSOAP_WSDL2H GSOAP_SOAPCPP2) gfal2-v2.23.0/cmake/modules/MacroAddDoxygen.cmake000066400000000000000000000010611465240014500215200ustar00rootroot00000000000000## ## Doxygen macro, allow Doxygen generation from cmake ## do a ""make doc" for the generation macro(addDoxyGeneration DOXYFILE_LOCATION) find_package(Doxygen) if(DOXYGEN_FOUND) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/${DOXYFILE_LOCATION} ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile @ONLY) add_custom_target(doc ${DOXYGEN_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} COMMENT "Generating API documentation with Doxygen" VERBATIM ) endif(DOXYGEN_FOUND) endmacro(addDoxyGeneration DOXYFILE_LOCATION) gfal2-v2.23.0/cmake/modules/MacroAddepydoc.cmake000066400000000000000000000036451465240014500214000ustar00rootroot00000000000000## ## Doxygen macro, allow Doxygen generation from cmake ## do a ""make doc" for the generation # for old version < 3.0 cmake variables replace config files ( not supported ) ( needed for EPEL 5 support ) # EPYDOC_MODULE_PATH path of the module to configure # EPYDOC_MODULE_URL project url # EPYDOC_MODULE_NAME project name # macro(addEpydocGeneration EPYDOC_CONFIG_LOCATION) IF(NOT EPYDOC_FOUND) execute_process(COMMAND epydoc --version OUTPUT_VARIABLE EPYDOC_VERSION_UNPARSED ERROR_VARIABLE EPYDOC_ERROR ) IF(${EPYDOC_ERROR}) message(SEND_ERROR "epydoc not found....") ELSE(${EPYDOC_ERROR}) string(REGEX REPLACE ".*version (.*)\n" "\\1" EPYDOC_VERSION ${EPYDOC_VERSION_UNPARSED}) message(STATUS " epydoc version ..... ${EPYDOC_VERSION} ") ENDIF(${EPYDOC_ERROR}) ENDIF(NOT EPYDOC_FOUND) IF(${EPYDOC_VERSION} VERSION_GREATER "3.0.0") configure_file(${CMAKE_CURRENT_SOURCE_DIR}/${EPYDOC_CONFIG_LOCATION} ${CMAKE_CURRENT_BINARY_DIR}/epydoc_config @ONLY) add_custom_target(doc epydoc --config ${CMAKE_CURRENT_BINARY_DIR}/epydoc_config -v WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} COMMENT "Generate API documentation with epydoc" VERBATIM ) ELSE(${EPYDOC_VERSION} VERSION_GREATER "3.0.0") # VERSION TOO OLD, NO CONFIG FILE MANAGEMENT add_custom_target(doc epydoc --html -u ${EPYDOC_MODULE_URL} -n ${EPYDOC_MODULE_NAME} -v -o ${CMAKE_CURRENT_BINARY_DIR}/html ${EPYDOC_MODULE_PATH} WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} COMMENT "Generate API documentation with epydoc" VERBATIM ) ENDIF(${EPYDOC_VERSION} VERSION_GREATER "3.0.0") endmacro(addEpydocGeneration EPYDOC_CONFIG_LOCATION) gfal2-v2.23.0/cmake/modules/MacroCopyFile.cmake000066400000000000000000000017141465240014500212110ustar00rootroot00000000000000# - macro_copy_file(_src _dst) # Copies a file to ${_dst} only if ${_src} is different (newer) than ${_dst} # # Example: # macro_copy_file(${CMAKE_CURRENT_SOURCE_DIR}/icon.png ${CMAKE_CURRENT_BINARY_DIR}/.) # Copies file icon.png to ${CMAKE_CURRENT_BINARY_DIR} directory # # Copyright (c) 2006-2007 Wengo # Copyright (c) 2006-2008 Andreas Schneider # # Redistribution and use is allowed according to the terms of the BSD license. # For details see the accompanying COPYING file. macro(copy_files GLOBPAT DESTINATION) file(GLOB COPY_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} ${GLOBPAT}) add_custom_target(copy ALL COMMENT "Copying files: ${GLOBPAT}") foreach(FILENAME ${COPY_FILES}) set(SRC "${CMAKE_CURRENT_SOURCE_DIR}/${FILENAME}") set(DST "${DESTINATION}/${FILENAME}") add_custom_command( TARGET copy COMMAND ${CMAKE_COMMAND} -E copy ${SRC} ${DST} ) endforeach(FILENAME) endmacro(copy_files) gfal2-v2.23.0/cmake/modules/ReleaseDebugAutoFlags.cmake000066400000000000000000000006201465240014500226450ustar00rootroot00000000000000## debug / release autoManagement set(CMAKE_C_FLAGS_RELEASE "-O2") set(CMAKE_C_FLAGS_DEBUG "-g3 -O0 -ggdb -Wall -fstack-protector-all") set(CMAKE_CXX_FLAGS_RELEASE "-O2") set(CMAKE_CXX_FLAGS_DEBUG "-g3 -O0 -ggdb -Wall -fstack-protector-all") set(CMAKE_C_FLAGS_RELWITHDEBINFO "-O2 -g -Wall -fstack-protector-all") set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O2 -g -Wall -fstack-protector-all") gfal2-v2.23.0/coverage.sh000077500000000000000000000020221465240014500151060ustar00rootroot00000000000000#!/bin/bash # Build source for coverage tests if [ "$#" -eq 1 ]; then BUILD_DIR=`readlink -f $1` else BUILD_DIR=~+/build fi SOURCE_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) RUNNING_DIR=`pwd` set -x mkdir -p "${BUILD_DIR}" pushd "${BUILD_DIR}" # Build CFLAGS=--coverage CXXFLAGS=--coverage cmake "${SOURCE_DIR}" \ -DUNIT_TESTS=ON -DFUNCTIONAL_TESTS=ON -DPLUGIN_MOCK=ON make -j2 # Clear counters lcov --directory . --zerocounters # Run export GFAL_PLUGIN_DIR="${BUILD_DIR}/plugins" export GFAL_CONFIG_DIR="${SOURCE_DIR}/test/conf_test" ctest -T test # Extract coverage lcov --directory . --capture --output-file="${RUNNING_DIR}/coverage.info" if [ ! -f "/tmp/lcov_cobertura.py" ]; then wget "https://raw.github.com/eriwen/lcov-to-cobertura-xml/master/lcov_cobertura/lcov_cobertura.py" -O "/tmp/lcov_cobertura.py" fi python /tmp/lcov_cobertura.py "${RUNNING_DIR}/coverage.info" -b "${SOURCE_DIR}" -e ".+usr.include." -o "${RUNNING_DIR}/coverage.xml" # Done popd echo "Done extracting coverage information" gfal2-v2.23.0/dist/000077500000000000000000000000001465240014500137235ustar00rootroot00000000000000gfal2-v2.23.0/dist/etc/000077500000000000000000000000001465240014500144765ustar00rootroot00000000000000gfal2-v2.23.0/dist/etc/gfal2.d/000077500000000000000000000000001465240014500157135ustar00rootroot00000000000000gfal2-v2.23.0/dist/etc/gfal2.d/bdii.conf000066400000000000000000000006151465240014500174730ustar00rootroot00000000000000# # basic configuration for the gfal 2 BDII related options ( information system for wlcg ) [BDII] # enable or disable bdii query system ( default : enabled ) ENABLED=true # default BDII server to contact LCG_GFAL_INFOSYS=lcg-bdii.cern.ch:2170 # Cache file for the BDII system # Compatible with FTS3 cache format # This file must be updated externally CACHE_FILE=/var/lib/fts3/bdii_cache.xml gfal2-v2.23.0/dist/etc/gfal2.d/dcap_plugin.conf000066400000000000000000000002541465240014500210500ustar00rootroot00000000000000# # basic configuration for the gfal 2 DCAP plugin [DCAP PLUGIN] # set the passive mode enabled or now # active mode can cause troubles with firewalls MODE_PASSIVE=true gfal2-v2.23.0/dist/etc/gfal2.d/gfal2_core.conf000066400000000000000000000015321465240014500205660ustar00rootroot00000000000000# Core configuration of GFAL 2.0 [CORE] # Enable or disable DNS resolution within the copy function RESOLVE_DNS=false # Namespace operations timeout in seconds. # Other protocols may override this if set (i.e. GRIDFTP PLUGIN:OPERATION_TIMEOUT) NAMESPACE_TIMEOUT=300 # Checksum timeout in seconds. # Other protocol specific may override this if set (i.e. GRIDFTP PLUGIN:CHECKSUM_TIMEOUT) CHECKSUM_TIMEOUT=1800 # Buffersize for non-3rd party copies, in bytes COPY_BUFFERSIZE=4194304 # Use direct IO (if the affected plugins accept it) for the copies # Use this only if you know what you are doing # See notes on man 2 open COPY_DIRECT_IO=false # If direct IO is enabled, the buffer may need to be aligned # 512 seems normally safe # COPY_BUFFER_ALIGNMENT=512 # When enabled, always return Adler32 checksum as 8-byte string FORMAT_ADLER32_CHECKSUM=true gfal2-v2.23.0/dist/etc/gfal2.d/gsiftp_plugin.conf000066400000000000000000000030371465240014500214370ustar00rootroot00000000000000# # basic configuration for the gfal 2 griftp plugin [GRIDFTP PLUGIN] # enable or disable gridFTPv2 support GRIDFTP_V2=true # enable or disable session re-use support # warning: # disabling this feature can slow-down a lot the performance # enabling this feature can cause trouble with Castor SESSION_REUSE=true # default number of streams used for file transfers # 0 means in-order-stream mode RD_NB_STREAM=0 # default checksum algorithm type used for transfer content verification COPY_CHECKSUM_TYPE=ADLER32 # enable or disable the encryption for the data channel DCAU=false # enable or disable the ipv6 support IPV6=false # enable or disable striped passive support # Note, if IPv6 is enabled, this may be needed in order to get GridFTP redirection to work SPAS=false # define the maximum time in s # for a gsiftp checksum request # Overrides CORE:CHECKSUM_TIMEOUT # CHECKSUM_CALC_TIMEOUT=1800 # maximum time between two performance markers # disable if equal to 0 PERF_MARKER_TIMEOUT=360 # timeout for namespace operations # Overrides CORE:NAMESPACE_TIMEOUT # OPERATION_TIMEOUT=300 ## enable or disable the delay passive option of gridftpv2 ## this option need to be enabled to support the gridftp redirection features DELAY_PASSV=true # Enable UDT transfers # Not all servers implement this, so gfal2 will fallback to a normal transfer if # not supported ENABLE_UDT=false # Enable the PASV plugin # Required to trigger events with the final destination IP and port ENABLE_PASV_PLUGIN=false # Block size for third party copies # BLOCK_SIZE = 0 gfal2-v2.23.0/dist/etc/gfal2.d/http_plugin.conf000066400000000000000000000035751465240014500211310ustar00rootroot00000000000000# Configuration file for HTTP plugin [HTTP PLUGIN] # Enable HTTP Third Party Copies ENABLE_REMOTE_COPY=true # Enable streamed copies (data passes via the client) ENABLE_STREAM_COPY=true # Enable TPC fallback mechanism # Start with DEFAULT_COPY_MODE and fallback in case of error ENABLE_FALLBACK_TPC_COPY=true ## Default mode for http copy ## possible values are: 3rd pull, 3rd push or streamed ## By default all methods will be tried in case of failures starting from 3rd pull ## if 3rd push is set as default only 3rd push and then streamed will be executed ## if streamed is set as default only streamed transfer will be executed DEFAULT_COPY_MODE=3rd pull # Enable or disable the SSL CA check INSECURE=false # Enable or disable Davix Metalink support METALINK=false ## Force Davix log level (0 - 5) ## Level 0 --> No explicit Davix log level. Applies Gfal2 verbosity to Davix as well ## Level 1 --> Critical messages only ## Level 2 --> Warning messages and higher ## Level 3 --> Verbose, but still reasonable ## Level 4 --> Debug (most of the logging data) ## Level 5 --> Trace (dump of almost everything) ## Recommended setting should be 0 or 3. For debugging, use level 4 LOG_LEVEL=0 # Enable Davix logging of sensitive information LOG_SENSITIVE=false # Enable Davix logging of HTTP request/response body (default false) # Note: Meant for debugging # - Do not use in production, risk of leaking secrets # - Do not use for transfers, as logging will have same file as transferred file LOG_CONTENT=false # Use HTTP Keep-Alive KEEP_ALIVE=true # Attempt to retrieve SE-issued tokens RETRIEVE_BEARER_TOKEN=true # AWS S3 related options [S3] ## AWS Secret key #SECRET_KEY= ## AWS Access key #ACCESS_KEY= ## AWS Short lived token #TOKEN= ## AWS Region #REGION= # GCloud related options [GCLOUD] ## Google JSON auth file path #JSON_AUTH_FILE= ## Google JSON auth content as string #JSON_AUTH_STRING= gfal2-v2.23.0/dist/etc/gfal2.d/lfc_plugin.conf000066400000000000000000000004561465240014500207110ustar00rootroot00000000000000# # basic configuration for the gfal 2 lfc plugin [LFC PLUGIN] # Logical File Catalog hostname # default is the central one, at cern LFC_HOST=lfc-puppet01.cern.ch # connexion timeout in second LFC_CONNTIMEOUT=15 # maximum number of try for opening a connexion LFC_CONRETRY=2 # LFC_CONRETRYINT=1 gfal2-v2.23.0/dist/etc/gfal2.d/mock_plugin.conf000066400000000000000000000001511465240014500210660ustar00rootroot00000000000000## configuration file for http plugin ## [MOCK PLUGIN] MAX_TRANSFER_TIME=5 MIN_TRANSFER_TIME=5 SIGNALS=0 gfal2-v2.23.0/dist/etc/gfal2.d/rfio_plugin.conf000066400000000000000000000002671465240014500211040ustar00rootroot00000000000000# # basic configuration for the gfal 2 rfio plugin [RFIO PLUGIN] # type of RFIO library to use ( cf : rfio castor vs rfio dpm ) # value can be castor or dpm LCG_RFIO_TYPE="dpm" gfal2-v2.23.0/dist/etc/gfal2.d/sftp_plugin.conf000066400000000000000000000003631465240014500211160ustar00rootroot00000000000000# # basic configuration for the gfal 2 sftp plugin [SFTP PLUGIN] ## Defaults to current user name # USER= ## Defaults to empty # PASSWORD= ## Defaults to $HOME/.ssh/id_rsa # PRIVKEY= ## Private key passphrase. Defaults to empty # PASSPHRASE= gfal2-v2.23.0/dist/etc/gfal2.d/srm_plugin.conf000066400000000000000000000021231465240014500207370ustar00rootroot00000000000000# # basic configuration for the gfal 2 srm plugin [SRM PLUGIN] # timeout for SRM operations in seconds # Overrides CORE:NAMESPACE_TIMEOUT #OPERATION_TIMEOUT=180 # timeout for the HTTP connexion in seconds CONN_TIMEOUT=60 # desired request lifetime REQUEST_LIFETIME=3600 # default checksum type for transfer check COPY_CHECKSUM_TYPE=ADLER32 # ordered list of turls protocols for remote I/O # the top priority protocol is the first one TURL_PROTOCOLS=gsiftp;rfio;gsidcap;dcap;kdcap # ordered list of turls protocols for third party transfer # only protocol supporting third party copy should be here # the top priority protocol is the first one TURL_3RD_PARTY_PROTOCOLS=gsiftp;https;root # enable or disable the srm session re-use # no parameter : disabled KEEP_ALIVE=true # enable or disable the check for source file locality # in SRM copy. If enabled and the locality is NEARLINE # the SRM copy is not executed # no parameter : disabled COPY_FAIL_NEARLINE=false # enable or disable locality check for REPLICAS XATTR # If enabled, obtain TURLs only if the file is ONLINE XATTR_FAIL_NEARLINE=false gfal2-v2.23.0/dist/etc/gfal2.d/x509.conf000066400000000000000000000000631465240014500172660ustar00rootroot00000000000000# X509 certificate and proxy [X509] #CERT= #KEY= gfal2-v2.23.0/dist/etc/gfal2.d/xrootd_plugin.conf000066400000000000000000000006101465240014500214540ustar00rootroot00000000000000# # basic configuration for the gfal2 XROOTD plugin # [XROOTD PLUGIN] # Default checksum type for third party copies COPY_CHECKSUM_TYPE=ADLER32 # Normalize the path (this is, turn root://host/path into root://host//path) NORMALIZE_PATH=true # To pass any custom flag via URL to the xrootd library, any variable that starts with XRD. will be used # (lowercase) # XRD.WANTPROT=unix,gsi,krb5 gfal2-v2.23.0/doc/000077500000000000000000000000001465240014500135255ustar00rootroot00000000000000gfal2-v2.23.0/doc/CMakeLists.txt000066400000000000000000000010151465240014500162620ustar00rootroot00000000000000 SET(MAIN_DOC TRUE CACHE STRING "enable installation of the main library documentation") IF(MAIN_DOC) file(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/build/html/") addDoxyGeneration("Doxyfile.in") install(DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/build/html/" DESTINATION ${DOC_INSTALL_DIR}/html/) install(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/examples/" DESTINATION ${DOC_INSTALL_DIR}/examples) ENDIF(MAIN_DOC) install(FILES "man/gfal2_version.1" DESTINATION ${MAN_INSTALL_DIR}/man1/) # install examples gfal2-v2.23.0/doc/Doxyfile.in000066400000000000000000001456561465240014500156610ustar00rootroot00000000000000# Doxyfile 1.4.7 # This file describes the settings to be used by the documentation system # doxygen (www.doxygen.org) for a project # # All text after a hash (#) is considered a comment and will be ignored # The format is: # TAG = value [value, ...] # For lists items can also be appended using: # TAG += value [value, ...] # Values that contain spaces should be placed between quotes (" ") #--------------------------------------------------------------------------- # Project related configuration options #--------------------------------------------------------------------------- # The PROJECT_NAME tag is a single word (or a sequence of words surrounded # by quotes) that should identify the project. PROJECT_NAME = "GFAL2" # The PROJECT_NUMBER tag can be used to enter a project or revision number. # This could be handy for archiving the generated documentation or # if some version control system is used. PROJECT_NUMBER = @VERSION_STRING@ # The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) # base path where the generated documentation will be put. # If a relative path is entered, it will be relative to the location # where doxygen was started. If left blank the current directory will be used. OUTPUT_DIRECTORY = @CMAKE_CURRENT_BINARY_DIR@/build/ # If the CREATE_SUBDIRS tag is set to YES, then doxygen will create # 4096 sub-directories (in 2 levels) under the output directory of each output # format and will distribute the generated files over these directories. # Enabling this option can be useful when feeding doxygen a huge amount of # source files, where putting all generated files in the same directory would # otherwise cause performance problems for the file system. CREATE_SUBDIRS = NO # The OUTPUT_LANGUAGE tag is used to specify the language in which all # documentation generated by doxygen is written. Doxygen will use this # information to generate all constant output in the proper language. # The default language is English, other supported languages are: # Brazilian, Catalan, Chinese, Chinese-Traditional, Croatian, Czech, Danish, # Dutch, Finnish, French, German, Greek, Hungarian, Italian, Japanese, # Japanese-en (Japanese with English messages), Korean, Korean-en, Norwegian, # Polish, Portuguese, Romanian, Russian, Serbian, Slovak, Slovene, Spanish, # Swedish, and Ukrainian. OUTPUT_LANGUAGE = English # This tag can be used to specify the encoding used in the generated output. # The encoding is not always determined by the language that is chosen, # but also whether or not the output is meant for Windows or non-Windows users. # In case there is a difference, setting the USE_WINDOWS_ENCODING tag to YES # forces the Windows encoding (this is the default for the Windows binary), # whereas setting the tag to NO uses a Unix-style encoding (the default for # all platforms other than Windows). USE_WINDOWS_ENCODING = NO # If the BRIEF_MEMBER_DESC tag is set to YES (the default) Doxygen will # include brief member descriptions after the members that are listed in # the file and class documentation (similar to JavaDoc). # Set to NO to disable this. BRIEF_MEMBER_DESC = YES # If the REPEAT_BRIEF tag is set to YES (the default) Doxygen will prepend # the brief description of a member or function before the detailed description. # Note: if both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the # brief descriptions will be completely suppressed. REPEAT_BRIEF = YES # This tag implements a quasi-intelligent brief description abbreviator # that is used to form the text in various listings. Each string # in this list, if found as the leading text of the brief description, will be # stripped from the text and the result after processing the whole list, is # used as the annotated text. Otherwise, the brief description is used as-is. # If left blank, the following values are used ("$name" is automatically # replaced with the name of the entity): "The $name class" "The $name widget" # "The $name file" "is" "provides" "specifies" "contains" # "represents" "a" "an" "the" ABBREVIATE_BRIEF = # If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then # Doxygen will generate a detailed section even if there is only a brief # description. ALWAYS_DETAILED_SEC = NO # If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all # inherited members of a class in the documentation of that class as if those # members were ordinary class members. Constructors, destructors and assignment # operators of the base classes will not be shown. INLINE_INHERITED_MEMB = NO # If the FULL_PATH_NAMES tag is set to YES then Doxygen will prepend the full # path before files name in the file list and in the header files. If set # to NO the shortest path that makes the file name unique will be used. FULL_PATH_NAMES = YES # If the FULL_PATH_NAMES tag is set to YES then the STRIP_FROM_PATH tag # can be used to strip a user-defined part of the path. Stripping is # only done if one of the specified strings matches the left-hand part of # the path. The tag can be used to show relative paths in the file list. # If left blank the directory from which doxygen is run is used as the # path to strip. STRIP_FROM_PATH = # The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of # the path mentioned in the documentation of a class, which tells # the reader which header file to include in order to use a class. # If left blank only the name of the header file containing the class # definition is used. Otherwise one should specify the include paths that # are normally passed to the compiler using the -I flag. STRIP_FROM_INC_PATH = # If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter # (but less readable) file names. This can be useful is your file systems # doesn't support long names like on DOS, Mac, or CD-ROM. SHORT_NAMES = NO # If the JAVADOC_AUTOBRIEF tag is set to YES then Doxygen # will interpret the first line (until the first dot) of a JavaDoc-style # comment as the brief description. If set to NO, the JavaDoc # comments will behave just like the Qt-style comments (thus requiring an # explicit @brief command for a brief description. JAVADOC_AUTOBRIEF = NO # The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make Doxygen # treat a multi-line C++ special comment block (i.e. a block of //! or /// # comments) as a brief description. This used to be the default behaviour. # The new default is to treat a multi-line C++ comment block as a detailed # description. Set this tag to YES if you prefer the old behaviour instead. MULTILINE_CPP_IS_BRIEF = NO # If the DETAILS_AT_TOP tag is set to YES then Doxygen # will output the detailed description near the top, like JavaDoc. # If set to NO, the detailed description appears after the member # documentation. DETAILS_AT_TOP = NO # If the INHERIT_DOCS tag is set to YES (the default) then an undocumented # member inherits the documentation from any documented member that it # re-implements. INHERIT_DOCS = YES # If the SEPARATE_MEMBER_PAGES tag is set to YES, then doxygen will produce # a new page for each member. If set to NO, the documentation of a member will # be part of the file/class/namespace that contains it. SEPARATE_MEMBER_PAGES = NO # The TAB_SIZE tag can be used to set the number of spaces in a tab. # Doxygen uses this value to replace tabs by spaces in code fragments. TAB_SIZE = 8 # This tag can be used to specify a number of aliases that acts # as commands in the documentation. An alias has the form "name=value". # For example adding "sideeffect=\par Side Effects:\n" will allow you to # put the command \sideeffect (or @sideeffect) in the documentation, which # will result in a user-defined paragraph with heading "Side Effects:". # You can put \n's in the value part of an alias to insert newlines. ALIASES = # Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C # sources only. Doxygen will then generate output that is more tailored for C. # For instance, some of the names that are used will be different. The list # of all members will be omitted, etc. OPTIMIZE_OUTPUT_FOR_C = YES # Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java # sources only. Doxygen will then generate output that is more tailored for Java. # For instance, namespaces will be presented as packages, qualified scopes # will look different, etc. OPTIMIZE_OUTPUT_JAVA = NO # If you use STL classes (i.e. std::string, std::vector, etc.) but do not want to # include (a tag file for) the STL sources as input, then you should # set this tag to YES in order to let doxygen match functions declarations and # definitions whose arguments contain STL classes (e.g. func(std::string); v.s. # func(std::string) {}). This also make the inheritance and collaboration # diagrams that involve STL classes more complete and accurate. BUILTIN_STL_SUPPORT = NO # If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC # tag is set to YES, then doxygen will reuse the documentation of the first # member in the group (if any) for the other members of the group. By default # all members of a group must be documented explicitly. DISTRIBUTE_GROUP_DOC = NO # Set the SUBGROUPING tag to YES (the default) to allow class member groups of # the same type (for instance a group of public functions) to be put as a # subgroup of that type (e.g. under the Public Functions section). Set it to # NO to prevent subgrouping. Alternatively, this can be done per class using # the \nosubgrouping command. SUBGROUPING = YES #--------------------------------------------------------------------------- # Build related configuration options #--------------------------------------------------------------------------- # If the EXTRACT_ALL tag is set to YES doxygen will assume all entities in # documentation are documented, even if no documentation was available. # Private class members and static file members will be hidden unless # the EXTRACT_PRIVATE and EXTRACT_STATIC tags are set to YES EXTRACT_ALL = NO # If the EXTRACT_PRIVATE tag is set to YES all private members of a class # will be included in the documentation. EXTRACT_PRIVATE = NO # If the EXTRACT_STATIC tag is set to YES all static members of a file # will be included in the documentation. EXTRACT_STATIC = NO # If the EXTRACT_LOCAL_CLASSES tag is set to YES classes (and structs) # defined locally in source files will be included in the documentation. # If set to NO only classes defined in header files are included. EXTRACT_LOCAL_CLASSES = YES # This flag is only useful for Objective-C code. When set to YES local # methods, which are defined in the implementation section but not in # the interface are included in the documentation. # If set to NO (the default) only methods in the interface are included. EXTRACT_LOCAL_METHODS = NO # If the HIDE_UNDOC_MEMBERS tag is set to YES, Doxygen will hide all # undocumented members of documented classes, files or namespaces. # If set to NO (the default) these members will be included in the # various overviews, but no documentation section is generated. # This option has no effect if EXTRACT_ALL is enabled. HIDE_UNDOC_MEMBERS = NO # If the HIDE_UNDOC_CLASSES tag is set to YES, Doxygen will hide all # undocumented classes that are normally visible in the class hierarchy. # If set to NO (the default) these classes will be included in the various # overviews. This option has no effect if EXTRACT_ALL is enabled. HIDE_UNDOC_CLASSES = NO # If the HIDE_FRIEND_COMPOUNDS tag is set to YES, Doxygen will hide all # friend (class|struct|union) declarations. # If set to NO (the default) these declarations will be included in the # documentation. HIDE_FRIEND_COMPOUNDS = NO # If the HIDE_IN_BODY_DOCS tag is set to YES, Doxygen will hide any # documentation blocks found inside the body of a function. # If set to NO (the default) these blocks will be appended to the # function's detailed documentation block. HIDE_IN_BODY_DOCS = NO # The INTERNAL_DOCS tag determines if documentation # that is typed after a \internal command is included. If the tag is set # to NO (the default) then the documentation will be excluded. # Set it to YES to include the internal documentation. INTERNAL_DOCS = NO # If the CASE_SENSE_NAMES tag is set to NO then Doxygen will only generate # file names in lower-case letters. If set to YES upper-case letters are also # allowed. This is useful if you have classes or files whose names only differ # in case and if your file system supports case sensitive file names. Windows # and Mac users are advised to set this option to NO. CASE_SENSE_NAMES = YES # If the HIDE_SCOPE_NAMES tag is set to NO (the default) then Doxygen # will show members with their full class and namespace scopes in the # documentation. If set to YES the scope will be hidden. HIDE_SCOPE_NAMES = NO # If the SHOW_INCLUDE_FILES tag is set to YES (the default) then Doxygen # will put a list of the files that are included by a file in the documentation # of that file. SHOW_INCLUDE_FILES = YES # If the INLINE_INFO tag is set to YES (the default) then a tag [inline] # is inserted in the documentation for inline members. INLINE_INFO = YES # If the SORT_MEMBER_DOCS tag is set to YES (the default) then doxygen # will sort the (detailed) documentation of file and class members # alphabetically by member name. If set to NO the members will appear in # declaration order. SORT_MEMBER_DOCS = YES # If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the # brief documentation of file, namespace and class members alphabetically # by member name. If set to NO (the default) the members will appear in # declaration order. SORT_BRIEF_DOCS = NO # If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be # sorted by fully-qualified names, including namespaces. If set to # NO (the default), the class list will be sorted only by class name, # not including the namespace part. # Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES. # Note: This option applies only to the class list, not to the # alphabetical list. SORT_BY_SCOPE_NAME = NO # The GENERATE_TODOLIST tag can be used to enable (YES) or # disable (NO) the todo list. This list is created by putting \todo # commands in the documentation. GENERATE_TODOLIST = YES # The GENERATE_TESTLIST tag can be used to enable (YES) or # disable (NO) the test list. This list is created by putting \test # commands in the documentation. GENERATE_TESTLIST = YES # The GENERATE_BUGLIST tag can be used to enable (YES) or # disable (NO) the bug list. This list is created by putting \bug # commands in the documentation. GENERATE_BUGLIST = YES # The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or # disable (NO) the deprecated list. This list is created by putting # \deprecated commands in the documentation. GENERATE_DEPRECATEDLIST= YES # The ENABLED_SECTIONS tag can be used to enable conditional # documentation sections, marked by \if sectionname ... \endif. ENABLED_SECTIONS = # The MAX_INITIALIZER_LINES tag determines the maximum number of lines # the initial value of a variable or define consists of for it to appear in # the documentation. If the initializer consists of more lines than specified # here it will be hidden. Use a value of 0 to hide initializers completely. # The appearance of the initializer of individual variables and defines in the # documentation can be controlled using \showinitializer or \hideinitializer # command in the documentation regardless of this setting. MAX_INITIALIZER_LINES = 30 # Set the SHOW_USED_FILES tag to NO to disable the list of files generated # at the bottom of the documentation of classes and structs. If set to YES the # list will mention the files that were used to generate the documentation. SHOW_USED_FILES = YES # If the sources in your project are distributed over multiple directories # then setting the SHOW_DIRECTORIES tag to YES will show the directory hierarchy # in the documentation. The default is NO. SHOW_DIRECTORIES = NO # The FILE_VERSION_FILTER tag can be used to specify a program or script that # doxygen should invoke to get the current version for each file (typically from the # version control system). Doxygen will invoke the program by executing (via # popen()) the command , where is the value of # the FILE_VERSION_FILTER tag, and is the name of an input file # provided by doxygen. Whatever the program writes to standard output # is used as the file version. See the manual for examples. FILE_VERSION_FILTER = #--------------------------------------------------------------------------- # configuration options related to warning and progress messages #--------------------------------------------------------------------------- # The QUIET tag can be used to turn on/off the messages that are generated # by doxygen. Possible values are YES and NO. If left blank NO is used. QUIET = NO # The WARNINGS tag can be used to turn on/off the warning messages that are # generated by doxygen. Possible values are YES and NO. If left blank # NO is used. WARNINGS = YES # If WARN_IF_UNDOCUMENTED is set to YES, then doxygen will generate warnings # for undocumented members. If EXTRACT_ALL is set to YES then this flag will # automatically be disabled. WARN_IF_UNDOCUMENTED = YES # If WARN_IF_DOC_ERROR is set to YES, doxygen will generate warnings for # potential errors in the documentation, such as not documenting some # parameters in a documented function, or documenting parameters that # don't exist or using markup commands wrongly. WARN_IF_DOC_ERROR = YES # This WARN_NO_PARAMDOC option can be abled to get warnings for # functions that are documented, but have no documentation for their parameters # or return value. If set to NO (the default) doxygen will only warn about # wrong or incomplete parameter documentation, but not about the absence of # documentation. WARN_NO_PARAMDOC = NO # The WARN_FORMAT tag determines the format of the warning messages that # doxygen can produce. The string should contain the $file, $line, and $text # tags, which will be replaced by the file and line number from which the # warning originated and the warning text. Optionally the format may contain # $version, which will be replaced by the version of the file (if it could # be obtained via FILE_VERSION_FILTER) WARN_FORMAT = "$file:$line: $text" # The WARN_LOGFILE tag can be used to specify a file to which warning # and error messages should be written. If left blank the output is written # to stderr. WARN_LOGFILE = #--------------------------------------------------------------------------- # configuration options related to the input files #--------------------------------------------------------------------------- # The INPUT tag can be used to specify the files and/or directories that contain # documented source files. You may enter file names like "myfile.cpp" or # directories like "/usr/src/myproject". Separate the files or directories # with spaces. INPUT = @CMAKE_SOURCE_DIR@/doc/src INPUT += @CMAKE_SOURCE_DIR@/src/core/posix/gfal_posix_api.h INPUT += @CMAKE_SOURCE_DIR@/src/core/file/gfal_file_api.h INPUT += @CMAKE_SOURCE_DIR@/src/core/gfal_api.h @CMAKE_SOURCE_DIR@/src/core/transfer/gfal_transfer.h INPUT += @CMAKE_SOURCE_DIR@/src/core/logger/gfal_logger.h @CMAKE_SOURCE_DIR@/src/core/common/gfal_constants.h INPUT += @CMAKE_SOURCE_DIR@/src/core/common/gfal_common.h INPUT += @CMAKE_SOURCE_DIR@/src/core/common/gfal_plugin_interface.h INPUT += @CMAKE_SOURCE_DIR@/src/core/common/gfal_config.h # If the value of the INPUT tag contains directories, you can use the # FILE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp # and *.h) to filter out the source-files in the directories. If left # blank the following patterns are tested: # *.c *.cc *.cxx *.cpp *.c++ *.java *.ii *.ixx *.ipp *.i++ *.inl *.h *.hh *.hxx # *.hpp *.h++ *.idl *.odl *.cs *.php *.php3 *.inc *.m *.mm *.py FILE_PATTERNS = *.c *.h *.cpp *.hpp # The RECURSIVE tag can be used to turn specify whether or not subdirectories # should be searched for input files as well. Possible values are YES and NO. # If left blank NO is used. RECURSIVE = YES # The EXCLUDE tag can be used to specify files and/or directories that should # excluded from the INPUT source files. This way you can easily exclude a # subdirectory from a directory tree whose root is specified with the INPUT tag. EXCLUDE = # The EXCLUDE_SYMLINKS tag can be used select whether or not files or # directories that are symbolic links (a Unix filesystem feature) are excluded # from the input. EXCLUDE_SYMLINKS = NO # If the value of the INPUT tag contains directories, you can use the # EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude # certain files from those directories. Note that the wildcards are matched # against the file with absolute path, so to exclude all test directories # for example use the pattern */test/* EXCLUDE_PATTERNS = *.man */externals/* *gfal_posix_local_file* *srm* */common/*/* # The EXAMPLE_PATH tag can be used to specify one or more files or # directories that contain example code fragments that are included (see # the \include command). EXAMPLE_PATH = @CMAKE_CURRENT_SOURCE_DIR@/examples/ # If the value of the EXAMPLE_PATH tag contains directories, you can use the # EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp # and *.h) to filter out the source-files in the directories. If left # blank all files are included. EXAMPLE_PATTERNS = # If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be # searched for input files to be used with the \include or \dontinclude # commands irrespective of the value of the RECURSIVE tag. # Possible values are YES and NO. If left blank NO is used. EXAMPLE_RECURSIVE = NO # The IMAGE_PATH tag can be used to specify one or more files or # directories that contain image that are included in the documentation (see # the \image command). IMAGE_PATH = @CMAKE_CURRENT_SOURCE_DIR@/diagrams/ # The INPUT_FILTER tag can be used to specify a program that doxygen should # invoke to filter for each input file. Doxygen will invoke the filter program # by executing (via popen()) the command , where # is the value of the INPUT_FILTER tag, and is the name of an # input file. Doxygen will then use the output that the filter program writes # to standard output. If FILTER_PATTERNS is specified, this tag will be # ignored. INPUT_FILTER = # The FILTER_PATTERNS tag can be used to specify filters on a per file pattern # basis. Doxygen will compare the file name with each pattern and apply the # filter if there is a match. The filters are a list of the form: # pattern=filter (like *.cpp=my_cpp_filter). See INPUT_FILTER for further # info on how filters are used. If FILTER_PATTERNS is empty, INPUT_FILTER # is applied to all files. FILTER_PATTERNS = # If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using # INPUT_FILTER) will be used to filter the input files when producing source # files to browse (i.e. when SOURCE_BROWSER is set to YES). FILTER_SOURCE_FILES = NO #--------------------------------------------------------------------------- # configuration options related to source browsing #--------------------------------------------------------------------------- # If the SOURCE_BROWSER tag is set to YES then a list of source files will # be generated. Documented entities will be cross-referenced with these sources. # Note: To get rid of all source code in the generated output, make sure also # VERBATIM_HEADERS is set to NO. SOURCE_BROWSER = NO # Setting the INLINE_SOURCES tag to YES will include the body # of functions and classes directly in the documentation. INLINE_SOURCES = NO # Setting the STRIP_CODE_COMMENTS tag to YES (the default) will instruct # doxygen to hide any special comment blocks from generated source code # fragments. Normal C and C++ comments will always remain visible. STRIP_CODE_COMMENTS = YES # If the REFERENCED_BY_RELATION tag is set to YES (the default) # then for each documented function all documented # functions referencing it will be listed. REFERENCED_BY_RELATION = YES # If the REFERENCES_RELATION tag is set to YES (the default) # then for each documented function all documented entities # called/used by that function will be listed. REFERENCES_RELATION = YES # If the REFERENCES_LINK_SOURCE tag is set to YES (the default) # and SOURCE_BROWSER tag is set to YES, then the hyperlinks from # functions in REFERENCES_RELATION and REFERENCED_BY_RELATION lists will # link to the source code. Otherwise they will link to the documentstion. REFERENCES_LINK_SOURCE = NO # If the USE_HTAGS tag is set to YES then the references to source code # will point to the HTML generated by the htags(1) tool instead of doxygen # built-in source browser. The htags tool is part of GNU's global source # tagging system (see http://www.gnu.org/software/global/global.html). You # will need version 4.8.6 or higher. USE_HTAGS = NO # If the VERBATIM_HEADERS tag is set to YES (the default) then Doxygen # will generate a verbatim copy of the header file for each class for # which an include is specified. Set to NO to disable this. VERBATIM_HEADERS = NO #--------------------------------------------------------------------------- # configuration options related to the alphabetical class index #--------------------------------------------------------------------------- # If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index # of all compounds will be generated. Enable this if the project # contains a lot of classes, structs, unions or interfaces. ALPHABETICAL_INDEX = NO # If the alphabetical index is enabled (see ALPHABETICAL_INDEX) then # the COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns # in which this list will be split (can be a number in the range [1..20]) COLS_IN_ALPHA_INDEX = 5 # In case all classes in a project start with a common prefix, all # classes will be put under the same header in the alphabetical index. # The IGNORE_PREFIX tag can be used to specify one or more prefixes that # should be ignored while generating the index headers. IGNORE_PREFIX = #--------------------------------------------------------------------------- # configuration options related to the HTML output #--------------------------------------------------------------------------- # If the GENERATE_HTML tag is set to YES (the default) Doxygen will # generate HTML output. GENERATE_HTML = YES # The HTML_OUTPUT tag is used to specify where the HTML docs will be put. # If a relative path is entered the value of OUTPUT_DIRECTORY will be # put in front of it. If left blank `html' will be used as the default path. HTML_OUTPUT = html # The HTML_FILE_EXTENSION tag can be used to specify the file extension for # each generated HTML page (for example: .htm,.php,.asp). If it is left blank # doxygen will generate files with .html extension. HTML_FILE_EXTENSION = .html # The HTML_HEADER tag can be used to specify a personal HTML header for # each generated HTML page. If it is left blank doxygen will generate a # standard header. HTML_HEADER = # The HTML_FOOTER tag can be used to specify a personal HTML footer for # each generated HTML page. If it is left blank doxygen will generate a # standard footer. HTML_FOOTER = # The HTML_STYLESHEET tag can be used to specify a user-defined cascading # style sheet that is used by each HTML page. It can be used to # fine-tune the look of the HTML output. If the tag is left blank doxygen # will generate a default style sheet. Note that doxygen will try to copy # the style sheet file to the HTML output directory, so don't put your own # stylesheet in the HTML output directory as well, or it will be erased! HTML_STYLESHEET = # If the HTML_ALIGN_MEMBERS tag is set to YES, the members of classes, # files or namespaces will be aligned in HTML using tables. If set to # NO a bullet list will be used. HTML_ALIGN_MEMBERS = YES # If the GENERATE_HTMLHELP tag is set to YES, additional index files # will be generated that can be used as input for tools like the # Microsoft HTML help workshop to generate a compressed HTML help file (.chm) # of the generated HTML documentation. GENERATE_HTMLHELP = NO # If the GENERATE_HTMLHELP tag is set to YES, the CHM_FILE tag can # be used to specify the file name of the resulting .chm file. You # can add a path in front of the file if the result should not be # written to the html output directory. CHM_FILE = # If the GENERATE_HTMLHELP tag is set to YES, the HHC_LOCATION tag can # be used to specify the location (absolute path including file name) of # the HTML help compiler (hhc.exe). If non-empty doxygen will try to run # the HTML help compiler on the generated index.hhp. HHC_LOCATION = # If the GENERATE_HTMLHELP tag is set to YES, the GENERATE_CHI flag # controls if a separate .chi index file is generated (YES) or that # it should be included in the master .chm file (NO). GENERATE_CHI = NO # If the GENERATE_HTMLHELP tag is set to YES, the BINARY_TOC flag # controls whether a binary table of contents is generated (YES) or a # normal table of contents (NO) in the .chm file. BINARY_TOC = NO # The TOC_EXPAND flag can be set to YES to add extra items for group members # to the contents of the HTML help documentation and to the tree view. TOC_EXPAND = NO # The DISABLE_INDEX tag can be used to turn on/off the condensed index at # top of each HTML page. The value NO (the default) enables the index and # the value YES disables it. DISABLE_INDEX = NO # This tag can be used to set the number of enum values (range [1..20]) # that doxygen will group on one line in the generated HTML documentation. ENUM_VALUES_PER_LINE = 4 # If the GENERATE_TREEVIEW tag is set to YES, a side panel will be # generated containing a tree-like index structure (just like the one that # is generated for HTML Help). For this to work a browser that supports # JavaScript, DHTML, CSS and frames is required (for instance Mozilla 1.0+, # Netscape 6.0+, Internet explorer 5.0+, or Konqueror). Windows users are # probably better off using the HTML help feature. GENERATE_TREEVIEW = NO # If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be # used to set the initial width (in pixels) of the frame in which the tree # is shown. TREEVIEW_WIDTH = 250 #--------------------------------------------------------------------------- # configuration options related to the LaTeX output #--------------------------------------------------------------------------- # If the GENERATE_LATEX tag is set to YES (the default) Doxygen will # generate Latex output. GENERATE_LATEX = YES # The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put. # If a relative path is entered the value of OUTPUT_DIRECTORY will be # put in front of it. If left blank `latex' will be used as the default path. LATEX_OUTPUT = latex # The LATEX_CMD_NAME tag can be used to specify the LaTeX command name to be # invoked. If left blank `latex' will be used as the default command name. LATEX_CMD_NAME = latex # The MAKEINDEX_CMD_NAME tag can be used to specify the command name to # generate index for LaTeX. If left blank `makeindex' will be used as the # default command name. MAKEINDEX_CMD_NAME = makeindex # If the COMPACT_LATEX tag is set to YES Doxygen generates more compact # LaTeX documents. This may be useful for small projects and may help to # save some trees in general. COMPACT_LATEX = NO # The PAPER_TYPE tag can be used to set the paper type that is used # by the printer. Possible values are: a4, a4wide, letter, legal and # executive. If left blank a4wide will be used. PAPER_TYPE = a4wide # The EXTRA_PACKAGES tag can be to specify one or more names of LaTeX # packages that should be included in the LaTeX output. EXTRA_PACKAGES = # The LATEX_HEADER tag can be used to specify a personal LaTeX header for # the generated latex document. The header should contain everything until # the first chapter. If it is left blank doxygen will generate a # standard header. Notice: only use this tag if you know what you are doing! LATEX_HEADER = # If the PDF_HYPERLINKS tag is set to YES, the LaTeX that is generated # is prepared for conversion to pdf (using ps2pdf). The pdf file will # contain links (just like the HTML output) instead of page references # This makes the output suitable for online browsing using a pdf viewer. PDF_HYPERLINKS = NO # If the USE_PDFLATEX tag is set to YES, pdflatex will be used instead of # plain latex in the generated Makefile. Set this option to YES to get a # higher quality PDF documentation. USE_PDFLATEX = NO # If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \\batchmode. # command to the generated LaTeX files. This will instruct LaTeX to keep # running if errors occur, instead of asking the user for help. # This option is also used when generating formulas in HTML. LATEX_BATCHMODE = NO # If LATEX_HIDE_INDICES is set to YES then doxygen will not # include the index chapters (such as File Index, Compound Index, etc.) # in the output. LATEX_HIDE_INDICES = NO #--------------------------------------------------------------------------- # configuration options related to the RTF output #--------------------------------------------------------------------------- # If the GENERATE_RTF tag is set to YES Doxygen will generate RTF output # The RTF output is optimized for Word 97 and may not look very pretty with # other RTF readers or editors. GENERATE_RTF = NO # The RTF_OUTPUT tag is used to specify where the RTF docs will be put. # If a relative path is entered the value of OUTPUT_DIRECTORY will be # put in front of it. If left blank `rtf' will be used as the default path. RTF_OUTPUT = rtf # If the COMPACT_RTF tag is set to YES Doxygen generates more compact # RTF documents. This may be useful for small projects and may help to # save some trees in general. COMPACT_RTF = NO # If the RTF_HYPERLINKS tag is set to YES, the RTF that is generated # will contain hyperlink fields. The RTF file will # contain links (just like the HTML output) instead of page references. # This makes the output suitable for online browsing using WORD or other # programs which support those fields. # Note: wordpad (write) and others do not support links. RTF_HYPERLINKS = NO # Load stylesheet definitions from file. Syntax is similar to doxygen's # config file, i.e. a series of assignments. You only have to provide # replacements, missing definitions are set to their default value. RTF_STYLESHEET_FILE = # Set optional variables used in the generation of an rtf document. # Syntax is similar to doxygen's config file. RTF_EXTENSIONS_FILE = #--------------------------------------------------------------------------- # configuration options related to the man page output #--------------------------------------------------------------------------- # If the GENERATE_MAN tag is set to YES (the default) Doxygen will # generate man pages GENERATE_MAN = YES # The MAN_OUTPUT tag is used to specify where the man pages will be put. # If a relative path is entered the value of OUTPUT_DIRECTORY will be # put in front of it. If left blank `man' will be used as the default path. MAN_OUTPUT = man # The MAN_EXTENSION tag determines the extension that is added to # the generated man pages (default is the subroutine's section .3) MAN_EXTENSION = .3 # If the MAN_LINKS tag is set to YES and Doxygen generates man output, # then it will generate one additional man file for each entity # documented in the real man page(s). These additional files # only source the real man page, but without them the man command # would be unable to find the correct page. The default is NO. MAN_LINKS = NO #--------------------------------------------------------------------------- # configuration options related to the XML output #--------------------------------------------------------------------------- # If the GENERATE_XML tag is set to YES Doxygen will # generate an XML file that captures the structure of # the code including all documentation. GENERATE_XML = YES #e XML_OUTPUT tag is used to specify where the XML pages will be put. # If a relative path is entered the value of OUTPUT_DIRECTORY will be # put in front of it. If left blank `xml' will be used as the default path. XML_OUTPUT = xml # The XML_SCHEMA tag can be used to specify an XML schema, # which can be used by a validating XML parser to check the # syntax of the XML files. XML_SCHEMA = # The XML_DTD tag can be used to specify an XML DTD, # which can be used by a validating XML parser to check the # syntax of the XML files. XML_DTD = # If the XML_PROGRAMLISTING tag is set to YES Doxygen will # dump the program listings (including syntax highlighting # and cross-referencing information) to the XML output. Note that # enabling this will significantly increase the size of the XML output. XML_PROGRAMLISTING = YES #--------------------------------------------------------------------------- # configuration options for the AutoGen Definitions output #--------------------------------------------------------------------------- # If the GENERATE_AUTOGEN_DEF tag is set to YES Doxygen will # generate an AutoGen Definitions (see autogen.sf.net) file # that captures the structure of the code including all # documentation. Note that this feature is still experimental # and incomplete at the moment. GENERATE_AUTOGEN_DEF = NO #--------------------------------------------------------------------------- # configuration options related to the Perl module output #--------------------------------------------------------------------------- # If the GENERATE_PERLMOD tag is set to YES Doxygen will # generate a Perl module file that captures the structure of # the code including all documentation. Note that this # feature is still experimental and incomplete at the # moment. GENERATE_PERLMOD = NO # If the PERLMOD_LATEX tag is set to YES Doxygen will generate # the necessary Makefile rules, Perl scripts and LaTeX code to be able # to generate PDF and DVI output from the Perl module output. PERLMOD_LATEX = NO # If the PERLMOD_PRETTY tag is set to YES the Perl module output will be # nicely formatted so it can be parsed by a human reader. This is useful # if you want to understand what is going on. On the other hand, if this # tag is set to NO the size of the Perl module output will be much smaller # and Perl will parse it just the same. PERLMOD_PRETTY = YES # The names of the make variables in the generated doxyrules.make file # are prefixed with the string contained in PERLMOD_MAKEVAR_PREFIX. # This is useful so different doxyrules.make files included by the same # Makefile don't overwrite each other's variables. PERLMOD_MAKEVAR_PREFIX = #--------------------------------------------------------------------------- # Configuration options related to the preprocessor #--------------------------------------------------------------------------- # If the ENABLE_PREPROCESSING tag is set to YES (the default) Doxygen will # evaluate all C-preprocessor directives found in the sources and include # files. ENABLE_PREPROCESSING = YES # If the MACRO_EXPANSION tag is set to YES Doxygen will expand all macro # names in the source code. If set to NO (the default) only conditional # compilation will be performed. Macro expansion can be done in a controlled # way by setting EXPAND_ONLY_PREDEF to YES. MACRO_EXPANSION = NO # If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES # then the macro expansion is limited to the macros specified with the # PREDEFINED and EXPAND_AS_DEFINED tags. EXPAND_ONLY_PREDEF = NO # If the SEARCH_INCLUDES tag is set to YES (the default) the includes files # in the INCLUDE_PATH (see below) will be search if a #include is found. SEARCH_INCLUDES = YES # The INCLUDE_PATH tag can be used to specify one or more directories that # contain include files that are not input files but should be processed by # the preprocessor. INCLUDE_PATH = # You can use the INCLUDE_FILE_PATTERNS tag to specify one or more wildcard # patterns (like *.h and *.hpp) to filter out the header-files in the # directories. If left blank, the patterns specified with FILE_PATTERNS will # be used. INCLUDE_FILE_PATTERNS = # The PREDEFINED tag can be used to specify one or more macro names that # are defined before the preprocessor is started (similar to the -D option of # gcc). The argument of the tag is a list of macros of the form: name # or name=definition (no spaces). If the definition and the = are # omitted =1 is assumed. To prevent a macro definition from being # undefined via #undef or recursively expanded use the := operator # instead of the = operator. PREDEFINED = # If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then # this tag can be used to specify a list of macro names that should be expanded. # The macro definition that is found in the sources will be used. # Use the PREDEFINED tag if you want to use a different macro definition. EXPAND_AS_DEFINED = # If the SKIP_FUNCTION_MACROS tag is set to YES (the default) then # doxygen's preprocessor will remove all function-like macros that are alone # on a line, have an all uppercase name, and do not end with a semicolon. Such # function macros are typically used for boiler-plate code, and will confuse # the parser if not removed. SKIP_FUNCTION_MACROS = YES #--------------------------------------------------------------------------- # Configuration::additions related to external references #--------------------------------------------------------------------------- # The TAGFILES option can be used to specify one or more tagfiles. # Optionally an initial location of the external documentation # can be added for each tagfile. The format of a tag file without # this location is as follows: # TAGFILES = file1 file2 ... # Adding location for the tag files is done as follows: # TAGFILES = file1=loc1 "file2 = loc2" ... # where "loc1" and "loc2" can be relative or absolute paths or # URLs. If a location is present for each tag, the installdox tool # does not have to be run to correct the links. # Note that each tag file must have a unique name # (where the name does NOT include the path) # If a tag file is not located in the directory in which doxygen # is run, you must also specify the path to the tagfile here. TAGFILES = # When a file name is specified after GENERATE_TAGFILE, doxygen will create # a tag file that is based on the input files it reads. GENERATE_TAGFILE = # If the ALLEXTERNALS tag is set to YES all external classes will be listed # in the class index. If set to NO only the inherited external classes # will be listed. ALLEXTERNALS = NO # If the EXTERNAL_GROUPS tag is set to YES all external groups will be listed # in the modules index. If set to NO, only the current project's groups will # be listed. EXTERNAL_GROUPS = YES # The PERL_PATH should be the absolute path and name of the perl script # interpreter (i.e. the result of `which perl'). PERL_PATH = /usr/bin/perl #--------------------------------------------------------------------------- # Configuration options related to the dot tool #--------------------------------------------------------------------------- # If the CLASS_DIAGRAMS tag is set to YES (the default) Doxygen will # generate a inheritance diagram (in HTML, RTF and LaTeX) for classes with base # or super classes. Setting the tag to NO turns the diagrams off. Note that # this option is superseded by the HAVE_DOT option below. This is only a # fallback. It is recommended to install and use dot, since it yields more # powerful graphs. CLASS_DIAGRAMS = YES # If set to YES, the inheritance and collaboration graphs will hide # inheritance and usage relations if the target is undocumented # or is not a class. HIDE_UNDOC_RELATIONS = YES # If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is # available from the path. This tool is part of Graphviz, a graph visualization # toolkit from AT&T and Lucent Bell Labs. The other options in this section # have no effect if this option is set to NO (the default) HAVE_DOT = NO # If the CLASS_GRAPH and HAVE_DOT tags are set to YES then doxygen # will generate a graph for each documented class showing the direct and # indirect inheritance relations. Setting this tag to YES will force the # the CLASS_DIAGRAMS tag to NO. CLASS_GRAPH = YES # If the COLLABORATION_GRAPH and HAVE_DOT tags are set to YES then doxygen # will generate a graph for each documented class showing the direct and # indirect implementation dependencies (inheritance, containment, and # class references variables) of the class with other documented classes. COLLABORATION_GRAPH = YES # If the GROUP_GRAPHS and HAVE_DOT tags are set to YES then doxygen # will generate a graph for groups, showing the direct groups dependencies GROUP_GRAPHS = YES # If the UML_LOOK tag is set to YES doxygen will generate inheritance and # collaboration diagrams in a style similar to the OMG's Unified Modeling # Language. UML_LOOK = NO # If set to YES, the inheritance and collaboration graphs will show the # relations between templates and their instances. TEMPLATE_RELATIONS = NO # If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDE_GRAPH, and HAVE_DOT # tags are set to YES then doxygen will generate a graph for each documented # file showing the direct and indirect include dependencies of the file with # other documented files. INCLUDE_GRAPH = YES # If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDED_BY_GRAPH, and # HAVE_DOT tags are set to YES then doxygen will generate a graph for each # documented header file showing the documented files that directly or # indirectly include this file. INCLUDED_BY_GRAPH = YES # If the CALL_GRAPH and HAVE_DOT tags are set to YES then doxygen will # generate a call dependency graph for every global function or class method. # Note that enabling this option will significantly increase the time of a run. # So in most cases it will be better to enable call graphs for selected # functions only using the \callgraph command. CALL_GRAPH = NO # If the CALLER_GRAPH and HAVE_DOT tags are set to YES then doxygen will # generate a caller dependency graph for every global function or class method. # Note that enabling this option will significantly increase the time of a run. # So in most cases it will be better to enable caller graphs for selected # functions only using the \callergraph command. CALLER_GRAPH = NO # If the GRAPHICAL_HIERARCHY and HAVE_DOT tags are set to YES then doxygen # will graphical hierarchy of all classes instead of a textual one. GRAPHICAL_HIERARCHY = YES # If the DIRECTORY_GRAPH, SHOW_DIRECTORIES and HAVE_DOT tags are set to YES # then doxygen will show the dependencies a directory has on other directories # in a graphical way. The dependency relations are determined by the #include # relations between the files in the directories. DIRECTORY_GRAPH = YES # The DOT_IMAGE_FORMAT tag can be used to set the image format of the images # generated by dot. Possible values are png, jpg, or gif # If left blank png will be used. DOT_IMAGE_FORMAT = png # The tag DOT_PATH can be used to specify the path where the dot tool can be # found. If left blank, it is assumed the dot tool can be found in the path. DOT_PATH = # The DOTFILE_DIRS tag can be used to specify one or more directories that # contain dot files that are included in the documentation (see the # \dotfile command). DOTFILE_DIRS = # The MAX_DOT_GRAPH_WIDTH tag can be used to set the maximum allowed width # (in pixels) of the graphs generated by dot. If a graph becomes larger than # this value, doxygen will try to truncate the graph, so that it fits within # the specified constraint. Beware that most browsers cannot cope with very # large images. MAX_DOT_GRAPH_WIDTH = 1024 # The MAX_DOT_GRAPH_HEIGHT tag can be used to set the maximum allows height # (in pixels) of the graphs generated by dot. If a graph becomes larger than # this value, doxygen will try to truncate the graph, so that it fits within # the specified constraint. Beware that most browsers cannot cope with very # large images. MAX_DOT_GRAPH_HEIGHT = 1024 # The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the # graphs generated by dot. A depth value of 3 means that only nodes reachable # from the root by following a path via at most 3 edges will be shown. Nodes # that lay further from the root node will be omitted. Note that setting this # option to 1 or 2 may greatly reduce the computation time needed for large # code bases. Also note that a graph may be further truncated if the graph's # image dimensions are not sufficient to fit the graph (see MAX_DOT_GRAPH_WIDTH # and MAX_DOT_GRAPH_HEIGHT). If 0 is used for the depth value (the default), # the graph is not depth-constrained. MAX_DOT_GRAPH_DEPTH = 0 # Set the DOT_TRANSPARENT tag to YES to generate images with a transparent # background. This is disabled by default, which results in a white background. # Warning: Depending on the platform used, enabling this option may lead to # badly anti-aliased labels on the edges of a graph (i.e. they become hard to # read). DOT_TRANSPARENT = NO # Set the DOT_MULTI_TARGETS tag to YES allow dot to generate multiple output # files in one run (i.e. multiple -o and -T options on the command line). This # makes dot run faster, but since only newer versions of dot (>1.8.10) # support this, this feature is disabled by default. DOT_MULTI_TARGETS = NO # If the GENERATE_LEGEND tag is set to YES (the default) Doxygen will # generate a legend page explaining the meaning of the various boxes and # arrows in the dot generated graphs. GENERATE_LEGEND = YES # If the DOT_CLEANUP tag is set to YES (the default) Doxygen will # remove the intermediate dot files that are used to generate # the various graphs. DOT_CLEANUP = YES #--------------------------------------------------------------------------- # Configuration::additions related to the search engine #--------------------------------------------------------------------------- # The SEARCHENGINE tag specifies whether or not a search engine should be # used. If set to NO the values of all tags below this one will be ignored. SEARCHENGINE = NO gfal2-v2.23.0/doc/diagrams/000077500000000000000000000000001465240014500153145ustar00rootroot00000000000000gfal2-v2.23.0/doc/diagrams/gfal_access.dia000066400000000000000000000076231465240014500202350ustar00rootroot00000000000000]r0?+ 3n&m:MNl@$Dmk:=b=gI]xbAfp~~eyre%ß^޾vo߸'"mUM\\|dw*Js2K.gY|D<qUJ<$x۸,Z (OqvF.\ciΒG&c8 xM*=Gjz%aQ6她jSf[NMmu+y ߎMe X&l}]xT2·[4kOh5'>8*TWcC5cfQWlP'Ɩƌu&lPQ8% +-}VG%RϑHA >ZQI~I>l3W5xO* _N>LKel8ͣ>@4xd6=TI [FE/?K]I&[4͸ZYnJFÎҳ|6GWaz 6Ί-Rb黮ҴbrS3In>iY ӀZkbMO_sh.Ь56݇T6RWM%U:o=e$Wl+,QA?%B1[ 6mY:'oưi,5v0E!A>V1HPj&]!C5u2¹8H:H9U1OxfU2qy mXS;0nDW%:m"zhLhb34 %p 17޼̼(a(!Xu%C!Q(OP؋ļu(1`XM'J;ccMPS"CGNhtTkt/^E3CfS䌨نLq?1YB&+3wn*cY" [&f]FF27wڼq^> a: K I:lna#ek$r+2w@unАW$zhH X+ ǘgHG Dc6P 'b mS0tMDQ4u/\ReeY.Io;I0 GKSYwI"w\Rp( Oab̨aT8yn*io[0 Hǽ6=9qȜ|}w9=@BunBrP^^cUp ;^N_ުQE< 3,-fbZ Է4$vkB]{yW^I1J_J_'"]V:1,) BQBkۥ/MNy{D`2"kl鷯b>IGmլGiԵQK,<ԢK- -l0fnm,"׌$7 p.FAQؾ``5+p`:y[kںݮRr ^pI~X[AI;m3')M Q/(Q]Գ) -pNEXq)ڽ;zdwQ>@0PA_ E& AH"(:F%͆s{wњ5n}0ݬ=l{?Jy0hx^>eݵikPa}%sχxl+EgٺXP{C|}N]4k>$`gvb$48ѡ@]}}i;,ndev%B.g738]u hTx"#P/P} 3]ʮ}#h јmb%Vd;J9% bnp5ѓp#HsȠYl&[f@wsڗP`c/[çMh$&Ndjj$ ]2")BKs'~7NMa䚂cjYn S!^n"dCu.> ۙM |6eМ,rxVpxDxI#`gvbf2肽U,8i*|7C("q-|Yʾd)ڵ m?ycB)\e}]RVjۨuOKD~S5M;t^f~)T#䂓OD?r:Aƒk_1 GJ(T2_BX{o[r™Ki\/35Dy{gA"mEr C¢˗Ոoe3rBh+DtMBP34vjޝ2n*{Zvy|p&˾.M/:c4 lJPٔlW[o<*ǼesQѦ؉1Q9At$:ˀ(2`#n ˱Pc5*P#!9V°G뮝#"W*#LЉDmxDDzG$ X5%}/K#t[1I`і1?=vcLjeʲ;Iͬ'wv;^ҖO-Gq93zYкoY kRmgiļeծKsh'D+D.]񮼨x9w3b^KLbpv!UОՉ]Vy`|x*@z!\ yQr$j? [Q gzb7h$@FE|7(C D~`ٓAH:sC:4@gnnt($d`W.-t(ߋnwV$qYYОىU@>?\+ [X/H9%]F-Rjr}pS:oR2 ZxnZU ki]V9r bH< UKn: ZH,*l>3Fr`AO<9в nkġ-ldl+Bȱ+s4IҶDPhMR"]½ VsIc%%~:չFرc%{ř?;n޼y_fʕ+Eة8͚5KgVlllmo>۷O3gT۶m5m4#vءǛ ,{L*TC ۅ w)::Z#Fڲel٢Yfiɒ%jРAt ۍ7A`g.\-[*44|̙#IZnKY,ԓa_7o֡CtYB Mz'z}(h˖-x_.)uZZjk׮ѣ .l^sarU_}]ӦMլY3s?44 6+f5Onݪ3gHV]y/k]dI=V٣cǎI6l-Z碣rJIҵk.]^zꩧUV,WW՛~\ñ{zrt-@2`ϟoH2̟?GvnjuÍRJe{1bڵkK/M0|g}pss3 :Ըzj6^|EPBi?~<-[fT\9~ ,hkFBBB}%$$Ǐ7r8pZjᑣ~)bDFF3N:5t&}Vvlٰa5I-ꫯ~,ѧO#,,,}߸qØ ԩS5d;w<^T)ִ1n'|^z)>> oܸ>}ӠY,1Bՙ3gWÚ>}Up/B3g4﯀Z###{!W_}5|||vZILӧOҥK:xXmܸQÇt[nifֱcGի:v+&&F3Ly?رjzݻۤZ*UrtMҥ}O:tЗ_~`EEEѥKiӦ_s^ȲD 0ZRz)ٳG׮]ӹsh?~\~arҤI2 CN:~ ? {ｗg.^z)s…9nѢEfԤIկ_?WWYsjׯgyUP!Iǵk׮lw9رcsU/@WEDDX^sĉl2_ggg#""m9}UVŐd*T(W^:˶L>^7n˲r%п=㒤d.""B7ǏiҜFOitwy'~ÇKJnܹ\pn۶mIR…3;ݬY39;;gۏty>3{~jbŊms2رջjժLkΜ9AgUdb+]tQJ$V石lgN5_y^^|Es_~ѕ+W611Q ,0<,Wų0 ;6?7o̫ԣGs;^<ի')ӦM%&&jɒ%]>lp>w]_tlRWIIIwW^  aÆ٬׵ʩu r#'of2%O1bԇ;_!B۶mհaCIŋvƍŋ]/bye>ȗ]|YVttUdn?yr[ ӱcW*11Ucǎ>ӆ =hs ͙3G6l0+W^f͚S]2 .ӧZ'''3 y͞=[111JLLԔ)S;C޽7o-[@zQzuOk֬$_^ׯWFԳgOlRڵ[nm߾]{6Gխ[W~~~*QM>,uiut}QHHHPF nZ{$-ZH|Iv+W4gwww֙ވ#W׮]ǵsNuٳgoia`dKg8i&6-sW_)m͛_~\W+ 涇=enj֬Ywacrj* 4ȼG||6oެ͛7KJ}{쩡C'c ,PDDve;x9JQmڴѓO>>GSxދ]vO>fTZ5mٲ%O+Z>NZ"n0[Yr}Y:r~˨Qkɒ%9sڤpذaOŊW_IJu{`scf2<@%TbEsR@@@K? ]Gڴ4k֬ф )+TiӦ𐇇J.m|ryRڰaP\9 2|2eX=Ú5k4z,L?Mbv#>(\ʕ+wWdK.]($$D?V^{8Rם[t.]N:i*_|~J*;vhÆ Zt6md%''kΝڹsO}sWϑC_~V\9믙m e˖5s:w5[S.]ZC QULO vڥ}f٧-e8q͛7[!!!Vn~Nyfe~͙3 uwߙmǎ_eyզM-[LRHx3oahfh4tP*TPZh޸qc5n8_Lnݺj۶vڥ[nizW%I>>>|g:th~ ؜C~ҥ} 3{:t\'Y_u1ѣG,I*XlŜҤ=;o<37oy|ذaVSr N :TnnnG}7ocEӧZj5kyP9sܮ^(;3i޾}@Im۶5oܸa5 [^kժ6nܨMvؑ~ʔ)>@c͂#GK.fp=OG|ڵ(??l˫~yR[fΞ=ksssSjղ&}rZ{/CYܹ*U$)u=`}v;wNRꔡÆ =?<׀TLIǵsN:uJ[l1B (Z&L`;vLSLNJsڵkٶΝ;\3h۶mٶKŋ/]JQ~O>NNNڵ}}NJJRll=&~OtYW\~Ֆ-[TV{;'*Tf͚}F7G Jwƍن/Zzu\+666!;ŊO4rHs?mWQ_{6ղ# ,hyߛ#հa{/x~ؑ)SI&ǛkrʙgΜѥK611Qcǎ5G<(??c풓5vhܸ:wlO8||7M6´u$#NS*TH%J Qn̩K*%___5hkܸq7n\sښKu,â< (W^y%˾;VcǎդIrK+00m m*UiӦw]gɒ%֖zjwcԨQf={hժUVm)irk5k~0ύ3 P_vpZr_}իe˖wڵkZdfΜi󺼽ͩ ШQ]DDﯝ;wu ///GWfhj۶m9~9sH"Rãvڙk-eŋzu  ޽{cǎӳ/[L]tѧ~|ӦM5ad˗/ҥKuH{ｧݻkժUYaO zajܸt钺wn>}Ce͛_믿vӠAԨQ#s矷0Æ S͚5{֭7o͛g5nUZUիW7_xL)~{9X"~h~?]vM:tЯzzw} ( ///sV!-U^]-[4{9sF777˦kѢ=|纏UߋgSdIN`I҉'4tP(P@7V DEFF:qDux wPBzW5}tIY ڵk0 :tHk֬Qtt\\\4a}|o[qrrɓ5uTI5ըQCTڵeXta^ZQQQ*Z&Mdv՝4h@ ,ȑ#3gΨ]vR6m)]|Yо}gA?:v@%''_|=zaÆf{wG՗_~/RժUS˖-UN.]Z t|||t:d0 mݺU[nZjMS TXX/mذ5?0_[;(ʔ)c0AK.U˖-u5EEEK.֭4hd޽*hU̙cԩSi/լYS R:u`^Z/_z-cV~0=_:t蠲e˪@ԟݻw+))I֭˴cǚlٲrvvjd=z\ 5j>CݼyS/_Vd5գnݚe?/||| 6L6xP~ء5jh߾}zwߚS%%%?^^^zw֣Wo9+VHJ_ghWT)-]4&_WPP~'I#վ J(~)WT :TJ҈#5F喫vڥ1chRe˖iٲewbXnLHHBBBlөS'^:Z111ڼy6o|6Zbj׮6uѺu4|ps4ZZ(x&Mhʕ91Zt,=:>̼ǂ 2+Rϟe=zh۶mP``z|%KT߾}믫N::xY.]:۾[j'N諯իu #P˔)6mh޽{X,}z饗j*'\NM&+W.Wɐ$U\Y^>l>W/xȽzw$IׯO<ၲ|r 2DRԲ.\P"E* x8~~~-[|aq|~C,99YIIIһﾛbf̘!ISN0!^.p{.e˖iJIIщ'ta܄ ᑏR``vy+WNAE9+Wf8޶m[FA}vM4iӦ R"ETzu 0@'OV…$ N̘1C3f2|e1 "[3 C)))܏b *v v v v v v v v v v v v v v v v v v v v v v v v v v v v v v v v v v v v v v v v v v v v v v v v v v v v v v v v v v v v v v v v v v v v@~(99Y:s.]$I*WT"///9::s2C@:AAA={|||tʕL۸'ԤIT^\!X 0[DD^{5-YD)))9Q#F,77<@N~y۷BCCkڵj֬+[_Gڑ#GԦMS?ڻw֭k /#+<<\-Zйsl_ʕuyxxؤ?_&OlKΝ;)Sج?/#i޽j۶lAŢy6@0 Hꫯl|Ia/l/as-yxx(&&&O/YU@<1 ϳKg3/#ٳy~3g=dDx\t)s?dIM ~9ʕ{TP! #/#J*y~{, #LGLrrʖ-+WIx7&oGGG; |/#_lگTfM ggGRZ4b;rH/ 1 ȊUVtW^=rqqIr_G/_.WW{U+V V^=N:wGݺuﯺuڰ2w ȫ^vܩ#G!ittt3<;wzyX!b/ ٳ+Wd]O>&Mz d L$''_׺u$IO=%GG|@fթS'I:v옿k~n~n~n~n~n~n~n~n~n~n~n~n~n~n~n~n~n~n~n~n~n~n~n~n~n~n~n~n~`ݫ͛UEΝz뭷򰺇s=~꘿UxqY,;vL<իgٶm ,]駟ֳ>e;U g իkݺurqqQٲes|[><<ŋgy(QB֭SUJB f%&&*%%~@>}4iW^yE7r!… 9s:v<>CujDWll9SN>(?J|ԯ__۷ׇ~˗w9~ ܹsuIըQC'NԞ={tu7N6m%I+WԡCTF =ZgΜњ5k(UREW׮]辶WG;v?#OOOmV< ߍ(͟?_n޼ujjѢԩSua999I&6lVjٱcVX/_^ ?num۶駟~ŋ h̘1\$ܹs;w=*0TbE < "##vZ͝;s֭Zn$iڵ:u==<<ꫯfy9sFסC쬮]gQBnjӦ\\\*<<\M6դIfŋ7oo(P@;vѣUhQom۪DZ`Ο?ٳgúxF?X*_|MURE'N'|ϫE:u .l{q}W:u,TÇl3h +Pҥ|n`/"jѢ6mڤ5k*&&Fڵ%KH#Iqㆢ+Iz7tR^z:v옺uoZjMD5nX7nj.88X 4DYom6lؠ hʕTʕ+'}wfO>D;vÇU^=]|Y{֌36}vh5iD+W믿JJ 4iM6nݺS"##5{lm۶)!!j͛77..N5kXՙ={aÆ_cXb8qz,={>CW VŊ_m۶vjԨ.\MӦMSNg=z(::ZJHHƍ_SN:~<==~zmV[lQǎ%J>аa>դIݻWWZt XNm۶ef;3$ ??.ܹQB#22m6..DF͍k׮e&00ppp0} ԩSaF۶m~e5))0 Ø={Qpaƍ7 Ø4iba Z5jQn]s˖-$# 0 HHH0*VhtHLL4ٳptt4ϟ\\\ c汽{x7 0^zF-7o>l*TȘ9syXb0 cȑ$c޼y"E2Ϛ5˰X,ƹs 0Se˖z>ð\,Z1iҤ, C"))Ivҳ>RJ dN㗝RJ0 ]tIʗ/+W(,,ksuuuap²X,:|$ڿ^yURzgggIүWjƌrqqɴ?B ^:/0 ݻWTP!8qBj(I*Xnݺ_Unp^BCC&rןMz۷oׅ 4e [n͛k׮]ѭ[7)"[jΝ;k͒KGiׯ]fG>}ԨQ )^ƌchB...ҥ4h`޽ Ppp5&&F۶mSrr.*IExb CQbbWnubFqF}zt /^\Ŋ3PyzzUm!!!z+WWW9;;aUhhKRAK9qTl,1cz)5nX5jP>}4l0yyyIFKj*Yz~irpHAqqqVkYݫ'OJo#3ڵkg8VNs˴{3Fcǎjgn Sg\]]Ub Ǥu$iܸqZvz)K P>}2l5 #-~(Z$e$իW/m߾]R\)))w]رc?kpIRBB)b| t@(65tNJuiiZr>3}3f+;v( @[l5x`[>>>Ȥ?3wF,\P*Tp$͛7O ޽jۚ5ko> 8P԰aCnZ|?/ڵK111;99Y2?i=CCdƌ֭'www*UJҥKkS5h@z/.]W^jժ܋ɓ'kڵjԨ:uv*XUsJ*ܹZlzJիWѣ%I ҥK szUB}WRרZd|}}Uzu[O>j׮VZs=+{4h6lKo$駟~RŊզM=ڵ|MK*S$EU6+GGG^Z_ުZva?j*3PLSOi̘1޽u]vz6K.6m:hjժ*W 6Ydܹ*_ڷoA}={Mf]VjՒw=ߨ;N8aM6QLqrDRL+sm*[M,K2Y556V6e鸍Rj["p?,A).9}>zϩٺU:u$mٲE;vlٹs.\@kNjӦ>cI7[fڵkgl-\P{1թSG[nU.]%I0lٲԙ֮]Kb5ilkl-]T{Qnn4ib* $%%iɒ%:r䈜ԢE u]1ZtN8!;;;5h@<򈂃%IOw}cǎ)==]ׯԩ#IJNN?*..NҥKHofϞgA+W3]OYX%%%k׮6lP=mKTff*""l͛yZhS%&&UV8p`1''G˗/מ={&uQڵ3=kٲejذv{n]vtVZEzzm۶S\m۶JHHz֭[e ҨQ4bI__yy^ IDATy70)**J ,-:eggI&СC嚗.]R:utQ_SL)w˼yoѣJ;?@%&&*""BzR@@~g}׿񬒣.\3gΔ5]]]~˚xxxh…_஠ȑ#g>1Ř:uX:2ՠ7ꫯR&M,ot*:=䓖XAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAq;u׿*<<\9~ʔ)jѢ,YR[|>S?~\_~ecSRR6m (J P :T'뭷RRR+lll%#ҡTyNNN=z[ ~gEv̙#IԳgrt(/*00P4u"W;~$i@P~ ZjzW%+))lW==#Fܾ+??ޭ&L`|~SZ5g;vLd٪Gyb0 !( .Cׯ_233%IV*0V~pjժi̘1d0++KԦM/cqׯxښ|r=N$6z$T|iӆ X@nܸPIbPY "8;;kђ֭[gϞN4-P9ԩS:y򤒒ׯ+//]cǎp*TU_PEHm۶M666Zz, VjݺN:%GGG:tHaaa*=Xhm&I1b(mHXU@VVBBB =zT+4|pkתk׮N_P,ZHqqqSR|ᮙXTM_`HԿ2s*22Rkז~͞=[׮]$egg롇ҥKh۷O=~r0ydEGG߱͛'___o^}Q>}1cܱk"##%I˗/XTM_`6o,Ijݺts#F_&LɓդIѣ݈RJIIȑ#4%%%)))TFM6M<<իK zg4n8i˖-:yZlYϟWegg-6tEyyyiرӄ ԧO5jH9992eN>SVZ$ǫCrttw}WݺuӚ5kԥKIҦM裏*22R._!CAj2ezڴi#[[[?^AAAZzz)_,࠱cǪ_~ W:u{q%$$W_U@@4m4uYG5}˖-յkWM4Iz࠺u[[[Õl(]tsڵt3g쬠Rccc[.u]:t+Whر1bfϞm:CjԨ.\ÇkΝ5aM44',RTTBCC|ryzzf֌ z͎uQիկ_?IҸqԺukZJvvvm۶tfCncCbtnVVV#㏛IZ$iݺuPǎuiӟ>^ڳg `ynZf/[Lmڴэ7>O.]҅ Ju?%;;[+W;Ｃcjrqqщ'$?~\ 0_ԦMs=߹?~@) X$QZZ222Rsnj༌ Iŋ%I=z(|///I72.ILL矗:$լYuASxnnrrrdggK.m۶W$)!!AU```qAAAr tկ_ȜΝ+v;ڷoqUV]\]]eccWJVzYi|nw+~NKK+ӹm۶U~~;`:u (g˖-ܩ*M/^իsΦ"4J4\\\\k$I6;vB=sE~a]9ok]qqqڴi>3-Xl$s._[b@y+a¦$_)S(771O.Ӝڵ$^q{rssnݺߜ388X[lQjj:t蠸2e*-[~PVV7SWm۶v>lv]vڴinܸq2;vL:t0q޽fbzyyN:nN:#G~ ح[իhܹSaH8N:dggΝ;eUÆ SBBFLiZnjz^֋/+V>S~~K/)33lq߿)wnn:3fܑ̡Z~$I'N m9j(]Vƍӑ#Gm6=䓥~ sm _`4h`e>Zl~uIQxx֯_2_SN8p\]].OOOK~[[N5j/WWW(""BKo?;ڵf͚?XիW kĉqcƌQ^SOMJKKSС>SرCAAArwwZhM6ݑo]`yxxI&z' =lذa7nyGSO=-Zy֕+W({m6Fջwo-Yw͑GӲQHH5jdZdjժ+$={V{ռysۛ1 C{ٳg⢖-[_4rHRvvd߭UZ5򔒒"777QYYYȐs®^NjٲBCC֮]tyiӦJOOW~~~`eeeiǎzjԨ{Gk.rJMMU>Tؘ=-99YwVVV^ժUKׯ_\\\- ]]]eoo`=c;wnynl2=㒤 6K.>#( 0@/V5P` ddd(..Tk?~V^]hZI_NNNJJJ*4=+׭[7I7W*[iP]zU 4PFԹsgիWOǏWttt}גnnHį{t#7(Ol{U>O>D:v6lhHbڵ? ]pcƶP5JפI,Vl$IÆ @cT}W_}%m޼Y;vt$XLԩS^~WըQұPŰ 3gEahȐ!JOOt$XI&ԩSX/XTu5myxqoq^Zf͒$nZNm 1 C<֬Y#I3fx Be?K.JOOۧ[:(V~@bccE),,L4c /!WϞ=.{{{}g_(/bj֬u) @4n89RyyyNfӦMԩ\"}ѣc*(88XׯWPP$魷҃> .X8*\M8Q]vekk9shРA/ԭ[7>|XI&iذap:TD;vСCu!I>#XUvڥ_|Q5h@WVVضmuۛMjϞ=_PX$mܸQ/NTn]y V͚5RFF<==eN-'''իWO...UJJik׮Iiذ$)>>&@/n /x*֭[?\7VXX7nVZҥKZdԼys꥗^aꫯOjڴwߙ]'!!A۷WƍմiSkժUӧW[VV秅 pႼG顇RPP7n,___\llZZ{9*""BŋMcۧp5nX],Ә [K.UPP |ocǎ ߯DhѦkhBc155U */з~HIR޽}9~ IDAT-ooomܸl;;;JF ʕ+5d666E߰aFWTg7n.]UV0`&NoF2eϟoVZz75m4]xQ̞=կ_?8qBW֥K'_.c8qL8izdܸqԹst խ[׬2 CzҒ%K4|?^ZdieW\\z!988h޽JHHЛo?Pх/kʔ):q>#SE:p85jhȑz4o<ٳ食~,_ 2~嗪Y"##M_$%%%K.f ʯYT)Uܼy I$c͚5'N4U6m IF@@SqҥѸqR_~!ɘ:u޽{m۶k6 fadddF޽=}aooo~C1|q!ɘ9safT^݈1o4o۷oҭ[7#00ٳ$fVZeH2gal޼ِd̙3عccc 㧟~2;>h HHH0 t!x͎ƌ3̎gggu1nalܸѐd̝;،ڵkcQF cРA%a/@O+66V/_k]kŊڵk$)::wv6Cرcz|'u֕ڵkg:fkk ]xQtaGf֫WO͛7Ν;M$)**lۧTeee?4{Mǎ+}IRN^H-bVm6IR߾}c׮]W4ibvW^?={}=P~|Qrrr{4si2֔)S4f?^[oFsuuULLƍQVRϞ= 3 C$___ 6k$0(GG2fA...mn=^e^JJ$ǧ|m/]T8___III{m(I/=($)''G,[[BynR}q5j9 yڻw{~~~Wi~;;"3խ''?<@o˚8qb߂nڿ$)&&FXzKԹsgի]/nuES`WJRg9rsW2GS```cL+nU 7uTEFF1//O,0zh=#׿v@EPK߯+Wo&M$?`pt?UV:w\\+<<\5j7|cvرc:|i֭[Kkq!///}'w/tt"IZpac"##uB+9;;m۶ӟ\bdffիjѢET1Mbcc裏cۜLmzW5zh 6LիW5j(yxxhᒤ wފ:v-[w1Yf /'xB>͛7Wƍq_~+$$)((H}x IRHHΝk<6oެ#F˓&Nh]!!!5$N!!!^zܯ|||4w\ 0;;;o^4[hCӧO7e|Ll2M>]orrrfV~|7n 1 ðt*4˪UVYL>]cǎP:`͛aÆ)!!A~~~v|]~]ʦ[+;;[)qTãPt)%%Errr*rLFF~K啸4sAhB˖-+Tf_fڴiEvN>]? ^5j`hBcǎU˖-- yf-X@?RSSu߽U#@UQ>~@%K/u{aÆQ|^zꫯut$T5kTDD&MǏS|+(XT*xMT|_S/rTl_cĈrvv3/@UPQ~S|*./@G`@DRDU x(UƝ, P JW 0XY|*/U+`y_UW 0(VW 0r(VǒW 02(V"_(G*@//UW 0aX:DE.nI&I|}}={aÆ9r瞳trA*KUQNI7nܰtrJ__go*+I`>>>Z|5jdTF'O֜9s,\Q~* &@[R_bcc3g۷mۦ`KGqrrtrǶJ)++(0KTJf2=;… zt *0+h̙nnٸc5lʏ Piƚ Duܹ`Ye@e@*SF}_J2`_@X\Q| `5*bF//UHP(V"`_eP~% 0/r(V`Y_V`y_WP1P~YQ|ʸPP~;YQ|ʹP1Q~?RQ|=PQ~`_@GJSQ|**/짟~*{K"UfffΜDkNiii@[ Xmf̘ѣGKYeZ2@*ȑ#-D6mdr>|XW^tr1c =_@@,^Xj޼ƍg8%ZxuvG猋Ә1cԬY3y{{~zuBcΝzHf/_޽{wJLL$͟?_׌3h抪K.)ʁ ՙ4iڴiT]qլY3}޽f̘{N6lP֭tRǏƍeةSt}i׮]2e|}}%ISzz}YM8QzoPwڹsԷo_9;;[:JJKKS^TZ5} 67|puEO=4i"8|v*777mݺlIWǎ(..NAAAwXdee h_^Zڶmm۶I{=5nXѣ\bvիW+00P |P:փ>(oooխ[Ws)2Off&O [>}JO?TgΜ[oUrssӿ/eff{ڵK;vmVhn~PѰ U)xvm'OԩSZvٳgGUtt6oެ#G*33SǏנAb IRvvw#Gh„ ӗ_~oPLL:t ???͙3G={RRR̲{:pFf͚Դit9-XXnlmmիW" Sxx֬YS۷õj*yzz{I2** /V2 C5j(yqqq:zBCC%I 6T۶m_ѣd=z]&///}ڽ{=㒤?JKK3e5kuVժUKԣGիW,ŋuVرCڵ$EEEK7n\[ :uJkVjՊСCqٶ=zPÆ fU^KtE5lذıPU8q-ZzJ5kT=t~ӦMMWkfu͛7a:{$ic=f6߳>tm߾]sNuT|I}Q-[:uOO6i޼b!;;[nnn%ާ[֩SGGΝ;K<_ڶm h㏵o߾< {n3F;vСCP}||^y`TFF$ҥK 4mXnݺIҙ3gTNB׽}ŋu9կ_O%>85j0|+ٳgUZ5yxx߶mճgOӖqrrҘ1c/hرZzuV\YuN>^#GԥK4a„b@y`U+I駟z-[ԅ ?IND1C̔VJl7jV[J;FԪMT?FZKPF-BDxJ8rqx#3gΤi߲e$J*gyFvRrraӌѣGZz){{{5*_ӦM͛7;d8>5\ڵkivqvڥ wwwL&=|Mֵk$ݹl̙j֬|}}%I{ֹs%&&jܸq:yd{=M6 ٳg5n8ݺu+Z*UCjҥΞ=+Iv횂4|p?Lȗ/6md t}R K/xJҚ5ktyyyMM6/e˖YhBÆ ӤI.777\Rn/͛7+!!AUT\\\TlY-^yIѣ5uTmذAe˖d&L>}믿n`Ů/#qI':wӧgTTL&Ҵ߸qC...rvv%%%)**Jrpph튎VҥUj ŋzJLLTll<==KIIo]˗Wٲe(..NԥK驪U*_|(>>>h988ҾpB[%#GԨQ$#82d&L gggZG]`dɒj۶=-`!vWWW+r$ۧ R٩z^}̈j֬y~ruu𙣣-l٢'Oj̙X_L{,\Pݻw׶m۴yfkc3ݫ~A5҆ ]d_lN…5i$kas];`3`3`3`3`3`3`3`3]t.S_ex~r+ݻ[ O=       <}_W $I'NTlllƎ+IrttTPP#Ӈ -aVf߯ 6Hwή/ P:ud5aťy>vX!GGG1[^ÇtgW_}eyf]_/Cر*V(I;wcG ĉ%IE5|pf+WQt!UVMO?I&`@. l=zf͚%I*SvܩZjY*r557Vj$IRRR\d#{%I[Vhx={VgϞUTTT'Oڵk숊uG&2[oI.\۷[ GqFJvW\Q޽U`A-[Ve˖U|TF Yoڸqc}?oĉulܹL&L&TX1?Ot5K{%d6ŋ3ff._nׯl6kߗdRLLL5L&9֡C ydF lpر%I֭{yH.g۶mtgkӧU^=iذajР<==u)7P۶ms쬥K0 kٳ;X}rrrRBBVXa96rҤIi¢3f… 2eJu<==t$$%''{̓',Yby˗/W>}XݫP͚5}vmݺ5'Z_p$߲3'z۷ok߾}PWBBB2w ?^ʕS'OСC5b]tIzջwomٲﭷٳ5vX֏?( >|jԨ ($$DŊMHHɓ'UD Kے%KTtiկ__ZRdɒl{p?>͏K.=|uM=z40a/_ӧOӀc 7nܰPB.Ii ɗ/,=XK.զM /(..NǏW>}Zƍ;V7oܹsըQ#8qB˗/G}_@@@۷O-ZP5W_T))):p$tE˱7nкu4p@L&9::cǎZbN1RZlyqqqw111Zr4~\Y\Yx~@&,?wssX;;kSRRܹ̙#UZ^*TH$IꫯnݺYtMs7|ίoT`4mzW=WVkJHHEnݺw/ŋ$tYswl6+))IҝVt4wkI8)r5o߮u)$ɓG.\$9;;ɵl2iŖ60訥K>PUZt1^v-KcSt;$)|qqq߿z{NիWv /I=boU r]OWΝ3%Νekܸqڹs7o/RmڴR-&I>;~~/_>=dҝc 7oެ/;KkŋN uf̘-[W^eoo5޺uyիϫI&;wno5K/ ٯA~9h"G:tвe˲USF/^,٬={hi~ϒ%K<:QNIի=Ɍ>9rD˗/,XpLx{{Moɓ'=~ѢEjРի{Ou֕\`(666999iʔ)޽^{5SE˗m6yzz?Vɒ%hڴiW۶m⢣G0 +ͼ&IӧOW޼yo*))Iof߯$%&&jٲejҤI$XbS,Y LZ'ׯڵkkĉJNNV@@n:-\PmڴQŊ;O4vX3F;wNsTԩSe6/RH?˻ƞy,3F0L*W:Gf{|%~?^K.դI$I PZoFaQɒ%U@nݺU'OVVdL&UaÆYM,G;;; :TAAAڶm4i'Ýv*UruK˗ 0]<>LΊcn޼),77JLL2>6u]'''ȺOEFFJzSvڥʕ+СC.6bȐ!0auՓ;ի'IΝ;x$OO $M_wK """믿JsgGиqcne˖YتUV)!!AԢE +W<<<<ԪU+IҺua`R +Qׯob'G^$Iњ1cټy~7IRϞ=y%>/I6m.]d`+O7o^O>/ȂQFI4p@+W[1g:tHԿyzzZ"GYиqcmVrJ^IaÆIJ*@+W/Ȣ3gPBv+W'Uddڵk(iѢEruuvYM ,*V.]*;;;EFFYf vYx$&&]vtjذldCӦM5rHIRxx6l3gX(<1բE mٲEԼysY*~@6 >\$ٳ]vm;O?Ij֬իWʕ ̙35x`Iҕ+WT~}3FV̺[v6UV_ԯ*Ij֬6mڤ}:sرc?2=199YGсtLrqqQ%G٬?C111={}?~bbb~v~Prr$.㿚?^M6U%TjU/_^5jйs9QT`A=szUpa5k,M+WG/V*/// :T)))ioo3x dۨQ$Q1eʔL]tIϗ$5nX>OICcbbԨQ#kΝ~߯hj/#GjСt5:tHm۶^SXXo͛7O'O̙33^9@n!d[͚5ըQ#IҼy2=lɊ3Cl6hѢ-_\O֚5kTvmϟ_~~~?9]ve8ɓ'%I{ (ʕ+W^>#FPzgG RϞ=յkW͙3'y+G0x 7Z`ڲeeק~ݻ4i®/?^{Ѷm+O>^NNNi}GܹV*Uhʗ/_| ~7n͛7kܹ4hPSL9r*sqqt.AGG \~])))Zzug 600$___رCSL'|CXbԺukEDD0 >}:ù4htl9::fzO/5j6oޜ(͛7Od߿ÇKf,XPfY6mdZfo>cǎ:u/;;;jJ&LǏl]x4VUW&M$IsεٳdY*U˗/{֨Q#]~]}䤺uj̘1JLLӧ5jhժU\.] /@c_2rHmڴvI%IM6U5U'D1IIIuEo@ժUK&IZjfϞe˦7g>}Z-[Tue3F P*U$ISNUݺuL2v~7?O7orrrG4;_$;;;[;((ȚxBޛlO?]Zª:uh*P<==%ݹ?L(QB!!!jذ|}}ոqc%''QBBB7o^uE/^y/Z ǜ0 E:uJ'OTUf4n޼}R+]vI&$[NeʔQʕQ~۵kԨQ{۾}bbbTB)SU䌽{׫YfT'ɞ={TV-j֬Yu-feiEFF*o޼ϴ_ll&ٜa.\X{ĉ>6dM0AΊv9߶աCjРAcR=zTjӦƍݻwС.\`׳gOxmm۶՘1cҴZJ+W𘥜2e >vܩ>@e˖ջﾛkFjڴi#FXO*UDz7ԡC]6~r%UyKs3ŋСW7oUVY^\NG}秹? x}aF @9;;?ԩS/hѢ.) gggw:vXb͸t$Y.0Z.e*000kܾ}[ʗ/L&S}Pz,m111ڴi.\y֭[WVVM))) C}ZNRTTnݺHk'O͛Wnnn*U|}}UXDsIIJw&Y]r:TZ~eWԨQI&Y¯bŊGPB6mRJJJ^o5|{hӦfΜPs*THt=9QFQFY^ILL?7jӦM:{KS@jԨ7nv`[֮]{jz҅O#!!A!!!jժvؑY…XB ]'w[~ׯ/777K޽{uUl2j_Zj_yݻWkVs: w|*UJONo IDATS… >}mru\R+WT~kVK@.#C ͛F鞗*UJׯ_~$}w3fL_Tɒ%s][^7֯-[x9P; SJ*Vm۶H"֠A:u($$$+""BAAAZxq##խ[W/|}}%IVqQQQJJJ۷?S?vܩG111 Vpp6mӧ7nB6x` .HbK[JJvڕwu_ݺusm a+J5p@M2E,ӧy 믿n 6m})$$D;w&cNZbN>>}^ 3f5jq6 9UyƍǏ}_uu*88X۶m`KӅ\ׯWҥ/w퀀k שSԻwo]p!࿤|ι0|"vܺuK͛7ל9s$I^^^Zz6nܨ_~WdI͙3G*W,IڱcW'NX:$/#y4i+Wرc^ ,gyF|=?u̙ ï?YZ|3fUxqUXQIIIٳC{IrE e'JEƍ PHH$z:pڵkgʀR~u]tyթSGna1&# <Μ9riŊ+k***J VѢEeooi߱c?י3g,m/_6mڤ fyX[ xp?g￯7o#WAM:U $y{{sOX5nXw$mV˖-xC؄'jȐ!2 C ֮];*9C ф XkH~!$%%B JLLTǎUV,ݫ[֬Y#GGGhviYF jȗ_~h]`W*0<۷_-Iܹ.]*;;7o#I*W7"4_9A[loXmۦ ; |/IV||>C}.ȩKD4g϶_7֒%K`sy >\ҝoի+b S9|ݍ`<.\+*::Zŋ.///k䊔5o\6m$]V[rU9_r+`dҊ+`l2ymf ݻfIRU^=+WB iĉk֬YV ƣRxJ 5n8+W<:ݻwK/$I>}Ss,!X<+3gﾓ$曹wqb24x`I`+W`k_h!5 Qrؚd?/Bad2i9\.]U^=\Yߎ=۷oF92_JJmۦgyF%Jȑ9ֹs_x։'3-W~P/__cwFN8!0Tb]vŋ_gϞ9:?^DDΝ;'Iu떒\l}B ʓ'+<V RszP5J 6oFTjU/_>k|8[._]vI&ǫqƚ1cZ\vMM67o^jٲj֬;vnݺ_{.GCe˖)00P)))2L96+0e˖N:tЧ~={ի5u ڲeBBB/ԩS]*_ԩթSGvvtCx=W*@>> ԅ ]O)J0Nշo_͟?_ڽ{Zlf͚?c#0$IֺGU=oVy< ޽{uʕ{SO'OWK)SFϷbU\ZjrqqQllvڥN:ZHرc 땒bi/{N˗'Gv|*/>>^qC2 C/^ԤI4}tuUFRŭ]6 Si$CQbE.))SX6N:ex\v1ba3UQfMcÆ 5j!ɰ3bbb^ǎ … ;|CqqsF%K}5bcc}d)S4ʗ/oL>>Q~}ZnQV-(\ѹsg#<<<ի/_>Ͳ?}"##glR2dZhazܰ\\\VJJ~'ժUKY׮];կ__}=I&SF$%&&q:ug}VSPP._lڵk TJtE}JII`CVPPPR=sj߾Ə=zdɒs2'|-[رcrpO3f?ԛo>Hn'|ڵkСCrwwO5i$EEE EEEY>3g^|f/AiŊ9r*V7oj׮] WʕN:nݺ*UٹsuVmٲE}ʕ+K7PrrwḸ88p@icccӵ[+WԠATfMYf:soK#GXv*Jҍ7tuA##?h֬Yj߾}:uT!=N:Y]uppP.] *X:< -wyGۧqiÆ ҇~ 6h77v32 VF cÆ FJJѴiSCQzl-ݻw=ztN:J;waoofaƤI l߆aom w\{׈# l,\~$iSw~UXHLLL6 .㣏>J3ܹs1s{ߪU+f͚icH2vai+Zn\z5Mg}x﹆1v{IթS'C_[RR?;TDEA{-͑VUiYގ>RHW9nqd~*xo˅˅<_u8";l4o(Ye?g~;v̐di;t!_ 0ÇYfsZZnmjr3 6l⒩׸q ICEm޼ȟ?{RFZ,Q|ykxơC]X8+'a?3Fgϖ$ ԿJyyyd2CSN̰1Rf:ܹS/ZlIp&҃Kt^=תUKϟWRR$i*W\]]nY_I6nܨm>tlѣ5}t}zײToFtbzR )IڴiԬY4`6UBݻ_א_iʔ)>|}-,1)ԯ_??^ΝWc>/R^zez^WwQdddjIݷ.!!AwҹȜN۷WLLlmm5emݺU5j԰viZjX_xQM4QHH+<,{Ǣw vSŋt$WKR3FQQQ:u"""Tvmf 6_Vh8;;+&&rB I4lmm,IjJNNV||P)S&ϝ;'0t.];qℎ= WUF I .Hڷo Ζ:$eׂ 4h =ZTV-OÇs)ɒ%KJΜ9#__GWϟSTك&YyY311Q>yf_rrrի, dggg}WUF;wSN Qխ] ~8vL&uMV*˼ݺu+4h^u-ZHoFǓ!L٨Q#}׺x/~~ז-[t݇yegg~)::Z_|EYLJy#!!!u$yy\/]k׮͛-ᗓSʕ+:~*T`i 2<'5Ⱥxbݻw9V$駟~J,?KjW`F5oV=-;;;^ E>}7|S.]R޽e˖l @wwW#Go.].]믿VժUeN:+Wjʗ/~m 0` .\XZfz>z {JJJҢE{~~~2LZn$)K3ի6mhĉ*\v*ݾ}[?ռyLUӦM/lYɓ:~4ibSLW^e ̌iС@-YD} )UZU}:w,m۶ͲkYf7n ,5kj׮]ϳȚԥ&͈#t)I)!#GɓϵuV}GV^vY!*THoܹjҥ*U/wwwUPA5k̙3uIUXQeʔ_|1VrʩH"*Xڵk:uX~fO>DgΜ (Ν;kر^zݻ*V>@/rѣ^ppVX!)#\FfͲ5}\;1ј&dZjԩ3\ u˹uYg=iה)S3<ݻwA˗o߾TCLL8K.C˗W%,+66V dzh{rrvڥ .E˗ph(66Vjذg)99ٲ$&&%= lllt-9;;[!Iad.IgϞ޽{egg'oooըQmEGG0 ,XҖ(,Xr۷eoo/ggtc:}fJ.}ߥS ''.٫W/?~\WXX>,=ifݹsG7nH͛7gf5iDs ( {{4}cbb 777رC<󌂃-K]rE KHHPLL\]]r9}e Ǚ3gy?0T^=۷O:r ;m۶iӦ2 C/%@0aO.GGGYǂ  GcjԨÇ[nzw-0*\n߾}j9\-4_9ٳ*T\]]%f]tgwiW_i}v9~ͼ_~Qv$I'O֔)S[z/V\)[[[>|X+VvIce)@$љL&͙3G2|I)KK֯_؜*uKӳ>+___ڵK~c $ﾓ$+VL5k|,Ν+I*X`r;#%''O?v9lFO|nZ>#aTT, y@^r<-V^-,IzWi;w/=zhرڹsXL&}RsflڴI&Izl7c߿_ q5{l999͛|Ν;ձcG܃ +0/r{{{5JtX" ג%K$I/B $??,}QF-jԨU]tQΝuL٬x%''grvvV"Erd_|Q$o>׳>Fiƍ>}zxt_I ] &I)?#DEEY"x7u]L&M4EG IDAT IIIڶm$EY>РAZr>c={Vk֬Q@@9_U111]2raÆܹs926l(I zh]v^o.)F% cƌF`AAA*[l] ܧ@6m  .h?rT@@~'IR>}ԠA+W7>}ڲ^Ft}jС0aB3 8+66VUTј1c蘦OXX>}ZjѢ'{{4ݫ+W… rrrR5/u7 Xӭ[TlYrqqIꫯ*[[[5nX%I .5aﭾe˖ĉ8q8qB4~xIR2e4hР ۽{֮]'jڽ{թS'u.IjԨvڥiTN hŊTɒ%5zhrrr,uVoz%Iqqq:ut"ɤ *>>^ժUӨQ,32gQHH?988hɖKJ FѣG?f~x3Ƚ^}U5mT`^9yY}v}wjƌ裏ԵkWf%%%iҤIѣkH2))INJg}V[_NNN7nΜ9ӧ|@neA͚5 IFN] L={!ɐdx{{'NnjcRJƥKu|EDDE1$3lb풀la c.)OYp .d:FrrC 4Y`!8pѦM#>>?0L&rJ0 ԩSѥK#!!!͵bcc 0@CqA͛F˖-%KC-XlZ֬YcH26mda$%%˗7ׯoDGG[8pppp0̙ci?a2kO6܌;fhݺCk3 / IFVD0 l6//_>ŋaƞ={ In9DƘ1cҍYH㭷޲W^ݨTei/ IF,mӦM3} ƍ$Xb˗-s5$G=?-_0Cƍ- '''cܸqƕ+W4>Ǐ7$.aC@rr Df|d(QV^m*>>^:uR``ykd˗/M6:{$?s=g媞\/ŋ%,().0*Pll2pQ~Txqq۶m%I'NP͚5믿ի4iRZl*Uh֭ݻ;%&&jBtrrJs|%꫊QHHJ.Z;w *X۴i#ɤ'Nݻk*~5kT&MuV5J4tPo[oJ*)&&FVj8f^ԬY3uUoeV[[*,7tP˲ƍS ,iggwfCr_IRa\… }uUݺur+;;;޽[ݺu_|YgϞM7,'ݹsy͚5oȑ#1cVZd]xQ/_V׮]?'Ŕ)S4uTkXdϟH1chْd Ns[oe-Յ ,SNNN6lXtڵtmV6m?^zԩuӱc|XsU&MTre-_\ѐ!Ct4b3Y>֬Y9rΝ+5lPM6zJ*ei<{{{_^wU*UԪU+uYՓOI)˵o>+WNm۶U㣟9ñ6m@oԩSgYdR@@\]]UfM5o\]vUF 6HJʕ+ e/'''&꫺v횊)2e_~h իWWjԹsgUVM/-ked񊊊RʕթS'UREwMܹsUbEnZE\]]Un]9::g.kR_vvv#'h^ ժUKTNn:k_O,.{{{w D//j֬Yꫯ$???/_^E"66V өSt4بSNzwTvm+UݻUMIIIrssStt>w6lؠ0O?ed]VjJIbb֯_:uY/66Vk׮Ձ ooo{m?^k֬щ'TP!կ__=temݺUi ÇرcW|}}3u֩\rZj5k֨RJ%fׯ޽{%JEjذ?WK3ѣGul<]xQW\\<==լY [x 7n(((H!!!*Ptb+I+͵]vM5RϞ=O?WիW3ڶm\"???թSG^^^ׯf̘!I￵~+cÆ 龦wՏ?=#͛7WHHTÇ?]4uTZ5l0}駙&er &hrtt ' / yXFZjVX={X ___G}U ]Nv7>SөSU-G ;;;oa5k֨[nԵkx3uÇ~a?~MW4_~MSi̘1RB%S0/.9rF .(88XRXX>+W(::e"sttT˫jժ%˺ZnM6)<<\g$,7رcGuA:vV^-u԰qFK1U* @;wԴi'J<"/#3f4uTEDD_:u,_~ H%ԧOڥ<^}U-YD>>>:s挵^zIcǎlֲe%JP֭eooc(Uѣrvv֧~e|rI,߰aCk܅ L2E4uT9s&M=im޼YK,є)Srl)=dvک]v9z WWW?>G ǎӦM$I={T|V < 6eM#fj.bʔ)2eJ.M ֭[~3g'j۶mZje9˖-kٍ m&O|Hg޼yQll "lJ}k. %ȅ4hJ~WBXuQIҤITdI+W _x,f͚2eHJY9rEx|gZt$I&3fu /< P^zI. O+Vhђ$///ZJvvvV SԮ][?6mh. yجYԯ_?f*00P. X^S||z+))ڥ!Q>}4vX!mܸQ5j԰viFǮs Ќ3ԤI:tڥ!?TvmZJTT)m۶MM6reǁ VѢE ݻ2gΝ[ P+W'Ѯ]ԱcGnZaaa$ݻWժUruDž VSlYرCƍtRUREZʕ+u5k\,""B-RհaCmܸQT`A͙3G*R]ӻﾫ 6l6k˖-ڲeL&*W |QBdkkk%&&͛:s挎;;vԩSi888_￯%JXR5~ W\֮]G/P@@"##e9#GXDb%KT^odɒ.`E_U*W >Ν;yfmݺUzC.+EzTvmذ 4i&MXܹׯ֭[2 Êlmm"///999Y@.E'\\\]Ř < y /_3g~ @A< y /_3g~ @A< y /_3g~ @A< y /_3g~ @A< y /_3g~!ٳ{ %&&pEiAl_O|hdo^˗W@@cu_VgϞիW(88}/_09sF׮]{<< jܸq*P$iԩ헔>H^{6/d+뒤`doŊ:u$i„ ʗ/c]_v?Pì/m<<<4tPIRPPP_;v{ ovțƎE)***/0,|}}ѡCm߾eg$~!9sTlYIY7oZ Aok…˗%IǏĬ/*///uer 9s(::Z}$i̘1r{%%%)<<ܲK+W~6.yĉU`AI_2d5\… *Sv%I:v옕+ _Q ְaҴ7Y_ G~!Ǎ5JΒ$ooo 0@5tPX>o' r\ѢE5tPI)g Wwn` kkpmݺUaaa:}uEGGg%KJJ,YKZ...rqqQUlYUPA͚5K<92ڲe-[`EDDdg]ȣo˕/_^-[ԀT~}kxd9JHHЊ+4c NKlmm-[nݺ%0sl6KEnݺ8q:v(Vi<\¯77ԙ3g,mjӦ{95kL*T`=rYCi֭"٬{K.U,X ZT@.ҥ:vh `]tI6lUjU/Y/_>կ__cƌі-[t9͘1C%J$8p@7o8+W ~m۶M5jкu$IZpBCC5tPx.%Jرcu)}rrrlU~}?~%r_ׯ.\ I޽BCC3 @˗/y>|XM4$>|X7֮]\ 7onݺݻעE"E<@~~~ ĉ%,ڪU+m߾ʕr ï]}Urrn: 2q裏h"*&&F:tÇ] I~{JHHծ];k 2D˖-d͛7ձcGݺuer4Wrrzey#yٲejڴU ӧ>I)=3S҄_ ,ݻ%IÇW=R<̄ ԦMI){Z"@n` ._I&I*VO>jEhٲerww$9Rw޵rUk_g۷%IS|VdeÓ'OjٲeV`m6t--^XԤI[(ȬAG4sL%''["5Hի%I8qU رc%I rEk˗KJ*{Ϊ=5akHBB!ݐ IDATB駟f/_Vxxe)MdͥK4}t;wҶrJYƊU^}}?͖-[d$]vYQQQQ駟=tR*T_(Km޼9[ eֵkW}2{ｧɓ'wڻwN8`CgϞ͖kk/}\<ڶm+)e}YٳGzey'''=oԥKKӧoiСZ`A󢣣uYf-[ֲdY*٬۷o+rppӧuMURE{`MnݒlV||n޼)Iʗ/-}bbb*;;;UZUΝ;JLLTbbeL;;;,X0M$M%Jx'NН;wTX{T2 C TLLN<)tad:uTvm;wd\]]􋏏Wll ( {{ f5o\Ϸ-[V͚5K7kY Qegg'IUxxf/.I)_xf&]aaayS|| *?~\...yΝ;e*wFbcc-/tuݺuKϟʖ-+L/_˗"__ ULL<==Udt'%%ȑ#2V"EsN8!{{{U\ggd2;k111QgΜQtt<==ywڥg|@`s1IRJ kͺxmŊK|2d<<t}WUPA*URݺuUT)UR%M K5kTڵUhQ+5rH)RDuU5feRR… zV .ѣG[UU|yծ][ҽ~_<}lN<))eΣWŵb I)Ӓ%KԷo_LtK.-[˺tV\~M&LHwԩj׮"""ta/_^]ty2zOVҥ+ƍqO?TFs)44T*TPΝ{e˖jҤe{"0뫯իW~zܹS~ϙ3gԺukyzzVdd6oެ[_EVX?SW^Շ~ŋ~9#GTN4`;wN4~x>^K.K/ ֜9s4{l]~]W^޽{զMI)!]kzѤI+W~KԪUKwֵkt͙3G|͛3l6륗^RhhnܸSNo߾5jT}mۦvکTRڷo"##>}X̛7OÇWuq]~]{QÆ -} ˗kɒ%|Μ9Ms lZJ˗/Wdd\[QFRfڵ"""t =zT#F~Kx #FYknnnaĉʕ+aaccc9s0$۶m{X'N4|||,gϞ5$-ZHܹs1qWLcife4m4M׍ =zx?ѲetӦM3$7nL޻wobŊÇE1[paccc\vמ2e!8}tbcc Iưa,m:t0 .Xfk 86~x5x7o4J.m4m d,\𾵥xѺukC!pvv6ڵkg^:Me˖&8~3${1 0>{}7d/]СC OOt #220 x IFHHOrre7o޴˽N>m<ҿ^z͚5ڵːdlٲָqc}Yl6~~~Ƹq,mZ2-jDGGgxNBB߽N8aL&cicbb w5 0Fm(QHNNpm۶۷gi׮!ɨSο - *dH2 *dR24yd}ƏoH2]cc`UֿM6MwֲeԬY3d8*..NK,޽{X]pAϟWBBB۶mܒ%KZj믿\ٳgu%5*M{…դIر#cޫEi˔) 6XUVwyf?~ܲUvP/n96Lx^BtRlR;vPǎ5dȐ^KBCC۷kӦMb G=zk͜9SW_vڪ[$I&JJJRڵ5p@jJիWp&?y<ϩ,K)*UJNNNi*]$… -3Zh=HԩSٲ졔2pѢEPddt-L,0}v3=7n<^BBBd^ŋyͲ`f4g5lP PUR%KߪUC]vUծ];5h{=H|QQQY>7e@ʕ3}\]]<҇]uƍl)ѷo_ :TWΝ3Gj׮]_իrJK.L2Ycw\CDAʼn6=KZΜ5wB6(52S+r\i\i* *zd >:u_s<.\8sf;l޼K*T0\J"Esʔ)޽{MBYf''LW2RfLzya۶m3:plllP-"99C3l0UFӦMrY#""""""""""OkWWW\ȓoGeŋ(]-99UVSNqQj׮i-Τn4s^x;wٶ0+sfG-ؽ{7NƍI&]۽ k[~Glll޽;)))((I{34i‚ ox饗2|888мys&Nȭ[8{,ѢE v)M4Ύ/2>.]OOOc[LL 6mʑ.^HM?3{y\z5y*V7-pE?,Z(,X;2tP._666Ԯ]9sN̪=<}l}}} ɓ< !!!iР666L2hBBBN޽:t(W\aḺҿLǪU={6TX&M0ydzͫJ߾}}6&Mʊ7xs^(Z(;vIĉYbZb„ T^8=w}3[J:um۶֖72{l7nl ÇywNv>|8 &22iӦQLF̙3_PB;w+رc)T͚5ӓ'N0o&xٺu+u!%%Zjlٲt׻ﾋ-_57oޤk׮4i҄^z@HH/r?3+VtΡCru{^Jzر#...-[6MWWWʔ)c|ΤI=z &///cKٳ(T 694sxyyCJɒ%HII!88u4i҄'h"^z 7n6l`ԨQܾ}5^yӜIֱcG(Yd|%Ko`0?~ٴiॗ^"$$~+VM`` E1>ϣF2yvjr+ye˖%{O5m4\\\YpӦMm۶3sL>c1b΄ɟ??J2y -JB2qg$$$PxqOƍMVUV-[0aHII<@yi۶-pvoPBO?>pMJ,'|”)S+ڷoό3ɓ'ر#3ϰzjcЙQQQ[Sʕ+ ]t`۶mZinܸ]!MTTJ"44@bccIJJz`ׯ_֖ ؜Y[O0"V 夛7obggg =si*Tܹs8p`}y&...&^V=(_ 󎎎0/'%%%s~n"!!!ӭr֍70 ~&&&u)PIPU_|nXFl!""""b\]]% "88 cT|&L@HHܹs &M`ee`য়~AXyPœek,LBL8… ꫯfQGs5 0K/_(ƍpssz=<ެ)bU+ѣ^^^ UOOO.]?õkט7oK aЩS'6oLXr%+Vd޼y$&&I"􊍍孷ޢjժܹUoQn]3W'"""""ɉzꙻF2eXx1+W6w)yۛ *iRDDDBe~4jԈÇӹsg]P{JD lٲL:;w`mm͐!Cطo*T0w"""""˗;wϏC1h <<<[nDGGz*777VjrDòex7h׮~~~fx? RhQ+FfXd><~~~_7x/// .L@@ϟtԩS֭['ҽ{w۷Sn](S | 7ĉ4nܘz ذa5j͍5jgZ[BB&M777J(A- hѢ4u1i4hC姟~Ϗ g:wFO||C'mV:.\Çə3g~Htt4/_6~3 ػw/+WdСpB9uJXh IDATa7uT>LBjՈaݜ;w.\x1MtSgՌ9c-eeeesȽ~ziӦ />ԩS$$$??ޝ֖ŋ@rhy|s<==?|ðܹÅ Ϗ' /`m"""""bfgϞ=O\¼yϰfҥ۷UV/ /˔)Sӧ˗7nޡCYn:peϟi@lܸ[ۻ^Y&>,_}C gڴiϟ۷^xҥKgis8y$%KUV/^K2~x `2ftؑ7xֵkGEEEqj֬il>>t "88Çi?~WRN O?ptt49*Uj?j*~W#GѣYiڮ]F%2cmmA_*OOO4R Eaٳ˗3a;wnһ;w`0ɑZNݻ9y$eʔɑ9EDDɡCtDDDDDDD.\-Zpȑf͚;wp-[PV-HHHxaaaϟM ۶mr];w4y|e¨XbcxIXȟiүzl޼^s#ڴi/qGUVOJJJwrϒ%K+&LcDz\׺uXt))))0h ݻwc6lȕ+W1cIII2jԨ4[7=ӽ{w<֭Y/f̘a< **+0Ӳeؿ?'Od̘1V' r"""R%""""""b틻;۷Ow[{Ю];۷oUh>KfժU9rhݺ5ue~5j`ĈL0x<ŋӝk׮ +R`A~WƎ˘1cprrՕ 60|*U:о}{ƍ3 ŋ &5j7|۩]6Ã={cŊ舛e˖%|Ǚ{7N:89930x4gTIcg~;{,ܺuI&{ewĉ3rss#&&ؿM6\v ggg;r5 *dlkݺ5gΜa۶mܺu2efYf1sLnܸ`2ǽlllXf ٳ)S5kϔ)StժU#<<~h}Y-j篿2?+++֭[ѣGJ*ܺud=z[nܸq{{{رV< gΜضut3=sk߾}?~p(U5k|`Kxx84lwww^xVk֬Iw%Kr)uuUʕ+"""tQ%""""""bkJ(``Μ9;X[ݰ%%%>kkkۼ.5C 6$))ӰaC:t@b2/_>\]]Ӵ٥9::Ү]amm3 -Z%KLjժTZ@i_T^x/֭cÆ QtiqjWʕ\ƍ,X0ݱi!=nbɒ%ȑ#]<(^8͛7?5ytM4;wмys<==iڴ)˖-Ν;\˗/Z˖-)Y$͚5RJtؑM2vXsf﫿O>$ӳDDD馕_"""""""oXb0{l~mϧ0g{ϻ\FHHժUڵݻwsEUz%swwg.CDDD DDDDDDD,'͚5c˖-U_Oƍ'ׯ#"""Ѷ"""""""o55{lY/KKDDDDDDĂxzz@\\qW-KDDDD$ DDDDDDD,ŋMU_"""""S%""""""ba<==iڴ)+++ )RČՉX6_"""""""hҥ_5,KDDDD)@EE&m-[Ԫ/P%""""""bK/0sE"""""Oᗈ*^8͚5y/^X>_"""""""o߾̕/::%5w"""""""M6ѧOzꅋ [6sU"bn)))\xSNqbccy&.Ml (3 TRRX1lll]/22˗3uT3V$KDDDDDD"mڴΝ;@\\:ub͚5 DBZ͛7ss$b .L˖-iٲ%(TKzDFFϥKmAAA Ĭ7'(({{{cO?D#W^-Z^zK$^ʲe۷/Ŋ{.뉑|EFF0bJ. WMPP111̘1H+DDDDDDD,DF@ƍٰaNNN$$$5k\䴔}]Vʏ?`֖.]qF_V^͈#hѢJU< ,+%Jy 6/euF|+WԩSIJJ2s叟/ooonJr9s&cƌ1[R%""""""b2 R)ymۖI&5}ɓX6m`m_d5͛7gɒ%>}bccCBBo-[49J2YJXRDDDDDD̲|R&d~T\]vSlY3W'd(Y$c߾}<3l۶s 3Wg|R&KDDDDDDČ|R&d QF/w^իgDLjb gҸqcOf|R&KDDDDDDL&JL?кuk.^)S ̕<ꫯ ʸhj-aT \~_}:oVVVfL1n8Ν K۶my󦙫|R&KDDDDDD$DJkС=z#G"f2h &O kf,CN_I^S%""""""r2JLeK5j|`Dnj +V<+0K DDDDDDDHn_<>ypqqaҥښ*5/#Fpm3We|R&yEᗈH+0 8y$ӦM@"E ٳ|f(fJ_""""""",/T D,[bb"3g| 4ȽzM5={6qqqf(EJ6_"""""""(/T D,?ٳg;v,666fHDeeetf(oeJ&_"""""""W*`"'99~WӳgϜ*ѣGٻwo̝;wr̎iOJJԩS/?سgپnN c׮]z+Ku%ك`ȵkkk_YYYѧOʗ/oҾ~7n+VLwRdd$zhѢQ^=6nܘߞ={x饗(RԬY?8Ӻ?sԩúu2`ڶm ÇOǝ;w?Ӹqc?RRR#&&868q">>>QD Zli (Ϲs&}/5k*T`Μ9iμ:<Ŋ͍ڵk?SRN<==)QZbڵ&}?~̅ ߿+#.]K.׏f /M6̙3ԭ[E{QpaV\IYfqk۷Ӻuk|}}yw)^8GСC^;%%#GpB,YB2322]vPxqƍǘ1c(UC _|鎽|2XYY1n8zM֭+VVZ+V8vʔ)̝; V͛7ٽ{7Ν˰ÇFjՌL)Sׯӹsg^}U^{56nȈ#(W۷_~~~0m4Xz5/"i|[ҽ{wʗ/Orr2_|:u_~1'Nd{qvԭ[q+ҵkW*U5Ϟ=֭[ٱcFbŊү_?3k|vZ`8pQF YTo0ϟ?[BDDDsߟǏg̘Ug?c(WѣG ܥm{(""""""_kܸ1!!!.C?VVV2eʰ`[ ֭[gܚƍ,X0[uBBvvvYSX1ET^Hjk׮iӦ\rG 8rߜ8qboo5_Nٿ? }%6n̙39{lYP!Znm|lmmM59իiܸ1ׯ_7yׯ ,7dĉpI}||X~=~!QQQtӬR{W 8` =000s`KUfMxbjqvv0.폸,VLϟϩS[~ϩSN-X_(Cڴi$&&boow}gW޽{ceeŘ1cHHHΎ3gҠAs&8p 0w...RRRXFe7c)ёWp5n߾͢ELPׅ aedʕdzxbYIj֬Y߸bjѣ#FVx۷op-_Ί+XbEsDEEQj,]/66Ν;m6(V$%%l!8gرc=z4իWW^aذa}EƤDݭ,S|ϟˤm+Zd>?aÆY~\phcqFӧ}1_AAA|f̙3)X SN5wib~Jw}++ƶmشigimmMJJ |M>vQҥꫯw7YffL,/ܹ3qqqP~*۴i>pw ˗+p%Kdܸq.CuVLnl3Ƹ"ٙ7n WWWs!jԨOjꚥ9 ,HjذaAAA6R _>}d: 9 k׮|W+Y'888tԉN:O?֖:uO? Xb&WBB{;wsܾ}^{__T^ض}vlmm)_|c<==… &3y:ux׍k׮r}9%PB?>WQT)GÙ3g|1貱1Yɶm~Hzm|KHg~<6m?D\\?}:gΜ1>>{,G}v}e˖eϞ==zlFܱH;fr>>>l߾'Ndʗ/C l޼ ХK\\\?~j׮ܹs{4YWN֭ DDDDDDDAƍٰaNNN$$$`ؚ5kڵ+899aÆ2Kz N<&J"ɓ l]ˋ]vQ^=VVV2zhyJ.9t… (R  M6aÆ1a ( nnn|=m4y'|{˗\2>>>|[le˖777|||ppp`Μ9DJJ ժULJ'f7߰yf-j|֭kr0+HFh۶-nnnIϞ=M-Y2eʘܫ3f2y8w*Us)Wj'''ZjEժUeӹpŋɉv1v4,YBFxWX"'Of…kf3}ѲeKׯϖ-[ R,Yۛ[3gZ|Ff͌a3 /H6 yܹv˗/SN.+ _"fѢE{uƔ)S 3p@=J*U˗/s!bbb(^8+V4 nݺErr2&nݺEJJ9Jp̥;v'+W=u/\={[ժUxҍ7ȗ/&c_5 J^y/o޼  yuW\Dؤ\BBwqIJJTRԬY3g:ݾ}xϏ#o&111}fZܹs~h<<<\񬫌\g.]DVZ;v)U5jH^cccILLyݻ7{>}#GJ&ML[ի*UZ7n`߾}$%%Ѹqcy&NNNiL}]\\8tjbӦMƕʩ[yyywWlHm/TP[|/""¸k۷I&/z 0QFQTLǟ;wN> ѣ1cF0| &=w1w9yB*_"""""4Lȣz*eʔ!!!0zh0"3NB ̚5#Ff"+DEEl uoܸA׮]ٻw/gϞ0LI ,`|jժ\c)[ *]PDDDDDDDr%S%byʖ-K.]XlY$"΢Eh߾|eS%AᗈH.1gKrM4 x׵[>|8qqqXYY1es')ܢKDDDDDD$#S%b٪VJغu+_~+˗~z|u5sEy/M DDDDDDDrY^` D 6Ç"ۉ't"E\QL6_"""""""y /0_"WWW.]-oߦm۶9se<Ο?O֭y&,^///sey)KDDDDDD$fKӨQ#f͚ŋiժQQQfJr%Znm{whժ2 |I^Q%""""""r#S%zי4iOgȑ#fJpI}Y;݂tf2FK/</ߴiӌT>wcfJ_GDD~,9)KDDDDDD r"S%db̞=ܹÀh޼9aaa.OriڶmK^ښS2|lll]ɉL/3yLȓg/x{{믿RZ5#:q={rlڴ ///6mDPPVVVfr=JKEᗈ=LKsqQ ,[իӠAf̘Ç1 .U=zYfѸqc*W7|Cbb" 0cǎѲeKsXxL yڥ`ڵ#66/_NNU%+T}.+W$99={gcrQ|y)P1۷osUN:ӧ65;wfҤIԬYLU>R0N>̙3|Yg"_$<Ν;]L)""o%Kpis#)]4ݻwW_<Ν;g Fm)<&L $${{{ܹcrV~XV)yz-[ 8q;wȑ#:ucǎqէ旙"͍*UKժUiҤ *Uҙ^9(` R( 7fʕtܙ8?~}gҥK2<֭[qI2BPޟ<"""""R/ u@DDDDDDs=rH :Tްan:TRnnn5kVǗ5$""""/"""""b"11z*nܸRqFP|ylݺC\\:v˗˗DFFb˖-/qNRog} $''zDDDDDT0EDDDDDTL\pr߽{wcÆ RYZZ6oތnݺL2t :::ebx-Ο?qqL<7XZZ⧟~BZZZ_ P(`llg}sss?(X`򋈈2e Zlŋ&&&jWT)6mR#::R4lڴ :u=L;[nTʝ=UcA4k֬@Ν;ҴDDDDDT0EDDDDDT9::iӦ +P[ȑ# ...R1c666?~<-Z+WB&IS-ǫW*Ҩa?yyBCCQjUn """""ܹ fff5j|||`ii[֯_f͚aϞ=3f r֭>|8.\(EEE!==@q?~XzEh:::֋h76mڄ(\|9_$Qđ_DDDDDDDZ111n#Я_?d 0@ڟ7oޠt*~נA`ݺu*ׯW{ #!!A* GhhhcNll,"""""*8򋈈۷/fΜ3fիWroK, qQݻWetX~c„ 9s&444ЩS'\t ֭nѫW/XlTR8r"?tLܤ bYZ ƍ?ǏKezźuB ҶZnJ.e˖a۶mhԨ0m4TV-8mmmcᰲB`` "ճD۶m=FӦMQFl2EDDDD !2~eIDDDDD/""666Xb{uS!ЦM$&&… @qI899ĉ( ~\M<~~~9[q/"""""bZjhݺ5FFa…Xh߾=W?Se3"""""*8Q1!pq\t yN#H===4lUVUw8DDDDD1EDDDDDTr888AݡSwDDDDDqC""""""""""""*1""""""""""""/""""""""""""*1""""""""""""/""""""""""""*1""""""""""""/""""""""""""*1""""""""""""/""""""""""""*1""""""""""""/""""""""""""*1"""""""*0< u@DDDDDDDy{9Qd2C$%%;φ#0]]]u@DDDDTpQֲeK:tHam۶N|rDEE;ϊ/"""""""""""h-Z@-`Ϟ=_\QL~QL~QL~QL~QXjj*֬Y⫯BJJC"9sd-:tC}>({^^^?7oި;b/"""""""""* 2pqqBPwH_sի £G>YΟ?;;;;!!!E ~![l 333̚5 F|'OI&XxCBd``{B k !!Aݡݻw %d߿"0EDDDDDDDDD {c899ƍ8vʗ/WWWd5=ziiiG6m ǹspE-[{.RRRpADFFnݺh߾}cc݈1w###iwpyԯ_JB||3fǙ;w*))IIEΝ]PL>]z mmmadd$߿/KOO?dB\.:v(޾}>Ə/ E:uBO>M6UTVVV6lJgϊ *RJ [[[QLQlY,չu ƍ'(WHHHQQQN:BOOOڊʕ+ B!~<\. &&&ZjR R׸qcѳgOiW.]:M&UF%333QF Q|y1zh@<, IDATyD!D\\ VZ%sww 6}066ujժ">>>JMM7kp]amm-J.-ׯ/LLLJ,;v000||_pssvvvtB]V mmm1w\QLQ^=#ʔ)#ܹØ1cL&B...ygf#GeʔBOOOWJm&ӧ aff&5j$b׮]tP ]]]aff&_.b޽RJ055VVVJ ! RJ&&&~m@L2ETTIԬYSTPA/VW'NUTDJϝ;m)))}B.닆  *[[[N>}}=zM41~xQtiakk+8}T/88XGzzz^zHhjj7m6'ʖ-+lmmBܹSBSSS [[[ѥK+tuuЯ_\E&ool_9SR &&W޽{O,%||}}EzzBI& \."""B{NTXQԪUK!xhܸ044/_B//N<)BT4agg'j׮-<!ٳgs]. L&֭['e?`֭R&8 3fHeǏZZZJ~d2)RRRB>}Zr1gΜ\+99YKacc#ӧOuԨQBCCCJ6d&J*%;&"--M L&&N(Ҥc I?B!.\ 'Nz̈́HPy}7nHM~ըQCz ?{LԫWOJGe&͛7B$&&:===#!:::Ez?w0555k֔ifBܺuKqOm۶53g KL~?`2"""""""""*ݻwf͚j'˥)ڪWΝ;KӤ@`` U)SHeݺuC.]vZǒ%KiڷoKKKhjjbڴiuÆ Czz:._ cZOb޼yX")~z ;vPghݺ5@P̙3|2d2ajj]vy}`ĉƆ lm۶rʘ:uT^z嫽 hѢ7n,]hjj?w8p,ZHSPয়~BRw^Tr\={6rtL044XZZ믿왛G*ܹ3z 6 %%%s-[Ve}v4m4ԨQC~CBBPti)ӧ޽{8ydڵ ^‚ +U9s &&3#GaÆ!>>LJ^z*SCrԫW+˗/9s8qJ#Fnݺ,Y k}˖-HJJ²eˤkkkL:w3gT7eԪU s1lllT*"""""""""|믿`mm [[[`ܸqj+++J*!66V C/7oׯ_γKKKˣVZRBXSXG """͚5Si\7onϕ+W+W7|$o\Mݻy˽{`kk+%2e&;RjUiL޻d2l2۵ޗyϟr Y_N&BCC#KB9&MdF͛7Gbb"ݻ9 e;wVhh($}2}(44rԩNV5*>>ԹsgTRuAǎb DEE8=^Q\9 ccc)I>|8Ə??\}:z|/(=111ضmwIL)"""""""""|144;z聠 _@ɯ\2߿޽{\rjժ頻yRRR`jjk{ +ZhQ1dwޑRlGRRT;*UuGZ,,,kVK ={PJ\ۧÇ^T ̾?7oHӂ>0!~$DGGa#}cNʕ+cbرHIIq~nݺ(]zߓ'O=UQܿ_+11ʕ+#&&*\SLQ ={ ..7oFr ܞ8!BES}:ĉ*Sx6mB ut5kL%K/\ЩS\۷oJœ9sK&@|||1޽=oܸSNUV9RŋgϞ?PץKc֭RYdd'N-==k֬SrMMM̞=;K -- _axΞ=իWX~=U*9 [KBWW7%;tt,]T%K^ xbri>r1;k~eҒ^k p5AB@UW\Ti{ŊP*RBCP*X|T'==K,<˗/׿_DDDDDDDDDT` Ц{ߴiӰw^899aذaŚ5k z1ahӦ Μ9۷cСycll+WM6EϞ=Ǐc׮]7n 1W͛ )))Xl1~tٳ' ƍQF J;v_~XjpEk۷ov T*1x`/ש,,,`9QQQꫯ;vѣGOcI&aΝpqqaPLXhQ555qq͚5Çqa/022ʵm۶mѫW/L:wEÆ q+VԪU  ~ QQQpttDpp0ߏɓ'__affZjaصk`[nc`ǎpss)޾}e˖vҺn}fΜmۢwHHHMr! m۶ɓ'{nL:5ԣprrBϞ=qmY=z;vD׮]1aܺu þ}pq,\0_c˖-puuEٲeQn]L>=vB,R0EDDDDDDDDD2j"/ҺX? .^yȑ#HKKC-0q<˩]PL.KeC???ԪU [l ```+WKchh///i5jҥKk.!P\9 <_}U@޽ajjիW#!!]ty`ll,qssS СCG@@J%?r;w͛q+WÇ~PYcYm׮]# <<>׮]+c:::… 9򋈈IKK~td2iQy"mFŊ&苸uԁ%c۶m1ېdXj)֘"""""""*bbbΝ;yNj*,]'ND~.}vx"""5L27o} 4i Zb#"""_ҾG}[N}L~12 }666EDDD0G=yD*F}L~IݻwGڵr>7ߪU`cc777\vMfffhѢ --Me_I0EDDDDDDT$rd8p:""""*2J%~cq}uV$@K//"""""""_G}OCCDP(T*vZT^}CT+4o 1k `򋈈>}: Po@DDDDEL:uaܸqCJc[.qu6k֬Qծ]? d"3G HtQ#"""ݻwDxx8+gHagg[[[uBTR(S aaaT\#Hm߿BTAQ=0sLԬYS~~pr9abb>9&b򋈈Wb׮]˗""*dpuu+zuD@HHuAD%yf>}ZnZSS:u*,-- ϸ8U&+d>)&(HLLĚ5kf\vM}1еkW5 0gggF!FDDDDDTcժUQ P(`kk֭[iӦr9J. MMM5FMDT|yJo߾Exx8xelٲ:tEPF uMDD${n߾%K`x'˔0a˖-hݺ5 ޽{CC`,O#H]nݺ௿ʬ0x`xxxrjdKOOlj'۷#55I&ۻ_bQɖ9qXj! ""+N82c=uM{J2~XפIԩSիW a``!>>_p={6KR SNŀ=5"_I1EDDDDuVxyy۷Ub֬YA""w0o<^ZZ [nEŊ/'''8qBQ1vx{{ѣR.F1cƠB :^bb"|}}`rNNN8p :uꄲe9|pel޼7oF\\+WD]]]]~~~dx:vHuFDDŐRЀ':֭[ѥK{ >>>8}tۣ§s̙3_]tARRRm"""""boÆ ҈/kkk9s666 r4qD,[ ;j L::::9r$߿@)A10`BWWwƌ3P( 1rM4˗ѬY3')>&X}6 (_4uѢE戈K!q  SǎѳgO;pL~Q1xb5GCDDL&äIO>Ŗ-[}):aiiހ,XY0EDDDDDХK>|8455 IDATQ|2]0J0;"'227nP[z)`ݺuj, .cȑj> 8x n߾ DDDDDDTp{( '_WOpuua}HOOhiijՂʖ-[hÇ_Me: !cԯ_666*ƌ۷OMѕlD*Uk׮q>|Eѣj*!qF&ڸqcTR%J%=ڵkQFYZWJkcǎN:ر#RRR_ .ĕ+W`ll\h}өS0`ĠbŊiiipww1vXuEԩG%v7n܈ٳg * u놅 "==ǎC޽`۶m~h>w!-- eʔɳnLL sߢjժ^Ο?~/"""""*^q@V>;v 3b~~~5k0x`lذGVcdDoƌr`oo]]]$&&̙3L~'uQU?+  i]<~ӦMCٲeQT,[lYd_гgO?<_DDDDDTܽ{2$/_&M֭[*3f@֭\ 40rH}V]BB|||PV-q.] WW,q,Zڵ>|#FK;;;Ν;R,hܸ1`jjaÆu 2u<~+Wk׮(W*%#3 !O?FFFh׮ܹ''',[Lɓ'g_~Y###ѿ͚5t5o*U1jժ6o }͛7 :ӦM`ذa]6accoF=7n?#/i۷o1eXYY5kSNbcc1j(TZFFFƠA 9oc7h=^|)oAHHΝ;'ŗq]xzzz022B:uǏ[LL ʕ+߳;~8addjժa*azn߾]DDDT*q)Zѣ,|?6mH͑_DDDDDTܻwOI*, B`` v튐cݘ9s&VX!M? 00#G>|___\~'N\.Gjj*pi5 ZB||<ۇhTRѸ~z8?~,׮]3g<==aii 011;K.ň#0sL<~z*Μ9BQk*Sh׮7oe˖ASS+WD-p%ԭ[9s`ذaС.^WWWxBJ>,}߾}JRھ{.`ee[nEpAko޼Av`cc>̲GΝl2 >\Z3t9^}? #G`ĉB@@͛7 6l#66B~zzz?~<6mڠK.[[[NXX0|p`pwwGXXjԨ|2РA,YXf pi4i$;{,7W\ܹslڴIw ܾ}>}:444?ԩݻw}vx{{bŊXr%w{J#={_5AAWWϞ=C֭!YPlY,_;wN:3f|Ԉ,]Tӧj/Qj,Iuݻ' x044}b*}| ^R/44Td2R~@ݻ7P(DDppׯueʔwޕڊVZ4,%%E؈>}!xCU_SL{x5j$z-mWF{NpttT9x֭DLLLC!N:%竔B&I%~lYH.HMMͶ!hٲT^zzzbڴiRYEƍEJJT&4h ty~hŊBPW^Ie+WѣG*-[V|7RY6mJeJRԫWO4h@*6lX B!nݺ% QfMlܸqXhJ}ҥKׯ_ !Xz Ə<4i"u뵈ZZZb͚5*}gZ]x7nP)={/_̱ \.BBBrC$:v옯W^DppTַo_Qn]i{8qT.Zh!ʗ//J*DERTk„ .p[*P o>sŹs СCܹZ BqFP(DÆ ŨQDϞ=\.ʕ+puuժU?pww2L+!5jr\ZJ7iΝD߾}… (W^^^b֬Y9ץK7?ԟ߿{*U044ݺu̙3HÇ={ B!Zl){8s>1f4sQO3HF8 yJ_ IJBk409Ǚayfkna{uw=̴]vmӢE l:vhzm-[VkӨQ## uܹsK2isAn6{_),,L|qZJVoI7oմiSٳZ^Z5(!!Am۶J*]ykϟ/I뮻j Q啐` cXEiѢE˗Wllջnڸqƌ}9e n:effZ=BfcǎY>={Vqqq׿Zxl6#4bĈ|}~Ir J$B ǕI&iҤI\fΜAcǎS֭5t WǎS@@5d/ *T$]Ґom3oy̻o|Sdĉ2d*VB ֶO<ԶZj֯^:\yxxڮZj2h t~οy1 M~Q)S&ߗ,y޷oSUrB6mRǎu)5jH*UR@@l6ӼZ9tvܩ/5xȑbS>}UP:V?,ݮHUXQ~~~rss+rcǎz_::z%օJr rj999_+rT7h +9s(33SG/M; {NcƌѢEԮ];}r8zW'<==5|p-^X?ze?`R>}:Ue+g2Iv:'Nhz嗝>3CGܹs¯M^R%$p]pTk+aԩZ~*U7xC<OgϞ.ޤ"&ÑoyI___yyy)%%咿$VPP/{1M>]={œZ=z(t;cta,nnnqq7/\1tY:uƍ/hĉVbb5/UILN_ğ9sF(-yfzꩧ.jt=z[WXٳgkoӧ/]޲ )h:???<---ߵ)h?F)%%%9UY???jJ˗//Q E8pZhyY_|߿_}Y1dժUKД)S\kr]7tS!ډ'JT_}vM)##%mVZN?رC6ͩ'|#[n߿Zv{-Kϫo߾Ŷ+l]w܅=ԯ__[[y݊iה ,,Keݺuzgkʕڻwo=t/EnI&ڵkvQh+*999_;Gyvv(eggkɒ%%:xGÇ+++Kv`"~ʕN Çd-[B{t㙗&Mɓ';駟4`ɱ޳m6Gy.++un| _& uaŊ rͥݻaÆ7njժO?T} IC=zwZlܱc4qD矒~ZzG短К5k&áQFI2335fmܸѩ[nEnnn7o٣={(++K;v?O->~&Oo;%a4i|||4d}=zRSS%uɒ%>}wn}z7(CYf׏?hO~ĈZz}YklYF'N$%$$諯w8Zp$YIsgΜ9+pIҥ*W^xz^$&&W^Qxx%r!ͦ5kjڱcsQ ~AuYI?KW.///M>]r8Gرc:tcjƍz9=3:wfΜyo߾ņӧ5`eddnkʔ)O;Ћ/h~W;wgtIiɒ%wV޽ YfMhڳgu?/ct颭[Zbǎ5jT3Izu v[+ @5k[jFo[5kƍn:ٳGGt[=֯__|^qO=1ׯΞ=+cΝS{V9x`ڵKz땐oNa$.P.VxxTbEi&߿FgϞվ}ڵk+;;|'sã"WCZ 68-ڰaMNu- p9H2<p$%%Y1Zٳֺ8p1Ƙnݺ˛8ߧOonj1e˖F6~~~FXY~^\\[$`S'|H2s=_~B N5k۪yڵc>lڶmk$///y^M>} |y&44Ԥn^}Uc$rY=zΞ={Lzuss3&L0fذaVsΙ#ɔ)Sl63x`ӰaCӽ{w:>#h$@cٌy'1,[ZuOOO3d>>|?4̢Ev+W6z2M41l6ӿcۭvZ2mڴ1>qss}gҬv>QF5M<wwwkl6yMfff&$$Č=l6vmNO?H2'O̷&MΝ;[?oٲTZxzzefn4h0wuʗ/o=gjժe$e$:u"̙3M7d=M``9{1Ƙ___og}f$?޽{ )_ IDATMddU_v1Ƭ\oooG2eʘQFY?#[oum&((ȸYWx<| ~8<ӦN:E{AVj$~+::H2ѥ] ď?hg]Ժ .4>cɓ'Mr]weN8؎;Ϟ֭>K\rРAfxc13^^^C&##cLrr;Lr̙3g1&5Ƙ^z{.n$O>vdbcc=VJ3`|;t`ʖ-k6l`-7ndLb-]R}Sq#K4j(n4j\*W#Gk׮OTcS e˖UPPΜ9#OOO!ФÒ;wNNCqlݺU[l7tGNNVZ'OL2jڴivء-[Fjذ222Uf vam۶iӦM7߬ 䫥my{{; '##CN9}VZow?+++K7V*UT\9_n+11QTDDn=\?iiiZrRSS Xs՚7\ٲeum9MpXP5k|='ʔ)4\NNRSS]œ;wNsܹslkaϞ=8eff*44Tv[~zYfqV*U衇;Ｃ߿_׭괭֭[S-͛}vU\YQQQNӕSh}Oʕ+o=C Ѵitq߿_W˖-AVVU\|Ύ9"8xeddQFRRRRﵑw}m6n+>>^V``ԩc͡WtXB*_9zᡨ(-[VgΜ_!233{ԩSp8ԴiS/_^gΜɓVʕw9*33Su?n+%%E>>>UVEHҞ={'Lx]\ZlXEGG;0\;w>gL2E})-R~z~N/ڵ1jҤo>mڴI˖-SV gϞZpt%$$hȐ!?~>LSJ~MΝ7|_៣믿iӦNlٲEM6nWJT~B?ԤIyxx?z)+**J{TZU>ӼtjJw߭SNiݺuѣOnN:{駟j?áp%~JxG_|:z޸v~ڗ~{E0. pظqS(7ߨGzϗx;|'܋V\\m Rjj)SxݣG*..N111ިjӦM_޾}{kxk*SΝ+ݮo>nݪ S׮]n9qbccժU+/_>'NnSq:uJ?CuԩSt3$-X@UVPZZfΜ 6W[V۶m,\P!!!jܸqufgBpݹ{_ԩSZx:tP% -**J!!!Y:;wC׿uQۙ1c$RJrw߭E]tZAAAz'K|=*<<\>Rd=!!!%|z'JԮw}_l¿短~tYJKKg}Fu5j6lXe" 6L+]߾}brU^'11Q:vtwy,ݻrJIR= pckӦ-Z={hݺu|V͙3Gv_;z衇kڱcGyp4hPiKп{ᇯp%lR-[*BթSo~뭷p8dJt9WK/phڴiW-*SڴiS4;vP||KuiȐ!rss\vޭ?\ԡCvm[PjUkS*99VZZt5jtUw#xwefc=F]J:?GLllldTffl6FQn $;wN|A)WKqA&x@#p7nd3<. p㏒G}TM4)76mX=Ǐ׾}J \^xfذaDu7[Cn۶MO?1\mۦzJTB?+܈>)==](PBל9s$I={L/׵_|Q۷$M>ݺpmKJJRv픒"777} -7;S}$-YDof)Wطoz-I t uf/Pݺu%IoƎ[UrQkN$j۶m)WM0Aaaa#G~(PgϪSNJNN$M>>CjZtԩ#IڷoJ*o߾;wΞ=[ʕټy}]5o\/#777O[lQLLLi @Zl~풤x5nXzҶmJCFF>CիWOcǎUNN5i$M2Ek3ƘX+n0Fѣ%I<PW&Lо}PժUUvmըQCʕc-H:uvء]v̙3Ns>|"##KJ\ZlXy\ZѣGwQNNmݻڶmRu?Ќ3W_cM6դI@(_pӧTvvvi.o#<{  n֭zWwpXm6մiSթS.RjjN8;vhƍZfRSSr->|z%7 hq%kt.](55UWo۷k۶mڻwΜ9M[p||| N:[բE կ_9.#<<\s֭[j֬Y:uꔌ1ڲelR% {u]g~]vj׮]i7/,;M +/<<\'N￯x-^Xqqqڶm?^]UNEDD]vjӦ*Vx#j޼7on-KIIɓ'%>C}Қ5kJ'~~~ Thhh4, /lٲ*[liOhhC 6jԨ:._2/ / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / / .޽{-I RPPP)W\{u"<<\aaa _$/ =֩S'Y?w֭}_\իzvua2222222222]٥]$)66VgϾMII̙3ubۮ[NSL5\mqqqZ|yڮ_^}6mt\m_!ũJ*+''K$}5jeգG[Z6c ͚5+_Q5\m&LЫjo>7Nǎv۶mzW͛n_RWnSL:իW߿KBBB+N:ֲO?TSN-Ū;wjСJJJ?M6/ԪUvRp%~paխ[BT%''pn+996JNNVVVEq1*T1c8_%aQrr%JIIщ'szzz̼^;CtС_CBnn<==/jkתe˖ Tpp*V?Nmh-^Xr^zIT`UPAof5J*zeIII VTTSN:$رCժUj޼5'V^yml޼Y V``z衴"O?[o$eggkذa Uppʖ-zꩧ`jժꫯ6+W;３o!!!zw kܹҥ$e˖1_ީ]sZ~+7ݴi7o.OOOڲe AiҤIV4m޼Y>}h„ v ,P޽ժU+;wfΜ Y4h Z~g~ѣG%IZd7n,|s!eddH&O5nXK.ҥK7ݮ~X<ohz/''Gݻ֭ XGĉki֭Zp86mX=ƏJSLџuW_3gXqKRttƍ'I8qu `/p]pj݊):jԨ~Aޒp޽[ǏO?mMNN֊+ԠAIR&M4w\}ر$iر _-&IUfMUXZڴiW_}UҥKյkW-_\K.UϞ=zjeddz*SըQ|v9R=$;ҥKjŞN:?~^f̙oFݺu$կ__sQxxϟݻ?PӦMcYFFFv~6lX`]w\uj߾3ϔh=ݮe˖^zӲeˬ!!!ڻwRRR5j԰/IlW8 |Vҥ|IR*UԴibiժzjIlݺa /_Kl6ڷoﴬ~1'o</fKJJRPP6n(IЊ+ԯ_?ZgyFǏ5mڴG/4jڵjذW^RRR8+P͚5uI-[VN>-I:y222 UR%|zt^*W{:_b1En_<'N(55U-Z(QՒ+,,LoF;ƍI&ZfU%q9m۶M<yBĕb;0uM#GʡnP{%믫uz+VZJNNֺu딐 &\޽{_֐!CtY=%.&&FZl}*U4XA4h>HsQrT|K#8wn 0@銉ۧsk׮jҤ͛;TPPh"qN8uT=jԨ֮]Cۻvɓ.IjҤӰTy +pK 6(==]}yURcǎUzzu 9s$L2^JJf͚7WmڴQ6mkp84eEEEW3fѣGꫯjRV4}tmڴI:uRLLvϟ5kpI&ڵ<==}/_\gϞU_jƍj߾:t萯μ:6m*???͜9SGш# ͙3G֭Zh_nn/_|u*44T_~9HSN;uf̘m۶)88X۷WTTS)SQF*Wf̘CP\\$iڴi=ݚ7o٣*Us wڶn׬Yf+WN=z*7422fj׮]RHHԩsQՅx zU|yծ][իW׃>Xׯ*UfY=$u֒VZծ];O111:tEGIi{տ|ͪT.XܹBBB;CիWԩSm >\uUUBKǏw !!!O+&&F;vpjw-P[oUh$MƏFiܹڴi I4{l͞==sLկ__~9J?4m4ku]>6oެs=Zr߼04rH#T[nA;LBB9p[u;v$$$SN\ORRRLBBټyIIIxjjٺuILL4Ǐ/t;26m2999UVI&66mڴic$d1X#G43p@kٚ5kJZƍ$3k֬[o5 6tz={dfΜi1&++H2fN۸$3tPp8#F0ܹse˗/7nnnֲݻIfnYoNNUG\\c[6r9|lŊü{ֲIfֲ~H21anVSjUsA݀$l2kqwwwZf1|򉑔w׮]}d>ckxi$"U;v8]Kh#DGGv)Aͳ>$&&v9p{#nZҖ-[#hС5UڵըQ#UVM *QF [۹)SF5R _WDDBBB N*Ut뭷Co߾ b?k&_j۶-!v]SNfSDDDYf6oެQF9OtUZdS]YfSlY9ҩG3ԼysuZ֪U+uA_Sfo{vEw}ݻ|r˪TEj۶/^~y>hBs̑$m޼Y7o /UZƌ#???}NС38ӦMn׸q777k̴zv͝;WaaazꩧuNuҥDSzukHƌZn}ѬYTRR׆L={VݺuӪU%QQQ$}:|o5jB>#UڵK^Æ SƍݻWvrj_\aaaNs:޽aÆJNNǭe5jP@@@j.xݻw;/;wc Yve˖UXXo^d=EٲeTgf]v)"""_ x7Ox{{kܹ{QL=Cz衇J\^{}=w")P Ra|ظ8?7H1')r2 }B39)Ƙ 3sN"b%E)ъN>zr:}r|=,Ç[bל9s2z$V}5 J~ry3̾o|<>K 80ڵ[bn޼RSUV/Wmz?Z{̾k$6mbۏ8TTTdXl;３>(=;Kx㍌7n۷oB+/w~LÆ ӢE̙3'IrYgy:L dgrW|{˦n+kf~L81g}vN>䕚eذaӿ~iܸq?fuiРAnn|ͳ{]vyw#_\:wg}A7n\6pÜyI>_URjmɞ{_=&LAo3'kv뗭ڪnm. <8O9}M7ݔ_~97o!CdM7Mj8qb޺Q/XE|첋U5h ]vYz衴i&{wGL:무h"W^yeN;4 j*W^߿x{0`qҸ :u6mڔ{w)di۶mZhQqWB:wf͚cǎԴiS__駟.M6"@]ᵇ6lXƌyo߾2eJG:GUa2|)S`q_ _;wI'D|U9S3gΜ ><ӦM˶nΝ;I& ^}r@_@TQQ~cP`_}:w\1VȰa#Μ9sϗ{$XaPu5믿~&Oɓ''I~䡇Zذa'IZj-2͚5+P@V[{ @?>'tR . <8SLI޽Ce6*X˴*~W,MrErꩧK/ML6-}믿^橾@_PV&|P4ԀU! _5.0 &|Pd԰/NZP@} ~@-)g//E`6@}#~@F/( `eTL>̪3 _w/:Lω_PL_PǬH`qA/XuW0 qܩ9sd6mZvu>˓$;w/Yuܰar&ILXz衇ҹs2Nu/D/㵇6lX4hc`)Uȹ瞛| V1Z*Pg_@a_@a_@a_@a_@a_@a_@a_@a_@a_@a_@a_@a_@a_@a_@a_@a_@a_@a_@a_@a_@a_@a_@a_@a_@a_@a_@a_@a_@a_@a_@a_@a_@a_@a_@a_@a_@a_@a_@a_@a_@a_@a_@a_@a_@a_@a_@a_@a_@a_@a_@a_@a_@a_@a_@a_@a_@a_@a_@a_@a_@a_@a4.5\Sj\nңGZPˎ;rP4io9{wkv ,ȁ;Vk@ :4C-ծO>y饗J6W_eҴiӴnݺcTmC̛7/_Ǝ;T 70wqG5kuQkPgL:u̴ijx]wݵɓ~syꩧyޚ.]?a>ZULʮUV{SYY>+șg {M˖-kiJVVm0 (mKL81=ؗ;nܸkICfWzj+_@piyI;o|g6lXcǎ0`@mG5&~uB3p$ɽޛG}tcZt:cȐ!_b***2|$V}j2_@߫^|Ҿ'V} 2ĪU\M0 So.m꫓$:t j"_@Ҿ}wqI{JykڴiYf7n\4is1;Vzq6jqꩧfԨQ?~V[mTVV"۷Ϗ~rrkr]w-믿~x,X x`Nvڭ}/СC/< .,m?3JD:cK+V_?> ,'|R:iСkJC!Rr!+u^3~uoIkv>O>w}LPg~iڴiڷo{V^{Y;v̠AҹshѢرcs/u!#FDuiC IV=F̚5+zm.;N/Κ1cF{ｼСCZn]NoB|'iܸq4i͚5k?̞=;z7o\3gN5kƍ˓|Pg̞=;W]uUkh"۷V[m-2mڴI֭s=nsfɓӫWh"͚5 'qݺuСCs 7}iӦMڶmxtȑ#Ӯ]i&-[n햷~{{0 ;S[niժUZh.Ɵwi/***2rlF;|}c9&mYn2L[w͟??>W^3f̘TVV.vٳssɅ^{'?$ITB IDAT?=3?2nܸ<3ĵ>[9rG{/~̙3֞(3fd}_:oʔ)9蠃r뭷[򭪺uĶn!vb͛C9$gqFi~%}+SNIw߽/φn뮻. 6V[ms裏_΋]kʔ)s|'P63f;^ziq7g0aBE]:,gϞK=裏^_ _^~헆 %[ϣǯVZe_=k=k*>hNFago6m/_.\> m۶]l7$ӗ8gK y;Ogyڮw=׿u]oUոq Qڗnoݺuf͚ڵ[kԺ^z)W]uU_wذa9sf_ҥK{ŶU}x1rC]ve~=zt~ӟVmy־}ׯ5`wq6lX=L6-C lWԪ [o|ͅ_vvi_j; ><+n4iҤWUO?L6Ʈ_N:,Zh<Ë;bĈ̙3gל:uj/ߠA\z饙1cF&M^x!/rr%馛Ϯs+Z+-ZW^y%;vkVZkrBjkN߃Ij'|R{P7_@Zgu q& U뮻nߣC5~& U;S5jTco۶mz>uԪvڥG5v=ܳFuԺG䵩/q]oҫWj.4h+"˂ 嚭[E]T-ת-_|qrQgTǟ (={sG~k5i$wyg6hj̟??/"~esGd̙9SSQQRh޼yƎJc-u^˖-W< ('dM(o 馛榛n6lSCՌ#Vr^_u]woo~~ _,+:Yf2dHN;^=.$I/]u'I4i:_@_^z# `%~B=ҿ$ѣ3}_pIXՉ_@1bĈ$կ~U>uqI>_թS'~F=ү_$ɵ^[~}% /^Z駟_JqV}/^ٳgz$iԨQڴiEQF:thzg?Y"3gLuQx9@Ꝟ={o߾ϫZ<2N@uz鋱ΆnXi.P/?{NƍsYg{Irf̘~8&MʫiӦeܹ={v>r>4j(;cGJ 4Țk5\37ҵklׯ_6|4hР#¦M馛seѢEk.IfΜYC=T7p9CӣG! /` >}bV_}lFYguҼyj*͛7/ӤRQQٳg?ܹsꫯb￟#GfȑfmrgdKÆn OLfʙg:ݺu~ejV)ɓ'Gmݖ &d…ygsf˨Qvە{LZ_il2jԨTVVaÆ92iҤ<93ӽ{w᫖m9swfҾ}$ɤI;dȐ!Ypa']T9Ӳ^{K| /o9yW<@ 4(~{dϣ>.]y*VT&M2z{I?w3cƌ2OP/䷿m$6l'f*T|Æ ˙g$2eJeּy3& 6̧~O<# ʌ1"I:+DԔw1 HL0!<@;K/%IN;lٲQ:묬Ie_P]{I֭[c-4Դ;Ou]1cF'^wߝ>lϛ1cF&L$9蠃ҢE>wʔ)|ϫ&LI&y 4{$y2o޼ l/XرcF%Ix}˵쩧J#5Z{i&ӽ{~7n\-Zkc{O-ܒ7|s}SN^&IL6m͘1csɓSQQYfeSYYɓ'磏>JEEE}o\8uj{.zb1cFnr-kL>=&Lȭޚ{,|Ii̙3y$nof…;뭷^=RO~$.g}6w_Uw\O/RSO=5?S8G}t6h 4\Irys92`;:u*⋗Oݿt.'O-R 2dS[n_}@]'~@Awx㍓_dM`ӦM˂ $]t$X+"^xaFyeܹ_:W]uUW^Iݳ>dѢEYhQ㎕wߝ {-m~O<1s93se^sܸqYwu{e…}3jԨ 6,W^yeϟso5\_:sr?̙33o޼3fl&9cJ?뮻nҥK4h$~;vlN8ᄯK:sӷo$?׬Y$o~&IZjs9';C}+پ}5*M4),=~92v }^+ٴi|߯g^M6M-2w/j UV裏k#_ڷk5Νɓ'O>+-Z/N˖-$zf̘w}7~b;yWhѢ ЧOnw;Xvء䭷o>`z뭷믧2o}:tw=O{iѢŊ>VhٲeΝs{j#~@=1gΜү4io}[y뭷$+u͝w9|{>yӾ}=+++X]fe6KEEE7^$YhRK޼y>WZgoݺb깺v횮]fiٲey׿.t[}3t 6+eWgP5jСO~R1g}6#F#<ҬYx9S殻Jx[YkFl96묳No߾Ok,-4m4[b3+kN һw >+m׮]F#F>ȹ瞛Çw߿~ɪ~߫VEj֬Y5kV1ꕧ~:~zҶu)]vI>w߭{w) / <8ַ6d̘19餓X}UQQQZuԦM~TN2u|i۶m_UOd 6WT۶m;GU:Oȑ#K[ԩSfϞߥ={vxeKѭ P =P=~|{wRj֬Y)S䢋.*$YguJ|Wm!CYf $կ2}3fLp 0`@9y;S93v$AEeύ7ޘ+"})*++"3go7.$.ꫯ΄ r]weYs5;'I{l2p@nᆌ92=zH6m9N;픉'N駟[nenbPY~:Æ w]zVz-M׮]== wwΖ[n?YfgϞy'3|}ٙ?~Zlm6,W2G믿&M$|emݖ3<3'xb:t/<׿2y:t.; |'A B@6U E@TvZRZh%:uAu:6 UTR@*VBA@B`< ɛKf}dwuiӦӧO:tL.]6޽{L6-̪UҲet-gyf[n8qbn,^8g}ϦcǎImfܸqsfː!Clٲ<35?"zh]fW_}uڵk>+VH۶m7|̙3k>ԩ&+nOϟ$=̕W^IG^z>|x4i%Kdml!~;쐅 Бc;2[e] oga+3(nWSO=$9C_@Qq!|u1of***6޾}m۶YdIwq-c=e˖%IN8[v~@6Ws(&M e,P~{u߿-4rgyfd͚5>|xPߦN'&I '[%ՅV㏧sɶn[XuN$I/_/OfOgϞEXrezꕗ_~97n\:B=$Zߟf͚eO~^{бY&?/'I.RPԔ_@/zkJJJdɒzVX|Kq &$I<\{NP_@- ʈ#RRRKO> oW^yꩧ$|pƎڪ E]QF,+WI'SO=5K.-t4~w}3cƌ$ɀ2qĴhѢ ZgqF{챴k.I2f̘t9zk֬YSt|_|1|p8|)))y慎 J r-\0gqFsdwy睗N8!:t(`:V^ &dy'jw1v[8hx/`$s3eʔL0!˖-~yyy?\}Վ9%֮]{7#FٳvMӬYNx|G_{"~z."$Ш)oe̙{2iҤ̞=;ׯ/tF}իW~_䨣R:Dԁ_yW2w,\0+Vڵk k֬Y1c$Iz ZnڵktRH[(-[,In)\pAPJ  h((/ h((/ h((/ h((/ h((/ h(*IDAT(/ h((/ h((/ h((/ h((/ h((/ h((M VZ>hiҤIZlYOJ -[,;,O?t&.8CQj޼yFp ;wn=$.(FiӦo~w}/-c(/ h((/ h((/ h((/ h((/ h(بʔ،SO=5ׯ/4J/ccƌQ@4)t;Z[/fԩSN˖-sAcIEEEE/ٳg#GfIիW[oMbKӳgό7.ܹ֭s6Ç/t4 {/I{5͛#FK.СCSRR$o8(={o߾_Rܯ/ׯ]__+:'|rM$o}~{e]Vsbuuu?zh'O΃>ŋe˖0`@>蔕լYjU㎼ YvmNKΝk^x!֭ÜuYiٲeK/e̘1YhQZh?'xbc׮]+SLIUUUڷo~ubߊc6W^$Y~}Zl׺7ojc^ 6,˗/V]]aÆeԩwٳgcǎYbEN9̛7fҥKӣG\|)++KsNY駟w޹KS^^6mdСݻw.$yҳg̞=;kJKK3lذL2Vs=7{n6m;３_כu/MЫW'O>d>$I߾}7ء7]tɅ^Gy${Gnݺ}y/buY[jv[iժUͺ+"?2eʔx5_~ϯ̘1#?|o$Ʌ^=3v[;$]wݕv)'ONi[n]>ӚY}Y\ve\lٷ=c&]^v}mC9$f{ҤI\Qe]ҦM$;vl=ZW+I߿ֵ=#ݻwϋ/Xs4K.͜9sj:,f'|Rsu֛;uK;kϩUV>|x}̙3'C +?>՛57̎;XY[iٲeYtiso\vڼ;Gy$w}w 8fs 's=n)v[̙S+ hxv~l<0_}zY();C*ɗeʕ+kּ׹s,\0 .٭[;쐗^z4i${ҧO >zQk[oW_}5{GƎܝv)gqFΌ7./3s,^A'k_>'OGɄ jyeڴi5_O>K>?2*m]͵UVmP}v}k4 ˗/τ 2mڴ|YdIڶmۧG9ꨣҪUB"\2\rI*++ӹsh"j>~)-W:uJ߾}3xL0!+V)ľk׮9rd9L:5{W,X3f_) 3f裏΁6md̙رcz$K2} 0 =zȎ;{//Rnt=Ir5^>쓶mfڴi裏2x\qܹs3q/tP}FToSٳgꫯf͚o\״i 0 ^{mvڀ Feee$ɦ8<7o^{e]j[jUFye}gJNj}7裏fi׮]~ӟ[n|Q@M2%+VH.]rI'ֺ'|2>l,Yv)~xw^> ƍk>,KN9l~K/I&O-ҽ{wq)+++N=Ԍ3&;szo5P~l`չ 2z_~+++˙go1͚5DŽ ks/ Ew޹[7Ju[nESB6Fo+WL~ /Os/+Wdl*y晙5kV̚5kV>:S~$y'r=̻;&Mәl hsUW쫮*2 )Foԩye3}z _@裏~%M2^|Iڂ _^EF4z-?u~C6lX#lcFE:m|! iʫZo{̜9%;Fs:K.:/)FcG}tK=%KdvJUUU.//￟6ml6dm6_|q̾K_ /$˗/ONv!sIVl&g@VZezd^3~x@S~~헑#GӤI92o%`S9?,Y$m۶MӬYB+z!SIENDB`gfal2-v2.23.0/doc/diagrams/gfal_access.svg000066400000000000000000002774211465240014500203040ustar00rootroot00000000000000 gfal2-v2.23.0/doc/diagrams/gfal_diagram.dia000066400000000000000000000054111465240014500203710ustar00rootroot00000000000000]ݒ易<ݲ$۳5$M.ld7Y .&ϓȋEmlpcv\5u 9zZփIG#,hq;ۗwǫA[Hoi_D޺ݧω$b/ 4} |/b2SaER܎ב?GfqK oGg4.90[SCWBN\Iϫ!55(Z||:WՁ I\Ѿ0W.X).0M+N+.HVL"q /ʥr-N̼P-C,Ao &'m޹ o҈Y?{I]=$4UGDig?w3ɬ[|Zx]xxX_$o,򘚙a㷴;b];`g!?IRܬB/Y5.ekK-b헯֍>HD蕇(vOQͦlVfYa ruv5oGdL": /Zb#9L0M 0;QR荅E(wXF|Ser2۵o}h Y"Xܧlp((we)GO1°2=ND )bzv%JzyTklUӿA6X!턖h1*c@cU؆)}]DKQ}=E^l(>(NG?le[7#i9 QΎ/:^s &_ƑvAf{@@ҵ[H $ĭ@ ϯ QaB<XwN^H YCmZ_<} 7áa;t;☼uX=C3CntDs â8Yy3?>0~z@[n_IQLnw˽|c{_4΢뷷#g!+~ߒo~z瀺%veD8˰QSGa'Kŀ\4SP)CO _N:<舱a=j=M&Ќ<\LYraN w`]%G3 xh|f ,XVt\ Qߓ{8P:mVqw]{0`"zֿ|ZHM>t]|C+'cM!6biG%!8Auؒe|.4{ je@Lg52)?JK6@o h9hH,:_[mBnojIZ'p"+=c!^DۙH: > tbyp;P.k b8:J +RӘ3(۰sk Rʿ&6Ͽ 3kAU Ȁ7L)iTLkϓs(؁ 'l]?VB w a!(BBtNlfH}; 6GB9b!ANp+SsT՘u_@q !b׆5`5(*lp{@v OlK_!N&]E;> ,3{z d7 LuP'rX-6\@d[0eJ~eˡhҖM\ $: ؒD٦4Ti42)9sJ3Gmdk49X!1Bڙ`0 L"ڮ`K9ܻE^iJDfU&}5,.8wI13rP2ܳ߁n.j:M@؆]5VVFc켅2Qιh)RE9l.v[5+A8 `ȈQdY)2:Wy[ܛbHyO~eƬ@.^7ϭ^XkfDKKiGXM( ˊh\V.'r\3^p Ս/!9'Mǵ`DH]QG5GQ/rLzC AKQ=Ac,.tn kkW1ۦ .S a-t*M)!2sS|t_zB~7ԿG9lgfal2-v2.23.0/doc/diagrams/gfal_diagram.png000066400000000000000000001240521465240014500204230ustar00rootroot00000000000000PNG  IHDRnQbKGD IDATxydeHD6Gh-UK$E~oh5 Zj5h#KCZAPED*ܿ?1I,̹̙}]H$I$I$I$I$I$I$I$I$I$I$I$I$I$I$I$I$I$I$I$Ijr[g#o`EJ1*]MMsflI% &<}Bة_\DdU0G(I$Ik;"S *b[|<t)Ϸ짜c q6XkLs#>`^i&moS8%4O<2¯&I$I'8! Sid9$mBB,$nJBeI%[fJKz$NJ`?vh+b&)cڤ$IRFl< L TR$QM:p3pV5Q%8ZdT_`gsD[`$:Ap>NJR$$č$I ^H;Pz"fd5iEw7݀Ax<1q#IZ U6$zx@*o6h>cBm' iP{ڥ$IR'L$ܾ' qЃ0VT>|&mVό%H3{nJX=؝*#jTm[cy0ӟ m~ƻ^HJ;n@$&n$IRkՅpaڹ "}m Pyk 2.die}4C„c_>^{ƴKq7˄KW*UX _"TtVRy/8'gB+yCa~I៕QL%6!T,}|BUwO'̼ΚLI،0ds3#"#[4ИM ;&A= 1>I3(I$5JMWZ'(\ 20,%TUT,T6w 7Q[l6F'zzV䚕'48'銛ɢ|"$Ğ^b ލ<}AH |\u1=\B|NH0j|J=Y7}Jx$I/v.M!{Mb;[Sf?w[T^E_jƴqYҁ7bWམ8VoE]?a~BL) q~J|PqTL'>/U7$)fޚU<(cǘ6 iXe x0jwĹZj; [/v"ָ e)W;*۹cY .9Z f/Ǵooss .5D)Щsк*1!&n$IRkӽcI: Z_9XVS<s̋%P)vzRkO!?DPd`!YCfY \@xَ0 vB(C'6J6!%9݂D"MTwjc?,&̃t !!u!U 8.s %I$ B%2"Y{{2͸aCn*n↝h§3y(,Dd)}r<GCi'"N1}CO;0N42mws*SwCsJ02 W;TjcW{5Wjh"/=*&mt^-пTVH֦k̩zP EW%]+0cpaj(?E$JI1'il*|~M UBC鳡0̏b(@cN"Ԙ?RrX@~66X WQ$%č$Ijmǣ)Ҷ1r|Hŭ.5ؠZ a,$1T$CtLRm 0!_R~"nRڸ I[94yɲR;Qȯj9H7jr&n$IRkSEAEbKpо 3x CSZQ冗*mkB3()nYRmS0Eet7DgmlI?b>P3 A,)6!WQ\BB5W\r!WH\ebVI$I"il, RjQ146L FK5 } <$}!.qӥv{PMd")wd\&)]뱩aDq~g`ܶ*B]įrUR+ɷVN$6oqL17'|V8*K ‰! jlp F8opMmV*n)Gd>$nP܄I؎NDD+@2lCX¤]IKxW{TbBI'$IͳEuRم>CnkhM$%Bm}cPZ }j"nK4UM5+&5Q?5}Hfu!arz!$G.|_h%M L(sLHf.aU|Kl I"jZmcM p5A :_JPMbW#q&' /vޑB+C峊|t*Uh(;.CfGJæPQ1n}~8Mj8aSS Ln:#c{83mX1{S>a13ToRxESNBPhBr)oC=LcfkBB“oJXzؿb'}{oZ7$5p&~c: YA5bI%_KLt~OHt28t.*WYE%nvr_7c\sLBqۆtV涆?!T'n \1$$If|? VroŜGqӉn1;m2%\ =?˪I1}^s~/B"2Q*e\LqPLIh#_g^p%S!C_T b7y]s"ªiJyKq˪JЕ{Cqц(+n$IRk50K*-%|V6h>7" }!~(R>3 $# }rs…%>-Y_myw#\#,[w h,",9] K|l]ہM^w|&`t mn BHMU+C V1'hD^mkŠyu $I$#RAIp1,aEsӮsL=nMsdgT1ވJ+n <U7>BUw%Ό?"$+"TMBHvػv&.!z(Bb<~1 ɱ= |087~ \ԁc.381;%]1wK  3s_"L>/w!wL"ښ@wO19fTm:>"TI\a;+)nJ& )π_V2|9TN&TNĿ'mr2č$IR00 a!TIxIvPJ%;U":|D* WLWB5IS%mVtzW@NYUv'$A*5L{崓JDS~7$H^Lq) $qPp\<縑$I*Щ= ؑ>T…Div0t.;q^(YKߓ[ ?} a{ sCUj#_U?vWIyi*,j % M.1-^$T7^h-_|VoPi۔pgc$I$IT 2uځ]}m"7[ &I;  <@qI*bF| xx=&F`Fێ^Ӆ&I 6N;I2q#5왻V`dP4I3evQ Vh(%Ič֜ݾ@zI q] 0!peE'I$Pm@TP lBj{U="I$0q#5)uvIJmpPS*} Ss\sdmFF5x,W\\1->&wÊ%E3/٩$I$P)Iʧ-ppZ!$`>s$u&*bgWrq̈́%ϗ%ocVH$I*Ijܭ>2!28=w# }8p?a5/V_Mm&%ވP53:_f.'$yکT-3wj$I$ITPcXs8Rc u\.쿚VZ}Iߑg?@WB$v"\96rLsS6/ IJˉ[DFpa#/#T,}}Zk.WZEXuD"EsYIN7귡ÀSrpI$IRLHR~|g}_%LBZOX7Ն0QEqBn f*L1m/ƗKOMocmHH +I$&n*qnԲͳo45Wɛ1Y86߳q$puJ$Ič$5/pI#X?Ek폀r{7F 0ԫ 98~:Ou/!̭#I$Fd; U2Cs<;nmw*\nV;w98Y@<L p@wBes՞LJ$IRYq#I$I"ę$I$IJI$I2q#I$IQ&n$I$I2č$I$IRF$I$I(7$I$I.Zi I$I$I$I$I$I$I$I$I$I$I$I M;Ir9pI$ :}DLH$I$eI$I2q#I$IQ&n$I$I2č$I$IRF$I$I(7$I$IeF$I$)LH$I$eI$I2q#I$IQ&n$I$I2č$I$IRF܆M$I$)Q8w[j$I$űF$I$)LH$I$eI$I$XQn$IR tO;I,7$I$**%I$IQ&n$I$I2č$I$IRF$I$I(7$I$IeF$I$)LH$I$eI$I2q#I$IQ&n$I$I2č$I$IRF$I$I(7$I$I.Zi I$I$I$I$I$I$I$I$I$I$I$I M;Ir9pI$ :}DLH$I$eI$I2q#I$IQ&n$I$I2$ IDAT$I$IRF$I$I(7$I$IeF$I$)LH$I$eI$I2q#I$IQ&n$I$I2č$I$IRF܆M$I$)Q8w[j$I$űF$I$)LH$I$eI$I$XQn$IR tO;I,7$I$**%I$IQ&n$I$I2č$I$IRF$I$I(7$I$IeF$I$)LH$I$eI$I2q#I$IQ&n$I$I2č$I$IRF$I$I(7$I$I.Zi I$I$I$I$I$I$&$IlJf$I7,7 fp:$IbFxQ%RMb={6XaX༤$R&n$&=~!kTMni:uJЦ38F%III{Yh#I$鿵I;I$I$5č$I$IRF$I$I(7$I$IeF$I$)LH$I$eI$I2q#I$IQ&n$I$I2]H$eJ5K)S$ILH͓5ZhtԩhV$LH͓5Z&NrK$brnZ%/j$dW%G-IԬǹۚTPE$5_gNW_e]vG=1cp6A$) \UJRf͞=ÇӣGouN]]w}w#^ߒo6ی{$IjLHʬvQs9.裏իCcǎx\<Ӈn kZtϽ[RK,;dtܙvډ7x ru1p@ ƸqXlY禛nb}[np|'tMߟwܑǗ}|1_uU2zhz_xIDMjJeU-X 32dHt]wE:t(N8ᄨcǎQΝ-2[(k&ׯ_Թsh5}矏XEQt'Fg}ѿhc=u)z7ǽ}ߍvmh„ ʕ+ӧG͋(9FkE/b;D?Oϭf'f͚ 6,aoѬYlPX߿h̙ʕ+}{щ'X5~7V?H\[$Ж$I7J5e^F}f̘X"+w޹~  Dna4iҤhժUѢES6pwމhm/^h\&n54>,СC4cƌǮ5*+.bŊ{?_-[u9oߞߟC9z:  @v8묳>}:se<\veߟCrYg՗԰l2f̘ŋҥ ;S~o?N:$o?~sԲ 0w9 &0zh @޽yG^TӦMc… 2}} +_ SLaɒ%Ƹ曳[ӭ[5;wno_{̟?+WaSu(IR1LHJE5 8~۷//bEnx?~<=zCeҤI:ә9s&{JjvqG(^lYt)#GdL8ٳg3bAѵkW5ϴi{饗CE>ݿ$I͙IHf'㏙9s&kQ{7?8ͣ_~=:o?9#G^h,KJz>_̜9sxbW[x1 .dذat҅3f1ovqGpErp3sL&N 7aVS-JKԜ4.jV{yy9s4hzxי0a˖-cק[nt%o?_|1/o[nN`Μ9%+yׯ_W~iF׮]߽{wZ?p yȑ#KsرwqG?k׮5ŋp׳{r!3<38X$I;`GjJPrIS q}p vmՋѣGsWϟpI'ѵkWsnfnVڶmKr-?>> uuu3&|9O=*Gy$g}6{wq~;;wO#9SyԩCϯհ{#F0ydf}SO~cr衇6EI͈+.]jK;}Q3n8ƎvXj96 Ǵ ,+}7x-wtya* jܹ1E1tP.r<Ò$h oZٳgs!3q#5c^Hym< ^OW 0O>9vW^yq; /}tVz 4?|0|p)IʶV*:uJ臭CrQ9T*Q/5ίYgK/:묳˗/g?]tQ|9T!KU>JIjF[ T7RTR-j!~NHv>.]w]|ɚ=أуYg=\ve}ݗe˖}^^$IҚ\\Ԓ T,MW)nF%ms=կ~˱6$I$$eq<}=/N8}k$I*P)IRKw 8O/O?~u? jIz$;)+IjE*q:wm&WxA]/nWF?]wݵ3J-z30>%IL57m(ms VU1lIIjIXRtR iG>&ve=&W-Dv~{$I VfKBVj5kV˗A(-Ac\uK;I-ʇiь=lJXR2ۘ lo3pN|WߝC9*uR[J,o؄}IZj$nOl\~zt%Kr2wܝׁ$57+BoCmz衒7V}$IJ@ҫJ6g}>}:#GOtЁQF1}tgo<;WJ<=/ju]|PI|]wmH$IJf`ÇcѾ}nQGŝw޹FhCK,aȑdqS7\=.,3gR[[ ;8)3_IAvkj¥|VkT/Ɋ-vےOlpN/A5'VE|GI mvF|'_y饗X@j`Hٳgs 'ꫯ.#ĞSWWǘ1c8c_N'dS`O?;v|.`L*lx2^H>.m^ =m=>0$IRt7f3W4>t9;FռD<@TGM4k'ST_͛7o6/՟,)2Z7xc~N̵uKkre,X?z 2$뮻:DQE'pBԱcǨsі[nzQE5\/ܹs~Esm<8뮻p>>;kSNѠA_jkkK/4mݢFcƌ̙7M74z꿞}о}{9zZr%w_~9{'m۶ely9蠃0`ڵ㬳bW,I1önK۶m9ꨣx'JIV,n=ztԣGaÆ<^㏏v.ȵUCV8M$IRA/2 lw=o„ 7^z%KxbOnziӦpB R9ߡCˋ/GQR&o}ݻ3|V\IvITKN>|8\s vmڬYسj*nF>l-[VCXcQy1q#I$wܑ(x'O.xҥK9r$cǎ뮣K.|,\0 D׮]^9'N?f̙뮍۫W/>/5ɂjNlٲq\wu}+{ｼ曫APa͗UR$IZog}⋹꫙5kw]xvŋpB F.]1cǏgN,vqGpEq%0l0^{56h#z9?8?0C /dРAիco?z!8>5)5ɂj$nVg \oK;n.} qi I$Ion~ՋN;+"ݻwkm۶tޝ#G2D;v,??я6m˛9s9S8# ιs)p嗳;бcG;8^y&YPE؆P6G`K-UEQq4&d5,zٳ'{@ϔÑ{2sLjkkf6w"a)[bNX Y %eI[te;pmU#߿?cǎ堃JI& ]WtO6I$I=ɓ֭}ᩧbܸq;6R_Y$DeIR7uj]bܹ1E1tP.r'^s|MLTm 5Zϲ$Y?ڃFjLH͓5Z2Y$I1q#5O^H$IR+&$I$I8+n$I$ Xv $5dF$I 6q$I$IRF$I$I(7$I$IeF$I$)LH$I$eJI$I)h.LJ7$UHeoҥi \FNZf™TϋX"%IjE.&n$FR ՚<|r,MIU1xpU o&,ךnWҺ$I-I}6($I$LHj֤$I$UI$IiJ;!7[v$I2ič2M庥$IL{86 TlVč$I$Uh~AHʈ6i I$Iƙ$I$I(7gs$II9 xv $57@t쨫K;č$I$I:9>7$I$)R)SʸJgF$I5UfhV-K$I$eI$I2q#I$IQ&n$5Gs$I$hNN,9z&w[j=VBYv $)&n$IZTi%IJI$I$eI$I2q#I$IQ&n$I$I2ɉ+avaQH$)KBvʖCP-]*횸ǹ[%γi IRۀi!)[Nu?čh$C C oZO?'r4cQ&XjӺ$ZzcF$In-i|M`VX3@rhJ6*K$IRtؑ{ｗ)$ȊI$ Z#FpGbŊ1T47$IRHؘQUT JI$IR1aSbF$II0q#9$I7J7gr#I缑T +n$I$Yy#X&n$I$)&o$áR$ITgϦ.={E9౪'0q#Irt!\}r,Rֻwo^|Ţ?xg!KR=7{8o+%ڈ&hxED24 nҮik%m5(fE[W235ݵu]5.*Ąc```c̜9d|#"Շt "׺ڵkM6-hD*SJJ3nD*DDDDDD*f܈HU'!""Rełb!((|8b)ߛ ;Gyvm̙3D}$''.b :wLƍu,PFDDDDCf̘Avv6˗/'55S:ێ=ʦM= ~?#233֭3f 33?nfΜIbb"OW_t8""WP2"""""%RF С&L`ժUdeetlXXou||<O:Ŕ)ScȐ!DGG3}tg[ofc\'$$ЫW/ZnMBB.V5jPV-V+.fc?f̘1kCLL cǎ-uֱn:|||ر#gϞn۵kGzz:L2+9s&۷o')) ˜1cfȑ{x饗x79ws{~Ϗ+/ IDAT{]6K.}رf͚9ɓ' #%%ϳk׮'"R͸???̞=-v֯_ɓw-[,q? ϏӳgO/*m}]f͚Evh۶-qqqW9ԵkW 77q.K.]&<<(vl_lL0___8p`)f܈xH||<'OqԬYm~ל={;r-%' jriW8|0Frnxy]mVضo… ٺu+ϟ'##aÆ97nH=Hq4FDDDDCV+M6-4i_&''状 ry봴4@|}}ٻwsϫ ///rss޺ukl6W&==t97|sžYzfۉsl߾t1ƹO޽ٲeU/I|""QFDȼt y$''_֫W/6l@NN+DY&}_ɓ,Y .]ŋ1Ɛŋ([Po lX,1S矓{VV'O$"":ugVZU`ÇW^ysqqV^]!ȵEa U9n|!U?{K= pJX,̞=ܹsm#Gj̔)S4hPcΝKfر# >>K%I&$&&ү_??-11C2yd|}}ĉnj2o<ƍGhh(SNecͤDXXX^T>*Nj9B.NjHAp?ɵa kЅ}#-C}ٷo,:t`֯_0111Ĕ_T+%R`~bǎԯ_ouqa"""<7"UC&,qUՏf9rﾛ/rwLZ<7"U@DDDh=TRJ܈TRJ܈TRq#"""R}9DnѣG=TǏwMV)q#"""U}`;*c|9N7"n!HčTV^@;ƾ'T v)@P`p,""%ُ{PeOED(%nDT+/"׎ʾ짲'"""Հ*%"""""""RI)qSv """"""""nReCWGjG3nDDDDDDDD*)%nDDD"ƱzO""""R(q#"""!=HU7"UC&ֳ"""~ˁ?WDDPFjTs(i#""JTRJ܈TRJ܈TRJ܈TRJ܈TRJ܈TRJ܈T @MO"""""""G@'O"""""""GJJJJg >л},`&SAHSF#y3}B(wŊJDDDDDD*7"Or3:()DDDDDD#)| 38x7e0!N 589@ 8'T/O "W>0 _H KDDDDDnK?)f-ﲶۈT?)xӁT!(q#.RFrq5j0?'npwZDDJeqCDDDرE .ЫW/%nD*q$n6=Hut4e"".$$D)?TXZۦeR"""4t """SF‘wu>8Q`\""""""RA.-@m`+" 7"@Лk5B3nD*@߼mx.)'7@Kj);SFe_{*q+?9-I`б};\`'e/"R)q#R%"""R0e ر#7p@C>3gJ;O=]KP7,""čH-;﹖IT}Kzoؾ};gΜ/رclѣB>Ldd$7p r阮]tR/Z \\*?͸)**"e8a^e^8i[6m۶w޼ٙ^~ף82JJ(Ri͟????>c,r:uT~IH%7RM?R"U3gΰ|~6nщ?W>D;s ?8tR1c0w\>Zl7H0`ovl֬aaa|WG/^Ldd$գK.|.ĬYի:t`ҤI=zXJ?>,~iynvHKW!c+"w}gn+Y\wu1 QFqݴ7q,-,oq1~[^E~pnAYjAg$SpTgÑLnnt b\oȐ!{1~OLM:ucƌ1?pc-[f:d.]dV^m֭kkg{HHYdIc: 8:t|&00'N4z?irrrW_}e; 4-[4)))&;;̞=t֭XҴiSyf;vMPPٿ1&M2cƌ)47\s2),qc7R彏(j#hT lLDD9w\?v3a„N~w+?[DJC'nL:uLJJs3Ec9 Zc؁h'|BxxK8qK/\;D k|>Ĕ&p ڶm֭[b?}G۶miѢ>vryLc ۷wnѣNFFݻw/ؗ]NزeK1t J~=ϛ4i)/"R)y#U@pf3v"8*q{KHHQQQk]ŋ3zhnf>p?ocVyEm?GgH5ЩS'1۷ϹmǎEsEFMdd$۶m :c\ܥSNX,[֝;wח[zm۶9߿@rJ?Nݻ [Igi*@Q&z@ @807'''gڵDEEq}dɒۆKd sٶ-/D\:?Α#Gؼys$##={R~}RRRJT`;+/^ѣ {yy1|p}Q>v;{.ɓ'ӟDΝ/4߿VXAvv6ǎcŸWӖz~N:E׮]|̘1o1֌c{<+gtj;\<`%[Ӏ]7%w?%Yq鞐E?>mڴ!,,Gy__BoҤ _6lݻwgʔ)=DcXV~_3dO?4FbƌKffcObܹtЁ믿+VOInݺtؑ@{Igi `r4SD.̜93|7" Yp]UDZL_mT"% Xrss?DTjԨ17k,筿 ~i\Υؿ}=ϟ?OݺurKo""Rjބ 7Pш]:Q਱qX㩀DDDJ˕Ⳟ;vaÆlْuoOstWE]O%nDDD}#q >CDDJٹsC(G2tPΞ=Kxx8O>$tXUVE]O%nDDDG͎J=f`ٱc[^א!C2dԒ6,ũn׳:3\=znĉ>Θ1c?~#""?^|rJ.t4jǗXrm?~|#R^wkz8OHA G<,0R*LU7c Y|9L:cĔK>"33nݺ1c 233qj̙$&&ri^}UO#")5mv'=HU5FxyyѡC&LUٳ mۖ>}b rssfeeO֭iԨzK[l6ʕ+ 7̒%KnƎr>>>XVjԨAZZx{{;HHHW^nݚwMIIf1~x˽K˖-iѢ)))cڜfc޽L8ƤI.XvaZiժ3gn׮]J$9"q*T3s)s=}:z!ӧOsA~zg9_~eӻwoaͦM̅ Rv2760  5fͺ:ǏgϚnl޼>E-JHH0f޽&%%t,Y>wqկ~e-[f.]dN:eN:U`M^11ƼfΝҥKf˖-aÆ&))~ Svm3w\a9b'ՓJIUh@DD}zg4nܘ5k:gdd~:t˝駟9r$N">>'x@?5jӦM#"" |nZ9}KǥqaFfG&?UR۷og…lݺϓLlܸ=z9>2 tR{:PFDDIUq۶m+VMG hӦ {q&?+w^Dٽ{7{W^y.GTT۶mfĈe:*P'_֭l|WWSYBvX&NȜ9sSÇǘWݛǗ$>w3MO?䍈HJܔ](/ `޼ys!֬Yol_f Pvmrssi޼ϟ?;wk.CTT!!!tKCNN/^ۻD3~BCCIOOo妛nrnX,1SOбcG8p pdeeqI"""S{aժU8>|8ӟxW… ܹ7/h⍈cǎѣG;l"3LDDtWkvONdd$QQQ(={_ O||bܸq>|5jp-[… b„ W$ "##۷/9۶mnvZυ f0m4ƍҹwԉ>3jժE\\x{{tYYYL4 6-š5kZ[L63gG=cgffҥK}QFwM Xpa{_⇢Wv`̙3ÇӴiSFөYK~~*b|",,?v;Huwࢧi\<8qԮ]̝;ddd#GC>}???j:t`?nΞ=kn&yfg{``yGey넄akRRRL͒%K xw_WfٲeҥKԩSԩS+WRǎMpp̩SLhhYt/޽{ m6md.\p8cƌ1?we7nl8`^}U|R q_o8 IDATٹstٲeiذIJJ*}JsVϏ;/}&=b|ux/)7ߍOPDJ'W\O^bIdffh5lٲe3a|}}w^{߿?Wns3h ٳ'6mgϞc;СCx"ûҥKi׮qqq\#Gk׮ 6 ƍ{Ψu]]?++sΑFHH%s̜99;\UQ_3_Ébݥ<+S㋈H00P cHI@nCؾ};fb.q+`;uЁ˗/ jri#F?L֭[Gԩiii>|عLnu?Iw&~o.7\;8~'FɩS'(RQF1m4"""Z_Y/E}v.\֭[9<;SW'={d˖-у?uO?4SNee `͚5۷hL2TO""IK QDJUq۶m1DI޽{_R ЦMW_ѽ{woժ;vdʕ;Ι,[fzjIOOȑ#|7WS,///rss].{n~_j%11T6mĂ ذaCƏ#**~vCa_Nll,;wfӯ_?1˯S1qe:u*GIj]xZPP+Α#G<T'J\Æ_^yΝ;Yz}̛7Ca֬YÀJ4FLL III|G1¹b0bNNnn.?,+η~[Z-[k.{=mŝ5k8x j׮Mnn.͛7wysNxwxKQ_YɓDDDPNêU/ ͛\1qeUVe.22g}{ｗ-[ҢE RRR\j?{,/m۶O>X⊄bqUV\xG|r ,]vXVZj̙30ի[&!!l?uSLϏ!C] -ZD=ҥ -%D.]xh֬sqOIIf1~xcLZh{ذa6G1ߌb1ӦMsU|ʕf'ON:Ϥ믿~Q\!*) '*9?zڪ%J\Wʼn*Ty>:ubʔ)t("%ǂ p9,O^ԳgOv_6vX~iHj׮]謬y8p cƌA<._~?tޝnO>ooBk܌9F׿pX۷/O<=((寗oiܸ1k׮%44p{y'1`Xt)z+/fժUXd^z%6nȰaø[0ajԩSԫW]_5n.xW]#i:0M}RspMU XBBܙ/vDrssXg~uqy֭""Rve)C=|mZاBWk/IBoڴ)}]\)T]B_5gϞ}osW RUV/^wxi T5 M5(DDo&DeMHH-R,UJDD|fHSFlŊtܙƍs."Waaan| Hw"##iڴ)/&99Zjy:,h9 @DDD.Rf3gdƌs=TzuQRSSi۶C2&::aTyJE3naYYYӺuk5jD^8}kJV9s&v>dl6{eĉl6&Mlp-Gt҅Eq%RRRl?bٰl$%%JZ8s挳]vQNt~gϞ^m۶Ӈ+Vmq}wvm@vvKDFFrҲeKZhAJJcgi5lѢE8p> ʖ-[Yv,]۳c.5kرz}+WϏ{={̙l߾$, cƌۛ#GҮ];INNfʔ)D.]XlҥK^z.߬Yؽ{7~!d4jԈnͥR\/^̢EW 89NQǝ^x^z%|MΝ;־EDDDDD|i5,++sΑF͚5K.]&<<(vR}]f͚Evh۶-qqqd+~߱dv;.wǯ^xCYzǻàAѣ۷'""M6>ڵ+Æ ۛƍӸqc!""""""C*.,, łb',ѱqqqDGG3rH7n̴i ,ھ};ƍ#88| \;--Ç3j(2ӧk.o˗_~IZZ7nۛ޽{tlFF'88عClݺ[nq>oذ!Os}\?H2:}4/¹'$,, pb_~+o XP+ǎ#}ijQ yСCY~D} )J^ذa8p-[8 @INNfС%:V"լYu2l0^z%.]Mdd$}?g۶mv֮]Kvp˖-c…dee1a„I,&MĆ a͚5XVQ(_g…ϐ!CQõIaaa;su1gRRR߿?.\ ##i7noӦM̙3ѣG_XXzb۶m;vhfϞLHU|OP~ڵkǣ>Jvly1n8v;կ>|xOLL;'OO?EWܵI&$&&ү_?2331cqqqH u%,\(5kƜ9s(Q).r}2rHCpp0 4[,^|E|I>z)&NX~D*ʗ_~Ν;K yWHBJ/ W Sr2W[n֭LJ;(."""sNO U7װzz_sĉnSy??~ն5jФIF]l԰aCHHH(rwUarkV൫ק0,]}6lPl?w}7w}wb(z+v,-7,_H|ҥv~!CBBH^{+K)W4h~~~ٓM6)q#""R6j#.SF*8p{ӡHؾ};1}t+/\&M0{l?3xZj ⋋㧟~bȑ:uxx j֬Yh˗5r5Za_h]DDDJMSmeTZ]veĉ,^{:)¶m0`q9i"^^^8O4mڴ~7&00s9^~eŞj ^t9r|sB響lڴ 8g]^(= _^(VJ[(DDDDsJ>ѣ$BD*B׮]Y`L׮]ӧO穧… v/x+B9߿?йϚ5k8x Ƙ+ υ:Ć XfE//Ksrrطo[lb +񉈈(q#""diѢz<Åu]ǫoSqQėСCEşfc|QF9v\VZ_xt}ϭV+Ov.gҾ}{rKί{OQٳ'[l 00;３xopɓz[c?~+*|#GfO^}С㏻ev~"eP{:iMm61cJ<'' ЦMW_ѽ{w޽{;%Nll,;wfӯ_+~/lR[fzjIOOȑ#|7e)Lq޽{\,gqn\f̘Avv6˗/'55S:ێ=Z!bbb2d]~֯_τ ge:?Hk\@@͛7'990o<:Ć Xf  p>+;wǏ;ReeeqI"""S{aժU%"6baĈL:?\χ~q]>=f͚ۗ_|'Odv~>ߖ8~jԨ:t`„ Z, 7puq+d^\*|qO<<˹j}Y~ӨQW+WTGJ\, /"gj2w\gӉ$**z9slcͤDXXs)je޼y7PNK4~YuƲeHJJfqw<~bb"Ce͉'ק,ϝ;f͚ѱcGx8pژӋ\n˗/gΝ,_ܹDf͚źuIHHv,] ,YܹsYh}„ o<, :ubvm_u jl]~~""""՝H ..{cl IDAT"/..  yj^ԱcGoޣ'04I$IGdPԁ5,29y/)xK;YTxItᘚr̖ii〥DZ]-KpPPx{vane[fMk\s5ͫ\3=viuYu3g-Ɋn??O9R__SO=]5ƎݢZ۞$-ʬYru׵kl-/Wwqnuj{׻ޕm&p@FosO>}O:z'?I]]]<̘1m[V]vY>}{_?~va\_Wڵk$+VW\ӛ͜93Ls_zvu\uUgĉe]{G?Q+Wf„ ?}1bDo$6lX=܌1"sfmW_}57tS:p馛M?ߎom~ZcI]]]{| _H]]]/7z„ 4iRE~ .ŋ.uuu뛏mm;$IIM[nŌ3k+/yqW{O?]\{E /|[38򗿼qZOo}O?yO>lѭ[bڴiESSSt;l}8Cb|xq-6v~kZ~>oy:thqM75?~|1qĊ~E64lhWc޽7I} h?|":VPusL5nFѼ`͚5ɓ'g]vYgvmpWf̘1}uرc,Ijժ̜93cǎm~]w|sa#]wռ>^{HCCCV^ݼoρ>;={Lmmm}G9i~mݖK/4{w ǿmFK[R-O[w󩄍*ߟ<$K71\6lX~%I.wxpKf̘:*=XLO<1~{-Z+Vd}iwhIԔ믿>=P=餓o;Jn=655eѢE8p`olM~zWZ=z7M: ?~k7n\L0!\rItҦ[ߖ7§7u=/B'SO=O>y_=]ϓ귤[S uT%??ؚXq@͝;7EQ(v6osgg̙yү_#/noѢEjr!Ї>cs 4(3f~Vׯ_?7裏f-7=Cnpݻ .… ?AfϞ[ߑ|L]]],Y$K,ҥK׿[R-O[wiM^ֻO<Ѯ[~%I׮]׻.M[UnڵYzux9rdu떮]fȑ[xprL6-F;vlWI'޶QFk_̞=;sOF'pBz\{y嗳|N5{xEnݺeڵ:?䤓NW<#Yvm-Zﾻ]7[Ӛ#FdYfMVC~o:th,Y'|un̘1֭[ƌݻ/ov5פ&_L~;|o_|lo1 97N8!sM2%#Ģ>|[UW]C9͵9sd߿ ީ$ .ȑ#ӳgϜz׾޻b;\p9cs9gϞ=zt}6nh槵|Z3nܸt=矟:]Ƿ%I޽sOmmmmG?z`KPS:7(e.ΨHﴝf͚5kr5״{ P=dpUnc$O{> *l!`Og„ BJKpVo߾9l7%%() @I n'555U3ɓ6ӳgvYgYǞ9sf>՞Uj7mٲeYpa P>ƎwU/WwqU%՞㚪@gbŊx㍹R[[ &cɻƢ>W\qEn ><555{o|M /̋/砃ZoaÆeĈ;wny晌=:76۴~k9<#Yti/s1~O0!vZd„ vmsUW$y(?ϳvk_ZN=,X GqD^}455.Irs\槵ϧV\/~={v}(sOwަ-м$]jLj7HRlj?җTvaŋYf})|1f̘bҥâ ۼMgqF/m:w}˗+V([̙3[;WtMͯǏ_L8=En݊iӦMMMҥK;s1f̘Q444߆槵ϧzhT^xW_}^X"v˺&iU:x&md v]w;NS__#8"wuW>я_ϬYr6~z+]ZVGuT$9XPIrg$ٳgjkk+6vKO[[re^~<ill!R͛Wk 9-aÆ ~$ɥ^)STnSSS-Z6;HgVX}٧y>moݻw /P7No>7~k7n\L0!\rItIzhlll/_cU ;wnHQڼy5k֬;~s{>h$ihhHϞ=c5oֶwԦ$zO[fΜmF5\/={v瞌5*IҥK|wK8j7o%jjj~77ӽ{L6y۔)S2r|Sʷ\uU]eڴiu]3hР̜93G}tvÇO]]]n\'?6Z7n\wQGծ9sd߿ ީJIһw\p9S[[뮻y[kӚ fȑٳgN=|k_{ݮ*d$Ivv#VN%EEQ>lРAsamM=>qdp*{Fܘ Մ[=da{z&Yvkl/_]v%I'٥-Xqm޼yYhQȽޛzjm@G[Kf̘1YjU>g̙yz|hw9U-SH*Us@'"() @I81lؼT5I2dȐ |N9 nh@-[l:Jp@ڕuEː!CKm2lKl HR$ڍЩ7gEU[XvmQF555E%{'E>}#}_z,*"k֬~ygSSS%K䢋.xʹ瞛ovҋ/.(9s뭷{I38#ӦM/$+А=zdԨQOڮ1SC{7N]]]^pԔ:O>?k^ziF}7_lٲ ߧOG?<%e]>viy'5O^ZɎ>t5=P,YnԤgϞ˛߯ͽޛ̚5+'xb~yn!ӧOϠA$gyzu]4iRg}v^|?K/g㦛nwߝ+]vYF^-ӆ|/~s9:ujnrgZqتU2k֬\|}3bĈ7?fm2jԨ=:ӧOocƌimZ=/s饗93qq-wGgڵk&NG}U74v^ҥKN>`L6~wY|y֬Y7Dp~헢(77o޼YjUN9唌92sSO=c=6EQk.].eҗ> wy6mZw߼=i:<Ǐ{ 4(G}t>ϬĖXhn_^hIkeTSSSlL\rIqo6bƌ~gyo3m8-ל7o^z=#w_~gԩ-OIp?~[e˖c͊+rF:vQYfMf:nG{olվwSɓ /_׋]wݵ>|xq-o}QE~޽{;}-nƢ(+(WGQ'?Yfcccq-lpGG>gϞ>K/mP|.FQ߿~q;c1jԨbŊ4hPqopK.)>O1iҤbҥww.̙z޼ysڜSbԩG?Ѣ_~ũZ-8餓$E+n`}.]vC=%K|o1555ٳg.kkks6fʉ'_~/fҤI2xO/^>7pCOA5wWdڴi{3nܸr!:th̝;7~z:tǽ馛rwge]ѣG}{+Is!7|s~_>sNNnck:XVʬYrgwψ#t]?>{Gf5*G4fMMM^{,X Jz8`}ƌ^h星ԧҷoߜxyo;} '}3g=3`t5'Ṇ>Q]9;vlkt%'xbv+$ɀ2dHidwϝwޙ~Mc+3gwܑ|3fV|~qok]vɞ{^z^KA>yIDAT|O}}}|6V;gl~޻w,_ϬsTy֫m5G\rIqo1c&)vmnmzǤyW^c=r}?qNuTc7-[Q.e˖c͊+rFn rqb7%%() @I nJJpPRTj7[:v[xe˖U*HpaٲeB69 I*fb'NO>bEkkk+Vt\S`ZQ)%o-,Kg ^nU 7%%() $%Ydj@C$;Ux+ @I nJJpPRZꫯmV6@ ,[,˖-v$nږ{۸81@I nJJpPRv\(︗j@c @I nJJp@U{zo$PY`p@I nJJpPR7%%() @I nJJpPRTj7 4U$Lv#VnITF7%%() @I nJJpPR7%%() @I nJJpPRqVNu?kXqPRilޛ>Ij7T$*PR7%%() @I nJJpPR7%յ tMn9.$Sہ@C$;Ux+aI`I>_zu:F_c:ڟRTI+X JQu:F_T?(bhIENDB`gfal2-v2.23.0/doc/diagrams/gfal_diagram.svg000066400000000000000000000332571465240014500204440ustar00rootroot00000000000000 gfal posix gfal common GFAL 2.0 design - Error Management - URL resolution - Plugin load/unload - Bdii/ vomcs conveniences functions - logging functions gfal lfc plugin gfal dcap plugin gfal rfio plugin libgfal2 - Posix Interface - convert posix to internal call - safety of the url - local file management gfal srm plugin Plugin Interface gfal2-v2.23.0/doc/diagrams/gfal_open_diagram.dia000066400000000000000000000073771465240014500214270ustar00rootroot00000000000000]rP1l[n-$bR;{Aʂbf\yyn@@|#:}>g\dE~=(i>.&Y>>0ɒߴL#_g׃zׯhHDl,AzIR%իIUJ<׃dǴ,dPSŬ(/zN?WeٳC2Mo4c!<iXd@R=>lXG|EZQ>' /k5]N&<)Yd3A$0x,P0dr<󲛝]y^v(2ɪmE1KZ˴=8-cY|&~YێiM+ŎUf#qի?:ZKngigyuO淣w[ j;D<Ǜ2:͎WdWIg* cY̒Ǵ\-SYfo~0KAWNӳWb)nߪsZ/!zp]bpKd}EoJa{ٵ%|$bdD#aD;nJI6ڭ͸(mT,N-]lz_|mre-u0 ۱f3nq,-d>ڼ}](.'I9~^).^).U\huE!10-9VJwf}50oEi^oYޯw{l[ȫKPw<=•&b,GÞ]KZe䵍: ԽI呈\Obs4/0Zw3 6Leʥщ8NQKTK'ؠjOe bV0$ ֱHL{!Dl=v2gfnyn}-앦Q{) .<|D$PvQ9RdDŒH4Xu?TQf6j}⚒eUhSqo gut?1+ȝ-?]B lpnW}p#}sR1fg4 MÆ_S)ȸzbUQ:Gzm ūRf.۸Ev:@DQ9!DF &Rqu\5&ˆ﵏8FD јvє{#3^VvP+ȋ*He*7&%-&msݰ1FԐGsdDRZX3=R3)˺R[w2*K,luc>lq﬚"?rƳIQ.WӀk)oM58A&5 I6Eԩ2*tb2\J#'3#+#ੳS$즋l!=2@|eɧ|河HP26uW{ܱL)߈ RIN*&f"(g5#$breyCm5:GØSw|{E9c'N(fZu,CWzL+b룸1C4nNøpְ.ļ"3:"^>U0I2{?!+yPshCB o> b"SFU5!=?k+=[8N`>T7$ Gי'u8,=t0^ӚkH~{ 'i&,f)eOXk^K2I pxos6[7Xҭk"F5•_Jtǻ ]T~L)-\^V5j3WFcn< b&C~9;D*ugZbUTiJעw\.Ɉ DEmNd[J$Ð/B{eQ 5R 'e_yަغ[U}$hNf&\w;2Xc-DBKsVtnOfJ8}Jh=*{߭Ip{BV} > >>g" ԂfF3ZΊ|[|[Pp]nk>!6V}F4)0"S=ul 6@v:/LPNK< CB;([vJy |@S8aHV|Fz2ٚeW M&[аpVF `͊e5Sg3&LN(uw5t? U ĈJݚ[L BK.Piĩ4ըy4x@C $'3fpO L%%J#q{lQyf\Ī7n b5 ֌Z7"Zr"x΄k 7ʡ̮FYM623q$PauEdwu;^m 0iPչح ,tX؋4Xhm&(qd;wG;Q؍qE3wvOG8y,jݤ@CuL@Dum^YJ8RM17m#[/Ola'‹FfS(,L 9u|ԺˈTGkfTJeVIA]X lJws=P@eǭ|}730VGLyAu["cdcZW1aT#Uqo,tr9V @l Hgۆ8DD %' Gfal_open activity Diagram { not exist } Must be created Resolve SURL to TURL {srm url } create fil descriptor from turl { valid turl } Report error { invalid turl} return file descriptor Resolve to SURL { Catalog URL } {reussite} Resolve to Catalog URL {echec} {guid} {valid url} {exist} test if file exist { invalid surl} put SURL to TURL gfal2-v2.23.0/doc/diagrams/surls_get_endpoint_activity_diagram.dia000066400000000000000000000073411465240014500253070ustar00rootroot00000000000000]n7-0")zg,zEŖueɕE<}X%%۲Cc9 09OxyGygFq6.4,#-Oweׯ(y(2Q/Q]'L.9&aךWòe9i8n<[AMЍ$˝0y?aZ} .e.Wٱ"Ey}i"ח/"aAe3t_Gj^VۃneR||@6I-B@Q&=.㲋Ey,o,´Z˨;b&v},O12{aR|'mWsgy<٭+[VOʻoU`ho<Y~S߮ն$X2D lfJw KR_SLXW,!ʛͣE7gbn~s\!eT 7FtUx^5j5?)J8ȳxAh9;;ByK&.4alwsGE,K8Wqöݶղl{.3 q)ipkGe</~͒sqP6Eܧ| .OxX]ۅӪկxbta>,G(]k^\Zmzu5(`v£ X TlQC_2<|6" RzlJMwL(x1A2މIy6wq^N̓WDM3w/4aQf&3=lzy&!L60GK\>;³>]{i4vhdhd `ߪ/m4Dv )[fy /e}{?S,<wbW~ψ#CljlKg _NLIdd*K=eO /8xbR'GUQX4b6 8(vNiit܅q,ʬJf8^ܼo&e@&#)GKؐ#o`J)ivP"IѨQII C&s;c'[TS4&Sj7@ 7 LC)E.ez#(~- 1 _1(^G&'QCiH}} p !iPGh! @ٸ,x a",\6 P0>=Nk V׾ڝv8LnRMb#"P (0j ])"pPt2ѴNfSk6AgseDM,P]Ց|ױPWCwǪ2K9̉wwn=ü D#U glO[Ig8 lm62! NIWfXH:Hse]}yr,0&e.[X:,iSYQ˫4FecD\G:'1FOf*uW&n! Hܲ:eZd2L- p6CBwpFg?V_=nYKAtXbs['bmʊȼsL'*F3Sk{#'OР evV)y] NuӖ!.9xo0/D^V Nv,M{݈Jox >eʹ j<^E6"@cGكTǠg89̕璇Dg]R垸Q n'й>W>3Q<|=+Ά>bF:Ʌ~a]0ţG{?}ۚMmhGo=pP`n4e&\ED=%}0iM%$vYė095wEpa:x{,0$TΕ5kdoW_&5ma85mz3{4JӠNL]egr*TLy}tV̖8ǥu1\ZWKa6-K<1BXᎪ)ĩ43T'N\HNı3T,8FՉ-B*7CbͭhT^Chr\5UCר5b d>|G@͊`vjjä4Lws(b6){yo=Yk*j?k]{vU fNX58h^x/4ceH^t=OʳjDRx}֨YxL<8д=Z=)MW]ڴӪM-Kϟl/g<@m!s6G$ݐ/Lҍ8LNHK끃jT:Wٲͪad_BL%f4E'aDaDXF\nG#in&&㽾SSjvب6ju(Fީ#j=xj/|\hrښM tcWޟչlvi˻[I- f!"\G`i[% 5=}f?z.PG2N~}`>XMUOU[>>!#Vm8LRÃhD^hbC9R$[$N4$#J4RjvQ[E޽#J7Jĺk>u[#=1S輊kȩZD[pJl&S"o9!78 c@&+տ^kUTNj>U]c]p*4mƚo s+b3X'6㙳XayCw-PU9x'r#O |A.Ū'<;G#)=qF&rSMs1"Ǚ2Y˄uFkF*@!A eyt32XDxGONɟmQَZAi#:1+f3Tu4M*& $Quw%lƆiH{l gH_׊jw325`"}"W6{MDA]uZ1r1o;Ρ,71=%ؕ`Yk"տ\gZ9Y|r}>'#™*Jc X o;paR-jrOgV'YkeԻ@0a8Hoa!=+ "jז4iԈl Le!^l%4“cgꟓ!?<xڪ1gfal2-v2.23.0/doc/diagrams/surls_get_endpoint_activity_diagram.svg000066400000000000000000000322601465240014500253470ustar00rootroot00000000000000 Activity Diagram of the SRM resolution SYSTEM : Extract endpoint from first surl {first url contain a full endpoint} {no bdii option specified and no full endpoint} { unable to extract the endpoint from the first surl} {successfull get endpoint} Force Srm type to srmv2 Request bdii to get endpoint set Error and return parse surls Search Endpoint { url is valid } { url is not a surl or surl is empty } Execute SRM to TURLS convertion bdii allowed and no full endpoint found in surl set Error and return { no response or bad response} { se_types contain se_default type specified ( srmv2)} Force SRM tp SRMv1 {else } gfal2-v2.23.0/doc/examples/000077500000000000000000000000001465240014500153435ustar00rootroot00000000000000gfal2-v2.23.0/doc/examples/Makefile000066400000000000000000000003621465240014500170040ustar00rootroot00000000000000# Makefile for all the examples all_examples:=$(shell find -iname "*.c") CFLAGS:=$(shell pkg-config --libs --cflags gfal2) $(shell pkg-config --libs --cflags gfal_transfer) -g OBJS=$(all_examples:%.c=%) all: $(OBJS) clean: $(RM) $(OBJS) gfal2-v2.23.0/doc/examples/gfal_copy.c000066400000000000000000000060751465240014500174620ustar00rootroot00000000000000/** * Example source code for copying files using GFAL2 */ #include #include // You only need this header #include static void _abort_on_gerror(const char* msg, GError* error) { if (!error) return; fprintf(stderr, "%s: (%d) %s\n", msg, error->code, error->message); exit(1); } // This will be called when an event happens static void my_event_callback(const gfalt_event_t e, gpointer user_data) { const char* side_str; switch (e->side) { case GFAL_EVENT_SOURCE: side_str = "SOURCE"; break; case GFAL_EVENT_DESTINATION: side_str = "DESTINATION"; break; default: side_str = "BOTH"; } const char* domain = g_quark_to_string(e->domain); const char* stage = g_quark_to_string(e->stage); printf("Event received: %s %s %s %s\n", side_str, domain, stage, e->description); } // This will be called when there is performance information static void my_monitor_callback(gfalt_transfer_status_t h, const char* src, const char* dst, gpointer user_data) { if (h) { size_t avg = gfalt_copy_get_average_baudrate(h, NULL); size_t inst = gfalt_copy_get_instant_baudrate(h, NULL); size_t trans = gfalt_copy_get_bytes_transferred(h, NULL); time_t elapsed = gfalt_copy_get_elapsed_time(h, NULL); printf("%zu bytes/second average (%zu instant). Transferred %zu, elapsed %d seconds \n", avg, inst, trans, (int) elapsed); } } int main(int argc, char** argv) { if (argc < 3) { fprintf(stderr, "Missing source and/or destination parameter"); abort(); } const char* source = argv[1]; const char* destination = argv[2]; // Errors will be put here GError* error = NULL; // Create a gfal2 context gfal2_context_t context; context = gfal2_context_new(&error); _abort_on_gerror("Could not create the gfal2 context", error); // Set some parameters for the copy gfalt_params_t params = gfalt_params_handle_new(&error); _abort_on_gerror("Could not create the gfal2 transfer parameters", error); gfalt_set_timeout(params, 60, &error); // 60 seconds timeout gfalt_set_dst_spacetoken(params, "TOKEN", &error); // Destination space token, support depends on the plugin (SRM, XROOTD do support this) gfalt_set_replace_existing_file(params, FALSE, &error); // Just in case, do not overwrite gfalt_set_checksum(params, GFALT_CHECKSUM_NONE, NULL, NULL, NULL); // No checksum gfalt_set_create_parent_dir(params, TRUE, &error); // Create the parent directory if needed // Callbacks gfalt_add_event_callback(params, my_event_callback, NULL, NULL, &error); // Called when some event is triggered gfalt_add_monitor_callback(params, my_monitor_callback, NULL, NULL, &error); // Performance monitor // Do the copy gfalt_copy_file(context, params, source, destination, &error); _abort_on_gerror("Copy failed", error); // Release memory gfalt_params_handle_delete(params, &error); gfal2_context_free(context); return 0; } gfal2-v2.23.0/doc/examples/gfal_ls.c000066400000000000000000000025361465240014500171240ustar00rootroot00000000000000/** * Example source code for directory listing using GFAL2 */ #include #include // You only need this header #include static void _abort_on_gerror(const char* msg, GError* error) { if (!error) return; fprintf(stderr, "%s: (%d) %s\n", msg, error->code, error->message); exit(1); } int main(int argc, char** argv) { const char* url; if (argc < 2) { fprintf(stderr, "Missing url parameter"); abort(); } url = argv[1]; // Errors will be put here GError* error = NULL; // Create a gfal2 context gfal2_context_t context; context = gfal2_context_new(&error); _abort_on_gerror("Could not create the gfal2 context", error); // Open the directory DIR* dir_handle = gfal2_opendir(context, url, &error); _abort_on_gerror("Could not open the directory", error); // Iterate and print struct dirent* ent = gfal2_readdir(context, dir_handle, &error); while (ent) { printf("%s\n", ent->d_name); ent = gfal2_readdir(context, dir_handle, &error); } _abort_on_gerror("Could not read the directory", error); // Close the directory gfal2_closedir(context, dir_handle, &error); _abort_on_gerror("Could not close the directory", error); // Release the context gfal2_context_free(context); return 0; } gfal2-v2.23.0/doc/examples/gfal_stat.c000066400000000000000000000021061465240014500174520ustar00rootroot00000000000000/** * Example source code for stating files and directories using GFAL2 */ #include #include // You only need this header #include static void _abort_on_gerror(const char* msg, GError* error) { if (!error) return; fprintf(stderr, "%s: (%d) %s\n", msg, error->code, error->message); exit(1); } int main(int argc, char** argv) { const char* url; if (argc < 2) { fprintf(stderr, "Missing url parameter"); abort(); } url = argv[1]; // Errors will be put here GError* error = NULL; // Create a gfal2 context gfal2_context_t context; context = gfal2_context_new(&error); _abort_on_gerror("Could not create the gfal2 context", error); // Stat the entry struct stat fstat; gfal2_stat(context, url, &fstat, &error); _abort_on_gerror("Could not stat", error); // Print printf("%s\n", url); printf("Mode: %04o\n", fstat.st_mode & 0777); printf("Size: %d\n", fstat.st_size); // Release the context gfal2_context_free(context); return 0; } gfal2-v2.23.0/doc/examples/gfal_unlink.c000066400000000000000000000016651465240014500200100ustar00rootroot00000000000000/** * Example source code for unlinking files using GFAL2 */ #include #include // You only need this header #include static void _abort_on_gerror(const char* msg, GError* error) { if (!error) return; fprintf(stderr, "%s: (%d) %s\n", msg, error->code, error->message); exit(1); } int main(int argc, char** argv) { const char* url; if (argc < 2) { fprintf(stderr, "Missing url parameter"); abort(); } url = argv[1]; // Errors will be put here GError* error = NULL; // Create a gfal2 context gfal2_context_t context; context = gfal2_context_new(&error); _abort_on_gerror("Could not create the gfal2 context", error); // Unlink gfal2_unlink(context, url, &error); _abort_on_gerror("Could not unlink", error); printf("DELETED %s\n", url); // Release the context gfal2_context_free(context); return 0; } gfal2-v2.23.0/doc/man/000077500000000000000000000000001465240014500143005ustar00rootroot00000000000000gfal2-v2.23.0/doc/man/gfal2_version.1000066400000000000000000000006561465240014500171310ustar00rootroot00000000000000.\" @(#)$RCSfile: gfal2_version.man,v $ $Revision: 2 $ $Date: 2012/04/13 13:09:16 $ CERN Adrien Devresse .\" Copyright (C) 2012 by CERN .\" All rights reserved .\" .TH GFAL2_VERSION 1 "$Date: 2012/04/13 13:09:16 $" GFAL 2.0 "Library Utility" .SH NAME gfal2_version \- Version tool for gfal2 .SH SYNOPSIS \fBgfal2_version\fR .sp .BI "gfal2_version .SH DESCRIPTION Simple utility printing the current version of the gfal2 library. gfal2-v2.23.0/doc/src/000077500000000000000000000000001465240014500143145ustar00rootroot00000000000000gfal2-v2.23.0/doc/src/API_CHANGE2.0.c000066400000000000000000000031341465240014500163170ustar00rootroot00000000000000/** \page api_change

GFAL 1.0 to GFAL 2.0 CHANGES

BRIEF OF THE API CHANGES

- The old POSIX API is still unchanged and follow the same signature. - the old non-POSIX functions do not exist anymore, most of them are converted to a POSIX one ( ex : gfal_ls -> gfal_opendir, gfal_readdir; gfal_turlsfromsurls -> gfal_getxattr ) - SRM related API is now in the gfal_plugin_lib. All the generic use case of the SRM API can be done with the POSIX API. - The old confusing error system disappear for a new GError system : \ref gfal_posix_check_error , \ref gfal_posix_print_error, \ref gfal_posix_strerror_r - 32 bits GFAL 2.0 need the usage of the Large File Support in Linux

FEATURE CHANGES

- LRC/EDG legacy support is dropped. - The LFC host parameter is not configured from the BDII value anymore. This feature is replaced by the usage of the lfc://server/path URL scheme. - The SRM plugin does not need the BDII by default, this one is used only if nor the port or a FULL SURL format is provided. - SRM specific feature can now be accessed by the extended attribute mechanism if needed (turl resolution)

ARCHITECTURE CHANGES

- Lots of the old direct dependencies are no more needed and are removed ( globus, gsoap, ccheck ) - The new File API is thread-safe - Any parameter can be overwriten at runtime by the \ref config_group functions

OTHER DOCS

More informations about the GError system http://developer.gnome.org/glib/stable/glib-Error-Reporting.html **/ gfal2-v2.23.0/doc/src/EXAMPLES.c000066400000000000000000000025131465240014500156770ustar00rootroot00000000000000/** @author Devresse Adrien ( adrien.devresse@cern.ch ) */ /** * @example gfal_testread.c * This example show how to open and read a file * ex : use case with LFC plugin : ./gfal_testread lfn:/grid/dteam/test_skywalker * */ /** * @example gfal_testrw.c * This example show how to open, write a file and read it after * ex : use case with SRM plugin : ./gfal_testrw srm://grid05.lal.in2p3.fr:8446/dpm/lal.in2p3.fr/home/dteam/test_yoda * */ /** * @example gfal_testget.c * This example show how to convert a surl to a turl * ex : use case with SRM plugin : ./gfal_testget srm://grid05.lal.in2p3.fr:8446/dpm/lal.in2p3.fr/home/dteam/test_darkvader * */ /** * @example gfal_testdir.c * This example show how to list the files in a directory * ex : use case with SRM plugin : ./gfal_testdir srm://grid05.lal.in2p3.fr:8446/dpm/lal.in2p3.fr/home/dteam/test_r2d2 * */ /** * @example gfal_testchmod.c * This example show how to change the right on a file/directory * ex : use case with SRM plugin : ./gfal_testdir srm://grid05.lal.in2p3.fr:8446/dpm/lal.in2p3.fr/home/dteam/test_obiwankenobi * */ /** * @example gfal_testcreatedir.c * This example show how to create a directory * ex : use case with SRM plugin : ./gfal_testdir srm://grid05.lal.in2p3.fr:8446/dpm/lal.in2p3.fr/home/dteam/test_tatooin * */ gfal2-v2.23.0/doc/src/FAQ.c000066400000000000000000000002741465240014500150720ustar00rootroot00000000000000/** \page faq * Doc File for a FAQ * * \author : Devresse Adrien See here : http://dmc.web.cern.ch/projects/gfal-2/faq */ gfal2-v2.23.0/doc/src/POSIX_API.c000066400000000000000000000001361465240014500160530ustar00rootroot00000000000000/* * Doc File for the POSIX API in the 2.X revisions * * author : Devresse Adrien * */ gfal2-v2.23.0/doc/src/lib_design.c000066400000000000000000000001571465240014500165620ustar00rootroot00000000000000/** \page page_design \author Devresse Adrien ( adrien.devresse@cern.ch ) \image html gfal_diagram.png */ gfal2-v2.23.0/doc/src/license.c000066400000000000000000000217311465240014500161060ustar00rootroot00000000000000/** \page apl Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: You must give any other recipients of the Work or Derivative Works a copy of this License; and You must cause any modified files to carry prominent notices stating that You changed the files; and You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS **/ gfal2-v2.23.0/doc/src/mainpage.c000066400000000000000000000025361465240014500162470ustar00rootroot00000000000000/** \mainpage GFAL 2.0 Documentation \author Data Management Clients support, CERN IT-ST ( dmc-support@cern.ch ) \author Devresse Adrien ( adrien.devresse@cern.ch ) \author Alejandro Alvarez Ayllon ( Alejandro.Alvarez.Ayllon@cern.ch ) \author Andres Manzi ( andrea.manzi.cern.ch)

GFAL 2.0 Wiki:

- Data Management Clients web page

API:

Main file : gfal_api.h - GFAL2 File API: - \ref file_group - Transfer API: - \ref transfer_group - Configuration & parameters: - \ref config_group - GFAL API for plugin development: - \ref _gfal_plugin_interface - gfal1.0 Deprecated API, provided for compatibility purpose only: - \ref posix_group ( see \ref api_change for more details )

FAQ:

- GFAL 2.0 \ref faq

Examples:

- examples

GFAL 2.0, Library Design:

- \ref page_design

Summary of the changes:

- Changes between 1.X and 2.X are resumed \ref api_change

How to compile locally GFAL 2.0

- Compile : - " 1. git clone https://gitlab.cern.ch/dmc/gfal2 " - " 2. cd gfal2 " - " 3. mkdir build; cd build" - " 4. cmake −D CMAKE_INSTALL_PREFIX=/usr" - " 5. make " - Compile tests : - " 4. cmake -DUNIT_TESTS=TRUE -DFUNCTIONAL_TESTS=TRUE ../ " - " 5. make; make test" - make RPMS : - cd packaging/; make rpm */ gfal2-v2.23.0/packaging/000077500000000000000000000000001465240014500147045ustar00rootroot00000000000000gfal2-v2.23.0/packaging/Dockerfile000066400000000000000000000021321465240014500166740ustar00rootroot00000000000000# # Copyright (c) CERN 2016 # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # gfal2 image FROM centos:7 ADD "https://dmc-repo.web.cern.ch/dmc-repo/dmc-devel-el7.repo" "/etc/yum.repos.d/" ADD "http://repository.egi.eu/sw/production/cas/1/current/repo-files/EGI-trustanchors.repo" "/etc/yum.repos.d/" # Enable CentOS Plus RUN yum-config-manager --enable centosplus # Install base packages RUN yum install -y epel-release RUN yum install -y gfal2-all gfal2-tests RUN yum install -y git gfal2-devel # Comes handy... RUN yum install -y golang RUN mkdir /go ENV GOPATH=/ # Leave yum cache clean RUN yum clean all gfal2-v2.23.0/packaging/Makefile000066400000000000000000000043261465240014500163510ustar00rootroot00000000000000NAME=gfal2 SPEC=rpm/$(NAME).spec VERSION=${shell grep '^Version:' $(SPEC) | awk '{print $$2}' } # Leave blank. To be overriden by CI tools. RELEASE= CWD=${shell pwd} RPMBUILD=/tmp/rpmbuild SRPMS=$(CWD) RPMS=$(CWD)/out MOCK_CHROOT=epel-7-x86_64 MOCK_FLAGS=--verbose RPMDEFINES_SRC=--define='_topdir $(RPMBUILD)' \ --define='_sourcedir $(CWD)' \ --define='_builddir %{_topdir}/BUILD' \ --define='_srcrpmdir $(SRPMS)' \ --define='_rpmdir $(RPMS)' \ --define "_source_filedigest_algorithm md5" \ $(RPMBUILD_SRC_EXTRA_FLAGS) RPMDEFINES_BIN=--define='_topdir $(RPMBUILD)' \ --define='_sourcedir %{_topdir}/SOURCES' \ --define='_builddir %{_topdir}/BUILD' \ --define='_srcrpmdir $(SRPMS).' \ --define='_rpmdir $(RPMS)' \ --define "_binary_filedigest_algorithm md5" \ $(RPMBUILD_BIN_EXTRA_FLAGS) PBUILDER_FLAGS= PBUILDER_TMP="/tmp/" all: srpm clean: rm -fv *.tar rm -fv *.tar.gz rm -fv *.rpm rm -fv *.log rm -rfv out rm -fv *.deb rm -fv *.gz rm -fv *.dsc rm -fv *.changes rm -rfv "$(RPMBUILD)" dist: clean tar vczf "$(NAME)-$(VERSION).tar.gz" --exclude="packaging" --exclude="ci/repo" --exclude=".git*" --exclude="*.pyc" --transform="s,^,$(NAME)-$(VERSION)/," .. $(RPMBUILD): mkdir -p "$(RPMBUILD)" override_release: $(SPEC) $(if $(RELEASE), sed -i "s/Release:.*/Release: $(RELEASE)%{?dist}/g" "$(SPEC)") srpm: dist $(SPEC) $(RPMBUILD) override_release /usr/bin/rpmbuild --nodeps -bs $(RPMDEFINES_SRC) $(SPEC) rpm: srpm /usr/bin/rpmbuild --rebuild $(RPMDEFINES_BIN) $(NAME)-$(VERSION)-*.src.rpm mock: srpm /usr/bin/mock $(MOCK_FLAGS) -r $(MOCK_CHROOT) $(NAME)-$(VERSION)-*.src.rpm deb-src: dist rm -rf "$(PBUILDER_TMP)/$(NAME)-$(VERSION)" tar xzf "$(NAME)-$(VERSION).tar.gz" -C "$(PBUILDER_TMP)" cp -rv debian "$(PBUILDER_TMP)/$(NAME)-$(VERSION)" $(if $(RELEASE), sed -ri "s/($(NAME) )\((([0-9]+\.)+[0-9]+)-[0-9]+\)/\\1(\\2-$(RELEASE))/g" "$(PBUILDER_TMP)/$(NAME)-$(VERSION)/debian/changelog") cp -f "$(NAME)-$(VERSION).tar.gz" "$(PBUILDER_TMP)/$(NAME)_$(VERSION).orig.tar.gz" cd "$(PBUILDER_TMP)/$(NAME)-$(VERSION)"; \ debuild -us -uc -S mv $(PBUILDER_TMP)/$(NAME)_$(VERSION)*.gz . mv $(PBUILDER_TMP)/$(NAME)_$(VERSION)*.dsc . deb: deb-src pbuilder build $(PBUILDER_FLAGS) $(NAME)_$(VERSION)*.dsc gfal2-v2.23.0/packaging/debian/000077500000000000000000000000001465240014500161265ustar00rootroot00000000000000gfal2-v2.23.0/packaging/debian/changelog000066400000000000000000000130661465240014500200060ustar00rootroot00000000000000gfal2 (2.23.0-1) unstable; urgency=low * New version -- DMC Devel Wed, Jul 31 2024 11:00 gfal2 (2.22.2-1) unstable; urgency=low * New version -- DMC Devel Wed, Mar 20 2024 14:00 gfal2 (2.22.1-1) unstable; urgency=low * New version -- DMC Devel Mon, Dec 11 2023 12:00 gfal2 (2.22.0-1) unstable; urgency=low * New version -- DMC Devel Wed, Oct 18 2023 14:00 gfal2 (2.21.5-1) unstable; urgency=low * New version -- DMC Devel Tue, Jul 25 2023 13:00 gfal2 (2.21.4-1) unstable; urgency=low * New version -- DMC Devel Mon, Apr 03 2023 16:00 gfal2 (2.21.3-1) unstable; urgency=low * New version -- DMC Devel Tue, Feb 07 2023 14:00 gfal2 (2.21.2-1) unstable; urgency=low * New version -- DMC Devel Thu, Dec 01 2022 11:00 gfal2 (2.21.1-1) unstable; urgency=low * New version -- DMC Devel Thu, Sep 29 2022 16:00 gfal2 (2.21.0-1) unstable; urgency=low * New version -- DMC Devel Fri, Jul 08 2022 11:00 gfal2 (2.20.5-1) unstable; urgency=low * New version -- DMC Devel Fri, 04 Mar 2022 16:00 gfal2 (2.20.4-1) unstable; urgency=low * New version -- DMC Devel Tue, 22 Feb 2022 14:00 gfal2 (2.20.3-1) unstable; urgency=low * New version -- DMC Devel Wed, 19 Jan 2022 17:00 gfal2 (2.20.2-1) unstable; urgency=low * New version -- DMC Devel Mon, 06 Dec 2021 12:00 gfal2 (2.20.1-1) unstable; urgency=low * New version -- DMC Devel Mon, 25 Oct 2021 12:00 gfal2 (2.20.0-1) unstable; urgency=low * New version -- DMC Devel Mon, 20 Sep 2021 14:00 gfal2 (2.19.2-1) unstable; urgency=low * New version -- DMC Devel Wed, 26 May 2021 10:00 gfal2 (2.19.1-1) unstable; urgency=low * New version -- DMC Devel Thu, 25 Mar 2021 11:00 gfal2 (2.19.0-1) unstable; urgency=low * New version -- DMC Devel Fri, 04 Dec 2020 17:00 gfal2 (2.18.2-1) unstable; urgency=low * New version -- DMC Devel Mon, 27 Jul 2020 17:00 gfal2 (2.18.1-1) unstable; urgency=low * New version -- DMC Devel Wed, 17 Jun 2020 14:00 gfal2 (2.18.0-1) unstable; urgency=low * New version -- DMC Devel Fri, 05 Jun 2020 14:00 gfal2 (2.17.3-1) unstable; urgency=low * New version -- DMC Devel Thu, 07 May 2020 17:00 gfal2 (2.17.2-1) unstable; urgency=low * New version -- DMC Devel Tue, 28 Jan 2020 14:00 gfal2 (2.17.1-1) unstable; urgency=low * New version -- DMC Devel Thu, 7 Nov 2019 12:00 gfal2 (2.17.0-1) unstable; urgency=low * New version -- DMC Devel Wed, 9 Oct 2019 12:00 gfal2 (2.16.4-1) unstable; urgency=low * New version -- DMC Devel Tue, 9 Jul 2019 14:00 gfal2 (2.16.3-1) unstable; urgency=low * New version -- DMC Devel Fri, 5 Apr 2019 14:00:00 +0100 gfal2 (2.16.2-1) unstable; urgency=low * New version -- DMC Devel Tue, 27 Nov 2018 14:00:00 +0100 gfal2 (2.16.1-1) unstable; urgency=low * New version -- DMC Devel Thu, 04 Oct 2018 16:00:00 +0100 gfal2 (2.16.0-1) unstable; urgency=low * New version -- DMC Devel Tue, 14 June 2018 16:00:00 +0100 gfal2 (2.15.5-1) unstable; urgency=low * New version -- DMC Devel Tue, 17 April 2018 16:00:00 +0100 gfal2 (2.15.4-1) unstable; urgency=low * New version -- DMC Devel Mon, 26 March 2018 16:00:00 +0100 gfal2 (2.15.3-1) unstable; urgency=low * New version -- DMC Devel Mon, 12 March 2018 16:00:00 +0100 gfal2 (2.15.2-1) unstable; urgency=low * New version -- DMC Devel Mon, 12 Feb 2018 15:06:00 +0100 gfal2 (2.15.1-1) unstable; urgency=low * New version -- DMC Devel Thu, 01 Feb 2018 11:06:00 +0100 gfal2 (2.15.0-1) unstable; urgency=low * New version -- DMC Devel Wed, 26 Jul 2017 09:12:00 +0100 gfal2 (2.14.2-1) unstable; urgency=low * New version -- DMC Devel Wed, 26 Jul 2017 17:30:00 +0100 gfal2 (2.14.1-1) unstable; urgency=low * New version -- DMC Devel Fri, 14 Jul 2017 09:52:00 +0100 gfal2 (2.14.0-1) unstable; urgency=low * New version -- DMC Devel Thu, 17 Feb 2017 08:54:00 +0100 gfal2 (2.13.0-1) unstable; urgency=low * New version -- DMC Devel Mon, 13 Sep 2016 13:46:00 +0100 gfal2 (2.12.2-1) unstable; urgency=low * New version -- DMC Devel Mon, 06 Sep 2016 14:31:00 +0100 gfal2 (2.12.1-1) unstable; urgency=low * New version -- DMC Devel Fri, 02 Sep 2016 14:31:00 +0100 gfal2 (2.12.0-1) unstable; urgency=low * New version -- DMC Devel Fri, 15 Apr 2016 08:48:00 +0100 gfal2 (2.11.1-1) unstable; urgency=low * New version -- DMC Devel Mon, 14 Mar 2016 09:13:00 +0100 gfal2 (2.11.0-1) unstable; urgency=low * New version -- DMC Devel Tue, 20 Oct 2015 10:39:00 +0200 gfal2 (2.10.1-1) unstable; urgency=low * New patch -- DMC Devel Tue, 20 Oct 2015 10:44:00 +0200 gfal2 (2.10.0-1) unstable; urgency=low * Deb packaging on CI -- DMC Devel Thu, 09 Jul 2015 09:52:00 +0200 gfal2-v2.23.0/packaging/debian/compat000066400000000000000000000000021465240014500173240ustar00rootroot000000000000008 gfal2-v2.23.0/packaging/debian/control000066400000000000000000000114521465240014500175340ustar00rootroot00000000000000Source: gfal2 Priority: optional Maintainer: DMC Devel Build-Depends: debhelper (>= 8.0.0), cmake, doxygen, libglib2.0-dev, libglibmm-2.4-dev, libattr1-dev, libldap2-dev, uuid-dev, liblfc-dev, libdpm-dev, srm-ifce-dev (>= 1.16.0), dcap-dev, libglobus-gass-copy-dev, davix-dev (>= 0.4.2), libgridsite-dev, libjson-c-dev, gsoap, pkg-config, libssh2-1-dev Standards-Version: 3.9.5 Section: net Homepage: http://dmc.web.cern.ch/ Package: gfal2 Architecture: any Depends: libgfal2-2 (= ${binary:Version}), libgfal-transfer2 (= ${binary:Version}), ${shlibs:Depends}, ${misc:Depends} Recommends: gfal2-plugin-lfc (= ${binary:Version}), gfal2-plugin-rfio (= ${binary:Version}), gfal2-plugin-dcap (= ${binary:Version}), gfal2-plugin-srm (= ${binary:Version}), gfal2-plugin-gridftp (= ${binary:Version}), gfal2-plugin-http (= ${binary:Version}) Description: Grid file access library 2.0 GFAL 2.0 offers an a single and simple POSIX-like API for the file operations in grids and cloud environments. The set of supported protocols depends on the gfal2 installed plugins. Package: libgfal2-2 Section: libs Architecture: any Depends: ${shlibs:Depends}, ${misc:Depends} Description: Core of the Grid File access Library 2.0 The main library of gfal2. The gfal protocol support relies on a plugin system. Package: libgfal-transfer2 Section: libs Architecture: any Depends: libgfal2-2 (= ${binary:Version}), ${shlibs:Depends}, ${misc:Depends} Description: File Transfer logic of gfal2 gfal-transfer is the high level API for file transfer operations in gfal2. It supports third-party copy. Package: libgfal2-dev Section: libdevel Architecture: any Depends: libgfal2-2 (= ${binary:Version}), libgfal-transfer2 (= ${binary:Version}), libglib2.0-dev, libattr1-dev, ${misc:Depends} Description: Development files of gfal2 Development files for gfal2 Package: gfal2-doc Section: doc Architecture: all Depends: ${misc:Depends} Description: Documentation for gfal2 Documentation, doxygen and examples of gfal2. Package: gfal2-plugin-file Architecture: any Depends: libgfal2-2 (= ${binary:Version}), ${shlibs:Depends}, ${misc:Depends} Replaces: libgfal2-2 (<< 2.8.4) Breaks: libgfal2-2 (<< 2.8.4) Description: Provides file support for gfal2 Provides the file support (file://) for gfal2. The file plugin provides local file operations, as copying from local to remote or the other way around. Package: gfal2-plugin-lfc Architecture: any Depends: libgfal2-2 (= ${binary:Version}), ${shlibs:Depends}, ${misc:Depends} Description: Provide the lfc access for gfal2 Provide the lfc support (LFN://) for gfal2. The LFC plugin allows read-only POSIX operations for the LFC catalog. Package: gfal2-plugin-rfio Architecture: any Depends: libgfal2-2 (= ${binary:Version}), ${shlibs:Depends}, ${misc:Depends} Description: Provide the rfio access for gfal2 Provide the rfio support (RFIO://) for gfal2. The rfio plugin provides the POSIX operations for the rfio URLs, the rfio protocol is used on the DPM and on the Castor storage systems. Package: gfal2-plugin-dcap Architecture: any Depends: libgfal2-2 (= ${binary:Version}), dcap-tunnel-gsi, ${shlibs:Depends}, ${misc:Depends} Description: Provide the dcap access for gfal2 Provide the dcap support (GSIDCAP://, DCAP://) for gfal2. The dcap plugin provides the POSIX operations for the dcap URLs, the dcap protocol is used on the DCACHE storage system Package: gfal2-plugin-srm Architecture: any Depends: libgfal2-2 (= ${binary:Version}), libgfal-srm-ifce1 (>= 1.16.0), ${shlibs:Depends}, ${misc:Depends} Description: Provide the srm access for gfal2 Provide the srm support (SRM://) for gfal2. The srm plugin provides the POSIX operations and the third party transfer support on the SRM URLs. Package: gfal2-plugin-gridftp Architecture: any Depends: libgfal2-2 (= ${binary:Version}), ${shlibs:Depends}, ${misc:Depends} Description: Provide the gridftp access for gfal2 Provide the gridftp support (GSIFTP://) for gfal2. The gridftp plugin provides the POSIX operations and the third party transfer support on the GSIFTP URLs. Package: gfal2-plugin-http Architecture: any Depends: libgfal2-2 (= ${binary:Version}), ${shlibs:Depends}, ${misc:Depends} Description: Provide the http access for gfal2 Provide the HTTP and WebDAV support for gfal2. The http plugin provides the POSIX operations and the third party transfer support on the HTTP URLs. This plugin is able to do third-party copy with WebDAV. Package: gfal2-plugin-sftp Architecture: any Depends: libgfal2-2 (= ${binary:Version}), ${shlibs:Depends}, ${misc:Depends} Description: Provide sftp access for gfal2 Provide sftp support for gfal2. Package: gfal2-plugin-mock Architecture: any Depends: libgfal2-2 (= ${binary:Version}), ${shlibs:Depends}, ${misc:Depends} Description: Provides a mock dummy protocol for gfal2 Provides a dummy mock:// protocol for gfal2. gfal2-v2.23.0/packaging/debian/copyright000066400000000000000000000022621465240014500200630ustar00rootroot00000000000000Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ Upstream-Name: gfal2 Source: https://gitlab.cern.ch/dmc/gfal2/repository/archive.tar.gz Files: * Copyright: Copyright (c) Members of the EGEE Collaboration. 2004-2013. See http://www.eu-egee.org/partners/ for details on the copyright holders. Copyright (c) CERN. 2013-2015 License: Apache-2.0 Files: debian/* Copyright: 2012-2015, Mattias Ellert 2015, DMC Devel / CERN License: Apache-2.0 License: Apache-2.0 Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at . http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. . On Debian systems the full text of the Apache-2.0 license can be found in the /usr/share/common-licenses/Apache-2.0 file. gfal2-v2.23.0/packaging/debian/gfal2-dev.install000066400000000000000000000001271465240014500212650ustar00rootroot00000000000000usr/include/* usr/lib/lib*.a usr/lib/lib*.so usr/lib/pkgconfig/* usr/share/pkgconfig/* gfal2-v2.23.0/packaging/debian/gfal2-plugin-dcap.docs000066400000000000000000000000441465240014500221720ustar00rootroot00000000000000src/plugins/dcap/README_PLUGIN_DCAP gfal2-v2.23.0/packaging/debian/gfal2-plugin-dcap.install000066400000000000000000000001131465240014500227050ustar00rootroot00000000000000usr/lib/gfal2-plugins/libgfal_plugin_dcap.so* etc/gfal2.d/dcap_plugin.conf gfal2-v2.23.0/packaging/debian/gfal2-plugin-file.docs000066400000000000000000000000441465240014500222020ustar00rootroot00000000000000src/plugins/file/README_PLUGIN_FILE gfal2-v2.23.0/packaging/debian/gfal2-plugin-file.install000066400000000000000000000000561465240014500227230ustar00rootroot00000000000000usr/lib/gfal2-plugins/libgfal_plugin_file.so* gfal2-v2.23.0/packaging/debian/gfal2-plugin-gridftp.docs000066400000000000000000000000521465240014500227210ustar00rootroot00000000000000src/plugins/gridftp/README_PLUGIN_GRIDFTP gfal2-v2.23.0/packaging/debian/gfal2-plugin-gridftp.install000066400000000000000000000001201465240014500234330ustar00rootroot00000000000000usr/lib/gfal2-plugins/libgfal_plugin_gridftp.so* etc/gfal2.d/gsiftp_plugin.conf gfal2-v2.23.0/packaging/debian/gfal2-plugin-http.docs000066400000000000000000000000441465240014500222420ustar00rootroot00000000000000src/plugins/http/README_PLUGIN_HTTP gfal2-v2.23.0/packaging/debian/gfal2-plugin-http.install000066400000000000000000000001131465240014500227550ustar00rootroot00000000000000usr/lib/gfal2-plugins/libgfal_plugin_http.so* etc/gfal2.d/http_plugin.conf gfal2-v2.23.0/packaging/debian/gfal2-plugin-lfc.docs000066400000000000000000000000421465240014500220250ustar00rootroot00000000000000src/plugins/lfc/README_PLUGIN_LFC gfal2-v2.23.0/packaging/debian/gfal2-plugin-lfc.install000066400000000000000000000001111465240014500225400ustar00rootroot00000000000000usr/lib/gfal2-plugins/libgfal_plugin_lfc.so* etc/gfal2.d/lfc_plugin.conf gfal2-v2.23.0/packaging/debian/gfal2-plugin-mock.docs000066400000000000000000000000441465240014500222140ustar00rootroot00000000000000src/plugins/mock/README_PLUGIN_MOCK gfal2-v2.23.0/packaging/debian/gfal2-plugin-mock.install000066400000000000000000000001131465240014500227270ustar00rootroot00000000000000usr/lib/gfal2-plugins/libgfal_plugin_mock.so* etc/gfal2.d/mock_plugin.conf gfal2-v2.23.0/packaging/debian/gfal2-plugin-rfio.docs000066400000000000000000000000441465240014500222220ustar00rootroot00000000000000src/plugins/rfio/README_PLUGIN_RFIO gfal2-v2.23.0/packaging/debian/gfal2-plugin-rfio.install000066400000000000000000000001131465240014500227350ustar00rootroot00000000000000usr/lib/gfal2-plugins/libgfal_plugin_rfio.so* etc/gfal2.d/rfio_plugin.conf gfal2-v2.23.0/packaging/debian/gfal2-plugin-sftp.docs000066400000000000000000000000441465240014500222370ustar00rootroot00000000000000src/plugins/sftp/README_PLUGIN_SFTP gfal2-v2.23.0/packaging/debian/gfal2-plugin-sftp.install000066400000000000000000000001131465240014500227520ustar00rootroot00000000000000usr/lib/gfal2-plugins/libgfal_plugin_sftp.so* etc/gfal2.d/sftp_plugin.conf gfal2-v2.23.0/packaging/debian/gfal2-plugin-srm.docs000066400000000000000000000000421465240014500220620ustar00rootroot00000000000000src/plugins/srm/README_PLUGIN_SRM gfal2-v2.23.0/packaging/debian/gfal2-plugin-srm.install000066400000000000000000000001111465240014500225750ustar00rootroot00000000000000usr/lib/gfal2-plugins/libgfal_plugin_srm.so* etc/gfal2.d/srm_plugin.conf gfal2-v2.23.0/packaging/debian/gfal2.docs000066400000000000000000000000141465240014500177660ustar00rootroot00000000000000DESCRIPTION gfal2-v2.23.0/packaging/debian/gfal2.install000066400000000000000000000000711465240014500205070ustar00rootroot00000000000000usr/bin/gfal2_version usr/share/man/man1/gfal2_version.1 gfal2-v2.23.0/packaging/debian/libgfal-transfer2.install000066400000000000000000000000361465240014500230210ustar00rootroot00000000000000usr/lib/libgfal_transfer.so.* gfal2-v2.23.0/packaging/debian/libgfal2-2.docs000066400000000000000000000000441465240014500206170ustar00rootroot00000000000000src/plugins/file/README_PLUGIN_FILE gfal2-v2.23.0/packaging/debian/libgfal2-2.install000066400000000000000000000001361465240014500213370ustar00rootroot00000000000000usr/lib/libgfal2.so.* etc/gfal2.d/bdii.conf etc/gfal2.d/gfal2_core.conf etc/gfal2.d/x509.conf gfal2-v2.23.0/packaging/debian/libgfal2-dev.install000066400000000000000000000002561465240014500217570ustar00rootroot00000000000000usr/include/gfal2 usr/include/gfal2/*.h usr/include/gfal2/*/*.h usr/lib/pkgconfig/gfal2.pc usr/lib/pkgconfig/gfal_transfer.pc usr/lib/libgfal2.so usr/lib/libgfal_transfer.so gfal2-v2.23.0/packaging/debian/libgfal2.docs000066400000000000000000000000251465240014500204570ustar00rootroot00000000000000README RELEASE-NOTES gfal2-v2.23.0/packaging/debian/rules000077500000000000000000000015561465240014500172150ustar00rootroot00000000000000#!/usr/bin/make -f # -*- makefile -*- -include /usr/share/dpkg/buildflags.mk CFLAGS += $(CPPFLAGS) CXXFLAGS += $(CPPFLAGS) export CFLAGS CXXFLAGS LDFLAGS %: dh $@ override_dh_auto_configure: dh_auto_configure -- -DLIB_SUFFIX="" -DSKIP_TESTS=TRUE -DPLUGIN_XROOTD=FALSE -DSYSCONF_INSTALL_DIR=/etc/ -DPLUGIN_MOCK=TRUE override_dh_auto_build: dh_auto_build -- all doc override_dh_install: mkdir -p debian/gfal2-doc/usr/share/doc/gfal2-doc mv debian/tmp/usr/share/doc/gfal2/examples \ debian/tmp/usr/share/doc/gfal2/html \ debian/gfal2-doc/usr/share/doc/gfal2-doc rm -rf debian/tmp/usr/share/doc dh_install --fail-missing override_dh_auto_clean: dh_auto_clean override_dh_auto_test: objdir=`ls -d obj-*` ; \ export GFAL_PLUGIN_DIR=$(CURDIR)/$${objdir}/plugins ; \ export GFAL_CONFIG_DIR=$(CURDIR)/$${objdir}/test/conf_test ; \ dh_auto_test -- "ARGS+=-V" gfal2-v2.23.0/packaging/gfal2-repo-manager.py000077500000000000000000000164601465240014500206360ustar00rootroot00000000000000#!/usr/bin/env python3 # Author: Georgios Bitzes import argparse import errno import os import re import shutil import subprocess import sys DRY_RUN = False NO_CREATE_REPO = False RAWHIDE_VERSIONS = ["fc41"] def sh(cmd): # poor man's subprocess.check_output, not supported on SL6 process = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True, stderr=subprocess.STDOUT) output, unused_err = process.communicate() retcode = process.poll() if retcode: raise Exception("Command {0} exited with code {1}. Output: {2}".format(cmd, retcode, output)) return output def ensure_valid_choice(parser, choice, text, available): if choice: if type(choice) == str: choice = [choice] for item in choice: if item not in available: parser.error("unrecognized {0}: '{1}'. Available choices: {2}".format(text, item, available)) def add_dependency(parser, when_present, dependency): if hasattr(parser, when_present): if not hasattr(parser, dependency): parser.error("argument --{0} is required when --{1} is present".format(dependency, when_present)) def declare_required(parser, args, choice): choice = choice.replace("-", "_") if not hasattr(args, choice): parser.error("argument --{0} is required".format(choice)) def bailout(msg): raise ValueError(msg) # poor man's enum class PackageType: Binary, NoArch, Source = range(1, 4) class Package(object): def __init__(self, path): self.path = path if not os.path.isfile(self.path): bailout("Not a file: {0}".format(self.path)) self.filename = os.path.basename(self.path) if self.filename.endswith(".src.rpm"): self.type = PackageType.Source tmp = self.filename[0:-8] elif self.filename.endswith(".noarch.rpm"): self.type = PackageType.NoArch tmp = self.filename[0:-11] elif self.filename.endswith(".rpm"): self.type = PackageType.Binary tmp = self.filename[0:-4] else: bailout("Unable to parse RPM type for {0}".format(self.path)) if self.type == PackageType.Source or self.type == PackageType.NoArch: self.arch = None elif tmp.endswith(".x86_64"): tmp = tmp[0:-7] self.arch = "x86_64" elif tmp.endswith(".i386"): tmp = tmp[0:-5] self.arch = "i386" else: bailout("Unable to determine architecture for {0}".format(self.path)) self.platform = tmp.split(".")[-1] tmp = tmp[0:-len(self.platform)-1] if self.platform == "cern": self.platform = tmp.split(".")[-1] tmp = tmp[0:-len(self.platform)-1] if self.platform in RAWHIDE_VERSIONS: self.platform = "fc-rawhide" self.packagename = tmp def construct_location(platform, arch, filename): return "{0}/{1}/{2}".format(platform, arch, filename) def is_tag(ref): return (re.compile("""^(v)(\d+)\.(\d+)\.(\d+)(-(rc)?(\d+))?$""").match(ref) != None or re.compile("""^(v)(\d+)\.(\d+)$""").match(ref) != None) def mkdir_p(path): try: os.makedirs(path) except OSError as exc: # Python >2.5 if exc.errno == errno.EEXIST and os.path.isdir(path): pass else: raise def createrepo(repo): print("-- Running createrepo on {0}".format(repo)) if NO_CREATE_REPO: return sh("rm -rf {0}".format(repo + "/.olddata")) sh("createrepo -q {0}".format(repo)) def copy_to_repo(source, repo): print("-- Copying {0} to {1}".format(source, repo)) if DRY_RUN: return mkdir_p(repo) shutil.copyfile(source, "{0}/{1}".format(repo, os.path.basename(source))) class Repository(object): def __init__(self, base): self.base = base if not os.path.isdir(self.base): bailout("Not a directory: {0}".format(self.base)) def store(self, ref, packages): platforms = set([x.platform for x in packages]) if len(platforms) != 1: raise ValueError("Cannot mix packages of different platforms in the same invocation: {0}".format(list(platforms))) archs = set([x.arch for x in packages]) archs.discard(None) if len(archs) != 1: raise ValueError("Cannot mix packages of different architectures in the same invocation: {0}".format(list(archs))) tag = is_tag(ref) if tag: base = "{0}/rc".format(self.base) else: base = "{0}/testing".format(self.base) if ref != "develop": base = "{0}/testing/{1}".format(self.base, ref) base += "/" + list(platforms)[0] reposToCreate = set() for package in packages: repo = "{0}/{1}".format(base, list(archs)[0]) copy_to_repo(package.path, repo) reposToCreate.add(repo) for repo in reposToCreate: createrepo(repo) def declare_incompatible_options(parser, option, group): if option not in sys.argv: return for item in group: if item in sys.argv: parser.error("argument {0} is incompatible with argument {1}".format(option, item)) def parseargs(): parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter, description="An opinionated yum repository manager.\n") parser.add_argument('--base', type=str, required=True, help="The base directory for your project.") parser.add_argument('--action', type=str, required=True, help="The action to perform. Choices: ['add', 'cleanup']") parser.add_argument('--dry-run', action="store_true", help="If set, don't actually change any files, just show what would happen if ran.") parser.add_argument('--no-create-repo', action="store_true", help="If set, don't run createrepo at the end.") parser.set_defaults(dry_run=False, no_create_repo=False) group = parser.add_argument_group('add options') group.add_argument('--ref', type=str, help="The branch or tag that is being built. Tag names must match 'x.y' or 'x.y.z' (may be prepended by 'v')") group.add_argument('--packages', type=str, nargs='+', help="The list of packages to add") group = parser.add_argument_group('cleanup options') group.add_argument('--keep-last-days', type=int, help="How many days worth of RPMs to keep. (only affects branches)") args = parser.parse_args() ensure_valid_choice(parser, args.action, "action", ["add", "cleanup"]) declare_incompatible_options(parser, "--no-create-repo", ["--dry-run"]) if args.action == "add": declare_required(parser, args, "ref") declare_required(parser, args, "packages") if args.action == "cleanup": declare_required(parser, args, "keep-last-days") bailout("NYI") if args.ref == "tags" or args.ref == "tag": bailout("A branch named '{0}'? Really?".format(args.ref)) global DRY_RUN global NO_CREATE_REPO if args.dry_run: DRY_RUN = True NO_CREATE_REPO = True if args.no_create_repo: NO_CREATE_REPO = True return args def main(): args = parseargs() repository = Repository(args.base) packages = [Package(x) for x in args.packages] repository.store(args.ref, packages) if __name__ == '__main__': main() gfal2-v2.23.0/packaging/rpm/000077500000000000000000000000001465240014500155025ustar00rootroot00000000000000gfal2-v2.23.0/packaging/rpm/gfal2.spec000066400000000000000000000517071465240014500173630ustar00rootroot00000000000000%undefine __cmake_in_source_build %undefine __cmake3_in_source_build # Require --with=tests in order to build functional tests %bcond_with tests Name: gfal2 Version: 2.23.0 Release: 1%{?dist} Summary: Grid file access library 2.0 License: ASL 2.0 URL: https://dmc-docs.web.cern.ch/dmc-docs/gfal2/gfal2.html # git clone --depth=1 --branch=v2.23.0 https://gitlab.cern.ch/dmc/gfal2.git gfal2-2.23.0 # tar czf gfal2-2.23.0.tar.gz --exclude-vcs gfal2-2.23.0 Source0: %{name}-%{version}.tar.gz #main lib dependencies BuildRequires: gcc-c++ BuildRequires: cmake3 BuildRequires: doxygen BuildRequires: json-c-devel BuildRequires: glib2-devel >= 2.28 Requires: glib2 >= 2.28 BuildRequires: libattr-devel BuildRequires: openldap-devel BuildRequires: pugixml-devel BuildRequires: libuuid-devel #file plugin dependencies BuildRequires: zlib-devel #srm plugin dependencies BuildRequires: srm-ifce-devel >= 1.23.1 #dcap plugin dependencies BuildRequires: dcap-devel #gridftp plugin dependencies BuildRequires: globus-gass-copy-devel #http plugin dependencies BuildRequires: davix-devel >= 0.8.4 BuildRequires: cryptopp-devel >= 5.6.2 #xrootd plugin dependencies BuildRequires: xrootd-client-devel >= 1:5.0.0 # sftp plugin dependencies BuildRequires: libssh2-devel #tests dependencies BuildRequires: gtest-devel Obsoletes: %{name}-core < %{version}-%{release} Provides: %{name}-core = %{version}-%{release} Obsoletes: %{name}-transfer < %{version}-%{release} Provides: %{name}-transfer = %{version}-%{release} %description GFAL 2.0 offers an a single and simple POSIX-like API for the file operations in grids and cloud environments. The set of supported protocols depends of the %{name} installed plugins. %package devel Summary: Development files of %{name} Requires: %{name}%{?_isa} = %{version}-%{release} Requires: glib2-devel%{?_isa} Requires: libattr-devel%{?_isa} Requires: pkgconfig %description devel development files for %{name} %package doc Summary: Documentation for %{name} BuildArch: noarch %description doc Documentation, Doxygen and examples of %{name}. %package plugin-file Summary: Provides file support for %{name} Requires: %{name}%{?_isa} = %{version}-%{release} %description plugin-file Provides the file support (file://) for %{name}. The file plugin provides local file operations, as copying from local to remote or the other way around. %package plugin-dcap Summary: Provides the support access for %{name} Requires: %{name}%{?_isa} = %{version}-%{release} Requires: dcap-tunnel-gsi%{?_isa} %description plugin-dcap Provides the dcap support (gsidcap://, dcap://) for %{name}. The dcap plugin provides the POSIX operations for the dcap \ URLs, the dcap protocol is used on the DCACHE storage system %package plugin-srm Summary: Provides the srm access for %{name} Requires: %{name}%{?_isa} = %{version}-%{release} Requires: srm-ifce >= 1.23.1 %description plugin-srm Provides the srm support (srm://) for %{name}. The srm plugin provides the POSIX operations and the third party transfer support on the SRM URLs. %package plugin-gridftp Summary: Provides the gridftp support for %{name} Requires: %{name}%{?_isa} = %{version}-%{release} %description plugin-gridftp Provides the gridftp support (gsiftp://) for %{name}. The gridftp plugin provides the POSIX operations and the third party transfer support on the GSIFTP URLs. %package plugin-http Summary: Provides the HTTP/DAV support for %{name} Requires: %{name}%{?_isa} = %{version}-%{release} Requires: davix-libs >= 0.8.4 %description plugin-http Provides the HTTP (http[s]://) and WevDAV (dav[s]://) support for %{name}. This plugin is able to do Third-Party Copy with WebDAV, if the storage supports it. %package plugin-xrootd Summary: Provide xrootd support for GFAL2 Requires: %{name}%{?_isa} = %{version}-%{release} %description plugin-xrootd The Grid File Access Library, GFAL2, provides a simple POSIX-like API for file operations in grid and cloud environments. Plug-ins are available to allow access via a variety of protocols. This package contains a plugin for the xrootd protocol (root://). %package plugin-sftp Summary: Provide sftp support for GFAL2 Requires: %{name}%{?_isa} = %{version}-%{release} %description plugin-sftp The Grid File Access Library, GFAL2, provides a simple POSIX-like API for file operations in grid and cloud environments. Plug-ins are available to allow access via a variety of protocols. This package contains a plugin for the sftp protocol (sftp://). %package plugin-mock Summary: Provides a Mock dummy protocol for %{name} Requires: %{name}%{?_isa} = %{version}-%{release} %description plugin-mock Provides a dummy mock:// protocol for %{name}. %package all Summary: Meta package for GFAL 2.0 install Requires: %{name}%{?_isa} = %{version}-%{release} Requires: %{name}-plugin-file%{?_isa} = %{version}-%{release} Requires: %{name}-plugin-dcap%{?_isa} = %{version}-%{release} Requires: %{name}-plugin-srm%{?_isa} = %{version}-%{release} Requires: %{name}-plugin-gridftp%{?_isa} = %{version}-%{release} Requires: %{name}-plugin-http%{?_isa} = %{version}-%{release} Requires: %{name}-plugin-xrootd%{?_isa} = %{version}-%{release} Requires: %{name}-plugin-sftp%{?_isa} = %{version}-%{release} %description all Meta-package for complete install of GFAL2 with all the protocol plugins. %package tests Summary: gfal2 tests Requires: gfal2-all%{?_isa} = %{version}-%{release} Requires: gfal2-plugin-mock%{?_isa} = %{version}-%{release} %description tests gfal2 tests %clean %cmake3_build --target clean %prep %setup -q %build # Make sure the version in the spec file and the version used # for building matches gfal2_cmake_ver=`sed -n 's/^set *(VERSION_\(MAJOR\|MINOR\|PATCH\) \+\([0-9]\+\).*/\2/p' CMakeLists.txt | paste -sd '.'` gfal2_spec_ver=`expr "%{version}" : '\([0-9]*\\.[0-9]*\\.[0-9]*\)'` if [ "$gfal2_cmake_ver" != "$gfal2_spec_ver" ]; then echo "The version in the spec file does not match the CMakeLists.txt version!" echo "$gfal2_cmake_ver != %{version}" exit 1 fi %cmake3 \ -DDOC_INSTALL_DIR=%{_pkgdocdir} \ -DUNIT_TESTS=TRUE \ -DPLUGIN_MOCK=TRUE \ -DFUNCTIONAL_TESTS=%{?with_tests:ON}%{?!with_tests:OFF} %cmake3_build %cmake3_build --target doc %install %cmake3_install %ldconfig_scriptlets %files %{_bindir}/gfal2_version %{_libdir}/libgfal2.so.* %{_libdir}/libgfal_transfer.so.* %dir %{_libdir}/%{name}-plugins %dir %{_sysconfdir}/%{name}.d %config(noreplace) %{_sysconfdir}/%{name}.d/bdii.conf %config(noreplace) %{_sysconfdir}/%{name}.d/gfal2_core.conf %config(noreplace) %{_sysconfdir}/%{name}.d/x509.conf %{_mandir}/man1/gfal2_version.1* %dir %{_pkgdocdir} %{_pkgdocdir}/DESCRIPTION %{_pkgdocdir}/README.md %{_pkgdocdir}/LICENSE %{_pkgdocdir}/RELEASE-NOTES %files devel %{_includedir}/%{name}/ %{_libdir}/pkgconfig/gfal2.pc %{_libdir}/pkgconfig/gfal_transfer.pc %{_libdir}/libgfal2.so %{_libdir}/libgfal_transfer.so %files doc %{_pkgdocdir}/readme.html %{_pkgdocdir}/html/ %{_pkgdocdir}/examples/ %files plugin-file %{_libdir}/%{name}-plugins/libgfal_plugin_file.so* %{_pkgdocdir}/README_PLUGIN_FILE %files plugin-dcap %{_libdir}/%{name}-plugins/libgfal_plugin_dcap.so* %{_pkgdocdir}/README_PLUGIN_DCAP %config(noreplace) %{_sysconfdir}/%{name}.d/dcap_plugin.conf %files plugin-srm %{_libdir}/%{name}-plugins/libgfal_plugin_srm.so* %{_pkgdocdir}/README_PLUGIN_SRM %config(noreplace) %{_sysconfdir}/%{name}.d/srm_plugin.conf %files plugin-gridftp %{_libdir}/%{name}-plugins/libgfal_plugin_gridftp.so* %{_pkgdocdir}/README_PLUGIN_GRIDFTP %config(noreplace) %{_sysconfdir}/%{name}.d/gsiftp_plugin.conf %files plugin-http %{_libdir}/%{name}-plugins/libgfal_plugin_http.so* %{_pkgdocdir}/README_PLUGIN_HTTP %config(noreplace) %{_sysconfdir}/%{name}.d/http_plugin.conf %files plugin-xrootd %{_libdir}/%{name}-plugins/libgfal_plugin_xrootd.so* %{_pkgdocdir}/README_PLUGIN_XROOTD %config(noreplace) %{_sysconfdir}/%{name}.d/xrootd_plugin.conf %files plugin-sftp %{_libdir}/%{name}-plugins/libgfal_plugin_sftp.so* %{_pkgdocdir}/README_PLUGIN_SFTP %config(noreplace) %{_sysconfdir}/%{name}.d/sftp_plugin.conf %files plugin-mock %{_libdir}/%{name}-plugins/libgfal_plugin_mock.so* %{_pkgdocdir}/README_PLUGIN_MOCK %config(noreplace) %{_sysconfdir}/%{name}.d/mock_plugin.conf %files tests %{_bindir}/gfal2-unit-tests %{_libdir}/libgfal2_test_shared.so %files all %changelog * Wed Jul 31 2024 Mihai Patrascoiu - 2.23.0-1 - New upstream release - Renamed HTTP COPY "TransferMetadata" to "ArchiveMetadata" - Introduce option to disable the automatic clean-up on transfer failure * Wed Mar 20 2024 Mihai Patrascoiu - 2.22.2-1 - New upstream release - Pass query string in XRootd bringonline operations * Mon Dec 11 2023 Joao Lopes - 2.22.1-1 - New upstream release - Introduces retrieval of storage tokens using OAuth2 flow * Wed Oct 18 2023 Joao Lopes - 2.22.0-1 - New upstream release - Introduces support for scitags * Tue Jul 25 2023 Mihai Patrascoiu - 2.21.5-1 - New upstream release * Mon Apr 03 2023 Joao Lopes - 2.21.4-1 - New upstream release * Tue Feb 07 2023 Joao Lopes - 2.21.3-1 - New upstream release * Thu Dec 01 2022 Mihai Patrascoiu - 2.21.2-1 - New upstream release * Thu Sep 29 2022 Mihai Patrascoiu - 2.21.1-1 - New upstream release * Fri Jul 08 2022 Joao Lopes - 2.21.0-1 - New upstream release - Introduces support for HTTP tape operations * Fri Mar 04 2022 Mihai Patrascoiu - 2.20.5-1 - New upstream release * Tue Feb 22 2022 Mihai Patrascoiu - 2.20.4-1 - New upstream release * Wed Jan 19 2022 Joao Lopes - 2.20.3-1 - New upstream release * Mon Dec 06 2021 Mihai Patrascoiu - 2.20.2-1 - New upstream release * Mon Oct 25 2021 Mihai Patrascoiu - 2.20.1-1 - New upstream release * Mon Sep 20 2021 Mihai Patrascoiu - 2.20.0-1 - New upstream release - Introduces SE-Token retrieval * Wed May 26 2021 Mihai Patrascoiu - 2.19.2-1 - New upstream release * Thu Mar 25 2021 Mihai Patrascoiu - 2.19.1-1 - New upstream release * Fri Dec 04 2020 Mihai Patrascoiu - 2.19.0-1 - New upstream release - Introduces Archiving API * Mon Jul 27 2020 Mihai Patrascoiu - 2.18.2-1 - New upstream release * Wed Jun 17 2020 Mihai Patrascoiu - 2.18.1-1 - New upstream release * Fri Jun 05 2020 Mihai Patrascoiu - 2.18.0-1 - New upstream release - Compatibility ensured with XRootD v5 - CDMI QoS interface implemented in HTTP plugin * Thu May 07 2020 Mihai Patrascoiu - 2.17.3-1 - New upstream release * Tue Jan 28 2020 Michal Simon - 2.17.2-1 - New upstream release * Thu Nov 07 2019 Andrea Manzi - 2.17.1-1 - New upstream release * Tue Oct 22 2019 Andrea Manzi - 2.17.0-1 - New upstream release * Wed Sep 18 2019 Andrea Manzi - 2.16.4-2 - New upstream release * Fri May 17 2019 Andrea Manzi - 2.16.3-1 - New upstream release * Mon Sep 24 2018 Andrea Manzi - 2.16.0-1 - New upstream release * Thu Apr 05 2018 Andrea Manzi - 2.15.4-1 - New upstream release * Mon Mar 12 2018 Andrea Manzi - 2.15.3-1 - New upstream release * Mon Feb 20 2017 Alejandro Alvarez Ayllon - 2.13.1-1 - Upgraded to upstream release 2.13.1 * Fri Feb 10 2017 Fedora Release Engineering - 2.12.3-4 - Rebuilt for https://fedoraproject.org/wiki/Fedora_26_Mass_Rebuild * Thu Jan 26 2017 Alejandro Alvarez Ayllon - 2.12.3-3 - Remove trailing whitespaces on pkgconfig file * Mon Nov 14 2016 Alejandro Alvarez Ayllon - 2.12.3-2 - Enable build of mock plugin * Thu Nov 10 2016 Alejandro Alvarez Ayllon - 2.12.3-1 - Upgraded to upstream release 2.12.3 * Wed Oct 26 2016 Richard Shaw - 2.12.2-2.el7.1 - Rebuild to fix soname dependency on aarch64. * Thu Sep 29 2016 Richard Shaw - 2.12.2-2 - Rebuild for pugixml with c++11 support required by mkvtoonix. * Thu Sep 22 2016 Alejandro Alvarez Ayllon - 2.12.2-1 - Upgraded to upstream release 2.12.2 * Wed Jul 20 2016 Richard Shaw - 2.11.1-2 - Rebuild for new pugixml. * Tue Apr 19 2016 Alejandro Alvarez Ayllon - 2.11.1-1 - Upgraded to upstream release 2.11.1 * Mon Mar 07 2016 Alejandro Alvarez Ayllon - 2.11.0-1 - Upgraded to upstream release 2.11.0 * Wed Feb 03 2016 Fedora Release Engineering - 2.10.3-2 - Rebuilt for https://fedoraproject.org/wiki/Fedora_24_Mass_Rebuild * Wed Dec 09 2015 Alejandro Alvarez Ayllon - 2.10.3-1 - Upgraded to upstream release 2.10.3 * Fri Nov 06 2015 Alejandro Alvarez Ayllon - 2.10.2-1 - Upgraded to upstream release 2.10.2 * Fri Jul 03 2015 Alejandro Alvarez Ayllon - 2.9.3-1 - Upgraded to upstream release 2.9.3 * Wed Jun 17 2015 Fedora Release Engineering - 2.9.1-2 - Rebuilt for https://fedoraproject.org/wiki/Fedora_23_Mass_Rebuild * Thu Apr 16 2015 Alejandro Alvarez Ayllon - 2.9.1-1 - Upgraded to upstream release 2.9.1 * Mon Mar 02 2015 Alejandro Alvarez Ayllon - 2.8.4-1 - Upgraded to upstream release 2.8.4 * Mon Jan 12 2015 Alejandro Alvarez Ayllon - 2.8.1-1 - Upgraded to upstream release 2.8.1 * Mon Dec 15 2014 Alejandro Alvarez Ayllon - 2.7.8-3 - Applied patch moving buffer to heap to avoid SIGSEGV when the stack size is limited * Tue Dec 02 2014 Alejandro Alvarez Ayllon - 2.7.8-2 - Patched a bug in a call to gfal2_set_error * Mon Nov 17 2014 Alejandro Alvarez Ayllon - 2.7.8-1 - Upstream backported fix for protocol honoring on SRM GET and PUT * Mon Nov 10 2014 Alejandro Alvarez Ayllon - 2.7.7-1 - Upgraded to upstream release 2.7.7 * Fri Nov 07 2014 Alejandro Alvarez Ayllon - 2.7.6-1 - New upstream release * Mon Sep 08 2014 Alejandro Alvarez Ayllon - 2.6.8-6 - Patch to use lseek64 instead of lseek in the http plugin * Thu Sep 04 2014 Orion Poplawski - 2.6.8-5 - Rebuild for pugixml 1.4 * Sat Aug 16 2014 Fedora Release Engineering - 2.6.8-4 - Rebuilt for https://fedoraproject.org/wiki/Fedora_21_22_Mass_Rebuild * Mon Aug 11 2014 Alejandro Alvarez Ayllon - 2.6.8-3 - Disable GridFTP session reuse by default (see LCGUTIL-448) * Fri Aug 08 2014 Alejandro Alvarez Ayllon - 2.6.8-2 - Patch for symbol that dissapeared in Davix * Mon Jul 28 2014 Alejandro Alvarez Ayllon - 2.6.8-1 - Release 2.6.8 of GFAL2 * Thu Mar 13 2014 Alejandro Alvarez - 2.5.5-2 - Backported patch that fixes segfault on the SRM plugin when listing empty directories * Wed Feb 26 2014 Adrien Devresse - 2.5.5-1 - Release 2.5.5 of GFAL2 * Thu Dec 05 2013 Alejandro Alvarez - 2.4.8-1 - Release 2.4.8 of GFAL2 * Mon Dec 02 2013 Alejandro Alvarez - 2.4.7-1 - Release 2.4.7 of GFAL2 * Thu Nov 07 2013 Alejandro Alvarez - 2.4.6-1 - Release 2.4.6 of GFAL 2 * Wed Oct 23 2013 Alejandro Alvarez - 2.4.5-3 - Release 2.4.5 of GFAL 2 * Tue Jul 02 2013 Adrien Devresse - 2.3.0-0 - Release 2.3.0 of GFAL 2.0 * Tue Apr 30 2013 Adrien Devresse - 2.2.1-0 - export transfer plugin API ( needed for xrootd plugin ) * Mon Apr 29 2013 Michail Salichos - 2.2.0-5 - make all gridftp ops async to avoid stalling processes * Fri Apr 26 2013 Michail Salichos - 2.2.0-4 - replace gass stat with gridftp stat * Mon Apr 22 2013 Michail Salichos - 2.2.0-3 - change gridftp error string pattern to satisfy Griffin * Wed Apr 10 2013 Michail Salichos - 2.2.0-2 - display turls in verbose mode, needed by fts3 * Mon Mar 25 2013 Michail Salichos - 2.2.0-1 - fix memory leaks in bringonline SRM op * Wed Mar 20 2013 Adrien Devresse - 2.2.0-0 - fix thread safety issue with gsiftp plugin - add the bring online API - support for the http plugin by default - remove executable stack need - remove openMP dependency - add synchronous cancellation API - add gsiftp performance marker timeout - support for srm session reuse - reduce memory footprint * Fri Feb 22 2013 Adrien Devresse - 2.1.6-0 - FTS 3.0 EMI 3 update - minor fix on the cancel logic - change the performance marker auto-cancel threading model - change the performance marker default timeout value * Mon Feb 11 2013 Adrien Devresse - 2.1.5-0 - FTS 3.0 EMI 3 release sync - include event hooks support - include cancel logic support - include performance marker auto-cancel for gsiftp - include checksum timeout support for gsiftp - include srm session re-use support * Thu Jan 10 2013 Adrien Devresse - 2.1.1-0 - fix a minor memory issue with the gfal_transfer stack - fix a wrong error report problem with srm third party copy * Wed Dec 05 2012 Adrien Devresse - 2.1.0-2 - fix an issue this surl to turl resolution for SRM third party copy * Fri Nov 30 2012 Adrien Devresse - 2.1.0-0 - One-globus session system for gsiftp plugin ( FTS 3.0 need ) - correct a major issue with the gass attribute system in gsiftp plugin - change the lfc set/get env var for a one compatible with set/get opt - add set/nb streams option for gsiftp - add the mkdir rec function for SRM transfer - correct an issue with opendir and srm_ls ( ENOTDIR error silent ) - correct a memory leak in the cache system - correct timeout support for gsiftp transfer - implement tcp buffer size support for gsiftp layer - apply a correction on the SRM over-write logic, related to a BeStMan errcode problem on File Not Found with srmRm ( EOS ) - apply a fix on the transfer gsiftp timeout ( protection against multiple cancel ) - fix for SRM filesize problem ( defined to 0, workaround ) related to globus 426 error bad filesize - secure the callback system for globus gass timeout - base implementation of the http plugin - improve reliability of the bdii resolution - add a fallback mechanism in case of bdii bad resolution - correct several race conditions in the bdii layer - add thread safe support for set/get variables in liblfc - correct a deadlock problem with globus and gisftp plugin - implement the mkdir_rec logic for general purpose - implement the parent folder creation logic with gridftp - add support for lfc://host/path URL style for the lfc plugin - switch off_t to 64bits size by default ( _FILE_OFFSET_BITS=64) - provide a "nobdii" like option - provide the choice of turl protocol resolution for srm plugin * Fri Jul 20 2012 Adrien Devresse - 2.0.0-1 - Official initial release candidate of gfal 2.0 - Transfer API is official - gridftp support for performance marker, checksum - gridftp support for gridftpv2, dcau param - SRM support for spacetoken in transfer - SRM abort auto-management - parallel operations in transfers - file protocol dedicated in a plugin - configuration file support - srm timeout support - general purpose checksum operation support - POSIX operation support for gridftp - cleaner plugin API - new documentation - I hope that you will enjoy gfal 2.0 :) * Sat Jun 23 2012 Adrien Devresse - 2.0.0-0.10.2012062323snap - Snapshot of the 0.10 version for testing * Fri Jun 15 2012 Adrien Devresse - 2.0.0-0.9.2012061511snap - Snapshot of the 0.9 version for testing * Fri May 04 2012 Adrien Devresse - 2.0.0-0.8.2012052812snap - Snapshot of the 0.8 version for testing. * Fri May 04 2012 Adrien Devresse - 2.0.0-0.7.2012050413snap - Improve gridftp plugin with severals other calls - Correct dcap/rfio/srm bugs related to error report - big work on the documentation * Mon Dec 12 2011 Adrien Devresse - 2.0.0-0.6.2012041515snap - Initial gfal 2.0 preview release gfal2-v2.23.0/qa/000077500000000000000000000000001465240014500133615ustar00rootroot00000000000000gfal2-v2.23.0/qa/Dockerfile000066400000000000000000000030551465240014500153560ustar00rootroot00000000000000# # Copyright (c) CERN 2016 # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # FROM centos:7 # Required repos ADD "https://dmc-repo.web.cern.ch/dmc-repo/dmc-devel-el7.repo" "/etc/yum.repos.d/" ADD "http://repository.egi.eu/sw/production/cas/1/current/repo-files/EGI-trustanchors.repo" "/etc/yum.repos.d/" RUN yum install -y epel-release yum-utils # Build requirements RUN yum install -y gcc gcc-c++ RUN yum-builddep -y gfal2 # Run dependencies RUN yum install -y cppcheck java-1.8.0-openjdk lcg-CA lcov rats unzip voms-clients voms-config-vo-dteam which # Coverage requirements ADD "http://repo1.maven.org/maven2/org/codehaus/sonar/runner/sonar-runner-dist/2.4/sonar-runner-dist-2.4.zip" /tmp RUN unzip "/tmp/sonar-runner-dist-2.4.zip" -d "/tmp" ADD sonar-runner.properties "/tmp/sonar-runner-2.4/conf" ADD "https://raw.github.com/eriwen/lcov-to-cobertura-xml/master/lcov_cobertura/lcov_cobertura.py" /tmp RUN chmod a+r /tmp/lcov_cobertura.py # Required input VOLUME ["/gfal2", "/.ssh", "/.globus"] # Entry point COPY "qa.sh" "sonar-project.properties" / ENTRYPOINT ["/qa.sh"] gfal2-v2.23.0/qa/qa.sh000077500000000000000000000022071465240014500143220ustar00rootroot00000000000000#!/usr/bin/bash set -x export HOME=/ export SONAR_USER_HOME=/tmp VOMS=${VOMS:=dteam} PASSWD=$1 if [[ -z "$PASSWD" ]]; then echo "Missing password" exit 1 fi mkdir -p /tmp/build pushd /tmp/build # Build CFLAGS=--coverage CXXFLAGS=--coverage cmake "/gfal2" \ -DUNIT_TESTS=ON -DFUNCTIONAL_TESTS=ON -DPLUGIN_MOCK=ON make -j2 # Static checkers cppcheck -v --enable=all \ -I "/usr/include" \ -I "/usr/include/glib-2.0" \ -I "/gfal2/src/core" \ -I "/gfal2/src/utilis" \ --xml "/gfal2" 2> cppcheck.xml rats -w 3 --xml "/gfal2" > rats.xml # Clear counters lcov --directory . --zerocounters # Run echo ${PASSWD} | voms-proxy-init --pwstdin --voms ${VOMS} export GFAL_PLUGIN_DIR="/tmp/build/plugins" export GFAL_CONFIG_DIR="/gfal2/test/conf_test" ctest -T test # Extract coverage lcov --directory . --capture --output-file="/tmp/coverage.info" python /tmp/lcov_cobertura.py "/tmp/coverage.info" -b "/gfal2" -e ".+usr.include." -o "coverage.xml" # Run sonar cp /sonar-project.properties . export SONAR_RUNNER_OPTS="-Duser.timezone=+01:00 -Djava.security.egd=file:///dev/urandom" /tmp/sonar-runner-2.4/bin/sonar-runner -X popd gfal2-v2.23.0/qa/sonar-project.properties000066400000000000000000000007461465240014500202740ustar00rootroot00000000000000sonar.projectKey=dmc:gfal2 sonar.projectName=gfal2 sonar.projectVersion=2.14.0 sonar.working.directory=/tmp/sonar-wd sonar.projectBaseDir=/gfal2/ sonar.sources=/gfal2/src/ sonar.exclusions=src/plugins/mock/* sonar.language=c++ sonar.core.codeCoveragePlugin=cobertura sonar.cxx.coverage.overallReportPath=/tmp/build/coverage.xml sonar.cxx.coverage.itReportPath=/tmp/build/coverage.xml sonar.cxx.cppcheck.reportPath=/tmp/build/cppcheck.xml sonar.cxx.rats.reportPath=/tmp/build/rats.xml gfal2-v2.23.0/qa/sonar-runner.properties000066400000000000000000000000551465240014500201300ustar00rootroot00000000000000sonar.host.url=http://baluarte.cern.ch:8080 gfal2-v2.23.0/readme.html000066400000000000000000000055771465240014500151210ustar00rootroot00000000000000 gfal2

gfal2

Description

The Grid File Access Library 2 offers a POSIX interface to the distributed files system supported by WLCG.
Differents file systems are supported by a collections of GFAL 2 plugins, this is a non complete list:

  • GRIDFTP with gsiftp://
  • LFC (Logical File Catalog), with lfn://
  • SRM (Storage Resource Manager) with srm://
  • RFIO (Remote File I/O) with rfio://
  • DCAP (Remote FTP for dCache ) with gsidcap://
  • HTTP/Webdav with cluster, third party copy and client side credential support
  • XRootD protocll with root://
  • S3 and Google Cloud, with s3:// and gcloud://.

Installation

GFAL2 itself abstracts the access to different underlying storage systems - including local -, but in order to get support for a given protocol, its corresponding plugin needs to be installed.

  • gfal2-plugin-gridftp
  • gfal2-plugin-lfc
  • gfal2-plugin-srm
  • gfal2-plugin-rfio
  • gfal2-plugin-dcap
  • gfal2-plugin-http
  • gfal2-plugin-xrootd
  • gfal2-plugin-file
  • gfal2-plugin-mock

For convenience we provide a package called gfal2-all, which will trigger the installation of all these packages.
gfal2-devel is required only for development.
A typical installation can be done as easy as bellow: 

yum install gfal2-all gfal2-devel

API Documentation

Doxygen documentation
You can also have a look at the examples folder to see some simple usage of the API.

License

This software is licensed under the Apache 2 License. See LICENSE for details.

Release Notes

See RELEASE-NOTES for a detailed changelog.

Contact

You can notify bugs or ask for feature requests via

Tracker
https://its.cern.ch/jira/projects/DMC/
Mailing list
dmc-support@cern.ch
gfal2-v2.23.0/src/000077500000000000000000000000001465240014500135475ustar00rootroot00000000000000gfal2-v2.23.0/src/CMakeLists.txt000066400000000000000000000007561465240014500163170ustar00rootroot00000000000000cmake_minimum_required (VERSION 2.6) # Shared utilities can be compiled as static, and then # included at linking time add_subdirectory (utils) set(gfal2_utils_src ${gfal2_utils_src} PARENT_SCOPE) set(gfal2_utils_c_src ${gfal2_utils_c_src} PARENT_SCOPE) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/utils) # Core library if (MAIN_CORE) add_subdirectory (core) # Command line tool to get GFAL2 version add_subdirectory (version) endif (MAIN_CORE) # Plugins add_subdirectory (plugins) gfal2-v2.23.0/src/core/000077500000000000000000000000001465240014500144775ustar00rootroot00000000000000gfal2-v2.23.0/src/core/CMakeLists.txt000066400000000000000000000057711465240014500172510ustar00rootroot00000000000000cmake_minimum_required (VERSION 2.6) # Sources needed for the core library file (GLOB src_cancel "cancel/*.c*") file (GLOB src_common "common/*.c" "future/*.c") file (GLOB src_config "config/*.c") file (GLOB src_exceptions "exceptions/*.c*") file (GLOB src_gconfig "g_config_manager/*.c*") file (GLOB src_file "file/*.c*") file (GLOB src_global "global/*c*") file (GLOB src_logger "logger/*.c*") file (GLOB src_posix "posix/*.c*") # Core library add_definitions(-D__GFAL2_BUILD__) include_directories(${gfal2_utils_includes}) add_definitions (${gfal2_utils_definitions}) add_library (gfal2 SHARED ${src_cancel} ${src_common} ${src_config} ${src_gconfig} ${src_file} ${src_global} ${src_logger} ${src_posix} ${src_exceptions} ${gfal2_utils_c_src}) target_link_libraries (gfal2 "stdc++" ) # hotfix for libc TLS init bug in EPEL 5 target_link_libraries (gfal2 ${GLIB2_PKG_LIBRARIES} ${GTHREAD2_PKG_LIBRARIES}) target_link_libraries (gfal2 "dl") target_link_libraries (gfal2 ${gfal2_utils_libraries}) set_target_properties (gfal2 PROPERTIES VERSION ${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_PATCH} SOVERSION ${VERSION_MAJOR} CLEAN_DIRECT_OUTPUT 1 OUTPUT_NAME ${OUTPUT_NAME_MAIN}) # pkgconfig for gfal2 add_PkgConfigFile_for_Library ("gfal2.pc" gfal2 HEADER_DIRS "gfal2" CFLAGS " -D_FILE_OFFSET_BITS=64 " DESCRIPTION "GFAL 2.0 pkgconfig file" REQUIRES "glib-2.0") # Install core library install (TARGETS gfal2 LIBRARY DESTINATION ${LIB_INSTALL_DIR}) install (FILES ${CMAKE_CURRENT_BINARY_DIR}/gfal2.pc DESTINATION ${PKGCONFIG_FILES_DIR}) # Install development headers install (FILES "gfal_api.h" "gfal_plugins_api.h" DESTINATION ${INCLUDE_INSTALL_DIR}/gfal2/) install (FILES "logger/gfal_logger.h" DESTINATION ${INCLUDE_INSTALL_DIR}/gfal2/logger) install (FILES "posix/gfal_posix_api.h" DESTINATION ${INCLUDE_INSTALL_DIR}/gfal2/posix) install (FILES "common/gfal_cancel.h" "common/gfal_common.h" "common/gfal_config.h" "common/gfal_constants.h" "common/gfal_cred_mapping.h" "common/gfal_deprecated.h" "common/gfal_error.h" "common/gfal_plugin.h" "common/gfal_file_handle.h" "common/gfal_plugin_interface.h" DESTINATION ${INCLUDE_INSTALL_DIR}/gfal2/common) install (FILES "file/gfal_file_api.h" DESTINATION ${INCLUDE_INSTALL_DIR}/gfal2/file) # Transfer library if (MAIN_TRANSFER) add_subdirectory (transfer) endif (MAIN_TRANSFER) gfal2-v2.23.0/src/core/common/000077500000000000000000000000001465240014500157675ustar00rootroot00000000000000gfal2-v2.23.0/src/core/common/gfal_cancel.c000066400000000000000000000064071465240014500203600ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include "gfal_handle.h" // // @author : Devresse Adrien // // cancel logic of gfal 2 // int gfal2_cancel(gfal2_context_t context) { if (!context) return -1; else if (context->cancel == TRUE) // avoid recursive calls return 0; g_mutex_lock(context->mux_cancel); const int n_cancel = g_atomic_int_get(&(context->running_ops)); context->cancel = TRUE; g_hook_list_invoke(&context->cancel_hooks, TRUE); g_mutex_unlock(context->mux_cancel); while ((g_atomic_int_get(&(context->running_ops))) > 0) { usleep(50); } context->cancel = FALSE; return n_cancel; } gboolean gfal2_is_canceled(gfal2_context_t context) { return context->cancel; } // Increase number of the running task for the cancel logic // Return negative value if task is canceled int gfal2_start_scope_cancel(gfal2_context_t context, GError** err) { if (context && context->cancel) { g_set_error(err, gfal_cancel_quark(), ECANCELED, "[gfal2_cancel] operation canceled by user"); return -1; } g_atomic_int_inc(&(context->running_ops)); return 0; } int gfal2_end_scope_cancel(gfal2_context_t context) { if (context) (void) g_atomic_int_dec_and_test(&(context->running_ops)); return 0; } struct gfal_hook_data_s { void* userdata; gfal2_context_t context; gfal_cancel_hook_cb cb; }; static void gfal_ghook_cancel_wrapper(gpointer data) { struct gfal_hook_data_s* d = data; d->cb(d->context, d->userdata); } gfal_cancel_token_t gfal2_register_cancel_callback(gfal2_context_t context, gfal_cancel_hook_cb cb, void* userdata) { g_assert(context && cb); g_mutex_lock(context->mux_cancel); GHook* h = g_hook_alloc(&context->cancel_hooks); struct gfal_hook_data_s* d = g_new(struct gfal_hook_data_s, 1); d->context = context; d->userdata = userdata; d->cb = cb; h->data = d; h->destroy = &g_free; h->func = &gfal_ghook_cancel_wrapper; g_hook_append(&context->cancel_hooks, h); g_mutex_unlock(context->mux_cancel); return (gfal_cancel_token_t) h; } void gfal2_remove_cancel_callback(gfal2_context_t context, gfal_cancel_token_t token) { g_assert(context && token); g_mutex_lock(context->mux_cancel); GHook* cb = (GHook*) token; g_hook_destroy_link(&context->cancel_hooks, cb); g_mutex_unlock(context->mux_cancel); } GQuark gfal_cancel_quark() { return g_quark_from_string("[gfal2_cancel]"); } gfal2-v2.23.0/src/core/common/gfal_cancel.h000066400000000000000000000056071465240014500203660ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #pragma once #ifndef GFAL_CANCEL_H_ #define GFAL_CANCEL_H_ #if !defined(__GFAL2_H_INSIDE__) && !defined(__GFAL2_BUILD__) # warning "Direct inclusion of gfal2 headers is deprecated. Please, include only gfal_api.h or gfal_plugins_api.h" #endif #include "gfal_common.h" #include "gfal_constants.h" #ifdef __cplusplus extern "C" { #endif typedef struct gfal_cancel_token_s* gfal_cancel_token_t; typedef void (*gfal_cancel_hook_cb)(gfal2_context_t context, void* userdata); /** * @brief cancel operation * * cancel all pending operation on the given context * blocking until all operations finish * all operations will return and trigger an ECANCELED if interrupted. * Thread safe * @param context : gfal 2 context * @return number of operations canceled */ int gfal2_cancel(gfal2_context_t context); /** * @brief cancel status * @return true if \ref gfal2_cancel has been called * * @param context * @return true if success */ gboolean gfal2_is_canceled(gfal2_context_t context); /** * Register a cancel hook, called in each cancellation * Thread-safe */ gfal_cancel_token_t gfal2_register_cancel_callback(gfal2_context_t context, gfal_cancel_hook_cb cb, void* userdata); /** * Remove a cancel hook * Thread-safe */ void gfal2_remove_cancel_callback(gfal2_context_t context, gfal_cancel_token_t token); /** * Mark the beginning of a cancellable scope */ int gfal2_start_scope_cancel(gfal2_context_t context, GError** err); /** * Mark the end of a cancellable scope */ int gfal2_end_scope_cancel(gfal2_context_t context); /** * GQuark of a cancel action */ GQuark gfal_cancel_quark(); /** Convenience macro for gfal2_start_scope_cancel */ #define GFAL2_BEGIN_SCOPE_CANCEL(context, ret_err_value, err) \ do{ \ if(gfal2_start_scope_cancel(context, err) < 0){ \ return ret_err_value; \ } \ }while(0) /** Convenience macro for gfal2_end_scope_cancel */ #define GFAL2_END_SCOPE_CANCEL(context) \ gfal2_end_scope_cancel(context) #ifdef __cplusplus } #endif #endif /* GFAL_CANCEL_H_ */ gfal2-v2.23.0/src/core/common/gfal_common.c000066400000000000000000000141241465240014500204160ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include #include #include #include "gfal_file_handler_container.h" // initialization __attribute__((constructor)) void core_init() { #if (!GLIB_CHECK_VERSION (2, 32, 0)) if (!g_thread_supported()) g_thread_init(NULL); #endif } GQuark gfal2_get_core_quark() { return g_quark_from_static_string(GFAL2_QUARK_CORE); } GQuark gfal2_get_config_quark() { return g_quark_from_static_string(GFAL2_QUARK_CONFIG); } GQuark gfal2_get_plugins_quark() { return g_quark_from_static_string(GFAL2_QUARK_PLUGINS); } static void gfal_setCredentialLocation(const char *where, gfal2_context_t handle, const char *cert, const char *key) { GError *error = NULL; gfal2_set_opt_string(handle, "X509", "CERT", cert, &error); g_clear_error(&error); gfal2_set_opt_string(handle, "X509", "KEY", key, &error); g_clear_error(&error); gfal2_log(G_LOG_LEVEL_DEBUG, "Using credentials from %s", where); gfal2_log(G_LOG_LEVEL_DEBUG, "Certificate: %s", cert); gfal2_log(G_LOG_LEVEL_DEBUG, "Private key: %s", key); } static void gfal_setBearerToken(gfal2_context_t handle, const char *token) { GError *error = NULL; gfal2_set_opt_string(handle, "BEARER", "TOKEN", token, &error); g_clear_error(&error); gfal2_log(G_LOG_LEVEL_DEBUG, "Using BEARER token credentials from the env"); } // Setup default credentials depending on the environment static void gfal_initCredentialLocation(gfal2_context_t handle) { //check first if BEARER is on the env const char *token = getenv("BEARER_TOKEN"); if (token != NULL) { gfal_setBearerToken(handle, token); return; } // X509_USER_PROXY const char *proxy = getenv("X509_USER_PROXY"); if (proxy != NULL) { gfal_setCredentialLocation("X509_USER_PROXY", handle, proxy, proxy); return; } // /tmp/x509up_u char default_proxy[64]; snprintf(default_proxy, sizeof(default_proxy), "/tmp/x509up_u%d", getuid()); if (access(default_proxy, F_OK) == 0) { gfal_setCredentialLocation("default proxy location", handle, default_proxy, default_proxy); return; } // X509_USER_CERT and X509_USER_KEY const char *cert, *key; cert = getenv("X509_USER_CERT"); key = getenv("X509_USER_KEY"); if (cert != NULL && key != NULL) { gfal_setCredentialLocation("X509_USER_CERT and X509_USER_KEY", handle, cert, key); return; } // Default certificate location const char *home = getenv("HOME"); if (home != NULL) { char *default_cert = g_strconcat(home, "/.globus/usercert.pem", NULL); char *default_key = g_strconcat(home, "/.globus/userkey.pem", NULL); int canAccess = (access(default_cert, F_OK) == 0 && access(default_key, F_OK)); g_free(default_cert); g_free(default_key); if (canAccess) { gfal_setCredentialLocation("default certificate location", handle, default_cert, default_key); return; } } // No idea! gfal2_log(G_LOG_LEVEL_DEBUG, "Could not find the credentials in any of the known locations"); } gfal2_context_t gfal2_context_new(GError **err) { GError *tmp_err = NULL; gfal2_context_t context = g_new0(struct gfal_handle_, 1); if (context == NULL) { g_set_error(err, gfal2_get_plugins_quark(), errno, "[%s] bad allocation, no more memory free", __func__); return NULL; } context->initiated = TRUE; context->config = gfal2_init_config(&tmp_err); if (!context->config) { gfal2_propagate_prefixed_error(err, tmp_err, __func__); g_free(context); return NULL; } gfal_initCredentialLocation(context); context->plugin_opt.plugin_number = 0; int ret = gfal_plugins_instance(context, &tmp_err); if (ret <= 0 && tmp_err) { gfal2_propagate_prefixed_error(err, tmp_err, __func__); g_key_file_free(context->config); g_free(context); return NULL; } context->client_info = g_ptr_array_new(); context->mux_cancel = g_mutex_new(); g_hook_list_init(&context->cancel_hooks, sizeof(GHook)); context->fdescs = gfal_file_descriptor_handle_create(NULL); G_RETURN_ERR(context, tmp_err, err); } void gfal2_context_free(gfal2_context_t context) { if (context == NULL) { errno = EFAULT; return; } gfal_plugins_delete(context, NULL); gfal_file_descriptor_handle_destroy(context->fdescs); g_key_file_free(context->config); g_list_free(context->plugin_opt.sorted_plugin); g_mutex_free(context->mux_cancel); g_hook_list_clear(&context->cancel_hooks); g_free(context->agent_name); g_free(context->agent_version); g_ptr_array_foreach(context->client_info, gfal_free_keyvalue, NULL); g_ptr_array_free(context->client_info, FALSE); gfal2_cred_clean(context, NULL); g_free(context); } gchar **gfal2_get_plugin_names(gfal2_context_t context) { gchar **array = g_new0(gchar*, context->plugin_opt.plugin_number + 1); int i; for (i = 0; i < context->plugin_opt.plugin_number; ++i) { array[i] = g_strdup(context->plugin_opt.plugin_list[i].getName()); } array[i] = NULL; return array; } // return a string of the current gfal version const char *gfal2_version() { return VERSION; } gfal2-v2.23.0/src/core/common/gfal_common.h000066400000000000000000000056321465240014500204270ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #pragma once #ifndef GFAL_GLOBAL_H_ #define GFAL_GLOBAL_H_ #if !defined(__GFAL2_H_INSIDE__) && !defined(__GFAL2_BUILD__) # warning "Direct inclusion of gfal2 headers is deprecated. Please, include only gfal_api.h or gfal_plugins_api.h" #endif #include #include "gfal_deprecated.h" #ifdef __cplusplus extern "C" { #endif /** * A gfal2 context is a separated instance of the gfal2 library * Each context owns his parameters, file descriptors * Context allows to have separated instance of GFAL with different parameters * providing an advanced interface to GFAL */ typedef struct gfal_handle_* gfal2_context_t; /** * @brief Create a gfal2 context * * Each context contain its own set of parameters and configurations ( \ref config_group ) * A context can be used in multiple threads at the same time ( Thread-safe ). * * @param err : GError error report system * @return a context if success, NULL if error */ gfal2_context_t gfal2_context_new(GError ** err); /** * Free a gfal2 context * It is safe to delete a NULL context */ void gfal2_context_free(gfal2_context_t context); /** * Get list of loaded plugins * The returned list must be freed using g_strfreev */ gchar** gfal2_get_plugin_names(gfal2_context_t context); /** For errors, gfal2 core quark */ #define GFAL2_QUARK_CORE "GFAL2::CORE" /** For errors, gfal2 configuration quark */ #define GFAL2_QUARK_CONFIG "GFAL2::CONFIG" /** For errors, gfal2 plugins quark */ #define GFAL2_QUARK_PLUGINS "GFAL2::PLUGINS" /** * GQuark for the gfal2 core namespace * GQuark are used by the GError gfal2 error system in order to determine the scope of one error * GQuark String : "GFAL2::CORE" */ GQuark gfal2_get_core_quark(); /** * GQuark for the gfal2 config namespace * GQuark are used by the GError gfal2 error system in order to determine the scope of one error * GQuark String : "GFAL2::CONFIG" */ GQuark gfal2_get_config_quark(); /** * GQuark for the gfal2 plugin namespace * GQuark String : "GFAL2::PLUGINS" * Any plugin specific GQuark follows this pattern GFAL2::PLUGINS::NAME * Example srm plugin : GFAL2::PLUGINS::SRM */ GQuark gfal2_get_plugins_quark(); #ifdef __cplusplus } #endif #endif /* GFAL_GLOBAL_H_ */ gfal2-v2.23.0/src/core/common/gfal_config.c000066400000000000000000000332741465240014500204020ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "gfal_handle.h" #include #include #ifndef GFAL_CONFIG_DIR_DEFAULT #error "GFAL_CONFIG_DIR_DEFAULT should be define at compile time" #endif const gchar *config_env_var = GFAL_CONFIG_DIR_ENV; const gchar *default_config_dir = GFAL_CONFIG_DIR_DEFAULT "/" GFAL_CONFIG_DIR_SUFFIX "/"; typedef struct _gfal_key_value { char *key, *value; } *gfal_key_value_t; void gfal_free_keyvalue(gpointer data, gpointer user_data) { gfal_key_value_t keyval = (gfal_key_value_t) data; g_free(keyval->key); g_free(keyval->value); g_free(keyval); } static gchar *check_configuration_dir(GError **err) { struct stat st; int res; gchar *dir_config = NULL; const gchar *env_str = g_getenv(config_env_var); if (env_str != NULL) { gfal2_log(G_LOG_LEVEL_DEBUG, " %s env var found, try to load configuration from %s", config_env_var, env_str); dir_config = g_strdup(env_str); } else { gfal2_log(G_LOG_LEVEL_DEBUG, " no %s env var found, try to load configuration from default directory %s", config_env_var, default_config_dir); dir_config = g_strdup(default_config_dir); } res = stat(dir_config, &st); if (res != 0 || S_ISDIR(st.st_mode) == FALSE) { g_set_error(err, gfal2_get_config_quark(), EINVAL, " %s is not a valid directory for " "gfal2 configuration files, please specify %s properly", dir_config, config_env_var); g_free(dir_config); dir_config = NULL; } return dir_config; } int gfal_load_configuration_to_conf_manager(GKeyFile *dest, const gchar *path, GError **err) { GError *tmp_err = NULL; GKeyFile *new_conf = g_key_file_new(); int groupIndex, keyIndex; if (g_key_file_load_from_file(new_conf, path, G_KEY_FILE_NONE, &tmp_err) == FALSE) { gfal2_propagate_prefixed_error_extended(err, tmp_err, __func__, "Error while loading configuration file %s: ", path); return -1; } gsize nGroups = 0; gchar **groups = g_key_file_get_groups(new_conf, &nGroups); for (groupIndex = 0; groupIndex < nGroups; ++groupIndex) { gsize nKeys = 0; GError *tmp_err = NULL; gchar **keys = g_key_file_get_keys(new_conf, groups[groupIndex], &nKeys, &tmp_err); if (keys == NULL) { g_clear_error(&tmp_err); continue; } for (keyIndex = 0; keyIndex < nKeys; ++keyIndex) { gchar *value = g_key_file_get_value(new_conf, groups[groupIndex], keys[keyIndex], &tmp_err); if (value == NULL) { g_clear_error(&tmp_err); continue; } g_key_file_set_value(dest, groups[groupIndex], keys[keyIndex], value); g_free(value); } g_strfreev(keys); } g_strfreev(groups); g_key_file_free(new_conf); return 0; } gboolean is_config_dir(const char *conffile) { char *p = NULL; if ((p = strstr(conffile, ".conf")) != NULL) { if (*(p + 5) == '\0') { return TRUE; } } return FALSE; } GKeyFile* gfal2_init_config(GError **err) { GError *tmp_err = NULL; gchar *dir_config = NULL; GKeyFile *res = g_key_file_new(); if ((dir_config = check_configuration_dir(&tmp_err)) != NULL) { DIR *d = opendir(dir_config); struct dirent *dirinfo; if (d != NULL) { while ((dirinfo = readdir(d)) != NULL) { if (is_config_dir(dirinfo->d_name)) { char *config_file = g_strdup_printf("%s/%s", dir_config, dirinfo->d_name); gfal2_log(G_LOG_LEVEL_DEBUG, " try to load configuration file %s ...", config_file); int rc = gfal_load_configuration_to_conf_manager(res, config_file, &tmp_err); g_free(config_file); if (rc != 0) { break; } } } closedir(d); } else { g_set_error(&tmp_err, gfal2_get_config_quark(), ENOENT, "Unable to open configuration directory %s", dir_config); } g_free(dir_config); } if (tmp_err) { g_key_file_free(res); res = NULL; } G_RETURN_ERR(res, tmp_err, err); } gchar *gfal2_get_opt_string(gfal2_context_t context, const gchar *group_name, const gchar *key, GError **error) { g_assert(context != NULL); return g_key_file_get_string(context->config, group_name, key, error); } gchar *gfal2_get_opt_string_with_default(gfal2_context_t handle, const gchar *group_name, const gchar *key, const gchar *default_value) { g_assert(handle != NULL); GError *tmp_err = NULL; gchar *value = gfal2_get_opt_string(handle, group_name, key, &tmp_err); if (tmp_err) { gfal2_log(G_LOG_LEVEL_DEBUG, "Impossible to get string parameter %s:%s, set to default value %s, err %s", group_name, key, default_value, tmp_err->message); g_clear_error(&tmp_err); value = g_strdup(default_value); } return value; } gint gfal2_set_opt_string(gfal2_context_t context, const gchar *group_name, const gchar *key, const gchar *value, GError **error) { g_assert(context != NULL); g_key_file_set_string(context->config, group_name, key, value); return 0; } gint gfal2_get_opt_integer(gfal2_context_t context, const gchar *group_name, const gchar *key, GError **error) { g_assert(context != NULL); return g_key_file_get_integer(context->config, group_name, key, error); } gint gfal2_get_opt_integer_with_default(gfal2_context_t context, const gchar *group_name, const gchar *key, gint default_value) { GError *tmp_err = NULL; gint res = gfal2_get_opt_integer(context, group_name, key, &tmp_err); if (tmp_err) { gfal2_log(G_LOG_LEVEL_DEBUG, "Impossible to get integer parameter %s:%s, set to default value %d, err %s", group_name, key, default_value, tmp_err->message); g_clear_error(&tmp_err); res = default_value; } return res; } gint gfal2_set_opt_integer(gfal2_context_t context, const gchar *group_name, const gchar *key, gint value, GError **error) { g_assert(context != NULL); g_key_file_set_integer(context->config, group_name, key, value); return 0; } gboolean gfal2_get_opt_boolean(gfal2_context_t context, const gchar *group_name, const gchar *key, GError **error) { g_assert(context != NULL); return g_key_file_get_boolean(context->config, group_name, key, error); } gboolean gfal2_get_opt_boolean_with_default(gfal2_context_t context, const gchar *group_name, const gchar *key, gboolean default_value) { GError *tmp_err = NULL; gboolean res = gfal2_get_opt_boolean(context, group_name, key, &tmp_err); if (tmp_err) { gfal2_log(G_LOG_LEVEL_DEBUG, "Impossible to get boolean parameter %s:%s, set to default value %s, err %s", group_name, key, ((default_value) ? "TRUE" : "FALSE"), tmp_err->message); g_clear_error(&tmp_err); res = default_value; } return res; } gint gfal2_set_opt_boolean(gfal2_context_t context, const gchar *group_name, const gchar *key, gboolean value, GError **error) { g_assert(context != NULL); g_key_file_set_boolean(context->config, group_name, key, value); return 0; } gchar **gfal2_get_opt_string_list(gfal2_context_t context, const gchar *group_name, const gchar *key, gsize *length, GError **error) { g_assert(context != NULL); return g_key_file_get_string_list(context->config, group_name, key, length, error); } gint gfal2_set_opt_string_list(gfal2_context_t context, const gchar *group_name, const gchar *key, const gchar *const list[], gsize length, GError **error) { g_assert(context != NULL); g_key_file_set_string_list(context->config, group_name, key, list, length); return 0; } gchar **gfal2_get_opt_string_list_with_default(gfal2_context_t context, const gchar *group_name, const gchar *key, gsize *length, char **default_value) { GError *tmp_err = NULL; gchar **res = gfal2_get_opt_string_list(context, group_name, key, length, &tmp_err); if (tmp_err) { if (gfal2_log_get_level() >= G_LOG_LEVEL_DEBUG) { gchar *list_default = default_value ? g_strjoinv(",", default_value) : NULL; gfal2_log(G_LOG_LEVEL_DEBUG, "Impossible to get string_list parameter %s:%s, set to a default value %s, err %s", group_name, key, list_default, tmp_err->message); g_free(list_default); } g_clear_error(&tmp_err); res = g_strdupv(default_value); } return res; } gint gfal2_load_opts_from_file(gfal2_context_t context, const char *path, GError **error) { return gfal_load_configuration_to_conf_manager(context->config, path, error); } gchar **gfal2_get_opt_keys(gfal2_context_t context, const gchar *group_name, gsize *length, GError **error) { return g_key_file_get_keys(context->config, group_name, length, error); } gboolean gfal2_remove_opt(gfal2_context_t context, const gchar *group_name, const gchar *key, GError **error) { return g_key_file_remove_key(context->config, group_name, key, error); } gint gfal2_set_user_agent(gfal2_context_t handle, const char *user_agent, const char *version, GError **error) { g_free(handle->agent_name); handle->agent_name = g_strdup(user_agent); g_free(handle->agent_version); handle->agent_version = g_strdup(version); return 0; } gint gfal2_get_user_agent(gfal2_context_t handle, const char **user_agent, const char **version) { *user_agent = handle->agent_name; *version = handle->agent_version; return 0; } gint gfal2_add_client_info(gfal2_context_t handle, const char *key, const char *value, GError **error) { gfal2_remove_client_info(handle, key, error); g_clear_error(error); gfal_key_value_t keyval = g_new0(struct _gfal_key_value, 1); keyval->key = g_strdup(key); keyval->value = g_strdup(value); g_ptr_array_add(handle->client_info, keyval); return 0; } gint gfal2_remove_client_info(gfal2_context_t handle, const char *key, GError **error) { const char *value; int i = gfal2_get_client_info_value(handle, key, &value, error); if (i < 0) { return i; } gfal_key_value_t keyval = (gfal_key_value_t) g_ptr_array_index(handle->client_info, i); gfal_free_keyvalue(keyval, NULL); g_ptr_array_remove_index_fast(handle->client_info, i); return 0; } gint gfal2_clear_client_info(gfal2_context_t handle, GError **error) { g_ptr_array_foreach(handle->client_info, gfal_free_keyvalue, NULL); g_ptr_array_free(handle->client_info, FALSE); handle->client_info = g_ptr_array_new(); return 0; } gint gfal2_get_client_info_count(gfal2_context_t handle, GError **error) { return handle->client_info->len; } gint gfal2_get_client_info_pair(gfal2_context_t handle, int index, const char **key, const char **value, GError **error) { gfal_key_value_t keyval = (gfal_key_value_t) g_ptr_array_index(handle->client_info, index); if (keyval) { *key = keyval->key; *value = keyval->value; } else { *key = NULL; *value = NULL; } return 0; } gint gfal2_get_client_info_value(gfal2_context_t handle, const char *key, const char **value, GError **error) { size_t i = 0; for (i = 0; i < handle->client_info->len; ++i) { gfal_key_value_t keyval = (gfal_key_value_t) g_ptr_array_index(handle->client_info, i); if (strcmp(keyval->key, key) == 0) { *value = keyval->value; return i; } } g_set_error(error, gfal2_get_config_quark(), EINVAL, "Key %s not found", key); return -1; } static char *gfal2_urlencode(const char *original) { size_t len = strlen(original); char *encoded = g_malloc0(len * 3 + 1); // Worst case const char *ip; // input pointer char *op; // output pointer for (ip = original, op = encoded; *ip; ++ip) { if (g_ascii_isalnum(*ip) || *ip == '.' || *ip == '-' || *ip == '_') { *op = *ip; ++op; } else { g_snprintf(op, 4, "%%%02X", (int) *ip); op += 3; } } return encoded; } char *gfal2_get_client_info_string(gfal2_context_t handle) { size_t i, nitems = handle->client_info->len; if (nitems == 0) { return NULL; } char **entries = g_new0(char*, nitems + 1); for (i = 0; i < nitems; ++i) { gfal_key_value_t keyval = (gfal_key_value_t) g_ptr_array_index(handle->client_info, i); char *encoded_key = gfal2_urlencode(keyval->key); char *encoded_value = gfal2_urlencode(keyval->value); entries[i] = g_strdup_printf("%s=%s", encoded_key, encoded_value); g_free(encoded_key); g_free(encoded_value); } char *joined = g_strjoinv(";", entries); g_strfreev(entries); return joined; } gfal2-v2.23.0/src/core/common/gfal_config.h000066400000000000000000000265221465240014500204050ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #pragma once #ifndef GFAL_CONFIG_H_ #define GFAL_CONFIG_H_ #if !defined(__GFAL2_H_INSIDE__) && !defined(__GFAL2_BUILD__) # warning "Direct inclusion of gfal2 headers is deprecated. Please, include only gfal_api.h or gfal_plugins_api.h" #endif #include "gfal_common.h" #ifdef __cplusplus extern "C" { #endif #define CORE_CONFIG_GROUP "CORE" #define CORE_CONFIG_CHECKSUM_TIMEOUT "CHECKSUM_TIMEOUT" #define CORE_CONFIG_NAMESPACE_TIMEOUT "NAMESPACE_TIMEOUT" /** * @file gfal_config.h * @brief gfal2 configuration API * set/get option for the running configuration of GFAL 2.0 * @author Adrien Devresse */ /*! \defgroup config_group Parameter API Allows to overwrite or/and define any parameter of gfal2. A complete list of parameter is accessible in the gfal2 configuration files directory ( by default : /etc/gfal2.d/ ) Example ( enable IPv6 support for the gridFTP plugin ) : gfal2_set_opt_boolean("GRIDFTP PLUGIN", "IPV6", true, NULL); */ /*! \addtogroup config_group @{ */ /** * @brief get a string parameter in the current GFAL 2.0 configuration * see gfal2.d configuration files or gfal2 documentation to know group/key/values * * @param context : context of gfal2 * @param group_name : group name of the parameter * @param key : key of the parameter * @param error : GError error report system * @return parameter value. Must be freed using g_free **/ gchar * gfal2_get_opt_string(gfal2_context_t context, const gchar *group_name, const gchar *key, GError **error); /** * @brief similar to \ref gfal2_get_opt_string but return a default value if * an error occurs * * @param handle : context of gfal2 * @param group_name : group name of the parameter * @param key : key of the parameter * @param default_value : Default value * @return parameter value. Must be freed using g_free **/ gchar * gfal2_get_opt_string_with_default(gfal2_context_t handle, const gchar *group_name, const gchar *key, const gchar* default_value); /** * @brief set a string parameter in the current GFAL 2.0 configuration * see gfal2.d configuration files or gfal2 documentation to know group/key/values * * @param context : context of gfal2 * @param group_name : group name of the parameter * @param key : key of the parameter * @param value : value to set * @param error : GError error report system * @return parameter value **/ gint gfal2_set_opt_string(gfal2_context_t context, const gchar *group_name, const gchar *key, const gchar* value, GError **error); /** * @brief get an integer parameter in the current GFAL 2.0 configuration * see gfal2.d configuration files or gfal2 documentation to know group/key/values * * @param context : context of gfal2 * @param group_name : group name of the parameter * @param key : key of the parameter * @param error : GError error report system * @return parameter value **/ gint gfal2_get_opt_integer(gfal2_context_t context, const gchar *group_name, const gchar *key, GError **error); /** * @brief similar to \ref gfal2_get_opt_integer but return a default value if * an error occurs * * @param handle : context of gfal2 * @param group_name : group name of the parameter * @param key : key of the parameter * @param default_value : default value returned if not present * @return parameter value **/ gint gfal2_get_opt_integer_with_default(gfal2_context_t handle, const gchar *group_name, const gchar *key, gint default_value); /** * @brief set an integer parameter in the current GFAL 2.0 configuration * see gfal2.d configuration files or gfal2 documentation to know group/key/values * * @param context : context of gfal2 * @param group_name : group name of the parameter * @param key : key of the parameter * @param value : value to set * @param error : GError error report system * @return parameter value **/ gint gfal2_set_opt_integer(gfal2_context_t context, const gchar *group_name, const gchar *key, gint value, GError** error); /** * @brief set a boolean parameter in the current GFAL 2.0 configuration * see gfal2.d configuration files or gfal2 documentation to know group/key/values * * @param context : context of gfal2 * @param group_name : group name of the parameter * @param key : key of the parameter * @param value : value to set * @param error : GError error report system * @return parameter value **/ gint gfal2_set_opt_boolean(gfal2_context_t context, const gchar *group_name, const gchar *key, gboolean value, GError **error); /** * @brief get a boolean parameter in the current GFAL 2.0 configuration * see gfal2.d configuration files or gfal2 documentation to know group/key/values * * @param context : context of gfal2 * @param group_name : group name of the parameter * @param key : key of the parameter * @param error : GError error report system * @return parameter value **/ gboolean gfal2_get_opt_boolean(gfal2_context_t context, const gchar *group_name, const gchar *key, GError **error); /** * @brief similar to \ref gfal2_get_opt_boolean but return a default value if * an error occures * * @param context : context of gfal2 * @param group_name : group name of the parameter * @param key : key of the parameter * @param default_value : default value returned if not present * @return parameter value **/ gboolean gfal2_get_opt_boolean_with_default(gfal2_context_t context, const gchar *group_name, const gchar *key, gboolean default_value); /** * @brief set a list of string parameter in the current GFAL 2.0 configuration * see gfal2.d configuration files or gfal2 documentation to know group/key/values * * @param context : context of gfal2 * @param group_name : group name of the parameter * @param key : key of the parameter * @param list : list of strings * @param length : length of the list * @param error : GError error report system * @return parameter value **/ gint gfal2_set_opt_string_list(gfal2_context_t context, const gchar *group_name, const gchar *key, const gchar * const list[], gsize length, GError ** error); /** * @brief get a list of string parameter in the current GFAL 2.0 configuration * see gfal2.d configuration files or gfal2 documentation to know group/key/values * * @param context : context of gfal2 * @param group_name : group name of the parameter * @param key : key of the parameter * @param length : the length of the string is stored here * @param error : GError error report system * @return parameter value **/ gchar ** gfal2_get_opt_string_list(gfal2_context_t context, const gchar *group_name, const gchar *key, gsize *length, GError **error); /** * @brief get a list of string parameter in the current GFAL 2.0 configuration * see gfal2.d configuration files or gfal2 documentation to know group/key/values * * @param context : context of gfal2 * @param group_name : group name of the parameter * @param key : key of the parameter * @param length : the length of the string is stored here * @param default_value : Default array of not found * @return parameter value **/ gchar ** gfal2_get_opt_string_list_with_default(gfal2_context_t context, const gchar *group_name, const gchar *key, gsize *length, char** default_value); /** * @brief load configuration parameters from the file specified by path */ gint gfal2_load_opts_from_file(gfal2_context_t context, const char* path, GError** error); /** * Get all keys defined for the given group_name * @param context : context of gfal2 * @param group_name : group name of the parameters * @param length : the number of keys is stored here * @param error : GError error report system * @return NULL on error. A NULL-terminated array with the list of keys. Use g_strfreev() to free it. */ gchar **gfal2_get_opt_keys(gfal2_context_t context, const gchar *group_name, gsize *length, GError **error); /** * Removes a key from the settings * @param context : context of gfal2 * @param group_name : group name of the parameters * @param key : key of the parameter * @param error : GError error report system * @return TRUE if the key was removed, FALSE otherwise */ gboolean gfal2_remove_opt(gfal2_context_t context, const gchar *group_name, const gchar *key, GError **error); /** * Set the user agent for those protocols that support this */ gint gfal2_set_user_agent(gfal2_context_t handle, const char* user_agent, const char* version, GError** error); /** * Returns the user agent and version specified before with gfal2_set_user_agent * Leave user_agent and version to NULL if not found */ gint gfal2_get_user_agent(gfal2_context_t handle, const char** user_agent, const char** version); /** * Add a new key/value pair with additional information to be passed to the storage * for protocols that support it. * For instance, this will be passed via CLIENTINFO for GridFTP, or the ClientInfo header for SRM and HTTP * Return < 0 on error */ gint gfal2_add_client_info(gfal2_context_t handle, const char* key, const char* value, GError** error); /** * Removes a key/value pair set previously with gfal2_add_client_info * Return < 0 on error */ gint gfal2_remove_client_info(gfal2_context_t handle, const char* key, GError** error); /** * Clear the client information * Return < 0 on error */ gint gfal2_clear_client_info(gfal2_context_t handle, GError** error); /** * Return how many custom pairs have been set * Return < 0 on error */ gint gfal2_get_client_info_count(gfal2_context_t handle, GError** error); /** * Put into key and value the pair at position index, or NULL if it does not exist * Return < 0 on error */ gint gfal2_get_client_info_pair(gfal2_context_t handle, int index, const char** key, const char** value, GError** error); /** * Put into value the value associated with the given key * Return < 0 on error */ gint gfal2_get_client_info_value(gfal2_context_t handle, const char* key, const char** value, GError** error); /** * For convenience, return all the key/value information in the form * key1=value1;key2=value2 * The return value is NULL if there is no information. Otherwise, use g_free on it when done. */ char* gfal2_get_client_info_string(gfal2_context_t handle); /** @} End of the FILE group */ #ifdef __cplusplus } #endif #endif /* GFAL_CONFIG_H_ */ gfal2-v2.23.0/src/core/common/gfal_config_internal.h000066400000000000000000000020231465240014500222670ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #pragma once #ifndef GFAL_CONFIG_INTERNAL_H_ #define GFAL_CONFIG_INTERNAL_H_ #include // create or delete configuration manager for gfal2, internal GKeyFile* gfal2_init_config(GError **err); void gfal_free_keyvalue(gpointer data, gpointer user_data); #endif /* GFAL_CONFIG_INTERNAL_H_ */ gfal2-v2.23.0/src/core/common/gfal_constants.h000066400000000000000000000064041465240014500211510ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #pragma once #ifndef GFAL_CONSTANTS_H_ #define GFAL_CONSTANTS_H_ #if !defined(__GFAL2_H_INSIDE__) && !defined(__GFAL2_BUILD__) # warning "Direct inclusion of gfal2 headers is deprecated. Please, include only gfal_api.h or gfal_plugins_api.h" #endif #ifdef __cplusplus extern "C" { #endif /** Maximum number of plugins */ #define MAX_PLUGIN_LIST 15 /** Module name size */ #define GFAL_MODULE_NAME_SIZE 1024 /** GFAL error level for gfal_errmsg */ #define GFAL_ERRMSG_LEN 2048 /** default buffer size for address */ #define GFAL_URL_MAX_LEN 2048 /** Adler32 formatted checksum byte length */ #define GFAL_ADLER_CHKSUM_LEN 8 /** environment variable for personalized plugin directory */ #define GFAL_PLUGIN_DIR_ENV "GFAL_PLUGIN_DIR" /** default directory name for gfal 2 plugin search */ #define GFAL_PLUGIN_DIR_SUFFIX "gfal2-plugins" /** plugin entry point */ #define GFAL_PLUGIN_INIT_SYM "gfal_plugin_init" /** environment variable for personalized configuration directory */ #define GFAL_CONFIG_DIR_ENV "GFAL_CONFIG_DIR" /** folder name under /etc for the configuration files */ #define GFAL_CONFIG_DIR_SUFFIX "gfal2.d" /* xattr standard keys for getxattr / setxattr */ /** replicas listing */ #define GFAL_XATTR_REPLICA "user.replicas" /** guid information */ #define GFAL_XATTR_GUID "user.guid" /** file comment */ #define GFAL_XATTR_COMMENT "user.comment" /** file checksum type */ #define GFAL_XATTR_CHKSUM_TYPE "user.chksumtype" /** file checksum */ #define GFAL_XATTR_CHKSUM_VALUE "user.checksum" /** * File availability status * This key can be used to check or set the stage status of a file */ #define GFAL_XATTR_STATUS "user.status" /** String value to use/compare for extended attribute user.status * user.status possible value, similar to SRM meaning of status ( brings_online ) */ #define GFAL_XATTR_STATUS_ONLINE "ONLINE" #define GFAL_XATTR_STATUS_NEARLINE "NEARLINE" #define GFAL_XATTR_STATUS_NEARLINE_ONLINE "ONLINE_AND_NEARLINE" #define GFAL_XATTR_STATUS_UNKNOWN "UNKNOWN" #define GFAL_XATTR_STATUS_LOST "LOST" #define GFAL_XATTR_STATUS_UNAVAILABLE "UNAVAILABLE" #define GFAL_XATTR_STATUS_NONE "NONE" /** space reporting */ #define GFAL_XATTR_SPACETOKEN "spacetoken" /** tape rest api attributes */ #define GFAL_XATTR_TAPE_API_SITENAME "taperestapi.sitename" #define GFAL_XATTR_TAPE_API_URI "taperestapi.uri" #define GFAL_XATTR_TAPE_API_VERSION "taperestapi.version" /** SciTag legal values */ #define GFAL_SCITAG_MIN_VALUE ((1<<6) + 1) #define GFAL_SCITAG_MAX_VALUE ((1<<16) - 1) #ifdef __cplusplus } #endif #endif /* GFAL_CONSTANTS_H_ */ gfal2-v2.23.0/src/core/common/gfal_cred_mapping.c000066400000000000000000000130361465240014500215570ustar00rootroot00000000000000/* * Copyright (c) CERN 2017 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include "gfal_handle.h" typedef struct { size_t prefix_len; char *url_prefix; gfal2_cred_t *cred; } gfal2_cred_node_t ; static gint node_compare(gconstpointer a, gconstpointer b) { const gfal2_cred_node_t *node_a, *node_b; node_a = a; node_b = b; int comp = -strcmp(node_a->url_prefix, node_b->url_prefix); if (comp == 0) { return strcmp(node_a->cred->type, node_b->cred->type); } return comp; } static void node_copy(gpointer src, gpointer user_data) { gfal2_context_t dest = user_data; const gfal2_cred_node_t *node = src; gfal2_cred_set(dest, node->url_prefix, node->cred, NULL); } static void node_free(gpointer ptr) { gfal2_cred_node_t *node = ptr; g_free(node->url_prefix); gfal2_cred_free(node->cred); g_free(node); } gfal2_cred_t *gfal2_cred_new(const char* type, const char *value) { gfal2_cred_t *cred = g_malloc0(sizeof(gfal2_cred_t)); cred->type = g_strdup(type); cred->value = g_strdup(value); return cred; } void gfal2_cred_free(gfal2_cred_t *cred) { if (cred) { g_free(cred->type); g_free(cred->value); g_free(cred); } } gfal2_cred_t* gfal2_cred_dup(const gfal2_cred_t *cred) { if (cred == NULL) { return NULL; } return gfal2_cred_new(cred->type, cred->value); } int gfal2_cred_set(gfal2_context_t handle, const char *url_prefix, const gfal2_cred_t *cred, GError **error) { gfal2_cred_node_t *node = g_malloc0(sizeof(gfal2_cred_node_t)); node->prefix_len = strlen(url_prefix); node->url_prefix = g_strdup(url_prefix); node->cred = gfal2_cred_dup(cred); // Remove existing value GList *item = g_list_find_custom(handle->cred_mapping, node, node_compare); if (item) { gfal2_cred_node_t *match = item->data; node_free(match); handle->cred_mapping = g_list_delete_link(handle->cred_mapping, item); } // If cred is NULL, done if (cred == NULL) { node_free(node); return 0; } handle->cred_mapping = g_list_insert_sorted(handle->cred_mapping, node, node_compare); return 0; } char *gfal2_cred_get(gfal2_context_t handle, const char *type, const char *url, char const** baseurl, GError **error) { // Pick the first, which is the longest match, since the list is sorted GList *item; for (item = g_list_first(handle->cred_mapping); item != NULL; item = g_list_next(item)) { gfal2_cred_node_t *node = item->data; if (strcmp(node->cred->type, type) == 0 && strncmp(node->url_prefix, url, node->prefix_len) == 0) { // Prefix must match a directory in the target URL if (node->prefix_len < strlen(url) && (url[node->prefix_len - 1] != '/' && url[node->prefix_len] != '/')) { continue; } if (baseurl) { *baseurl = (char const*)(node->url_prefix); } return g_strdup(node->cred->value); } } if (baseurl) { *baseurl = ""; } // If there is no match, use the config if (strcmp(type, GFAL_CRED_X509_CERT) == 0) { return gfal2_get_opt_string_with_default(handle, "X509", "CERT", NULL); } else if (strcmp(type, GFAL_CRED_X509_KEY) == 0) { return gfal2_get_opt_string_with_default(handle, "X509", "KEY", NULL); } else if (strcmp(type, GFAL_CRED_BEARER) == 0) { return gfal2_get_opt_string_with_default(handle, "BEARER", "TOKEN", NULL); } return NULL; } int gfal2_cred_del(gfal2_context_t handle, const char *type, const char *url, GError **error) { GList *item; for (item = g_list_first(handle->cred_mapping); item != NULL; item = g_list_next(item)) { gfal2_cred_node_t *node = item->data; if ((strcmp(node->cred->type, type) == 0) && (node->prefix_len == strlen(url)) && (strcmp(node->url_prefix, url) == 0)) { node_free(node); handle->cred_mapping = g_list_delete_link(handle->cred_mapping, item); return 0; } } return -1; } int gfal2_cred_clean(gfal2_context_t handle, GError **error) { g_list_free_full(handle->cred_mapping, node_free); handle->cred_mapping = NULL; return 0; } int gfal2_cred_copy(gfal2_context_t dest, const gfal2_context_t src, GError **error) { if (gfal2_cred_clean(dest, error) != 0) { return -1; } g_list_foreach(src->cred_mapping, node_copy, dest); return 0; } typedef struct { gfal_cred_func_t callback; void *user_data; } callback_data; static void foreach_callback_wrapper(gpointer item, gpointer user_data) { callback_data *data = user_data; gfal2_cred_node_t *node = item; data->callback(node->url_prefix, node->cred, data->user_data); } void gfal2_cred_foreach(gfal2_context_t handle, gfal_cred_func_t callback, void *user_data) { callback_data data = {callback, user_data}; g_list_foreach(handle->cred_mapping, foreach_callback_wrapper, &data); } gfal2-v2.23.0/src/core/common/gfal_cred_mapping.h000066400000000000000000000105451465240014500215660ustar00rootroot00000000000000/* * Copyright (c) CERN 2017 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #ifndef GFAL2_GFAL_CRED_MAPPING_H #define GFAL2_GFAL_CRED_MAPPING_H #include "gfal_common.h" #ifdef __cplusplus extern "C" { #endif /* Predefined credential types * Some plugins may define their own */ /// No credential set #define GFAL_CRED_NONE NULL /// X509 user certificate, or proxy certificate #define GFAL_CRED_X509_CERT "X509_CERT" /// X509 user private key #define GFAL_CRED_X509_KEY "X509_KEY" /// User, usually combined with GFAL_CRED_PASSWD #define GFAL_CRED_USER "USER" /// Password, usually combined with GFAL_CRED_USER #define GFAL_CRED_PASSWD "PASSWORD" /// Bearer token-type credential #define GFAL_CRED_BEARER "BEARER" /** * Stores a credential value together with its type * The specific value depends on the type. * @note Use gfal2_cred_new to create a new instance */ typedef struct { char *type; char *value; } gfal2_cred_t; /** * Callback type for gfal2_cred_foreach */ typedef void (*gfal_cred_func_t)(const char *url_prefix, const gfal2_cred_t *cred, void *user_data); /** * Create a new gfal2_cred_t * @return An initialized gfal2_cred_t */ gfal2_cred_t *gfal2_cred_new(const char* type, const char *value); /** * Release a gfal2_cred_t instance */ void gfal2_cred_free(gfal2_cred_t *cred); /** * Duplicate a gfal2_cred_t instance */ gfal2_cred_t *gfal2_cred_dup(const gfal2_cred_t *cred); /** * Set a credential for a given url prefix * @param handle The gfal2 context * @param url_prefix The URL prefix * @param cred The credential to use for this prefix * @param error In case of error * @return 0 on success, -1 on error * @note The empty prefix is initialized by default with the environment X509_USER_* variables * or the [X509] configuration * @note It will store its own copy of url_prefix and cred */ int gfal2_cred_set(gfal2_context_t handle, const char *url_prefix, const gfal2_cred_t *cred, GError **error); /** * Get a credential for a given url * @param handle The gfal2 context * @param type Credential type * @param url Full URL. Best matching prefix will be picked. * @param baseurl If not NULL, the chosen base url will be put here. * @param error In case of error * @return A credential suitable for the given url. NULL if nothing has been found. Remember to g_free it. */ char *gfal2_cred_get(gfal2_context_t handle, const char *type, const char *url, char const** baseurl, GError **error); /** * Remove the credential for a given type and url * @param handle The gfal2 context * @param type Credential type * @param url Full URL. Only exact matching URL will be deleted * @param error In case of error * @return 0 on success, -1 on error */ int gfal2_cred_del(gfal2_context_t handle, const char *type, const char *url, GError **error); /** * Remove all loaded credentials * @param handle The gfal2 context * @param error In case of error * @return 0 on success, -1 on error */ int gfal2_cred_clean(gfal2_context_t handle, GError **error); /** * Copy the credential list from one context to another * @param dest Destination gfal2 context * @param src Source gfal2 context * @param error In case of error * @return 0 on success, -1 on error */ int gfal2_cred_copy(gfal2_context_t dest, const gfal2_context_t src, GError **error); /** * Iterate over all registered credentials * @param handle The gfal2 context * @param callback Callback for each item * @param user_data To be passed to the callback */ void gfal2_cred_foreach(gfal2_context_t handle, gfal_cred_func_t callback, void *user_data); #ifdef __cplusplus } #endif #endif //GFAL2_GFAL_CRED_MAPPING_H gfal2-v2.23.0/src/core/common/gfal_deprecated.h000066400000000000000000000034621465240014500212360ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #pragma once #ifndef GFAL_DEPRECATED_H_ #define GFAL_DEPRECATED_H_ #if !defined(__GFAL2_H_INSIDE__) && !defined(__GFAL2_BUILD__) # warning "Direct inclusion of gfal2 headers is deprecated. Please, include only gfal_api.h or gfal_plugins_api.h" #endif #if __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 5) #define GFAL2_DEPRECATED(alt) __attribute__ ((deprecated("Use '" #alt "' instead"))) #elif __GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 1) #define GFAL2_DEPRECATED(alt) __attribute__((__deprecated__)) #elif defined(_MSC_VER) && (_MSC_VER >= 1300) #define GFAL2_DEPRECATED(alt) __declspec(deprecated) #else #warning "No deprecation rule for your compiler!" #endif #if __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 5) #define GFAL2_DEPRECATED_NOALT __attribute__ ((deprecated)) #elif __GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 1) #define GFAL2_DEPRECATED_NOALT __attribute__((__deprecated__)) #elif defined(_MSC_VER) && (_MSC_VER >= 1300) #define GFAL2_DEPRECATED_NOALT __declspec(deprecated) #else #warning "No deprecation rule for your compiler!" #endif #endif gfal2-v2.23.0/src/core/common/gfal_error.c000066400000000000000000000043321465240014500202570ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include "gfal_error.h" #if (GLIB_CHECK_VERSION(2,16,0) != TRUE) #include "future/glib.h" #endif void gfal2_set_error(GError **err, GQuark domain, gint code, const gchar *function, const gchar *format, ...) { char buffer[512]; va_list args; va_start(args, format); vsnprintf(buffer, sizeof(buffer), format, args); va_end(args); if (gfal2_log_get_level() >= G_LOG_LEVEL_DEBUG) g_set_error(err, domain, code, "[%s] %s", function, buffer); else g_set_error_literal(err, domain, code, buffer); } void gfal2_propagate_prefixed_error_extended(GError **dest, GError *src, const gchar *function, const gchar *format, ...) { if (dest == NULL) { g_error_free(src); return; } if (gfal2_log_get_level() >= G_LOG_LEVEL_DEBUG) { if (src->message[0] == '[') g_propagate_prefixed_error(dest, src, "[%s]", function); else g_propagate_prefixed_error(dest, src, "[%s] ", function); } else { *dest = src; } if (format != NULL) { char buffer[512]; va_list args; va_start(args, format); vsnprintf(buffer, sizeof(buffer), format, args); va_end(args); g_prefix_error(dest, "%s", buffer); } } void gfal2_propagate_prefixed_error(GError **dest, GError *src, const gchar *function) { gfal2_propagate_prefixed_error_extended(dest, src, function, NULL); } gfal2-v2.23.0/src/core/common/gfal_error.h000066400000000000000000000051421465240014500202640ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #pragma once #ifndef GFAL_COMMON_ERR_HELPERS_H_ #define GFAL_COMMON_ERR_HELPERS_H_ #include "gfal_common.h" #include #if !defined(__GFAL2_H_INSIDE__) && !defined(__GFAL2_BUILD__) # warning "Direct inclusion of gfal2 headers is deprecated. Please, include only gfal_api.h or gfal_plugins_api.h" #endif #ifdef __cplusplus extern "C" { #endif /** @def Wraps g_set_error and appends the function name * only if DEBUG is enabled */ void gfal2_set_error(GError **err, GQuark domain, gint code, const gchar *function, const gchar *format, ...) G_GNUC_PRINTF (5, 6); /** @def Wraps g_propagate_prefixed_error, and appends * the function name only if DEBUG is enabled * fmt is always appended */ void gfal2_propagate_prefixed_error_extended(GError **dest, GError *src, const gchar *function, const gchar *format, ...) G_GNUC_PRINTF (4, 5); /** @def Wraps g_propagate_prefixed_error, and appends * the function name only if DEBUG is enabled */ void gfal2_propagate_prefixed_error(GError **dest, GError *src, const gchar *function); #ifdef __cplusplus } #endif /** @def macro for error error report on args * */ #define g_return_val_err_if_fail(exp, val, err, msg) if(!(exp)){ g_set_error(err, gfal2_get_core_quark(), EINVAL, msg); return val; } /** @def macro for one-line return with error management exception-like */ #define G_RETURN_ERR(ret, tmp_err, err) \ if(tmp_err)\ gfal2_propagate_prefixed_error(err, tmp_err, __func__);\ return ret #endif /* GFAL_COMMON_ERR_HELPERS_H_ */ gfal2-v2.23.0/src/core/common/gfal_file_handle.c000066400000000000000000000042341465240014500213610ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "gfal_file_handle.h" #include "gfal_file_handler_container.h" gfal_file_handle gfal_file_handle_new(const char* module_name, gpointer fdesc) { gfal_file_handle f = g_new(struct _gfal_file_handle, 1); g_strlcpy(f->module_name, module_name, GFAL_MODULE_NAME_SIZE); f->lock = g_mutex_new(); f->offset = 0; f->fdesc = fdesc; f->ext_data = NULL; f->path = NULL; return f; } gfal_file_handle gfal_file_handle_new2(const char *module_name, gpointer fdesc, gpointer user_data, const char *file_path) { gfal_file_handle f = gfal_file_handle_new(module_name, fdesc); if (file_path) f->path = g_strdup(file_path); f->ext_data = user_data; return f; } gpointer gfal_file_handle_get_fdesc(gfal_file_handle fh) { return fh->fdesc; } void gfal_file_handle_set_fdesc(gfal_file_handle fh, gpointer fdesc) { fh->fdesc = fdesc; } gpointer gfal_file_handle_get_user_data(gfal_file_handle fh) { return fh->ext_data; } const gchar* gfal_file_handle_get_path(gfal_file_handle fh) { return fh->path; } void gfal_file_handle_lock(gfal_file_handle fh) { g_assert(fh); g_mutex_lock(fh->lock); } void gfal_file_handle_unlock(gfal_file_handle fh) { g_assert(fh); g_mutex_unlock(fh->lock); } // Delete a gfal_file handle void gfal_file_handle_delete(gfal_file_handle fh) { if (fh) { g_mutex_free(fh->lock); g_free(fh->path); g_free(fh); } } gfal2-v2.23.0/src/core/common/gfal_file_handle.h000066400000000000000000000052201465240014500213620ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #ifndef GFAL_FILE_HANDLE_H #define GFAL_FILE_HANDLE_H #pragma once #if !defined(__GFAL2_H_INSIDE__) && !defined(__GFAL2_BUILD__) # warning "Direct inclusion of gfal2 headers is deprecated. Please, include only gfal_api.h or gfal_plugins_api.h" #endif #include #include #ifdef __cplusplus extern "C" { #endif typedef struct _gfal_file_handle_container *gfal_file_handle_container; typedef struct _gfal_file_handle* gfal_file_handle; /** * @brief create a gfal file handle * @param module_name : module name must be the plugin_name of the plugin creating the gfal_file_handle, \ref _gfal_plugin_interface * @param gpointer : internal file descriptor of the plugin to store* */ gfal_file_handle gfal_file_handle_new(const char* module_name, gpointer fdesc); /** * same than \ref gfal_file_handle_new but allows to store user data in the gfal file descriptor * allow the usage of the readdirpp operation in case of directory file handle */ gfal_file_handle gfal_file_handle_new2(const char *module_name, gpointer fdesc, gpointer user_data, const char *file_path); /** * return the file descriptor of this gfal file handle */ gpointer gfal_file_handle_get_fdesc(gfal_file_handle fh); /** * set the file descriptor */ void gfal_file_handle_set_fdesc(gfal_file_handle fh, gpointer fdesc); /** * return the user data of this gfal file descriptor */ gpointer gfal_file_handle_get_user_data(gfal_file_handle fh); /** * return the file path associated to the file handle */ const gchar* gfal_file_handle_get_path(gfal_file_handle fh); /** * delete an existing gfal file descriptor * a file descriptor must be deleted by the plugin in the "close" functions */ void gfal_file_handle_delete(gfal_file_handle fh); /** * Lock file handler */ void gfal_file_handle_lock(gfal_file_handle fh); /** * Unlock file handler */ void gfal_file_handle_unlock(gfal_file_handle fh); #ifdef __cplusplus } #endif #endif /* GFAL_FILE_HANDLE_H */ gfal2-v2.23.0/src/core/common/gfal_file_handler_container.c000066400000000000000000000101511465240014500236000ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include #include "gfal_error.h" #include "gfal_handle.h" #include "gfal_file_handler_container.h" // generate a new unique key static int gfal_file_key_generatorG(gfal_file_handle_container fhandle, GError** err) { g_return_val_err_if_fail(fhandle, 0, err, "[gfal_file_descriptor_generatorG] Invalid arg file handle"); int ret = rand(); GHashTable* c = fhandle->container; if (g_hash_table_size(c) > G_MAXINT / 2) { gfal2_set_error(err, gfal2_get_plugins_quark(), EMFILE, __func__, "Too many files open"); ret = 0; } else { while (ret == 0 || g_hash_table_lookup(c, GINT_TO_POINTER(ret)) != NULL) { ret = rand(); } } return ret; } /* * Add the given file handle to the and return a file descriptor * return the associated key if success else 0 and set err */ int gfal_add_new_file_desc(gfal_file_handle_container fhandle, gpointer pfile, GError** err) { g_return_val_err_if_fail(fhandle && pfile, 0, err, "[gfal_add_new_file_desc] Invalid arg fhandle and/or pfile"); pthread_mutex_lock(&(fhandle->m_container)); GError* tmp_err = NULL; GHashTable* c = fhandle->container; int key = gfal_file_key_generatorG(fhandle, &tmp_err); if (key != 0) { g_hash_table_insert(c, GINT_TO_POINTER(key), pfile); } if (tmp_err) { gfal2_propagate_prefixed_error(err, tmp_err, __func__); } pthread_mutex_unlock(&(fhandle->m_container)); return key; } // remove the associated file handle associated with the given file descriptor // return true if success else false gboolean gfal_remove_file_desc(gfal_file_handle_container fhandle, int key, GError** err) { pthread_mutex_lock(&(fhandle->m_container)); GHashTable* c = fhandle->container; gboolean p = g_hash_table_remove(c, GINT_TO_POINTER(key)); if (!p) gfal2_set_error(err, gfal2_get_plugins_quark(), EBADF, __func__, "bad file descriptor"); pthread_mutex_unlock(&(fhandle->m_container)); return p; } //create a new file descriptor container with the given destroyer function to an element of the container gfal_file_handle_container gfal_file_descriptor_handle_create(GDestroyNotify destroyer) { gfal_file_handle_container d = g_malloc0(sizeof(struct _gfal_file_handle_container)); d->container = g_hash_table_new_full(NULL, NULL, NULL, destroyer); pthread_mutex_init(&(d->m_container), NULL); return d; } void gfal_file_descriptor_handle_destroy(gfal_file_handle_container fhandle) { if (fhandle->container) { g_hash_table_destroy(fhandle->container); } pthread_mutex_destroy(&fhandle->m_container); g_free(fhandle); } /* * * return the file handle associated with the file_desc * @warning does not free the handle * * */ gfal_file_handle gfal_file_handle_bind(gfal_file_handle_container h, int fd, GError** err) { g_return_val_err_if_fail(fd, 0, err, "invalid dir descriptor"); pthread_mutex_lock(&(h->m_container)); gpointer p = g_hash_table_lookup(h->container, GINT_TO_POINTER(fd)); if (!p) { gfal2_set_error(err, gfal2_get_plugins_quark(), EBADF, __func__, "bad file descriptor"); } pthread_mutex_unlock(&(h->m_container)); return (gfal_file_handle)p; } gfal2-v2.23.0/src/core/common/gfal_file_handler_container.h000066400000000000000000000034511465240014500236120ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #pragma once #ifndef GFAL_COMMON_FILEDESCRIPTOR_H_ #define GFAL_COMMON_FILEDESCRIPTOR_H_ #include "gfal_constants.h" #include "gfal_file_handle.h" #include #include #include #ifdef __cplusplus extern "C" { #endif struct _gfal_file_handle_container { GHashTable* container; pthread_mutex_t m_container; }; struct _gfal_file_handle { char module_name[GFAL_MODULE_NAME_SIZE]; // This MUST be the Name of the plugin associated with this handle! GMutex* lock; off_t offset; gpointer ext_data; gpointer fdesc; gchar* path; }; gfal_file_handle_container gfal_file_descriptor_handle_create(GDestroyNotify destroyer); void gfal_file_descriptor_handle_destroy(gfal_file_handle_container fhandle); int gfal_add_new_file_desc(gfal_file_handle_container fhandle, gpointer pfile, GError** err); gboolean gfal_remove_file_desc(gfal_file_handle_container fhandle, int key, GError** err); gfal_file_handle gfal_file_handle_bind(gfal_file_handle_container h, int fd, GError** err); #ifdef __cplusplus } #endif #endif /* GFAL_COMMON_FILEDESCRIPTOR_H_ */ gfal2-v2.23.0/src/core/common/gfal_handle.h000066400000000000000000000034141465240014500203660ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #pragma once #ifndef GFAL_TYPES_H_ #define GFAL_TYPES_H_ #if !defined(__GFAL2_H_INSIDE__) && !defined(__GFAL2_BUILD__) # warning "Direct inclusion of gfal2 headers is deprecated. Please, include only gfal_api.h or gfal_plugins_api.h" #endif #include "gfal_plugin_interface.h" /* enforce proper calling convention */ #ifdef __cplusplus extern "C" { #endif struct _gfal_plugin_opts { gfal_plugin_interface plugin_list[MAX_PLUGIN_LIST]; GList* sorted_plugin; int plugin_number; }; typedef struct _gfal_plugin_opts gfal_plugin_opts; struct gfal_handle_ { gboolean initiated; // struct of the plugin opts gfal_plugin_opts plugin_opt; //struct for the file descriptors gfal_file_handle_container fdescs; GKeyFile *config; // cancel logic volatile gint running_ops; gboolean cancel; GMutex* mux_cancel; GHookList cancel_hooks; // Credential mapping GList *cred_mapping; // client information char* agent_name; char* agent_version; GPtrArray* client_info; }; #ifdef __cplusplus } #endif #endif gfal2-v2.23.0/src/core/common/gfal_plugin.c000066400000000000000000001217501465240014500204300ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include #include #include #include #include #include "gfal_plugin.h" #include "gfal_constants.h" #include "gfal_error.h" #include "gfal_file_handler_container.h" #include #ifndef GFAL_PLUGIN_DIR_DEFAULT #error "GFAL_PLUGIN_DIR_DEFAULT should be define at compile time" #endif /* * function to use in order to create a new plugin interface * permit to keep the ABI compatibility * must be use in ALL the plugin's "gfal_plugin_init" functions */ gfal_plugin_interface* gfal_plugin_interface_new() { size_t s_plugin = sizeof(gfal_plugin_interface) + 1; gfal_plugin_interface* res = malloc(s_plugin); // bigger allocation that needed for future extensions memset(res, 0, s_plugin); return res; } plugin_handle gfal_get_plugin_handle(gfal_plugin_interface* cata_list) { return cata_list->plugin_data; } gboolean gfal_feature_is_supported(void *ptr, GQuark scope, const char *func_name, const char *surl, GError **err) { if (ptr == NULL) { g_set_error(err, gfal2_get_plugins_quark(), EPROTONOSUPPORT, "[%s] Protocol not supported or path/url invalid: %s", func_name, surl); return FALSE; } return TRUE; } //convenience function for safe calls to the plugin checkers gboolean gfal_plugin_checker_safe(gfal_plugin_interface* cata_list, const char* path, plugin_mode call_type, GError** terr) { if (cata_list->check_plugin_url) return cata_list->check_plugin_url(cata_list->plugin_data, path, call_type, terr); else return FALSE; } // // Resolve entry point in a plugin and add it to the current plugin list // static int gfal_module_init(gfal2_context_t handle, void* dlhandle, const char* module_name, GError** err) { GError* tmp_err = NULL; static gfal_plugin_interface (*constructor)(gfal2_context_t, GError**); int* n = &handle->plugin_opt.plugin_number; int res = -1; constructor = (gfal_plugin_interface (*)(gfal2_context_t, GError**)) dlsym(dlhandle, GFAL_PLUGIN_INIT_SYM); if (constructor == NULL) { g_set_error(&tmp_err, gfal2_get_plugins_quark(), EINVAL, "No symbol %s found in the plugin %s, failure", GFAL_PLUGIN_INIT_SYM, module_name); *n = 0; } else { handle->plugin_opt.plugin_list[*n] = constructor(handle, &tmp_err); handle->plugin_opt.plugin_list[*n].gfal_data = dlhandle; if (tmp_err) { g_prefix_error(&tmp_err, "Unable to load plugin %s : ", module_name); *n = 0; } else { *n += 1; gfal2_log(G_LOG_LEVEL_MESSAGE, "[gfal_module_load] plugin %s loaded with success ", module_name); res = 0; } } G_RETURN_ERR(res, tmp_err, err); } // unload each loaded plugin int gfal_plugins_delete(gfal2_context_t handle, GError** err) { g_return_val_err_if_fail(handle, -1, err, "[gfal_plugins_delete] Invalid value of handle"); const int plugin_number = handle->plugin_opt.plugin_number; if (plugin_number > 0) { int i; for (i = 0; i < plugin_number; ++i) { gfal_plugin_interface* p = &(handle->plugin_opt.plugin_list[i]); if (p->plugin_delete) p->plugin_delete(gfal_get_plugin_handle(p)); } handle->plugin_opt.plugin_number = 0; } return 0; } // return the proper plugin linked to this file handle gfal_plugin_interface* gfal_plugin_map_file_handle(gfal2_context_t handle, gfal_file_handle fh, GError** err) { GError* tmp_err = NULL; int i; gfal_plugin_interface* cata_list = NULL; int n = gfal_plugins_instance(handle, &tmp_err); if (n > 0) { cata_list = handle->plugin_opt.plugin_list; for (i = 0; i < n; ++i) { if (strncmp(cata_list[i].getName(), fh->module_name, GFAL_MODULE_NAME_SIZE) == 0) return &(cata_list[i]); } g_set_error(&tmp_err, gfal2_get_plugins_quark(), EBADF, "No gfal_module with the handle name : %s", fh->module_name); } else { g_set_error(&tmp_err, gfal2_get_plugins_quark(), EINVAL, "No gfal_module loaded"); } if (tmp_err) gfal2_propagate_prefixed_error(err, tmp_err, __func__); return cata_list; } // external function to get the list of the plugins loaded char** gfal_plugins_get_list(gfal2_context_t handle, GError** err) { GError* tmp_err = NULL; char** resu = NULL; int n = gfal_plugins_instance(handle, &tmp_err); if (n > 0) { resu = g_new0(char*, n + 1); int i; gfal_plugin_interface* cata_list = handle->plugin_opt.plugin_list; for (i = 0; i < n; ++i, ++cata_list) { resu[i] = strndup(cata_list->getName(), GFAL_URL_MAX_LEN); } } if (tmp_err) gfal2_propagate_prefixed_error(err, tmp_err, __func__); return resu; } // external function to return a gfal_plugin_interface from a given plugin name gfal_plugin_interface* gfal_search_plugin_with_name(gfal2_context_t handle, const char* name, GError** err) { g_return_val_err_if_fail(name && handle, NULL, err, "must be non NULL value"); GError* tmp_err = NULL; gfal_plugin_interface* resu = NULL; int n = gfal_plugins_instance(handle, &tmp_err); if (n > 0) { int i; gfal_plugin_interface* cata_list = handle->plugin_opt.plugin_list; for (i = 0; i < n; ++i, ++cata_list) { const char* plugin_name = cata_list->getName(); if (plugin_name != NULL && strcmp(plugin_name, name) == 0) { resu = cata_list; break; } } if (resu == NULL) g_set_error(&tmp_err, gfal2_get_plugins_quark(), ENOENT, " No plugin loaded with this name %s", name); } if (tmp_err) gfal2_propagate_prefixed_error(err, tmp_err, __func__); return resu; } // load the gfal_plugins in the listed library static int gfal_module_load(gfal2_context_t handle, char* module_name, GError** err) { void* dlhandle = dlopen(module_name, RTLD_NOW); GError * tmp_err = NULL; int res = 0; if (dlhandle == NULL) { gfal2_log(G_LOG_LEVEL_WARNING, "Unable to open the %s plugin specified in the plugin directory: %s", module_name, dlerror()); } else { res = gfal_module_init(handle, dlhandle, module_name, &tmp_err); } if (tmp_err) { gfal2_propagate_prefixed_error(err, tmp_err, __func__); } return res; } /* * Provide a list of the gfal2 plugins path * Return NULL terminated table of plugins */ char ** gfal_list_directory_plugins(const char * dir, GError ** err) { GError * tmp_err = NULL; char ** res, **p_res; res = p_res = NULL; int n = 0; GDir* d = g_dir_open(dir, 0, &tmp_err); if (d) { gchar * d_name = NULL; while ((d_name = (char*) g_dir_read_name(d)) != NULL) { if (strstr(d_name, G_MODULE_SUFFIX) != NULL) { GString * strbuff = g_string_new(dir); n++; if (n == 1) { res = malloc(sizeof(char*) * 2); p_res = res; } else { res = realloc(res, sizeof(char*) * (n + 1)); p_res = res + n - 1; } gfal2_log(G_LOG_LEVEL_DEBUG, " [gfal_list_directory_plugins] add plugin to list to load %s%s%s ", dir, G_DIR_SEPARATOR_S, d_name); g_string_append(strbuff, G_DIR_SEPARATOR_S); g_string_append(strbuff, d_name); *p_res = g_string_free(strbuff, FALSE); } else { gfal2_log(G_LOG_LEVEL_DEBUG, " [gfal_list_directory_plugins] WARNING : File that is not a plugin in the plugin directory %s%s%s ", dir, G_DIR_SEPARATOR_S, d_name); } } if (p_res != NULL) { p_res++; // finish string tab *p_res = NULL; } g_dir_close(d); } if (tmp_err) { g_prefix_error(&tmp_err, "Error, gfal2 plugins directory : %s -> ", dir); gfal2_propagate_prefixed_error(err, tmp_err, __func__); } return res; } char ** gfal_localize_plugins(GError** err) { GError * tmp_err = NULL; char** res = NULL; char * gfal_plugin_dir = (char*) g_getenv(GFAL_PLUGIN_DIR_ENV); if (gfal_plugin_dir != NULL) { gfal2_log(G_LOG_LEVEL_DEBUG, "... %s environment variable specified, try to load the plugins in given dir : %s", GFAL_PLUGIN_DIR_ENV, gfal_plugin_dir); } else { /* GFAL_PLUGIN_DIR_DEFAULT defined at compilation time */ gfal_plugin_dir = GFAL_PLUGIN_DIR_DEFAULT G_DIR_SEPARATOR_S; gfal2_log(G_LOG_LEVEL_DEBUG, "... no %s environment variable specified, try to load plugins in the default directory : %s", GFAL_PLUGIN_DIR_ENV, gfal_plugin_dir); } res = gfal_list_directory_plugins(gfal_plugin_dir, &tmp_err); G_RETURN_ERR(res, tmp_err, err); } int gfal_modules_resolve(gfal2_context_t handle, GError** err) { GError* tmp_err = NULL; int res = -1; char** tab_args; if ((tab_args = gfal_localize_plugins(&tmp_err)) != NULL) { char** p = tab_args; while (*p != NULL) { if (**p == '\0') break; if (gfal_module_load(handle, *p, &tmp_err) != 0) { res = -1; break; } gfal2_log(G_LOG_LEVEL_DEBUG, " gfal_plugin loaded successfully : %s", *p); res = 0; p++; } g_strfreev(tab_args); } if (tmp_err) gfal2_propagate_prefixed_error(err, tmp_err, __func__); return res; } // // compare of the plugin function // gint gfal_plugin_compare(gconstpointer a, gconstpointer b) { gfal_plugin_interface* pa = (gfal_plugin_interface*) a; gfal_plugin_interface* pb = (gfal_plugin_interface*) b; return (pa->priority > pb->priority) ? (-1) : (((pa->priority == pb->priority) ? 0 : 1)); } // // Sort plugins by priority // int gfal_plugins_sort(gfal2_context_t handle, GError ** err) { if (handle->plugin_opt.sorted_plugin) { g_list_free(handle->plugin_opt.sorted_plugin); handle->plugin_opt.sorted_plugin = NULL; } int i; for (i = 0; i < handle->plugin_opt.plugin_number; ++i) { handle->plugin_opt.sorted_plugin = g_list_append( handle->plugin_opt.sorted_plugin, &(handle->plugin_opt.plugin_list[i])); } handle->plugin_opt.sorted_plugin = g_list_sort( handle->plugin_opt.sorted_plugin, &gfal_plugin_compare); if (gfal2_log_get_level() >= G_LOG_LEVEL_DEBUG) { // print plugin order GString* strbuff = g_string_new(" plugin priority order: "); GList* l = handle->plugin_opt.sorted_plugin; for (i = 0; i < handle->plugin_opt.plugin_number; ++i) { strbuff = g_string_append(strbuff, ((gfal_plugin_interface*) l->data)->getName()); strbuff = g_string_append(strbuff, " -> "); l = g_list_next(l); } gfal2_log(G_LOG_LEVEL_DEBUG, "%s", strbuff->str); g_string_free(strbuff, TRUE); } return 0; } // // Instantiate all plugins for use if it's not the case // return the number of plugins available // int gfal_plugins_instance(gfal2_context_t handle, GError** err) { g_return_val_err_if_fail(handle, -1, err, "[gfal_plugins_instance] invalid value of handle"); const int plugin_number = handle->plugin_opt.plugin_number; if (plugin_number <= 0) { GError* tmp_err = NULL; gfal_modules_resolve(handle, &tmp_err); if (tmp_err) { gfal2_propagate_prefixed_error(err, tmp_err, __func__); handle->plugin_opt.plugin_number = -1; } else if (handle->plugin_opt.plugin_number > 0) { gfal_plugins_sort(handle, &tmp_err); if (tmp_err) { gfal2_propagate_prefixed_error(err, tmp_err, __func__); return -1; } } return handle->plugin_opt.plugin_number; } return plugin_number; } gfal_plugin_interface* gfal_find_plugin(gfal2_context_t handle, const char * url, plugin_mode acc_mode, GError** err) { GError* tmp_err = NULL; gboolean compatible = FALSE; const int n_plugins = gfal_plugins_instance(handle, &tmp_err); if (n_plugins > 0) { GList * plugin_list = g_list_first(handle->plugin_opt.sorted_plugin); while (plugin_list != NULL) { gfal_plugin_interface* plugin_ifce = plugin_list->data; compatible = gfal_plugin_checker_safe(plugin_ifce, url, acc_mode, &tmp_err); if (tmp_err) break; if (compatible) return plugin_ifce; plugin_list = g_list_next(plugin_list); } } if (tmp_err) { gfal2_propagate_prefixed_error(err, tmp_err, __func__); } else { gfal2_set_error(err, gfal2_get_plugins_quark(), EPROTONOSUPPORT, __func__, "Protocol not supported or path/url invalid: %s", url); } return NULL; } int gfal2_register_plugin(gfal2_context_t handle, const gfal_plugin_interface* ifce, GError** error) { if (handle->plugin_opt.plugin_number >= MAX_PLUGIN_LIST) { gfal2_set_error(error, gfal2_get_plugins_quark(), ENOMEM, __func__, "Not enough space to allocate a new plugin"); return -1; } int i = handle->plugin_opt.plugin_number; handle->plugin_opt.plugin_number++; handle->plugin_opt.plugin_list[i] = *ifce; return gfal_plugins_sort(handle, error); } // Execute an access function on the first plugin compatible in the plugin list // return the result of the first valid plugin for a given URL // result of the access method or -1 if error and set GError with the correct value // error : EPROTONOSUPPORT means that the URL is not matched by a plugin int gfal_plugins_accessG(gfal2_context_t handle, const char* path, int mode, GError** err) { g_return_val_err_if_fail(handle && path, EINVAL, err, "[gfal_plugins_accessG] Invalid arguments"); int res = -1; GError * tmp_err = NULL; gfal_plugin_interface* p = gfal_find_plugin(handle, path, GFAL_PLUGIN_ACCESS, &tmp_err); if (p) res = p->accessG(gfal_get_plugin_handle(p), path, mode, &tmp_err); G_RETURN_ERR(res, tmp_err, err); } // Execute a stat function on the appropriate plugin int gfal_plugin_statG(gfal2_context_t handle, const char* path, struct stat* st, GError** err) { g_return_val_err_if_fail(handle && path, EINVAL, err, "[gfal_plugin_statG] Invalid arguments"); int res = -1; GError* tmp_err = NULL; gfal_plugin_interface* p = gfal_find_plugin(handle, path, GFAL_PLUGIN_STAT, &tmp_err); if (p) res = p->statG(gfal_get_plugin_handle(p), path, st, &tmp_err); G_RETURN_ERR(res, tmp_err, err); } // Execute a stat function on the appropriate plugin int gfal_plugin_lstatG(gfal2_context_t handle, const char* path, struct stat* st, GError** err) { g_return_val_err_if_fail(handle && path, EINVAL, err, "[gfal_plugin_statG] Invalid arguments"); int res = -1; GError* tmp_err = NULL; gfal_plugin_interface* p = gfal_find_plugin(handle, path, GFAL_PLUGIN_LSTAT, &tmp_err); if (p) res = p->lstatG(gfal_get_plugin_handle(p), path, st, &tmp_err); G_RETURN_ERR(res, tmp_err, err); } // Execute a readlink function on the appropriate plugin ssize_t gfal_plugin_readlinkG(gfal2_context_t handle, const char* path, char* buff, size_t buffsiz, GError** err) { g_return_val_err_if_fail(handle && path, EINVAL, err, "[gfal_plugin_readlinkG] Invalid arguments"); GError* tmp_err = NULL; ssize_t resu = -1; gfal_plugin_interface* p = gfal_find_plugin(handle, path, GFAL_PLUGIN_READLINK, &tmp_err); if (p) resu = p->readlinkG(gfal_get_plugin_handle(p), path, buff, buffsiz, &tmp_err); G_RETURN_ERR(resu, tmp_err, err); } // Execute a chmod function on the appropriate plugin int gfal_plugin_chmodG(gfal2_context_t handle, const char* path, mode_t mode, GError** err) { g_return_val_err_if_fail(handle && path, -1, err, "[gfal_plugin_chmodG] Invalid arguments"); GError* tmp_err = NULL; int res = -1; gfal_plugin_interface* p = gfal_find_plugin(handle, path, GFAL_PLUGIN_CHMOD, &tmp_err); if (p) res = p->chmodG(gfal_get_plugin_handle(p), path, mode, &tmp_err); G_RETURN_ERR(res, tmp_err, err); } // Execute a rename function on the appropriate plugin int gfal_plugin_renameG(gfal2_context_t handle, const char* oldpath, const char* newpath, GError** err) { g_return_val_err_if_fail(oldpath && newpath, -1, err, "[gfal_plugin_renameG] invalid value in args oldpath, handle or newpath"); GError* tmp_err = NULL; int res = -1; gfal_plugin_interface *src_p, *dst_p; src_p = gfal_find_plugin(handle, oldpath, GFAL_PLUGIN_RENAME, &tmp_err); if (src_p) { dst_p = gfal_find_plugin(handle, newpath, GFAL_PLUGIN_RENAME, &tmp_err); if (src_p == dst_p) res = dst_p->renameG(gfal_get_plugin_handle(dst_p), oldpath, newpath, &tmp_err); } G_RETURN_ERR(res, tmp_err, err); } // Execute a symlink function on the appropriate plugin int gfal_plugin_symlinkG(gfal2_context_t handle, const char* oldpath, const char* newpath, GError** err) { g_return_val_err_if_fail(oldpath && newpath, -1, err, "[gfal_plugin_symlinkG] invalid value in args oldpath, handle or newpath"); GError* tmp_err = NULL; int res = -1; gfal_plugin_interface *src_p, *dst_p; src_p = gfal_find_plugin(handle, oldpath, GFAL_PLUGIN_SYMLINK, &tmp_err); if (src_p) { dst_p = gfal_find_plugin(handle, newpath, GFAL_PLUGIN_SYMLINK, &tmp_err); if (src_p == dst_p) res = dst_p->symlinkG(gfal_get_plugin_handle(dst_p), oldpath, newpath, &tmp_err); } G_RETURN_ERR(res, tmp_err, err); } // Execute a mkdir function on the appropriate plugin int gfal_plugin_mkdirp(gfal2_context_t handle, const char* path, mode_t mode, gboolean pflag, GError** err) { g_return_val_err_if_fail(handle && path, -1, err, "[gfal_plugin_mkdirp] Invalid argumetns in path or/and handle"); GError* tmp_err = NULL; int res = -1; gfal_plugin_interface* p = gfal_find_plugin(handle, path, GFAL_PLUGIN_MKDIR, &tmp_err); if (p) res = p->mkdirpG(gfal_get_plugin_handle(p), path, mode, pflag, &tmp_err); if (pflag && res < 0 && tmp_err->code == EEXIST) { g_error_free(tmp_err); tmp_err = NULL; res = 0; } G_RETURN_ERR(res, tmp_err, err); } // Execute a rmdir function on the appropriate plugin int gfal_plugin_rmdirG(gfal2_context_t handle, const char* path, GError** err) { g_return_val_err_if_fail(handle && path, -1, err, "[gfal_plugin_rmdirp] Invalid arguments in path or/and handle"); GError* tmp_err = NULL; int res = -1; gfal_plugin_interface* p = gfal_find_plugin(handle, path, GFAL_PLUGIN_RMDIR, &tmp_err); if (p) res = p->rmdirG(gfal_get_plugin_handle(p), path, &tmp_err); G_RETURN_ERR(res, tmp_err, err); } // Execute a opendir function on the appropriate plugin gfal_file_handle gfal_plugin_opendirG(gfal2_context_t handle, const char* name, GError** err) { g_return_val_err_if_fail(handle && name, NULL, err, "[gfal_plugin_opendir] invalid value"); GError* tmp_err = NULL; gfal_file_handle resu = NULL; gfal_plugin_interface* p = gfal_find_plugin(handle, name, GFAL_PLUGIN_OPENDIR, &tmp_err); if (p) resu = p->opendirG(gfal_get_plugin_handle(p), name, &tmp_err); G_RETURN_ERR(resu, tmp_err, err); } // Execute a closedir function on the appropriate plugin int gfal_plugin_closedirG(gfal2_context_t handle, gfal_file_handle fh, GError** err) { g_return_val_err_if_fail(handle && fh, -1, err, "[gfal_plugin_closedirG] Invalid args "); GError* tmp_err = NULL; int res = -1; gfal_plugin_interface* if_cata = gfal_plugin_map_file_handle(handle, fh, &tmp_err); if (!tmp_err) res = if_cata->closedirG(if_cata->plugin_data, fh, &tmp_err); G_RETURN_ERR(res, tmp_err, err); } // Execute a open function on the appropriate plugin gfal_file_handle gfal_plugin_openG(gfal2_context_t handle, const char * path, int flag, mode_t mode, GError ** err) { GError* tmp_err = NULL; gfal_file_handle resu = NULL; gfal2_log(G_LOG_LEVEL_DEBUG, " %s ->", __func__); gfal_plugin_interface* p = gfal_find_plugin(handle, path, GFAL_PLUGIN_OPEN, &tmp_err); if (p) resu = p->openG(gfal_get_plugin_handle(p), path, flag, mode, &tmp_err); G_RETURN_ERR(resu, tmp_err, err); } // Execute a close function on the appropriate plugin int gfal_plugin_closeG(gfal2_context_t handle, gfal_file_handle fh, GError** err) { g_return_val_err_if_fail(handle && fh, -1, err, "[gfal_plugin_closeG] Invalid args "); GError* tmp_err = NULL; int res = -1; gfal2_log(G_LOG_LEVEL_DEBUG, " <- %s", __func__); gfal_plugin_interface* if_cata = gfal_plugin_map_file_handle(handle, fh, &tmp_err); if (!tmp_err) res = if_cata->closeG(if_cata->plugin_data, fh, &tmp_err); G_RETURN_ERR(res, tmp_err, err); } // Execute a readdir function on the appropriate plugin struct dirent* gfal_plugin_readdirG(gfal2_context_t handle, gfal_file_handle fh, GError** err) { g_return_val_err_if_fail(handle && fh, NULL, err, "[gfal_plugin_readdirG] Invalid args "); GError* tmp_err = NULL; struct dirent* res = NULL; gfal_plugin_interface* if_cata = gfal_plugin_map_file_handle(handle, fh, &tmp_err); if (!tmp_err) res = if_cata->readdirG(if_cata->plugin_data, fh, &tmp_err); G_RETURN_ERR(res, tmp_err, err); } // Execute a readdir function on the appropriate plugin struct dirent* gfal_plugin_readdirppG(gfal2_context_t handle, gfal_file_handle fh, struct stat* st, GError** err) { g_return_val_err_if_fail(handle && fh, NULL, err, "[gfal_plugin_readdirppG] Invalid args "); GError* tmp_err = NULL; struct dirent* res = NULL; gfal_plugin_interface* if_cata = gfal_plugin_map_file_handle(handle, fh, &tmp_err); if (!tmp_err) { if (gfal_feature_is_supported(if_cata->readdirppG, g_quark_from_string(GFAL2_PLUGIN_SCOPE), __func__, fh->path, &tmp_err)) res = if_cata->readdirppG(if_cata->plugin_data, fh, st, &tmp_err); } G_RETURN_ERR(res, tmp_err, err); } // Execute a getxattr function on the appropriate plugin ssize_t gfal_plugin_getxattrG(gfal2_context_t handle, const char* path, const char*name, void* buff, size_t s_buff, GError** err) { GError* tmp_err = NULL; ssize_t resu = -1; gfal_plugin_interface* p = gfal_find_plugin(handle, path, GFAL_PLUGIN_GETXATTR, &tmp_err); if (p) resu = p->getxattrG(gfal_get_plugin_handle(p), path, name, buff, s_buff, &tmp_err); // If asking for checksum, and got an error, try ourselves if (resu < 0 && tmp_err) { const int checksum_prop_len = sizeof(GFAL_XATTR_CHKSUM_VALUE) - 1; if (strncmp(name, GFAL_XATTR_CHKSUM_VALUE, checksum_prop_len) == 0 && name[checksum_prop_len] == '.') { g_error_free(tmp_err); tmp_err = NULL; const char* chk_type = name + checksum_prop_len + 1; resu = gfal2_checksum(handle, path, chk_type, 0, 0, buff, s_buff, &tmp_err); } } G_RETURN_ERR(resu, tmp_err, err); } // Execute a listxattr function on the appropriate plugin ssize_t gfal_plugin_listxattrG(gfal2_context_t handle, const char* path, char* list, size_t s_list, GError** err) { GError* tmp_err = NULL; ssize_t resu = -1; gfal_plugin_interface* p = gfal_find_plugin(handle, path, GFAL_PLUGIN_LISTXATTR, &tmp_err); if (p) resu = p->listxattrG(gfal_get_plugin_handle(p), path, list, s_list, &tmp_err); G_RETURN_ERR(resu, tmp_err, err); } // Execute a setxattr function on the appropriate plugin int gfal_plugin_setxattrG(gfal2_context_t handle, const char* path, const char* name, const void* value, size_t size, int flags, GError** err) { GError* tmp_err = NULL; int resu = -1; gfal_plugin_interface* p = gfal_find_plugin(handle, path, GFAL_PLUGIN_SETXATTR, &tmp_err); if (p) resu = p->setxattrG(gfal_get_plugin_handle(p), path, name, value, size, flags, &tmp_err); G_RETURN_ERR(resu, tmp_err, err); } // Execute a read function on the appropriate plugin int gfal_plugin_readG(gfal2_context_t handle, gfal_file_handle fh, void* buff, size_t s_buff, GError** err) { g_return_val_err_if_fail(handle && fh && buff && s_buff > 0, -1, err, "[gfal_plugin_readG] Invalid args "); GError* tmp_err = NULL; int res = -1; gfal_plugin_interface* if_cata = gfal_plugin_map_file_handle(handle, fh, &tmp_err); if (!tmp_err) res = if_cata->readG(if_cata->plugin_data, fh, buff, s_buff, &tmp_err); G_RETURN_ERR(res, tmp_err, err); } // Simulate a pread operation in case of non-parallels read support // this is slower than a normal pread/pwrite operation static ssize_t gfal_plugin_simulate_preadG(gfal2_context_t handle, gfal_plugin_interface* if_cata, gfal_file_handle fh, void* buff, size_t s_buff, off_t offset, GError** err) { GError* tmp_err = NULL; ssize_t res = -1; gfal_file_handle_lock(fh); res = if_cata->lseekG(if_cata->plugin_data, fh, offset, SEEK_SET, &tmp_err); if (res == offset) { res = if_cata->readG(if_cata->plugin_data, fh, buff, s_buff, &tmp_err); } else if (!tmp_err) { g_set_error(&tmp_err, gfal2_get_plugins_quark(), EOVERFLOW, "Unknown return from plugin_lseek call"); res = -1; } gfal_file_handle_unlock(fh); G_RETURN_ERR(res, tmp_err, err); } // Execute a pread function on the appropriate plugin ssize_t gfal_plugin_preadG(gfal2_context_t handle, gfal_file_handle fh, void* buff, size_t s_buff, off_t offset, GError** err) { g_return_val_err_if_fail(handle && fh && buff, -1, err, "[gfal_plugin_preadG] Invalid args "); GError* tmp_err = NULL; ssize_t res = -1; gfal_plugin_interface* if_cata = gfal_plugin_map_file_handle(handle, fh, &tmp_err); if (!tmp_err) { if (if_cata->preadG) res = if_cata->preadG(if_cata->plugin_data, fh, buff, s_buff, offset, &tmp_err); else { res = gfal_plugin_simulate_preadG(handle, if_cata, fh, buff, s_buff, offset, &tmp_err); } } G_RETURN_ERR(res, tmp_err, err); } // Simulate a pread operation in case of non-parallels write support // this is slower than a normal pread/pwrite operation static ssize_t gfal_plugin_simulate_pwriteG(gfal2_context_t handle, gfal_plugin_interface* if_cata, gfal_file_handle fh, void* buff, size_t s_buff, off_t offset, GError** err) { GError* tmp_err = NULL; ssize_t res = -1; gfal_file_handle_lock(fh); res = if_cata->lseekG(if_cata->plugin_data, fh, offset, SEEK_SET, &tmp_err); if (res == offset) { res = if_cata->writeG(if_cata->plugin_data, fh, buff, s_buff, &tmp_err); } else if (!tmp_err) { g_set_error(&tmp_err, gfal2_get_plugins_quark(), EOVERFLOW, "Unknown return from plugin_lseek call"); res = -1; } gfal_file_handle_unlock(fh); G_RETURN_ERR(res, tmp_err, err); } // Execute a pwrite function on the appropriate plugin ssize_t gfal_plugin_pwriteG(gfal2_context_t handle, gfal_file_handle fh, void* buff, size_t s_buff, off_t offset, GError** err) { g_return_val_err_if_fail(handle && fh && buff, -1, err, "[gfal_plugin_pwriteG] Invalid args "); GError* tmp_err = NULL; ssize_t res = -1; gfal_plugin_interface* if_cata = gfal_plugin_map_file_handle(handle, fh, &tmp_err); if (!tmp_err) { if (if_cata->pwriteG) res = if_cata->pwriteG(if_cata->plugin_data, fh, buff, s_buff, offset, &tmp_err); else { res = gfal_plugin_simulate_pwriteG(handle, if_cata, fh, buff, s_buff, offset, &tmp_err); } } G_RETURN_ERR(res, tmp_err, err); } // Execute a lseek function on the appropriate plugin int gfal_plugin_lseekG(gfal2_context_t handle, gfal_file_handle fh, off_t offset, int whence, GError** err) { g_return_val_err_if_fail(handle && fh, -1, err, "[gfal_plugin_lseekG] Invalid args "); GError* tmp_err = NULL; int res = -1; gfal_plugin_interface* if_cata = gfal_plugin_map_file_handle(handle, fh, &tmp_err); if (!tmp_err) res = if_cata->lseekG(if_cata->plugin_data, fh, offset, whence, &tmp_err); G_RETURN_ERR(res, tmp_err, err); } // Execute a write function on the appropriate plugin int gfal_plugin_writeG(gfal2_context_t handle, gfal_file_handle fh, void* buff, size_t s_buff, GError** err) { g_return_val_err_if_fail(handle && fh && buff && s_buff > 0, -1, err, "[gfal_plugin_writeG] Invalid args "); GError* tmp_err = NULL; int res = -1; gfal_plugin_interface* if_cata = gfal_plugin_map_file_handle(handle, fh, &tmp_err); if (!tmp_err) res = if_cata->writeG(if_cata->plugin_data, fh, buff, s_buff, &tmp_err); G_RETURN_ERR(res, tmp_err, err); } // Execute a unlink function on the appropriate plugin int gfal_plugin_unlinkG(gfal2_context_t handle, const char* path, GError** err) { GError* tmp_err = NULL; int resu = -1; gfal_plugin_interface* p = gfal_find_plugin(handle, path, GFAL_PLUGIN_UNLINK, &tmp_err); if (p) resu = p->unlinkG(gfal_get_plugin_handle(p), path, &tmp_err); G_RETURN_ERR(resu, tmp_err, err); } int gfal_plugin_bring_onlineG(gfal2_context_t handle, const char* uri, time_t pintime, time_t timeout, char* token, size_t tsize, int async, GError ** err) { GError* tmp_err = NULL; int resu = -1; gfal_plugin_interface* p = gfal_find_plugin(handle, uri, GFAL_PLUGIN_BRING_ONLINE, &tmp_err); if (p) resu = p->bring_online(gfal_get_plugin_handle(p), uri, pintime, timeout, token, tsize, async, &tmp_err); G_RETURN_ERR(resu, tmp_err, err); } int gfal_plugin_bring_online_v2G(gfal2_context_t handle, const char* uri, const char* metadata, time_t pintime, time_t timeout, char* token, size_t tsize, int async, GError ** err) { GError* tmp_err = NULL; int resu = -1; gfal_plugin_interface* p = gfal_find_plugin(handle, uri, GFAL_PLUGIN_BRING_ONLINE, &tmp_err); if (p) resu = p->bring_online_v2(gfal_get_plugin_handle(p), uri, metadata, pintime, timeout, token, tsize, async, &tmp_err); G_RETURN_ERR(resu, tmp_err, err); } ssize_t gfal_plugin_qos_check_classes(gfal2_context_t handle, const char* url, const char* type, char* buff, size_t s_buff, GError** err) { // GFAL_PLUGIN_QOS_CHECK_CLASSES GError* tmp_err = NULL; ssize_t res = -1; gfal_plugin_interface* p = gfal_find_plugin(handle, url, GFAL_PLUGIN_QOS_CHECK_CLASSES, &tmp_err); if (p) res = p->check_qos_classes(gfal_get_plugin_handle(p), url, type, buff, s_buff, &tmp_err); G_RETURN_ERR(res, tmp_err, err); } ssize_t gfal_plugin_check_file_qos(gfal2_context_t handle, const char* url, char* buff, size_t s_buff, GError** err) { // GFAL_PLUGIN_CHECK_FILE_QOS GError* tmp_err = NULL; ssize_t res = -1; gfal_plugin_interface* p = gfal_find_plugin(handle, url, GFAL_PLUGIN_CHECK_FILE_QOS, &tmp_err); if (p) res = p->check_file_qos(gfal_get_plugin_handle(p), url, buff, s_buff, &tmp_err); G_RETURN_ERR(res, tmp_err, err); } ssize_t gfal_plugin_check_qos_available_transitions(gfal2_context_t handle, const char* qos_class_url, char* buff, size_t s_buff, GError** err) { // GFAL_PLUGIN_CHECK_QOS_AVAILABLE_TRANSITIONS GError* tmp_err = NULL; ssize_t res = -1; gfal_plugin_interface* p = gfal_find_plugin(handle, qos_class_url, GFAL_PLUGIN_CHECK_QOS_AVAILABLE_TRANSITIONS, &tmp_err); if (p) res = p->check_qos_available_transitions(gfal_get_plugin_handle(p), qos_class_url, buff, s_buff, &tmp_err); G_RETURN_ERR(res, tmp_err, err); } ssize_t gfal_plugin_check_target_qos(gfal2_context_t handle, const char* url, char* buff, size_t s_buff, GError** err) { // GFAL_PLUGIN_CHECK_TARGET_QOS GError* tmp_err = NULL; ssize_t res = -1; gfal_plugin_interface* p = gfal_find_plugin(handle, url, GFAL_PLUGIN_CHECK_TARGET_QOS, &tmp_err); if (p) res = p->check_target_qos(gfal_get_plugin_handle(p), url, buff, s_buff, &tmp_err); G_RETURN_ERR(res, tmp_err, err); } int gfal_plugin_change_object_qos(gfal2_context_t handle, const char* url, const char* target_qos, GError** err) { // GFAL_PLUGIN_CHANGE_OBJECT_QOS GError* tmp_err = NULL; int res = -1; gfal_plugin_interface* p = gfal_find_plugin(handle, url, GFAL_PLUGIN_CHANGE_OBJECT_QOS, &tmp_err); if (p) res = p->change_object_qos(gfal_get_plugin_handle(p), url, target_qos, &tmp_err); G_RETURN_ERR(res, tmp_err, err); } int gfal_plugin_bring_online_pollG(gfal2_context_t handle, const char* uri, const char* token, GError ** err) { GError* tmp_err = NULL; int resu = -1; gfal_plugin_interface* p = gfal_find_plugin(handle, uri, GFAL_PLUGIN_BRING_ONLINE, &tmp_err); if (p) resu = p->bring_online_poll(gfal_get_plugin_handle(p), uri, token, &tmp_err); G_RETURN_ERR(resu, tmp_err, err); } int gfal_plugin_release_fileG(gfal2_context_t handle, const char* uri, const char* token, GError ** err) { GError* tmp_err = NULL; int resu = -1; gfal_plugin_interface* p = gfal_find_plugin(handle, uri, GFAL_PLUGIN_BRING_ONLINE, &tmp_err); if (p) resu = p->release_file(gfal_get_plugin_handle(p), uri, token, &tmp_err); G_RETURN_ERR(resu, tmp_err, err); } int gfal_plugin_bring_online_listG(gfal2_context_t handle, int nbfiles, const char* const* uris, time_t pintime, time_t timeout, char* token, size_t tsize, int async, GError ** errors) { GError* tmp_err = NULL; int resu = -1; gfal_plugin_interface* p = gfal_find_plugin(handle, *uris, GFAL_PLUGIN_BRING_ONLINE, &tmp_err); if (p && p->bring_online_list) { resu = p->bring_online_list(gfal_get_plugin_handle(p), nbfiles, uris, pintime, timeout, token, tsize, async, errors); } else { if (p && !p->bring_online_list) { gfal2_set_error(&tmp_err, gfal2_get_plugins_quark(), EPROTONOSUPPORT, __func__, "The plugin does not implement bulk bring online"); } int i; for (i = 0; i < nbfiles; ++i) { errors[i] = g_error_copy(tmp_err); } g_error_free(tmp_err); } return resu; } int gfal_plugin_bring_online_list_v2G(gfal2_context_t handle, int nbfiles, const char* const* uris, const char* const* metadata, time_t pintime, time_t timeout, char* token, size_t tsize, int async, GError ** errors) { GError* tmp_err = NULL; int resu = -1; gfal_plugin_interface* p = gfal_find_plugin(handle, *uris, GFAL_PLUGIN_BRING_ONLINE, &tmp_err); if (p && p->bring_online_list_v2) { resu = p->bring_online_list_v2(gfal_get_plugin_handle(p), nbfiles, uris, metadata, pintime, timeout, token, tsize, async, errors); } else { if (p && !p->bring_online_list) { gfal2_set_error(&tmp_err, gfal2_get_plugins_quark(), EPROTONOSUPPORT, __func__, "The plugin does not implement bulk bring online"); } int i; for (i = 0; i < nbfiles; ++i) { errors[i] = g_error_copy(tmp_err); } g_error_free(tmp_err); } return resu; } int gfal_plugin_bring_online_poll_listG(gfal2_context_t handle, int nbfiles, const char* const* uris, const char* token, GError ** errors) { GError* tmp_err = NULL; int resu = -1; gfal_plugin_interface* p = gfal_find_plugin(handle, *uris, GFAL_PLUGIN_BRING_ONLINE, &tmp_err); if (p && p->bring_online_poll_list) { resu = p->bring_online_poll_list(gfal_get_plugin_handle(p), nbfiles, uris, token, errors); } else { if (p && !p->bring_online_poll_list) { gfal2_set_error(&tmp_err, gfal2_get_plugins_quark(), EPROTONOSUPPORT, __func__, "The plugin does not implement bulk bring online polling"); } int i; for (i = 0; i < nbfiles; ++i) { errors[i] = g_error_copy(tmp_err); } g_error_free(tmp_err); } return resu; } int gfal_plugin_release_file_listG(gfal2_context_t handle, int nbfiles, const char* const* uris, const char* token, GError ** errors) { GError* tmp_err = NULL; int resu = -1; gfal_plugin_interface* p = gfal_find_plugin(handle, *uris, GFAL_PLUGIN_BRING_ONLINE, &tmp_err); if (p && p->release_file_list) { resu = p->release_file_list(gfal_get_plugin_handle(p), nbfiles, uris, token, errors); } else { if (p && !p->release_file_list) { gfal2_set_error(&tmp_err, gfal2_get_plugins_quark(), EPROTONOSUPPORT, __func__, "The plugin does not implement bulk releases"); } int i; for (i = 0; i < nbfiles; ++i) { errors[i] = g_error_copy(tmp_err); } g_error_free(tmp_err); } return resu; } int gfal_plugin_unlink_listG(gfal2_context_t handle, int nbfiles, const char* const* uris, GError ** errors) { GError* tmp_err = NULL; int resu = -1; gfal_plugin_interface* p = gfal_find_plugin(handle, *uris, GFAL_PLUGIN_UNLINK, &tmp_err); if (p) { plugin_handle handle = gfal_get_plugin_handle(p); if (p->unlink_listG) { resu = p->unlink_listG(handle, nbfiles, uris, errors); } // Fallback else { int i; resu = 0; for (i = 0; i < nbfiles; ++i) { resu += p->unlinkG(handle, uris[i], &(errors[i])); } } } else { int i; for (i = 0; i < nbfiles; ++i) { errors[i] = g_error_copy(tmp_err); } g_error_free(tmp_err); } return resu; } int gfal_plugin_abort_filesG(gfal2_context_t handle, int nbfiles, const char* const * uris, const char* token, GError ** errors) { GError* tmp_err = NULL; int resu = -1; gfal_plugin_interface* p = gfal_find_plugin(handle, *uris, GFAL_PLUGIN_BRING_ONLINE, &tmp_err); if (p) { plugin_handle handle = gfal_get_plugin_handle(p); resu = p->abort_files(handle, nbfiles, uris, token, errors); } else { int i; for (i = 0; i < nbfiles; ++i) { errors[i] = g_error_copy(tmp_err); } g_error_free(tmp_err); } return resu; } int gfal_plugin_archive_pollG(gfal2_context_t handle, const char* uri, GError ** err) { GError* tmp_err = NULL; int resu = -1; gfal_plugin_interface* p = gfal_find_plugin(handle, uri, GFAL_PLUGIN_ARCHIVE, &tmp_err); if (p) resu = p->archive_poll(gfal_get_plugin_handle(p), uri, &tmp_err); G_RETURN_ERR(resu, tmp_err, err); } int gfal_plugin_archive_poll_listG(gfal2_context_t handle, int nbfiles, const char* const* uris, GError ** errors) { GError* tmp_err = NULL; int resu = -1; gfal_plugin_interface* p = gfal_find_plugin(handle, *uris, GFAL_PLUGIN_ARCHIVE, &tmp_err); if (p && p->archive_poll_list) { resu = p->archive_poll_list(gfal_get_plugin_handle(p), nbfiles, uris, errors); } else { if (p && !p->archive_poll_list) { gfal2_set_error(&tmp_err, gfal2_get_plugins_quark(), EPROTONOSUPPORT, __func__, "The plugin does not support bulk archive polling"); } int i; for (i = 0; i < nbfiles; ++i) { errors[i] = g_error_copy(tmp_err); } g_error_free(tmp_err); } return resu; } ssize_t gfal_plugin_token_retrieveG(gfal2_context_t handle, const char* url, const char* issuer, gboolean write_access, unsigned validity, const char* const* activities, char* buff, size_t s_buff, GError** err) { GError* tmp_err = NULL; ssize_t resu = -1; gfal_plugin_interface* p = gfal_find_plugin(handle, url, GFAL_PLUGIN_TOKEN, &tmp_err); if (p) resu = p->token_retrieve(gfal_get_plugin_handle(p), url, issuer, write_access, validity, activities, buff, s_buff, err); G_RETURN_ERR(resu, tmp_err, err); } gfal2-v2.23.0/src/core/common/gfal_plugin.h000066400000000000000000000053251465240014500204340ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #pragma once #ifndef GFAL_COMMON_PLUGIN_H #define GFAL_COMMON_PLUGIN_H #if !defined(__GFAL2_H_INSIDE__) && !defined(__GFAL2_BUILD__) # warning "Direct inclusion of gfal2 headers is deprecated. Please, include only gfal_api.h or gfal_plugins_api.h" #endif #include "gfal_handle.h" #include "gfal_plugin_interface.h" #include #include #include #include #include #include #include #define GFAL2_PLUGIN_SCOPE "GFAL2::PLUGIN" #ifdef __cplusplus extern "C" { #endif // __cplusplus typedef struct _plugin_pointer_handle { gfal_plugin_interface* plugin_api; /**< plugin official API */ void* dlhandle; /**< dlhandle of the plugin */ void* plugin_data; /**< plugin internal data */ char plugin_name[GFAL_URL_MAX_LEN]; /**< plugin name */ char plugin_lib[GFAL_URL_MAX_LEN]; /**< plugin library path */ }*plugin_pointer_handle; /** * This API is a plugin reserved API and should be used by GFAL2's plugins only * Backward compatibility of this API is not guaranteed */ plugin_handle gfal_get_plugin_handle(gfal_plugin_interface* p_interface); int gfal_plugins_instance(gfal2_context_t, GError** err); char** gfal_plugins_get_list(gfal2_context_t, GError** err); int gfal_plugins_delete(gfal2_context_t, GError** err); gboolean gfal_feature_is_supported(void *ptr, GQuark scope, const char *func_name, const char *surl, GError **err); /** * Find a compatible catalog or return NULL + error */ gfal_plugin_interface* gfal_find_plugin(gfal2_context_t handle, const char * url, plugin_mode acc_mode, GError** err); gfal_plugin_interface* gfal_plugin_map_file_handle(gfal2_context_t handle, gfal_file_handle fh, GError** err); #ifdef __cplusplus } #endif // __cplusplus #endif /* GFAL_COMMON_PLUGIN_H */ gfal2-v2.23.0/src/core/common/gfal_plugin_interface.h000066400000000000000000001107041465240014500224520ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #pragma once #ifndef _GFAL_COMMON_PLUGIN_INTERFACE_H_ #define _GFAL_COMMON_PLUGIN_INTERFACE_H_ #if !defined(__GFAL2_H_INSIDE__) && !defined(__GFAL2_BUILD__) # warning "Direct inclusion of gfal2 headers is deprecated. Please, include only gfal_api.h" #endif #include "gfal_common.h" #include "gfal_constants.h" #include "gfal_file_handle.h" #include #include #include #include #ifdef __cplusplus extern "C" { #endif /** * Helper for the versioned plugin names */ #define GFAL2_PLUGIN_VERSIONED(plugin, version) plugin "-" version /** * classical data access plugin */ #define GFAL_PLUGIN_PRIORITY_DATA 0; /**< The plugin provides IO */ #define GFAL_PLUGIN_PRIORITY_CATALOG 100; /**< The plugin provides namespace operations */ #define GFAL_PLUGIN_PRIORITY_CACHE 200; /**< The plugin provides a cache */ typedef struct _gfal_plugin_interface gfal_plugin_interface; typedef gpointer plugin_handle; /** * Plugin check type */ typedef enum _plugin_mode { GFAL_PLUGIN_ALL=0, GFAL_PLUGIN_ACCESS, GFAL_PLUGIN_CHMOD, GFAL_PLUGIN_RENAME, GFAL_PLUGIN_SYMLINK, GFAL_PLUGIN_STAT, GFAL_PLUGIN_LSTAT, GFAL_PLUGIN_MKDIR, GFAL_PLUGIN_RMDIR, GFAL_PLUGIN_OPENDIR, /**< concat of opendir readdir, closedir*/ GFAL_PLUGIN_OPEN, /**< concat of open read, close*/ GFAL_PLUGIN_RESOLVE_GUID, GFAL_PLUGIN_GETXATTR, GFAL_PLUGIN_SETXATTR, GFAL_PLUGIN_LISTXATTR, GFAL_PLUGIN_READLINK, GFAL_PLUGIN_UNLINK, GFAL_PLUGIN_CHECKSUM, GFAL_PLUGIN_MKDIR_REC, GFAL_PLUGIN_BRING_ONLINE, GFAL_PLUGIN_ARCHIVE, GFAL_PLUGIN_QOS_CHECK_CLASSES, GFAL_PLUGIN_CHECK_FILE_QOS, GFAL_PLUGIN_CHECK_QOS_AVAILABLE_TRANSITIONS, GFAL_PLUGIN_CHECK_TARGET_QOS, GFAL_PLUGIN_CHANGE_OBJECT_QOS, GFAL_PLUGIN_TOKEN } plugin_mode; /** * Check modes for transfers */ typedef enum _gfal_url2_check { GFAL_FILE_COPY, GFAL_BULK_COPY } gfal_url2_check; /** * Prototype of the plugins entry point * * return gfal_plugin_interface must be allocated with \êef gfal_plugin_interface_new * * @param handle : gfal2_context_t of the current call * @param err : Error report in case of fatal error while the plugin load. * * */ typedef gfal_plugin_interface* (*gfal_plugin_init_t)(gfal2_context_t handle, GError** err); /** * @struct _gfal_plugin_interface * * Main interface that MUST be returned the entry point function "gfal_plugin_init" of each GFAL 2.0 plugin. * the minimum calls are : getName, plugin_delete, check_plugin_url * all the unused function pointers must be set to NULL */ struct _gfal_plugin_interface { //! @cond // internal gfal data: do not modify it void * gfal_data; //! @endcond // plugin management /** * plugin reserved pointer, free to use for plugin's internal data, passed to any function * * */ plugin_handle plugin_data; /** * plugin priority * SHOULD be defined to GFAL_PLUGIN_PRIORITY_DATA by default * */ int priority; /** * MANDATORY : return a the string id of the plugin. * string id must be short, constant and unique ( ex : "plugin_gridftp" ) * @return the constant identity string * */ const char* (*getName)(); /** * OPTIONAL : Last call before the unload of the plugin for the associated handle * You can use this to clear your internal context * @param plugin_data : internal plugin data */ void (*plugin_delete)(plugin_handle plugin_data); /** * OPTIONAL: Check the availability of the given operation on this plugin for the given URL * * @warning This function is a key function of GFAL 2.0, It MUST be as fast as possible. * @param plugin_data : internal plugin data * @param url : URL to check for the protocol compatibility * @param operation : operation to check * @param err : error handle, should be used ONLY in case of major failure. * @return must return TRUE if the operation is compatible with this plugin, else FALSE */ gboolean (*check_plugin_url)(plugin_handle plugin_data, const char* url, plugin_mode operation, GError** err); // FILE API /** * OPTIONAL : gfal_access function support * @param plugin_data : internal plugin data * @param url : URL for access checking * @param mode : mode to check ( see man 2 access ) * @param err : Error report, the code field of err should be set to errno value when possible * @return 0 or -1 if error occurs, * err MUST be set in case of error * */ int (*accessG)(plugin_handle plugin_data, const char* url, int mode, GError** err); /** * OPTIONAL : gfal_chmod function support * @param plugin_data : internal plugin data * @param url : URL of the file * @param mode : mode to set * @param err : Error report, the code field of err should be set to errno value when possible * @return 0 or -1 if error occurs, * err MUST be set in case of error * */ int (*chmodG)(plugin_handle plugin_data, const char * url, mode_t mode, GError** err); /** * OPTIONAL : gfal_rename function support * @param plugin_data : internal plugin data * @param oldurl : old url of the file * @param urlnew : new url of the file * @param err : Error report, the code field of err should be set to errno value when possible * @return 0 or -1 if error occurs, * err MUST be set in case of error * */ int (*renameG)(plugin_handle plugin_data, const char * oldurl, const char * urlnew, GError** err); /** * OPTIONAL : gfal_symlink function support * @param plugin_data : internal plugin data * @param oldurl : old url of the file * @param urlnew : symlink to create * @param err : Error report, the code field of err should be set to errno value when possible * @return 0 or -1 if error occurs, * err MUST be set in case of error * */ int (*symlinkG)(plugin_handle plugin_data, const char* oldurl, const char* newold, GError** err); /** * OPTIONAL : gfal_stat function support * @param plugin_data : internal plugin data * @param url : url to stat * @param buf : informations of the file * @param err : Error report, the code field of err should be set to errno value when possible * @return 0 or -1 if error occurs, * err MUST be set in case of error * */ int (*statG)(plugin_handle plugin_data , const char* url, struct stat *buf, GError** err); /** * OPTIONAL : gfal_lstat function support * In case of non support for this function, calls to @ref gfal_lstat are mapped to @ref gfal_stat. * * @param plugin_data : internal plugin data * @param url : url to stat * @param buf : informations of the file * @param err : Error report, the code field of err should be set to errno value when possible * @return 0 or -1 if error occurs, * err MUST be set in case of error * */ int (*lstatG)(plugin_handle plugin_data, const char* url, struct stat *buf, GError** err); /** * OPTIONAL : gfal_readlink function support * * @param plugin_data : internal plugin data * @param url : url to stat * @param buff : buffer for the readlink result * @param size_t : buffsize maximum size to fill in the buffer * @param err : Error report, the code field of err should be set to errno value when possible * @return number of bytes in buff in case of success or -1 if error occurs, * err MUST be set in case of error * */ ssize_t (*readlinkG)(plugin_handle plugin_data, const char* url, char* buff, size_t buffsiz, GError** ); /** * OPTIONAL : gfal_opendir function support * * @param plugin_data : internal plugin data * @param url : url of directory to list * @param err : Error report, the code field of err should be set to errno value when possible * @return gfa_file_handle in case of success or NULL if error, * err MUST be set in case of error * */ gfal_file_handle (*opendirG)(plugin_handle plugin_data, const char* url, GError** err); /** * MANDATORY IF OPENDIR : gfal_closedir function support * * @param plugin_data : internal plugin data * @param dir_desc : directory descriptor to use * @param err : Error report, the code field of err should be set to errno value when possible * @return 0 in case of success or -1 if error, * err MUST be set in case of error * */ int (*closedirG)(plugin_handle plugin_data, gfal_file_handle dir_desc, GError** err); /** * MANDATORY IF OPENDIR : gfal_readdir function support * * @param plugin_data : internal plugin data * @param dir_desc : directory descriptor to use * @param err : Error report, the code field of err should be set to errno value when possible * @return dirent information in case of success or NULL if end of listing or error, * err MUST be set in case of error * */ struct dirent* (*readdirG)(plugin_handle plugin_data, gfal_file_handle dir_desc, GError** err); /** * OPTIONAL : gfal_mkdir function support * * @param plugin_data : internal plugin data * @param url : url of the directory to create * @param mode : right mode of the created directory * @param rec_flag : recursive mode, if enabled the plugin MUST create the parent directories if needed, * if the rec_flag is not supported by this plugin, the plugin MUST return a negative value and set the GError errcode to ENOENT * @param err : Error report, the code field of err should be set to errno value when possible * @return 0 in case of success or -1 if error occurs, * err MUST be set in case of error * */ int (*mkdirpG)(plugin_handle plugin_data, const char* url, mode_t mode, gboolean rec_flag, GError** err); /** * OPTIONAL : gfal_rmdir function support * * @param plugin_data : internal plugin data * @param url : url of the directory to delete * @param err : Error report, the code field of err should be set to errno value when possible * @return 0 in case of success or -1 if error occurs, * err MUST be set in case of error * */ int (*rmdirG)(plugin_handle plugin_data, const char* url, GError** err); // basic file operations /** * OPTIONAL : gfal_open function support * * @param plugin_data : internal plugin data * @param url : url of the directory to open * @param flag : open flags * @param mode : mode of the file, in case of creation * @param err : Error report, the code field of err should be set to errno value when possible * @return gfal_file_handle in case of success or NULL if error occurs, * err MUST be set in case of error * */ gfal_file_handle (*openG)(plugin_handle plugin_data, const char* url, int flag, mode_t mode, GError**); /** * MANDATORY IF OPEN : gfal_read function support * * */ ssize_t (*readG)(plugin_handle, gfal_file_handle fd, void* buff, size_t count, GError**); /** * MANDATORY IF OPEN : gfal_write function support * * */ ssize_t (*writeG)(plugin_handle, gfal_file_handle fd, const void* buff, size_t count, GError**); /** * MANDATORY IF OPEN : gfal_close function support * * */ int (*closeG)(plugin_handle, gfal_file_handle fd, GError **); /** * MANDATORY IF OPEN : gfal_lseek function support * * */ off_t (*lseekG)(plugin_handle, gfal_file_handle fd, off_t offset, int whence, GError** err); // vector operations /** * OPTIONAL : gfal_pread function support * * Allow fast parallels read support, If not implemented, this function is simulated by GFAL 2.0 * * */ ssize_t (*preadG)(plugin_handle, gfal_file_handle fd, void* buff, size_t count, off_t offset, GError**); /** * OPTIONAL : gfal_pwriteG function support * * Allow fast parallels write support, If not implemented, this function is simulated by GFAL 2.0 * * */ ssize_t (*pwriteG)(plugin_handle, gfal_file_handle fd, const void* buff, size_t count, off_t offset, GError**); // remove operations /** * OPTIONAL : gfal_unlink function support * * @param plugin_data : internal plugin data * @param url : url of the file * @param err : Error report, the code field of err should be set to errno value when possible * @return 0 in case of success or -1 if error occurs, * err MUST be set in case of error * */ int (*unlinkG)(plugin_handle plugin_data, const char* url, GError** err); // advanced attributes management /** * OPTIONAL : gfal_getxattr function support * * @param plugin_data : internal plugin data * @param url : url of the file * @param key : key of the attribute to get * @param buff : buffer for the attribute content * @param s_buff : maximum buffer size * @param err : Error report, the code field of err should be set to errno value when possible * @return size of the attribute in case of success or -1 if error occurs, * err MUST be set in case of error * */ ssize_t (*getxattrG)(plugin_handle plugin_data, const char* url, const char* key, void* buff, size_t s_buff, GError** err); /** * OPTIONAL : gfal_listxattr function support * * @param plugin_data : internal plugin data * @param url : url of the file * @param list : buffer for the list attribute content * @param s_buff : maximum buffer size * @param s_list : Error report, the code field of err should be set to errno value when possible * @return size of the list in case of success or -1 if error occurs, * err MUST be set in case of error * */ ssize_t (*listxattrG)(plugin_handle plugin_data, const char* url, char* list, size_t s_list, GError** err); /** * OPTIONAL : gfal_setxattr function support * * @param plugin_data : internal plugin data * @param url : url of the file * @param key : key of the attribute to set * @param buff : buffer for the attribute content * @param s_buff : maximum buffer size * @param flas : set/get flags * @param err : Error report, the code field of err should be set to errno value when possible * @return 0 or -1 if error occurs, * err MUST be set in case of error * */ int (*setxattrG)(plugin_handle plugin_data, const char* url, const char* key, const void* buff , size_t s_buff, int flags, GError** err); /** * OPTIONAL : checksum calculation function support ( transfer consistency check, gfal_checksum ) * * @param plugin_data : internal plugin data * @param url : url of the file * @param check_type : string of the checksum type ( \ref GFAL_CHKSUM_MD5, \ref GFAL_CHKSUM_SHA1, .. ) * @param start_offset : offset in the file where the checksum calculation will start ( 0 : from begining ) * @param data_length : size of data to compute for the checksum after start_offset ( 0 -: full file ) * @param checksum_buffer : buffer with checksum string as result * @param buffer_length : maximum buffer length * @param err : GError error support * @return dynamically allocated checksum string if success, else NULL and err MUST be set * error code MUST be ENOSYS in case of : * - checksum calculation with offset is not supported * - the specified checksum algorithm is not supported */ int (*checksum_calcG)(plugin_handle data, const char* url, const char* check_type, char * checksum_buffer, size_t buffer_length, off_t start_offset, size_t data_length, GError ** err); // TRANSFER API /** * OPTIONAL: if transfer support, * should return TRUE if the plugin is able to execute third party transfer from src to dst url * */ int(*check_plugin_url_transfer)(plugin_handle plugin_data, gfal2_context_t, const char* src, const char* dst, gfal_url2_check check); /** * * OPTIONAL: if transfer support, * Execute a filecopy operation for the given parameters * @param plugin_data : internal plugin context * @param gfal2_context_t context : gfal 2 handle * @param params: parameters for the current transfer, see gfalt_params calls * @param src : source file to copy * @param dst : destination file * @param err : GError err report * */ int (*copy_file)(plugin_handle plugin_data, gfal2_context_t context, gfalt_params_t params, const char* src, const char* dst, GError** ); /** * * OPTIONAL: Requests to stage a file to the fist layer on a hierarchical SE. * @param plugin_data : internal plugin context * @param url : The url of the file * @param pintime : Time the file should stay in the cache * @param timeout : Operation timeout * @param token Where to put the retrieved token. * @param tsize The size of the buffer pointed by token. * @param async If true (!= 0), the call will not block. The caller will need * to use bring_online_poll later. * @param err: GError error support * @return -1 on error (and err is set). 0 on success. 1 if the file has been staged. */ int (*bring_online)(plugin_handle plugin_data, const char* url, time_t pintime, time_t timeout, char* token, size_t tsize, int async, GError** err); /** * * OPTIONAL: Requests to stage a file to the fist layer on a hierarchical SE. * @param plugin_data : internal plugin context * @param url : The url of the file * @param metadata : Staging metadata * @param pintime : Time the file should stay in the cache * @param timeout : Operation timeout * @param token Where to put the retrieved token. * @param tsize The size of the buffer pointed by token. * @param async If true (!= 0), the call will not block. The caller will need * to use bring_online_poll later. * @param err: GError error support * @return -1 on error (and err is set). 0 on success. 1 if the file has been staged. */ int (*bring_online_v2)(plugin_handle plugin_data, const char* url, const char* metadata, time_t pintime, time_t timeout, char* token, size_t tsize, int async, GError** err); /** * OPTIONAL: Polling the bring_online request (mandatory if bring online is supported) * @param url The same URL as was passed to bring_online_async * @param token The token as returned by bring_online_async * @return -1 on error (and err is set). 0 on success. 1 if the file has been staged. */ int (*bring_online_poll)(plugin_handle plugin_data, const char* url, const char* token, GError** err); /** * OPTIONAL: Releases a previously staged file (mandatory if bring online is supported) * @param plugin_data : internal plugin context * @param url : The url of the file * @param token: The request token. If NULL, * @param err: GError error support */ int (*release_file)(plugin_handle plugin_data, const char* url, const char* token, GError** err); /** * OPTIONAL : gfal_readdirpp function support * Allow directory listing + get meta-data in one operation * * @param plugin_data : internal plugin data * @param dir_desc : directory descriptor to use * @param st : struct stat to fill * @param err : Error report, the code field of err should be set to errno value when possible * @return dirent information in case of success or NULL if end of listing or error, * err MUST be set in case of error * */ struct dirent* (*readdirppG)(plugin_handle plugin_data, gfal_file_handle dir_desc, struct stat* st, GError** err); /** * * OPTIONAL: Requests to stage a file to the fist layer on a hierarchical SE. * @param plugin_data : internal plugin context * @param nbfiles : number of files * @param urls : The urls of the files * @param pintime : Time the file should stay in the cache * @param timeout : Operation timeout * @param token Where to put the retrieved token. * @param tsize The size of the buffer pointed by token. * @param async If true (!= 0), the call will not block. The caller will need * to use bring_online_poll later. * @param err: GError error support * @return -1 on error (and err is set). 0 on success. 1 if the file has been staged. */ int (*bring_online_list)(plugin_handle plugin_data, int nbfiles, const char* const* urls, time_t pintime, time_t timeout, char* token, size_t tsize, int async, GError** err); /** * * OPTIONAL: Requests to stage a file to the fist layer on a hierarchical SE. * @param plugin_data : internal plugin context * @param nbfiles : number of files * @param urls : The urls of the files * @param metadata : Staging metadata array * @param pintime : Time the file should stay in the cache * @param timeout : Operation timeout * @param token Where to put the retrieved token. * @param tsize The size of the buffer pointed by token. * @param async If true (!= 0), the call will not block. The caller will need * to use bring_online_poll later. * @param err: GError error support * @return -1 on error (and err is set). 0 on success. 1 if the file has been staged. */ int (*bring_online_list_v2)(plugin_handle plugin_data, int nbfiles, const char* const* urls, const char* const* metadata, time_t pintime, time_t timeout, char* token, size_t tsize, int async, GError** err); /** * OPTIONAL: Polling the bring_online request (mandatory if bring online is supported) * @param nbfiles : number of files * @param urls The same URLs as were passed to bring_online_async * @param token The token as returned by bring_online_async * @return -1 on error (and err is set). 0 on success. 1 if the file has been staged. */ int (*bring_online_poll_list)(plugin_handle plugin_data, int nbfiles, const char* const* urls, const char* token, GError** err); /** * OPTIONAL: Releases a previously staged file (mandatory if bring online is supported) * @param plugin_data : internal plugin context * @param nbfiles : number of files in the list * @param url : The urls of the files * @param token: The request token. If NULL, * @param err: GError error support */ int (*release_file_list)(plugin_handle plugin_data, int nbfiles, const char* const* urls, const char* token, GError** err); /** * OPTIONAL: Bulk deletion */ int (*unlink_listG)(plugin_handle plugin_data, int nbfiles, const char* const* uris, GError ** errors); /** * OPTIONAL: allows clients to abort selective file requests from the asynchronous requests of any type * @param plugin_data : internal plugin context * @param nbfiles : number of files in the list * @param url : The urls of the files * @param token: The request token * @param err: GError error support */ int(*abort_files)(plugin_handle handle, int nbfiles, const char* const* uris, const char* token, GError ** err); /** * OPTIONAL: copy nbfiles files * @param plugin_data : internal plugin context * @param context : gfal2 context * @param params: gfal2 transfer parameters * @param nbfiles: how many files are to be transferred * @param srcs: array of nbfiles sources * @param dsts: array of nbfiles destinations * @param checkums: array of nbfiles checksums. it can be NULL * @param op_error: Operation error * @param file_errors: Per file error */ int (*copy_bulk)(plugin_handle plugin_data, gfal2_context_t context, gfalt_params_t params, size_t nbfiles, const char* const* srcs, const char* const* dsts, const char* const* checksums, GError** op_error, GError*** file_errors); /** * OPTIONAL: executed by the core before entering a copy, so a plugin can install its own * event listeners. */ int (*copy_enter_hook)(plugin_handle plugin_data, gfal2_context_t context, gfalt_params_t params, GError** error); // QoS API /** * OPTIONAL: Check what kind of QoS classes exist on the CDMI-enabled URL provided * * @param plugin_data : internal plugin data * @param url : CDMI-enabled URL to check for the protocol compatibility * @param type : CDMI-enabld type of QoS class * @param buff : buffer for the QoS classes content * @param s_buff : maximum buffer size * @param err : error handle, should be used ONLY in case of major failure. * @return number of bytes in buff in case of success or -1 if error occurs * * @note Result is provided as a comma-separated list of available QoS classes */ ssize_t (*check_qos_classes)(plugin_handle plugin_data, const char* url, const char* type, char* buff, size_t s_buff, GError** err); /** * OPTIONAL: Check the QoS of a file with the CDMI-enabled url provided * * @param plugin_data : internal plugin data * @param url : CDMI-enabled URL to check for the protocol compatibility * @param buff : buffer for the QoS class content * @param s_buff : maximum buffer size * @param err : error handle, should be used ONLY in case of major failure. * @return number of bytes in buff in case of success or -1 if error occurs */ ssize_t (*check_file_qos)(plugin_handle plugin_data, const char* url, char* buff, size_t s_buff, GError** err); /** * OPTIONAL: Check the available transitions of a QoS class to other QoS classes * * @param plugin_data : internal plugin data * @param qos_class_url : CDMI-enabled URL of QoS class * @param buff : buffer for the QoS class transitions content * @param s_buff : maximum buffer size * @param err : error handle, should be used ONLY in case of major failure. * @return number of bytes in buff in case of success or -1 if error occurs * * @note Result is provided as comma-separated list of available QoS class transitions */ ssize_t (*check_qos_available_transitions)(plugin_handle plugin_data, const char* qos_class_url, char* buff, size_t s_buff, GError** err); /** * OPTIONAL: Check the target QoS of an entity * * @param plugin_data : internal plugin data * @param url : CDMI-enabled URL of an entity * @param buff : buffer for the target QoS class content * @param s_buff : maximum buffer size * @param err : error handle, should be used ONLY in case of major failure. * @return number of bytes in buff in case of success or -1 if error occurs */ ssize_t (*check_target_qos)(plugin_handle plugin_data, const char* url, char* buff, size_t s_buff, GError** err); /** * OPTIONAL: Check the QoS of a file with the CDMI-enabled url provided * * @param plugin_data : internal plugin data * @param url : CDMI-enabled URL of an entity * @param target_qos: the requested target QoS class * @param err : error handle, should be used ONLY in case of major failure. * @return 0 or -1 if error occurs */ int (*change_object_qos)(plugin_handle plugin_data, const char* url, const char* target_qos, GError** err); // ARCHIVE API /** * OPTIONAL: Polling the archive request (mandatory if archiving is supported) * * @param plugin_data: internal plugin data * @param url: the URL for which to check archive status * @param err: error handle * @return -1 on error (and err is set). 0 on success. 1 if the file has been archived. */ int (*archive_poll)(plugin_handle plugin_data, const char* url, GError** err); /** * OPTIONAL: Polling the archive request (mandatory if archiving is supported) * * @param plugin_data: internal plugin data * @param nbfiles: number of files * @param urls: the URLs for which to check archive status * @param err: error handle * @return -1 on error (and err is set). 0 on success. 1 if the files have been archived. * 2 if some files have bene archived, others encountered errors */ int (*archive_poll_list)(plugin_handle plugin_data, int nbfiles, const char* const* urls, GError** err); // TOKEN API /** * OPTIONAL: Retrieve a token from Storage Element for a given resource * * @param plugin_data: internal plugin data * @param url: the URL of the resource * @param issuer: the token issuer endpoint (optional) * @param write_access: write access flag * @param validity: token validity time in minutes * @param activities: array of activities for access request * @param buff : buffer for the token content * @param s_buff : maximum buffer size * @param err : error handle * @return number of bytes in buff in case of success or -1 if error occurs */ ssize_t (*token_retrieve)(plugin_handle plugin_data, const char* url, const char* issuer, gboolean write_access, unsigned validity, const char* const* activities, char* buff, size_t s_buff, GError** err); // reserved for future usage //! @cond void* future[4]; //! @endcond }; /** * Trigger plugin instantiation from the client code * The passed interface is copied, so you can, if needed, free it after the call */ int gfal2_register_plugin(gfal2_context_t handle, const gfal_plugin_interface* ifce, GError** error); // internal API for inter plugin communication //! @cond int gfal_plugins_accessG(gfal2_context_t handle, const char* path, int mode, GError** err); int gfal_plugin_rmdirG(gfal2_context_t handle, const char* path, GError** err); ssize_t gfal_plugin_readlinkG(gfal2_context_t handle, const char* path, char* buff, size_t buffsiz, GError** err); int gfal_plugin_chmodG(gfal2_context_t handle, const char* path, mode_t mode, GError** err); int gfal_plugin_statG(gfal2_context_t handle,const char* path, struct stat* st, GError** err); int gfal_plugin_renameG(gfal2_context_t handle, const char* oldpath, const char* newpath, GError** err); int gfal_plugin_symlinkG(gfal2_context_t handle, const char* oldpath, const char* newpath, GError** err); int gfal_plugin_lstatG(gfal2_context_t handle,const char* path, struct stat* st, GError** err); int gfal_plugin_mkdirp(gfal2_context_t handle, const char* path, mode_t mode, gboolean pflag, GError** err); gfal_file_handle gfal_plugin_opendirG(gfal2_context_t handle, const char* name, GError** err); struct dirent* gfal_plugin_readdirppG(gfal2_context_t handle, gfal_file_handle fh, struct stat* st, GError** err); int gfal_plugin_closedirG(gfal2_context_t handle, gfal_file_handle fh, GError** err); struct dirent* gfal_plugin_readdirG(gfal2_context_t handle, gfal_file_handle fh, GError** err); gfal_file_handle gfal_plugin_openG(gfal2_context_t handle, const char * path, int flag, mode_t mode, GError ** err); int gfal_plugin_closeG(gfal2_context_t handle, gfal_file_handle fh, GError** err); int gfal_plugin_writeG(gfal2_context_t handle, gfal_file_handle fh, void* buff, size_t s_buff, GError** err); int gfal_plugin_lseekG(gfal2_context_t handle, gfal_file_handle fh, off_t offset, int whence, GError** err); int gfal_plugin_readG(gfal2_context_t handle, gfal_file_handle fh, void* buff, size_t s_buff, GError** err); ssize_t gfal_plugin_preadG(gfal2_context_t handle, gfal_file_handle fh, void* buff, size_t s_buff, off_t offset, GError** err); ssize_t gfal_plugin_pwriteG(gfal2_context_t handle, gfal_file_handle fh, void* buff, size_t s_buff, off_t offset, GError** err); int gfal_plugin_unlinkG(gfal2_context_t handle, const char* path, GError** err); ssize_t gfal_plugin_getxattrG(gfal2_context_t, const char*, const char*, void* buff, size_t s_buff, GError** err); ssize_t gfal_plugin_listxattrG(gfal2_context_t, const char*, char* list, size_t s_list, GError** err); int gfal_plugin_setxattrG(gfal2_context_t, const char*, const char*, const void*, size_t, int, GError**); int gfal_plugin_bring_onlineG(gfal2_context_t handle, const char* uri, time_t pintime, time_t timeout, char* token, size_t tsize, int async, GError ** err); int gfal_plugin_bring_online_v2G(gfal2_context_t handle, const char* uri, const char* metadata, time_t pintime, time_t timeout, char* token, size_t tsize, int async, GError ** err); int gfal_plugin_bring_online_pollG(gfal2_context_t handle, const char* uri, const char* token, GError ** err); int gfal_plugin_release_fileG(gfal2_context_t handle, const char* uri, const char* token, GError ** err); int gfal_plugin_bring_online_listG(gfal2_context_t handle, int nbfiles, const char* const* uris, time_t pintime, time_t timeout, char* token, size_t tsize, int async, GError ** err); int gfal_plugin_bring_online_list_v2G(gfal2_context_t handle, int nbfiles, const char* const* uris, const char* const* metadata, time_t pintime, time_t timeout, char* token, size_t tsize, int async, GError ** err); int gfal_plugin_bring_online_poll_listG(gfal2_context_t handle, int nbfiles, const char* const* uris, const char* token, GError ** err); int gfal_plugin_release_file_listG(gfal2_context_t handle, int nbfiles, const char* const* uris, const char* token, GError ** err); int gfal_plugin_unlink_listG(gfal2_context_t handle, int nbfiles, const char* const* uris, GError ** errors); int gfal_plugin_abort_filesG(gfal2_context_t handle, int nbfiles, const char* const* uris, const char* token, GError ** err); ssize_t gfal_plugin_qos_check_classes(gfal2_context_t handle, const char* url, const char* type, char* buff, size_t s_buff, GError** err); ssize_t gfal_plugin_check_file_qos(gfal2_context_t handle, const char* url, char* buff, size_t s_buff, GError** err); ssize_t gfal_plugin_check_qos_available_transitions(gfal2_context_t handle, const char* qos_class_url, char* buff, size_t s_buff, GError** err); ssize_t gfal_plugin_check_target_qos(gfal2_context_t handle, const char* url, char* buff, size_t s_buff, GError** err); int gfal_plugin_change_object_qos(gfal2_context_t handle, const char* url, const char* target_qos, GError** err); int gfal_plugin_archive_pollG(gfal2_context_t handle, const char* uri, GError ** err); int gfal_plugin_archive_poll_listG(gfal2_context_t handle, int nbfiles, const char* const* uris, GError ** err); ssize_t gfal_plugin_token_retrieveG(gfal2_context_t handle, const char* url, const char* issuer, gboolean write_access, unsigned validity, const char* const* activities, char* buff, size_t s_buff, GError** err); //! @endcond #ifdef __cplusplus } #endif #endif /* GFAL_COMMON_PLUGIN_INTERFACE_H_ */ gfal2-v2.23.0/src/core/file/000077500000000000000000000000001465240014500154165ustar00rootroot00000000000000gfal2-v2.23.0/src/core/file/gfal2_directory_operations.c000066400000000000000000000127661465240014500231200ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include #include #ifdef __APPLE__ static char* mempcpy(char* dst, const char* src, size_t len) { memcpy(src, dst, len); return dst + len; } #endif static int gfal_rw_dir_handle_store(gfal2_context_t handle, gfal_file_handle fhandle, GError **err) { g_return_val_err_if_fail(handle && fhandle, 0, err, "[gfal_rw_dir_handle_store] handle invalid"); GError *tmp_err = NULL; int key = 0; key = gfal_add_new_file_desc(handle->fdescs, (gpointer) fhandle, &tmp_err); G_RETURN_ERR(key, tmp_err, err); } DIR *gfal2_opendir(gfal2_context_t handle, const char *name, GError **err) { GError *tmp_err = NULL; gfal_file_handle ret = NULL; GFAL2_BEGIN_SCOPE_CANCEL(handle, NULL, err); if (name == NULL || handle == NULL) { g_set_error(&tmp_err, gfal2_get_core_quark(), EFAULT, "uri or/and handle are NULL"); } else { ret = gfal_plugin_opendirG(handle, name, &tmp_err); } int key = 0; if (ret) { key = gfal_rw_dir_handle_store(handle, ret, &tmp_err); } GFAL2_END_SCOPE_CANCEL(handle); G_RETURN_ERR(GINT_TO_POINTER(key), tmp_err, err); } struct dirent *gfal2_readdir(gfal2_context_t handle, DIR *dir, GError **err) { GError *tmp_err = NULL; struct dirent *res = NULL; GFAL2_BEGIN_SCOPE_CANCEL(handle, NULL, err); if (dir == NULL || handle == NULL) { g_set_error(&tmp_err, gfal2_get_core_quark(), EFAULT, "file descriptor or/and handle are NULL"); } else { const int key = GPOINTER_TO_INT(dir); gfal_file_handle fh = gfal_file_handle_bind(handle->fdescs, key, &tmp_err); if (fh != NULL) { res = gfal_plugin_readdirG(handle, fh, &tmp_err); } } GFAL2_END_SCOPE_CANCEL(handle); G_RETURN_ERR(res, tmp_err, err); } static size_t gfal_rw_get_root_length(const char *surl) { regex_t rx; regmatch_t matches[1]; regcomp(&rx, "(\\w+://[^/]*/)", REG_EXTENDED); if (regexec(&rx, surl, 1, matches, 0) == 0) { return matches[0].rm_eo - matches[0].rm_so; } else { return 0; } } static struct dirent * gfal_rw_gfalfilehandle_readdirpp(gfal2_context_t context, gfal_file_handle fh, struct stat *st, GError **err) { g_return_val_err_if_fail(context && fh, NULL, err, "[gfal_posix_gfalfilehandle_readdirpp] incorrect args"); GError *tmp_err = NULL; struct dirent *ret = gfal_plugin_readdirppG(context, fh, st, &tmp_err); // try to simulate readdirpp if (tmp_err && tmp_err->code == EPROTONOSUPPORT && fh->path != NULL) { g_clear_error(&tmp_err); ret = gfal_plugin_readdirG(context, fh, &tmp_err); if (!tmp_err && ret != NULL) { char *url = NULL; if (ret->d_name[0] != '/') { url = g_strconcat(fh->path, "/", ret->d_name, NULL); } else { size_t root_len = gfal_rw_get_root_length(fh->path); char *root = g_strndup(fh->path, root_len); url = g_strconcat(root, ret->d_name, NULL); g_free(root); } if (gfal2_stat(context, url, st, &tmp_err) < 0) { ret = NULL; } g_free(url); } } G_RETURN_ERR(ret, tmp_err, err); } struct dirent *gfal2_readdirpp(gfal2_context_t context, DIR *dir, struct stat *st, GError **err) { GError *tmp_err = NULL; struct dirent *res = NULL; GFAL2_BEGIN_SCOPE_CANCEL(context, NULL, err); if (dir == NULL || context == NULL) { g_set_error(&tmp_err, gfal2_get_core_quark(), EFAULT, "file descriptor or/and handle are NULL"); } else { const int key = GPOINTER_TO_INT(dir); gfal_file_handle fh = gfal_file_handle_bind(context->fdescs, key, &tmp_err); if (fh != NULL) { res = gfal_rw_gfalfilehandle_readdirpp(context, fh, st, &tmp_err); } } GFAL2_END_SCOPE_CANCEL(context); G_RETURN_ERR(res, tmp_err, err); } int gfal2_closedir(gfal2_context_t handle, DIR *d, GError **err) { GError *tmp_err = NULL; int ret = -1; if (d == NULL || handle == NULL) { g_set_error(&tmp_err, gfal2_get_core_quark(), EFAULT, "file descriptor or/and handle are NULL"); } else { int key = GPOINTER_TO_INT(d); gfal_file_handle fh = gfal_file_handle_bind(handle->fdescs, key, &tmp_err); if (fh != NULL) { ret = gfal_plugin_closedirG(handle, fh, &tmp_err); if (ret == 0) { ret = (gfal_remove_file_desc(handle->fdescs, key, &tmp_err)) ? 0 : -1; } } } G_RETURN_ERR(ret, tmp_err, err); } gfal2-v2.23.0/src/core/file/gfal2_read_write_file_operations.c000066400000000000000000000145411465240014500242310ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include /* * store a gfal_file_handle in the base, in a key/value model * the key, else 0 if error occured and err is set correctly */ static int gfal_rw_file_handle_store(gfal2_context_t handle, gfal_file_handle fhandle, GError **err) { g_return_val_err_if_fail(handle && fhandle, -1, err, "[gfal_rw_file_handle_store] invalid args"); GError *tmp_err = NULL; int key = 0; key = gfal_add_new_file_desc(handle->fdescs, (gpointer) fhandle, &tmp_err); G_RETURN_ERR(key, tmp_err, err); } int gfal2_open(gfal2_context_t handle, const char *uri, int flag, GError **err) { return gfal2_open2(handle, uri, flag, (S_IRWXU | S_IRGRP | S_IROTH), err); } int gfal2_open2(gfal2_context_t handle, const char *uri, int flag, mode_t mode, GError **err) { GError *tmp_err = NULL; gfal_file_handle fhandle = NULL; int key = -1; GFAL2_BEGIN_SCOPE_CANCEL(handle, -1, err); gfal2_log(G_LOG_LEVEL_DEBUG, "%s ->", __func__); if (uri == NULL || handle == NULL) { g_set_error(&tmp_err, gfal2_get_core_quark(), EFAULT, "name is empty"); } else { fhandle = gfal_plugin_openG(handle, uri, flag, mode, &tmp_err); } if (fhandle) { key = gfal_rw_file_handle_store(handle, fhandle, &tmp_err); } GFAL2_END_SCOPE_CANCEL(handle); G_RETURN_ERR(key, tmp_err, err); } int gfal2_creat(gfal2_context_t handle, const char *filename, mode_t mode, GError **err) { return (gfal2_open2(handle, filename, (O_WRONLY | O_CREAT | O_TRUNC), mode, err)); } ssize_t gfal2_read(gfal2_context_t handle, int fd, void *buff, size_t s_buff, GError **err) { GError *tmp_err = NULL; int res = -1; GFAL2_BEGIN_SCOPE_CANCEL(handle, -1, err); if (fd <= 0 || handle == NULL) { g_set_error(&tmp_err, gfal2_get_core_quark(), EBADF, "Incorrect file descriptor or incorrect handle"); } else { const int key = fd; gfal_file_handle fh = gfal_file_handle_bind(handle->fdescs, key, &tmp_err); if (fh != NULL) { res = gfal_plugin_readG(handle, fh, buff, s_buff, &tmp_err); } } GFAL2_END_SCOPE_CANCEL(handle); G_RETURN_ERR(res, tmp_err, err); } int gfal2_close(gfal2_context_t handle, int fd, GError **err) { GError *tmp_err = NULL; int ret = -1; if (fd <= 0 || handle == NULL) { g_set_error(&tmp_err, gfal2_get_core_quark(), EBADF, "Incorrect file descriptor or incorrect handle"); } else { int key = GPOINTER_TO_INT(fd); gfal_file_handle fh = gfal_file_handle_bind(handle->fdescs, key, &tmp_err); if (fh != NULL) { ret = gfal_plugin_closeG(handle, fh, &tmp_err); if (ret == 0) { ret = (gfal_remove_file_desc(handle->fdescs, key, &tmp_err)) ? 0 : -1; } } } G_RETURN_ERR(ret, tmp_err, err); } off_t gfal2_lseek(gfal2_context_t handle, int fd, off_t offset, int whence, GError **err) { GError *tmp_err = NULL; int res = -1; GFAL2_BEGIN_SCOPE_CANCEL(handle, -1, err); if (fd <= 0 || handle == NULL) { g_set_error(&tmp_err, gfal2_get_core_quark(), EBADF, "Incorrect file descriptor"); } else { const int key = fd; gfal_file_handle fh = gfal_file_handle_bind(handle->fdescs, key, &tmp_err); if (fh != NULL) { res = gfal_plugin_lseekG(handle, fh, offset, whence, &tmp_err); } } GFAL2_END_SCOPE_CANCEL(handle); G_RETURN_ERR(res, tmp_err, err); } int gfal2_flush(gfal2_context_t handle, int fd, GError **err) { return 0; } ssize_t gfal2_pread(gfal2_context_t handle, int fd, void *buff, size_t s_buff, off_t offset, GError **err) { GError *tmp_err = NULL; ssize_t res = -1; GFAL2_BEGIN_SCOPE_CANCEL(handle, -1, err); if (fd <= 0 || handle == NULL) { g_set_error(&tmp_err, gfal2_get_core_quark(), EBADF, "Incorrect file descriptor or incorrect handle"); } else { const int key = fd; gfal_file_handle fh = gfal_file_handle_bind(handle->fdescs, key, &tmp_err); if (fh != NULL) { res = gfal_plugin_preadG(handle, fh, buff, s_buff, offset, &tmp_err); } } GFAL2_END_SCOPE_CANCEL(handle); G_RETURN_ERR(res, tmp_err, err); } ssize_t gfal2_write(gfal2_context_t handle, int fd, const void *buff, size_t s_buff, GError **err) { GError *tmp_err = NULL; int res = -1; GFAL2_BEGIN_SCOPE_CANCEL(handle, -1, err); if (fd <= 0 || handle == NULL) { g_set_error(&tmp_err, gfal2_get_core_quark(), EBADF, "Incorrect file descriptor or incorrect handle"); } else { const int key = fd; gfal_file_handle fh = gfal_file_handle_bind(handle->fdescs, key, &tmp_err); if (fh != NULL) { res = gfal_plugin_writeG(handle, fh, (void *) buff, s_buff, &tmp_err); } } GFAL2_END_SCOPE_CANCEL(handle); G_RETURN_ERR(res, tmp_err, err); } ssize_t gfal2_pwrite(gfal2_context_t handle, int fd, const void *buff, size_t s_buff, off_t offset, GError **err) { GError *tmp_err = NULL; int res = -1; GFAL2_BEGIN_SCOPE_CANCEL(handle, -1, err); if (fd <= 0) { g_set_error(&tmp_err, gfal2_get_core_quark(), EBADF, "Incorrect file descriptor"); } else { const int key = fd; gfal_file_handle fh = gfal_file_handle_bind(handle->fdescs, key, &tmp_err); if (fh != NULL) { res = gfal_plugin_pwriteG(handle, fh, (void *) buff, s_buff, offset, &tmp_err); } } GFAL2_END_SCOPE_CANCEL(handle); G_RETURN_ERR(res, tmp_err, err); } gfal2-v2.23.0/src/core/file/gfal2_standard_file_operations.c000066400000000000000000000601661465240014500237100ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include #include int gfal2_access(gfal2_context_t context, const char *url, int amode, GError **err) { int res = -1; GError *tmp_err = NULL; GFAL2_BEGIN_SCOPE_CANCEL(context, -1, err); if (url == NULL || context == NULL) { g_set_error(&tmp_err, gfal2_get_core_quark(), EFAULT, "context or/and url are incorrect arguments"); } else { res = gfal_plugins_accessG(context, (char *) url, amode, &tmp_err); } GFAL2_END_SCOPE_CANCEL(context); G_RETURN_ERR(res, tmp_err, err); } int gfal2_chmod(gfal2_context_t context, const char *url, mode_t mode, GError **err) { GError *tmp_err = NULL; int res = -1; GFAL2_BEGIN_SCOPE_CANCEL(context, -1, err); if (url == NULL || context == NULL) { g_set_error(&tmp_err, gfal2_get_core_quark(), EFAULT, "context or/and url are incorrect arguments"); } else { res = gfal_plugin_chmodG(context, url, mode, &tmp_err); } GFAL2_END_SCOPE_CANCEL(context); G_RETURN_ERR(res, tmp_err, err); } int gfal2_rename(gfal2_context_t context, const char *oldurl, const char *newurl, GError **err) { GError *tmp_err = NULL; int res = -1; GFAL2_BEGIN_SCOPE_CANCEL(context, -1, err); if (oldurl == NULL || newurl == NULL || context == NULL) { g_set_error(&tmp_err, gfal2_get_core_quark(), EFAULT, "oldurl/newurl/context are incorrect arguments"); } else { res = gfal_plugin_renameG(context, oldurl, newurl, &tmp_err); } GFAL2_END_SCOPE_CANCEL(context); G_RETURN_ERR(res, tmp_err, err); } int gfal2_stat(gfal2_context_t context, const char *url, struct stat *buff, GError **err) { GError *tmp_err = NULL; int res = -1; GFAL2_BEGIN_SCOPE_CANCEL(context, -1, err); if (url == NULL || context == NULL || buff == NULL) { g_set_error(&tmp_err, gfal2_get_core_quark(), EFAULT, "context or/and url or/and buff are incorrect arguments"); } else { res = gfal_plugin_statG(context, url, buff, &tmp_err); } GFAL2_END_SCOPE_CANCEL(context); G_RETURN_ERR(res, tmp_err, err); } int gfal2_lstat(gfal2_context_t context, const char *url, struct stat *buff, GError **err) { GError *tmp_err = NULL; int res = -1; GFAL2_BEGIN_SCOPE_CANCEL(context, -1, err); if (url == NULL || context == NULL || buff == NULL) { g_set_error(&tmp_err, gfal2_get_core_quark(), EFAULT, "context or/and url are incorrect arguments"); } else { res = gfal_plugin_lstatG(context, url, buff, &tmp_err); // protocol does not support lstat, try to map to stat if (res != 0 && tmp_err && tmp_err->code == EPROTONOSUPPORT) { res = gfal2_stat(context, url, buff, err); g_error_free(tmp_err); tmp_err = NULL; } } GFAL2_END_SCOPE_CANCEL(context); G_RETURN_ERR(res, tmp_err, err); } int gfal2_mkdir(gfal2_context_t context, const char *url, mode_t mode, GError **err) { GError *tmp_err = NULL; int res = -1; GFAL2_BEGIN_SCOPE_CANCEL(context, -1, err); if (url == NULL || context == NULL) { g_set_error(&tmp_err, gfal2_get_core_quark(), EFAULT, "context or/and url are incorrect arguments"); } else { res = gfal_plugin_mkdirp(context, url, mode, FALSE, &tmp_err); } GFAL2_END_SCOPE_CANCEL(context); G_RETURN_ERR(res, tmp_err, err); } int gfal2_mkdir_rec(gfal2_context_t context, const char *url, mode_t mode, GError **err) { GError *tmp_err = NULL; int res = -1; GFAL2_BEGIN_SCOPE_CANCEL(context, -1, err); if (url == NULL || context == NULL) { g_set_error(&tmp_err, gfal2_get_core_quark(), EFAULT, "context or/and url are incorrect arguments"); } else { gfal2_log(G_LOG_LEVEL_DEBUG, "execute directory creation for %s", url); res = gfal_plugin_mkdirp(context, url, mode, TRUE, &tmp_err); if (tmp_err) { if (tmp_err->code == EEXIST) { g_clear_error(&tmp_err); res = 0; } else if (tmp_err->code == ENOENT) { gfal2_log(G_LOG_LEVEL_DEBUG, "execute step-by-step recursive directory creation for %s", url); GList *stack_url = NULL; char current_url[GFAL_URL_MAX_LEN]; g_strlcpy(current_url, url, GFAL_URL_MAX_LEN); while (tmp_err && tmp_err->code == ENOENT) { stack_url = g_list_prepend(stack_url, g_strdup(current_url)); g_clear_error(&tmp_err); const size_t s_url = strlen(current_url); char *p_url = current_url + s_url - 1; while (p_url > current_url && *p_url == '/') { // remove trailing '/' *p_url = '\0'; p_url--; } while (p_url > current_url && *p_url != '/') { // find the parent directory p_url--; } if (p_url > current_url) { *p_url = '\0'; res = gfal_plugin_mkdirp(context, current_url, mode, FALSE, &tmp_err); if (res == 0) { gfal2_log(G_LOG_LEVEL_DEBUG, "created directory %s", current_url); } } } // Directory might have been created by a separate process if (tmp_err && tmp_err->code == EEXIST) { g_clear_error(&tmp_err); } if (!tmp_err) { gfal2_log(G_LOG_LEVEL_DEBUG, "recursive directory create from stack root %s", current_url); res = 0; GList *tmp_list = stack_url; while (tmp_list != NULL && res == 0) { res = gfal_plugin_mkdirp(context, (char *) tmp_list->data, mode, FALSE, &tmp_err); if (res == 0) { gfal2_log(G_LOG_LEVEL_DEBUG, "created directory %s", (char *) tmp_list->data); } // Due to a race condition, maybe someone else created the directory else if (tmp_err->code == EEXIST) { res = 0; g_clear_error(&tmp_err); } tmp_list = g_list_next(tmp_list); } } g_list_free_full(stack_url, g_free); } } } GFAL2_END_SCOPE_CANCEL(context); G_RETURN_ERR(res, tmp_err, err); } int gfal2_rmdir(gfal2_context_t context, const char *url, GError **err) { GError *tmp_err = NULL; int res = -1; GFAL2_BEGIN_SCOPE_CANCEL(context, -1, err); if (url == NULL || context == NULL) { g_set_error(&tmp_err, gfal2_get_core_quark(), EFAULT, "context or/and url are incorrect arguments"); } else { res = gfal_plugin_rmdirG(context, url, &tmp_err); } GFAL2_END_SCOPE_CANCEL(context); G_RETURN_ERR(res, tmp_err, err); } int gfal2_symlink(gfal2_context_t context, const char *oldurl, const char *newurl, GError **err) { GError *tmp_err = NULL; int res = -1; GFAL2_BEGIN_SCOPE_CANCEL(context, -1, err); if (oldurl == NULL || newurl == NULL || context == NULL) { g_set_error(&tmp_err, gfal2_get_core_quark(), EFAULT, "oldurl and/or newurl and/or context are incorrect arguments"); } else { res = gfal_plugin_symlinkG(context, oldurl, newurl, &tmp_err); } GFAL2_END_SCOPE_CANCEL(context); G_RETURN_ERR(((res) ? -1 : 0), tmp_err, err); } ssize_t gfal2_getxattr(gfal2_context_t context, const char *url, const char *name, void *value, size_t size, GError **err) { GError *tmp_err = NULL; ssize_t res = -1; GFAL2_BEGIN_SCOPE_CANCEL(context, -1, err); if (url == NULL || context == NULL || name == NULL) { g_set_error(&tmp_err, gfal2_get_core_quark(), EFAULT, "url or/and context or/and name are incorrect arguments"); } else { res = gfal_plugin_getxattrG(context, url, name, value, size, &tmp_err); } GFAL2_END_SCOPE_CANCEL(context); G_RETURN_ERR(res, tmp_err, err); } ssize_t gfal2_readlink(gfal2_context_t context, const char *url, char *buff, size_t buffsiz, GError **err) { GError *tmp_err = NULL; ssize_t res = -1; GFAL2_BEGIN_SCOPE_CANCEL(context, -1, err); if (url == NULL || buff == NULL || context == NULL) { g_set_error(&tmp_err, gfal2_get_core_quark(), EFAULT, "url and/or buff or/and context are incorrect arguments"); } else { res = gfal_plugin_readlinkG(context, url, buff, buffsiz, &tmp_err); } GFAL2_END_SCOPE_CANCEL(context); G_RETURN_ERR(res, tmp_err, err); } int gfal2_unlink(gfal2_context_t context, const char *url, GError **err) { GError *tmp_err = NULL; ssize_t res = -1; GFAL2_BEGIN_SCOPE_CANCEL(context, -1, err); if (url == NULL || context == NULL) { g_set_error(&tmp_err, gfal2_get_core_quark(), EFAULT, "url and/or context are incorrect arguments"); } else { res = gfal_plugin_unlinkG(context, url, &tmp_err); } GFAL2_END_SCOPE_CANCEL(context); G_RETURN_ERR(res, tmp_err, err); } ssize_t gfal2_listxattr(gfal2_context_t context, const char *url, char *list, size_t size, GError **err) { GError *tmp_err = NULL; ssize_t res = -1; GFAL2_BEGIN_SCOPE_CANCEL(context, -1, err); if (url == NULL || context == NULL) { g_set_error(&tmp_err, gfal2_get_core_quark(), EFAULT, "context or/and url or/and list are incorrect arguments"); } else { res = gfal_plugin_listxattrG(context, url, list, size, &tmp_err); } GFAL2_END_SCOPE_CANCEL(context); G_RETURN_ERR(res, tmp_err, err); } int gfal2_setxattr(gfal2_context_t context, const char *url, const char *name, const void *value, size_t size, int flags, GError **err) { GError *tmp_err = NULL; int res = -1; GFAL2_BEGIN_SCOPE_CANCEL(context, -1, err); if (url == NULL || name == NULL || context == NULL) { g_set_error(&tmp_err, gfal2_get_core_quark(), EFAULT, "url or/and name or/and context are an incorrect arguments"); } else { res = gfal_plugin_setxattrG(context, url, name, value, size, flags, &tmp_err); } GFAL2_END_SCOPE_CANCEL(context); G_RETURN_ERR(res, tmp_err, err); } int gfal2_bring_online(gfal2_context_t context, const char *url, time_t pintime, time_t timeout, char *token, size_t tsize, int async, GError **err) { GError *tmp_err = NULL; int res = -1; GFAL2_BEGIN_SCOPE_CANCEL(context, -1, err); if (url == NULL || context == NULL) { g_set_error(&tmp_err, gfal2_get_core_quark(), EFAULT, "context or/and url are incorrect arguments"); } else { res = gfal_plugin_bring_onlineG(context, url, pintime, timeout, token, tsize, async, &tmp_err); } GFAL2_END_SCOPE_CANCEL(context); G_RETURN_ERR(res, tmp_err, err); } int gfal2_bring_online_v2(gfal2_context_t context, const char *url, const char *metadata, time_t pintime, time_t timeout, char *token, size_t tsize, int async, GError **err) { GError *tmp_err = NULL; int res = -1; GFAL2_BEGIN_SCOPE_CANCEL(context, -1, err); if (url == NULL || context == NULL || metadata == NULL) { g_set_error(&tmp_err, gfal2_get_core_quark(), EFAULT, "context, url or/and metadata are incorrect arguments"); } else { res = gfal_plugin_bring_online_v2G(context, url, metadata, pintime, timeout, token, tsize, async, &tmp_err); } GFAL2_END_SCOPE_CANCEL(context); G_RETURN_ERR(res, tmp_err, err); } int gfal2_bring_online_poll(gfal2_context_t context, const char *url, const char *token, GError **err) { GError *tmp_err = NULL; int res = -1; GFAL2_BEGIN_SCOPE_CANCEL(context, -1, err); if (url == NULL || context == NULL) { g_set_error(&tmp_err, gfal2_get_core_quark(), EFAULT, "context or/and url are incorrect arguments"); } else { res = gfal_plugin_bring_online_pollG(context, url, token, &tmp_err); } GFAL2_END_SCOPE_CANCEL(context); G_RETURN_ERR(res, tmp_err, err); } int gfal2_release_file(gfal2_context_t context, const char *url, const char *token, GError **err) { GError *tmp_err = NULL; int res = -1; GFAL2_BEGIN_SCOPE_CANCEL(context, -1, err); if (url == NULL || context == NULL) { g_set_error(&tmp_err, gfal2_get_core_quark(), EFAULT, "context or/and url are incorrect arguments"); } else { res = gfal_plugin_release_fileG(context, url, token, &tmp_err); } GFAL2_END_SCOPE_CANCEL(context); G_RETURN_ERR(res, tmp_err, err); } ssize_t gfal2_qos_check_classes(gfal2_context_t context, const char *url, const char *type, char *buff, size_t s_buff, GError **err) { GError *tmp_err = NULL; ssize_t res = -1; GFAL2_BEGIN_SCOPE_CANCEL(context, -1, err); if (url == NULL || type == NULL || context == NULL || buff == NULL) { g_set_error(&tmp_err, gfal2_get_core_quark(), EFAULT, "context, buff, type or/and url are incorrect arguments"); } else { res = gfal_plugin_qos_check_classes(context, url, type, buff, s_buff, &tmp_err); } GFAL2_END_SCOPE_CANCEL(context); G_RETURN_ERR(res, tmp_err, err); } ssize_t gfal2_check_file_qos(gfal2_context_t context, const char *url, char* buff, size_t s_buff, GError ** err) { GError *tmp_err = NULL; ssize_t res = -1; GFAL2_BEGIN_SCOPE_CANCEL(context, -1, err); if (url == NULL || context == NULL || buff == NULL) { g_set_error(&tmp_err, gfal2_get_core_quark(), EFAULT, "context, buff or/and url are incorrect arguments"); } else { res = gfal_plugin_check_file_qos(context, url, buff, s_buff, &tmp_err); } GFAL2_END_SCOPE_CANCEL(context); G_RETURN_ERR(res, tmp_err, err); } ssize_t gfal2_check_available_qos_transitions(gfal2_context_t context, const char *qos_class_url, char *buff, size_t s_buff, GError **err) { GError *tmp_err = NULL; ssize_t res = -1; GFAL2_BEGIN_SCOPE_CANCEL(context, -1, err); if (qos_class_url == NULL || context == NULL || buff == NULL) { g_set_error(&tmp_err, gfal2_get_core_quark(), EFAULT, "context, buff or/and url are incorrect arguments"); } else { res = gfal_plugin_check_qos_available_transitions(context, qos_class_url, buff, s_buff, &tmp_err); } GFAL2_END_SCOPE_CANCEL(context); G_RETURN_ERR(res, tmp_err, err); } ssize_t gfal2_check_target_qos(gfal2_context_t context, const char *url, char *buff, size_t s_buff, GError ** err) { GError *tmp_err = NULL; ssize_t res = -1; GFAL2_BEGIN_SCOPE_CANCEL(context, -1, err); if (url == NULL || context == NULL || buff == NULL) { g_set_error(&tmp_err, gfal2_get_core_quark(), EFAULT, "context, buff or/and url are incorrect arguments"); } else { res = gfal_plugin_check_target_qos(context, url, buff, s_buff, &tmp_err); } GFAL2_END_SCOPE_CANCEL(context); G_RETURN_ERR(res, tmp_err, err); } int gfal2_change_object_qos(gfal2_context_t context, const char *url, const char *target_qos, GError ** err) { GError *tmp_err = NULL; int res = -1; GFAL2_BEGIN_SCOPE_CANCEL(context, -1, err); if (url == NULL || context == NULL || target_qos == NULL) { g_set_error(&tmp_err, gfal2_get_core_quark(), EFAULT, "context, url or/and target qos are incorrect arguments"); } else { res = gfal_plugin_change_object_qos(context, url, target_qos, &tmp_err); } GFAL2_END_SCOPE_CANCEL(context); G_RETURN_ERR(res, tmp_err, err); } ssize_t gfal2_token_retrieve(gfal2_context_t context, const char* url, const char* issuer, gboolean write_access, unsigned validity, const char* const* activities, char* buff, size_t s_buff, GError ** err) { GError *tmp_err = NULL; ssize_t res = -1; GFAL2_BEGIN_SCOPE_CANCEL(context, -1, err); if (url == NULL || context == NULL || buff == NULL) { g_set_error(&tmp_err, gfal2_get_core_quark(), EFAULT, "context, buff or/and resource url are incorrect arguments"); } else { res = gfal_plugin_token_retrieveG(context, url, issuer, write_access, validity, activities, buff, s_buff, &tmp_err); } GFAL2_END_SCOPE_CANCEL(context); G_RETURN_ERR(res, tmp_err, err); } int gfal2_bring_online_list(gfal2_context_t context, int nbfiles, const char *const *urls, time_t pintime, time_t timeout, char *token, size_t tsize, int async, GError **errors) { int res = -1; GFAL2_BEGIN_SCOPE_CANCEL(context, -1, errors); if (urls == NULL || *urls == NULL || context == NULL) { int i; for (i = 0; i < nbfiles; ++i) { g_set_error(&errors[i], gfal2_get_core_quark(), EFAULT, "context or/and urls are incorrect arguments"); } res = -1; } else { res = gfal_plugin_bring_online_listG(context, nbfiles, urls, pintime, timeout, token, tsize, async, errors); } GFAL2_END_SCOPE_CANCEL(context); return res; } int gfal2_bring_online_list_v2(gfal2_context_t context, int nbfiles, const char *const *urls, const char *const *metadata, time_t pintime, time_t timeout, char *token, size_t tsize, int async, GError **errors) { int res = -1; GFAL2_BEGIN_SCOPE_CANCEL(context, -1, errors); if (urls == NULL || *urls == NULL || context == NULL || metadata == NULL || *metadata == NULL) { int i; for (i = 0; i < nbfiles; ++i) { g_set_error(&errors[i], gfal2_get_core_quark(), EFAULT, "context, urls or/and metadata are incorrect arguments"); } res = -1; } else { res = gfal_plugin_bring_online_list_v2G(context, nbfiles, urls, metadata, pintime, timeout, token, tsize, async, errors); } GFAL2_END_SCOPE_CANCEL(context); return res; } int gfal2_bring_online_poll_list(gfal2_context_t context, int nbfiles, const char *const *urls, const char *token, GError **errors) { int res = -1; GFAL2_BEGIN_SCOPE_CANCEL(context, -1, errors); if (urls == NULL || *urls == NULL || context == NULL) { int i; for (i = 0; i < nbfiles; ++i) { g_set_error(&errors[i], gfal2_get_core_quark(), EFAULT, "context or/and urls are incorrect arguments"); } res = -1; } else { res = gfal_plugin_bring_online_poll_listG(context, nbfiles, urls, token, errors); } GFAL2_END_SCOPE_CANCEL(context); return res; } int gfal2_release_file_list(gfal2_context_t context, int nbfiles, const char *const *urls, const char *token, GError **errors) { int res = -1; GFAL2_BEGIN_SCOPE_CANCEL(context, -1, errors); if (urls == NULL || *urls == NULL || context == NULL) { int i; for (i = 0; i < nbfiles; ++i) { g_set_error(&errors[i], gfal2_get_core_quark(), EFAULT, "context or/and urls are incorrect arguments"); } res = -1; } else { res = gfal_plugin_release_file_listG(context, nbfiles, urls, token, errors); } GFAL2_END_SCOPE_CANCEL(context); return res; } int gfal2_unlink_list(gfal2_context_t context, int nbfiles, const char *const *urls, GError **errors) { GError *tmp_err = NULL; int res = 0; if (urls == NULL || *urls == NULL || context == NULL) { g_set_error(&tmp_err, gfal2_get_core_quark(), EFAULT, "urls or/and name or/and context are an incorrect arguments"); res = -1; } else { res = gfal2_start_scope_cancel(context, &tmp_err); if (res == 0) { res = gfal_plugin_unlink_listG(context, nbfiles, urls, errors); gfal2_end_scope_cancel(context); } } if (tmp_err) { int i; for (i = 0; i < nbfiles; ++i) { errors[i] = g_error_copy(tmp_err); } g_error_free(tmp_err); } return res; } int gfal2_abort_files(gfal2_context_t context, int nbfiles, const char *const *urls, const char *token, GError **err) { GError *tmp_err = NULL; int res = -1; GFAL2_BEGIN_SCOPE_CANCEL(context, -1, err); if (urls == NULL || *urls == NULL || context == NULL) { g_set_error(&tmp_err, gfal2_get_core_quark(), EFAULT, "context or/and urls are incorrect arguments"); } else { res = gfal_plugin_abort_filesG(context, nbfiles, urls, token, err); } GFAL2_END_SCOPE_CANCEL(context); G_RETURN_ERR(res, tmp_err, err); } int gfal2_archive_poll(gfal2_context_t context, const char* url, GError ** err) { GError *tmp_err = NULL; int res = -1; GFAL2_BEGIN_SCOPE_CANCEL(context, -1, err); if (url == NULL || context == NULL) { g_set_error(&tmp_err, gfal2_get_core_quark(), EFAULT, "context or/and url are incorrect arguments"); } else { res = gfal_plugin_archive_pollG(context, url, &tmp_err); } GFAL2_END_SCOPE_CANCEL(context); G_RETURN_ERR(res, tmp_err, err); } int gfal2_archive_poll_list(gfal2_context_t context, int nbfiles, const char* const *urls, GError **errors) { int res = -1; GFAL2_BEGIN_SCOPE_CANCEL(context, -1, errors); if (urls == NULL || *urls == NULL || context == NULL) { int i; for (i = 0; i < nbfiles; ++i) { g_set_error(&errors[i], gfal2_get_core_quark(), EFAULT, "context or/and urls are incorrect arguments"); } res = -1; } else { res = gfal_plugin_archive_poll_listG(context, nbfiles, urls, errors); } GFAL2_END_SCOPE_CANCEL(context); return res; } int gfal2_checksum(gfal2_context_t handle, const char *url, const char *check_type, off_t start_offset, size_t data_length, char *checksum_buffer, size_t buffer_length, GError **err) { if (!(handle != NULL && url != NULL && check_type != NULL && checksum_buffer != NULL && buffer_length != 0)) { g_set_error(err, gfal2_get_core_quark(), EFAULT, "Invalid parameters to %s", __func__); return -1; } GFAL2_BEGIN_SCOPE_CANCEL(handle, -1, err); int res = -1; GError *tmp_err = NULL; gfal_plugin_interface *p = gfal_find_plugin(handle, url, GFAL_PLUGIN_CHECKSUM, &tmp_err); if (p) { res = p->checksum_calcG(gfal_get_plugin_handle(p), url, check_type, checksum_buffer, buffer_length, start_offset, data_length, &tmp_err); } GFAL2_END_SCOPE_CANCEL(handle); // If configured, always return Adler32 checksum as 8-byte string gboolean format_checksum = gfal2_get_opt_boolean_with_default(handle, "CORE", "FORMAT_ADLER32_CHECKSUM", TRUE); if (format_checksum && checksum_buffer != NULL && strncasecmp(check_type, "adler32", strlen(check_type)) == 0) { size_t checksum_len = strlen(checksum_buffer); if (checksum_len < GFAL_ADLER_CHKSUM_LEN && buffer_length > GFAL_ADLER_CHKSUM_LEN) { size_t diff = GFAL_ADLER_CHKSUM_LEN - checksum_len; char* tmp_buffer = g_strdup(checksum_buffer); memset(checksum_buffer, '0', diff); g_strlcpy(checksum_buffer + diff, tmp_buffer, buffer_length); gfal2_log(G_LOG_LEVEL_DEBUG, "Formatted adler32 checksum: %s --> %s", tmp_buffer, checksum_buffer); g_free(tmp_buffer); } } G_RETURN_ERR(res, tmp_err, err); } gfal2-v2.23.0/src/core/file/gfal_file_api.h000066400000000000000000000746521465240014500203460ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #pragma once #ifndef GFAL_FILE_API_H_ #define GFAL_FILE_API_H_ #if !defined(__GFAL2_H_INSIDE__) && !defined(__GFAL2_BUILD__) # warning "Direct inclusion of gfal2 headers is deprecated. Please, include only gfal_api.h or gfal_plugins_api.h" #endif #include #include #include #include #include #include #ifndef ENOATTR #define ENOATTR ENODATA #endif #if defined __APPLE__ #include #else #if defined __GLIBC_PREREQ && __GLIBC_PREREQ(2,27) #include #else #include #endif #endif #include #include #include #ifdef __cplusplus extern "C" { #endif /*! \defgroup file_group GFAL 2.0 generic file API GFAL 2.0 file API is the main entry point for file/directory operations All functions report a EPROTONOSUPPORT GError if the url protocol does not support this operation. */ /*! \addtogroup file_group @{ */ /** * @briefcompute checksum * * Compute checksum function for a file url * @param context : gfal2 handle, see \ref gfal2_context_new * @param url : url of the file * @param check_type : string of the checksum type ( \ref GFAL_CHKSUM_MD5, \ref GFAL_CHKSUM_SHA1, .. ) * @param start_offset : offset in the file where the checksum calculation will start ( 0 : from begining ) * @param data_length : size of data to compute for the checksum after start_offset ( 0 -: full file ) * @param checksum_buffer : buffer with checksum string as result * @param buffer_length : maximum buffer length * @param err : GError error report * @return 0 if success, else -1 and err is be set * See gfal2 error system for more information \ref gfal2_error_system */ int gfal2_checksum(gfal2_context_t context, const char* url, const char* check_type, off_t start_offset, size_t data_length, char * checksum_buffer, size_t buffer_length, GError ** err); /** * @brief permission check * * Check real user's permissions for a file * @param context : gfal2 handle, see \ref gfal2_context_new * @param url : url of the file * @param amode : mode of the access * @param err : GError error report * @return 0 if success, else -1 and err MUST be set properly * See gfal2 error system for more information \ref gfal2_error_system */ int gfal2_access(gfal2_context_t context, const char *url, int amode, GError** err); /** * @brief change file access permissions * * Change the permissions of a file according to "mode" * @param context : gfal2 handle, see \ref gfal2_context_new * @param url : url of the file or the folder * @param mode : right to configure * @param err : GError error report * @return return 0 if success else -1 and err is be set * See gfal2 error system for more information \ref gfal2_error_system */ int gfal2_chmod(gfal2_context_t context, const char* url, mode_t mode, GError ** err); /** * @brief change the name or location of a file * * Move ( or rename ) the file 'oldurl' to 'newurl' * @param context : gfal2 handle, see \ref gfal2_context_new * @param oldurl : the old url of the file * @param newurl : the new url of the file * @param err : GError error report * @return : return 0 if success, else -1 if errors. * See gfal2 error system for more information \ref gfal2_error_system */ int gfal2_rename(gfal2_context_t context, const char *oldurl, const char *newurl, GError ** err); /** * @brief posix file status * * Get meta-data information about the file 'url' * @param context : gfal2 handle, see \ref gfal2_context_new * @param url : url of the file * @param buff : stat structure filled * @param err : GError error report */ int gfal2_stat(gfal2_context_t context, const char* url, struct stat* buff, GError ** err); /** * @brief posix file status * * Get meta-data information about the file 'url' * same behavior than \ref gfal2_stat but return information * about the link itself if "url" is a symbolic link * * @param context : gfal2 handle, see \ref gfal2_context_new * @param url : url of the file * @param buff : stat structure filled * @param err : GError error report */ int gfal2_lstat(gfal2_context_t context, const char* url, struct stat* buff, GError ** err); /** * @brief create directory * * Create a directory at the address 'url' * * @param context : gfal2 handle, see \ref gfal2_context_new * @param url : url of the file * @param mode : directory file rights * @param err : GError error report */ int gfal2_mkdir(gfal2_context_t context, const char* url, mode_t mode, GError ** err); /** * @brief create directory * * Create a directory at the address 'url' * Create all the parent drectories and * does not return an error if the directory already exist * * @param context : gfal2 handle, see \ref gfal2_context_new * @param url : url of the file * @param mode : directory file rights * @param err : GError error report */ int gfal2_mkdir_rec(gfal2_context_t context, const char* url, mode_t mode, GError ** err); /** * @brief suppress a directory * * Suppress a directory at the address 'url' * @param context : gfal2 handle, see \ref gfal2_context_new * @param url : url of the file * @param err : GError error report * @return 0 if success, negative value if error, set err properly in case of error */ int gfal2_rmdir(gfal2_context_t context, const char* url, GError ** err); /** * @brief open a directory for content listing * * Return a directory handle for content listing * * @param context : gfal2 handle, see \ref gfal2_context_new * @param url : url of the file * @param err : GError error report * @return a directory handle in case of success or NULL if error occurs, set err properly in case of error */ DIR* gfal2_opendir(gfal2_context_t context, const char* url, GError ** err); /** * @brief return the next directory entry * * @param context : gfal2 handle, see \ref gfal2_context_new * @param d : directory handle created by \ref gfal2_opendir * @param err : GError error report * @return pointer to a dirent struct if success, NULL if end of listing or error, * set err properly in case of error */ struct dirent* gfal2_readdir(gfal2_context_t context, DIR* d, GError ** err); /** * @brief return the next directory entry in addition of the entry meta-data * * readdirpp get both of the directory entry informations and the stat * informations in one operation, improving the performance in case of remote file system * * @param context : gfal2 handle, see \ref gfal2_context_new * @param d : directory handle created by \ref gfal2_opendir * @param st : the file stats will be stored here * @param err : GError error report * @return pointer to a dirent struct and configure st with the meta-data information if success, NULL if end of listing or error, set err properly in case of error */ struct dirent* gfal2_readdirpp(gfal2_context_t context, DIR* d, struct stat* st, GError ** err); /** * @brief close a directory handle * * @param context : gfal2 handle, see \ref gfal2_context_new * @param d : directory handle created by \ref gfal2_opendir * @param err : GError error report * @return 0 if success, negative value if error, set err properly in case of error */ int gfal2_closedir(gfal2_context_t context, DIR* d, GError ** err); /** * @brief create a symbolic link * * Symbolic links are not supported by all protocols, in case of non-supported feature * GFAL2 always return an error and set err to the code EPROTONOSUPPORT * * @param context : gfal2 handle, see \ref gfal2_context_new * @param oldurl : origin file * @param newurl : symbolic link path * @param err : GError error report * @return 0 if success, negative value if error, set err properly in case of error */ int gfal2_symlink(gfal2_context_t context, const char* oldurl, const char* newurl, GError ** err); /** * @brief read a symbolic link value, provide the linked file path * * Symbolic links are not supported by all protocols, in case of non-supported feature * GFAL2 always return an error and set err to the code EPROTONOSUPPORT. * gfal2_readlink follows the POSIX behavior and does not add a null byte at the end of the buffer if the link is truncated. * * @param context : gfal2 handle, see \ref gfal2_context_new * @param url : path of the symbolic link to read * @param buff : buffer for symbolic link value * @param buffsiz : maximum number of bytes to write * @param err : GError error report * @return size of the link value in bytes if success, negative value if error. set err properly in case of error */ ssize_t gfal2_readlink(gfal2_context_t context, const char* url, char* buff, size_t buffsiz, GError ** err); /** * @brief Delete a file entry * * Does not work for Collections or directory. * * @param context : gfal2 handle, see \ref gfal2_context_new * @param url : path of the file to delete * @param err : GError error report * @return 0 if success, -1 if error. set err properly in case of error */ int gfal2_unlink(gfal2_context_t context, const char* url, GError ** err); /** * @brief list extended attributes of a resource. * * Extended attributes keys are concatenated in the buffer and separated by a null character * * @param context : gfal2 handle, see \ref gfal2_context_new * @param url : path of the resource * @param list : buffer for the extended attribute keys * @param size : maximum size of the buffer to write * @param err : GError error report * @return the size of the concatenated xattr keys in bytes if success, -1 if error. set err properly in case of errors. */ ssize_t gfal2_listxattr (gfal2_context_t context, const char *url, char *list, size_t size, GError ** err); /** * @brief get an extended attribute value of a resource. * * @param context : gfal2 handle, see \ref gfal2_context_new * @param url : path of the resource * @param name : key of the extended attribute * @param value : buffer for the extended attribute value * @param size : maximum size of the buffer to write * @param err : GError error report * @return the size of the xattr value in bytes if success, -1 if error. set err properly in case of errors. */ ssize_t gfal2_getxattr (gfal2_context_t context, const char *url, const char *name, void *value, size_t size, GError ** err); /** * @brief set an extended attribute value of a resource. * * @param context : gfal2 handle, see \ref gfal2_context_new * @param url : path of the resource * @param name : key of the extended attribute to define/set * @param value : new value of the xattr * @param size : size of the data * @param flags : xattr flag, XATTR_CREATE specifies a pure create, which fails if the named attribute exists already. XATTR_REPLACE specifies a pure * replace operation, which fails if the named attribute does not already exist. By default (no flags), the extended attribute will be created if need be, or will simply replace the value if the * attribute exists. * * @param err : GError error report * @return 0 if success, -1 if error. set err properly in case of errors. */ int gfal2_setxattr (gfal2_context_t context, const char *url, const char *name, const void *value, size_t size, int flags, GError ** err); /** * @brief Bring online a file * * @param context : gfal2 handle, see \ref gfal2_context_new * @param url : url of the file * @param pintime : pin time * @param timeout : timeout * @param token : The token will be put in the buffer pointed by this * @param tsize: The size of the buffer pointed by token * @param async: Asynchronous request (does not block if != 0) * @param err : GError error report * @return 0 if the request has been queued, > 0 if the file is pinned, < 0 on error */ int gfal2_bring_online(gfal2_context_t context, const char* url, time_t pintime, time_t timeout, char* token, size_t tsize, int async, GError ** err); /** * @brief Bring online a file * * @param context : gfal2 handle, see \ref gfal2_context_new * @param url : url of the file * @param metadata : Staging metadata * @param pintime : pin time * @param timeout : timeout * @param token : The token will be put in the buffer pointed by this * @param tsize : The size of the buffer pointed by token * @param async : Asynchronous request (does not block if != 0) * @param err : GError error report * @return 0 if the request has been queued, > 0 if the file is pinned, < 0 on error */ int gfal2_bring_online_v2(gfal2_context_t context, const char* url, const char* metadata, time_t pintime, time_t timeout, char* token, size_t tsize, int async, GError ** err); /** * @brief Check for a bring online request * * @param context : gfal2 handle, see \ref gfal2_context_new * @param url : url of the file * @param token : as set by gfal2_bring_online * @param err : GError error report * @return 0 if the request is queued, > 0 if the file is pinned, < 0 on error */ int gfal2_bring_online_poll(gfal2_context_t context, const char* url, const char* token, GError ** err); /** * @brief Release a file * * @param context : gfal2 handle, see \ref gfal2_context_new * @param url : url of the file * @param token : as set by gfal2_bring_online * @param err : GError error report * @return 0 if the file was released, < 0 on error */ int gfal2_release_file(gfal2_context_t context, const char* url, const char* token, GError ** err); /** * @brief Check available QoS classes supported by the endpoint * * @param context : gfal2 handle, see \ref gfal2_context_new * @param url : url of the endpoint * @param type : QoS type of the entity * @param buff : buffer for the QoS classes string * @param s_buff : maximum size of the buffer to write * @param err : GError error report * @return the size of the QoS classes string in bytes if success, -1 if error */ ssize_t gfal2_qos_check_classes(gfal2_context_t context, const char *url, const char *type, char *buff, size_t s_buff, GError **err); /** * @brief Check QoS of a file * * @param context : gfal2 handle, see \ref gfal2_context_new * @param url : url of the file * @param buff : buffer for the file QoS class * @param s_buff : maximum size of the buffer to write * @param err : GError error report * @return the size of the QoS class value in bytes if success, -1 if error */ ssize_t gfal2_check_file_qos(gfal2_context_t context, const char *url, char *buff, size_t s_buff, GError **err); /** * @brief Check available QoS transitions of a QoS class * * @param context : gfal2 handle, see \ref gfal2_context_new * @param qos_class_url : url of the QoS class * @param buff : buffer for the QoS transitions string * @param s_buff : maximum size of the buffer to write * @param err : GError error report * @return the size of the QoS transitions string in bytes if success, -1 if error */ ssize_t gfal2_check_available_qos_transitions(gfal2_context_t context, const char *qos_class_url, char *buff, size_t s_buff, GError **err); /** * @brief Check target QoS of a file * * @param context : gfal2 handle, see \ref gfal2_context_new * @param url : url of the file * @param buff : buffer for the target QoS class * @param s_buff : maximum size of the buffer to write * @param err : GError error report * @return the size of the target QoS class value in bytes if success, -1 if error * * @note Usually available during QoS transitions */ ssize_t gfal2_check_target_qos(gfal2_context_t context, const char *url, char *buff, size_t s_buff, GError **err); /** * @brief Request the QoS transition of a CDMI object * * @param context : gfal2 handle, see \ref gfal2_context_new * @param url : url of the file * @param target_qos : the target QoS class * @param err : GError error report * @return 0 if success, -1 if error */ int gfal2_change_object_qos(gfal2_context_t context, const char *url, const char *target_qos, GError **err); /** * @brief Retrieve a Storage Element token for a given resource * * @param context : gfal2 handle, see \ref gfal2_context_new * @param url : url of the resource * @param issuer : the token issuer endpoint (optional) * @param write_access : write access flag * @param validity : token validity in minutes * @param activities : array of activities for access request * @param buff : buffer for the token content * @param s_buff : maximum buffer size * @param err : GError error report * @return the size of the token value in bytes if success, -1 if error */ ssize_t gfal2_token_retrieve(gfal2_context_t context, const char* url, const char* issuer, gboolean write_access, unsigned validity, const char* const* activities, char* buff, size_t s_buff, GError** err); /** * @brief Bring online a file * * @param context : gfal2 handle, see \ref gfal2_context_new * @param nbfiles : number of files * @param urls : urls of files * @param pintime : pin time * @param timeout : timeout * @param token : The token will be put in the buffer pointed by this * @param tsize : The size of the buffer pointed by token * @param async: Asynchronous request (does not block if != 0) * @param errors : Preallocated array of nbfiles pointers to GError. User must allocate and free. * @return 0 if the request has been queued, > 0 if the file is pinned, < 0 on error * @note Even if the result is > 0, you need to check each individual file status */ int gfal2_bring_online_list(gfal2_context_t context, int nbfiles, const char* const* urls, time_t pintime, time_t timeout, char* token, size_t tsize, int async, GError ** errors); /** * @brief Bring online a file * * @param context : gfal2 handle, see \ref gfal2_context_new * @param nbfiles : number of files * @param urls : urls of files * @param metadata : Staging metadata array * @param pintime : pin time * @param timeout : timeout * @param token : The token will be put in the buffer pointed by this * @param tsize : The size of the buffer pointed by token * @param async : Asynchronous request (does not block if != 0) * @param errors : Preallocated array of nbfiles pointers to GError. User must allocate and free. * @return 0 if the request has been queued, > 0 if the file is pinned, < 0 on error * @note Even if the result is > 0, you need to check each individual file status */ int gfal2_bring_online_list_v2(gfal2_context_t context, int nbfiles, const char* const* urls, const char* const* metadata, time_t pintime, time_t timeout, char* token, size_t tsize, int async, GError ** errors); /** * @brief Check for a bring online request * * @param context : gfal2 handle, see \ref gfal2_context_new * @param nbfiles : number of files * @param urls : urls of files * @param token : As set by gfal2_bring_online * @param errors : Preallocated array of nbfiles pointers to GError. User must allocate and free. * @return 0 if the request is queued, > 0 if the file is pinned, < 0 on error * @note Even if the result is > 0, you need to check each individual file status */ int gfal2_bring_online_poll_list(gfal2_context_t context, int nbfiles, const char* const* urls, const char* token, GError ** errors); /** * @brief Perform a bulk release file * * @param context : gfal2 handle, see \ref gfal2_context_new * @param nbfiles : number of files * @param urls : paths of the files to delete * @param token : the token from the bring online request * @param errors : Preallocated array of nbfiles pointers to GError. User must allocate and free. * @note Even if the result is > 0, you need to check each individual file status */ int gfal2_release_file_list(gfal2_context_t context, int nbfiles, const char* const* urls, const char* token, GError ** errors); /** * @brief Perform a bulk deletion * * Does not work for Collections or directories. * * @param context : gfal2 handle, see \ref gfal2_context_new * @param nbfiles : number of files * @param urls : paths of the files to delete * @param errors : Pre-allocated array with nbfiles pointers to errors. * It is the user's responsability to allocate and free. * @return 0 if success, -1 if error. set err properly in case of error * @note The plugin tried will be the one that matches the first url * @note If bulk deletion is not supported, gfal2_unlink will be called nbfiles times */ int gfal2_unlink_list(gfal2_context_t context, int nbfiles, const char* const* urls, GError ** errors); /** * @brief abort a list of files * @param context : gfal2 handle, see \ref gfal2_context_new * @param nbfiles : number of files * @param urls : paths of the files to delete * @param token : the token from the bring online request * @param errors : Preallocated array of nbfiles pointers to GError. User must allocate and free. * @note Even if the result is > 0, you need to check each individual file status */ int gfal2_abort_files(gfal2_context_t context, int nbfiles, const char* const* urls, const char* token, GError ** errors); /** * @brief Check status of an archive request * * @param context : gfal2 handle, see \ref gfal2_context_new * @param url : url of the file * @param err : GError error report * @return 0 if the request is queued, > 0 if the file is archived, < 0 on error */ int gfal2_archive_poll(gfal2_context_t context, const char* url, GError ** err); /** * @brief Check status for a list of archive requests * * @param context : gfal2 handle, see \ref gfal2_context_new * @param nbfiles : number of files * @param urls : urls of files * @param errors : Preallocated array of nbfiles pointers to GError. User must allocate and free. * @return 0 if the request is queued, > 0 if the file is archived, < 0 on error * @note Even if the result is > 0, you need to check each individual file status */ int gfal2_archive_poll_list(gfal2_context_t context, int nbfiles, const char* const* urls, GError ** errors); /** * @brief Open a file, return GFAL2 file descriptor * * gfal_open supports the same flags than the POSIX open call * * O_TRUNC * If the file already exists and is a regular file and the open mode allows writing (i.e., is O_RDWR or O_WRONLY) it will be truncated to length 0. If the file is a FIFO or termi‐ * nal device file, the O_TRUNC flag is ignored. Otherwise the effect of O_TRUNC is unspecified. * * O_APPEND * The file is opened in append mode. Before each write(2), the file offset is positioned at the end of the file, as if with lseek(2). O_APPEND may lead to corrupted files on NFS * file systems if more than one process appends data to a file at once. This is because NFS does not support appending to a file, so the client kernel has to simulate it, which * can't be done without a race condition. * * O_CREAT * If the file does not exist it will be created. The owner (user ID) of the file is set to the effective user ID of the process. The group ownership (group ID) is set either to * the effective group ID of the process or to the group ID of the parent directory (depending on file system type and mount options, and the mode of the parent directory, see the * mount options bsdgroups and sysvgroups described in mount(8)). * * mode specifies the permissions to use in case a new file is created. This argument must be supplied when O_CREAT is specified in flags; if O_CREAT is not specified, then mode is * ignored. The effective permissions are modified by the process's umask in the usual way: The permissions of the created file are (mode & ~umask). Note that this mode only * applies to future accesses of the newly created file; the open() call that creates a read-only file may well return a read/write file descriptor. * * O_DIRECT * Try to minimize cache effects of the I/O to and from this file. In general this will degrade performance, but it is useful in special situations, such as when applications do * their own caching. File I/O is done directly to/from user space buffers. The O_DIRECT flag on its own makes at an effort to transfer data synchronously, but does not give the * guarantees of the O_SYNC that data and necessary metadata are transferred. To guarantee synchronous I/O the O_SYNC must be used in addition to O_DIRECT. See NOTES below for fur‐ * ther discussion. * * O_LARGEFILE * (LFS) Allow files whose sizes cannot be represented in an off_t (but can be represented in an off64_t) to be opened. The _LARGEFILE64_SOURCE macro must be defined (before includ‐ * ing any header files) in order to obtain this definition. Setting the _FILE_OFFSET_BITS feature test macro to 64 (rather than using O_LARGEFILE) is the preferred method of * obtaining method of accessing large files on 32-bit systems * * @param context : gfal2 handle, see \ref gfal2_context_new * @param url : url of the file to open * @param flags : flags to use ( similar to open ) * @param err : GError error report * @return This routine return a valid file descriptor if the operation is a success * or -1 if error occured. In case of Error, err is set properly */ int gfal2_open(gfal2_context_t context, const char * url, int flags, GError ** err); /** * * Same than \ref gfal2_open but allow to specify the default right of the file */ int gfal2_open2(gfal2_context_t context, const char * url, int flag, mode_t mode, GError ** err); /** * \ref gfal2_creat is equivalent to \ref gfal2_open2 with flags equal to O_CREAT|O_WRONLY|O_TRUNC. */ int gfal2_creat (gfal2_context_t context, const char *filename, mode_t mode, GError ** err); /** * @brief read data from a GFAL2 file descriptor * * @param context : gfal2 handle, see \ref gfal2_context_new * @param fd : GFAL2 file descriptor of the file * @param buff : buffer for read data * @param s_buff : maximum size to read * @param err : GError error report* * @return On success, the number of bytes read is returned * (zero indicates end of file), and the file position is advanced * by this number. It is not an error if this number is * smaller than the number of bytes requested. or a negative value is returned when an error occurs * In case of Error, err is set properly */ ssize_t gfal2_read(gfal2_context_t context, int fd, void* buff, size_t s_buff, GError ** err); /** * @brief write data to a GFAL2 file descriptor * * @param context : gfal2 handle, see \ref gfal2_context_new * @param fd : GFAL2 file descriptor of the file * @param buff : buffer with data to write * @param s_buff : maximum size to write * @param err : GError error report* * @return On success, the number of bytes written is returned * (zero indicates end of file), and the file position is advanced * by this number. It is not an error if this number is * smaller than the number of bytes requested. or a negative value is returned when an error occurs. * In case of Error, err is set properly */ ssize_t gfal2_write(gfal2_context_t context, int fd, const void *buff, size_t s_buff, GError ** err); /** * @brief move the file cursor * * Move the file cursor of the GFAL2 file descriptor fd to offset * * @param context : gfal2 handle, see \ref gfal2_context_new * @param fd : file descriptor * @param offset : new position of the cursor * @param whence : directive to follow, can be SEEK_SET, SEEK_CUR or SEEK_END * @param err : GError error report * @return new cursor position if success, -1 if failure, set err properly in case of error. */ off_t gfal2_lseek (gfal2_context_t context, int fd, off_t offset, int whence, GError ** err); /** * @brief close a file GFAL2 descriptor * * @param context : gfal2 handle, see \ref gfal2_context_new * @param fd : file descriptor * @param err : GError error report * @return new cursor position if success, -1 if failure, set err properly in case of error. */ int gfal2_close(gfal2_context_t context, int fd, GError ** err); /** * @brief flush all buffered data for the given file descriptor * * @param context : gfal2 handle, see \ref gfal2_context_new * @param fd : file descriptor * @param err : GError error report * @return new cursor position if success, -1 if failure, set err properly in case of error. */ int gfal2_flush(gfal2_context_t context, int fd, GError ** err); /** * @brief read from file descriptor at a given offset * * @param context : gfal2 handle, see \ref gfal2_context_new * @param fd : file descriptor * @param buffer: buffer for read data * @param count : maximum size to read * @param offset : operation offset * @param err : GError error report * @return number of read bytes, -1 on failure, set err properly in case of error. */ ssize_t gfal2_pread(gfal2_context_t context, int fd, void * buffer, size_t count, off_t offset, GError ** err); /** * @brief write to file descriptor at a given offset * * @param context : gfal2 handle, see \ref gfal2_context_new * @param fd : file descriptor * @param buffer: buffer for read data * @param count : maximum size to write * @param offset : operation offset * @param err : GError error report * @return number of written bytes, -1 on failure, set err properly in case of error. */ ssize_t gfal2_pwrite(gfal2_context_t context, int fd, const void * buffer, size_t count, off_t offset, GError ** err); /** @} End of the FILE group */ #ifdef __cplusplus } #endif #endif /* GFAL_FILE_API_H_ */ gfal2-v2.23.0/src/core/future/000077500000000000000000000000001465240014500160115ustar00rootroot00000000000000gfal2-v2.23.0/src/core/future/glib.c000066400000000000000000000045511465240014500170770ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include "glib.h" #if (GLIB_CHECK_VERSION(2,16,0) != TRUE) // add code of glib 2.16 for link with a very old glib version static void g_error_add_prefix (gchar **string, const gchar *format, va_list ap) { gchar *oldstring; gchar *prefix; prefix = g_strdup_vprintf (format, ap); oldstring = *string; *string = g_strconcat (prefix, oldstring, NULL); g_free (oldstring); g_free (prefix); } void g_propagate_prefixed_error (GError **dest, GError *src, const gchar *format, ...) { g_propagate_error (dest, src); if (dest && *dest) { va_list ap; va_start (ap, format); g_error_add_prefix (&(*dest)->message, format, ap); va_end (ap); } } void g_prefix_error (GError **err, const gchar *format, ...) { if (err && *err) { va_list ap; va_start (ap, format); g_error_add_prefix (&(*err)->message, format, ap); va_end (ap); } } #endif #if (GLIB_CHECK_VERSION(2,18,0) != TRUE) void g_set_error_literal (GError **err, GQuark domain, gint code, const gchar *message) { g_set_error(err, domain, code, "%s", message); } #endif #if (GLIB_CHECK_VERSION(2,28,0) != TRUE) void g_list_free_full(GList *list, GDestroyNotify free_func) { GList* tmp_list= list; while( tmp_list != NULL){ free_func(tmp_list->data); tmp_list = g_list_next(tmp_list); } g_list_free(list); } #endif gfal2-v2.23.0/src/core/future/glib.h000066400000000000000000000023771465240014500171100ustar00rootroot00000000000000/** * @file glib.h * @brief Advanced functions of glib for the old versions * @author Devresse Adrien **/ #pragma once #ifdef __cplusplus extern "C" { #endif #include // GError #if (GLIB_CHECK_VERSION(2,16,0) != TRUE) #define ERROR_OVERWRITTEN_WARNING "GError set over the top of a previous GError or uninitialized memory.\n" void g_propagate_prefixed_error (GError **dest, GError *src, const gchar *format, ...) G_GNUC_PRINTF (3, 4); void g_prefix_error (GError **err, const gchar *format, ...) G_GNUC_PRINTF (2, 3); #endif #if (GLIB_CHECK_VERSION(2,18,0) != TRUE) void g_set_error_literal (GError **err, GQuark domain, gint code, const gchar *message); #endif // GList #if (GLIB_CHECK_VERSION(2,28,0) != TRUE) void g_list_free_full(GList *list, GDestroyNotify free_func); #endif #ifdef __cplusplus } #endif gfal2-v2.23.0/src/core/gfal_api.h000066400000000000000000000027611465240014500164200ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #pragma once #ifndef GFAL2_API_H_ #define GFAL2_API_H_ #define __GFAL2_H_INSIDE__ /* gfal2 uses 64 bits offset size by default */ #ifndef _FILE_OFFSET_BITS # define _FILE_OFFSET_BITS 64 #endif /* global context operations */ #include #include /* parameter and configuration API */ #include /* log API */ #include /* main gfal2 API for file operations */ #include /* operation control API */ #include /* posix compatibility layer */ #include /* transfers*/ #include /* error helpers*/ #include #undef __GFAL2_H_INSIDE__ #endif /* GFAL2_API_H_ */ gfal2-v2.23.0/src/core/gfal_plugins_api.h000066400000000000000000000020301465240014500201460ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #pragma once #ifndef GFAL2_PLUGIN_API_H_ #define GFAL2_PLUGIN_API_H_ #define __GFAL2_H_INSIDE__ #include #include #include #undef __GFAL2_H_INSIDE__ #include #endif /* GFAL2_PLUGIN_API_H_ */ gfal2-v2.23.0/src/core/logger/000077500000000000000000000000001465240014500157565ustar00rootroot00000000000000gfal2-v2.23.0/src/core/logger/gfal_logger.c000066400000000000000000000030721465240014500203740ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include "gfal_logger.h" static GLogLevelFlags gfal2_log_level = G_LOG_LEVEL_WARNING; void gfal2_log(GLogLevelFlags level, const char* msg, ...) { if (level <= gfal2_log_level) { va_list args; va_start(args, msg); g_logv("GFAL2", level, msg, args); va_end(args); } } void gfal2_logv(GLogLevelFlags level, const char* msg, va_list args) { if (level <= gfal2_log_level) { g_logv("GFAL2", level, msg, args); } } void gfal2_log_set_level(GLogLevelFlags level) { gfal2_log_level = level; } GLogLevelFlags gfal2_log_get_level(void) { return gfal2_log_level; } int gfal2_log_set_handler(GLogFunc func, gpointer user_data) { return g_log_set_handler("GFAL2", G_LOG_LEVEL_MASK, func, user_data); } gfal2-v2.23.0/src/core/logger/gfal_logger.h000066400000000000000000000041131465240014500203760ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #pragma once #ifndef GFAL_LOGGER_H_ #define GFAL_LOGGER_H_ #if !defined(__GFAL2_H_INSIDE__) && !defined(__GFAL2_BUILD__) # warning "Direct inclusion of gfal2 headers is deprecated. Please, include only gfal_api.h or gfal_plugins_api.h" #endif #include #include #ifdef __cplusplus extern "C" { #endif /** * Log a message with the given level. * The default handler prints to stderr. */ void gfal2_log(GLogLevelFlags level, const char* msg, ...); /** * Log a message with the given level. * The default handler prints to stderr. */ void gfal2_logv(GLogLevelFlags level, const char* msg, va_list args); /** * Set the log level. Only messages with level higher or equal will be passed to the handler. * For instance, if set to G_LOG_LEVEL_WARNING, * only messages with level WARNING, CRITICAL and ERROR will be considered. */ void gfal2_log_set_level(GLogLevelFlags level); /** * Return the log level configured */ GLogLevelFlags gfal2_log_get_level(void); /** * Set a custom handler * See Glib2 message logging system for more informations about log_func. * Internally, gfal2 uses the glib2 log system with the "GFAL2" domain. * Usual glib2 functions can be used to control the gfal2 messages. */ int gfal2_log_set_handler(GLogFunc func, gpointer user_data); #ifdef __cplusplus } #endif #endif /* GFAL_LOGGER_H_ */ gfal2-v2.23.0/src/core/posix/000077500000000000000000000000001465240014500156415ustar00rootroot00000000000000gfal2-v2.23.0/src/core/posix/gfal_posix_api.h000066400000000000000000000475371465240014500210160ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #pragma once #ifndef GFAL_POSIX_API_H_ #define GFAL_POSIX_API_H_ #if !defined(__GFAL2_H_INSIDE__) && !defined(__GFAL2_BUILD__) # warning "Direct inclusion of gfal2 headers is deprecated. Please, include only gfal_api.h or gfal_plugins_api.h" #endif #include #include #include #include #include #include #include #include #ifndef ENOATTR #define ENOATTR ENODATA #endif #if defined __APPLE__ #include #else #if defined __GLIBC_PREREQ && __GLIBC_PREREQ(2,27) #include #else #include #endif #endif #include #include #ifdef __cplusplus extern "C" { #endif /*! \defgroup posix_group POSIX-like API */ /*! \addtogroup posix_group @{ */ /** * Warnings about POSIX functions : some protocols do not permit a full POSIX support ( ex : no symlinks in srm:// URLs ) * In case of a call to a unsupported function, -1 code is return and an error is set to EPROTONOSUPPORT */ /** * @brief change file access permissions * * gfal_chmod changes the permissions of each given file according to mode, * Similar behavior to the POSIX chmod ( man 2 chmod ) * * @param url : url of the file or the folder, can be in all supported protocols (lfn, srm, file, guid,..) * @param mode : right to configure * @return return 0 if success else -1 if errors occures. * In case of errors, see @ref gfal_posix_check_error() for error management */ int gfal_chmod(const char *url, mode_t mode); /** * @brief change the name or location of a file * * gfal_rename will rename the specified files by replacing the first occurrence of from in their name by to. * gfal_rename implies two URL with the same protocols and generally on the same server. * Similar behavior to the POSIX rename ( man 2 rename ) * * @param oldurl : the old url of the file * @param newurl : the new url of the file * @return : return 0 if success, else -1 if errors. * In case of errors, see @ref gfal_posix_check_error() for error management */ int gfal_rename(const char *oldurl, const char *newurl); /** * @brief get the file status, follow links * * return informations about the given files * Similar behavior to the POSIX stat ( man 2 stat ) * * @param url : url of the file * @param st : pointer to an allocated struct stat * @return : return 0 if success, else -1 if errors. * In case of errors, see @ref gfal_posix_check_error() for error management */ int gfal_stat(const char *url, struct stat *st); /** * @brief get the file status, does not follow links * * return informations about the given files and links * Similar behavior to the POSIX lstat ( man 2 lstat ) * * In case of protocols without the support of links, GFAL 2.0 \ * convert this call to @ref gfal_stat * * * @param url : url of the file * @param st : pointer to an allocated struct stat * @return : return 0 if success, else -1 if errors. * In case of errors, see @ref gfal_posix_check_error() for error management */ int gfal_lstat(const char *url, struct stat *st); /** * @brief check user permissions for a file * * Similar behavior to the POSIX access ( man 2 access ) * * @param url : url of the file to access * @param amode : access mode to check (R_OK, W_OK, X_OK or F_OK) * @return This routine return 0 if the operation was successful, or -1 if error occured. * In case of errors, see @ref gfal_posix_check_error() for error management */ int gfal_access(const char *url, int amode); /** * @brief resolve a link destination * * Similar behavior to the POSIX readlink ( man 2 readlink ) * * Some protocols can not support this functionality, \ * check the protocol specification for more information. * * @param url : url of the file to access, can be in supported protocols (lfn, srm, file, guid,..) * @param buff : buffer to fill * @param buffsiz : maximum size of the link destination * @return This routine return the size of the link destination \ * if the operation was successful, or -1 if error occured. * In case of errors, see @ref gfal_posix_check_error() for error management */ ssize_t gfal_readlink(const char *url, char *buff, size_t buffsiz); /** * @brief create a symbolic link * * Similar behavior to the POSIX symlink ( man 2 symlink ) * * Some protocols can not support this functionality, \ * check the protocol specification for more information. * * @param oldurl : url of the original file * @param newurl : url of the symlink * @return This routine return 0 if the operation was successful, or -1 if error occured. * In case of errors, see @ref gfal_posix_check_error() for error management */ int gfal_symlink(const char *oldurl, const char *newurl); /** * @brief unlink a file, delete it * * Similar behavior to the POSIX unlink ( man 2 unlink ) * * @param url : url of the file to delete * @return This routine return 0 if the operation was successful, or -1 if error occured. * In case of errors, see @ref gfal_posix_check_error() for error management */ int gfal_unlink(const char *url); /** * @brief create a directory * * Similar behavior to the POSIX mkdir( man 2 mkdir ) * * @param url : url of the directory to create * @param mode : initial mode right of the directory * @return This routine return 0 if the operation was successful, or -1 if error occured. * In case of errors, see @ref gfal_posix_check_error() for error management */ int gfal_mkdir(const char *url, mode_t mode); /** * @brief open a directory * * Similar behavior to the POSIX opendir ( man 2 opendir ) * * @param url : url of the directory to list * @return This routine return a directory descriptor if the operation was successful, or -1 if error occured. * In case of errors, see @ref gfal_posix_check_error() for error management */ DIR *gfal_opendir(const char *url); /** * @brief read a directory * * the directory descriptor must be provided by @ref gfal_opendir * Similar behavior to the POSIX readdir ( man 2 readdir ) * * @param d : descriptor of the directory * @return This routine return the directory information if the operation was successful, * NULL if the end of the directory is reached * or if error occured. * In case of errors, see @ref gfal_posix_check_error() for error management */ struct dirent *gfal_readdir(DIR *d); /** * @brief close a directory * * Similar behavior to the POSIX closedir ( man 2 closedir ) * * @param d : descriptor of the directory * @return This routine return 0 if the operation was successful, or -1 if error occured. * In case of errors, see @ref gfal_posix_check_error() for error management */ int gfal_closedir(DIR *d); /** * @brief delete a directory * * Similar behavior to the POSIX rmdir ( man 2 rmdir ) * * @param url : url of the directory to suppress * @return This routine return 0 if the operation was successful, or -1 if error occured. * In case of errors, see @ref gfal_posix_check_error() for error management */ int gfal_rmdir(const char *url); /** * @brief creat a file * * Similar behavior to the POSIX creat ( man 2 creat ) * gfal_creat is similar to @ref gfal_open with the flags O_CREAT|O_WRONLY|O_TRUNC * * @param url : url of the file to creat * @param mode : mode of the file. * @return This routine return a valid file descriptor if the operation is a success * or -1 if error occured. * In case of errors, see @ref gfal_posix_check_error() for error management */ int gfal_creat(const char *url, mode_t mode); /** * @brief open a file * * Similar behavior to the POSIX open ( man 2 open ) * gfal_open supports the same flags than the POSIX open call * * O_TRUNC * If the file already exists and is a regular file and the open mode allows writing (i.e., is O_RDWR or O_WRONLY) it will be truncated to length 0. If the file is a FIFO or termi‐ * nal device file, the O_TRUNC flag is ignored. Otherwise the effect of O_TRUNC is unspecified. * * O_APPEND * The file is opened in append mode. Before each write(2), the file offset is positioned at the end of the file, as if with lseek(2). O_APPEND may lead to corrupted files on NFS * file systems if more than one process appends data to a file at once. This is because NFS does not support appending to a file, so the client kernel has to simulate it, which * can't be done without a race condition. * * O_CREAT * If the file does not exist it will be created. The owner (user ID) of the file is set to the effective user ID of the process. The group ownership (group ID) is set either to * the effective group ID of the process or to the group ID of the parent directory (depending on file system type and mount options, and the mode of the parent directory, see the * mount options bsdgroups and sysvgroups described in mount(8)). * * mode specifies the permissions to use in case a new file is created. This argument must be supplied when O_CREAT is specified in flags; if O_CREAT is not specified, then mode is * ignored. The effective permissions are modified by the process's umask in the usual way: The permissions of the created file are (mode & ~umask). Note that this mode only * applies to future accesses of the newly created file; the open() call that creates a read-only file may well return a read/write file descriptor. * * * * O_DIRECT * Try to minimize cache effects of the I/O to and from this file. In general this will degrade performance, but it is useful in special situations, such as when applications do * their own caching. File I/O is done directly to/from user space buffers. The O_DIRECT flag on its own makes at an effort to transfer data synchronously, but does not give the * guarantees of the O_SYNC that data and necessary metadata are transferred. To guarantee synchronous I/O the O_SYNC must be used in addition to O_DIRECT. See NOTES below for fur‐ * ther discussion. * * O_LARGEFILE * (LFS) Allow files whose sizes cannot be represented in an off_t (but can be represented in an off64_t) to be opened. The _LARGEFILE64_SOURCE macro must be defined (before includ‐ * ing any header files) in order to obtain this definition. Setting the _FILE_OFFSET_BITS feature test macro to 64 (rather than using O_LARGEFILE) is the preferred method of * obtaining method of accessing large files on 32-bit systems * * * @param url : url of the file to open * @param flags : flags to use ( similar to open ) * @param mode (optional) : mode of the file in case of a new file. * @return This routine return a valid file descriptor if the operation is a success * or -1 if error occured. * In case of errors, see @ref gfal_posix_check_error() for error management */ int gfal_open(const char *url, int flags, ...); /** * @brief reposition read/write file offset * * Similar behavior to the POSIX lseek ( man 2 lseek ) * * @param fd : gfal file descriptor of the file * @param off : offset to set * @param flags : offset mode * @return This routine return the current offset if the operation is a success * or -1 if error occured. * In case of errors, see @ref gfal_posix_check_error() for error management */ off_t gfal_lseek(int fd, off_t off, int flags); /** * @brief close a gfal file descriptor * * Similar behavior to the POSIX close ( man 2 close ) * * @param fd : gfal file descriptor of the file * @return This routine return 0 if the operation is a success * or -1 if error occured. * In case of errors, see @ref gfal_posix_check_error() for error management */ int gfal_close(int fd); /** * @brief read from a gfal file descriptor * * Similar behavior to the POSIX read ( man 2 read ) * * @param fd : gfal file descriptor of the file * @param buff : buffer for the read content * @param size : maximum size to read * @return On success, the number of bytes read is returned * (zero indicates end of file), and the file position is advanced * by this number. It is not an error if this number is * smaller than the number of bytes requested.In case of errors, * see @ref gfal_posix_check_error() for error management * In case of errors, see @ref gfal_posix_check_error() for error management */ ssize_t gfal_read(int fd, void *buff, size_t size); /** * @brief write from a gfal file descriptor * * Similar behavior to the POSIX write ( man 2 write ) * * @param fd : gfal file descriptor of the file * @param buff : buffer for the write content * @param size : maximum size to write * @return On success, the number of bytes written is returned. * In case of errors, -1 is returned * see @ref gfal_posix_check_error() for error management */ ssize_t gfal_write(int fd, const void *buff, size_t size); /** * @brief flush the given file descriptor * * flush the current fiel descriptor, clear the cache \ * and commit the changes. * This call is reserved fora futur usage. * * @param fd : gfal file descriptor of the file * @return On success, return 0. * In case of errors, -1 is returned * see @ref gfal_posix_check_error() for error management */ int gfal_flush(int fd); /** * @brief parallel read from a gfal file descriptor * * Similar behavior to the POSIX pread ( man 2 pread ) * * In gfal2, this call is thread-safe and allows the usage of \ * parallels read at the same time on the same file descriptor. * * This is the recommended way to execute parallels transfers. * * This gfal_pread is emulated on filesystem that does not support it * explicitely ( ex : rfio ) * * @param fd : gfal file descriptor of the file * @param buff : buffer for the read content * @param size : maximum size to read * @param offset : offset where the read start * @return On success, the number of bytes read is returned * (zero indicates end of file), and the file position is advanced * by this number. It is not an error if this number is * smaller than the number of bytes requested.In case of errors, * see @ref gfal_posix_check_error() for error management * In case of errors, see @ref gfal_posix_check_error() for error management */ ssize_t gfal_pread(int fd, void *buff, size_t size, off_t offset); /** * @brief parallel write from a gfal file descriptor * * Similar behavior to the POSIX pwrite ( man 2 pwrite ) * * In gfal2, this function is thread-safe and allows the usage of * parallels write at the same time on the same file descriptor. * * This is the recommended way to execute parallels transfers. * * This gfal_pwrite is emulated on filesystem that does not support it * explicitely ( ex : rfio ) * * @param fd : gfal file descriptor of the file * @param buff : buffer for the write content * @param size : maximum size to write * @param offset : offset where the write start * @return On success, the number of bytes read is returned * (zero indicates end of file), and the file position is advanced * by this number. It is not an error if this number is * smaller than the number of bytes requested.In case of errors, * see @ref gfal_posix_check_error() for error management * In case of errors, see @ref gfal_posix_check_error() for error management */ ssize_t gfal_pwrite(int fd, const void *buff, size_t size, off_t offset); /** * @brief retrieve an extended attribute value * * similar to the getxattr function * * In gfal2, this is the standard way to interact \ * with advanced file properties * * ex : gfal_getxattr("srm://myurl/endpoint", "user.replicas", * buff, 256); * * @param url : url of the file * @param name : key of the extended attribute * @param value : buffer for the extended attribute value * @param size : maximum size of the buffer * @return On success, return the size of the extended attribute. * In case of errors, -1 is returned * see @ref gfal_posix_check_error() for error management */ ssize_t gfal_getxattr(const char *url, const char *name, void *value, size_t size); /** * @brief retrieve a list of the extended attributes availables * * similar to the etxattr function * * * @param url : url of the file * @param list : buffer for the list of availables attributes concatened. * @param size : maximum size of the buffer * @return On success, return the size of the list. * In case of errors, -1 is returned * see @ref gfal_posix_check_error() for error management */ ssize_t gfal_listxattr(const char *url, char *list, size_t size); /** * @brief define an extended attribute value * * similar to the listxattr function * * In gfal2, this is the standard way to interact \ * with advanced file properties * * ex : gfal_setxattr("srm://myurl/endpoint", "user.status", * "ONLINE", strlen("ONLINE"),0); * // -> simple brings_online * * @param url : url of the file * @param name : key of the extended attribute * @param value : buffer for the extended attribute value * @param size : size of the attribute value * @param flags : mode * @return On success, return 0 . * In case of errors, -1 is returned * see @ref gfal_posix_check_error() for error management */ int gfal_setxattr(const char *url, const char *name, const void *value, size_t size, int flags); /** * @brief delete a extended attribute value * * similar to the removexattr function * * * @param url : url of the file * @param name : key of the extended attribute * @return On success, return 0. * In case of errors, -1 is returned * see @ref gfal_posix_check_error() for error management */ int gfal_removexattr(const char *url, const char *name); /** * safe and easy error management : * -> if last error * -> print on stderr, return 1 * -> else do nothing * @return 0 if no error, 1 else */ GFAL2_DEPRECATED_NOALT int gfal_posix_check_error(); /** * clear the last error \ * provided for convenience. */ void gfal_posix_clear_error(); /** * display the last error to the standard output \ * and clear it. * provided for convenience. */ GFAL2_DEPRECATED_NOALT void gfal_posix_release_error(); /** * Get the string representation of the last error * @param buff_err : buffer where string representation will be stored. * @param s_err : maximum message size * @return return pointer to buff_err for convenience */ char *gfal_posix_strerror_r(char *buff_err, size_t s_err); /** * display the last error recorded to the standard output, * provided for convenience. */ GFAL2_DEPRECATED_NOALT void gfal_posix_print_error(); /** * return the code associated with the last error. * In case of POSIX calls and when possible, this code is similar to the local errno * @return last error code or 0 if nothing */ int gfal_posix_code_error(); /** * get a string representation of the gfal2 version */ const char *gfal2_version(); /** * @brief get context for advanced operation * Return the gfal2 context used for POSIX operations * Allow to do advanced operation ( config, checksum, transfer ) on this context * @warning Delete this context leads to undefined behavior. */ gfal2_context_t gfal_posix_get_handle(); /** @} End of the POSIX group */ #ifdef __cplusplus } #endif #endif /* GFAL_POSIX_API_H_ */ gfal2-v2.23.0/src/core/posix/gfal_posix_ng.c000066400000000000000000000322421465240014500206270ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include "gfal_posix_api.h" /** * Context per thread */ typedef struct _Gfal2ThreadContext { gfal2_context_t context; GError *error; } Gfal2ThreadContext; /** Release the context bound to a thread when the later ends */ static void gfal_posix_free_context(gpointer ptr) { Gfal2ThreadContext *thread_context = (Gfal2ThreadContext*)ptr; g_error_free(thread_context->error); gfal2_context_free(thread_context->context); g_free(thread_context); } /** Thread local storage */ static GPrivate *thread_private; __attribute__((constructor)) static void init_thread_private() { g_thread_init(NULL); thread_private = g_private_new(gfal_posix_free_context); } /** Return the context for the current thread */ static Gfal2ThreadContext* gfal_posix_get_thread_context() { Gfal2ThreadContext *thread_context = g_private_get(thread_private); if (thread_context == NULL) { thread_context = g_malloc0(sizeof(Gfal2ThreadContext)); g_private_set(thread_private, thread_context); } return thread_context; } /** Return the gfal2 handle for the current thread */ gfal2_context_t gfal_posix_get_handle() { errno = 0; Gfal2ThreadContext *thread_context = gfal_posix_get_thread_context(); if (thread_context->context == NULL) { thread_context->context = gfal2_context_new(&thread_context->error); if (thread_context->error != NULL) { errno = thread_context->error->code; } } return thread_context->context; } /** * Register the last error in the handle and display a warning if an error was registered and not deleted */ static void gfal_posix_register_internal_error(const char *prefix, GError *tmp_err) { Gfal2ThreadContext *thread_context = gfal_posix_get_thread_context(); if (thread_context->error != NULL) { gfal2_log(G_LOG_LEVEL_WARNING, "[%s] Warning : existing registered error replaced! old error: %s ", prefix, thread_context->error->message); g_clear_error(&thread_context->error); } gfal2_propagate_prefixed_error(&thread_context->error, tmp_err, prefix); errno = tmp_err->code; } void gfal_posix_clear_error() { g_clear_error(&gfal_posix_get_thread_context()->error); errno = 0; } int gfal_posix_code_error() { GError *err = gfal_posix_get_thread_context()->error; if (err == NULL) { return 0; } return err->code; } int gfal_posix_check_error() { GError *err = gfal_posix_get_thread_context()->error; if (err != NULL) { g_printerr("[gfal] %s\n", err->message); return 1; } return 0; } char *gfal_posix_strerror_r(char *buff_err, size_t s_err) { GError *err = gfal_posix_get_thread_context()->error; if (err == NULL) { strerror_r(0, buff_err, s_err); } else { snprintf(buff_err, s_err, "[gfal] %s", err->message); } return buff_err; } // Avoid deprecation warning static void _gfal_posix_print_error() { GError *err = gfal_posix_get_thread_context()->error; if (err != NULL ) { g_printerr("[gfal]%s\n", err->message); } else if (errno != 0) { char *sterr = strerror(errno); g_printerr("[gfal] errno reported by external lib: %s", sterr); } else { g_printerr("[gfal] No gfal error reported\n"); } } void gfal_posix_print_error() { _gfal_posix_print_error(); } void gfal_posix_release_error() { _gfal_posix_print_error(); gfal_posix_clear_error(); } int gfal_access(const char *path, int mode) { GError *tmp_err = NULL; gfal2_context_t handle; if ((handle = gfal_posix_get_handle()) == NULL) { return -1; } int ret = gfal2_access(handle, path, mode, &tmp_err); if (ret < 0) { gfal_posix_register_internal_error(__func__, tmp_err); } return ret; } int gfal_chmod(const char *path, mode_t mode) { GError *tmp_err = NULL; gfal2_context_t handle; if ((handle = gfal_posix_get_handle()) == NULL) { return -1; } int ret = gfal2_chmod(handle, path, mode, &tmp_err); if (tmp_err) { gfal_posix_register_internal_error(__func__, tmp_err); } return ret; } int gfal_rename(const char *oldpath, const char *newpath) { GError *tmp_err = NULL; gfal2_context_t handle; if ((handle = gfal_posix_get_handle()) == NULL) { return -1; } int ret = gfal2_rename(handle, oldpath, newpath, &tmp_err); if (tmp_err) { gfal_posix_register_internal_error(__func__, tmp_err); } return ret; } int gfal_stat(const char *path, struct stat *buff) { gfal2_context_t handle; GError *tmp_err = NULL; if ((handle = gfal_posix_get_handle()) == NULL) { return -1; } int ret = gfal2_stat(handle, path, buff, &tmp_err); if (tmp_err) { gfal_posix_register_internal_error(__func__, tmp_err); } return ret; } int gfal_lstat(const char *path, struct stat *buff) { gfal2_context_t handle; GError *tmp_err = NULL; if ((handle = gfal_posix_get_handle()) == NULL) { return -1; } int ret = gfal2_lstat(handle, path, buff, &tmp_err); if (ret) { gfal_posix_register_internal_error(__func__, tmp_err); } return ret; } int gfal_mkdirp(const char *path, mode_t mode) { GError *tmp_err = NULL; gfal2_context_t handle; if ((handle = gfal_posix_get_handle()) == NULL) { return -1; } int ret = gfal2_mkdir(handle, path, mode, &tmp_err); if (tmp_err) { gfal_posix_register_internal_error(__func__, tmp_err); errno = tmp_err->code; } return ret; } int gfal_mkdir(const char *path, mode_t mode) { return gfal_mkdirp(path, mode); } int gfal_rmdir(const char *path) { GError *tmp_err = NULL; gfal2_context_t handle; if ((handle = gfal_posix_get_handle()) == NULL) { return -1; } int ret = gfal2_rmdir(handle, path, &tmp_err); if (tmp_err) { gfal_posix_register_internal_error(__func__, tmp_err); } return ret; } DIR *gfal_opendir(const char *name) { GError *tmp_err = NULL; gfal2_context_t handle; if ((handle = gfal_posix_get_handle()) == NULL) { return NULL; } DIR *dir = gfal2_opendir(handle, name, &tmp_err); if (tmp_err) { gfal_posix_register_internal_error(__func__, tmp_err); } return dir; } struct dirent *gfal_readdir(DIR *d) { GError *tmp_err = NULL; gfal2_context_t handle; if ((handle = gfal_posix_get_handle()) == NULL) { return NULL; } struct dirent *res = gfal2_readdir(handle, d, &tmp_err); if (tmp_err) { gfal_posix_register_internal_error(__func__, tmp_err); } return res; } int gfal_closedir(DIR *d) { GError *tmp_err = NULL; gfal2_context_t handle; if ((handle = gfal_posix_get_handle()) == NULL) { return -1; } if (d == NULL) { g_set_error(&tmp_err, gfal2_get_core_quark(), EFAULT, "File descriptor is NULL"); gfal_posix_register_internal_error(__func__, tmp_err); return -1; } int ret = gfal2_closedir(handle, d, &tmp_err); if (tmp_err) { gfal_posix_register_internal_error(__func__, tmp_err); } return ret; } int gfal_open(const char *path, int flag, ...) { va_list va; va_start(va, flag); mode_t mode = va_arg(va, mode_t); va_end(va); GError *tmp_err = NULL; gfal2_context_t handle; if ((handle = gfal_posix_get_handle()) == NULL) { return -1; } int fd = gfal2_open2(handle, path, flag, mode, &tmp_err); if (tmp_err) { gfal_posix_register_internal_error(__func__, tmp_err); } return fd; } int gfal_creat(const char *filename, mode_t mode) { return (gfal_open(filename, O_WRONLY | O_CREAT | O_TRUNC, mode)); } ssize_t gfal_read(int fd, void *buff, size_t s_buff) { GError *tmp_err = NULL; gfal2_context_t handle; if ((handle = gfal_posix_get_handle()) == NULL) { return -1; } int ret = gfal2_read(handle, fd, buff, s_buff, &tmp_err); if (tmp_err) { gfal_posix_register_internal_error(__func__, tmp_err); } return ret; } ssize_t gfal_write(int fd, const void *buff, size_t s_buff) { GError *tmp_err = NULL; gfal2_context_t handle; if ((handle = gfal_posix_get_handle()) == NULL) { return -1; } int ret = gfal2_write(handle, fd, buff, s_buff, &tmp_err); if (tmp_err) { gfal_posix_register_internal_error(__func__, tmp_err); } return ret; } int gfal_close(int fd) { GError *tmp_err = NULL; gfal2_context_t handle; if ((handle = gfal_posix_get_handle()) == NULL) { return -1; } int ret = gfal2_close(handle, fd, &tmp_err); if (tmp_err) { gfal_posix_register_internal_error(__func__, tmp_err); } return ret; } int gfal_symlink(const char *oldpath, const char *newpath) { GError *tmp_err = NULL; gfal2_context_t handle; if ((handle = gfal_posix_get_handle()) == NULL) { return -1; } int ret = gfal2_symlink(handle, oldpath, newpath, &tmp_err); if (tmp_err) { gfal_posix_register_internal_error(__func__, tmp_err); } return ret; } off_t gfal_lseek(int fd, off_t offset, int whence) { GError *tmp_err = NULL; gfal2_context_t handle; if ((handle = gfal_posix_get_handle()) == NULL) { return -1; } int ret = gfal2_lseek(handle, fd, offset, whence, &tmp_err); if (tmp_err) { gfal_posix_register_internal_error(__func__, tmp_err); } return ret; } ssize_t gfal_getxattr(const char *path, const char *name, void *value, size_t size) { GError *tmp_err = NULL; gfal2_context_t handle; if ((handle = gfal_posix_get_handle()) == NULL) { return -1; } ssize_t ret = gfal2_getxattr(handle, path, name, value, size, &tmp_err); if (tmp_err) { gfal_posix_register_internal_error(__func__, tmp_err); } return ret; } ssize_t gfal_readlink(const char *path, char *buff, size_t buffsiz) { gfal2_context_t handle; GError *tmp_err = NULL; if ((handle = gfal_posix_get_handle()) == NULL) { return -1; } ssize_t ret = gfal2_readlink(handle, path, buff, buffsiz, &tmp_err); if (tmp_err) { gfal_posix_register_internal_error(__func__, tmp_err); } return ret; } int gfal_unlink(const char *path) { gfal2_context_t handle; GError *tmp_err = NULL; if ((handle = gfal_posix_get_handle()) == NULL) { return -1; } int ret = gfal2_unlink(handle, path, &tmp_err); if (tmp_err) { gfal_posix_register_internal_error(__func__, tmp_err); } return ret; } ssize_t gfal_listxattr(const char *path, char *list, size_t size) { GError *tmp_err = NULL; gfal2_context_t handle; if ((handle = gfal_posix_get_handle()) == NULL) { return -1; } ssize_t ret = gfal2_listxattr(handle, path, list, size, &tmp_err); if (tmp_err) { gfal_posix_register_internal_error(__func__, tmp_err); } return ret; } int gfal_setxattr(const char *path, const char *name, const void *value, size_t size, int flags) { GError *tmp_err = NULL; gfal2_context_t handle; if ((handle = gfal_posix_get_handle()) == NULL) { return -1; } int ret = gfal2_setxattr(handle, path, name, value, size, flags, &tmp_err); if (tmp_err) { gfal_posix_register_internal_error(__func__, tmp_err); } return ret; } int gfal_removexattr(const char *path, const char *name) { GError *tmp_err = NULL; g_set_error(&tmp_err, gfal2_get_core_quark(), ENOSYS, "Not implemented"); gfal_posix_register_internal_error(__func__, tmp_err); return -1; } ssize_t gfal_pread(int fd, void *buffer, size_t count, off_t offset) { GError *tmp_err = NULL; gfal2_context_t handle; if ((handle = gfal_posix_get_handle()) == NULL) { return -1; } ssize_t ret = gfal2_pread(handle, fd, buffer, count, offset, &tmp_err); if (tmp_err) { gfal_posix_register_internal_error(__func__, tmp_err); } return ret; } ssize_t gfal_pwrite(int fd, const void *buffer, size_t count, off_t offset) { GError *tmp_err = NULL; gfal2_context_t handle; if ((handle = gfal_posix_get_handle()) == NULL) { return -1; } ssize_t ret = gfal2_pwrite(handle, fd, buffer, count, offset, &tmp_err); if (tmp_err) { gfal_posix_register_internal_error( __func__, tmp_err); } return ret; } int gfal_flush(int fd) { return 0; } gfal2-v2.23.0/src/core/transfer/000077500000000000000000000000001465240014500163235ustar00rootroot00000000000000gfal2-v2.23.0/src/core/transfer/CMakeLists.txt000066400000000000000000000030171465240014500210640ustar00rootroot00000000000000cmake_minimum_required (VERSION 2.6) if (MAIN_TRANSFER) file (GLOB src_trans "*.c") list (APPEND header_transfer "gfal_transfer.h" "gfal_transfer_plugins.h" ) add_definitions( ${GLIB2_PKG_CFLAGS} ${GTHREAD2_PKG_CFLAGS}) add_library(gfal2_transfer SHARED ${src_trans} ${gfal2_utils_src}) target_link_libraries(gfal2_transfer ${GLIB2_PKG_LIBRARIES} ${GTHREAD2_PKG_LIBRARIES}) target_link_libraries(gfal2_transfer ${UUID_PKG_LIBRARIES} ${OUTPUT_NAME_MAIN}) set_target_properties(gfal2_transfer PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/src/core VERSION ${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_PATCH} SOVERSION ${VERSION_MAJOR} CLEAN_DIRECT_OUTPUT 1 OUTPUT_NAME "gfal_transfer") add_PkgConfigFile_for_Library("gfal_transfer.pc" gfal2_transfer HEADER_DIRS "gfal2" DESCRIPTION "gfal_transfer pkgconfig file" REQUIRES "gfal2") install(TARGETS gfal2_transfer LIBRARY DESTINATION ${LIB_INSTALL_DIR} ) install(FILES ${CMAKE_CURRENT_BINARY_DIR}/gfal_transfer.pc DESTINATION ${PKGCONFIG_FILES_DIR}) install(FILES ${header_transfer} DESTINATION ${INCLUDE_INSTALL_DIR}/gfal2/transfer) endif (MAIN_TRANSFER) gfal2-v2.23.0/src/core/transfer/gfal_transfer.h000066400000000000000000000416561465240014500213250ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #pragma once #ifndef GFAL_TRANSFER_H_ #define GFAL_TRANSFER_H_ #if !defined(__GFAL2_H_INSIDE__) && !defined(__GFAL2_BUILD__) # warning "Direct inclusion of gfal2 headers is deprecated. Please, include only gfal_api.h or gfal_plugins_api.h" #endif #include #include #include #ifdef __cplusplus extern "C" { #endif /*! \defgroup transfer_group File Transfer API */ /*! \addtogroup transfer_group @{ */ /** * @brief container for transfer related parameters * */ typedef struct _gfalt_params_t* gfalt_params_t; /** * @brief internal status of a copy file action * */ typedef struct _gfalt_transfer_status* gfalt_transfer_status_t; /** * @brief copy gfalt_monitor_transfer * This function is called callback_mperiod milli-seconds in order to provide information and a control on the transfers. * @param src : URL of the source file * @param dst : URL of the dest file * @param user_data : external pointer provided before * */ typedef void (*gfalt_monitor_func)(gfalt_transfer_status_t h, const char* src, const char* dst, gpointer user_data); /** * @brief Predefined stages. */ extern GQuark GFAL_EVENT_PREPARE_ENTER; /**< Triggered before entering preparation */ extern GQuark GFAL_EVENT_PREPARE_EXIT; /**< Triggered after exiting the preparation */ extern GQuark GFAL_EVENT_TRANSFER_ENTER; /**< Triggered before entering the transfer */ extern GQuark GFAL_EVENT_TRANSFER_EXIT; /**< Triggered after exiting the transfer */ extern GQuark GFAL_EVENT_CLOSE_ENTER; /**< Triggered before entering the closing (putdone) */ extern GQuark GFAL_EVENT_CLOSE_EXIT; /**< Triggered after exiting the closing (putdone) */ extern GQuark GFAL_EVENT_CHECKSUM_ENTER; /**< Triggered before entering checksum validation */ extern GQuark GFAL_EVENT_CHECKSUM_EXIT; /**< Triggered after exiting checksum validation */ extern GQuark GFAL_EVENT_CANCEL_ENTER; /**< Triggered before the cancellation logic */ extern GQuark GFAL_EVENT_CANCEL_EXIT; /**< Triggered after the cancellation logic */ extern GQuark GFAL_EVENT_OVERWRITE_DESTINATION; /**< Triggered before overwriting */ extern GQuark GFAL_EVENT_LIST_ENTER; /**< Triggered before listing the urls to be transferred */ extern GQuark GFAL_EVENT_LIST_ITEM; /**< Triggered once per url pair to be transferred */ extern GQuark GFAL_EVENT_LIST_EXIT; /**< Triggered after listing the urls to be transferred */ extern GQuark GFAL_EVENT_TRANSFER_TYPE; /**< Triggered to register the transfer type being done */ extern GQuark GFAL_EVENT_IPV4; /**< Triggered to register the transfer is done over IPv4 */ extern GQuark GFAL_EVENT_IPV6; /**< Triggered to register the transfer is done over IPv6 */ extern GQuark GFAL_EVENT_EVICT; /**< Triggered after a file eviction operation */ extern GQuark GFAL_EVENT_CLEANUP; /**< Triggered after a delete to cleanup a failed transfer */ /** * Types for for GFAL_EVENT_TRANSFER_TYPE */ #define GFAL_TRANSFER_TYPE_STREAMED "streamed" #define GFAL_TRANSFER_TYPE_PUSH "3rd push" #define GFAL_TRANSFER_TYPE_PULL "3rd pull" /** * Enable or disable DNS resolution within the copy function */ #define RESOLVE_DNS "RESOLVE_DNS" /** Trigger of the event */ typedef enum { GFAL_EVENT_SOURCE = 0, /**< Event triggered by the source */ GFAL_EVENT_DESTINATION, /**< Event triggered by the destination */ GFAL_EVENT_NONE /**< Event triggered by the transfer */ } gfal_event_side_t; /** * @brief Event message. */ struct _gfalt_event { gfal_event_side_t side; /**< Which side triggered the stage change */ gint64 timestamp; /**< Timestamp in milliseconds */ GQuark stage; /**< Stage. You can check the predefined ones. */ GQuark domain; /**< Domain/protocol of this stage. i.e. SRM*/ const char *description; /**< Additional description */ }; /** * Event message */ typedef struct _gfalt_event* gfalt_event_t; /** * This function is called when a transfer changes its stage. * @param e : Event message. * @param user_data : external pointer provided before */ typedef void (*gfalt_event_func)(const gfalt_event_t e, gpointer user_data); /** * Checksum verification mode */ typedef enum { /// Don't verify checksum GFALT_CHECKSUM_NONE = 0x00, /// Compare user provided checksum vs source GFALT_CHECKSUM_SOURCE = 0x01, /// Compare user provided checksum vs destination GFALT_CHECKSUM_TARGET = 0x02, /// Compare user provided checksum vs both, *or* source checksum vs target checksum GFALT_CHECKSUM_BOTH = (GFALT_CHECKSUM_SOURCE | GFALT_CHECKSUM_TARGET) } gfalt_checksum_mode_t; /** * Create a new parameter handle */ gfalt_params_t gfalt_params_handle_new(GError ** err); /** * Delete a created parameters handle */ void gfalt_params_handle_delete(gfalt_params_t params, GError ** err); /** Create a copy of a parameter handle */ gfalt_params_t gfalt_params_handle_copy(gfalt_params_t params, GError ** err); /** * Define the maximum time acceptable for the file transfer */ gint gfalt_set_timeout(gfalt_params_t, guint64 timeout, GError** err); /** * Get the maximum connexion timeout */ guint64 gfalt_get_timeout(gfalt_params_t handle, GError** err); /** * Define the maximum number of parallels connexion to use for the file transfer */ gint gfalt_set_nbstreams(gfalt_params_t, guint nbstreams, GError** err); /** * Get the maximum number of parallels streams to use for the transfer */ guint gfalt_get_nbstreams(gfalt_params_t params, GError** err); /** * Define the size of the tcp buffer size for network transfer */ gint gfalt_set_tcp_buffer_size(gfalt_params_t, guint64 tcp_buffer_size, GError** err); /** * get the size of the tcp buffer size for network transfer */ guint64 gfalt_get_tcp_buffer_size(gfalt_params_t params, GError** err); /** * Enable or disable the non-third party transfer execution ( default : true ) */ gint gfalt_set_local_transfer_perm(gfalt_params_t, gboolean local_transfer_status, GError ** err); /** * Get the current authorization for the non-third party transfer execution */ gboolean gfalt_get_local_transfer_perm(gfalt_params_t, GError ** err); /** * Set the source spacetoken for SRM transfers */ gint gfalt_set_src_spacetoken(gfalt_params_t params, const char* srm_spacetoken, GError** err); /** * Get the source spacetoken for SRM transfers */ const gchar* gfalt_get_src_spacetoken(gfalt_params_t params, GError** err); /** * Set the destination spacetoken for SRM transfers */ gint gfalt_set_dst_spacetoken(gfalt_params_t params, const char* srm_spacetoken, GError** err); /** * Get the destination spacetoken for SRM transfers */ const gchar* gfalt_get_dst_spacetoken(gfalt_params_t params, GError** err); /** * set the replace/overwrite option. * default : false * when True, if a destination file already exist, it is deleted before the copy. */ gint gfalt_set_replace_existing_file(gfalt_params_t, gboolean replace, GError** err); /** * Get the policy in case of destination file already existing ( replace or cancel ) * default : cancel */ gboolean gfalt_get_replace_existing_file(gfalt_params_t, GError** err); /** * Set the strict copy mode * default : false * In the strict copy mode, the destination/source checks are skipped. * only the minimum of the operations are done * This option can leads to undefined behavior depending of the underlying protocol */ gint gfalt_set_strict_copy_mode(gfalt_params_t, gboolean strict_mode, GError** err); /** * Get the strict copy mode value */ gboolean gfalt_get_strict_copy_mode(gfalt_params_t, GError** err); /** * @deprecated Equivalent to gfalt_get_checksum_check(params, GFALT_CHECKSUM_BOTH, err) * Force additional checksum verification between source and destination * an Error is return by the copy function is case of checksum failure. * @warning for safety reason, even in case of checksum failure the destination file is not removed. */ GFAL2_DEPRECATED(gfalt_set_checksum) gint gfalt_set_checksum_check(gfalt_params_t, gboolean value, GError** err); /** * @deprecated * Get the checksum verification boolean */ GFAL2_DEPRECATED(gfalt_get_checksum) gboolean gfalt_get_checksum_check(gfalt_params_t, GError** err); /** * @deprecated gfalt_set_checksum * Set an user-defined checksum for file content verification * Setting NULL & NULL clear the current one. * This function requires to enable global checksum verification with \ref gfalt_set_checksum_check * @param param : parameter handle * @param chktype : checksum type string ( MD5, ADLER32, CRC32, etc... ) * @param checksum : value of checksum in string format * @param err : GError error report */ GFAL2_DEPRECATED(gfalt_set_checksum) gint gfalt_set_user_defined_checksum(gfalt_params_t param, const gchar* chktype, const gchar* checksum, GError** err); /** * @deprecated gfalt_get_checksum * Get the current user-defined checksum for file content verification * If current user-defined checksum is NULL, both of the buffer are set to empty string * If the value is set, but not the type, ADLER32 will be assumed */ GFAL2_DEPRECATED(gfalt_get_checksum) gint gfalt_get_user_defined_checksum(gfalt_params_t params, gchar* chktype_buff, size_t chk_type_len, gchar* checksum_buff, size_t checksum_len, GError** err); /** * Set the checksum configuration to use * @param mode For GFALT_CHECKSUM_SOURCE or GFALT_CHECKSUM_TARGET only, the checksum value is mandatory. * For GFALT_CHECKSUM_BOTH, the checksum value can be NULL, as the verification can be done end to end. * @param type Checksum algorithm to use. Support depends on protocol and storage, but ADLER32 and MD5 * are normally safe bets. If NULL, previous type is kept. * @param checksum Expected checksum value. Can be NULL for GFALT_CHECKSUM_BOTH mode. If NULL, clears value. * @param err GError error report * @return 0 on success, < 0 on failure * @version 2.13.0 */ gint gfalt_set_checksum(gfalt_params_t params, gfalt_checksum_mode_t mode, const gchar* type, const gchar *checksum, GError **err); /** * Get the checksum configuration * @param type_buff Put in this buffer the configured algorithm (or "\0" if none). * @param type_buff_len algorithm_buffer capacity. * @param checksum_buff Put in this buffer the configured checksum value ("\0" if none). * @param checksum_buff_len checksum_buff capacity. * @param err GError error report * @return The configured checksum mode * @version 2.13.0 */ gfalt_checksum_mode_t gfalt_get_checksum(gfalt_params_t params, gchar* type_buff, size_t type_buff_len, gchar* checksum_buff, size_t checksum_buff_len, GError **err); /** * Get only the checksum mode configured * @return The configured checksum mode */ gfalt_checksum_mode_t gfalt_get_checksum_mode(gfalt_params_t params, GError **err); /** * Enable or disable the destination parent directory creation */ gint gfalt_set_create_parent_dir(gfalt_params_t, gboolean create_parent, GError** err); /** * Get the parent directory creation value */ gboolean gfalt_get_create_parent_dir(gfalt_params_t, GError** err); /** * Enable or disable the clean-up happening when a transfer fails */ gint gfalt_set_transfer_cleanup(gfalt_params_t, gboolean transfer_cleanup, GError** err); /** * Get the transfer clean-up mode */ gboolean gfalt_get_transfer_cleanup(gfalt_params_t, GError** err); /** * Enable or disable usage of TPC proxy delegation */ gint gfalt_set_use_proxy_delegation(gfalt_params_t, gboolean proxy_delegation, GError** err); /** * Get the usage of TPC proxy delegation value */ gboolean gfalt_get_use_proxy_delegation(gfalt_params_t, GError** err); /** * Set the SciTag transfer flow label */ gint gfalt_set_scitag(gfalt_params_t, guint scitag, GError** err); /** * Get the SciTag transfer flow label */ guint gfalt_get_scitag(gfalt_params_t, GError** err); /** * Enable or disable usage of file eviction */ gint gfalt_set_use_evict(gfalt_params_t, gboolean evict, GError** err); /** * Get the usage of file eviction */ gboolean gfalt_get_use_evict(gfalt_params_t, GError** err); /** * Set the request id used in the staging operation */ gint gfalt_set_stage_request_id(gfalt_params_t, const char* request_id, GError** err); /** * Get the request id used in the staging operation */ const gchar* gfalt_get_stage_request_id(gfalt_params_t, GError** err); /** * Set the transfer metadata to be sent to the transfer destination */ gint gfalt_set_transfer_metadata(gfalt_params_t, const char* metadata, GError** err); /** * Get the transfer metadata to be sent to the transfer destination */ const gchar* gfalt_get_transfer_metadata(gfalt_params_t, GError** err); /** * Set the archive metadata to be sent to the transfer destination */ gint gfalt_set_archive_metadata(gfalt_params_t, const char* metadata, GError** err); /** * Get the transfer metadata to be sent to the transfer destination */ const gchar* gfalt_get_archive_metadata(gfalt_params_t, GError** err); /** * @brief Add a new callback for monitoring the current transfer * Adding the same callback with a different udata will just change the udata and the free method, but the callback will not be called twice. * In this case, udata_free will be called with the old data. * udata_free can be left to NULL */ gint gfalt_add_monitor_callback(gfalt_params_t params, gfalt_monitor_func callback, gpointer udata, GDestroyNotify udata_free, GError** err); /** * @brief Remove an installed monitor callback * It will call the method registered to free the user data */ gint gfalt_remove_monitor_callback(gfalt_params_t params, gfalt_monitor_func callback, GError** err); /** * @brief Add a new callback for event monitoring * Adding the same callback with a different udata will just change the udata, but the callback will not be called twice. * In this case, udata_free will be called with the old data. * udata_free can be left to NULL */ gint gfalt_add_event_callback(gfalt_params_t params, gfalt_event_func callback, gpointer udata, GDestroyNotify udata_free, GError** err); /** * @brief Remove an installed callback * It will call the method registered to free the user data */ gint gfalt_remove_event_callback(gfalt_params_t params, gfalt_event_func callback, GError** err); /** * @brief copy function * start a synchronous copy of the file * @param context : gfal2 context * @param params parameter handle ( \ref gfalt_parameters_new ) * @param src source URL supported by GFAL * @param dst destination URL supported by GFAL * @param err the error is put here */ int gfalt_copy_file(gfal2_context_t context, gfalt_params_t params, const char* src, const char* dst, GError** err); /** * @brief bulk copy operation * If not provided by the plugin, it will fallback to a serialized implementation * Note that file_errors will point to an array of nbfiles pointers to GError, where each one * corresponds to the source and destination pair in the same position * op_error will contain an error if something happened _before_ file transferring could be attempted */ int gfalt_copy_bulk(gfal2_context_t context, gfalt_params_t params, size_t nbfiles, const char* const * srcs, const char* const * dsts, const char* const* checksums, GError** op_error, GError*** file_errors); /** * Get a transfer status indicator */ gint gfalt_copy_get_status(gfalt_transfer_status_t, GError ** err); /** * Get an estimation of the average baudrate in bytes/s */ size_t gfalt_copy_get_average_baudrate(gfalt_transfer_status_t, GError ** err); /** * Get an estimation of the instant baudrate in bytes/s */ size_t gfalt_copy_get_instant_baudrate(gfalt_transfer_status_t, GError ** err); /** * Get the current number of bytes transferred */ GFAL2_DEPRECATED(gfalt_copy_get_bytes_transferred) size_t gfalt_copy_get_bytes_transfered(gfalt_transfer_status_t, GError ** err); size_t gfalt_copy_get_bytes_transferred(gfalt_transfer_status_t, GError ** err); /** * Get the elapsed time since the call to \ref gfalt_copy_file */ time_t gfalt_copy_get_elapsed_time(gfalt_transfer_status_t, GError ** err); /** @} End of the File Transfer API */ #ifdef __cplusplus } #endif #endif /* GFAL_TRANSFER_H_ */ gfal2-v2.23.0/src/core/transfer/gfal_transfer_filecopy.c000066400000000000000000000222301465240014500231750ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include static GQuark scope_copy_domain() { return g_quark_from_static_string("GFAL2:CORE:COPY"); } static gfal_plugin_interface* find_copy_plugin(gfal2_context_t context, gfal_url2_check operation, const char* src, const char* dst, void** plugin_data, GError** error) { GList* item = g_list_first(context->plugin_opt.sorted_plugin); void* resu = NULL; while (item != NULL && resu == NULL) { gfal_plugin_interface* plugin_ifce = (gfal_plugin_interface*)item->data; if (plugin_ifce->check_plugin_url_transfer != NULL) { gboolean compatible; if ((compatible = plugin_ifce->check_plugin_url_transfer(plugin_ifce->plugin_data, context, src, dst, operation)) == TRUE) { *plugin_data = plugin_ifce->plugin_data; resu = plugin_ifce; } } item = g_list_next(item); } return resu; } static int trigger_listener_plugins(gfal2_context_t context, gfalt_params_t params, GError** error) { GList *item = g_list_first(context->plugin_opt.sorted_plugin); while (item != NULL) { gfal_plugin_interface* plugin_ifce = (gfal_plugin_interface*)item->data; if (plugin_ifce->copy_enter_hook) { GError* tmp_error = NULL; plugin_ifce->copy_enter_hook(plugin_ifce->plugin_data, context, params, &tmp_error); if (tmp_error) { gfal2_log(G_LOG_LEVEL_MESSAGE, "Copy enter hook failed: %s", tmp_error->message); g_error_free(tmp_error); } } item = g_list_next(item); } return 0; } static int notify_copy_list(gfal2_context_t context, gfalt_params_t params, size_t nbfiles, const char* const * srcs, const char* const * dsts, GError** error) { plugin_trigger_event(params, scope_copy_domain(), GFAL_EVENT_NONE, GFAL_EVENT_LIST_ENTER, NULL); size_t i; for (i = 0; i < nbfiles; ++i) { gchar* src = g_markup_escape_text(srcs[i], -1); gchar* dst = g_markup_escape_text(dsts[i], -1); plugin_trigger_event(params, scope_copy_domain(), GFAL_EVENT_NONE, GFAL_EVENT_LIST_ITEM, "%s => %s", src, dst); g_free(src); g_free(dst); } plugin_trigger_event(params, scope_copy_domain(), GFAL_EVENT_NONE, GFAL_EVENT_LIST_EXIT, NULL); return 0; } static int perform_copy(gfal2_context_t context, gfalt_params_t params, const char* src, const char* dst, GError** error) { gfal2_log(G_LOG_LEVEL_DEBUG, " -> Gfal::Transfer::FileCopy"); GError *tmp_err = NULL; int res = -1; if (trigger_listener_plugins(context, params, &tmp_err) < 0) { gfal2_propagate_prefixed_error(error, tmp_err, __func__); return -1; } if (notify_copy_list(context, params, 1, &src, &dst, &tmp_err)) { gfal2_propagate_prefixed_error(error, tmp_err, __func__); return -1; } void *plugin_data = NULL; gfal_plugin_interface* plugin = find_copy_plugin(context, GFAL_FILE_COPY, src, dst, &plugin_data, &tmp_err); if (tmp_err == NULL) { if (plugin == NULL) { if (gfalt_get_local_transfer_perm(params, NULL)) { res = perform_local_copy(context, params, src, dst, &tmp_err); } else { gfal2_set_error(error, scope_copy_domain(), EPROTONOSUPPORT, __func__, "No plugin supports a transfer from %s to %s, and local streaming is disabled", src, dst); res = -1; } } else { res = plugin->copy_file(plugin_data, context, params, src, dst, &tmp_err); } } gfal2_log(G_LOG_LEVEL_DEBUG, " <- Gfal::Transfer::FileCopy"); if (tmp_err != NULL) gfal2_propagate_prefixed_error(error, tmp_err, __func__); return res; } static int set_checksum(gfalt_params_t params, const char* checksum, GError **err) { gfalt_checksum_mode_t mode = gfalt_get_checksum_mode(params, err); if (*err) { return -1; } if (checksum == NULL) { return gfalt_set_checksum(params, mode, NULL, NULL, err); } else { const char* colon = strchr(checksum, ':'); if (colon == NULL) { return gfalt_set_checksum(params, mode, NULL, checksum, err); } else { char chktype[64]; size_t chktype_len = colon - checksum; g_strlcpy(chktype, checksum, chktype_len < 64 ? chktype_len : 64); return gfalt_set_checksum(params, mode, chktype, colon + 1, err); } } } static int bulk_fallback(gfal2_context_t context, gfalt_params_t params, size_t nbfiles, const char* const * srcs, const char* const * dsts, const char* const * checksums, GError** op_error, GError*** file_errors) { *file_errors = g_new0(GError*, nbfiles); int ret = 0; size_t i; for (i = 0; i < nbfiles; ++i) { int subret = 0; if (checksums) { const char* checksum = checksums[i]; subret = set_checksum(params, checksum, &(*file_errors)[i]); } else { subret = set_checksum(params, NULL, &(*file_errors)[i]); } if (subret < 0) { ret -= 1; continue; } subret = perform_copy(context, params, srcs[i], dsts[i], &(*file_errors)[i]); if (subret < 0) { ret -= 1; } } return ret; } static int perform_bulk_copy(gfal2_context_t context, gfalt_params_t params, size_t nbfiles, const char* const * srcs, const char* const * dsts, const char* const * checksums, GError** op_error, GError*** file_errors) { GError* tmp_err = NULL; int res = -1; gfal2_log(G_LOG_LEVEL_DEBUG, " -> Gfal::Transfer::BulkFileCopy"); if (trigger_listener_plugins(context, params, &tmp_err) < 0) { gfal2_propagate_prefixed_error(op_error, tmp_err, __func__); return -1; } if (notify_copy_list(context, params, nbfiles, srcs, dsts, &tmp_err)) { gfal2_propagate_prefixed_error(op_error, tmp_err, __func__); return -1; } void *plugin_data = NULL; gfal_plugin_interface *plugin = find_copy_plugin(context, GFAL_BULK_COPY, srcs[0], dsts[0], &plugin_data, &tmp_err); if (tmp_err == NULL) { if (plugin == NULL) { res = bulk_fallback(context, params, nbfiles, srcs, dsts, checksums, op_error, file_errors); } else { res = plugin->copy_bulk(plugin_data, context, params, nbfiles, srcs, dsts, checksums, op_error, file_errors); } } gfal2_log(G_LOG_LEVEL_DEBUG, " <- Gfal::Transfer::BulkFileCopy"); if (tmp_err != NULL) gfal2_propagate_prefixed_error(op_error, tmp_err, __func__); return res; } int gfalt_copy_file(gfal2_context_t handle, gfalt_params_t params, const char* src, const char* dst, GError** err) { g_return_val_err_if_fail(handle && src && dst, -1, err, "invalid source or/and destination values"); gfalt_params_t p = NULL; GFAL2_BEGIN_SCOPE_CANCEL(handle, -1, err); int ret = -1; GError* nested_error = NULL; if (params == NULL) { p = gfalt_params_handle_new(NULL); ret = perform_copy(handle, p, src, dst, &nested_error); } else { ret = perform_copy(handle, params, src, dst, &nested_error); } gfalt_params_handle_delete(p, NULL); GFAL2_END_SCOPE_CANCEL(handle); G_RETURN_ERR(ret, nested_error, err); } int gfalt_copy_bulk(gfal2_context_t context, gfalt_params_t params, size_t nbfiles, const char* const * srcs, const char* const * dsts, const char* const * checksums, GError** op_error, GError*** file_errors) { g_return_val_err_if_fail(context && srcs && dsts, -1, op_error, "invalid source or/and destination values"); gfalt_params_t p = NULL; int ret = -1; if (params == NULL) { p = gfalt_params_handle_new(NULL); params = p; } GFAL2_BEGIN_SCOPE_CANCEL(context, -1, op_error); ret = perform_bulk_copy(context, params, nbfiles, srcs, dsts, checksums, op_error, file_errors); gfalt_params_handle_delete(p, NULL); GFAL2_END_SCOPE_CANCEL(context); return ret; } gfal2-v2.23.0/src/core/transfer/gfal_transfer_internal.h000066400000000000000000000051301465240014500232040ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #pragma once #ifndef GFAL_TRANSFER_INTERNAL_H_ #define GFAL_TRANSFER_INTERNAL_H_ #include "gfal_transfer.h" #include "gfal_transfer_plugins.h" #include struct _gfalt_params_t { gboolean lock; // lock enabled after the start of the transfer guint64 timeout; // connection timeout guint64 tcp_buffer_size; gboolean replace_existing; // replace destination or not off_t start_offset; // start offset in case of restart guint nb_data_streams; // nb of parallels streams gboolean strict_mode; // state of the strict copy mode gboolean local_transfers; // local transfer authorized gboolean parent_dir_create; // force the creation of the parent dir gboolean transfer_cleanup; // enable/disable automatic clean-up on failed transfers gboolean proxy_delegation; // use TPC proxy delegation guint scitag; // transfer SciTag flow in the [65-65535] range // disk residency management for tape endpoints gboolean evict; // evict file from disk buffer gchar *stage_request_id; // request id used in the staging operation gchar *transfer_metadata; // transfer metadata sent in the copy request to the destination gchar *archive_metadata; // archive metadata sent in the copy request to the destination // spacetoken management for SRM gchar *src_space_token; gchar *dst_space_token; // checksums gfalt_checksum_mode_t checksum_mode; gchar *checksum_value; gchar *checksum_type; // callback lists GSList *monitor_callbacks; GSList *event_callbacks; }; struct _gfalt_callback_entry { gpointer func, udata; GDestroyNotify udata_free; }; int perform_local_copy(gfal2_context_t context, gfalt_params_t params, const char *src, const char *dst, GError **error); #endif /* GFAL_TRANSFER_INTERNAL_H_ */ gfal2-v2.23.0/src/core/transfer/gfal_transfer_localcopy.c000066400000000000000000000305131465240014500233530ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include "gfal_transfer_plugins.h" #include "gfal_transfer_internal.h" const size_t DEFAULT_BUFFER_SIZE = 4194304; static GQuark local_copy_domain() { return g_quark_from_static_string("GFAL2:CORE:COPY:LOCAL"); } static char* get_parent(const char* url) { char *parent = g_strdup(url); char *slash = strrchr(parent, '/'); if (slash) { *slash = '\0'; } else { g_free(parent); parent = NULL; } return parent; } static int create_parent(gfal2_context_t context, gfalt_params_t params, const char* surl, GError** error) { if (!gfalt_get_create_parent_dir(params, NULL)) return 0; char *parent = get_parent(surl); if (!parent) { gfalt_set_error(error, local_copy_domain(), EINVAL, __func__, GFALT_ERROR_DESTINATION, GFALT_ERROR_PARENT, "Could not get the parent directory of %s", surl); return -1; } GError* nested_error = NULL; struct stat st; if (gfal2_stat(context, parent, &st, &nested_error) < 0) { if (nested_error->code != ENOENT) { gfal2_propagate_prefixed_error(error, nested_error, __func__); return -1; } g_clear_error(&nested_error); } else { return 0; } gfal2_mkdir_rec(context, parent, 0755, &nested_error); if (nested_error != NULL) { gfal2_propagate_prefixed_error(error, nested_error, __func__); return -1; } return 0; } static int unlink_if_exists(gfal2_context_t context, gfalt_params_t params, const char* surl, GError** error) { GError* nested_error = NULL; struct stat st; if (gfal2_stat(context, surl, &st, &nested_error) != 0) { if (nested_error->code == ENOENT) { g_error_free(nested_error); return 0; } gfal2_propagate_prefixed_error(error, nested_error, __func__); return -1; } if (S_ISFIFO(st.st_mode) || S_ISCHR(st.st_mode) || S_ISBLK(st.st_mode) || S_ISSOCK(st.st_mode)) { gfal2_log(G_LOG_LEVEL_MESSAGE, "%s is a special file (%o), so keep going", surl, S_IFMT & st.st_mode); return 0; } if (!gfalt_get_replace_existing_file(params,NULL)) { gfal2_set_error(error, local_copy_domain(), EEXIST, __func__, "The file exists and overwrite is not set"); return -1; } gfal2_unlink(context, surl, &nested_error); if (nested_error != NULL) { if (nested_error->code != ENOENT) { gfal2_propagate_prefixed_error(error, nested_error, __func__); return -1; } g_error_free(nested_error); } else { plugin_trigger_event(params, local_copy_domain(), GFAL_EVENT_DESTINATION, GFAL_EVENT_OVERWRITE_DESTINATION, "Deleted %s", surl); } return 0; } struct perf_data_t { time_t start, last_update, now; size_t done; size_t done_since_last_update; }; static void send_performance_data(gfalt_params_t params, const char* src, const char* dst, const struct perf_data_t* perf) { struct _gfalt_transfer_status status; time_t total_time = perf->now - perf->start; time_t inc_time = perf->now - perf->last_update; status.average_baudrate = (size_t)(perf->done / total_time); status.bytes_transfered = (size_t)(perf->done); status.instant_baudrate = (size_t)(perf->done_since_last_update / inc_time); status.transfer_time = total_time; plugin_trigger_monitor(params, &status, src, dst); } static int streamed_copy(gfal2_context_t context, gfalt_params_t params, const char* src, const char* dst, GError** error) { GError *nested_error = NULL; plugin_trigger_event(params, local_copy_domain(), GFAL_EVENT_NONE, GFAL_EVENT_TRANSFER_ENTER, "%s => %s", src, dst); plugin_trigger_event(params, local_copy_domain(), GFAL_EVENT_NONE, GFAL_EVENT_TRANSFER_TYPE, "%s", GFAL_TRANSFER_TYPE_STREAMED); size_t alignment = gfal2_get_opt_integer_with_default(context, "CORE", "COPY_BUFFER_ALIGNMENT", 512); size_t buffersize = gfal2_get_opt_integer_with_default(context, "CORE", "COPY_BUFFERSIZE", DEFAULT_BUFFER_SIZE); char *buffer; errno = posix_memalign((void**)&buffer, alignment, buffersize); if (errno) { g_set_error(error, local_copy_domain(), errno, "Failed to allocate aligned buffer"); return -1; } int src_open_flags = O_RDONLY; #ifdef O_DIRECT gboolean direct_io = gfal2_get_opt_boolean_with_default(context, "CORE", "COPY_DIRECT_IO", FALSE); if (direct_io) { src_open_flags |= O_DIRECT; gfal2_log(G_LOG_LEVEL_DEBUG, " open src file with direct io : %s ", src); } else { gfal2_log(G_LOG_LEVEL_DEBUG, " open src file : %s ", src); } #endif gfal_file_handle f_src = gfal_plugin_openG(context, src, src_open_flags, 0, &nested_error); if (nested_error) { free(buffer); gfal2_propagate_prefixed_error_extended(error, nested_error, __func__, "Could not open source: "); return -1; } int dst_open_flags = O_WRONLY | O_CREAT; #ifdef O_DIRECT if (direct_io) { dst_open_flags |= O_DIRECT; gfal2_log(G_LOG_LEVEL_DEBUG, " open dst file with direct io : %s ", dst); } else { gfal2_log(G_LOG_LEVEL_DEBUG, " open dst file : %s ", dst); } #endif gfal_file_handle f_dst = gfal_plugin_openG(context, dst, dst_open_flags, 0755, &nested_error); if (nested_error) { free(buffer); gfal_plugin_closeG(context, f_src, NULL); gfal2_propagate_prefixed_error_extended(error, nested_error, __func__, "Could not open destination: "); return -1; } struct perf_data_t perf_data; perf_data.start = perf_data.now = perf_data.last_update = time(NULL); perf_data.done = perf_data.done_since_last_update = 0; const time_t timeout = perf_data.start + gfalt_get_timeout(params, NULL); ssize_t s_file = 1; gfal2_log(G_LOG_LEVEL_DEBUG, " begin local transfer %s -> %s with buffer size %zd", src, dst, buffersize); while (s_file > 0 && !nested_error) { s_file = gfal_plugin_readG(context, f_src, buffer, buffersize, &nested_error); if (s_file > 0) { gfal_plugin_writeG(context, f_dst, buffer, s_file, &nested_error); } perf_data.done += s_file; perf_data.done_since_last_update += s_file; // Make sure we don't have to cancel if (gfal2_is_canceled(context)) { if (nested_error == NULL) g_set_error(&nested_error, local_copy_domain(), ECANCELED, "Transfer canceled"); } // Timed-out? else { perf_data.now = time(NULL); if (perf_data.now >= timeout) { if (nested_error == NULL) g_set_error(&nested_error, local_copy_domain(), ETIMEDOUT, "Transfer canceled because the timeout expired"); } else if (perf_data.now - perf_data.last_update > 5) { send_performance_data(params, src, dst, &perf_data); perf_data.done_since_last_update = 0; perf_data.last_update = perf_data.now; } } } free(buffer); gfal_plugin_closeG(context, f_dst, (nested_error)?NULL:(&nested_error)); gfal_plugin_closeG(context, f_src, (nested_error)?NULL:(&nested_error)); if (nested_error) { gfal2_propagate_prefixed_error(error, nested_error, __func__); return -1; } else { plugin_trigger_event(params, local_copy_domain(), GFAL_EVENT_NONE, GFAL_EVENT_TRANSFER_EXIT, "%s => %s", src, dst); return 0; } } int perform_local_copy(gfal2_context_t context, gfalt_params_t params, const char* src, const char* dst, GError** error) { GError* nested_error = NULL; gfal2_log(G_LOG_LEVEL_DEBUG, " -> Gfal::Transfer::start_local_copy "); char checksum_type[1024] = {0}; char user_checksum[1024] = {0}; char source_checksum[1024] = {0}; gboolean is_strict_mode = gfalt_get_strict_copy_mode(params, NULL); gfalt_checksum_mode_t checksum_mode = GFALT_CHECKSUM_NONE; if (!is_strict_mode) { checksum_mode = gfalt_get_checksum(params, checksum_type, sizeof(checksum_type), user_checksum, sizeof(user_checksum), error); } if (checksum_type[0] == '\0') { g_strlcpy(checksum_type, "ADLER32", sizeof(checksum_type)); } // Source checksum if (checksum_mode & GFALT_CHECKSUM_SOURCE) { plugin_trigger_event(params, local_copy_domain(), GFAL_EVENT_SOURCE, GFAL_EVENT_CHECKSUM_ENTER, ""); gfal2_checksum(context, src, checksum_type, 0, 0, source_checksum, sizeof(source_checksum), &nested_error); if (nested_error != NULL) { gfal2_propagate_prefixed_error_extended(error, nested_error, __func__, "Could not get the source checksum: "); return -1; } plugin_trigger_event(params, local_copy_domain(), GFAL_EVENT_SOURCE, GFAL_EVENT_CHECKSUM_EXIT, ""); } if (user_checksum[0] && source_checksum[0]) { if (gfal_compare_checksums(user_checksum, source_checksum, 1024) != 0) { gfalt_set_error(error, local_copy_domain(), EIO, __func__, GFALT_ERROR_SOURCE, GFALT_ERROR_CHECKSUM_MISMATCH, "Source checksum and user-specified checksum do not match: %s != %s", source_checksum, user_checksum); return -1; } } if (!is_strict_mode) { // Parent directory create_parent(context, params, dst, &nested_error); if (nested_error != NULL) { gfal2_propagate_prefixed_error(error, nested_error, __func__); return -1; } // Remove if exists and overwrite is set if (!is_strict_mode) { unlink_if_exists(context, params, dst, &nested_error); if (nested_error != NULL) { gfal2_propagate_prefixed_error(error, nested_error, __func__); return -1; } } } // Do the transfer streamed_copy(context, params, src, dst, &nested_error); if (nested_error != NULL) { gfal2_propagate_prefixed_error(error, nested_error, __func__); return -1; } // Destination checksum char *compare_against = user_checksum; char *compare_side = "User defined"; if (user_checksum[0] == '\0') { compare_against = source_checksum; compare_side = "Source"; } if (checksum_mode & GFALT_CHECKSUM_TARGET) { char destination_checksum[1024]; plugin_trigger_event(params, local_copy_domain(), GFAL_EVENT_DESTINATION, GFAL_EVENT_CHECKSUM_ENTER, ""); gfal2_checksum(context, dst, checksum_type, 0, 0, destination_checksum, sizeof(destination_checksum), &nested_error); if (nested_error != NULL) { gfal2_propagate_prefixed_error_extended(error, nested_error, __func__, "Could not get the destination checksum: "); return -1; } plugin_trigger_event(params, local_copy_domain(), GFAL_EVENT_DESTINATION, GFAL_EVENT_CHECKSUM_EXIT, ""); if (gfal_compare_checksums(compare_against, destination_checksum, 1204) != 0) { gfalt_set_error(error, local_copy_domain(), EIO, __func__, GFALT_ERROR_DESTINATION, GFALT_ERROR_CHECKSUM_MISMATCH, "%s checksum and destination checksum do not match: %s != %s", compare_side, compare_against, destination_checksum); return -1; } } gfal2_log(G_LOG_LEVEL_DEBUG, " <- Gfal::Transfer::start_local_copy"); return 0; } gfal2-v2.23.0/src/core/transfer/gfal_transfer_params.c000066400000000000000000000422411465240014500226520ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include "gfal_transfer_internal.h" static void gfalt_params_handle_init(gfalt_params_t p, GError ** err) { *p = (struct _gfalt_params_t){ .lock = FALSE, .nb_data_streams = 0, .timeout = 3600, .start_offset = 0, .tcp_buffer_size = 0, .replace_existing = FALSE, .local_transfers = TRUE, .strict_mode = FALSE, .parent_dir_create = FALSE, .transfer_cleanup = TRUE, .proxy_delegation = TRUE, .scitag = 0, .evict = FALSE, .monitor_callbacks = NULL, .event_callbacks = NULL, }; } static GSList* gfalt_params_copy_callbacks(const GSList* original) { GSList* copy = NULL; const GSList* p = original; while (p) { struct _gfalt_callback_entry* original_entry = (struct _gfalt_callback_entry*)p->data; struct _gfalt_callback_entry* new_entry = g_new0(struct _gfalt_callback_entry, 1); new_entry->func = original_entry->func; new_entry->udata = original_entry->udata; copy = g_slist_append(copy, new_entry); p = g_slist_next(p); } return copy; } gfalt_params_t gfalt_params_handle_copy(gfalt_params_t params, GError ** err) { gfalt_params_t p = g_new0(struct _gfalt_params_t, 1); memcpy(p, params, sizeof(struct _gfalt_params_t)); p->stage_request_id = g_strdup(params->stage_request_id); p->transfer_metadata = g_strdup(params->transfer_metadata); p->archive_metadata = g_strdup(params->archive_metadata); p->src_space_token = g_strdup(params->src_space_token); p->dst_space_token = g_strdup(params->dst_space_token); p->checksum_type = g_strdup(params->checksum_type); p->checksum_value = g_strdup(params->checksum_value); p->monitor_callbacks = gfalt_params_copy_callbacks(params->monitor_callbacks); p->event_callbacks = gfalt_params_copy_callbacks(params->event_callbacks); return p; } gfalt_params_t gfalt_params_handle_new(GError ** err) { gfalt_params_t p = g_new0(struct _gfalt_params_t, 1); gfalt_params_handle_init(p, err); return p; } static void gfalt_params_free_callback(gpointer data, gpointer user_data) { struct _gfalt_callback_entry* entry = data; if (entry->udata_free) entry->udata_free(entry->udata); g_free(entry); } void gfalt_params_handle_delete(gfalt_params_t params, GError ** err) { if (params) { params->lock = FALSE; g_free(params->stage_request_id); g_free(params->transfer_metadata); g_free(params->archive_metadata); g_free(params->src_space_token); g_free(params->dst_space_token); g_free(params->checksum_type); g_free(params->checksum_value); g_slist_foreach(params->monitor_callbacks, gfalt_params_free_callback , NULL); g_slist_free(params->monitor_callbacks); g_slist_foreach(params->event_callbacks, gfalt_params_free_callback , NULL); g_slist_free(params->event_callbacks); g_free(params); } } gint gfalt_set_timeout(gfalt_params_t params, guint64 timeout, GError** err) { g_return_val_err_if_fail(params != NULL, -1, err, "[BUG] invalid parameter handle value"); params->timeout = timeout; return 0; } guint64 gfalt_get_timeout(gfalt_params_t params, GError** err) { g_return_val_err_if_fail(params != NULL, -1, err, "[BUG] invalid params handle"); return params->timeout; } gint gfalt_set_nbstreams(gfalt_params_t params, guint nbstreams, GError** err) { g_return_val_err_if_fail(params != NULL, -1, err, "[BUG] invalid parameter handle"); params->nb_data_streams = nbstreams; return 0; } gint gfalt_set_tcp_buffer_size(gfalt_params_t params, guint64 tcp_buffer_size, GError** err) { g_return_val_err_if_fail(params != NULL, -1, err, "[BUG] invalid params handle"); params->tcp_buffer_size = tcp_buffer_size; return 0; } guint64 gfalt_get_tcp_buffer_size(gfalt_params_t params, GError** err) { return params->tcp_buffer_size; } gint gfalt_set_local_transfer_perm(gfalt_params_t params, gboolean local_transfer_status, GError ** err) { g_return_val_err_if_fail(params != NULL, -1, err, "[BUG] invalid params handle"); params->local_transfers = local_transfer_status; return 0; } gboolean gfalt_get_local_transfer_perm(gfalt_params_t params, GError ** err) { g_return_val_err_if_fail(params != NULL, -1, err, "[BUG] invalid params handle"); return params->local_transfers; } gint gfalt_set_replace_existing_file(gfalt_params_t params, gboolean replace, GError** err) { g_return_val_err_if_fail(params != NULL, -1, err, "[BUG] invalid params handle"); params->replace_existing = replace; return 0; } gboolean gfalt_get_replace_existing_file(gfalt_params_t params, GError** err) { g_return_val_err_if_fail(params != NULL, -1, err, "[BUG] invalid params handle"); return params->replace_existing; } gint gfalt_set_offset_from_source(gfalt_params_t params, off_t offset, GError** err) { g_return_val_err_if_fail(params != NULL, -1, err, "[BUG] invalid params handle"); params->start_offset = offset; return 0; } static GSList* gfalt_search_callback(GSList* list, gpointer callback) { struct _gfalt_callback_entry* entry; GSList* p = list; while (p) { entry = (struct _gfalt_callback_entry*)p->data; if (entry->func == callback) return p; p = g_slist_next(p); } return NULL; } gint gfalt_add_monitor_callback(gfalt_params_t params, gfalt_monitor_func callback, gpointer udata, GDestroyNotify udata_free, GError** err) { g_return_val_err_if_fail(params != NULL, -1, err, "[BUG] invalid params handle"); struct _gfalt_callback_entry* entry; GSList* i = gfalt_search_callback(params->monitor_callbacks, callback); if (i) { entry = (struct _gfalt_callback_entry*)i->data; if (entry->udata_free) entry->udata_free(entry->udata); entry->udata = udata; entry->udata_free = udata_free; } else { entry = g_new0(struct _gfalt_callback_entry, 1); entry->func = callback; entry->udata = udata; entry->udata_free = udata_free; params->monitor_callbacks = g_slist_append(params->monitor_callbacks, entry); } return 0; } gint gfalt_remove_monitor_callback(gfalt_params_t params, gfalt_monitor_func callback, GError** err) { g_return_val_err_if_fail(params != NULL, -1, err, "[BUG] invalid params handle"); GSList* i = gfalt_search_callback(params->monitor_callbacks, callback); if (i) { struct _gfalt_callback_entry* entry = i->data; if (entry->udata_free) entry->udata_free(entry->udata); g_free(i->data); params->monitor_callbacks = g_slist_delete_link(params->monitor_callbacks, i); return 0; } gfal2_set_error(err, gfal2_get_core_quark(), ENOENT, __func__, "Could not find the callback"); return -1; } gint gfalt_add_event_callback(gfalt_params_t params, gfalt_event_func callback, gpointer udata, GDestroyNotify udata_free, GError** err) { g_return_val_err_if_fail(params != NULL, -1, err, "[BUG] invalid params handle"); struct _gfalt_callback_entry* entry; GSList* i = gfalt_search_callback(params->event_callbacks, callback); if (i) { entry = (struct _gfalt_callback_entry*)i->data; if (entry->udata_free) entry->udata_free(entry->udata); entry->udata = udata; entry->udata_free = udata_free; } else { entry = g_new0(struct _gfalt_callback_entry, 1); entry->func = callback; entry->udata = udata; entry->udata_free = udata_free; params->event_callbacks = g_slist_append(params->event_callbacks, entry); } return 0; } gint gfalt_remove_event_callback(gfalt_params_t params, gfalt_event_func callback, GError** err) { g_return_val_err_if_fail(params != NULL, -1, err, "[BUG] invalid params handle"); GSList* i = gfalt_search_callback(params->event_callbacks, callback); if (i) { struct _gfalt_callback_entry* entry = i->data; if (entry->udata_free) entry->udata_free(entry->udata); g_free(i->data); params->event_callbacks = g_slist_delete_link(params->event_callbacks, i); return 0; } gfal2_set_error(err, gfal2_get_core_quark(), ENOENT, __func__, "Could not find the callback"); return -1; } guint gfalt_get_nbstreams(gfalt_params_t params, GError** err) { g_return_val_err_if_fail(params != NULL, -1, err, "[BUG] invalid parameter handle"); return params->nb_data_streams; } gint gfalt_set_strict_copy_mode(gfalt_params_t params, gboolean strict_mode, GError** err) { g_return_val_err_if_fail(params != NULL, -1, err, "[BUG] invalid parameter handle"); params->strict_mode = strict_mode; return 0; } gboolean gfalt_get_strict_copy_mode(gfalt_params_t params, GError** err) { g_return_val_err_if_fail(params != NULL, -1, err, "[BUG] invalid parameter handle"); return params->strict_mode; } gint gfalt_set_src_spacetoken(gfalt_params_t params, const char* srm_spacetoken, GError** err) { if (params->src_space_token) g_free(params->src_space_token); params->src_space_token = g_strdup(srm_spacetoken); return 0; } const gchar* gfalt_get_src_spacetoken(gfalt_params_t params, GError** err) { return params->src_space_token; } gint gfalt_set_dst_spacetoken(gfalt_params_t params, const char* srm_spacetoken, GError** err) { if (params->dst_space_token) g_free(params->dst_space_token); params->dst_space_token = g_strdup(srm_spacetoken); return 0; } const gchar* gfalt_get_dst_spacetoken(gfalt_params_t params, GError** err) { return params->dst_space_token; } gint gfalt_set_create_parent_dir(gfalt_params_t params, gboolean create_parent, GError** err) { params->parent_dir_create = create_parent; return 0; } gboolean gfalt_get_create_parent_dir(gfalt_params_t params, GError** err) { return params->parent_dir_create; } gint gfalt_set_transfer_cleanup(gfalt_params_t params, gboolean transfer_cleanup, GError** err) { params->transfer_cleanup = transfer_cleanup; return 0; } gboolean gfalt_get_transfer_cleanup(gfalt_params_t params, GError** err) { return params->transfer_cleanup; } gint gfalt_set_use_proxy_delegation(gfalt_params_t params, gboolean proxy_delegation, GError** err) { g_return_val_err_if_fail(params != NULL, -1, err, "[BUG] invalid params handle"); params->proxy_delegation = proxy_delegation; return 0; } gboolean gfalt_get_use_proxy_delegation(gfalt_params_t params, GError** err) { g_return_val_err_if_fail(params != NULL, -1, err, "[BUG] invalid params handle"); return params->proxy_delegation; } gint gfalt_set_scitag(gfalt_params_t params, guint scitag, GError** err) { g_return_val_err_if_fail(params != NULL, -1, err, "[BUG] invalid params handle"); g_return_val_err_if_fail((scitag >= GFAL_SCITAG_MIN_VALUE && scitag <= GFAL_SCITAG_MAX_VALUE), -1, err, "Invalid SciTag value (must be in the [65, 65535] range)"); params->scitag = scitag; return 0; } guint gfalt_get_scitag(gfalt_params_t params, GError** err) { g_return_val_err_if_fail(params != NULL, -1, err, "[BUG] invalid params handle"); return params->scitag; } gint gfalt_set_use_evict(gfalt_params_t params, gboolean evict, GError** err) { g_return_val_err_if_fail(params != NULL, -1, err, "[BUG] invalid params handle"); params->evict = evict; return 0; } gboolean gfalt_get_use_evict(gfalt_params_t params , GError** err) { g_return_val_err_if_fail(params != NULL, -1, err, "[BUG] invalid params handle"); return params->evict; } gint gfalt_set_stage_request_id(gfalt_params_t params, const char* request_id, GError** err) { if (params->stage_request_id) g_free(params->stage_request_id); params->stage_request_id = g_strdup(request_id); return 0; } const gchar* gfalt_get_stage_request_id(gfalt_params_t params, GError** err) { return params->stage_request_id; } gint gfalt_set_transfer_metadata(gfalt_params_t params, const char* metadata, GError** err) { g_return_val_err_if_fail(params != NULL, -1, err, "[BUG] invalid params handle"); if (params->transfer_metadata) g_free(params->transfer_metadata); params->transfer_metadata = g_strdup(metadata); return 0; } const gchar* gfalt_get_transfer_metadata(gfalt_params_t params, GError** err) { return params->transfer_metadata; } gint gfalt_set_archive_metadata(gfalt_params_t params, const char* metadata, GError** err) { g_return_val_err_if_fail(params != NULL, -1, err, "[BUG] invalid params handle"); if (params->archive_metadata) g_free(params->archive_metadata); params->archive_metadata = g_strdup(metadata); return 0; } const gchar* gfalt_get_archive_metadata(gfalt_params_t params, GError** err) { return params->archive_metadata; } gint gfalt_set_checksum_check(gfalt_params_t params, gboolean value, GError** err) { if (value) { return gfalt_set_checksum(params, GFALT_CHECKSUM_BOTH, params->checksum_type, params->checksum_value, err); } return gfalt_set_checksum(params, GFALT_CHECKSUM_NONE, NULL, NULL, err); } gboolean gfalt_get_checksum_check(gfalt_params_t params, GError** err) { return params->checksum_mode != GFALT_CHECKSUM_NONE; } gint gfalt_set_user_defined_checksum(gfalt_params_t params, const gchar* chktype, const gchar* checksum, GError** err) { return gfalt_set_checksum(params, GFALT_CHECKSUM_BOTH, chktype, checksum, err); } gint gfalt_get_user_defined_checksum(gfalt_params_t params, gchar* chktype_buff, size_t chk_type_len, gchar* checksum_buff, size_t checksum_len, GError** err) { g_assert(chktype_buff && checksum_buff); gfalt_get_checksum(params, chktype_buff, chk_type_len, checksum_buff, checksum_len, err); return 0; } gint gfalt_set_checksum(gfalt_params_t params, gfalt_checksum_mode_t mode, const gchar* type, const gchar *checksum, GError **err) { if ((mode == GFALT_CHECKSUM_SOURCE || mode == GFALT_CHECKSUM_TARGET) && (checksum == NULL || checksum[0] == '\0')) { gfal2_set_error(err, gfal2_get_core_quark(), EINVAL, __func__, "Checksum value required if mode is not end to end"); return -1; } params->checksum_mode = mode; if (type) { g_free(params->checksum_type); params->checksum_type = g_strdup(type); } g_free(params->checksum_value); params->checksum_value = g_strdup(checksum); return 0; } gfalt_checksum_mode_t gfalt_get_checksum(gfalt_params_t params, gchar* type_buff, size_t type_buff_len, gchar* checksum_buff, size_t checksum_buff_len, GError **err) { if (params->checksum_type) { g_strlcpy(type_buff, params->checksum_type, type_buff_len); } else { g_strlcpy(type_buff, "", type_buff_len); } if (params->checksum_value) { g_strlcpy(checksum_buff, params->checksum_value, checksum_buff_len); } else { g_strlcpy(checksum_buff, "", checksum_buff_len); } return params->checksum_mode; } gfalt_checksum_mode_t gfalt_get_checksum_mode(gfalt_params_t params, GError **err) { return params->checksum_mode; } gint gfalt_copy_get_status(gfalt_transfer_status_t s, GError ** err) { g_return_val_err_if_fail(s != NULL, -1, err, "[BUG] invalid transfer status handle"); return s->status; } size_t gfalt_copy_get_average_baudrate(gfalt_transfer_status_t s, GError ** err) { g_return_val_err_if_fail(s != NULL, -1, err, "[BUG] invalid transfer status handle"); return s->average_baudrate; } size_t gfalt_copy_get_instant_baudrate(gfalt_transfer_status_t s, GError ** err) { g_return_val_err_if_fail(s != NULL, -1, err, "[BUG] invalid transfer status handle"); return s->instant_baudrate; } size_t gfalt_copy_get_bytes_transfered(gfalt_transfer_status_t s, GError ** err) { return gfalt_copy_get_bytes_transferred(s, err); } size_t gfalt_copy_get_bytes_transferred(gfalt_transfer_status_t s, GError ** err) { g_return_val_err_if_fail(s != NULL, -1, err, "[BUG] invalid transfer status handle"); return s->bytes_transfered; } time_t gfalt_copy_get_elapsed_time(gfalt_transfer_status_t s, GError ** err) { g_return_val_err_if_fail(s != NULL, -1, err, "[BUG] invalid transfer status handle"); return s->transfer_time; } gfal2-v2.23.0/src/core/transfer/gfal_transfer_plugins.h000066400000000000000000000061261465240014500230570ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #pragma once #ifndef GFAL_TRANSFER_PLUGINS_H_ #define GFAL_TRANSFER_PLUGINS_H_ #if !defined(__GFAL2_H_INSIDE__) && !defined(__GFAL2_BUILD__) # warning "Direct inclusion of gfal2 headers is deprecated. Please, include only gfal_api.h or gfal_plugins_api.h" #endif #include #ifdef __cplusplus extern "C" { #endif // __cplusplus struct _gfalt_transfer_status { gpointer plugin_transfer_data; int status; size_t average_baudrate; size_t instant_baudrate; time_t transfer_time; size_t bytes_transfered; }; /** * Convenience method for event callback * @param params The transfer parameters. * @param domain The plugin domain. * @param side The side that triggered the change, if any. * @param stage The new stage. * @param fmt A format string for a message */ int plugin_trigger_event(gfalt_params_t params, GQuark domain, gfal_event_side_t side, GQuark stage, const char* fmt, ...); /** * Convenience method for monitoring callbacks * @param params The transfer parameters. * @param status The transfer status. * @param src Source surl. * @param dst Destination surl. */ int plugin_trigger_monitor(gfalt_params_t params, gfalt_transfer_status_t status, const char* src, const char* dst); /** * Convenience error methods for copy implementations */ void gfalt_propagate_prefixed_error(GError **dest, GError *src, const gchar *function, const gchar *side, const gchar *note); void gfalt_set_error(GError **err, GQuark domain, gint code, const gchar *function, const char *side, const gchar *note, const gchar *format, ...) G_GNUC_PRINTF (7, 8); #define GFALT_ERROR_SOURCE "SOURCE" #define GFALT_ERROR_DESTINATION "DESTINATION" #define GFALT_ERROR_TRANSFER "TRANSFER" #define GFALT_ERROR_CHECKSUM "CHECKSUM" #define GFALT_ERROR_EXISTS "EXISTS" #define GFALT_ERROR_OVERWRITE "OVERWRITE" #define GFALT_ERROR_PARENT "MAKE_PARENT" /** * GFALT_ERROR_CHECKSUM occurs during the retrieval of the checksum * GFALT_ERROR_CHECKSUM_MISMATCH means the checksum was successfully retrieved, * but the comparison failed */ #define GFALT_ERROR_CHECKSUM_MISMATCH "CHECKSUM MISMATCH" /** * Size verification failed */ #define GFALT_ERROR_SIZE_MISMATCH "SIZE MISMATCH" #ifdef __cplusplus } #endif // __cplusplus #endif /* GFAL_TRANSFER_PLUGINS_H_ */ gfal2-v2.23.0/src/core/transfer/gfal_transfer_util.c000066400000000000000000000135461465240014500223520ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include GQuark GFAL_EVENT_PREPARE_ENTER; GQuark GFAL_EVENT_PREPARE_EXIT; GQuark GFAL_EVENT_TRANSFER_ENTER; GQuark GFAL_EVENT_TRANSFER_EXIT; GQuark GFAL_EVENT_CLOSE_ENTER; GQuark GFAL_EVENT_CLOSE_EXIT; GQuark GFAL_EVENT_CHECKSUM_ENTER; GQuark GFAL_EVENT_CHECKSUM_EXIT; GQuark GFAL_EVENT_CANCEL_ENTER; GQuark GFAL_EVENT_CANCEL_EXIT; GQuark GFAL_EVENT_OVERWRITE_DESTINATION; GQuark GFAL_EVENT_LIST_ENTER; GQuark GFAL_EVENT_LIST_ITEM; GQuark GFAL_EVENT_LIST_EXIT; GQuark GFAL_EVENT_TRANSFER_TYPE; GQuark GFAL_EVENT_IPV4; GQuark GFAL_EVENT_IPV6; GQuark GFAL_EVENT_EVICT; GQuark GFAL_EVENT_CLEANUP; __attribute__((constructor)) static void init_event_quarks() { GFAL_EVENT_PREPARE_ENTER = g_quark_from_static_string("PREPARE:ENTER"); GFAL_EVENT_PREPARE_EXIT = g_quark_from_static_string("PREPARE:EXIT"); GFAL_EVENT_TRANSFER_ENTER = g_quark_from_static_string("TRANSFER:ENTER"); GFAL_EVENT_TRANSFER_EXIT = g_quark_from_static_string("TRANSFER:EXIT"); GFAL_EVENT_CLOSE_ENTER = g_quark_from_static_string("CLOSE:ENTER"); GFAL_EVENT_CLOSE_EXIT = g_quark_from_static_string("CLOSE:EXIT"); GFAL_EVENT_CHECKSUM_ENTER = g_quark_from_static_string("CHECKSUM:ENTER"); GFAL_EVENT_CHECKSUM_EXIT = g_quark_from_static_string("CHECKSUM:EXIT"); GFAL_EVENT_CANCEL_ENTER = g_quark_from_static_string("CANCEL:ENTER"); GFAL_EVENT_CANCEL_EXIT = g_quark_from_static_string("CANCEL:EXIT"); GFAL_EVENT_OVERWRITE_DESTINATION = g_quark_from_static_string("OVERWRITE"); GFAL_EVENT_LIST_ENTER = g_quark_from_static_string("LIST:ENTER"); GFAL_EVENT_LIST_ITEM = g_quark_from_static_string("LIST:ITEM"); GFAL_EVENT_LIST_EXIT = g_quark_from_static_string("LIST:EXIT"); GFAL_EVENT_TRANSFER_TYPE = g_quark_from_static_string("TRANSFER:TYPE"); GFAL_EVENT_IPV4 = g_quark_from_static_string("IPV4"); GFAL_EVENT_IPV6 = g_quark_from_static_string("IPV6"); GFAL_EVENT_EVICT = g_quark_from_static_string("EVICT"); GFAL_EVENT_CLEANUP = g_quark_from_static_string("CLEANUP"); } static void plugin_trigger_event_callback(gpointer data, gpointer user_data) { gfalt_event_t event = (gfalt_event_t)user_data; struct _gfalt_callback_entry* entry = (struct _gfalt_callback_entry*)data; gfalt_event_func callback = (gfalt_event_func)entry->func; callback(event, entry->udata); } int plugin_trigger_event(gfalt_params_t params, GQuark domain, gfal_event_side_t side, GQuark stage, const char* fmt, ...) { char buffer[512] = { 0 }; va_list msg_args; va_start(msg_args, fmt); if (fmt) { vsnprintf(buffer, sizeof(buffer), fmt, msg_args); } va_end(msg_args); struct _gfalt_event event; GTimeVal tmst; g_get_current_time(&tmst); event.domain = domain; event.side = side; event.stage = stage; event.timestamp = tmst.tv_sec * 1000 + tmst.tv_usec / 1000; event.description = buffer; g_slist_foreach(params->event_callbacks, plugin_trigger_event_callback, &event); const char* side_str; switch (side) { case GFAL_EVENT_SOURCE: side_str = "SOURCE"; break; case GFAL_EVENT_DESTINATION: side_str = "DESTINATION"; break; default: side_str = "BOTH"; } gfal2_log(G_LOG_LEVEL_MESSAGE, "Event triggered: %s %s %s %s", side_str, g_quark_to_string(domain), g_quark_to_string(stage), buffer); return 0; } struct _gfalt_monitor_data { gfalt_transfer_status_t* status; const char* src, *dst; }; static void plugin_trigger_monitor_callback(gpointer data, gpointer user_data) { struct _gfalt_monitor_data* monitor = (struct _gfalt_monitor_data*)user_data; struct _gfalt_callback_entry* entry = (struct _gfalt_callback_entry*)data; gfalt_monitor_func callback = (gfalt_monitor_func)entry->func; callback(*monitor->status, monitor->src, monitor->dst, entry->udata); } int plugin_trigger_monitor(gfalt_params_t params, gfalt_transfer_status_t status, const char* src, const char* dst) { struct _gfalt_monitor_data monitor; monitor.status = &status; monitor.src = src; monitor.dst = dst; g_slist_foreach(params->monitor_callbacks, plugin_trigger_monitor_callback, &monitor); return 0; } void gfalt_propagate_prefixed_error(GError **dest, GError *src, const gchar *function, const gchar *side, const gchar *note) { if (note) { gfal2_propagate_prefixed_error_extended(dest, src, function, "%s %s ", side, note); } else { gfal2_propagate_prefixed_error_extended(dest, src, function, "%s ", side); } } void gfalt_set_error(GError **err, GQuark domain, gint code, const gchar *function, const char *side, const gchar *note, const gchar *format, ...) { char buffer[1024]; va_list args; va_start(args, format); vsnprintf(buffer, sizeof(buffer), format, args); va_end(args); if (note) gfal2_set_error(err, domain, code, function, "%s %s %s", side, note, buffer); else gfal2_set_error(err, domain, code, function, "%s %s", side, buffer); } gfal2-v2.23.0/src/plugins/000077500000000000000000000000001465240014500152305ustar00rootroot00000000000000gfal2-v2.23.0/src/plugins/CMakeLists.txt000066400000000000000000000005201465240014500177650ustar00rootroot00000000000000cmake_minimum_required (VERSION 2.6) link_directories(${CMAKE_CURRENT_BINARY_DIR}/../core) add_subdirectory (dcap) add_subdirectory (file) add_subdirectory (gridftp) add_subdirectory (http) add_subdirectory (lfc) add_subdirectory (rfio) add_subdirectory (sftp) add_subdirectory (srm) add_subdirectory (xrootd) add_subdirectory (mock) gfal2-v2.23.0/src/plugins/dcap/000077500000000000000000000000001465240014500161375ustar00rootroot00000000000000gfal2-v2.23.0/src/plugins/dcap/CMakeLists.txt000066400000000000000000000017731465240014500207070ustar00rootroot00000000000000cmake_minimum_required (VERSION 2.6) if (PLUGIN_DCAP) file (GLOB src_dcap "*.c*") find_package(DCAP REQUIRED) include_directories(${DCAP_INCLUDE_DIR}) add_definitions(-D_REENTRANT) add_definitions(${DCAP_CFLAGS}) add_library(plugin_dcap MODULE ${src_dcap}) target_link_libraries(plugin_dcap gfal2 ${DCAP_LIBRARIES}) set_target_properties(plugin_dcap PROPERTIES CLEAN_DIRECT_OUTPUT 1 OUTPUT_NAME "gfal_plugin_dcap" LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/plugins) install(TARGETS plugin_dcap LIBRARY DESTINATION ${PLUGIN_INSTALL_DIR}) install(FILES "README_PLUGIN_DCAP" DESTINATION ${DOC_INSTALL_DIR}) # install dcap configuration files LIST(APPEND dcap_conf_file "${CMAKE_SOURCE_DIR}/dist/etc/gfal2.d/dcap_plugin.conf") install(FILES ${dcap_conf_file} DESTINATION ${SYSCONF_INSTALL_DIR}/gfal2.d/) endif (PLUGIN_DCAP) gfal2-v2.23.0/src/plugins/dcap/README_PLUGIN_DCAP000066400000000000000000000000771465240014500206700ustar00rootroot00000000000000 gfal2 lfc plugin : - features : * data access functions gfal2-v2.23.0/src/plugins/dcap/gfal_dcap_plugin_bindings.c000066400000000000000000000202261465240014500234400ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include #include #include #include #include #include #include "gfal_dcap_plugin_layer.h" #include "gfal_dcap_plugin_bindings.h" #include "gfal_dcap_plugin_main.h" // errno conversion for dcap non-complete error messages static int dcap_errno_conversion(const char * msg, int old_err) { int ret; ret = old_err; switch (old_err) { case 0: ret = EIO; break; case EIO: if (strstr(msg, "o such")) ret = ENOENT; break; case EACCES: if (strstr(msg, "ectory not empty")) ret = ENOTEMPTY; break; default: ret = old_err; } return ret; } static void dcap_report_error(gfal_plugin_dcap_handle h, const char * func_name, GError** err) { char buff_error[2048]; const int status = *(h->ops->geterror()); g_strlcpy(buff_error, h->ops->strerror(status), 2048); // errno conversion errno = dcap_errno_conversion(buff_error, errno); gfal2_set_error(err, gfal2_get_plugin_dcap_quark(), errno, func_name, "Error reported by the external library dcap : %s, number : %d", buff_error, status); } gfal_file_handle gfal_dcap_openG(plugin_handle handle, const char* path, int flag, mode_t mode, GError** err) { gfal_plugin_dcap_handle h = (gfal_plugin_dcap_handle) handle; gfal_file_handle ret = NULL; const int internal_flag = flag & (~0100000); // filter non supported flags int fd = h->ops->open(path, internal_flag, mode); if (fd == -1) dcap_report_error(h, __func__, err); else ret = gfal_file_handle_new(gfal_dcap_getName(), GINT_TO_POINTER(fd)); return ret; } /* * map to the libdcap read call * */ ssize_t gfal_dcap_readG(plugin_handle handle, gfal_file_handle fd, void* buff, size_t s_buff, GError** err) { gfal_plugin_dcap_handle h = (gfal_plugin_dcap_handle) handle; ssize_t ret = h->ops->read(GPOINTER_TO_INT(gfal_file_handle_get_fdesc(fd)), buff, s_buff); if (ret < 0) dcap_report_error(h, __func__, err); else errno = 0; return ret; } /* * map to the libdcap pread call */ ssize_t gfal_dcap_preadG(plugin_handle handle, gfal_file_handle fd, void* buff, size_t s_buff, off_t offset, GError** err) { gfal_plugin_dcap_handle h = (gfal_plugin_dcap_handle) handle; ssize_t ret = h->ops->pread(GPOINTER_TO_INT(gfal_file_handle_get_fdesc(fd)), buff, s_buff, offset); if (ret < 0) dcap_report_error(h, __func__, err); else errno = 0; return ret; } /* * map to the libdcap pwrite call */ ssize_t gfal_dcap_pwriteG(plugin_handle handle, gfal_file_handle fd, const void* buff, size_t s_buff, off_t offset, GError** err) { gfal_plugin_dcap_handle h = (gfal_plugin_dcap_handle) handle; ssize_t ret = h->ops->pwrite(GPOINTER_TO_INT(gfal_file_handle_get_fdesc(fd)), buff, s_buff, offset); if (ret < 0) dcap_report_error(h, __func__, err); else errno = 0; return ret; } off_t gfal_dcap_lseekG(plugin_handle handle, gfal_file_handle fd, off_t offset, int whence, GError** err) { gfal_plugin_dcap_handle h = (gfal_plugin_dcap_handle) handle; off_t ret = h->ops->lseek(GPOINTER_TO_INT(gfal_file_handle_get_fdesc(fd)), offset, (int) whence); if (ret == ((off_t) 0) - 1) dcap_report_error(h, __func__, err); else errno = 0; return ret; } ssize_t gfal_dcap_writeG(plugin_handle handle, gfal_file_handle fd, const void* buff, size_t s_buff, GError** err) { gfal_plugin_dcap_handle h = (gfal_plugin_dcap_handle) handle; ssize_t ret = h->ops->write(GPOINTER_TO_INT(gfal_file_handle_get_fdesc(fd)), buff, s_buff); if (ret < 0) dcap_report_error(h, __func__, err); else errno = 0; return ret; } int gfal_dcap_closeG(plugin_handle handle, gfal_file_handle fd, GError ** err) { gfal_plugin_dcap_handle h = (gfal_plugin_dcap_handle) handle; int ret = h->ops->close(GPOINTER_TO_INT(gfal_file_handle_get_fdesc(fd))); if (ret != 0) { dcap_report_error(h, __func__, err); } gfal_file_handle_delete(fd); return ret; } int gfal_dcap_statG(plugin_handle handle, const char* name, struct stat* buff, GError ** err) { gfal_plugin_dcap_handle h = (gfal_plugin_dcap_handle) handle; int ret = h->ops->stat(name, buff); if (ret != 0) { dcap_report_error(h, __func__, err); } return ret; } int gfal_dcap_lstatG(plugin_handle handle, const char* name, struct stat* buff, GError ** err) { gfal_plugin_dcap_handle h = (gfal_plugin_dcap_handle) handle; int ret = h->ops->lstat(name, buff); if (ret != 0) { dcap_report_error(h, __func__, err); } return ret; } int gfal_dcap_accessG(plugin_handle plugin_data, const char* url, int mode, GError** err) { gfal_plugin_dcap_handle h = (gfal_plugin_dcap_handle) plugin_data; ssize_t ret = h->ops->access(url, mode); if (ret < 0) dcap_report_error(h, __func__, err); else errno = 0; return ret; } int gfal_dcap_mkdirG(plugin_handle handle, const char* name, mode_t mode, gboolean pflag, GError** err) { gfal_plugin_dcap_handle h = (gfal_plugin_dcap_handle) handle; int ret = h->ops->mkdir(name, mode); if (ret != 0) { dcap_report_error(h, __func__, err); } return ret; } int gfal_dcap_chmodG(plugin_handle handle, const char* name, mode_t mode, GError** err) { gfal_plugin_dcap_handle h = (gfal_plugin_dcap_handle) handle; int ret = h->ops->chmod(name, mode); if (ret != 0) { dcap_report_error(h, __func__, err); } return ret; } int gfal_dcap_rmdirG(plugin_handle handle, const char* name, GError** err) { gfal_plugin_dcap_handle h = (gfal_plugin_dcap_handle) handle; int ret = h->ops->rmdir(name); if (ret != 0) { dcap_report_error(h, __func__, err); } return ret; } gfal_file_handle gfal_dcap_opendirG(plugin_handle handle, const char* path, GError ** err) { gfal_plugin_dcap_handle h = (gfal_plugin_dcap_handle) handle; DIR * d = h->ops->opendir(path); gfal_file_handle ret = NULL; if (d == NULL) { dcap_report_error(h, __func__, err); } else { ret = gfal_file_handle_new2(gfal_dcap_getName(), (gpointer) d, NULL, path); } return ret; } int gfal_dcap_closedirG(plugin_handle handle, gfal_file_handle fh, GError** err) { gfal_plugin_dcap_handle h = (gfal_plugin_dcap_handle) handle; int ret = h->ops->closedir(gfal_file_handle_get_fdesc(fh)); if (ret != 0) { dcap_report_error(h, __func__, err); } else { gfal_file_handle_delete(fh); } return ret; } struct dirent* gfal_dcap_readdirG(plugin_handle handle, gfal_file_handle fh, GError** err) { gfal_plugin_dcap_handle h = (gfal_plugin_dcap_handle) handle; struct dirent* ret = h->ops->readdir(gfal_file_handle_get_fdesc(fh)); if (ret == NULL && *(h->ops->geterror()) != 0) { dcap_report_error(h, __func__, err); } return ret; } int gfal_dcap_unlinkG(plugin_handle handle, const char* url, GError** err) { gfal_plugin_dcap_handle h = (gfal_plugin_dcap_handle) handle; int ret = h->ops->unlink(url); if (ret != 0) { dcap_report_error(h, __func__, err); } return ret; } gfal2-v2.23.0/src/plugins/dcap/gfal_dcap_plugin_bindings.h000066400000000000000000000050001465240014500234360ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include gfal_file_handle gfal_dcap_openG(plugin_handle handle , const char* path, int flag, mode_t mode, GError** err); ssize_t gfal_dcap_readG(plugin_handle handle , gfal_file_handle fd, void* buff, size_t s_buff, GError** err); ssize_t gfal_dcap_preadG(plugin_handle handle , gfal_file_handle fd, void* buff, size_t s_buff, off_t offset, GError** err); ssize_t gfal_dcap_pwriteG(plugin_handle handle , gfal_file_handle fd, const void* buff, size_t s_buff, off_t offset, GError** err); off_t gfal_dcap_lseekG(plugin_handle handle , gfal_file_handle fd, off_t offset, int whence, GError** err); ssize_t gfal_dcap_writeG(plugin_handle handle , gfal_file_handle fd, const void* buff, size_t s_buff, GError** err); int gfal_dcap_accessG(plugin_handle plugin_data, const char* url, int mode, GError** err); int gfal_dcap_closeG(plugin_handle handle, gfal_file_handle fd, GError ** err); int gfal_dcap_statG(plugin_handle handle, const char* name, struct stat* buff, GError ** err); int gfal_dcap_lstatG(plugin_handle handle, const char* name, struct stat* buff, GError ** err); int gfal_dcap_mkdirG(plugin_handle handle, const char* name, mode_t mode, gboolean pflag, GError** err); int gfal_dcap_chmodG(plugin_handle handle, const char* name, mode_t mode, GError** err); int gfal_dcap_rmdirG(plugin_handle handle, const char* name, GError** err); gfal_file_handle gfal_dcap_opendirG(plugin_handle handle, const char* path, GError ** err); int gfal_dcap_closedirG(plugin_handle handle, gfal_file_handle fh, GError** err); struct dirent* gfal_dcap_readdirG(plugin_handle handle, gfal_file_handle fh, GError** err); int gfal_dcap_unlinkG(plugin_handle handle, const char* url, GError** err); const char* gfal_dcap_getName(); gfal2-v2.23.0/src/plugins/dcap/gfal_dcap_plugin_layer.c000066400000000000000000000045111465240014500227560ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #if defined __i386__ || defined _M_IX86 #define __USE_LARGEFILE64 1 #endif #include #include #include #include #include #include #include #include "gfal_dcap_plugin_layer.h" typedef int (*lstat_t)(const char *, struct stat *); typedef struct dirent *(*readdir_t)(DIR *); typedef int (*stat_t)(const char *, struct stat *); struct dcap_proto_ops * gfal_dcap_internal_loader_base(GError** err) { struct dcap_proto_ops * pops = NULL; GError* tmp_err = NULL; pops = g_new0(struct dcap_proto_ops, 1); pops->geterror = &__dc_errno; pops->strerror = &dc_strerror; pops->access = &dc_access; pops->chmod = &dc_chmod; pops->close = &dc_close; pops->closedir = &dc_closedir; pops->lseek = &dc_lseek; pops->lstat = (lstat_t) &dc_lstat; pops->mkdir = &dc_mkdir; pops->open = &dc_open; pops->opendir = &dc_opendir; pops->read = &dc_read; pops->pread = &dc_pread; pops->readdir = (readdir_t) &dc_readdir; pops->rename = NULL; pops->rmdir = &dc_rmdir; pops->stat = (stat_t) &dc_stat; pops->unlink = &dc_unlink; pops->write = &dc_write; pops->pwrite = &dc_pwrite; pops->debug_level = &dc_setDebugLevel; pops->active_mode = &dc_setClientActive; // pops->active_mode(); // switch to active mode to avoid firewalls problems if ((gfal2_log_get_level() >= G_LOG_LEVEL_DEBUG)) pops->debug_level(8 | 6 | 32); G_RETURN_ERR(pops, tmp_err, err); } struct dcap_proto_ops * (*gfal_dcap_internal_loader)(GError** err)= &gfal_dcap_internal_loader_base; gfal2-v2.23.0/src/plugins/dcap/gfal_dcap_plugin_layer.h000066400000000000000000000035331465240014500227660ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include #include #include struct dcap_proto_ops { int* (*geterror)(); const char*(*strerror)(int); int (*access)(const char *, int); int (*chmod)(const char *, mode_t); int (*close)(int); int (*closedir)(DIR *); void (*debug_level)(unsigned int); void (*active_mode)(); off_t (*lseek)(int, off_t, int); int (*lstat)(const char *, struct stat *); int (*mkdir)(const char *, mode_t); int (*open)(const char *, int, ...); DIR *(*opendir)(const char *); ssize_t (*read)(int, void *, size_t); ssize_t (*pread)(int fildes, void *buf, size_t nbytes, off_t offset); ssize_t (*pwrite)(int fildes, const void *buf, size_t nbytes, off_t offset); struct dirent *(*readdir)(DIR *); int (*rename)(const char *, const char *); int (*rmdir)(const char *); ssize_t (*setfilchg)(int, const void *, size_t); int (*stat)(const char *, struct stat *); int (*unlink)(const char *); ssize_t (*write)(int, const void *, size_t); }; extern struct dcap_proto_ops * (*gfal_dcap_internal_loader)(GError** err); gfal2-v2.23.0/src/plugins/dcap/gfal_dcap_plugin_main.c000066400000000000000000000112351465240014500225670ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include "gfal_dcap_plugin_layer.h" #include "gfal_dcap_plugin_main.h" #include "gfal_dcap_plugin_bindings.h" void gfal_dcap_destroyG(plugin_handle handle); gboolean gfal_dcap_check_url(plugin_handle ch, const char* url, plugin_mode mode, GError** err); static int gfal_dcap_regex_compile(regex_t * rex, GError** err) { int ret = regcomp(rex, "^(dcap|gsidcap)://([:alnum:]|-|/|.|_)+$", REG_ICASE | REG_EXTENDED); g_return_val_err_if_fail(ret == 0, -1, err, "[gfal_dcap_regex_compile] fail to compile regex, report this bug"); return ret; } // RFIO plugin GQuark GQuark gfal2_get_plugin_dcap_quark() { return g_quark_from_static_string(GFAL2_QUARK_PLUGINS "::DCAP"); } static gfal_plugin_dcap_handle gfal_dcap_init_handle(gfal2_context_t handle, GError** err) { gfal_plugin_dcap_handle ret = g_new0(struct _gfal_plugin_dcap_handle, 1); ret->ops = gfal_dcap_internal_loader(err); ret->handle = handle; gfal_dcap_regex_compile(&(ret->rex), err); return ret; } const char* gfal_dcap_getName() { return GFAL2_PLUGIN_VERSIONED("dcap", VERSION); } /* * Init function, called before all * */ gfal_plugin_interface gfal_plugin_init(gfal2_context_t handle, GError** err) { gfal_plugin_interface dcap_plugin; GError* tmp_err = NULL; memset(&dcap_plugin, 0, sizeof(gfal_plugin_interface)); // clear the plugin dcap_plugin.plugin_data = (plugin_handle) gfal_dcap_init_handle(handle, &tmp_err); dcap_plugin.plugin_delete = &gfal_dcap_destroyG; dcap_plugin.getName = &gfal_dcap_getName; dcap_plugin.openG = &gfal_dcap_openG; dcap_plugin.closeG = &gfal_dcap_closeG; dcap_plugin.readG = &gfal_dcap_readG; dcap_plugin.preadG = &gfal_dcap_preadG; dcap_plugin.writeG = &gfal_dcap_writeG; dcap_plugin.pwriteG = &gfal_dcap_pwriteG; dcap_plugin.lseekG = &gfal_dcap_lseekG; dcap_plugin.check_plugin_url = &gfal_dcap_check_url; dcap_plugin.statG = &gfal_dcap_statG; dcap_plugin.lstatG = &gfal_dcap_lstatG; dcap_plugin.mkdirpG = &gfal_dcap_mkdirG; dcap_plugin.chmodG = &gfal_dcap_chmodG; dcap_plugin.rmdirG = &gfal_dcap_rmdirG; dcap_plugin.closedirG = &gfal_dcap_closedirG; dcap_plugin.opendirG = &gfal_dcap_opendirG; dcap_plugin.readdirG = &gfal_dcap_readdirG; dcap_plugin.accessG = &gfal_dcap_accessG; dcap_plugin.unlinkG = &gfal_dcap_unlinkG; if (tmp_err) gfal2_propagate_prefixed_error(err, tmp_err, __func__); return dcap_plugin; } void gfal_dcap_destroyG(plugin_handle handle) { gfal_plugin_dcap_handle h = (gfal_plugin_dcap_handle) handle; free(h->ops); regfree(&(h->rex)); free(h); } gboolean gfal_dcap_internal_check_url(gfal_plugin_dcap_handle dh, const char* surl, GError** err) { if (surl == NULL || strnlen(surl, GFAL_URL_MAX_LEN) == GFAL_URL_MAX_LEN) { gfal2_set_error(err, gfal2_get_plugin_dcap_quark(), EINVAL, __func__, "Invalid surl, surl too long or NULL"); return FALSE; } int ret = regexec(&dh->rex, surl, 0, NULL, 0); return (ret == 0) ? TRUE : FALSE; } /* * Check the dcap url in the gfal module way * */ gboolean gfal_dcap_check_url(plugin_handle ch, const char* url, plugin_mode mode, GError** err) { int ret; GError* tmp_err = NULL; gfal_plugin_dcap_handle rh = (gfal_plugin_dcap_handle) ch; switch (mode) { case GFAL_PLUGIN_OPEN: case GFAL_PLUGIN_LSTAT: case GFAL_PLUGIN_STAT: case GFAL_PLUGIN_MKDIR: case GFAL_PLUGIN_CHMOD: case GFAL_PLUGIN_RMDIR: case GFAL_PLUGIN_OPENDIR: case GFAL_PLUGIN_ACCESS: case GFAL_PLUGIN_UNLINK: ret = gfal_dcap_internal_check_url(rh, url, &tmp_err); break; default: ret = FALSE; break; } if (tmp_err) gfal2_propagate_prefixed_error(err, tmp_err, __func__); return ret; } gfal2-v2.23.0/src/plugins/dcap/gfal_dcap_plugin_main.h000066400000000000000000000020661465240014500225760ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include typedef struct _gfal_plugin_dcap_handle{ gfal2_context_t handle; struct dcap_proto_ops* ops; regex_t rex; }* gfal_plugin_dcap_handle; GQuark gfal2_get_plugin_dcap_quark(); // Entry point of the plugin gfal_plugin_interface gfal_plugin_init(gfal2_context_t context, GError** err); gfal2-v2.23.0/src/plugins/file/000077500000000000000000000000001465240014500161475ustar00rootroot00000000000000gfal2-v2.23.0/src/plugins/file/CMakeLists.txt000066400000000000000000000013721465240014500207120ustar00rootroot00000000000000cmake_minimum_required (VERSION 2.6) if (PLUGIN_FILE) file (GLOB src_file "*.c*") find_package (ZLIB REQUIRED) include_directories(${ZLIB_INCLUDE_DIRS}) add_library (plugin_file MODULE ${src_file} ${gfal2_src_checksum}) target_link_libraries (plugin_file gfal2 ${ZLIB_LIBRARIES}) set_target_properties(plugin_file PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/src CLEAN_DIRECT_OUTPUT 1 OUTPUT_NAME "gfal_plugin_file" LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/plugins) install(TARGETS plugin_file LIBRARY DESTINATION ${PLUGIN_INSTALL_DIR} ) install(FILES "README_PLUGIN_FILE" DESTINATION ${DOC_INSTALL_DIR}) endif (PLUGIN_FILE) gfal2-v2.23.0/src/plugins/file/README_PLUGIN_FILE000066400000000000000000000001221465240014500206770ustar00rootroot00000000000000File plugin : - provide the map to the local POSIX calls for the gfal2 system gfal2-v2.23.0/src/plugins/file/gfal_file_plugin_main.c000066400000000000000000000440351465240014500226130ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include #include #include #include #if defined __APPLE__ #include #else #if defined __GLIBC_PREREQ && __GLIBC_PREREQ(2,27) #include #else #include #endif #endif #include #include #include #include #include typedef struct _chksum_interface{ // init checksum handle void* (*init)(void); // compute checksum chunk ssize_t (*update)(void* chk_handler, const char* buffer, size_t s_size); // return checksum result : > 0 -> success, -1 : buffer to short int (*getResult)(void* chk_handler, char* buffer, size_t s_b); } Chksum_interface; static const int FILE_PREFIX_LEN = 7; // file:// // File plugin GQuark GQuark gfal2_get_plugin_file_quark(){ return g_quark_from_static_string(GFAL2_QUARK_PLUGINS "::FILE"); } /* * srm plugin id */ const char* gfal_file_plugin_getName(){ return GFAL2_PLUGIN_VERSIONED("file", VERSION); } /* * Return 1 if url is a file url */ static int gfal_is_file(const char *url) { GError *err = NULL; gfal2_uri *parsed = gfal2_parse_uri(url, &err); if (!parsed) { g_error_free(err); return 0; } // Check if host is at least defined (even if empty), so only file:// is accepted! int check = parsed->scheme != NULL && strcmp(parsed->scheme, "file") == 0 \ && parsed->host != NULL && strcmp(parsed->host, "") == 0 \ && parsed->path != NULL && parsed->path[0] == '/'; gfal2_free_uri(parsed); return check; } /* * url checker for the file module */ static gboolean gfal_file_check_url(plugin_handle handle, const char* url, plugin_mode mode, GError** err){ g_return_val_err_if_fail(url != NULL, EINVAL, err, "[gfal_lfile_path_checker] Invalid url "); switch(mode){ case GFAL_PLUGIN_ACCESS: case GFAL_PLUGIN_MKDIR: case GFAL_PLUGIN_STAT: case GFAL_PLUGIN_LSTAT: case GFAL_PLUGIN_RMDIR: case GFAL_PLUGIN_OPENDIR: case GFAL_PLUGIN_OPEN: case GFAL_PLUGIN_CHMOD: case GFAL_PLUGIN_UNLINK: case GFAL_PLUGIN_GETXATTR: case GFAL_PLUGIN_LISTXATTR: case GFAL_PLUGIN_SETXATTR: case GFAL_PLUGIN_RENAME: case GFAL_PLUGIN_SYMLINK: case GFAL_PLUGIN_CHECKSUM: case GFAL_PLUGIN_READLINK: return gfal_is_file(url); default: return FALSE; } } void gfal_plugin_file_report_error(const char* funcname, GError** err){ gfal2_set_error(err, gfal2_get_plugin_file_quark(), errno, funcname, "errno reported by local system call %s", strerror(errno)); } int gfal_plugin_file_access(plugin_handle plugin_data, const char *path, int amode, GError **err) { const int res = access(path + FILE_PREFIX_LEN, amode); if (res < 0) { gfal_plugin_file_report_error(__func__, err); } else { errno = 0; } return res; } int gfal_plugin_file_chmod(plugin_handle plugin_data, const char* path, mode_t mode,GError** err){ const int res = chmod(path + FILE_PREFIX_LEN, mode); if (res < 0) { gfal_plugin_file_report_error(__func__, err); } else { errno = 0; } return res; } ssize_t gfal_plugin_file_getxattr(plugin_handle plugin_data, const char *path, const char *name, void *buff, size_t s_buff, GError **err) { #ifdef __APPLE__ const ssize_t res = getxattr(path + FILE_PREFIX_LEN, name, buff, s_buff, 0, XATTR_NOFOLLOW); #else const ssize_t res = getxattr(path + FILE_PREFIX_LEN, name, buff, s_buff); #endif if (res < 0) { gfal_plugin_file_report_error(__func__, err); } else { errno = 0; } return res; } ssize_t gfal_plugin_file_listxattr(plugin_handle plugin_data, const char *path, char *list, size_t s_list, GError **err) { #ifdef __APPLE__ const ssize_t res = listxattr(path + FILE_PREFIX_LEN, list, s_list, XATTR_NOFOLLOW); #else const ssize_t res = listxattr(path + FILE_PREFIX_LEN, list, s_list); #endif if (res < 0) { gfal_plugin_file_report_error(__func__, err); } else { errno = 0; } return res; } int gfal_plugin_file_setxattr(plugin_handle plugin_data, const char *path, const char *name, const void *value, size_t size, int flags, GError **err) { #ifdef __APPLE__ const int res = setxattr(path + FILE_PREFIX_LEN, name, value, size, 0, flags); #else const int res = setxattr(path + FILE_PREFIX_LEN, name, value, size, flags); #endif if (res < 0) { gfal_plugin_file_report_error(__func__, err); } else { errno = 0; } return res; } int gfal_plugin_file_rename(plugin_handle plugin_data, const char *oldpath, const char *newpath, GError **err) { const int res = rename(oldpath + FILE_PREFIX_LEN, newpath + FILE_PREFIX_LEN); if (res < 0) { gfal_plugin_file_report_error(__func__, err); } else { errno = 0; } return res; } int gfal_plugin_file_stat(plugin_handle plugin_data, const char *path, struct stat *buf, GError **err) { const int res = stat(path + FILE_PREFIX_LEN, buf); if (res < 0) { gfal_plugin_file_report_error(__func__, err); } else { errno = 0; } return res; } int gfal_plugin_file_lstat(plugin_handle plugin_data, const char *path, struct stat *buf, GError **err) { const int res = lstat(path + FILE_PREFIX_LEN, buf); if (res < 0) { gfal_plugin_file_report_error(__func__, err); } else { errno = 0; } return res; } int gfal_plugin_file_mkdir(plugin_handle plugin_data, const char *path, mode_t mode, gboolean pflag, GError **err) { const int res = mkdir(path + FILE_PREFIX_LEN, mode); if (res < 0) { gfal_plugin_file_report_error(__func__, err); } return res; } int gfal_plugin_file_unlink(plugin_handle plugin_data, const char *path, GError **err) { const int res = unlink(path + FILE_PREFIX_LEN); if (res < 0) { gfal_plugin_file_report_error(__func__, err); } return res; } gfal_file_handle gfal_plugin_file_opendir(plugin_handle plugin_data, const char *path, GError **err) { DIR *ret = opendir(path + FILE_PREFIX_LEN); gfal_file_handle resu = NULL; if (ret == NULL) { gfal_plugin_file_report_error(__func__, err); } if (ret) resu = gfal_file_handle_new2(gfal_file_plugin_getName(), (gpointer) ret, NULL, path); return resu; } struct dirent *gfal_plugin_file_readdir(plugin_handle plugin_data, gfal_file_handle fh, GError **err) { errno = 0; struct dirent *res = readdir(gfal_file_handle_get_fdesc(fh)); if (res == NULL && errno) { gfal_plugin_file_report_error(__func__, err); } return res; } gfal_file_handle gfal_plugin_file_open(plugin_handle plugin_data, const char *path, int flag, mode_t mode, GError **err) { errno = 0; const int ret = open(path + FILE_PREFIX_LEN, flag, mode); if (ret < 0) { gfal_plugin_file_report_error(__func__, err); return NULL; } else { return gfal_file_handle_new(gfal_file_plugin_getName(), GINT_TO_POINTER(ret)); } } /* * map the gfal_read call to the local read call for file:// * */ ssize_t gfal_plugin_file_read(plugin_handle plugin_data, gfal_file_handle fh, void *buff, size_t s_buff, GError **err) { errno = 0; const int fd = GPOINTER_TO_INT(gfal_file_handle_get_fdesc(fh)); const int ret = read(fd, buff, s_buff); if (ret < 0) gfal_plugin_file_report_error(__func__, err); return ret; } /* * map the gfal_pread call to the local pread call for file:// * */ ssize_t gfal_plugin_file_pread(plugin_handle plugin_data, gfal_file_handle fh, void *buff, size_t s_buff, off_t offset, GError **err) { errno = 0; const int fd = GPOINTER_TO_INT(gfal_file_handle_get_fdesc(fh)); const ssize_t ret = pread(fd, buff, s_buff, offset); if (ret < 0) gfal_plugin_file_report_error(__func__, err); return ret; } off_t gfal_plugin_file_lseek(plugin_handle plugin_data, gfal_file_handle fh, off_t offset, int whence, GError **err) { errno = 0; const int ret = lseek(GPOINTER_TO_INT(gfal_file_handle_get_fdesc(fh)), offset, whence); if (ret < 0) gfal_plugin_file_report_error(__func__, err); return ret; } /* * map to the local write call * */ ssize_t gfal_plugin_file_write(plugin_handle plugin_data, gfal_file_handle fh, const void *buff, size_t s_buff, GError **err) { errno = 0; const int ret = write(GPOINTER_TO_INT(gfal_file_handle_get_fdesc(fh)), buff, s_buff); if (ret < 0) gfal_plugin_file_report_error(__func__, err); return ret; } /* * map to the local pwrite call */ ssize_t gfal_plugin_file_pwrite(plugin_handle plugin_data, gfal_file_handle fh, const void *buff, size_t s_buff, off_t offset, GError **err) { errno = 0; const ssize_t ret = pwrite(GPOINTER_TO_INT(gfal_file_handle_get_fdesc(fh)), buff, s_buff, offset); if (ret < 0) gfal_plugin_file_report_error(__func__, err); return ret; } int gfal_plugin_file_close(plugin_handle plugin_data, gfal_file_handle fh, GError **err) { errno = 0; const int ret = close(GPOINTER_TO_INT(gfal_file_handle_get_fdesc(fh))); if (ret != 0) { gfal_plugin_file_report_error(__func__, err); } else { gfal_file_handle_delete(fh); } return ret; } ssize_t gfal_plugin_file_readlink(plugin_handle plugin_data, const char *path, char *buff, size_t buffsiz, GError **err) { const ssize_t res = readlink(path + FILE_PREFIX_LEN, buff, buffsiz); if (res < 0) { gfal_plugin_file_report_error(__func__, err); } else if (res == buffsiz) { gfal2_set_error(err, gfal2_get_plugin_file_quark(), ENOMEM, __func__, "Returned path may have been truncated"); } else { errno = 0; buff[res] = '\0'; } return res; } /* * local rmdir mapper * */ int gfal_plugin_file_rmdir(plugin_handle plugin_data, const char *path, GError **err) { const int res = rmdir(path + FILE_PREFIX_LEN); if (res < 0) gfal_plugin_file_report_error(__func__, err); return res; } /* * local closedir mapper * * */ int gfal_plugin_file_closedir(plugin_handle plugin_data, gfal_file_handle fh, GError** err){ const int res = closedir(gfal_file_handle_get_fdesc(fh)); if(res<0) gfal_plugin_file_report_error(__func__, err); else gfal_file_handle_delete(fh); return res; } int gfal_plugin_file_symlink(plugin_handle plugin_data, const char *oldpath, const char *newpath, GError **err) { const int res = symlink(oldpath + FILE_PREFIX_LEN, newpath + FILE_PREFIX_LEN); if (res != 0) gfal_plugin_file_report_error(__func__, err); return res; } // checksum wrapper static void *adler_init() { unsigned long *lp = malloc(sizeof(unsigned long)); *lp = adler32(0L, Z_NULL, 0); return (void *) lp; } static ssize_t adler32_update(void *chk_handler, const char *buffer, size_t s) { unsigned long *lp = (unsigned long *) chk_handler; *lp = adler32(*lp, (const Bytef *) buffer, (uInt) s); return (ssize_t) s; } static int adler32_getResult(void *chk_handler, char *resu, size_t s_b) { unsigned long *lp = (unsigned long *) chk_handler; snprintf(resu, s_b, "%08lx", *lp); free(lp); return 0; } static void *crc32_init() { unsigned long *lp = malloc(sizeof(unsigned long)); *lp = crc32(0L, Z_NULL, 0); return (void *) lp; } static ssize_t crc32_update(void *chk_handler, const char *buffer, size_t s) { unsigned long *lp = (unsigned long *) chk_handler; *lp = crc32(*lp, (const Bytef *) buffer, s); return (ssize_t) s; } static int crc32_getResult(void *chk_handler, char *resu, size_t s_b) { unsigned long *lp = (unsigned long *) chk_handler; snprintf(resu, s_b, "%ld", *lp); free(lp); return 0; } static void *md5_init() { GFAL_MD5_CTX *ctx = (GFAL_MD5_CTX *) malloc(sizeof(GFAL_MD5_CTX)); gfal2_md5_init(ctx); return (void *) ctx; } static ssize_t md5_update(void *chk_handler, const char *buffer, size_t s) { GFAL_MD5_CTX *ctx = (GFAL_MD5_CTX *) chk_handler; gfal2_md5_update(ctx, buffer, (unsigned long) s); return (ssize_t) s; } static int md5_getResult(void *chk_handler, char *resu, size_t s_b) { GFAL_MD5_CTX *ctx = (GFAL_MD5_CTX *) chk_handler; unsigned char buffer[16]; if (s_b < 33) // buffer to short return -1; gfal2_md5_final(buffer, ctx); gfal2_md5_to_hex_string(buffer, resu, sizeof(buffer)); free(ctx); return 0; } // checksum implem static int gfal_plugin_file_chk_compute(plugin_handle data, const char *url, const char *check_type, char *checksum_buffer, size_t buffer_length, off_t start_offset, size_t data_length, Chksum_interface *i_chk, GError **err) { GError *tmp_err = NULL; const ssize_t chunk_size = 2 << 20; gfal2_context_t handle = (gfal2_context_t) data; int fd; ssize_t ret = 0, remain_bytes = ((data_length > 0) ? (data_length) : (chunk_size)); if ((fd = gfal2_open(handle, url, O_RDONLY, &tmp_err)) < 0) { g_prefix_error(err, "Error during checksum calculation, open "); gfal2_propagate_prefixed_error(err, tmp_err, __func__); return -1; } if (gfal2_lseek(handle, fd, start_offset, SEEK_SET, &tmp_err) < 0) { g_prefix_error(err, "Error during checksum calculation, lseek "); gfal2_propagate_prefixed_error(err, tmp_err, __func__); return -1; } void *c_handle = i_chk->init(); char *buffer = malloc(chunk_size); do { ret = gfal2_read(handle, fd, buffer, MIN(chunk_size, remain_bytes), &tmp_err); if (data_length > 0) { remain_bytes -= ret; } if (ret > 0) { i_chk->update(c_handle, buffer, ret); } } while (ret > 0 && remain_bytes > 0); free(buffer); gfal2_close(handle, fd, NULL); if (i_chk->getResult(c_handle, checksum_buffer, buffer_length) < 0) { gfal2_set_error(err, gfal2_get_plugin_file_quark(), ENOBUFS, __func__, "buffer for checksum too short"); return -1; } if (ret < 0) { gfal2_set_error(err, gfal2_get_plugin_file_quark(), tmp_err->code, __func__, "Error during checksum calculation, read: %s", tmp_err->message); g_error_free(tmp_err); return -1; } return 0; } int gfal_plugin_filechecksum_calc(plugin_handle data, const char *url, const char *check_type, char *checksum_buffer, size_t buffer_length, off_t start_offset, size_t data_length, GError **err) { if (strcasecmp(check_type, "adler32") == 0) { Chksum_interface ie = {.init = &adler_init, .update = &adler32_update, .getResult = &adler32_getResult}; return gfal_plugin_file_chk_compute(data, url, check_type, checksum_buffer, buffer_length, start_offset, data_length, &ie, err); } else if (strcasecmp(check_type, "crc32") == 0) { Chksum_interface ie = {.init = &crc32_init, .update = &crc32_update, .getResult = &crc32_getResult}; return gfal_plugin_file_chk_compute(data, url, check_type, checksum_buffer, buffer_length, start_offset, data_length, &ie, err); } else if (strcasecmp(check_type, "md5") == 0) { Chksum_interface ie = {.init = &md5_init, .update = &md5_update, .getResult = &md5_getResult}; return gfal_plugin_file_chk_compute(data, url, check_type, checksum_buffer, buffer_length, start_offset, data_length, &ie, err); } gfal2_set_error(err, gfal2_get_plugin_file_quark(), ENOSYS, __func__, "Checksum type %s not supported for local files", check_type); return -1; } /* * Init function, called before all * */ gfal_plugin_interface gfal_plugin_init(gfal2_context_t handle, GError **err) { gfal_plugin_interface file_plugin; memset(&file_plugin, 0, sizeof(gfal_plugin_interface)); // clear the plugin file_plugin.plugin_data = handle; file_plugin.check_plugin_url = &gfal_file_check_url; file_plugin.getName = &gfal_file_plugin_getName; file_plugin.plugin_delete = NULL; file_plugin.accessG = &gfal_plugin_file_access; file_plugin.mkdirpG = &gfal_plugin_file_mkdir; file_plugin.statG = &gfal_plugin_file_stat; file_plugin.lstatG = &gfal_plugin_file_lstat; file_plugin.renameG = &gfal_plugin_file_rename; file_plugin.symlinkG = &gfal_plugin_file_symlink; file_plugin.rmdirG = &gfal_plugin_file_rmdir; file_plugin.opendirG = &gfal_plugin_file_opendir; file_plugin.readdirG = &gfal_plugin_file_readdir; file_plugin.closedirG = &gfal_plugin_file_closedir; file_plugin.readlinkG = &gfal_plugin_file_readlink; file_plugin.openG = &gfal_plugin_file_open; file_plugin.closeG = &gfal_plugin_file_close; file_plugin.readG = &gfal_plugin_file_read; file_plugin.preadG = &gfal_plugin_file_pread; file_plugin.writeG = &gfal_plugin_file_write; file_plugin.pwriteG = &gfal_plugin_file_pwrite; file_plugin.chmodG = &gfal_plugin_file_chmod; file_plugin.lseekG = &gfal_plugin_file_lseek; file_plugin.unlinkG = &gfal_plugin_file_unlink; file_plugin.getxattrG = &gfal_plugin_file_getxattr; file_plugin.listxattrG = &gfal_plugin_file_listxattr; file_plugin.setxattrG = &gfal_plugin_file_setxattr; file_plugin.checksum_calcG = &gfal_plugin_filechecksum_calc; return file_plugin; } gfal2-v2.23.0/src/plugins/gridftp/000077500000000000000000000000001465240014500166675ustar00rootroot00000000000000gfal2-v2.23.0/src/plugins/gridftp/CMakeLists.txt000066400000000000000000000032051465240014500214270ustar00rootroot00000000000000cmake_minimum_required (VERSION 2.6) if (PLUGIN_GRIDFTP) file (GLOB_RECURSE src_gridftp "*.c*") find_package (Globus_FTP_CLIENT REQUIRED) find_package (Globus_FTP_CONTROL REQUIRED) find_package (Globus_GASS_COPY REQUIRED) find_package (Globus_COMMON REQUIRED) find_package (Globus_GSS_ASSIST REQUIRED) find_package (Globus_GSSAPI_GSI REQUIRED) add_definitions( ${GLOBUS_GASS_COPY_CFLAGS}) add_library (plugin_gridftp MODULE ${src_gridftp}) target_link_libraries(plugin_gridftp gfal2 gfal2_transfer ${GLOBUS_FTP_CLIENT_LIBRARIES} ${GLOBUS_FTP_CONTROL_LIBRARIES} ${GLOBUS_GASS_COPY_LIBRARIES} ${GLOBUS_GSSAPI_GSI} ${GLOBUS_COMMON_LIBRARIES} ${GLOBUS_GSS_ASSIST_LIBRARIES} ${GLOBUS_GSSAPI_GSI_LIBRARIES}) include_directories(${GLOBUS_GASS_COPY_INCLUDE_DIRS}) set_target_properties(plugin_gridftp PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/src CLEAN_DIRECT_OUTPUT 1 OUTPUT_NAME "gfal_plugin_gridftp" LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/plugins) install(TARGETS plugin_gridftp LIBRARY DESTINATION ${PLUGIN_INSTALL_DIR} ) install(FILES "README_PLUGIN_GRIDFTP" DESTINATION ${DOC_INSTALL_DIR}) # install gsiftp configuration files LIST(APPEND gsiftp_conf_file "${CMAKE_SOURCE_DIR}/dist/etc/gfal2.d/gsiftp_plugin.conf") install(FILES ${gsiftp_conf_file} DESTINATION ${SYSCONF_INSTALL_DIR}/gfal2.d/) endif (PLUGIN_GRIDFTP) gfal2-v2.23.0/src/plugins/gridftp/README_PLUGIN_GRIDFTP000066400000000000000000000001111465240014500217750ustar00rootroot00000000000000 gfal2 gridftp plugins : - features : * thrid party transfert copy gfal2-v2.23.0/src/plugins/gridftp/gridftp_bulk.cpp000066400000000000000000000601251465240014500220530ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2015 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include "gridftp_filecopy.h" #include "gridftpwrapper.h" #include "gridftp_namespace.h" #include "gridftp_plugin.h" #include #include static const GQuark GSIFTP_BULK_DOMAIN = g_quark_from_static_string("GridFTP::Filecopy"); struct GridFTPBulkData { GridFTPBulkData(size_t nbfiles) : srcs(NULL), dsts(NULL), checksums(nbfiles), errn(new int[nbfiles]), fsize(new off_t[nbfiles]), index(0), nbfiles(nbfiles), started(new bool[nbfiles]), params(NULL), error(NULL), done(false) { for (size_t i = 0; i < nbfiles; ++i) { started[i] = false; errn[i] = 0; fsize[i] = 0; } globus_mutex_init(&lock, GLOBUS_NULL); globus_cond_init(&cond, GLOBUS_NULL); } ~GridFTPBulkData() { delete [] started; delete [] errn; delete [] fsize; if (error) globus_object_free(error); } const char* const* srcs; const char* const* dsts; std::vector checksums; int* errn; off_t* fsize; size_t index, nbfiles; bool *started; gfalt_params_t params; globus_mutex_t lock; globus_cond_t cond; globus_object_t* error; bool done; }; struct GridFTPBulkPerformance { std::string source, destination; gfalt_params_t params; bool ipv6; time_t start_time; globus_ftp_client_plugin_t* plugin; }; // Called by Globus when done static void gridftp_done_callback(void * user_arg, globus_ftp_client_handle_t * handle, globus_object_t * err) { GridFTPBulkData* data = static_cast(user_arg); if (err) { data->error = globus_object_copy(err); } else { for (size_t i = 0; i < data->nbfiles; ++i) { if (data->started[i]) { plugin_trigger_event(data->params, GSIFTP_BULK_DOMAIN, GFAL_EVENT_NONE, GFAL_EVENT_TRANSFER_EXIT, "Done %s => %s", data->srcs[i], data->dsts[i]); } } } globus_mutex_lock(&data->lock); data->done = true; globus_cond_signal(&data->cond); globus_mutex_unlock(&data->lock); } // Called by Globus when ready for a new pair static void gridftp_pipeline_callback(globus_ftp_client_handle_t * handle, char ** source_url, char ** dest_url, void * user_arg) { GridFTPBulkData* data = static_cast(user_arg); // Next data->index++; // Skip pairs marked as failed while (data->index < data->nbfiles && data->errn[data->index]) { gfal2_log(G_LOG_LEVEL_DEBUG, "Skipping pair %d as marked failed with %d", data->index, data->errn[data->index]); data->index++; } // Return next pair if (data->index < data->nbfiles) { *source_url = (char*)data->srcs[data->index]; *dest_url = (char*)data->dsts[data->index]; data->started[data->index] = true; gfal2_log(G_LOG_LEVEL_MESSAGE, "Providing pair %s => %s", *source_url, *dest_url); } else { *source_url = NULL; *dest_url = NULL; gfal2_log(G_LOG_LEVEL_MESSAGE, "No more pairs to give"); } } static void gridftp_bulk_cancel(gfal2_context_t context, void* userdata) { globus_ftp_client_handle_t* ftp_handle = static_cast(userdata); globus_ftp_client_abort(ftp_handle); } static void gridftp_bulk_begin_cb(void * user_specific, globus_ftp_client_handle_t * handle, const char * source_url, const char * dest_url) { GridFTPBulkPerformance* original = static_cast(user_specific); GridFTPBulkPerformance* pd; globus_ftp_client_throughput_plugin_get_user_specific(original->plugin, (void**)(&pd)); pd->source = source_url; pd->destination = dest_url; pd->start_time = time(NULL); plugin_trigger_event(pd->params, GSIFTP_BULK_DOMAIN, GFAL_EVENT_NONE, GFAL_EVENT_TRANSFER_ENTER, "(%s) %s => (%s) %s", return_host_and_port(source_url, pd->ipv6).c_str(), source_url, return_host_and_port(dest_url, pd->ipv6).c_str(), dest_url); plugin_trigger_event(pd->params, GFAL_GRIDFTP_DOMAIN_GSIFTP, GFAL_EVENT_NONE, GFAL_EVENT_TRANSFER_TYPE, "%s", GFAL_TRANSFER_TYPE_PUSH); } static void gridftp_bulk_throughput_cb(void *user_specific, globus_ftp_client_handle_t *handle, globus_off_t bytes, float instantaneous_throughput, float avg_throughput) { GridFTPBulkPerformance* original = static_cast(user_specific); GridFTPBulkPerformance* pd; globus_ftp_client_throughput_plugin_get_user_specific(original->plugin, (void**)(&pd)); _gfalt_transfer_status status; status.bytes_transfered = bytes; status.average_baudrate = (size_t) avg_throughput; status.instant_baudrate = (size_t) instantaneous_throughput; status.transfer_time = (time(NULL) - pd->start_time); plugin_trigger_monitor(pd->params, &status, pd->source.c_str(), pd->destination.c_str()); } static void gridftp_bulk_complete_cb(void * user_specific, globus_ftp_client_handle_t * handle, globus_bool_t success) { GridFTPBulkPerformance* original = static_cast(user_specific); GridFTPBulkPerformance* pd; globus_ftp_client_throughput_plugin_get_user_specific(original->plugin, (void**)(&pd)); } static void* gridftp_bulk_copy_perf_cb(void * user_specific) { GridFTPBulkPerformance* pd = static_cast(user_specific); return new GridFTPBulkPerformance(*pd); } static void gridftp_bulk_destroy_perf_cb(void * user_specific) { GridFTPBulkPerformance* pd = static_cast(user_specific); delete pd; } static void gridftp_pipeline_init_operationattr(globus_ftp_client_operationattr_t *ftp_operation_attr, const globus_ftp_client_operationattr_t *original, gss_cred_id_t *cred_id, gfal2_context_t context, bool udt, const char *url, GError **op_error) { globus_ftp_client_operationattr_copy(ftp_operation_attr, original); globus_ftp_client_operationattr_set_mode(ftp_operation_attr, GLOBUS_FTP_CONTROL_MODE_EXTENDED_BLOCK); globus_ftp_client_operationattr_set_delayed_pasv(ftp_operation_attr, GLOBUS_FALSE); if (udt) { globus_ftp_client_operationattr_set_net_stack(ftp_operation_attr, "udt"); } else { globus_ftp_client_operationattr_set_net_stack(ftp_operation_attr, "default"); } gchar *ucert = gfal2_cred_get(context, GFAL_CRED_X509_CERT, url, NULL, op_error); gchar *ukey = gfal2_cred_get(context, GFAL_CRED_X509_KEY, url, NULL, op_error); gfal_globus_set_credentials(ucert,ukey, NULL, NULL, cred_id, ftp_operation_attr); g_free(ucert); g_free(ukey); } static int gridftp_pipeline_transfer(plugin_handle plugin_data, gfal2_context_t context, bool udt, GridFTPBulkData* pairs, GError** op_error) { GridFTPModule* gsiftp = static_cast(plugin_data); GridFTPSessionHandler handler(gsiftp->get_session_factory(), pairs->srcs[0]); globus_ftp_client_plugin_t throughput_plugin; globus_ftp_client_handle_t ftp_handle; globus_ftp_client_operationattr_t ftp_operation_attr_src, ftp_operation_attr_dst; gss_cred_id_t cred_id_src = NULL, cred_id_dst = NULL; globus_ftp_client_handleattr_t* ftp_handle_attr = handler.get_ftp_client_handleattr(); // First viable pair goes with the globus call pairs->index = 0; while (pairs->index < pairs->nbfiles && pairs->errn[pairs->index]) pairs->index++; if (pairs->index >= pairs->nbfiles) return 0; pairs->started[pairs->index] = true; GridFTPBulkPerformance perf; perf.params = pairs->params; perf.ipv6 = gfal2_get_opt_boolean_with_default(context, GRIDFTP_CONFIG_GROUP, GRIDFTP_CONFIG_IPV6, false); perf.plugin = &throughput_plugin; globus_ftp_client_throughput_plugin_init(&throughput_plugin, gridftp_bulk_begin_cb, NULL, gridftp_bulk_throughput_cb, gridftp_bulk_complete_cb, &perf); globus_ftp_client_throughput_plugin_set_copy_destroy(&throughput_plugin, gridftp_bulk_copy_perf_cb, gridftp_bulk_destroy_perf_cb); globus_ftp_client_handleattr_add_plugin(ftp_handle_attr, &throughput_plugin); globus_ftp_client_handleattr_set_pipeline(ftp_handle_attr, 0, gridftp_pipeline_callback, pairs); globus_ftp_client_handle_init(&ftp_handle, ftp_handle_attr); gridftp_pipeline_init_operationattr( &ftp_operation_attr_src, handler.get_ftp_client_operationattr(), &cred_id_src, context, udt, pairs->srcs[pairs->index], op_error); gridftp_pipeline_init_operationattr( &ftp_operation_attr_dst, handler.get_ftp_client_operationattr(), &cred_id_dst, context, udt, pairs->dsts[pairs->index], op_error); int nbstreams = gfal2_get_opt_integer_with_default(context, GRIDFTP_CONFIG_GROUP, GRIDFTP_CONFIG_NB_STREAM, 0); if (nbstreams == 0) { nbstreams = gfalt_get_nbstreams(pairs->params, NULL); } guint64 buffer_size = gfalt_get_tcp_buffer_size(pairs->params, NULL); globus_ftp_control_parallelism_t parallelism; globus_ftp_control_tcpbuffer_t tcp_buffer_size; if (nbstreams > 1) { parallelism.fixed.size = nbstreams; parallelism.mode = GLOBUS_FTP_CONTROL_PARALLELISM_FIXED; globus_ftp_client_operationattr_set_mode(&ftp_operation_attr_src, GLOBUS_FTP_CONTROL_MODE_EXTENDED_BLOCK); globus_ftp_client_operationattr_set_parallelism(&ftp_operation_attr_src, ¶llelism); globus_ftp_client_operationattr_set_mode(&ftp_operation_attr_dst, GLOBUS_FTP_CONTROL_MODE_EXTENDED_BLOCK); globus_ftp_client_operationattr_set_parallelism(&ftp_operation_attr_dst, ¶llelism); } if (buffer_size > 0) { tcp_buffer_size.mode = GLOBUS_FTP_CONTROL_TCPBUFFER_FIXED; tcp_buffer_size.fixed.size = buffer_size; globus_ftp_client_operationattr_set_tcp_buffer(&ftp_operation_attr_src, &tcp_buffer_size); globus_ftp_client_operationattr_set_tcp_buffer(&ftp_operation_attr_dst, &tcp_buffer_size); } gfal_cancel_token_t cancel_token; cancel_token = gfal2_register_cancel_callback(context, gridftp_bulk_cancel, ftp_handle); int res = 0; try { globus_result_t globus_return; globus_return = globus_ftp_client_third_party_transfer(&ftp_handle, pairs->srcs[pairs->index], &ftp_operation_attr_src, pairs->dsts[pairs->index], &ftp_operation_attr_dst, GLOBUS_NULL, gridftp_done_callback, pairs); gfal_globus_check_result(GSIFTP_BULK_DOMAIN, globus_return); globus_mutex_lock(&pairs->lock); guint64 timeout = gfalt_get_timeout(pairs->params, NULL); globus_abstime_t timeout_expires; GlobusTimeAbstimeGetCurrent(timeout_expires); timeout_expires.tv_sec += timeout; int wait_ret = 0; while (!pairs->done && wait_ret != ETIMEDOUT) { if (timeout > 0) wait_ret = globus_cond_timedwait(&pairs->cond, &pairs->lock, &timeout_expires); else wait_ret = globus_cond_wait(&pairs->cond, &pairs->lock); } globus_mutex_unlock(&pairs->lock); if (pairs->error) { char *err_buffer; int err_code = gfal_globus_error_convert(pairs->error, &err_buffer); if (err_code) { gfal2_log(G_LOG_LEVEL_WARNING, "Bulk transfer failed with %s", err_buffer); gfal2_set_error(op_error, GSIFTP_BULK_DOMAIN, err_code, __func__, "%s", err_buffer); res = -1; g_free(err_buffer); } } else if (wait_ret == ETIMEDOUT) { gfal2_set_error(op_error, GSIFTP_BULK_DOMAIN, ETIMEDOUT, __func__, "Transfer timed out"); res = -1; } } catch (const Gfal::CoreException& e) { gfal2_log(G_LOG_LEVEL_MESSAGE, "Bulk transfer failed with %s", e.what()); gfal2_set_error(op_error, e.domain(), e.code(), __func__, "%s", e.what()); res = -1; } gfal2_remove_cancel_callback(context, cancel_token); globus_ftp_client_handleattr_remove_plugin(ftp_handle_attr, &throughput_plugin); globus_ftp_client_throughput_plugin_destroy(&throughput_plugin); globus_ftp_client_handle_destroy(&ftp_handle); globus_ftp_client_operationattr_destroy(&ftp_operation_attr_src); globus_ftp_client_operationattr_destroy(&ftp_operation_attr_dst); globus_ftp_client_handleattr_set_pipeline(ftp_handle_attr, 0, NULL, NULL); OM_uint32 minor_status; gss_release_cred(&minor_status, &cred_id_src); gss_release_cred(&minor_status, &cred_id_dst); return res; } static int gridftp_bulk_check_sources(plugin_handle plugin_data, gfal2_context_t context, GridFTPBulkData* pairs, GError** file_errors) { struct stat st; int nfailed = 0, ret = 0; char chk_type[32] = {0}, chk_value[128], dummy[1]; gfalt_checksum_mode_t checksum_mode = gfalt_get_checksum(pairs->params, chk_type, sizeof(chk_type), dummy, 0, NULL); for (size_t i = 0; i < pairs->nbfiles; ++i) { if (gfal2_is_canceled(context)) { gfal2_set_error(&(file_errors[i]), GSIFTP_BULK_DOMAIN, EINTR, __func__, "Operation canceled"); pairs->errn[i] = EINTR; } else if (gfal_gridftp_statG(plugin_data, pairs->srcs[i], &st, &(file_errors[i])) < 0) { pairs->errn[i] = file_errors[i]->code; } else if (S_ISDIR(st.st_mode)) { gfal2_set_error(&(file_errors[i]), GSIFTP_BULK_DOMAIN, EISDIR, __func__, "File is a directory"); pairs->errn[i] = EISDIR; } else { pairs->fsize[i] = st.st_size; if (checksum_mode & GFALT_CHECKSUM_SOURCE) { plugin_trigger_event(pairs->params, GSIFTP_BULK_DOMAIN, GFAL_EVENT_SOURCE, GFAL_EVENT_CHECKSUM_ENTER, "%s", pairs->srcs[i]); ret = gfal_gridftp_checksumG(plugin_data, pairs->srcs[i], chk_type, chk_value, sizeof(chk_value), 0, 0, &(file_errors[i])); if (ret == 0) { if (!pairs->checksums[i].empty()) { if (gfal_compare_checksums(pairs->checksums[i].c_str(), chk_value, sizeof(chk_value)) != 0) { gfalt_set_error(&(file_errors[i]), GSIFTP_BULK_DOMAIN, EIO, GFALT_ERROR_SOURCE, GFALT_ERROR_CHECKSUM_MISMATCH, __func__, "User checksum and source checksum do not match: %s != %s", pairs->checksums[i].c_str(), chk_value); pairs->errn[i] = EIO; } } else { pairs->checksums[i] = chk_value; } } else { pairs->errn[i] = file_errors[i]->code; } plugin_trigger_event(pairs->params, GSIFTP_BULK_DOMAIN, GFAL_EVENT_SOURCE, GFAL_EVENT_CHECKSUM_EXIT, "%s", pairs->srcs[i]); } } if (file_errors[i] != NULL) ++nfailed; } return nfailed; } static int gridftp_bulk_prepare_destination(plugin_handle plugin_data, gfal2_context_t context, GridFTPBulkData* pairs, GError** file_errors) { int nfailed = 0; std::vector created_parents; for (size_t i = 0; i < pairs->nbfiles; ++i) { // May have failed when preparing the source! if (pairs->errn[i] == 0) { if (gfal2_is_canceled(context)) { gfal2_set_error(&(file_errors[i]), GSIFTP_BULK_DOMAIN, EINTR, __func__, "Operation canceled"); pairs->errn[i] = EINTR; } else { try { const char* slash = strrchr(pairs->dsts[i], '/'); std::string parent; if (slash) parent.assign(pairs->dsts[i], 0, slash - pairs->dsts[i]); gridftp_filecopy_delete_existing( (GridFTPModule*) plugin_data, pairs->params, pairs->dsts[i]); if (slash && std::find(created_parents.begin(), created_parents.end(), parent) != created_parents.end()) { gfal2_log(G_LOG_LEVEL_DEBUG, "Skip mkdir of %s", parent.c_str()); } else { gridftp_create_parent_copy((GridFTPModule*) plugin_data, pairs->params, pairs->dsts[i]); created_parents.push_back(parent); } } catch (const Gfal::TransferException& e) { gfal2_set_error(&(file_errors[i]), GSIFTP_BULK_DOMAIN, e.code(), __func__, "%s", e.what()); pairs->errn[i] = e.code(); } } if (file_errors[i] != NULL) ++nfailed; } } return nfailed; } static int gridftp_bulk_prepare(plugin_handle plugin_data, gfal2_context_t context, GridFTPBulkData* pairs, GError** file_errors) { plugin_trigger_event(pairs->params, GSIFTP_BULK_DOMAIN, GFAL_EVENT_NONE, GFAL_EVENT_PREPARE_ENTER, ""); int src_failed = gridftp_bulk_check_sources(plugin_data, context, pairs, file_errors); int dst_failed = gridftp_bulk_prepare_destination(plugin_data, context, pairs, file_errors); plugin_trigger_event(pairs->params, GSIFTP_BULK_DOMAIN, GFAL_EVENT_NONE, GFAL_EVENT_PREPARE_EXIT, ""); return src_failed + dst_failed; } static int gridftp_bulk_close(plugin_handle plugin_data, gfal2_context_t context, GridFTPBulkData* pairs, GError** file_errors) { int nfailed = 0, ret = 0; struct stat st; char chk_type[32] = {0}, chk_value[128], dummy[1]; gfalt_checksum_mode_t checksum_mode = gfalt_get_checksum(pairs->params, chk_type, sizeof(chk_type), dummy, 0, NULL); plugin_trigger_event(pairs->params, GSIFTP_BULK_DOMAIN, GFAL_EVENT_NONE, GFAL_EVENT_CLOSE_ENTER, ""); for (size_t i = 0; i < pairs->nbfiles; ++i) { if (pairs->errn[i] == 0) { if (gfal2_is_canceled(context)) { gfal2_set_error(&(file_errors[i]), GSIFTP_BULK_DOMAIN, EINTR, __func__, "Operation canceled"); pairs->errn[i] = EINTR; } else if (gfal_gridftp_statG(plugin_data, pairs->dsts[i], &st, &(file_errors[i])) < 0) { pairs->errn[i] = file_errors[i]->code; } else { if (pairs->fsize[i] != st.st_size) { gfalt_set_error(&(file_errors[i]), GSIFTP_BULK_DOMAIN, EIO, GFALT_ERROR_DESTINATION, GFALT_ERROR_SIZE_MISMATCH, __func__, "Source and destination file sizes do not match: %lld != %lld", (long long)pairs->fsize, (long long)st.st_size); pairs->errn[i] = EIO; } else if (checksum_mode & GFALT_CHECKSUM_TARGET) { plugin_trigger_event(pairs->params, GSIFTP_BULK_DOMAIN, GFAL_EVENT_DESTINATION, GFAL_EVENT_CHECKSUM_ENTER, "%s", pairs->dsts[i]); ret = gfal_gridftp_checksumG(plugin_data, pairs->dsts[i], chk_type, chk_value, sizeof(chk_value), 0, 0, &(file_errors[i])); if (ret == 0) { if (!pairs->checksums[i].empty()) { if (gfal_compare_checksums( pairs->checksums[i].c_str(), chk_value, sizeof(chk_value)) != 0) { gfalt_set_error(&(file_errors[i]), GSIFTP_BULK_DOMAIN, EIO, __func__, GFALT_ERROR_DESTINATION, GFALT_ERROR_CHECKSUM_MISMATCH, "Destination checksum do not match: %s != %s", pairs->checksums[i].c_str(), chk_value); pairs->errn[i] = EIO; } } else { pairs->checksums[i] = chk_value; } } else { pairs->errn[i] = file_errors[i]->code; } plugin_trigger_event(pairs->params, GSIFTP_BULK_DOMAIN, GFAL_EVENT_DESTINATION, GFAL_EVENT_CHECKSUM_EXIT, "%s", pairs->srcs[i]); } } if (file_errors[i] != NULL) ++nfailed; } } plugin_trigger_event(pairs->params, GSIFTP_BULK_DOMAIN, GFAL_EVENT_NONE, GFAL_EVENT_CLOSE_EXIT, ""); return nfailed; } int gridftp_bulk_copy(plugin_handle plugin_data, gfal2_context_t context, gfalt_params_t params, size_t nbfiles, const char* const * srcs, const char* const * dsts, const char* const * checksums, GError** op_error, GError*** file_errors) { gfal2_log(G_LOG_LEVEL_DEBUG, "-> %s", __func__); if (nbfiles == 0 || srcs == NULL || dsts == NULL) { gfal2_set_error(op_error, GSIFTP_BULK_DOMAIN, EINVAL, __func__, "Invalid parameters"); return -1; } if (gfal2_start_scope_cancel(context, op_error) < 0) return -1; GridFTPBulkData pairs(nbfiles); pairs.srcs = srcs; pairs.dsts = dsts; if (checksums) { for (size_t i = 0; i < nbfiles; ++i) pairs.checksums[i] = checksums[i]; } pairs.nbfiles = nbfiles; pairs.params = params; // Preparation stage *file_errors = g_new0(GError*, nbfiles); int total_failed = gridftp_bulk_prepare(plugin_data, context, &pairs, *file_errors); // Transfer int transfer_ret = -1; if (!gfal2_is_canceled(context)) { bool udt = gfal2_get_opt_boolean_with_default(context, GRIDFTP_CONFIG_GROUP, GRIDFTP_CONFIG_TRANSFER_UDT, false); transfer_ret = gridftp_pipeline_transfer(plugin_data, context, udt, &pairs, op_error); // If UDT was tried and it failed, give it another shot if (transfer_ret < 0 && strstr((*op_error)->message, "udt driver not whitelisted") && !gfal2_is_canceled(context)) { udt = false; pairs.done = false; globus_object_free(pairs.error); pairs.error = NULL; g_error_free(*op_error); *op_error = NULL; gfal2_log(G_LOG_LEVEL_WARNING, "UDT transfer failed! Disabling and retrying..."); transfer_ret = gridftp_pipeline_transfer(plugin_data, context, udt, &pairs, op_error); } } if (transfer_ret < 0) total_failed = nbfiles; // Check destinations if (transfer_ret == 0) total_failed += gridftp_bulk_close(plugin_data, context, &pairs, *file_errors); // Done gfal2_log(G_LOG_LEVEL_DEBUG, "<- %s", __func__); gfal2_end_scope_cancel(context); return -total_failed; } gfal2-v2.23.0/src/plugins/gridftp/gridftp_dir_reader/000077500000000000000000000000001465240014500225065ustar00rootroot00000000000000gfal2-v2.23.0/src/plugins/gridftp/gridftp_dir_reader/GridFTPStreamBuffer.h000066400000000000000000000031001465240014500264160ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #pragma once #ifndef GRIDFTP_STREAMBUF_H #define GRIDFTP_STREAMBUF_H #include #include "../gridftpwrapper.h" class GridFTPStreamBuffer: public std::streambuf { protected: GridFTPStreamState* gstream; char buffer[4096]; GQuark quark; ssize_t fetch_more() { ssize_t rsize = gridftp_read_stream(quark, gstream, buffer, sizeof(buffer) - 1, false); this->setg(buffer, buffer, buffer + rsize); return rsize; } public: GridFTPStreamBuffer(GridFTPStreamState* gsiftp_stream, GQuark quark): gstream(gsiftp_stream), quark(quark) { fetch_more(); } virtual ~GridFTPStreamBuffer() { } int_type underflow() { ssize_t rsize = fetch_more(); if (rsize <= 0) return traits_type::eof(); return *buffer; } }; #endif // GRIDFTP_STREAMBUF_H gfal2-v2.23.0/src/plugins/gridftp/gridftp_dir_reader/GridFtpDirReader.h000066400000000000000000000046741465240014500260130ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #pragma once #ifndef GRIDFTP_DIR_READER_H #define GRIDFTP_DIR_READER_H #include #include #include "GridFTPStreamBuffer.h" #include "../gridftpmodule.h" #include "../gridftp_parsing.h" // Directory reader interface class GridFtpDirReader { protected: struct dirent dbuffer; GridFTPSessionHandler* handler; GridFTPRequestState* request_state; GridFTPStreamState *stream_state; GridFTPStreamBuffer *stream_buffer; public: GridFtpDirReader(): handler(NULL), request_state(NULL), stream_state(NULL), stream_buffer(NULL) { memset(&dbuffer, 0, sizeof(dbuffer)); }; virtual ~GridFtpDirReader() { delete this->stream_buffer; delete this->stream_state; delete this->request_state; delete this->handler; }; virtual struct dirent* readdir() = 0; virtual struct dirent* readdirpp(struct stat* st) = 0; }; // Implementation for simple list class GridFtpSimpleListReader: public GridFtpDirReader { public: GridFtpSimpleListReader(GridFTPModule* gsiftp, const char* path); ~GridFtpSimpleListReader(); struct dirent* readdir(); struct dirent* readdirpp(struct stat* st); }; // Implementation for MLSD class GridFtpMlsdReader: public GridFtpDirReader { public: GridFtpMlsdReader(GridFTPModule* gsiftp, const char* path); ~GridFtpMlsdReader(); struct dirent* readdir(); struct dirent* readdirpp(struct stat* st); }; // Implementation for STAT class GridFtpListReader: public GridFtpDirReader { public: GridFtpListReader(GridFTPModule* gsiftp, const char* path); ~GridFtpListReader(); struct dirent* readdir(); struct dirent* readdirpp(struct stat* st); }; #endif /* GRIDFTP_DIR_READER_H */ gfal2-v2.23.0/src/plugins/gridftp/gridftp_dir_reader/GridFtpListReader.cpp000066400000000000000000000064301465240014500265330ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2015 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "GridFtpDirReader.h" static const GQuark GridFtpListReaderQuark = g_quark_from_static_string("GridFtpListReader::readdir"); GridFtpListReader::GridFtpListReader(GridFTPModule* gsiftp, const char* path) { GridFTPFactory* factory = gsiftp->get_session_factory(); this->handler = new GridFTPSessionHandler(factory, path); this->request_state = new GridFTPRequestState(this->handler); this->stream_state = new GridFTPStreamState(this->handler); gfal2_log(G_LOG_LEVEL_DEBUG, " -> [GridftpListReader::GridftpListReader]"); globus_result_t res = globus_ftp_client_verbose_list( this->handler->get_ftp_client_handle(), path, this->handler->get_ftp_client_operationattr(), globus_ftp_client_done_callback, this->request_state); gfal_globus_check_result(GridFtpListReaderQuark, res); this->stream_buffer = new GridFTPStreamBuffer(this->stream_state, GridFtpListReaderQuark); gfal2_log(G_LOG_LEVEL_DEBUG, " <- [GridftpListReader::GridftpListReader]"); } GridFtpListReader::~GridFtpListReader() { this->request_state->wait(GridFtpListReaderQuark); } struct dirent* GridFtpListReader::readdir() { struct stat _; return readdirpp(&_); } static std::string& ltrim(std::string& str) { size_t i = 0; while (i < str.length() && isspace(str[i])) ++i; str = str.substr(i); return str; } static std::string& rtrim(std::string& str) { int i = str.length() - 1; while (i >= 0 && isspace(str[i])) --i; str = str.substr(0, i + 1); return str; } static std::string& trim(std::string& str) { return ltrim(rtrim(str)); } struct dirent* GridFtpListReader::readdirpp(struct stat* st) { std::string line; std::istream in(stream_buffer); if (!std::getline(in, line)) return NULL; if (trim(line).empty()) return NULL; char* unparsed = strdup(line.c_str()); if (parse_stat_line(unparsed, st, dbuffer.d_name, sizeof(dbuffer.d_name)) != GLOBUS_SUCCESS) { free(unparsed); throw Gfal::CoreException(GridFtpListReaderQuark, EINVAL, std::string("Error parsing GridFTP line: '").append(line).append("\'")); } free(unparsed); // Workaround for LCGUTIL-295 // Some endpoints return the absolute path when listing an empty directory if (dbuffer.d_name[0] == '\0') return NULL; if (S_ISDIR(st->st_mode)) dbuffer.d_type = DT_DIR; else if (S_ISLNK(st->st_mode)) dbuffer.d_type = DT_LNK; else dbuffer.d_type = DT_REG; return &dbuffer; } gfal2-v2.23.0/src/plugins/gridftp/gridftp_dir_reader/GridFtpMlsdReader.cpp000066400000000000000000000062551465240014500265240ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2015 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "GridFtpDirReader.h" static const GQuark GridFtpMlsdReaderQuark = g_quark_from_static_string("GridftpSimpleListReader::readdir"); GridFtpMlsdReader::GridFtpMlsdReader(GridFTPModule* gsiftp, const char* path) { GridFTPFactory* factory = gsiftp->get_session_factory(); this->handler = new GridFTPSessionHandler(factory, path); this->request_state = new GridFTPRequestState(this->handler); this->stream_state = new GridFTPStreamState(this->handler); gfal2_log(G_LOG_LEVEL_DEBUG, " -> [GridftpListReader::GridftpListReader]"); globus_result_t res = globus_ftp_client_machine_list( this->handler->get_ftp_client_handle(), path, this->handler->get_ftp_client_operationattr(), globus_ftp_client_done_callback, this->request_state); gfal_globus_check_result(GridFtpMlsdReaderQuark, res); this->stream_buffer = new GridFTPStreamBuffer(this->stream_state, GridFtpMlsdReaderQuark); gfal2_log(G_LOG_LEVEL_DEBUG, " <- [GridftpListReader::GridftpListReader]"); } GridFtpMlsdReader::~GridFtpMlsdReader() { this->request_state->wait(GridFtpMlsdReaderQuark); } struct dirent* GridFtpMlsdReader::readdir() { struct stat _; return readdirpp(&_); } static std::string& ltrim(std::string& str) { size_t i = 0; while (i < str.length() && isspace(str[i])) ++i; str = str.substr(i); return str; } static std::string& rtrim(std::string& str) { int i = str.length() - 1; while (i >= 0 && isspace(str[i])) --i; str = str.substr(0, i + 1); return str; } static std::string& trim(std::string& str) { return ltrim(rtrim(str)); } struct dirent* GridFtpMlsdReader::readdirpp(struct stat* st) { std::string line; std::istream in(stream_buffer); if (!std::getline(in, line)) return NULL; if (trim(line).empty()) return NULL; char* unparsed = strdup(line.c_str()); if (parse_mlst_line(unparsed, st, dbuffer.d_name, sizeof(dbuffer.d_name)) != GLOBUS_SUCCESS) { free(unparsed); throw Gfal::CoreException(GridFtpMlsdReaderQuark, EINVAL, std::string("Error parsing GridFTP line: '").append(line).append("\'")); } free(unparsed); if (dbuffer.d_name[0] == '\0') return NULL; if (S_ISDIR(st->st_mode)) dbuffer.d_type = DT_DIR; else if (S_ISLNK(st->st_mode)) dbuffer.d_type = DT_LNK; else dbuffer.d_type = DT_REG; return &dbuffer; } gfal2-v2.23.0/src/plugins/gridftp/gridftp_dir_reader/GridFtpSimpleListReader.cpp000066400000000000000000000063651465240014500277140ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2015 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include "GridFtpDirReader.h" static const GQuark GridFTPSimpleReaderQuark = g_quark_from_static_string("GridftpSimpleListReader::readdir"); GridFtpSimpleListReader::GridFtpSimpleListReader(GridFTPModule* gsiftp, const char* path) { GridFTPFactory* factory = gsiftp->get_session_factory(); this->handler = new GridFTPSessionHandler(factory, path); this->request_state = new GridFTPRequestState(this->handler); this->stream_state = new GridFTPStreamState(this->handler); gfal2_log(G_LOG_LEVEL_DEBUG, " -> [GridftpSimpleListReader::GridftpSimpleListReader]"); globus_result_t res = globus_ftp_client_list( // start req this->handler->get_ftp_client_handle(), path, this->handler->get_ftp_client_operationattr(), globus_ftp_client_done_callback, this->request_state); gfal_globus_check_result(GridFTPSimpleReaderQuark, res); stream_buffer = new GridFTPStreamBuffer(this->stream_state, GridFTPSimpleReaderQuark); gfal2_log(G_LOG_LEVEL_DEBUG, " <- [GridftpSimpleListReader::GridftpSimpleListReader]"); } GridFtpSimpleListReader::~GridFtpSimpleListReader() { this->request_state->wait(GridFTPSimpleReaderQuark); } // try to extract dir information static int gridftp_readdir_parser(const std::string& line, struct dirent* entry) { memset(entry->d_name, 0, sizeof(entry->d_name)); g_strlcpy(entry->d_name, line.c_str(), sizeof(entry->d_name)); char *p = stpncpy(entry->d_name, line.c_str(), sizeof(entry->d_name)); // clear new line madness do { *p = '\0'; --p; } while (p >= entry->d_name && isspace(*p)); return 0; } struct dirent* GridFtpSimpleListReader::readdir() { gfal2_log(G_LOG_LEVEL_DEBUG, " -> [GridftpSimpleListReader::readdir]"); std::string line; std::istream in(stream_buffer); if (!std::getline(in, line)) return NULL; if (gridftp_readdir_parser(line, &dbuffer) != 0) { throw Gfal::CoreException(GridFTPSimpleReaderQuark, EINVAL, std::string("Error parsing GridFTP line: ").append(line)); } if (dbuffer.d_name[0] == '\0') return NULL; gfal2_log(G_LOG_LEVEL_DEBUG, " list file %s ", dbuffer.d_name); gfal2_log(G_LOG_LEVEL_DEBUG, " [GridftpSimpleListReader::readdir] <- "); return &dbuffer; } struct dirent* GridFtpSimpleListReader::readdirpp(struct stat* st) { throw Gfal::CoreException(GridFTPSimpleReaderQuark, EBADF, "Can not call readdirpp after simple readdir"); } gfal2-v2.23.0/src/plugins/gridftp/gridftp_filecopy.cpp000066400000000000000000000614511465240014500227330ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2015 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include "gridftp_namespace.h" #include "gridftp_filecopy.h" #include "gridftp_plugin.h" #include #include #include #include #include #include #include #include #include #include #include #include static const GQuark GFAL_GRIDFTP_SCOPE_FILECOPY = g_quark_from_string("GridFTPFileCopyModule::FileCopy"); const GQuark GFAL_GRIDFTP_DOMAIN_GSIFTP = g_quark_from_string("GSIFTP"); /*IPv6 compatible lookup*/ std::string lookup_host(const char *host, bool ipv6_enabled, bool *got_ipv6) { struct addrinfo hints, *addresses = NULL; int errcode; char addrstr[100] = { 0 }; char ip4str[16] = { 0 }; char ip6str[46] = { 0 }; void *ptr = NULL; if (!host) { return std::string("cant.be.resolved"); } memset(&hints, 0, sizeof(hints)); hints.ai_family = PF_UNSPEC; hints.ai_socktype = SOCK_STREAM; hints.ai_flags |= AI_CANONNAME; errcode = getaddrinfo(host, NULL, &hints, &addresses); if (errcode != 0) { return std::string("cant.be.resolved"); } if (got_ipv6) { *got_ipv6 = false; } struct addrinfo *i = addresses; while (i) { inet_ntop(i->ai_family, i->ai_addr->sa_data, addrstr, sizeof(addrstr)); switch (i->ai_family) { case AF_INET: ptr = &((struct sockaddr_in *) i->ai_addr)->sin_addr; if (ptr) { inet_ntop(i->ai_family, ptr, ip4str, sizeof(ip4str)); } break; case AF_INET6: ptr = &((struct sockaddr_in6 *) i->ai_addr)->sin6_addr; if (ptr) { inet_ntop(i->ai_family, ptr, ip6str, sizeof(ip6str)); if (got_ipv6) { *got_ipv6 = true; } } break; } i = i->ai_next; } if (addresses) { freeaddrinfo(addresses); } if (ipv6_enabled && ip6str[0]) { return std::string("[").append(ip6str).append("]"); } else if (ip4str[0]) { return std::string(ip4str); } else { return std::string("cant.be.resolved"); } } std::string return_host_and_port(const std::string &uri, gboolean use_ipv6) { GError* error = NULL; gfal2_uri *parsed = gfal2_parse_uri(uri.c_str(), &error); if (error) { throw Gfal::CoreException(error); } std::ostringstream str; str << lookup_host(parsed->host, use_ipv6, NULL) << ":" << parsed->port; gfal2_free_uri(parsed); return str.str(); } // return 1 if deleted something int gridftp_filecopy_delete_existing(GridFTPModule* module, gfalt_params_t params, const char * url) { const bool replace = gfalt_get_replace_existing_file(params, NULL); bool exist = module->exists(url); if (exist) { if (replace) { gfal2_log(G_LOG_LEVEL_DEBUG, " File %s already exist, delete it for override ....", url); module->unlink(url); gfal2_log(G_LOG_LEVEL_DEBUG, " File %s deleted with success, proceed to copy ....", url); plugin_trigger_event(params, GFAL_GRIDFTP_DOMAIN_GSIFTP, GFAL_EVENT_DESTINATION, GFAL_EVENT_OVERWRITE_DESTINATION, "Deleted %s", url); return 1; } else { char err_buff[GFAL_ERRMSG_LEN]; snprintf(err_buff, GFAL_ERRMSG_LEN, " Destination already exist %s, Cancel", url); throw Gfal::TransferException(GFAL_GRIDFTP_SCOPE_FILECOPY, EEXIST, err_buff, GFALT_ERROR_DESTINATION, GFALT_ERROR_EXISTS); } } return 0; } // create the parent directory void gridftp_create_parent_copy(GridFTPModule* module, gfalt_params_t params, const char * gridftp_url) { const gboolean create_parent = gfalt_get_create_parent_dir(params, NULL); if (create_parent) { gfal2_log(G_LOG_LEVEL_DEBUG, " -> [gridftp_create_parent_copy]"); char current_uri[GFAL_URL_MAX_LEN]; const size_t s_uri = g_strlcpy(current_uri, gridftp_url, GFAL_URL_MAX_LEN); char* p_uri = current_uri + s_uri - 1; while (p_uri > current_uri && *p_uri == '/') { // remove trailing '/' *p_uri = '\0'; p_uri--; } while (p_uri > current_uri && *p_uri != '/') { // find the parent directory p_uri--; } if (p_uri > current_uri) { struct stat st; *p_uri = '\0'; try { module->stat(current_uri, &st); if (!S_ISDIR(st.st_mode)) { throw Gfal::TransferException(GFAL_GRIDFTP_SCOPE_FILECOPY, ENOTDIR, "The parent of the destination file exists, but it is not a directory", GFALT_ERROR_DESTINATION); } return; } catch (Gfal::CoreException& e) { if (e.code() != ENOENT) throw; } GError* tmp_err = NULL; (void) gfal2_mkdir_rec(module->get_session_factory()->get_gfal2_context(), current_uri, 0755, &tmp_err); Gfal::gerror_to_cpp(&tmp_err); } else { throw Gfal::TransferException(GFAL_GRIDFTP_SCOPE_FILECOPY, EINVAL, "Impossible to create directory " + std::string(current_uri) + " : invalid path", GFALT_ERROR_DESTINATION); } gfal2_log(G_LOG_LEVEL_DEBUG, " [gridftp_create_parent_copy] <-"); } } void gsiftp_3rd_callback(void* user_args, globus_gass_copy_handle_t* handle, globus_off_t total_bytes, float throughput, float avg_throughput); // // Performance callback object // contain the performance callback parameter // and the auto cancel logic on performance callback inactivity struct CallbackHandler { static void* func_timer(void* v) { CallbackHandler* args = (CallbackHandler*) v; while (time(NULL) < args->timeout_time) { if (pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0) { gfal2_log(G_LOG_LEVEL_DEBUG, "thread setcancelstate error, interrupt performance marker timer"); return NULL; } usleep(500000); if (pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, NULL) != 0) { gfal2_log(G_LOG_LEVEL_DEBUG, "thread setcancelstate error, interrupt performance marker timer"); return NULL; } } std::stringstream msg; msg << "Transfer canceled because the gsiftp performance marker timeout of " << args->timeout_value << " seconds has been exceeded, or all performance markers during that period indicated zero bytes transferred"; try { args->req->cancel(GFAL_GRIDFTP_SCOPE_FILECOPY, msg.str(), ETIMEDOUT); } catch (Gfal::CoreException& e) { gfal2_log(G_LOG_LEVEL_WARNING, "Exception while cancelling on performance marker timeout: %s", e.what()); } catch (...) { gfal2_log(G_LOG_LEVEL_WARNING, "Unknown exception while cancelling on performance marker timeout"); } pthread_exit(NULL); } CallbackHandler(gfal2_context_t context, gfalt_params_t params, GridFTPRequestState* req, const char* src, const char* dst, size_t src_size): params(params), req(req), src(src), dst(dst), start_time(0), timeout_value(0), timeout_time(0), timer_pthread(0), source_size(src_size) { timeout_value = gfal2_get_opt_integer_with_default(context, GRIDFTP_CONFIG_GROUP, GRIDFTP_CONFIG_TRANSFER_PERF_TIMEOUT, 180); start_time = time(NULL); if (timeout_value > 0) { timeout_time = start_time + timeout_value; pthread_create(&timer_pthread, NULL, CallbackHandler::func_timer, this); } globus_gass_copy_register_performance_cb( req->handler->get_gass_copy_handle(), gsiftp_3rd_callback, (gpointer) this); } virtual ~CallbackHandler() { if (timeout_value > 0) { pthread_cancel(timer_pthread); pthread_join(timer_pthread, NULL); } globus_gass_copy_register_performance_cb(req->handler->get_gass_copy_handle(), NULL, NULL); } gfalt_params_t params; GridFTPRequestState* req; const char* src; const char* dst; time_t start_time; int timeout_value; time_t timeout_time; pthread_t timer_pthread; globus_off_t source_size; }; void gsiftp_3rd_callback(void* user_args, globus_gass_copy_handle_t* handle, globus_off_t total_bytes, float throughput, float avg_throughput) { CallbackHandler* args = (CallbackHandler*)user_args; _gfalt_transfer_status status; status.bytes_transfered = total_bytes; status.average_baudrate = (size_t) avg_throughput; status.instant_baudrate = (size_t) throughput; status.transfer_time = (time(NULL) - args->start_time); plugin_trigger_monitor(args->params, &status, args->src, args->dst); if (args->timeout_time > 0) { // If throughput != 0, or the file has been already sent, reset timer callback // [LCGUTIL-440] Some endpoints calculate the checksum before closing, so we will // get throughput = 0 for a while, and the transfer should not fail if (throughput != 0.0 || (args->source_size > 0 && args->source_size <= total_bytes)) { //GridFTPRequestState* req = args->req; //Glib::RWLock::ReaderLock l(req->mux_req_state); if (args->timeout_value > 0) { gfal2_log(G_LOG_LEVEL_DEBUG, "Performance marker received, re-arm timer"); args->timeout_time = time(NULL) + args->timeout_value; } } // Otherwise, do not reset and notify else { gfal2_log(G_LOG_LEVEL_MESSAGE, "Performance marker received, but throughput is 0. Not resetting timeout!"); } } } void gridftp_set_credentials(gfal2_context_t context, GassCopyAttrHandler &gass_attr, const char *url) { gchar *ucert = NULL, *ukey = NULL, *user = NULL, *passwd = NULL; std::string baseurl = gfal_gridftp_get_credentials(context, url, &ucert, &ukey, &user, &passwd); gfal_globus_set_credentials(ucert, ukey, user, passwd, &gass_attr.cred_id, gass_attr.attr_gass.ftp_attr); gfal2_log(G_LOG_LEVEL_DEBUG, "Using %s:%s for %s", ucert, ukey, baseurl.c_str()); g_free(ucert); g_free(ukey); g_free(user); g_free(passwd); } static void gridftp_do_copy_inner(GridFTPModule* module, GridFTPFactory* factory, gfalt_params_t params, const char* src, const char* dst, GridFTPRequestState& req, time_t timeout) { GassCopyAttrHandler gass_attr_src(req.handler->get_ftp_client_operationattr()); GassCopyAttrHandler gass_attr_dst(req.handler->get_ftp_client_operationattr()); gfal2_log(G_LOG_LEVEL_DEBUG, "[GridFTPFileCopyModule::filecopy] start gridftp transfer %s -> %s", src, dst); // Required for the PASV plugin to be able to trigger events req.handler->session->params = params; // Override source/destination credentials with specifics // Source is properly set up already, as the session is retrieved using it gridftp_set_credentials(factory->get_gfal2_context(), gass_attr_dst, dst); try { globus_result_t res = globus_gass_copy_register_url_to_url( req.handler->get_gass_copy_handle(), (char*) src, &(gass_attr_src.attr_gass), (char*) dst, &(gass_attr_dst.attr_gass), globus_gass_client_done_callback, &req); gfal_globus_check_result(GFAL_GRIDFTP_SCOPE_FILECOPY, res); req.wait(GFAL_GRIDFTP_SCOPE_FILECOPY, timeout); req.handler->session->params = NULL; } catch (...) { req.handler->session->params = NULL; throw; } } static void gridftp_do_copy(GridFTPModule* module, GridFTPFactory* factory, gfalt_params_t params, const char* src, const char* dst, GridFTPRequestState& req, time_t timeout) { if (strncmp(src, "ftp:", 4) == 0 || strncmp(dst, "ftp:", 4) == 0) { gfal2_log(G_LOG_LEVEL_DEBUG, "[GridFTPFileCopyModule::filecopy] start gridftp transfer without performance markers"); gridftp_do_copy_inner(module, factory, params, src, dst, req, timeout); } else { CallbackHandler callback_handler(factory->get_gfal2_context(), params, &req, src, dst, 0); gfal2_log(G_LOG_LEVEL_DEBUG, "[GridFTPFileCopyModule::filecopy] start gridftp transfer with performance markers enabled (timeout %d)", callback_handler.timeout_value); gridftp_do_copy_inner(module, factory, params, src, dst, req, timeout); } } static int gridftp_filecopy_copy_file_internal(GridFTPModule* module, GridFTPFactory * factory, gfalt_params_t params, const char* src, const char* dst) { GError * tmp_err = NULL; gboolean is_strict_mode = gfalt_get_strict_copy_mode(params, NULL); const time_t timeout = gfalt_get_timeout(params, &tmp_err); Gfal::gerror_to_cpp(&tmp_err); unsigned int nbstream = gfalt_get_nbstreams(params, &tmp_err); Gfal::gerror_to_cpp(&tmp_err); const guint64 tcp_buffer_size = gfalt_get_tcp_buffer_size(params, &tmp_err); Gfal::gerror_to_cpp(&tmp_err); if (!is_strict_mode) { // If 1, the destination was deleted. So the parent directory is there! if (gridftp_filecopy_delete_existing(module, params, dst) == 0) gridftp_create_parent_copy(module, params, dst); } GridFTPSessionHandler handler(factory, src); GridFTPRequestState req(&handler, GRIDFTP_REQUEST_GASS); const unsigned int nb_streams_from_conf = gfal2_get_opt_integer_with_default( factory->get_gfal2_context(), GRIDFTP_CONFIG_GROUP, GRIDFTP_CONFIG_NB_STREAM, 0); //if the number of streams in the config file is 0 use the one passed by parameter if (nb_streams_from_conf !=0 ) { nbstream = nb_streams_from_conf; } //always set streams = 0 if source or dest are ftp, this disables the GLOBUS_FTP_CONTROL_MODE_EXTENDED_BLOCK if (strncmp(src, "ftp:", 4) == 0 || strncmp(dst, "ftp:", 4) == 0) { nbstream = 0; } handler.session->set_nb_streams(nbstream); gfal2_log(G_LOG_LEVEL_DEBUG, " [GridFTPFileCopyModule::filecopy] setup gsiftp number of streams to %d", nbstream); handler.session->set_tcp_buffer_size(tcp_buffer_size); gfal2_log(G_LOG_LEVEL_DEBUG, " [GridFTPFileCopyModule::filecopy] setup gsiftp buffer size to %d", tcp_buffer_size); gboolean enable_udt_transfers = gfal2_get_opt_boolean(factory->get_gfal2_context(), GRIDFTP_CONFIG_GROUP, GRIDFTP_CONFIG_TRANSFER_UDT, NULL); if (enable_udt_transfers) { gfal2_log(G_LOG_LEVEL_MESSAGE, "Trying UDT transfer"); plugin_trigger_event(params, GFAL_GRIDFTP_DOMAIN_GSIFTP, GFAL_EVENT_NONE, g_quark_from_static_string("UDT:ENABLE"), "Trying UDT"); handler.session->set_udt(true); } try { gridftp_do_copy(module, factory, params, src, dst, req, timeout); } catch (Gfal::CoreException& e) { // Try again if the failure was related to udt if (e.what_str().find("udt driver not whitelisted") != std::string::npos) { gfal2_log(G_LOG_LEVEL_WARNING, "UDT transfer failed! Disabling and retrying..."); plugin_trigger_event(params, GFAL_GRIDFTP_DOMAIN_GSIFTP, GFAL_EVENT_NONE, g_quark_from_static_string("UDT:DISABLE"), "UDT failed. Falling back to default mode: %s", e.what()); handler.session->set_udt(false); gridftp_do_copy(module, factory, params, src, dst, req, timeout); } // Else, rethrow else { handler.session->params = NULL; throw; } } return 0; } // clear dest if error occurs in transfer, does not clean if dest file is set as already exist before any transfer void GridFTPModule::autoCleanFileCopy(gfalt_params_t params, int code, const char* dst) { if (code != EEXIST) { gfal2_log(G_LOG_LEVEL_INFO, "\t\tError in transfer, clean destination file %s ", dst); try { this->unlink(dst); } catch (...) { gfal2_log(G_LOG_LEVEL_DEBUG, "\t\tFailure in cleaning ..."); } } } void GridFTPModule::filecopy(gfalt_params_t params, const char* src, const char* dst) { std::stringstream errstr; char checksum_type[GFAL_URL_MAX_LEN] = { 0 }; char checksum_user_defined[GFAL_URL_MAX_LEN]; char checksum_src[GFAL_URL_MAX_LEN] = { 0 }; char checksum_dst[GFAL_URL_MAX_LEN] = { 0 }; char resolved_src[GFAL_URL_MAX_LEN] = { 0 }; char resolved_dst[GFAL_URL_MAX_LEN] = { 0 }; gboolean use_ipv6 = gfal2_get_opt_boolean(_handle_factory->get_gfal2_context(), GRIDFTP_CONFIG_GROUP, GRIDFTP_CONFIG_IPV6, NULL); gboolean resolve_dns = gfal2_get_opt_boolean_with_default(_handle_factory->get_gfal2_context(), CORE_CONFIG_GROUP, RESOLVE_DNS, FALSE); // DMC-1348: DNS resolution mechanism if (resolve_dns) { char* resolved_src_ptr = resolve_dns_helper(src, "Resolving source"); char* resolved_dst_ptr = resolve_dns_helper(dst, "Resolving destination"); if (resolved_src_ptr) { g_strlcpy(resolved_src, resolved_src_ptr, sizeof(resolved_src)); src = resolved_src; free(resolved_src_ptr); } if (resolved_dst_ptr) { g_strlcpy(resolved_dst, resolved_dst_ptr, sizeof(resolved_dst)); dst = resolved_dst; free(resolved_dst_ptr); } } gfalt_checksum_mode_t checksum_mode = GFALT_CHECKSUM_NONE; if (!gfalt_get_strict_copy_mode(params, NULL)) { checksum_mode = gfalt_get_checksum(params, checksum_type, sizeof(checksum_type), checksum_user_defined, sizeof(checksum_user_defined), NULL); } gboolean skip_source_checksum = gfal2_get_opt_boolean( _handle_factory->get_gfal2_context(), GRIDFTP_CONFIG_GROUP, GRIDFTP_CONFIG_TRANSFER_SKIP_CHECKSUM, NULL); if (skip_source_checksum) { checksum_mode = gfalt_checksum_mode_t(checksum_mode & ~GFALT_CHECKSUM_SOURCE); } if (checksum_mode) { if (checksum_user_defined[0] == '\0' && checksum_type[0] == '\0') { GError *get_default_error = NULL; char *default_checksum_type; default_checksum_type = gfal2_get_opt_string( _handle_factory->get_gfal2_context(), GRIDFTP_CONFIG_GROUP, GRIDFTP_CONFIG_TRANSFER_CHECKSUM, &get_default_error); Gfal::gerror_to_cpp(&get_default_error); g_strlcpy(checksum_type, default_checksum_type, sizeof(checksum_type)); g_free(default_checksum_type); gfal2_log(G_LOG_LEVEL_DEBUG, "\t\tNo user defined checksum, fetch the default one from configuration"); } gfal2_log(G_LOG_LEVEL_DEBUG, "\t\tChecksum Algorithm for transfer verification %s", checksum_type); } // Source checksum if (checksum_mode & GFALT_CHECKSUM_SOURCE) { plugin_trigger_event(params, GFAL_GRIDFTP_DOMAIN_GSIFTP, GFAL_EVENT_SOURCE, GFAL_EVENT_CHECKSUM_ENTER, "%s", checksum_type); checksum(src, checksum_type, checksum_src, sizeof(checksum_src), 0, 0); plugin_trigger_event(params, GFAL_GRIDFTP_DOMAIN_GSIFTP, GFAL_EVENT_SOURCE, GFAL_EVENT_CHECKSUM_EXIT, "%s=%s", checksum_type, checksum_src); if (checksum_user_defined[0]) { if (gfal_compare_checksums(checksum_user_defined, checksum_src, GFAL_URL_MAX_LEN) != 0) { errstr << "USER_DEFINE and SRC checksums are different. " << checksum_user_defined << " != " << checksum_src; throw Gfal::TransferException(GFAL_GRIDFTP_SCOPE_FILECOPY, EIO, errstr.str(), GFALT_ERROR_TRANSFER, GFALT_ERROR_CHECKSUM_MISMATCH); } } } plugin_trigger_event(params, GFAL_GRIDFTP_DOMAIN_GSIFTP, GFAL_EVENT_NONE, GFAL_EVENT_TRANSFER_ENTER, "(%s) %s => (%s) %s", return_host_and_port(src, use_ipv6).c_str(), src, return_host_and_port(dst, use_ipv6).c_str(), dst); plugin_trigger_event(params, GFAL_GRIDFTP_DOMAIN_GSIFTP, GFAL_EVENT_NONE, GFAL_EVENT_TRANSFER_TYPE, "%s", GFAL_TRANSFER_TYPE_PUSH); try { gridftp_filecopy_copy_file_internal(this, _handle_factory, params, src, dst); } catch (Gfal::TransferException & e) { throw; } catch (const Gfal::CoreException & e) { autoCleanFileCopy(params, e.code(), dst); throw Gfal::TransferException(e.domain(), e.code(), e.what(), GFALT_ERROR_TRANSFER); } catch (std::exception & e) { autoCleanFileCopy(params, EIO, dst); throw Gfal::TransferException(GFAL_GRIDFTP_DOMAIN_GSIFTP, EIO, e.what(), GFALT_ERROR_TRANSFER, "UNEXPECTED"); } catch (...) { autoCleanFileCopy(params, EIO, dst); throw; } plugin_trigger_event(params, GFAL_GRIDFTP_DOMAIN_GSIFTP, GFAL_EVENT_NONE, GFAL_EVENT_TRANSFER_EXIT, "(%s) %s => (%s) %s", return_host_and_port(src, use_ipv6).c_str(), src, return_host_and_port(dst, use_ipv6).c_str(), dst); // Validate destination checksum if (checksum_mode & GFALT_CHECKSUM_TARGET) { plugin_trigger_event(params, GFAL_GRIDFTP_DOMAIN_GSIFTP, GFAL_EVENT_DESTINATION, GFAL_EVENT_CHECKSUM_ENTER, "%s", checksum_type); checksum(dst, checksum_type, checksum_dst, sizeof(checksum_dst), 0, 0); plugin_trigger_event(params, GFAL_GRIDFTP_DOMAIN_GSIFTP, GFAL_EVENT_DESTINATION, GFAL_EVENT_CHECKSUM_EXIT, "%s", checksum_type); if (checksum_mode & GFALT_CHECKSUM_SOURCE) { if (gfal_compare_checksums(checksum_src, checksum_dst, GFAL_URL_MAX_LEN) != 0) { errstr << "SRC and DST checksum are different. Source: " << checksum_src << " Destination: " << checksum_dst; throw Gfal::TransferException(GFAL_GRIDFTP_SCOPE_FILECOPY, EIO, errstr.str(), GFALT_ERROR_TRANSFER, GFALT_ERROR_CHECKSUM_MISMATCH); } } else { if (gfal_compare_checksums(checksum_user_defined, checksum_dst, GFAL_URL_MAX_LEN) != 0) { errstr << "USER_DEFINE and DST checksums are different. " << checksum_user_defined << " != " << checksum_dst; throw Gfal::TransferException(GFAL_GRIDFTP_SCOPE_FILECOPY, EIO, errstr.str(), GFALT_ERROR_TRANSFER, GFALT_ERROR_CHECKSUM_MISMATCH); } } } } /** * initialize a file copy from the given source to the given dest with the parameters params */ extern "C" int gridftp_plugin_filecopy(plugin_handle handle, gfal2_context_t context, gfalt_params_t params, const char* src, const char* dst, GError ** err) { g_return_val_err_if_fail(handle != NULL && src != NULL && dst != NULL, -1, err, "[plugin_filecopy][gridftp] Invalid parameters"); GError * tmp_err = NULL; int ret = -1; gfal2_log(G_LOG_LEVEL_DEBUG, " -> [gridftp_plugin_filecopy]"); CPP_GERROR_TRY (static_cast(handle))->filecopy(params, src, dst); ret = 0; CPP_GERROR_CATCH(&tmp_err); gfal2_log(G_LOG_LEVEL_DEBUG, " [gridftp_plugin_filecopy]<-"); G_RETURN_ERR(ret, tmp_err, err); } gfal2-v2.23.0/src/plugins/gridftp/gridftp_filecopy.h000066400000000000000000000041431465240014500223730ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #pragma once #ifndef GRIFTP_IFCE_FILECOPY_H #define GRIFTP_IFCE_FILECOPY_H #include "gridftpmodule.h" #include "gridftpwrapper.h" extern const GQuark GFAL_GRIDFTP_DOMAIN_GSIFTP; extern "C" int gridftp_plugin_filecopy(plugin_handle handle, gfal2_context_t context, gfalt_params_t params, const char* src, const char* dst, GError ** err); extern "C" int gridftp_bulk_copy(plugin_handle plugin_data, gfal2_context_t context, gfalt_params_t params, size_t nbfiles, const char* const * srcs, const char* const * dsts, const char* const * checksums, GError** op_error, GError*** file_errors); int gridftp_filecopy_delete_existing(GridFTPModule* module, gfalt_params_t params, const char * url); void gridftp_create_parent_copy(GridFTPModule* module, gfalt_params_t params, const char * gridftp_url); /** * Return the ip of the given host * @param host Hostname * @param ipv6_enabled If true, this function will look for an ipv6 preferably * @param got_ipv6 Will be set to true if ipv6_enabled is true, and the function finds an ipv6 for the host. * If NULL, it will be ignored. * @return An IP associated with host. */ std::string lookup_host(const char *host, bool ipv6_enabled, bool *got_ipv6); std::string return_host_and_port(const std::string &uri, gboolean use_ipv6); #endif /* GRIFTP_IFCE_FILECOPY_H */ gfal2-v2.23.0/src/plugins/gridftp/gridftp_io.cpp000066400000000000000000000405231465240014500215250ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2015 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include "gridftp_io.h" #include "gridftp_namespace.h" #include "gridftp_plugin.h" static const GQuark GFAL_GRIDFTP_SCOPE_OPEN = g_quark_from_static_string("GridFTPModule::open"); static const GQuark GFAL_GRIDFTP_SCOPE_READ = g_quark_from_static_string("GridFTPModule::read"); static const GQuark GFAL_GRIDFTP_SCOPE_INTERNAL_PREAD = g_quark_from_static_string("GridFTPModule::internal_pread"); static const GQuark GFAL_GRIDFTP_SCOPE_WRITE = g_quark_from_static_string("GridFTPModule::write"); static const GQuark GFAL_GRIDFTP_SCOPE_INTERNAL_PWRITE = g_quark_from_static_string("GridFTPModule::internal_pwrite"); static const GQuark GFAL_GRIDFTP_SCOPE_LSEEK = g_quark_from_static_string("GridFTPModule::lseek"); static const GQuark GFAL_GRIDFTP_SCOPE_CLOSE = g_quark_from_static_string("GridFTPModule::close"); const size_t readdir_len = 65000; struct GridFTPFileDesc { GridFTPSessionHandler* handler; GridFTPRequestState* request; GridFTPStreamState* stream; int open_flags; off_t current_offset; std::string url; globus_mutex_t mutex; GridFTPFileDesc(GridFTPSessionHandler* h, GridFTPRequestState* r, GridFTPStreamState * s, const std::string & _url, int flags) : handler(h), request(r), stream(s) { gfal2_log(G_LOG_LEVEL_DEBUG, "create descriptor for %s", _url.c_str()); this->open_flags = flags; current_offset = 0; url = _url; globus_mutex_init(&mutex, NULL); } virtual ~GridFTPFileDesc() { gfal2_log(G_LOG_LEVEL_DEBUG, "destroy descriptor for %s", url.c_str()); delete stream; delete request; delete handler; globus_mutex_destroy(&mutex); } bool is_not_seeked() { return (stream != NULL && current_offset == stream->offset); } bool is_eof() { return stream->eof; } void reset() { delete stream; stream = NULL; } }; inline bool is_read_only(int open_flags) { return ((open_flags & O_RDONLY) || ((open_flags & (O_WRONLY | O_RDWR)) == 0)); } inline bool is_write_only(int open_flags) { return (open_flags & ( O_WRONLY | O_CREAT)); } inline int gridftp_rw_commit_put(GQuark scope, GridFTPFileDesc* desc) { char buffer[2]; if (is_write_only(desc->open_flags) && desc->stream && !desc->stream->eof) { gfal2_log(G_LOG_LEVEL_DEBUG, "Commit change for the current stream PUT ... "); gridftp_write_stream(GFAL_GRIDFTP_SCOPE_WRITE, desc->stream, buffer, 0, true); gfal2_log(G_LOG_LEVEL_DEBUG, "Committed with success ... "); } return 0; } // internal pread, do a read query with offset on a different descriptor, do not change the position of the current one. ssize_t gridftp_rw_internal_pread(GridFTPFactory * factory, GridFTPFileDesc* desc, void* buffer, size_t s_buff, off_t offset) { // throw Gfal::CoreException gfal2_log(G_LOG_LEVEL_DEBUG, " -> [GridFTPModule::internal_pread]"); GridFTPSessionHandler handler(factory, desc->url); GridFTPRequestState request_state(&handler); GridFTPStreamState stream_state(&handler); globus_result_t res = globus_ftp_client_partial_get( handler.get_ftp_client_handle(), desc->url.c_str(), handler.get_ftp_client_operationattr(), NULL, offset, offset + s_buff, globus_ftp_client_done_callback, &request_state); gfal_globus_check_result(GFAL_GRIDFTP_SCOPE_INTERNAL_PREAD, res); ssize_t r_size = gridftp_read_stream(GFAL_GRIDFTP_SCOPE_INTERNAL_PREAD, &stream_state, buffer, s_buff, true); request_state.wait(GFAL_GRIDFTP_SCOPE_INTERNAL_PREAD); gfal2_log(G_LOG_LEVEL_DEBUG, "[GridFTPModule::internal_pread] <-"); return r_size; } // internal pwrite, do a write query with offset on a different descriptor, do not change the position of the current one. ssize_t gridftp_rw_internal_pwrite(GridFTPFactory * factory, GridFTPFileDesc* desc, const void* buffer, size_t s_buff, off_t offset) { // throw Gfal::CoreException gfal2_log(G_LOG_LEVEL_DEBUG, " -> [GridFTPModule::internal_pwrite]"); GridFTPSessionHandler handler(factory, desc->url); GridFTPRequestState request_state(&handler); GridFTPStreamState stream(&handler); globus_result_t res = globus_ftp_client_partial_put( stream.handler->get_ftp_client_handle(), desc->url.c_str(), stream.handler->get_ftp_client_operationattr(), NULL, offset, offset + s_buff, globus_ftp_client_done_callback, &request_state); gfal_globus_check_result(GFAL_GRIDFTP_SCOPE_INTERNAL_PWRITE, res); ssize_t r_size = gridftp_write_stream(GFAL_GRIDFTP_SCOPE_INTERNAL_PWRITE, &stream, buffer, s_buff, true); // write block request_state.wait(GFAL_GRIDFTP_SCOPE_INTERNAL_PWRITE); gfal2_log(G_LOG_LEVEL_DEBUG, "[GridFTPModule::internal_pwrite] <-"); return r_size; } // gridFTP open is restricted by the protocol : READ or Write but not both // gfal_file_handle GridFTPModule::open(const char* url, int flag, mode_t mode) { GridFTPSessionHandler *handler = new GridFTPSessionHandler(_handle_factory, url); GridFTPStreamState* stream = new GridFTPStreamState(handler); GridFTPRequestState* request = new GridFTPRequestState(handler); std::unique_ptr desc(new GridFTPFileDesc(handler, request, stream, url, flag)); gfal2_log(G_LOG_LEVEL_DEBUG, " -> [GridFTPModule::open] "); globus_result_t res; // check ENOENT condition for R_ONLY if (is_read_only(desc->open_flags)) { // Castor TURLs are really one-use-only, so with this dirty hack we allow // the SRM plugin to disable this check if the endpoint is Castor gboolean check_file_exists = gfal2_get_opt_boolean_with_default( get_session_factory()->get_gfal2_context(), "GRIDFTP PLUGIN", "STAT_ON_OPEN", TRUE); if (check_file_exists && !this->exists(url)) { char err_buff[2048]; snprintf(err_buff, 2048, " gridftp open error : %s on url %s", strerror(ENOENT), url); throw Gfal::CoreException(GFAL_GRIDFTP_SCOPE_OPEN, ENOENT, err_buff); } } if (is_read_only(desc->open_flags)) { gfal2_log(G_LOG_LEVEL_DEBUG, " -> initialize FTP GET global operations... "); res = globus_ftp_client_get( desc->stream->handler->get_ftp_client_handle(), url, desc->stream->handler->get_ftp_client_operationattr(), NULL, globus_ftp_client_done_callback, desc->request); gfal_globus_check_result(GFAL_GRIDFTP_SCOPE_OPEN, res); } else if (is_write_only(desc->open_flags)) { gfal2_log(G_LOG_LEVEL_DEBUG, " -> initialize FTP PUT global operations ... "); res = globus_ftp_client_put( desc->stream->handler->get_ftp_client_handle(), url, desc->stream->handler->get_ftp_client_operationattr(), NULL, globus_ftp_client_done_callback, desc->request); gfal_globus_check_result(GFAL_GRIDFTP_SCOPE_OPEN, res); } else { gfal2_log(G_LOG_LEVEL_DEBUG, " -> no operation initialization, switch to partial read/write mode..."); desc->reset(); } gfal2_log(G_LOG_LEVEL_DEBUG, " <- [GridFTPModule::open] "); return gfal_file_handle_new2(gridftp_plugin_name(), (gpointer) desc.release(), NULL, url); } ssize_t GridFTPModule::read(gfal_file_handle handle, void* buffer, size_t count) { GridFTPFileDesc* desc = static_cast(gfal_file_handle_get_fdesc(handle)); ssize_t ret; globus_mutex_lock(&desc->mutex); try { if (desc->is_not_seeked() && is_read_only(desc->open_flags) && desc->stream != NULL) { gfal2_log(G_LOG_LEVEL_DEBUG, " read in the GET main flow ... "); ret = gridftp_read_stream(GFAL_GRIDFTP_SCOPE_READ, desc->stream, buffer, count, false); } else { gfal2_log(G_LOG_LEVEL_DEBUG, " read with a pread ... "); ret = gridftp_rw_internal_pread(_handle_factory, desc, buffer, count, desc->current_offset); } } catch (...) { globus_mutex_unlock(&desc->mutex); throw; } desc->current_offset += ret; globus_mutex_unlock(&desc->mutex); return ret; } ssize_t GridFTPModule::write(gfal_file_handle handle, const void* buffer, size_t count) { GridFTPFileDesc* desc = static_cast(gfal_file_handle_get_fdesc(handle)); ssize_t ret; globus_mutex_lock(&desc->mutex); try { if (desc->is_not_seeked() && is_write_only(desc->open_flags) && desc->stream != NULL) { gfal2_log(G_LOG_LEVEL_DEBUG, " write in the PUT main flow ... "); ret = gridftp_write_stream(GFAL_GRIDFTP_SCOPE_WRITE, desc->stream, buffer, count, false); } else { gfal2_log(G_LOG_LEVEL_DEBUG, " write with a pwrite ... "); ret = gridftp_rw_internal_pwrite(_handle_factory, desc, buffer, count, desc->current_offset); } } catch (...) { globus_mutex_unlock(&desc->mutex); throw; } desc->current_offset += ret; globus_mutex_unlock(&desc->mutex); return ret; } ssize_t GridFTPModule::pread(gfal_file_handle handle, void* buffer, size_t count, off_t offset) { GridFTPFileDesc* desc = static_cast(gfal_file_handle_get_fdesc(handle)); return gridftp_rw_internal_pread(_handle_factory, desc, buffer, count, offset); } ssize_t GridFTPModule::pwrite(gfal_file_handle handle, const void* buffer, size_t count, off_t offset) { GridFTPFileDesc* desc = static_cast(gfal_file_handle_get_fdesc(handle)); return gridftp_rw_internal_pwrite(_handle_factory, desc, buffer, count, offset); } off_t GridFTPModule::lseek(gfal_file_handle handle, off_t offset, int whence) { GridFTPFileDesc* desc = static_cast(gfal_file_handle_get_fdesc(handle)); globus_mutex_lock(&desc->mutex); try { // Calculate new offset off_t new_offset; switch (whence) { case SEEK_SET: new_offset = offset; break; case SEEK_CUR: new_offset = desc->current_offset + offset; break; case SEEK_END: default: throw Gfal::CoreException(GFAL_GRIDFTP_SCOPE_LSEEK, EINVAL, "Invalid whence"); } // If the new offset is the same we have, we are good // This is done to avoid seeking when actually the reads/writes are done sequentially // This happens, for instance, with gfalFS, which will do parallel writes and reads, but // in order if (new_offset == desc->current_offset) { gfal2_log(G_LOG_LEVEL_DEBUG, "New and current offsets are the same (%lld), so do not seek", (long long)(new_offset)); globus_mutex_unlock(&desc->mutex); return desc->current_offset; } gfal2_log(G_LOG_LEVEL_DEBUG, "New offset set to %lld", (long long)(new_offset)); // If the new offset does not correspond with the current offset, // abort initial GET/PUT operation if running if (!desc->request->done) { gfal2_log(G_LOG_LEVEL_WARNING, "Abort GridFTP request done at open(...)"); globus_ftp_client_abort(desc->handler->get_ftp_client_handle()); try { desc->request->wait(GFAL_GRIDFTP_SCOPE_LSEEK); } catch (const Gfal::CoreException& e) { if (e.code() != ECANCELED) throw; } } desc->reset(); desc->current_offset = new_offset; } catch (...) { globus_mutex_unlock(&desc->mutex); throw; } globus_mutex_unlock(&desc->mutex); return desc->current_offset; } int GridFTPModule::close(gfal_file_handle handle) { GridFTPFileDesc* desc = static_cast(gfal_file_handle_get_fdesc(handle)); if (desc) { gridftp_rw_commit_put(GFAL_GRIDFTP_SCOPE_CLOSE, desc); if (is_write_only(desc->open_flags)) { desc->request->wait(GFAL_GRIDFTP_SCOPE_CLOSE); } else if (is_read_only(desc->open_flags)) { if (!desc->request->done) globus_ftp_client_abort(desc->handler->get_ftp_client_handle()); try { desc->request->wait(GFAL_GRIDFTP_SCOPE_CLOSE); } catch (const Gfal::CoreException& e) { if (e.code() != ECANCELED) throw; } } gfal_file_handle_delete(handle); delete desc; } return 0; } // open C bind extern "C" gfal_file_handle gfal_gridftp_openG(plugin_handle handle, const char* url, int flag, mode_t mode, GError** err) { g_return_val_err_if_fail(handle != NULL && url != NULL, NULL, err, "[gfal_gridftp_openG][gridftp] Invalid parameters"); GError * tmp_err = NULL; gfal_file_handle ret = NULL; gfal2_log(G_LOG_LEVEL_DEBUG, " -> [gfal_gridftp_openG]"); CPP_GERROR_TRY ret = ((static_cast(handle))->open(url, flag, mode)); CPP_GERROR_CATCH(&tmp_err); gfal2_log(G_LOG_LEVEL_DEBUG, " [gfal_gridftp_openG]<-"); G_RETURN_ERR(ret, tmp_err, err); } extern "C" ssize_t gfal_gridftp_readG(plugin_handle ch, gfal_file_handle fd, void* buff, size_t s_buff, GError** err) { g_return_val_err_if_fail(ch != NULL && fd != NULL, -1, err, "[gfal_gridftp_readG][gridftp] Invalid parameters"); GError * tmp_err = NULL; int ret = -1; gfal2_log(G_LOG_LEVEL_DEBUG, " -> [gfal_gridftp_readG]"); CPP_GERROR_TRY ret = (int) ((static_cast(ch))->read(fd, buff, s_buff)); CPP_GERROR_CATCH(&tmp_err); gfal2_log(G_LOG_LEVEL_DEBUG, " [gfal_gridftp_readG]<-"); G_RETURN_ERR(ret, tmp_err, err); } extern "C" ssize_t gfal_gridftp_writeG(plugin_handle ch, gfal_file_handle fd, const void* buff, size_t s_buff, GError** err) { g_return_val_err_if_fail(ch != NULL && fd != NULL, -1, err, "[gfal_gridftp_writeG][gridftp] Invalid parameters"); GError * tmp_err = NULL; int ret = -1; gfal2_log(G_LOG_LEVEL_DEBUG, " -> [gfal_gridftp_writeG]"); CPP_GERROR_TRY ret = (int) ((static_cast(ch))->write(fd, buff, s_buff)); CPP_GERROR_CATCH(&tmp_err); gfal2_log(G_LOG_LEVEL_DEBUG, " [gfal_gridftp_writeG] <-"); G_RETURN_ERR(ret, tmp_err, err); } extern "C" int gfal_gridftp_closeG(plugin_handle ch, gfal_file_handle fd, GError** err) { g_return_val_err_if_fail(ch != NULL && fd != NULL, -1, err, "[gfal_gridftp_closeG][gridftp] Invalid parameters"); GError * tmp_err = NULL; int ret = -1; gfal2_log(G_LOG_LEVEL_DEBUG, " -> [gfal_gridftp_closeG]"); CPP_GERROR_TRY ret = ((static_cast(ch))->close(fd)); CPP_GERROR_CATCH(&tmp_err); gfal2_log(G_LOG_LEVEL_DEBUG, " [gfal_gridftp_closeG]<-"); G_RETURN_ERR(ret, tmp_err, err); } extern "C" off_t gfal_gridftp_lseekG(plugin_handle ch, gfal_file_handle fd, off_t offset, int whence, GError** err) { g_return_val_err_if_fail(ch != NULL && fd != NULL, -1, err, "[gfal_gridftp_lseekG][gridftp] Invalid parameters"); GError * tmp_err = NULL; off_t ret = -1; gfal2_log(G_LOG_LEVEL_DEBUG, " -> [gfal_gridftp_lseekG]"); CPP_GERROR_TRY ret = ((static_cast(ch))->lseek(fd, offset, whence)); CPP_GERROR_CATCH(&tmp_err); gfal2_log(G_LOG_LEVEL_DEBUG, " [gfal_gridftp_lseekG]<-"); G_RETURN_ERR(ret, tmp_err, err); } gfal2-v2.23.0/src/plugins/gridftp/gridftp_io.h000066400000000000000000000027221465240014500211710ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #pragma once #ifndef GRIDFTP_RW_MODULE_H #define GRIDFTP_RW_MODULE_H #include "gridftpmodule.h" extern "C" gfal_file_handle gfal_gridftp_openG(plugin_handle ch, const char* url, int flag, mode_t mode, GError** err); extern "C" ssize_t gfal_gridftp_readG(plugin_handle ch, gfal_file_handle fd, void* buff, size_t s_buff, GError** err); extern "C" ssize_t gfal_gridftp_writeG(plugin_handle ch, gfal_file_handle fd, const void* buff, size_t s_buff, GError** err); extern "C" off_t gfal_gridftp_lseekG(plugin_handle ch, gfal_file_handle fd, off_t offset, int whence, GError** err); extern "C" int gfal_gridftp_closeG(plugin_handle ch, gfal_file_handle fd, GError** err); #endif /* GRIDFTP_RW_MODULE_H */ gfal2-v2.23.0/src/plugins/gridftp/gridftp_namespace.h000066400000000000000000000051511465240014500225150ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #pragma once #ifndef GRIDFTP_NAMESPACE_H #define GRIDFTP_NAMESPACE_H #include "gridftpwrapper.h" #include "gridftpmodule.h" #ifdef __cplusplus extern "C" { #endif int gfal_gridftp_statG(plugin_handle handle, const char* name, struct stat* buff, GError ** err); int gfal_gridftp_accessG(plugin_handle handle, const char* name, int mode, GError** err); int gfal_gridftp_chmodG(plugin_handle handle, const char* path, mode_t mode, GError** err); int gfal_gridftp_mkdirG(plugin_handle handle, const char* path, mode_t mode, gboolean pflag, GError** err); int gfal_gridftp_rmdirG(plugin_handle handle, const char* url, GError** err); void gridftp_unlink_internal(gfal2_context_t context, GridFTPSessionHandler* sess, const char * path); int gfal_gridftp_unlinkG(plugin_handle handle, const char* url, GError** err); int gfal_gridftp_renameG(plugin_handle plugin_data, const char * oldurl, const char * urlnew, GError** err); gfal_file_handle gfal_gridftp_opendirG(plugin_handle handle, const char* path, GError** err); struct dirent* gfal_gridftp_readdirG(plugin_handle handle, gfal_file_handle fh, GError** err); struct dirent* gfal_gridftp_readdirppG(plugin_handle handle, gfal_file_handle fh, struct stat*, GError** err); int gfal_gridftp_closedirG(plugin_handle handle, gfal_file_handle fh, GError** err); int gfal_gridftp_checksumG(plugin_handle handle, const char* url, const char* check_type, char * checksum_buffer, size_t buffer_length, off_t start_offset, size_t data_length, GError ** err); ssize_t gfal_gridftp_getxattrG(plugin_handle handle, const char* path, const char *name, void *buff, size_t s_buff, GError** err); ssize_t gfal_gridftp_listxattrG(plugin_handle plugin_data, const char* url, char* list, size_t s_list, GError** err); #ifdef __cplusplus } #endif #endif /* GRIDFTP_NAMESPACE_H */ gfal2-v2.23.0/src/plugins/gridftp/gridftp_ns_checksum.cpp000066400000000000000000000072521465240014500234220ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2015 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include "gridftp_namespace.h" #include "gridftp_plugin.h" #include #include static const GQuark GFAL_GRIDFTP_SCOPE_CHECKSUM = g_quark_from_static_string("GridFTPModule::checksum"); const char * gridftp_checksum_calc_timeout= "CHECKSUM_CALC_TIMEOUT"; extern "C" int gfal_gridftp_checksumG(plugin_handle handle, const char* url, const char* check_type, char * checksum_buffer, size_t buffer_length, off_t start_offset, size_t data_length, GError ** err) { g_return_val_err_if_fail(handle != NULL && url != NULL, -1, err, "[gfal_gridftp_checksumG][gridftp] Invalid parameeters"); GError * tmp_err = NULL; int ret = -1; gfal2_log(G_LOG_LEVEL_DEBUG, " -> [gfal_gridftp_checksumG]"); CPP_GERROR_TRY (static_cast(handle))->checksum(url, check_type, checksum_buffer, buffer_length, start_offset, data_length); ret = 0; CPP_GERROR_CATCH(&tmp_err); gfal2_log(G_LOG_LEVEL_DEBUG, " [gfal_gridftp_checksumG] <-"); G_RETURN_ERR(ret, tmp_err, err); } bool string_is_valid(const std::string &str) { for(size_t i=0; i < str.length(); i++){ if(!isalnum(str[i])){ return 0; } } return 1; } void GridFTPModule::checksum(const char* url, const char* check_type, char * checksum_buffer, size_t buffer_length, off_t start_offset, size_t data_length) { gfal2_log(G_LOG_LEVEL_DEBUG, " -> [GridFTPModule::checksum] "); gfal2_log(G_LOG_LEVEL_DEBUG, " Checksum calculation %s for url %s", check_type, url); GridFTPSessionHandler handler(_handle_factory, url); GridFTPRequestState req(&handler, GRIDFTP_REQUEST_FTP); if (buffer_length < 16) { throw Gfal::CoreException(GFAL_GRIDFTP_SCOPE_CHECKSUM, ENOBUFS, "buffer length for checksum calculation is not enough"); } globus_result_t res = globus_ftp_client_cksm(req.handler->get_ftp_client_handle(), url, req.handler->get_ftp_client_operationattr(), checksum_buffer, start_offset, ((data_length) ? (data_length) : (-1)), check_type, globus_ftp_client_done_callback, &req); gfal_globus_check_result(GFAL_GRIDFTP_SCOPE_CHECKSUM, res); // wait for answer with a timeout const time_t global_timeout = gfal2_get_opt_integer_with_default( _handle_factory->get_gfal2_context(), CORE_CONFIG_GROUP, CORE_CONFIG_CHECKSUM_TIMEOUT, 1800 ); const time_t timeout = gfal2_get_opt_integer_with_default( _handle_factory->get_gfal2_context(), GRIDFTP_CONFIG_GROUP, gridftp_checksum_calc_timeout, global_timeout); req.wait(GFAL_GRIDFTP_SCOPE_CHECKSUM, timeout); if (string_is_valid(checksum_buffer) != 1) { std::string s(16, '0'); strncpy(checksum_buffer, s.c_str(), buffer_length); } gfal2_log(G_LOG_LEVEL_DEBUG, " <- [GridFTPModule::checksum] "); } gfal2-v2.23.0/src/plugins/gridftp/gridftp_ns_chmod.cpp000066400000000000000000000044061465240014500227100ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2015 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "gridftp_namespace.h" #include static const GQuark GFAL_GRIDFTP_SCOPE_CHMOD = g_quark_from_static_string("GridFTPModule::chmod"); void GridFTPModule::chmod(const char* path, mode_t mode) { if (path == NULL) { throw Gfal::CoreException(GFAL_GRIDFTP_SCOPE_CHMOD, EINVAL, "Invalid arguments path or mode"); } gfal2_log(G_LOG_LEVEL_DEBUG, " -> [GridFTPModule::chmod] "); GridFTPSessionHandler handler(_handle_factory, path); GridFTPRequestState req(&handler); globus_result_t res = globus_ftp_client_chmod(req.handler->get_ftp_client_handle(), path, mode, req.handler->get_ftp_client_operationattr(), globus_ftp_client_done_callback, &req); gfal_globus_check_result(GFAL_GRIDFTP_SCOPE_CHMOD, res); // wait for answer req.wait(GFAL_GRIDFTP_SCOPE_CHMOD); gfal2_log(G_LOG_LEVEL_DEBUG, " <- [GridFTPModule::chmod] "); } extern "C" int gfal_gridftp_chmodG(plugin_handle handle, const char* path, mode_t mode, GError** err) { g_return_val_err_if_fail(handle != NULL && path != NULL, -1, err, "[gfal_gridftp_chmodG][gridftp] Invalid parameters"); GError * tmp_err = NULL; int ret = -1; gfal2_log(G_LOG_LEVEL_DEBUG, " -> [gfal_gridftp_chmod]"); CPP_GERROR_TRY (static_cast(handle))->chmod(path, mode); ret = 0; CPP_GERROR_CATCH(&tmp_err); gfal2_log(G_LOG_LEVEL_DEBUG, " [gfal_gridftp_chmod]<-"); G_RETURN_ERR(ret, tmp_err, err); } gfal2-v2.23.0/src/plugins/gridftp/gridftp_ns_exist.cpp000066400000000000000000000021531465240014500227470ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2015 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "gridftp_namespace.h" #include bool GridFTPModule::exists(const char* path) { try { struct stat fstat; memset(&fstat, 0, sizeof(fstat)); internal_globus_gass_stat(path, &fstat); } catch (Gfal::CoreException& e) { if (e.code() != ENOENT) throw; return false; } return true; } gfal2-v2.23.0/src/plugins/gridftp/gridftp_ns_mkdir.cpp000066400000000000000000000051761465240014500227310ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2015 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "gridftp_namespace.h" #include static const GQuark GFAL_GRIDFTP_SCOPE_MKDIR = g_quark_from_static_string("GridFTPModule::mkdir"); static const GQuark GFAL_GRIDFTP_SCOPE_REQ_STATE = g_quark_from_static_string("GridFTPModule::RequestState"); void GridFTPModule::mkdir(const char* path, mode_t mode) { if (path == NULL) { throw Gfal::CoreException(GFAL_GRIDFTP_SCOPE_MKDIR, EINVAL, "Invalid arguments path or mode"); } gfal2_log(G_LOG_LEVEL_DEBUG, " -> [GridFTPModule::mkdir] "); GridFTPSessionHandler handler(_handle_factory, path); GridFTPRequestState req(&handler); globus_result_t res = globus_ftp_client_mkdir(req.handler->get_ftp_client_handle(), path, req.handler->get_ftp_client_operationattr(), globus_ftp_client_done_callback, &req); gfal_globus_check_result(GFAL_GRIDFTP_SCOPE_MKDIR, res); // wait for answer req.wait(GFAL_GRIDFTP_SCOPE_MKDIR); gfal2_log(G_LOG_LEVEL_DEBUG, " <- [GridFTPModule::mkdir] "); } extern "C" int gfal_gridftp_mkdirG(plugin_handle handle, const char* path, mode_t mode, gboolean pflag, GError** err) { g_return_val_err_if_fail(handle != NULL && path != NULL, -1, err, "[gfal_gridftp_mkdirG][gridftp] Invalid parameters"); GError * tmp_err = NULL; int ret = -1; gfal2_log(G_LOG_LEVEL_DEBUG, " -> [gfal_gridftp_mkdirG]"); CPP_GERROR_TRY (static_cast(handle))->mkdir(path, mode); ret = 0; CPP_GERROR_CATCH(&tmp_err); gfal2_log(G_LOG_LEVEL_DEBUG, " [gfal_gridftp_mkdirG]<-"); // Distinguish between actual ENOENT or certificate loading error if (pflag && tmp_err && tmp_err->code == ENOENT && tmp_err->domain == GFAL_GRIDFTP_SCOPE_REQ_STATE) { // ENOENT error will be masked by parent flag tmp_err->code = EFAULT; } G_RETURN_ERR(ret, tmp_err, err); } gfal2-v2.23.0/src/plugins/gridftp/gridftp_ns_opendir.cpp000066400000000000000000000123371465240014500232600ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2015 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include "gridftp_dir_reader/GridFtpDirReader.h" #include "gridftp_namespace.h" #include "gridftp_plugin.h" static const GQuark GFAL_GRIDFTP_SCOPE_OPENDIR = g_quark_from_static_string("gfal_gridftp_opendirG"); extern "C" gfal_file_handle gfal_gridftp_opendirG(plugin_handle handle, const char* path, GError** err) { g_return_val_err_if_fail(handle != NULL && path != NULL, NULL, err, "[gfal_gridftp_opendirG][gridftp] Invalid parameters"); gfal2_log(G_LOG_LEVEL_DEBUG, " -> [gfal_gridftp_opendirG]"); // Since we are deferring the opening, at least check it is there and it is a directory, // and we have permissions struct stat st; if (gfal_gridftp_statG(handle, path, &st, err) != 0) { return NULL; } else if (!S_ISDIR(st.st_mode)) { gfal2_set_error(err, GFAL_GRIDFTP_SCOPE_OPENDIR, EISDIR, __func__, "%s is not a directory", path); return NULL; } else if ((st.st_mode & ( S_IRUSR | S_IRGRP | S_IROTH)) == 0) { gfal2_set_error(err, GFAL_GRIDFTP_SCOPE_OPENDIR, EACCES, __func__, "Can not read %s", path); return NULL; } // Do nothing, and defer until the first readdir or readdirpp // is called return gfal_file_handle_new2(gridftp_plugin_name(), NULL, NULL, path); } extern "C" struct dirent* gfal_gridftp_readdirG(plugin_handle handle, gfal_file_handle fh, GError** err) { g_return_val_err_if_fail(handle != NULL && fh != NULL, NULL, err, "[gfal_gridftp_readdirG][gridftp] Invalid parameters"); GError * tmp_err = NULL; struct dirent* ret = NULL; gfal2_log(G_LOG_LEVEL_DEBUG, " -> [gfal_gridftp_readdirG]"); CPP_GERROR_TRY GridFtpDirReader* reader = static_cast(gfal_file_handle_get_fdesc(fh)); // Not open yet, so instantiate the simple reader if (reader == NULL) { GridFTPModule* gsiftp = static_cast(handle); reader = new GridFtpSimpleListReader(gsiftp, gfal_file_handle_get_path(fh)); gfal_file_handle_set_fdesc(fh, reader); } ret = reader->readdir(); CPP_GERROR_CATCH(&tmp_err); gfal2_log(G_LOG_LEVEL_DEBUG, " [gfal_gridftp_readdirG] <-"); G_RETURN_ERR(ret, tmp_err, err); } static GridFtpDirReader* gfal_gridftp_readdirpp_instantiate(GridFTPModule* gsiftp, const char* path) { GridFTPSessionHandler handler(gsiftp->get_session_factory(), path); globus_ftp_client_tristate_t supported; globus_ftp_client_is_feature_supported(handler.get_ftp_features(), &supported, GLOBUS_FTP_CLIENT_FEATURE_MLST); if (supported != GLOBUS_FTP_CLIENT_FALSE) { return new GridFtpMlsdReader(gsiftp, path); } else { return new GridFtpListReader(gsiftp, path); } } extern "C" struct dirent* gfal_gridftp_readdirppG(plugin_handle handle, gfal_file_handle fh, struct stat* st, GError** err) { g_return_val_err_if_fail(handle != NULL && fh != NULL, NULL, err, "[gfal_gridftp_readdirG][gridftp] Invalid parameters"); GError * tmp_err = NULL; struct dirent* ret = NULL; gfal2_log(G_LOG_LEVEL_DEBUG, " -> [gfal_gridftp_readdirG]"); CPP_GERROR_TRY GridFtpDirReader* reader = static_cast(gfal_file_handle_get_fdesc(fh)); // Not open yet, so instantiate the reader if (reader == NULL) { GridFTPModule* gsiftp = static_cast(handle); reader = gfal_gridftp_readdirpp_instantiate(gsiftp, gfal_file_handle_get_path(fh)); gfal_file_handle_set_fdesc(fh, reader); } ret = reader->readdirpp(st); CPP_GERROR_CATCH(&tmp_err); gfal2_log(G_LOG_LEVEL_DEBUG, " [gfal_gridftp_readdirG] <-"); G_RETURN_ERR(ret, tmp_err, err); } extern "C" int gfal_gridftp_closedirG(plugin_handle handle, gfal_file_handle fh, GError** err) { g_return_val_err_if_fail(handle != NULL, -1, err, "[gfal_gridftp_readdirG][gridftp] Invalid parameters"); GError * tmp_err = NULL; int ret = -1; gfal2_log(G_LOG_LEVEL_DEBUG, " -> [gfal_gridftp_closedirG]"); CPP_GERROR_TRY GridFtpDirReader* reader = static_cast(gfal_file_handle_get_fdesc(fh)); delete reader; gfal_file_handle_delete(fh); ret = 0; CPP_GERROR_CATCH(&tmp_err); gfal2_log(G_LOG_LEVEL_DEBUG, " [gfal_gridftp_closedirG] <-"); G_RETURN_ERR(ret, tmp_err, err); } gfal2-v2.23.0/src/plugins/gridftp/gridftp_ns_rename.cpp000066400000000000000000000045151465240014500230660ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2015 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "gridftp_namespace.h" #include "gridftpwrapper.h" #include static const GQuark GFAL_GRIDFTP_SCOPE_RENAME = g_quark_from_static_string("GridFTPModule::rmdir"); void GridFTPModule::rename(const char* src, const char* dst) { if (src == NULL || dst == NULL) { throw Gfal::CoreException(GFAL_GRIDFTP_SCOPE_RENAME, EINVAL, "Invalid source and/or destination"); } gfal2_log(G_LOG_LEVEL_DEBUG, " -> [GridFTPModule::rename] "); GridFTPSessionHandler handler(_handle_factory, src); GridFTPRequestState req(&handler); globus_result_t res = globus_ftp_client_move(req.handler->get_ftp_client_handle(), src, dst, req.handler->get_ftp_client_operationattr(), globus_ftp_client_done_callback, &req); gfal_globus_check_result(GFAL_GRIDFTP_SCOPE_RENAME, res); // wait for answer req.wait(GFAL_GRIDFTP_SCOPE_RENAME); gfal2_log(G_LOG_LEVEL_DEBUG, " <- [GridFTPModule::rename] "); } int gfal_gridftp_renameG(plugin_handle handle, const char * oldurl, const char * newurl, GError** err) { g_return_val_err_if_fail(handle != NULL && oldurl != NULL && newurl != NULL, -1, err, "[gfal_gridftp_rename][gridftp] Invalid parameters"); GError * tmp_err = NULL; int ret = -1; gfal2_log(G_LOG_LEVEL_DEBUG, " -> [gfal_gridftp_rename]"); CPP_GERROR_TRY (static_cast(handle))->rename(oldurl, newurl); ret = 0; CPP_GERROR_CATCH(&tmp_err); gfal2_log(G_LOG_LEVEL_DEBUG, " [gfal_gridftp_rename]<-"); G_RETURN_ERR(ret, tmp_err, err); } gfal2-v2.23.0/src/plugins/gridftp/gridftp_ns_rmdir.cpp000066400000000000000000000047431465240014500227370ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2015 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "gridftp_namespace.h" #include static const GQuark GFAL_GRIDFTP_SCOPE_RMDIR = g_quark_from_static_string("GridFTPModule::rmdir"); void GridFTPModule::rmdir(const char* path) { if (path == NULL) { throw Gfal::CoreException(GFAL_GRIDFTP_SCOPE_RMDIR, EINVAL, "Invalid arguments path"); } gfal2_log(G_LOG_LEVEL_DEBUG, " -> [GridFTPModule::rmdir] "); try { GridFTPSessionHandler handler(_handle_factory, path); GridFTPRequestState req(&handler); globus_result_t res = globus_ftp_client_rmdir( req.handler->get_ftp_client_handle(), path, req.handler->get_ftp_client_operationattr(), globus_ftp_client_done_callback, &req); gfal_globus_check_result(GFAL_GRIDFTP_SCOPE_RMDIR, res); // wait for answer req.wait(GFAL_GRIDFTP_SCOPE_RMDIR); } catch (Gfal::CoreException & e) { if (e.code() == EEXIST) // false ENOTEMPTY errno, do conversion throw Gfal::CoreException(e.domain(), ENOTEMPTY, e.what()); throw e; } gfal2_log(G_LOG_LEVEL_DEBUG, " <- [GridFTPModule::rmdir] "); } extern "C" int gfal_gridftp_rmdirG(plugin_handle handle, const char* url, GError** err) { g_return_val_err_if_fail(handle != NULL && url != NULL, -1, err, "[gfal_gridftp_rmdir][gridftp] Invalid parameters"); GError * tmp_err = NULL; int ret = -1; gfal2_log(G_LOG_LEVEL_DEBUG, " -> [gfal_gridftp_rmdir]"); CPP_GERROR_TRY (static_cast(handle))->rmdir(url); ret = 0; CPP_GERROR_CATCH(&tmp_err); gfal2_log(G_LOG_LEVEL_DEBUG, " [gfal_gridftp_rmdir]<-"); G_RETURN_ERR(ret, tmp_err, err); } gfal2-v2.23.0/src/plugins/gridftp/gridftp_ns_stat.cpp000066400000000000000000000145031465240014500225700ustar00rootroot00000000000000/* * Copyright @ Members of the EMI Collaboration, 2010. * See www.eu-emi.eu for details on the copyright holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include "gridftp_namespace.h" #include "gridftp_parsing.h" static const GQuark GFAL_GRIDFTP_SCOPE_STAT = g_quark_from_static_string("Gridftp_stat_module::stat"); static const GQuark GFAL_GRIDFTP_SCOPE_ACCESS = g_quark_from_static_string("Gridftp_stat_module::access"); void GridFTPModule::stat(const char* path, struct stat * st) { if (path == NULL || st == NULL) { throw Gfal::CoreException(GFAL_GRIDFTP_SCOPE_STAT, EINVAL, "Invalid arguments path or stat "); } gfal2_log(G_LOG_LEVEL_DEBUG, " -> [GridFTPModule::stat] "); internal_globus_gass_stat(path, st); gfal2_log(G_LOG_LEVEL_DEBUG, " <- [GridFTPModule::stat] "); } void GridFTPModule::access(const char* path, int mode) { if (path == NULL) { throw Gfal::CoreException(GFAL_GRIDFTP_SCOPE_STAT, EINVAL, "Invalid arguments path or stat "); } gfal2_log(G_LOG_LEVEL_DEBUG, " -> [Gridftp_stat_module::access] "); struct stat st; internal_globus_gass_stat(path, &st); if (st.st_mode == (mode_t)-1) { // mode not managed by server gfal2_log(G_LOG_LEVEL_MESSAGE, "Access request is not managed by this server %s , return access authorized by default", path); return; } const mode_t file_mode = (mode_t) st.st_mode; if (((file_mode & ( S_IRUSR | S_IRGRP | S_IROTH)) == FALSE) && (mode & R_OK)) { throw Gfal::CoreException(GFAL_GRIDFTP_SCOPE_ACCESS, EACCES, "No read access"); } if (((file_mode & ( S_IWUSR | S_IWGRP | S_IWOTH)) == FALSE) && (mode & W_OK)) { throw Gfal::CoreException(GFAL_GRIDFTP_SCOPE_ACCESS, EACCES, "No write access"); } if (((file_mode & ( S_IXUSR | S_IXGRP | S_IXOTH)) == FALSE) && (mode & X_OK)) { throw Gfal::CoreException(GFAL_GRIDFTP_SCOPE_ACCESS, EACCES, "No execute access"); } gfal2_log(G_LOG_LEVEL_DEBUG, " <- [Gridftp_stat_module::access] "); } static void gridftp_stat_mlst(GridFTPSessionHandler *handler, const char* path, struct stat* fstat) { gfal2_log(G_LOG_LEVEL_DEBUG, "Stat via MLST"); globus_byte_t *buffer = NULL; globus_size_t buflen = 0; GridFTPRequestState req(handler); globus_result_t res = globus_ftp_client_mlst(handler->get_ftp_client_handle(), path, handler->get_ftp_client_operationattr(), &buffer, &buflen, globus_ftp_client_done_callback, &req); gfal_globus_check_result(GFAL_GRIDFTP_SCOPE_STAT, res); req.wait(GFAL_GRIDFTP_SCOPE_STAT); gfal2_log(G_LOG_LEVEL_DEBUG, " <- [%s]] Got '%s'", __func__, buffer); parse_mlst_line((char *) buffer, fstat, NULL, 0); globus_free(buffer); } static void gridftp_stat_stat(GridFTPSessionHandler *handler, const char* path, struct stat* fstat) { gfal2_log(G_LOG_LEVEL_DEBUG, "Stat via STAT"); globus_byte_t *buffer = NULL; globus_size_t buflen = 0; GridFTPRequestState req(handler); globus_result_t res = globus_ftp_client_stat(handler->get_ftp_client_handle(), path, handler->get_ftp_client_operationattr(), &buffer, &buflen, globus_ftp_client_done_callback, &req); gfal_globus_check_result(GFAL_GRIDFTP_SCOPE_STAT, res); req.wait(GFAL_GRIDFTP_SCOPE_STAT); gfal2_log(G_LOG_LEVEL_DEBUG, " <- [%s]] Got '%s'", __func__, buffer); char* p = (char*)buffer; if (strncmp(p, "211", 3) == 0) { p += 4; } else if (strncmp(p, "213", 3) == 0) { p = strchr(p, '\n'); if (p) ++p; } parse_stat_line(p, fstat, NULL, 0); globus_free(buffer); } void GridFTPModule::internal_globus_gass_stat(const char* path, struct stat* fstat) { gfal2_log(G_LOG_LEVEL_DEBUG, " -> [Gridftp_stat_module::globus_gass_stat] "); GridFTPSessionHandler handler(get_session_factory(), path); globus_ftp_client_tristate_t supported; globus_ftp_client_is_feature_supported(handler.get_ftp_features(), &supported, GLOBUS_FTP_CLIENT_FEATURE_MLST); if (supported != GLOBUS_FTP_CLIENT_FALSE) { gridftp_stat_mlst(&handler, path, fstat); } else { gridftp_stat_stat(&handler, path, fstat); } gfal2_log(G_LOG_LEVEL_DEBUG, " <- [Gridftp_stat_module::internal_globus_gass_stat] "); } extern "C" int gfal_gridftp_statG(plugin_handle handle, const char* name, struct stat* buff, GError ** err) { g_return_val_err_if_fail(handle != NULL && name != NULL && buff != NULL, -1, err, "[gfal_gridftp_statG][gridftp] Invalid parameters"); GError * tmp_err = NULL; int ret = -1; gfal2_log(G_LOG_LEVEL_DEBUG, " -> [gfal_gridftp_statG]"); CPP_GERROR_TRY (static_cast(handle))->stat(name, buff); ret = 0; CPP_GERROR_CATCH(&tmp_err); gfal2_log(G_LOG_LEVEL_DEBUG, " [gfal_gridftp_statG]<-"); G_RETURN_ERR(ret, tmp_err, err); } extern "C" int gfal_gridftp_accessG(plugin_handle handle, const char* name, int mode, GError** err) { g_return_val_err_if_fail(handle != NULL && name != NULL, -1, err, "[gfal_gridftp_statG][gridftp] Invalid parameters"); GError * tmp_err = NULL; int ret = -1; gfal2_log(G_LOG_LEVEL_DEBUG, " -> [gfal_gridftp_accessG]"); CPP_GERROR_TRY (static_cast(handle))->access(name, mode); ret = 0; CPP_GERROR_CATCH(&tmp_err); gfal2_log(G_LOG_LEVEL_DEBUG, " [gfal_gridftp_accessG]<-"); G_RETURN_ERR(ret, tmp_err, err); } gfal2-v2.23.0/src/plugins/gridftp/gridftp_ns_unlink.cpp000066400000000000000000000046751465240014500231260ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2015 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "gridftpmodule.h" #include "gridftp_namespace.h" #include static const GQuark GFAL_GRIDFTP_SCOPE_UNLINK = g_quark_from_static_string("GridFTPModule::unlink"); void gridftp_unlink_internal(gfal2_context_t context, GridFTPSessionHandler* sess, const char * path) { gfal2_log(G_LOG_LEVEL_DEBUG, " -> [GridFTPModule::unlink] "); GridFTPRequestState req(sess); // get connection session globus_result_t res = globus_ftp_client_delete(req.handler->get_ftp_client_handle(), path, req.handler->get_ftp_client_operationattr(), globus_ftp_client_done_callback, &req); gfal_globus_check_result(GFAL_GRIDFTP_SCOPE_UNLINK, res); req.wait(GFAL_GRIDFTP_SCOPE_UNLINK); gfal2_log(G_LOG_LEVEL_DEBUG, " <- [GridFTPModule::unlink] "); } void GridFTPModule::unlink(const char* path) { if (path == NULL) { throw Gfal::CoreException(GFAL_GRIDFTP_SCOPE_UNLINK, EINVAL, "Invalid arguments path"); } GridFTPSessionHandler handler(_handle_factory, path); gridftp_unlink_internal(_handle_factory->get_gfal2_context(), &handler, path); } extern "C" int gfal_gridftp_unlinkG(plugin_handle handle, const char* url, GError** err) { g_return_val_err_if_fail(handle != NULL && url != NULL, -1, err, "[gfal_gridftp_unlinkG][gridftp] Invalid parameters"); GError * tmp_err = NULL; int ret = -1; gfal2_log(G_LOG_LEVEL_DEBUG, " -> [gfal_gridftp_unlinkG]"); CPP_GERROR_TRY (static_cast(handle))->unlink(url); ret = 0; CPP_GERROR_CATCH(&tmp_err); gfal2_log(G_LOG_LEVEL_DEBUG, " [gfal_gridftp_unlinkG] <-"); G_RETURN_ERR(ret, tmp_err, err); } gfal2-v2.23.0/src/plugins/gridftp/gridftp_ns_xattr.cpp000066400000000000000000000353751465240014500227710ustar00rootroot00000000000000/* * Copyright (c) University of Nebraska-Lincoln 2016 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include "gridftp_namespace.h" #include "gridftp_plugin.h" #include #include "space/gfal2_space.h" #define GlobusErrorGeneric(reason) \ globus_error_put(GlobusErrorObjGeneric(reason)) #define GlobusErrorObjGeneric(reason) \ globus_error_construct_error( \ GLOBUS_NULL, \ GLOBUS_NULL, \ 1, \ __FILE__, \ "GFAL GridFTP getxattr", \ __LINE__, \ "%s", \ (reason)) static const GQuark GFAL_GRIDFTP_SCOPE_GETXATTR = g_quark_from_static_string("GridFTPModule::getxattr"); struct XAttrState; extern "C" { static void gfal_globus_done_callback(void* user_args, globus_object_t *globus_error); static void gridftp_cancel(gfal2_context_t context, void* userdata); } static int callback_cond_wait(XAttrState* state, time_t timeout); void globus_ftp_control_done_callback(void * user_arg, globus_ftp_control_handle_t *handle, globus_object_t *error, globus_ftp_control_response_t *resp) { gfal2_log(G_LOG_LEVEL_DEBUG, "FTP control operation done"); gfal_globus_done_callback(user_arg, error ? error : GLOBUS_SUCCESS); } struct XAttrState { XAttrState(const char *token, GridFTPFactory* factory) : m_token(token), m_url(NULL), m_handle(NULL), m_factory(factory), m_cred(GSS_C_NO_CREDENTIAL), m_error(NULL), m_done(true), m_needs_quit(false), m_usage(-1), m_free(-1), m_total(-1) { int global_timeout = gfal2_get_opt_integer_with_default( factory->get_gfal2_context(), CORE_CONFIG_GROUP, CORE_CONFIG_NAMESPACE_TIMEOUT, 300); m_default_timeout = gfal2_get_opt_integer_with_default( factory->get_gfal2_context(), GRIDFTP_CONFIG_GROUP, GRIDFTP_CONFIG_OP_TIMEOUT, global_timeout); globus_mutex_init(&m_mutex, NULL); globus_cond_init(&m_cond, NULL); memset(&m_auth, '\0', sizeof(m_auth)); } ~XAttrState() { if (!m_done) { cancel("XAttrState destructor called before the operation finished!"); callback_cond_wait(this, m_default_timeout); } globus_mutex_destroy(&m_mutex); globus_cond_destroy(&m_cond); delete m_error; if (m_url) { globus_url_destroy(m_url); } delete m_url; if (m_handle) { globus_ftp_control_handle_destroy(m_handle); } delete m_handle; } void cancel(const std::string& msg) { //if (!m_needs_quit) return; globus_result_t result = globus_ftp_control_force_close(m_handle, globus_ftp_control_done_callback, this); gfal_globus_check_result(GFAL_GRIDFTP_SCOPE_GETXATTR, result); m_error = new Gfal::CoreException(GFAL_GRIDFTP_SCOPE_GETXATTR, ECANCELED, msg); } void wait(time_t timeout = -1) { if (timeout < 0) timeout = m_default_timeout; gfal2_log(G_LOG_LEVEL_DEBUG, " [XAttrState::wait_callback] setup gsiftp timeout to %lld seconds", (long long) timeout); gfal_cancel_token_t cancel_token; cancel_token = gfal2_register_cancel_callback(m_factory->get_gfal2_context(), gridftp_cancel, this); int wait_ret = callback_cond_wait(this, timeout); gfal2_remove_cancel_callback(m_factory->get_gfal2_context(), cancel_token); // Operation expired, so cancel and raise an error if (wait_ret == ETIMEDOUT) { gfal2_log(G_LOG_LEVEL_DEBUG, " [XAttrState::wait_callback] Operation timeout of %d seconds expired, canceling...", timeout); gridftp_cancel(m_factory->get_gfal2_context(), this); // Wait again for the callback, ignoring timeout this time callback_cond_wait(this, timeout); throw Gfal::CoreException(GFAL_GRIDFTP_SCOPE_GETXATTR, ETIMEDOUT, "Operation timed out"); } if (m_error) { if (m_needs_quit) { m_done = false; globus_result_t result = globus_ftp_control_force_close(m_handle, globus_ftp_control_done_callback, this); gfal_globus_check_result(GFAL_GRIDFTP_SCOPE_GETXATTR, result); callback_cond_wait(this, timeout); } if (m_error->domain() != 0) throw Gfal::CoreException(GFAL_GRIDFTP_SCOPE_GETXATTR, m_error->code(), m_error->what()); else throw *m_error; } } const char *m_token; globus_url_t *m_url; globus_ftp_control_handle_t *m_handle; GridFTPFactory *m_factory; globus_ftp_control_auth_info_t m_auth; gss_cred_id_t m_cred; globus_mutex_t m_mutex; globus_cond_t m_cond; Gfal::CoreException *m_error; bool m_done; bool m_needs_quit; time_t m_default_timeout; long long m_usage; long long m_free; long long m_total; }; extern "C" { static void gfal_globus_done_callback(void* user_args, globus_object_t *globus_error) { XAttrState* state = (XAttrState*) user_args; globus_mutex_lock(&state->m_mutex); if (globus_error != GLOBUS_SUCCESS) { char *err_buffer; int err_code = gfal_globus_error_convert(globus_error, &err_buffer); char err_static[2048]; g_strlcpy(err_static, err_buffer, sizeof(err_static)); g_free(err_buffer); state->m_error = new Gfal::CoreException(GFAL_GRIDFTP_SCOPE_GETXATTR, err_code, err_static); // Log complete error dump char *chain = globus_error_print_chain(globus_error); if (chain != NULL) { gfal2_log(G_LOG_LEVEL_DEBUG, chain); globus_free(chain); } } state->m_done = true; globus_cond_signal(&state->m_cond); globus_mutex_unlock(&state->m_mutex); } static void gridftp_cancel(gfal2_context_t context, void* userdata) { XAttrState* state = (XAttrState*)userdata; state->cancel("Operation canceled from gfal2_cancel"); } static void site_usage_callback(void *arg, globus_ftp_control_handle_t *handle, globus_object_t *err, globus_ftp_control_response_t *resp) { if (resp == GLOBUS_NULL) { gfal_globus_done_callback(arg, err ? err : GlobusErrorObjGeneric("Site usage invoked with null response")); return; } if (resp->code != 250) { gfal_globus_done_callback(arg, err ? err : GlobusErrorObjGeneric(resp->response_buffer)); return; } //printf("Response: %s", resp->response_buffer); XAttrState *state = (XAttrState*)arg; if (3 != sscanf((const char *)resp->response_buffer, "250 USAGE %lld FREE %lld TOTAL %lld", &state->m_usage, &state->m_free, &state->m_total)) { gfal_globus_done_callback(arg, GlobusErrorObjGeneric("Invalid SITE USAGE response from server.")); return; } if ((state->m_total < 0) && (state->m_free >= 0) && (state->m_usage >= 0)) { state->m_total = state->m_free + state->m_usage; } gfal_globus_done_callback(arg, GLOBUS_SUCCESS); } static void authenticate_callback(void *arg, globus_ftp_control_handle_t *handle, globus_object_t *err, globus_ftp_control_response_t *resp) { if (resp == GLOBUS_NULL) { gfal_globus_done_callback(arg, err ? err : GlobusErrorObjGeneric("Authenticate invoked with null response")); return; } if (resp->code != 230) { gfal_globus_done_callback(arg, err ? err : GlobusErrorObjGeneric("Authentication failed.")); return; } XAttrState *state = (XAttrState*)arg; globus_result_t result; if (state->m_token) { //printf("SITE USAGE TOKEN %s /%s\n", state->m_token, state->m_url->url_path); result = globus_ftp_control_send_command(handle, "SITE USAGE TOKEN %s /%s\r\n", site_usage_callback, state, state->m_token, state->m_url->url_path); } else { //printf("SITE USAGE /%s\n", state->m_url->url_path); result = globus_ftp_control_send_command(handle, "SITE USAGE /%s\r\n", site_usage_callback, state, state->m_url->url_path); } if (result != GLOBUS_SUCCESS) {gfal_globus_done_callback(arg, globus_error_get(result));} } static void connect_callback(void *arg, globus_ftp_control_handle_t *handle, globus_object_t *err, globus_ftp_control_response_t *resp) { if (resp == GLOBUS_NULL) { gfal_globus_done_callback(arg, (err != GLOBUS_SUCCESS) ? err : GlobusErrorObjGeneric("Connect invoked with null response")); return; } //printf("Login message: %s\n", resp->response_buffer); XAttrState *state = (XAttrState*)arg; globus_mutex_lock(&state->m_mutex); { state->m_needs_quit = true; } globus_mutex_unlock(&state->m_mutex); if (resp->code != 220) { gfal_globus_done_callback(arg, GlobusErrorObjGeneric("Server did not indicate successful connection.")); return; } globus_result_t result = globus_ftp_control_auth_info_init( &state->m_auth, state->m_cred, GLOBUS_FALSE, NULL, NULL, NULL, NULL); if (result != GLOBUS_SUCCESS) {gfal_globus_done_callback(arg, globus_error_get(result));} result = globus_ftp_control_authenticate( handle, &state->m_auth, GLOBUS_TRUE, authenticate_callback, arg); if (result != GLOBUS_SUCCESS) {gfal_globus_done_callback(arg, globus_error_get(result));} } } static int callback_cond_wait(XAttrState* state, time_t timeout) { globus_abstime_t timeout_expires; GlobusTimeAbstimeGetCurrent(timeout_expires); timeout_expires.tv_sec += timeout; globus_mutex_lock(&state->m_mutex); int wait_ret = 0; while (!state->m_done && wait_ret != ETIMEDOUT) { wait_ret = globus_cond_timedwait(&state->m_cond, &state->m_mutex, &timeout_expires); } globus_mutex_unlock(&state->m_mutex); return wait_ret; } ssize_t GridFTPModule::getxattr(const char *path, const char *name, void *buff, size_t s_buff) { if (path == NULL) { throw Gfal::CoreException(GFAL_GRIDFTP_SCOPE_GETXATTR, EINVAL, "Invalid path argument"); } if (strncmp(name, GFAL_XATTR_SPACETOKEN, 10) != 0) { std::stringstream msg; msg << "'" << name << "' extended attributed not supported by GridFTP plugin"; throw Gfal::CoreException(GFAL_GRIDFTP_SCOPE_GETXATTR, ENOATTR, msg.str()); } const char *qmark = strchr(name, '?'); const char *token = NULL; if (qmark) { token = qmark+1; } bool is_descr = false; const char *dot = strchr(name, '.'); if ((dot != NULL) && (strncmp(dot, ".description", 12) == 0)) { is_descr = true; } gfal2_log(G_LOG_LEVEL_DEBUG, " -> [GridFTPModule::getxattr] "); XAttrState handler(token, _handle_factory); OM_uint32 min; OM_uint32 maj = gss_acquire_cred( &min, GSS_C_NO_NAME, 0, GSS_C_NO_OID_SET, GSS_C_BOTH, &handler.m_cred, NULL, NULL); if (maj != GSS_S_COMPLETE) { throw Gfal::CoreException(GFAL_GRIDFTP_SCOPE_GETXATTR, ENOATTR, "failed to acquire client credential"); } handler.m_url = new globus_url_t; globus_result_t result = globus_url_parse_rfc1738(path, handler.m_url); gfal_globus_check_result(GFAL_GRIDFTP_SCOPE_GETXATTR, result); handler.m_handle = new globus_ftp_control_handle_t; result = globus_ftp_control_handle_init(handler.m_handle); gfal_globus_check_result(GFAL_GRIDFTP_SCOPE_GETXATTR, result); short int port = handler.m_url->port == 0 ? 2811 : handler.m_url->port; handler.m_done = false; result = globus_ftp_control_connect(handler.m_handle, handler.m_url->host, port, connect_callback, &handler); if (result != GLOBUS_SUCCESS) {handler.m_done = true;} gfal_globus_check_result(GFAL_GRIDFTP_SCOPE_GETXATTR, result); handler.wait(); if (handler.m_needs_quit) { handler.m_done = false; result = globus_ftp_control_quit(handler.m_handle, globus_ftp_control_done_callback, &handler); gfal_globus_check_result(GFAL_GRIDFTP_SCOPE_GETXATTR, result); handler.wait(); } gfal2_log(G_LOG_LEVEL_DEBUG, " <- [GridFTPModule::getxattr] "); struct space_report report = {0}; report.used = handler.m_usage; report.free = handler.m_free; report.total = handler.m_total; return gfal2_space_generate_json(&report, (char*)buff, s_buff); } extern "C" ssize_t gfal_gridftp_getxattrG(plugin_handle handle, const char* path, const char *name, void *buff, size_t s_buff, GError** err) { g_return_val_err_if_fail(handle != NULL && path != NULL && name != NULL && buff != NULL, -1, err, "[gfal_gridftp_getxattrG][gridftp] Invalid parameters"); GError * tmp_err = NULL; int ret = -1; gfal2_log(G_LOG_LEVEL_DEBUG, " -> [gfal_gridftp_getxattrG]"); CPP_GERROR_TRY ret = (static_cast(handle))->getxattr(path, name, buff, s_buff); CPP_GERROR_CATCH(&tmp_err); gfal2_log(G_LOG_LEVEL_DEBUG, " [gfal_gridftp_getxattrG]<-"); G_RETURN_ERR(ret, tmp_err, err); } extern "C" ssize_t gfal_gridftp_listxattrG(plugin_handle handle, const char* url, char* list, size_t s_list, GError** err) { return g_strlcpy(list, GFAL_XATTR_SPACETOKEN, s_list); } gfal2-v2.23.0/src/plugins/gridftp/gridftp_parsing.cpp000066400000000000000000000270771465240014500225720ustar00rootroot00000000000000/* * Copyright @ CERN, 2015 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "gridftp_parsing.h" #include #include #include #include #include #include #include #include // Adapted from http://cvs.globus.org/viewcvs.cgi/gass/copy/source/globus_gass_copy_glob.c static int copy_mdtm_to_timet(char * mdtm_str, int * time_out) { char * p; struct tm tm; struct tm gmt_now_tm; struct tm * gmt_now_tm_p; time_t offset; time_t gmt_now; time_t now; time_t file_time; int rc; p = mdtm_str; memset(&tm, '\0', sizeof(struct tm)); /* 4 digit year */ rc = sscanf(p, "%04d", &tm.tm_year); if (rc != 1) { goto error_exit; } tm.tm_year -= 1900; p += 4; /* 2 digit month [01-12] */ rc = sscanf(p, "%02d", &tm.tm_mon); if (rc != 1) { goto error_exit; } tm.tm_mon--; p += 2; /* 2 digit day/month [01-31] */ rc = sscanf(p, "%02d", &tm.tm_mday); if (rc != 1) { goto error_exit; } p += 2; /* 2 digit hour [00-23] */ rc = sscanf(p, "%02d", &tm.tm_hour); if (rc != 1) { goto error_exit; } p += 2; /* 2 digit minute [00-59] */ rc = sscanf(p, "%02d", &tm.tm_min); if (rc != 1) { goto error_exit; } p += 2; /* 2 digit second [00-60] */ rc = sscanf(p, "%02d", &tm.tm_sec); if (rc != 1) { goto error_exit; } p += 2; file_time = mktime(&tm); if (file_time == (time_t) -1) { goto error_exit; } now = time(&now); if (now == (time_t) -1) { goto error_exit; } memset(&gmt_now_tm, '\0', sizeof(struct tm)); gmt_now_tm_p = globus_libc_gmtime_r(&now, &gmt_now_tm); if (gmt_now_tm_p == NULL) { goto error_exit; } gmt_now = mktime(&gmt_now_tm); if (gmt_now == (time_t) -1) { goto error_exit; } offset = now - gmt_now; *time_out = file_time + offset; return 0; error_exit: return -1; } globus_result_t parse_mlst_line(char *line, struct stat *stat_info, char *filename_buf, size_t filename_size) { globus_result_t result; int i; char * space; char * filename; char * startline; char * startfact; char * endfact; char * factval; char * unique_id = NULL; char * mode_s = NULL; char * symlink_target = NULL; char * modify_s = NULL; char * size_s = NULL; globus_gass_copy_glob_entry_t type = GLOBUS_GASS_COPY_GLOB_ENTRY_FILE; startline = line; space = strchr(startline, ' '); if (space == GLOBUS_NULL) { result = globus_error_put( globus_error_construct_string(GLOBUS_GASS_COPY_MODULE, GLOBUS_NULL, "[%s]: Bad MLSD response", __func__)); goto error_invalid_mlsd; } *space = '\0'; filename = space + 1; startfact = startline; if (filename_buf) { size_t len = g_strlcpy(filename_buf, filename, filename_size); char* trailing = filename_buf + len; do { *trailing = '\0'; --trailing; } while (trailing >= filename_buf && isspace(*trailing)); } while (startfact != space) { endfact = strchr(startfact, ';'); if (endfact) { *endfact = '\0'; } else { /* older MLST-draft spec says ending fact can be missing the final semicolon... not a problem to support this, no need to die. (ncftpd does this) result = globus_error_put( globus_error_construct_string( GLOBUS_GASS_COPY_MODULE, GLOBUS_NULL, "[%s]: Bad MLSD response", myname)); goto error_invalid_mlsd; */ endfact = space - 1; } factval = strchr(startfact, '='); if (!factval) { result = globus_error_put( globus_error_construct_string(GLOBUS_GASS_COPY_MODULE, GLOBUS_NULL, "[%s]: Bad MLSD response", __func__)); goto error_invalid_mlsd; } *(factval++) = '\0'; for (i = 0; startfact[i] != '\0'; i++) { startfact[i] = tolower(startfact[i]); } if (strcmp(startfact, "type") == 0) { if ((strcasecmp(factval, "dir") == 0) || (strcasecmp(factval, "pdir") == 0) || (strcasecmp(factval, "cdir") == 0)) { type = GLOBUS_GASS_COPY_GLOB_ENTRY_DIR; } else if (strcasecmp(factval, "file") == 0) { type = GLOBUS_GASS_COPY_GLOB_ENTRY_FILE; } else { type = GLOBUS_GASS_COPY_GLOB_ENTRY_OTHER; } } if (strcmp(startfact, "unique") == 0) { unique_id = factval; } if (strcmp(startfact, "unix.mode") == 0) { mode_s = factval; } if (strcmp(startfact, "modify") == 0) { modify_s = factval; } if (strcmp(startfact, "size") == 0) { size_s = factval; } if (strcmp(startfact, "unix.slink") == 0) { symlink_target = factval; } if (strcmp(startfact, "unix.uid") == 0) { stat_info->st_uid = atoi(factval); } if (strcmp(startfact, "unix.gid") == 0) { stat_info->st_gid = atoi(factval); } startfact = endfact + 1; } stat_info->st_nlink = 1; stat_info->st_mode = -1; stat_info->st_size = 0; stat_info->st_mtime = -1; if (mode_s) { stat_info->st_mode = strtoul(mode_s, NULL, 8); if (type == GLOBUS_GASS_COPY_GLOB_ENTRY_DIR) { stat_info->st_mode |= S_IFDIR; } else { stat_info->st_mode |= S_IFREG; } } if (size_s) { long long size; int rc; rc = sscanf(size_s, "%lld", &size); if (rc == 1) { stat_info->st_size = size; } } if (modify_s) { int mdtm; if (copy_mdtm_to_timet(modify_s, &mdtm) == GLOBUS_SUCCESS) { stat_info->st_mtime = mdtm; } } return GLOBUS_SUCCESS; error_invalid_mlsd: return result; } static mode_t parse_ls_submode(const char* substr) { return ((substr[0] == 'r') * S_IRUSR) | ((substr[1] == 'w') * S_IWUSR) | ((substr[2] == 'x') * S_IXUSR); } static mode_t parse_ls_mode(const char* mode_str) { mode_t mode = 0; if (strlen(mode_str) != 10) return mode; switch (mode_str[0]) { case 'd': mode = S_IFDIR; break; case '-': mode = S_IFREG; break; case 'l': mode = S_IFLNK; break; case 'b': mode = S_IFBLK; break; case 'c': mode = S_IFCHR; break; case 's': mode = S_IFSOCK; break; default: // Unknown break; } mode |= parse_ls_submode(mode_str + 1); mode |= (parse_ls_submode(mode_str + 4) >> 3); mode |= (parse_ls_submode(mode_str + 7) >> 6); return mode; } globus_result_t parse_stat_line(char* buffer, struct stat* fstat, char *filename_buf, size_t filename_size) { if (!buffer || !fstat) return GLOBUS_FAILURE; if (filename_buf && filename_size > 0) filename_buf[0] = '\0'; // Lines are like // -rw-rw-r-- 1 ftp ftp 49 Oct 29 2009 /pub/ubuntu-releases/robots.txt enum FtpField { FTP_FIELD_MODE, FTP_FIELD_NLINKS, FTP_FIELD_OWNER, FTP_FIELD_GROUP, FTP_FIELD_SIZE, FTP_FIELD_MONTH, FTP_FIELD_DAY, FTP_FIELD_YEAR_OR_TIME, FTP_FIELD_NAME, FTP_FIELD_LINK }; int field = FTP_FIELD_MODE; struct tm timedef; memset(&timedef, 0, sizeof(timedef)); time_t now = time(NULL); struct tm today; localtime_r(&now, &today); char *colon; char *start = buffer; while (*start && field < FTP_FIELD_LINK) { while (isspace(*start) && *start) ++start; if (!*start) break; bool eol = false; char* end = start; while (!isspace(*end) && *end) ++end; if (!*end) eol = true; *end = '\0'; switch (field) { case FTP_FIELD_MODE: fstat->st_mode = parse_ls_mode(start); break; case FTP_FIELD_NLINKS: fstat->st_nlink = atol(start); break; case FTP_FIELD_OWNER: if (isdigit(*start)) { fstat->st_uid = atoi(start); } else { char usrbuf[128]; struct passwd usr, *usr_ptr; if (getpwnam_r(start, &usr, usrbuf, sizeof(usrbuf), &usr_ptr) == 0) { fstat->st_uid = usr.pw_uid; } else { gfal2_log(G_LOG_LEVEL_WARNING, "Could not get uid for %s (%d)", start, errno); } } break; case FTP_FIELD_GROUP: if (isdigit(*start)) { fstat->st_gid = atoi(start); } else { char grbuf[128]; struct group grp, *grp_ptr; if (getgrnam_r(start, &grp, grbuf, sizeof(grbuf), &grp_ptr) == 0) { fstat->st_gid = grp.gr_gid; } else { gfal2_log(G_LOG_LEVEL_WARNING, "Could not get gid for %s (%d)", start, errno); } } break; case FTP_FIELD_SIZE: fstat->st_size = atol(start); break; case FTP_FIELD_MONTH: strptime(start, "%b", &timedef); break; case FTP_FIELD_DAY: timedef.tm_mday = atoi(start); break; case FTP_FIELD_YEAR_OR_TIME: if ((colon = strchr(start, ':'))) { timedef.tm_year = today.tm_year; timedef.tm_hour = atoi(start); timedef.tm_min = atoi(colon + 1); } else { timedef.tm_year = atoi(start) - 1900; } break; case FTP_FIELD_NAME: if (filename_buf && filename_size) { g_strlcpy(filename_buf, start, filename_size); } default: break; } if (eol) break; // Next field start = end + 1; field++; } struct tm gmt_now_tm; memset(&gmt_now_tm, '\0', sizeof(struct tm)); globus_libc_gmtime_r(&now, &gmt_now_tm); time_t gmt_now = mktime(&gmt_now_tm); time_t offset = now - gmt_now; fstat->st_atime = fstat->st_mtime = fstat->st_ctime = mktime(&timedef) - offset; return GLOBUS_SUCCESS; } gfal2-v2.23.0/src/plugins/gridftp/gridftp_parsing.h000066400000000000000000000017201465240014500222220ustar00rootroot00000000000000/* * Copyright @ CERN, 2015 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #ifndef GFAL2_GRIDFTP_PARSING_H #define GFAL2_GRIDFTP_PARSING_H #include /// Parse a MLST line globus_result_t parse_mlst_line(char *line, struct stat *stat_info, char *filename_buf, size_t filename_size); /// Parse a STAT or LIST line globus_result_t parse_stat_line(char* buffer, struct stat* fstat, char *filename_buf, size_t filename_size); #endif //GFAL2_GRIDFTP_PARSING_H gfal2-v2.23.0/src/plugins/gridftp/gridftp_pasv_plugin.cpp000066400000000000000000000233671465240014500234540ustar00rootroot00000000000000/* * Copyright @ CERN, 2015. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include "uri/gfal2_uri.h" #include "gridftp_pasv_plugin.h" #include "gridftp_filecopy.h" #include "gridftp_plugin.h" static const GQuark GFAL_GRIDFTP_PASV_STAGE_QUARK = g_quark_from_static_string("PASV"); static globus_ftp_client_plugin_t* gfal2_ftp_client_pasv_plugin_copy( globus_ftp_client_plugin_t* plugin_template, void* plugin_specific) { globus_ftp_client_plugin_t* plugin = (globus_ftp_client_plugin_t*) globus_malloc( sizeof(globus_ftp_client_plugin_t)); gfal2_ftp_client_pasv_plugin_init(plugin, reinterpret_cast(plugin_specific)); return plugin; } static void gfal2_ftp_client_pasv_plugin_destroy(globus_ftp_client_plugin_t* plugin, void* plugin_specific) { globus_ftp_client_plugin_destroy(plugin); globus_free(plugin); } static void gfal2_ftp_client_pasv_command(globus_ftp_client_plugin_t* plugin, void* plugin_specific, globus_ftp_client_handle_t* handle, const char* url, const char* command) { gfal2_log(G_LOG_LEVEL_DEBUG, ">> %s", command); } static void gfal2_ftp_client_pasv_fire_event(GridFTPSession* session, const char* hostname, const char* ip, unsigned port, bool is_ipv6) { if (session->params) { plugin_trigger_event(session->params, GFAL_GRIDFTP_DOMAIN_GSIFTP, GFAL_EVENT_DESTINATION, GFAL_GRIDFTP_PASV_STAGE_QUARK, "%s:%s:%u", hostname, ip, port); GQuark ipevent = (is_ipv6) ? GFAL_EVENT_IPV6 : GFAL_EVENT_IPV4; plugin_trigger_event(session->params, GFAL_GRIDFTP_DOMAIN_GSIFTP, GFAL_EVENT_DESTINATION, ipevent, "%s:%u", ip, port); } } // Entering Passive Mode (h1,h2,h3,h4,p1,p2). // Parenthesis are not guaranteed! static int parse_27(const char *resp, char *ip, size_t ip_size, unsigned *port, bool *is_ipv6) { static const char *regex_str = "[12]27 [^[0-9]+\\(?([0-9]+),([0-9]+),([0-9]+),([0-9]+),([0-9]+),([0-9]+)\\)?"; regex_t preg; // This response can not return never IPv6 values *is_ipv6 = false; assert(regcomp(&preg, regex_str, REG_EXTENDED | REG_ICASE) == 0); regmatch_t matches[7]; int ret = regexec(&preg, resp, 7, matches, 0); regfree(&preg); if (ret == REG_NOMATCH) { gfal2_log(G_LOG_LEVEL_DEBUG, "Failed to apply regex to 227 response"); return -1; } unsigned h1, h2, h3, h4, p1, p2; h1 = atoi(resp + matches[1].rm_so); h2 = atoi(resp + matches[2].rm_so); h3 = atoi(resp + matches[3].rm_so); h4 = atoi(resp + matches[4].rm_so); p1 = atoi(resp + matches[5].rm_so); p2 = atoi(resp + matches[6].rm_so); snprintf(ip, ip_size, "%u.%u.%u.%u", h1, h2, h3, h4); *port = (p1 * 256) + p2; return 0; } // Entering Long Passive Mode (long address, port). // Parenthesis are not guaranteed! static int parse_28(const char *, char *, size_t, unsigned *, bool*) { gfal2_log(G_LOG_LEVEL_WARNING, "Long Passive Mode not supported!"); return -1; } // Parse IPv6 replies static int parse_29_ipv6(const char *msg, char *ip, size_t ip_size, unsigned *port, bool *is_ipv6) { regex_t ipv6regex; int retregex = regcomp(&ipv6regex, "\\|([0-9]*)\\|([^|]*)\\|([0-9]+)\\|", REG_EXTENDED); g_assert(retregex == 0); regmatch_t matches[4]; retregex = regexec(&ipv6regex, msg, 4, matches, 0); regfree(&ipv6regex); if (retregex == REG_NOMATCH) { return -1; } // Type if (matches[1].rm_eo != matches[1].rm_so) { int type = atol(msg + matches[1].rm_so); if (type == 2) { *is_ipv6 = true; } } // Ip if (matches[2].rm_eo != matches[2].rm_so) { size_t len = matches[2].rm_eo - matches[2].rm_so; if (len > ip_size) { len = ip_size; } if (*is_ipv6) { char *buffer = g_strndup(msg + matches[2].rm_so, len); snprintf(ip, ip_size, "[%s]", buffer); g_free(buffer); } else { g_strlcpy(ip, msg + matches[2].rm_so, len); } } // Port *port = atoi(msg + matches[3].rm_so); return 0; } // Parse IPv4 replies static int parse_29_ipv4(const char *msg, char *ip, size_t ip_size, unsigned *port, bool *is_ipv6) { regex_t ipv4regex; int retregex = regcomp(&ipv4regex, "([0-9]+),([0-9]+),([0-9]+),([0-9]+),([0-9]+),([0-9]+)", REG_EXTENDED); g_assert(retregex == 0); regmatch_t matches[6]; retregex = regexec(&ipv4regex, msg, 6, matches, 0); regfree(&ipv4regex); if (retregex == REG_NOMATCH) { return -1; } *is_ipv6 = false; unsigned h1, h2, h3, h4, p1, p2; h1 = atoi(msg + matches[0].rm_so); h2 = atoi(msg + matches[1].rm_so); h3 = atoi(msg + matches[2].rm_so); h4 = atoi(msg + matches[3].rm_so); p1 = atoi(msg + matches[4].rm_so); p2 = atoi(msg + matches[5].rm_so); snprintf(ip, ip_size, "%u.%u.%u.%u", h1, h2, h3, h4); *port = (p1 * 256) + p2; return 0; } // Entering Extended Passive Mode (|protocol|ip|port|). // Parenthesis are standardized for EPSV, but they are not for SPAS static int parse_29(const char *msg, char *ip, size_t ip_size, unsigned *port, bool *is_ipv6) { *is_ipv6 = false; if (parse_29_ipv6(msg, ip, ip_size, port, is_ipv6) == 0) { return 0; } else if (parse_29_ipv4(msg, ip, ip_size, port, is_ipv6) == 0) { return 0; } gfal2_log(G_LOG_LEVEL_WARNING, "The passive mode response could not be parsed: %s", msg); return -1; } // Handle PASV responses static void gfal2_ftp_client_pasv_response(globus_ftp_client_plugin_t* plugin, void* plugin_specific, globus_ftp_client_handle_t* handle, const char* url, globus_object_t* error, const globus_ftp_control_response_t* ftp_response) { GridFTPSession* session = reinterpret_cast(plugin_specific); const char *p = reinterpret_cast(ftp_response->response_buffer); gfal2_log(G_LOG_LEVEL_DEBUG, ">> %s", p); char ip[65] = {0}; unsigned port = 0; bool got_pasv_ip = false, is_ipv6 = false; switch (ftp_response->response_class) { case GLOBUS_FTP_POSITIVE_PRELIMINARY_REPLY: case GLOBUS_FTP_POSITIVE_COMPLETION_REPLY: switch (ftp_response->code % 100) { case 27: got_pasv_ip = (parse_27(p, ip, sizeof(ip), &port, &is_ipv6) == 0); break; case 28: got_pasv_ip = (parse_28(p, ip, sizeof(ip), &port, &is_ipv6) == 0); break; case 29: got_pasv_ip = (parse_29(p, ip, sizeof(ip), &port, &is_ipv6) == 0); break; } break; default: break; } if (got_pasv_ip) { GError* err = NULL; gfal2_uri *parsed = gfal2_parse_uri(url, &err); if (parsed == NULL) { gfal2_log(G_LOG_LEVEL_WARNING, "Could not parse the URL: %s (%s)", url, err->message); g_error_free(err); } else { // Not specified in the response, so figure it out if (ip[0] == '\0') { bool ipv6_enabled = gfal2_get_opt_boolean_with_default(session->context, GRIDFTP_CONFIG_GROUP, GRIDFTP_CONFIG_IPV6, FALSE); g_strlcpy(ip, lookup_host(parsed->host, ipv6_enabled, &is_ipv6).c_str(), sizeof(ip)); } gfal2_ftp_client_pasv_fire_event(session, parsed->host, ip, port, is_ipv6); gfal2_free_uri(parsed); } } } static void gfal2_ftp_client_pasv_transfer(globus_ftp_client_plugin_t* plugin, void* plugin_specific, globus_ftp_client_handle_t* handle, const char* source_url, const globus_ftp_client_operationattr_t* source_attr, const char* dest_url, const globus_ftp_client_operationattr_t* dest_attr, globus_bool_t restart) { // NOOP } globus_result_t gfal2_ftp_client_pasv_plugin_init(globus_ftp_client_plugin_t* plugin, GridFTPSession* session) { globus_result_t result = GLOBUS_SUCCESS; result = globus_ftp_client_plugin_init(plugin, "gfal2_ftp_client_pasv_plugin", GLOBUS_FTP_CLIENT_CMD_MASK_ALL, session); if (result != GLOBUS_SUCCESS) { goto failure; } result = globus_ftp_client_plugin_set_copy_func(plugin, gfal2_ftp_client_pasv_plugin_copy); if (result != GLOBUS_SUCCESS) { goto failure; } result = globus_ftp_client_plugin_set_destroy_func(plugin, gfal2_ftp_client_pasv_plugin_destroy); if (result != GLOBUS_SUCCESS) { goto failure; } result = globus_ftp_client_plugin_set_command_func(plugin, gfal2_ftp_client_pasv_command); if (result != GLOBUS_SUCCESS) { goto failure; } result = globus_ftp_client_plugin_set_response_func(plugin, gfal2_ftp_client_pasv_response); if (result != GLOBUS_SUCCESS) { goto failure; } // If we do not register these two, the others will not be called for the operations we want result = globus_ftp_client_plugin_set_third_party_transfer_func(plugin, gfal2_ftp_client_pasv_transfer); if (result != GLOBUS_SUCCESS) { goto failure; } gfal2_log(G_LOG_LEVEL_DEBUG, "gfal2_ftp_client_pasv_plugin registered"); failure: return result; } gfal2-v2.23.0/src/plugins/gridftp/gridftp_pasv_plugin.h000066400000000000000000000020751465240014500231120ustar00rootroot00000000000000/* * Copyright @ CERN, 2015. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ /** * This implements a GridFTP client plugin that listen to PASV events, so it can * trigger gfal2 events and pass outside the information before the connection happens */ #ifndef GRIDFTPPASVPLUGIN_H #define GRIDFTPPASVPLUGIN_H #include #include class GridFTPSession; /** * Initialize the PASV plugin */ globus_result_t gfal2_ftp_client_pasv_plugin_init(globus_ftp_client_plugin_t* plugin, GridFTPSession* session); #endif // GRIDFTPPASVPLUGIN_H gfal2-v2.23.0/src/plugins/gridftp/gridftp_plugin.cpp000066400000000000000000000114231465240014500224110ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2015 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "gridftpmodule.h" #include "gridftp_namespace.h" #include "gridftp_filecopy.h" #include "gridftp_io.h" #include "gridftp_plugin.h" #include extern "C"{ static bool is_gridftp_uri(const char* src) { static const char gridftp_prefix[] = "gsiftp://"; static const char ftp_prefix[] = "ftp://"; return ( strncmp(src, gridftp_prefix, sizeof(gridftp_prefix) - 1) == 0 || strncmp(src, ftp_prefix, sizeof(ftp_prefix) - 1) == 0 ); } gboolean gridftp_check_url_transfer(plugin_handle handle, gfal2_context_t context, const char* src, const char* dst, gfal_url2_check type) { g_return_val_if_fail(handle != NULL, FALSE); gboolean res = FALSE; if( src != NULL && dst != NULL){ bool is_gridftp_transfer = is_gridftp_uri(src) && is_gridftp_uri(dst); if (type == GFAL_FILE_COPY || type == GFAL_BULK_COPY) res = is_gridftp_transfer; } return res; } int gridftp_check_url(plugin_handle handle, const char* src, plugin_mode check, GError ** err) { gboolean res = FALSE; if(is_gridftp_uri(src)) { switch(check){ case GFAL_PLUGIN_ACCESS: case GFAL_PLUGIN_STAT: case GFAL_PLUGIN_LSTAT: case GFAL_PLUGIN_MKDIR: case GFAL_PLUGIN_CHMOD: case GFAL_PLUGIN_RMDIR: case GFAL_PLUGIN_OPENDIR: case GFAL_PLUGIN_UNLINK: case GFAL_PLUGIN_OPEN: case GFAL_PLUGIN_CHECKSUM: case GFAL_PLUGIN_RENAME: case GFAL_PLUGIN_GETXATTR: case GFAL_PLUGIN_LISTXATTR: res = TRUE; break; default: break; } } return res; } plugin_handle gridftp_plugin_load(gfal2_context_t handle, GError ** err) { GError * tmp_err=NULL; plugin_handle h = NULL; CPP_GERROR_TRY gfal2_log(G_LOG_LEVEL_DEBUG, " -> [gridftp_plugin] try to load .."); h = static_cast( new GridFTPModule( new GridFTPFactory(handle) ) ); gfal2_log(G_LOG_LEVEL_DEBUG, " -> [gridftp_plugin] loaded .."); CPP_GERROR_CATCH(&tmp_err); G_RETURN_ERR(h, tmp_err, err); } void gridftp_plugin_unload(plugin_handle handle) { if(handle){ try{ delete (static_cast(handle)); }catch(...){ gfal2_log(G_LOG_LEVEL_MESSAGE, " bug found plugin gridFTP throws error while loading"); } } } const char *gridftp_plugin_name() { return GFAL2_PLUGIN_VERSIONED("gridftp", VERSION); } /** * Map function for the gridftp interface * this function provide the generic PLUGIN interface for the gridftp plugin. **/ gfal_plugin_interface gfal_plugin_init(gfal2_context_t handle, GError** err) { GError* tmp_err = NULL; gfal_plugin_interface ret; memset(&ret, 0, sizeof(gfal_plugin_interface)); plugin_handle r = gridftp_plugin_load(handle, &tmp_err); ret.plugin_data = r; ret.check_plugin_url = &gridftp_check_url; ret.plugin_delete = &gridftp_plugin_unload; ret.getName = &gridftp_plugin_name; ret.accessG = &gfal_gridftp_accessG; ret.statG = & gfal_gridftp_statG; ret.lstatG = &gfal_gridftp_statG; ret.unlinkG = &gfal_gridftp_unlinkG; ret.mkdirpG = &gfal_gridftp_mkdirG; ret.chmodG = &gfal_gridftp_chmodG; ret.rmdirG = &gfal_gridftp_rmdirG; ret.opendirG = &gfal_gridftp_opendirG; ret.readdirG = &gfal_gridftp_readdirG; ret.readdirppG = &gfal_gridftp_readdirppG; ret.closedirG = &gfal_gridftp_closedirG; ret.openG = &gfal_gridftp_openG; ret.closeG = &gfal_gridftp_closeG; ret.readG = &gfal_gridftp_readG; ret.writeG = &gfal_gridftp_writeG; ret.lseekG = &gfal_gridftp_lseekG; ret.checksum_calcG = &gfal_gridftp_checksumG; ret.renameG = &gfal_gridftp_renameG; ret.check_plugin_url_transfer = &gridftp_check_url_transfer; ret.copy_file = &gridftp_plugin_filecopy; ret.copy_bulk = &gridftp_bulk_copy; ret.getxattrG = &gfal_gridftp_getxattrG; ret.listxattrG = &gfal_gridftp_listxattrG; G_RETURN_ERR(ret, tmp_err, err); } }; gfal2-v2.23.0/src/plugins/gridftp/gridftp_plugin.h000066400000000000000000000041761465240014500220650ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #pragma once #ifndef GRIDFTP_PLUGIN_H #define GRIDFTP_PLUGIN_H #include // Configuration options #define GRIDFTP_CONFIG_GROUP "GRIDFTP PLUGIN" #define GRIDFTP_CONFIG_IPV6 "IPV6" #define GRIDFTP_CONFIG_SPAS "SPAS" #define GRIDFTP_CONFIG_V2 "GRIDFTP_V2" #define GRIDFTP_CONFIG_SESSION_REUSE "SESSION_REUSE" #define GRIDFTP_CONFIG_OP_TIMEOUT "OPERATION_TIMEOUT" #define GRIDFTP_CONFIG_DCAU "DCAU" #define GRIDFTP_CONFIG_DELAY_PASSV "DELAY_PASSV" #define GRIDFTP_CONFIG_ENABLE_PASV_PLUGIN "ENABLE_PASV_PLUGIN" #define GRIDFTP_CONFIG_BLOCK_SIZE "BLOCK_SIZE" #define GRIDFTP_CONFIG_NB_STREAM "RD_NB_STREAM" #define GRIDFTP_CONFIG_RESOLVE_DNS "RESOLVE_DNS" #define GRIDFTP_CONFIG_TRANSFER_CHECKSUM "COPY_CHECKSUM_TYPE" #define GRIDFTP_CONFIG_TRANSFER_PERF_TIMEOUT "PERF_MARKER_TIMEOUT" #define GRIDFTP_CONFIG_TRANSFER_SKIP_CHECKSUM "SKIP_SOURCE_CHECKSUM" #define GRIDFTP_CONFIG_TRANSFER_UDT "ENABLE_UDT" #ifdef __cplusplus extern "C" { #endif const char *gridftp_plugin_name(); plugin_handle gridftp_plugin_load(gfal2_context_t handle, GError ** err); void gridftp_plugin_unload(plugin_handle handle); gboolean gridftp_check_url_transfer(plugin_handle handle, gfal2_context_t context, const char* src, const char* dst, gfal_url2_check type); #ifdef __cplusplus } #endif #endif /* GRIDFTP_PLUGIN_H */ gfal2-v2.23.0/src/plugins/gridftp/gridftpmodule.cpp000066400000000000000000000037001465240014500222400ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2015 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include "gridftpmodule.h" #include "gridftpwrapper.h" #include #include #include #include #include //GOnce my_once = G_ONCE_INIT; static const GQuark GFAL_GRIDFTP_SCOPE_GLOBUS_INIT = g_quark_from_static_string("GridFTPModule::init_globus"); // initialization __attribute__((constructor)) void gridftp_plugin_init() { #if (!GLIB_CHECK_VERSION (2, 32, 0)) if (!g_thread_supported()) g_thread_init(NULL); #endif if (!getenv("GLOBUS_THREAD_MODEL")) globus_thread_set_model("pthread"); globus_module_activate(GLOBUS_GSI_GSS_ASSIST_MODULE); globus_module_activate(GLOBUS_GSI_GSSAPI_MODULE); } GridFTPModule::GridFTPModule(GridFTPFactory* factory) { _handle_factory = factory; globus_module_activate(GLOBUS_GASS_COPY_MODULE); globus_module_activate(GLOBUS_FTP_CLIENT_MODULE); globus_module_activate(GLOBUS_FTP_CONTROL_MODULE); globus_module_activate(GLOBUS_FTP_CLIENT_DEBUG_PLUGIN_MODULE); globus_module_activate(GLOBUS_FTP_CLIENT_THROUGHPUT_PLUGIN_MODULE); } GridFTPModule::~GridFTPModule() { delete _handle_factory; } gfal2-v2.23.0/src/plugins/gridftp/gridftpmodule.h000066400000000000000000000062011465240014500217040ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #pragma once #ifndef GRIDFTOMODULE_H #define GRIDFTOMODULE_H #include #include #include #include #include class GridFTPFactory; class GridFTPModule { public: GridFTPModule(GridFTPFactory *); ~GridFTPModule(); bool exists(const char* path); // Execute an access call, map on stat due to protocol restrictions void access(const char* path, int mode); // Execute a chmod query on path void chmod(const char* path, mode_t mode); // Execute a open query on path gfal_file_handle open(const char* url, int flag, mode_t mode); // Execute a read/pread query ssize_t read(gfal_file_handle handle, void* buffer, size_t count); ssize_t pread(gfal_file_handle handle, void* buffer, size_t count, off_t offset); // Execute a read/pread query ssize_t write(gfal_file_handle handle, const void* buffer, size_t count); ssize_t pwrite(gfal_file_handle handle, const void* buffer, size_t count, off_t offset); // seek a file off_t lseek(gfal_file_handle handle, off_t offset, int whence); // close a file int close(gfal_file_handle handle); //Execute a stat call on a gridftp URL void stat(const char* path, struct stat * st); // remove a file entry void unlink(const char* path); // Execute a mkdir query on path void mkdir(const char* path, mode_t mode); void checksum(const char* url, const char* check_type, char * checksum_buffer, size_t buffer_length, off_t start_offset, size_t data_length); // Rename void rename(const char* src, const char* dst); // rmdir query on path void rmdir(const char* path); // Query an extended attribute ssize_t getxattr(const char *path, const char *name, void *buff, size_t s_buff); void autoCleanFileCopy(gfalt_params_t params, int code, const char* dst); // Execute a file transfer operation for gridftp URLs void filecopy(gfalt_params_t params, const char* src, const char* dst); void internal_globus_gass_stat(const char* path, struct stat* fstat); GridFTPFactory* get_session_factory() { return _handle_factory; } private: GridFTPFactory * _handle_factory; }; void core_init(); #endif /* GRIDFTOMODULE_H */ gfal2-v2.23.0/src/plugins/gridftp/gridftpwrapper.cpp000066400000000000000000000735471465240014500224530ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2015 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include #include #include #include "gridftp_plugin.h" #include "gridftpwrapper.h" #include "gridftp_pasv_plugin.h" static const GQuark GFAL_GRIDFTP_SCOPE_REQ_STATE = g_quark_from_static_string("GridFTPModule::RequestState"); static const GQuark GFAL_GRIDFTP_GASS_COPY_HANDLER = g_quark_from_static_string("GridFTPModule::GassCopyAttrHandler"); static const GQuark GFAL_GRIDFTP_SESSION = g_quark_from_static_string("GridFTPModule::GridFTPSession"); static const GQuark GFAL_GLOBUS_DONE_SCOPE = g_quark_from_static_string("GridFTPModule::Done"); static std::string gridftp_hostname_from_url(const std::string& url) { GError *err = NULL; gfal2_uri *parsed = gfal2_parse_uri(url.c_str(), &err); if (err != NULL) { throw Gfal::CoreException(err); } char buffer[GFAL_URL_MAX_LEN]; snprintf(buffer, sizeof(buffer), "%s://%s:%d", parsed->scheme, parsed->host, parsed->port); gfal2_free_uri(parsed); return std::string(buffer); } GassCopyAttrHandler::GassCopyAttrHandler(globus_ftp_client_operationattr_t* ftp_operation_attr): cred_id(NULL) { // initialize gass copy attr globus_result_t res = globus_gass_copy_attr_init(&(attr_gass)); gfal_globus_check_result(GFAL_GRIDFTP_GASS_COPY_HANDLER, res); globus_ftp_client_operationattr_init(&(operation_attr_ftp_for_gass)); globus_ftp_client_operationattr_copy(&(operation_attr_ftp_for_gass), ftp_operation_attr); res = globus_gass_copy_attr_set_ftp(&(attr_gass), &operation_attr_ftp_for_gass); gfal_globus_check_result(GFAL_GRIDFTP_GASS_COPY_HANDLER, res); } GassCopyAttrHandler::~GassCopyAttrHandler() { globus_ftp_client_operationattr_destroy(&(operation_attr_ftp_for_gass)); if (cred_id) { OM_uint32 minor_status; gss_release_cred(&minor_status, &cred_id); } } GridFTPSessionHandler::GridFTPSessionHandler(GridFTPFactory* f, const std::string &uri): factory(f) { this->session = f->get_session(uri); GridFTPRequestState req(this); globus_result_t result = globus_ftp_client_feat(&this->session->handle_ftp, (char*)uri.c_str(), &this->session->operation_attr_ftp, &this->session->ftp_features, globus_ftp_client_done_callback, &req); gfal_globus_check_result(GFAL_GLOBUS_DONE_SCOPE, result); req.wait(GFAL_GLOBUS_DONE_SCOPE); // Enable SPAS if configured and supported gboolean spasEnabled = gfal2_get_opt_boolean_with_default(f->get_gfal2_context(), GRIDFTP_CONFIG_GROUP, GRIDFTP_CONFIG_SPAS, FALSE); globus_ftp_client_tristate_t spasSupported; globus_ftp_client_is_feature_supported(&this->session->ftp_features, &spasSupported, GLOBUS_FTP_CLIENT_FEATURE_MLST); if (spasEnabled && spasSupported == GLOBUS_FTP_CLIENT_TRUE) { globus_ftp_client_operationattr_set_striped(&this->session->operation_attr_ftp, GLOBUS_TRUE); } } GridFTPSessionHandler::~GridFTPSessionHandler() { try { factory->release_session(this->session); } catch (const std::exception& e) { gfal2_log(G_LOG_LEVEL_MESSAGE, "Caught an exception inside ~GridFTP_session()!! %s", e.what()); } catch (...) { gfal2_log(G_LOG_LEVEL_MESSAGE, "Caught an unknown exception inside ~GridFTP_session()!!"); } } GridFTPSession::GridFTPSession(gfal2_context_t context, const std::string& baseurl): baseurl(baseurl), cred_id(NULL), pasv_plugin(NULL), context(context), params(NULL) { globus_result_t res; res = globus_ftp_client_debug_plugin_init(&debug_ftp_plugin, stderr, "gridftp debug :"); gfal_globus_check_result(GFAL_GRIDFTP_SESSION, res); res = globus_ftp_client_operationattr_init(&operation_attr_ftp); gfal_globus_check_result(GFAL_GRIDFTP_SESSION, res); res = globus_ftp_client_handleattr_init(&attr_handle); gfal_globus_check_result(GFAL_GRIDFTP_SESSION, res); globus_ftp_client_handleattr_set_cache_all(&attr_handle, GLOBUS_TRUE); // enable session re-use if (getenv("GFAL2_GRIDFTP_DEBUG")) { globus_ftp_client_handleattr_add_plugin(&attr_handle, &debug_ftp_plugin); } gboolean register_pasv_plugin = gfal2_get_opt_boolean_with_default(context, GRIDFTP_CONFIG_GROUP, GRIDFTP_CONFIG_ENABLE_PASV_PLUGIN, FALSE); if (register_pasv_plugin) { res = gfal2_ftp_client_pasv_plugin_init(&pasv_plugin, this); gfal_globus_check_result(GFAL_GRIDFTP_SESSION, res); res = globus_ftp_client_handleattr_add_plugin(&attr_handle, &pasv_plugin); gfal_globus_check_result(GFAL_GRIDFTP_SESSION, res); } this->set_user_agent(context); res = globus_gass_copy_handleattr_init(&gass_handle_attr); gfal_globus_check_result(GFAL_GRIDFTP_SESSION, res); res = globus_gass_copy_handleattr_set_ftp_attr(&gass_handle_attr, &attr_handle); gfal_globus_check_result(GFAL_GRIDFTP_SESSION, res); res = globus_gass_copy_handle_init(&gass_handle, &gass_handle_attr); gfal_globus_check_result(GFAL_GRIDFTP_SESSION, res); int block_size = gfal2_get_opt_integer_with_default(context, GRIDFTP_CONFIG_GROUP, GRIDFTP_CONFIG_BLOCK_SIZE, 0); if (block_size > 0) { res = globus_gass_copy_set_buffer_length(&gass_handle, 0); gfal_globus_check_result(GFAL_GRIDFTP_SESSION, res); } res = globus_gass_copy_get_ftp_handle(&gass_handle, &handle_ftp); gfal_globus_check_result(GFAL_GRIDFTP_SESSION, res); res = globus_gass_copy_set_allocate(&gass_handle, GLOBUS_TRUE); gfal_globus_check_result(GFAL_GRIDFTP_SESSION, res); this->set_nb_streams(0); globus_ftp_client_features_init(&this->ftp_features); } GridFTPSession::~GridFTPSession() { globus_ftp_client_debug_plugin_destroy(&debug_ftp_plugin); globus_gass_copy_handle_destroy(&gass_handle); globus_ftp_client_operationattr_destroy(&operation_attr_ftp); globus_gass_copy_handleattr_destroy(&gass_handle_attr); globus_ftp_client_handleattr_destroy(&attr_handle); globus_ftp_client_features_destroy(&this->ftp_features); globus_ftp_client_plugin_destroy(&this->pasv_plugin); OM_uint32 minor_status; gss_release_cred(&minor_status, &this->cred_id); } void GridFTPSession::set_gridftpv2(bool v2) { globus_ftp_client_handleattr_set_gridftp2(&attr_handle, v2); } void GridFTPSession::set_ipv6(bool ipv6) { globus_ftp_client_operationattr_set_allow_ipv6(&operation_attr_ftp, ipv6); } void GridFTPSession::set_delayed_pass(bool delayed) { globus_ftp_client_operationattr_set_delayed_pasv(&operation_attr_ftp, delayed); } void GridFTPSession::set_dcau(bool dcau) { if (dcau) dcau_control.mode = GLOBUS_FTP_CONTROL_DCAU_DEFAULT; else dcau_control.mode = GLOBUS_FTP_CONTROL_DCAU_NONE; globus_ftp_client_operationattr_set_dcau(&operation_attr_ftp, &dcau_control); } void GridFTPSession::set_nb_streams(unsigned int nbstream) { if (nbstream == 0) { parallelism.fixed.size = 1; parallelism.mode = GLOBUS_FTP_CONTROL_PARALLELISM_NONE; mode = GLOBUS_FTP_CONTROL_MODE_NONE; } else { parallelism.fixed.size = nbstream; parallelism.mode = GLOBUS_FTP_CONTROL_PARALLELISM_FIXED; mode = GLOBUS_FTP_CONTROL_MODE_EXTENDED_BLOCK; } globus_ftp_client_operationattr_set_mode(&operation_attr_ftp, mode); globus_ftp_client_operationattr_set_parallelism(&operation_attr_ftp, ¶llelism); } void GridFTPSession::set_tcp_buffer_size(guint64 buffersize) { if (buffersize == 0) { tcp_buffer_size.mode = GLOBUS_FTP_CONTROL_TCPBUFFER_DEFAULT; tcp_buffer_size.fixed.size = 0ul; } else { tcp_buffer_size.mode = GLOBUS_FTP_CONTROL_TCPBUFFER_FIXED; tcp_buffer_size.fixed.size = buffersize; } globus_ftp_client_operationattr_set_tcp_buffer(&operation_attr_ftp, &tcp_buffer_size); } void GridFTPSession::set_user_agent(gfal2_context_t context) { const char *agent, *version; gfal2_get_user_agent(context, &agent, &version); // Client information char* client_info = gfal2_get_client_info_string(context); if (agent) { std::ostringstream full_version; full_version << version << " (gfal2 " << gfal2_version() << ")"; globus_ftp_client_handleattr_set_clientinfo(&attr_handle, agent, full_version.str().c_str(), client_info); } else { globus_ftp_client_handleattr_set_clientinfo(&attr_handle, "gfal2", gfal2_version(), client_info); } g_free(client_info); } void GridFTPSession::set_udt(bool udt) { if (udt) globus_ftp_client_operationattr_set_net_stack(&operation_attr_ftp, "udt"); else globus_ftp_client_operationattr_set_net_stack(&operation_attr_ftp, "default"); } std::string gfal_gridftp_get_credentials(gfal2_context_t context, const std::string &url, gchar **ucert, gchar **ukey, gchar **user, gchar **passwd) { GError *error = NULL; const char *baseurl; *ucert = gfal2_cred_get(context, GFAL_CRED_X509_CERT, url.c_str(), &baseurl, &error); Gfal::gerror_to_cpp(&error); *ukey = gfal2_cred_get(context, GFAL_CRED_X509_KEY, url.c_str(), &baseurl, &error); Gfal::gerror_to_cpp(&error); // User/password if (strncmp(url.c_str(), "ftp://", 6) == 0) { *user = gfal2_cred_get(context, GFAL_CRED_USER, url.c_str(), &baseurl, &error); Gfal::gerror_to_cpp(&error); if (!*user) { *user = gfal2_get_opt_string_with_default(context, "FTP", "USER", "anonymous"); } *passwd = gfal2_cred_get(context, GFAL_CRED_PASSWD, url.c_str(), &baseurl, &error); Gfal::gerror_to_cpp(&error); if (!*passwd) { *passwd = gfal2_get_opt_string_with_default(context, "FTP", "PASSWORD", "anonymous"); } } // Set credentials if (*ucert) { gfal2_log(G_LOG_LEVEL_DEBUG, "GSIFTP using certificate %s", *ucert); } if (*ukey) { gfal2_log(G_LOG_LEVEL_DEBUG, "GSIFTP using private key %s", *ukey); } if (*user) { gfal2_log(G_LOG_LEVEL_DEBUG, "FTP using user %s", *user); } if (baseurl != NULL && baseurl[0] != '\0') { return baseurl; } return gridftp_hostname_from_url(url); } void gfal_globus_set_credentials(const char* ucert, const char* ukey, const char *user, const char *passwd, gss_cred_id_t *cred_id, globus_ftp_client_operationattr_t* opattr) { if (ucert) { std::stringstream buffer; std::ifstream cert_stream(ucert); if (!cert_stream.good()) { throw Gfal::CoreException(GFAL_GRIDFTP_SCOPE_REQ_STATE, errno, "Could not open the user certificate"); } buffer << cert_stream.rdbuf(); if (ukey && strcmp(ucert, ukey) != 0) { std::ifstream key_stream(ukey); if (key_stream.bad()) { throw Gfal::CoreException(GFAL_GRIDFTP_SCOPE_REQ_STATE, errno, "Could not open the user private key"); } buffer << key_stream.rdbuf(); } OM_uint32 minor_status, major_status; gss_buffer_desc_struct buffer_desc; buffer_desc.value = g_strdup(buffer.str().c_str()); buffer_desc.length = buffer.str().size(); major_status = gss_import_cred(&minor_status, cred_id, GSS_C_NO_OID, 0, // 0 = Pass credentials; 1 = Pass path as X509_USER_PROXY=... &buffer_desc, 0, NULL); g_free(buffer_desc.value); if (major_status != GSS_S_COMPLETE) { std::stringstream err_buffer; err_buffer << "Could not load the user credentials: "; globus_object_t *error = globus_error_get(major_status); char *globus_errstr; int globus_errno = gfal_globus_error_convert(error, &globus_errstr); if (globus_errstr) { err_buffer << globus_errstr; g_free(globus_errstr); } globus_object_free(error); err_buffer << " (" << globus_errno << ")"; throw Gfal::CoreException(GFAL_GRIDFTP_SCOPE_REQ_STATE, globus_errno, err_buffer.str()); } } globus_ftp_client_operationattr_set_authorization( opattr, *cred_id, user, passwd, NULL, NULL); } globus_ftp_client_handle_t* GridFTPSessionHandler::get_ftp_client_handle() { globus_result_t res = globus_gass_copy_get_ftp_handle(&(session->gass_handle), &(session->handle_ftp)); gfal_globus_check_result(GFAL_GRIDFTP_SESSION, res); return &(session->handle_ftp); } globus_gass_copy_handle_t* GridFTPSessionHandler::get_gass_copy_handle() { return &(session->gass_handle); } globus_ftp_client_operationattr_t* GridFTPSessionHandler::get_ftp_client_operationattr() { return &(session->operation_attr_ftp); } globus_gass_copy_handleattr_t* GridFTPSessionHandler::get_gass_copy_handleattr() { return &(session->gass_handle_attr); } globus_ftp_client_handleattr_t* GridFTPSessionHandler::get_ftp_client_handleattr() { return &(session->attr_handle); } globus_ftp_client_features_t* GridFTPSessionHandler::get_ftp_features() { return (&session->ftp_features); } GridFTPFactory* GridFTPSessionHandler::get_factory() { return factory; } GridFTPFactory::GridFTPFactory(gfal2_context_t handle): gfal2_context(handle) { GError * tmp_err = NULL; session_reuse = gfal2_get_opt_boolean(gfal2_context, GRIDFTP_CONFIG_GROUP, GRIDFTP_CONFIG_SESSION_REUSE, &tmp_err); gfal2_log(G_LOG_LEVEL_DEBUG, " define GSIFTP session re-use to %s", (session_reuse) ? "TRUE" : "FALSE"); if (tmp_err) { throw Gfal::CoreException(tmp_err); } size_cache = 400; globus_mutex_init(&mux_cache, NULL); } void GridFTPFactory::clear_cache() { globus_mutex_lock(&mux_cache); gfal2_log(G_LOG_LEVEL_DEBUG, "gridftp session cache garbage collection ..."); std::multimap::iterator it; for (it = session_cache.begin(); it != session_cache.end(); ++it) { delete (*it).second; } session_cache.clear(); globus_mutex_unlock(&mux_cache); } void GridFTPFactory::recycle_session(GridFTPSession* session) { globus_mutex_lock(&mux_cache); if (session_cache.size() > size_cache) { clear_cache(); } gfal2_log(G_LOG_LEVEL_DEBUG, "insert gridftp session for %s in cache ...", session->baseurl.c_str()); session_cache.insert(std::pair(session->baseurl, session)); globus_mutex_unlock(&mux_cache); } // recycle a gridftp session object from cache if exist, return NULL else GridFTPSession* GridFTPFactory::get_recycled_handle(const std::string &baseurl) { globus_mutex_lock(&mux_cache); GridFTPSession* session = NULL; // try to find a session explicitly associated with this handle std::multimap::iterator it = session_cache.find(baseurl); // if no session found, take a generic one if (it == session_cache.end()) { gfal2_log(G_LOG_LEVEL_DEBUG, "no session associated with this baseurl, try find generic one .... "); it = session_cache.begin(); } if (it != session_cache.end()) { gfal2_log(G_LOG_LEVEL_DEBUG,"gridftp session for: %s found in cache !", baseurl.c_str()); session = (*it).second; session_cache.erase(it); } else { gfal2_log(G_LOG_LEVEL_DEBUG, "no session found in cache for %s!", baseurl.c_str()); } globus_mutex_unlock(&mux_cache); return session; } GridFTPFactory::~GridFTPFactory() { try { clear_cache(); } catch (const std::exception & e) { gfal2_log(G_LOG_LEVEL_MESSAGE, "Caught an exception inside ~GridFTPFactory()!! %s", e.what()); } catch (...) { gfal2_log(G_LOG_LEVEL_MESSAGE, "Caught an unknown exception inside ~GridFTPFactory()!!"); } globus_mutex_destroy(&mux_cache); } gfal2_context_t GridFTPFactory::get_gfal2_context() { return gfal2_context; } /* * dirty function to convert error code from globus * In the current state, globus provides no way to convert gridftp error code to errno properly.... * */ #ifndef ECOMM #define ECOMM EIO #endif static int scan_errstring(const char *p) { int ret = ECOMM; if (p == NULL) return ret; if (strcasestr(p, "No such file") || strcasestr(p, "not found") || strcasestr(p, "error 3011")) ret = ENOENT; else if (strstr(p, "Permission denied") || strcasestr(p, "credential")) ret = EACCES; else if ( (strcasestr(p, "exists")) || strcasestr(p, "error 3006")) ret = EEXIST; else if (strcasestr(p, "Not a direct")) ret = ENOTDIR; else if (strcasestr(p, "Operation not supported")) ret = ENOTSUP; else if (strcasestr(p, "Login incorrect") || strcasestr(p, "Could not get virtual id")) ret = EACCES; else if (strcasestr(p, "the operation was aborted")) ret = ECANCELED; else if (strcasestr(p, "Is a directory")) ret = EISDIR; else if (strcasestr(p, "isk quota exceeded")) ret = ENOSPC; return ret; } int gfal_globus_error_convert(globus_object_t * error, char ** str_error) { if (error) { *str_error = globus_error_print_friendly(error); char * p = *str_error; while (*p != '\0') { // string normalization of carriage return *p = (*p == '\n' || *p == '\r') ? ' ' : *p; ++p; } int errn = scan_errstring(*str_error); // try to get errno if (errn == 0) { globus_free(*str_error); *str_error = NULL; } return errn; } else { *str_error = NULL; } return 0; } static void gfal_globus_check_error(GQuark scope, globus_object_t * error) { if (error != GLOBUS_SUCCESS) { int globus_errno; char errbuff[GFAL_URL_MAX_LEN]; char * glob_str = NULL; *errbuff = '\0'; globus_errno = gfal_globus_error_convert(error, &glob_str); if (glob_str) { // security g_strlcpy(errbuff, glob_str, GFAL_URL_MAX_LEN); g_free(glob_str); } globus_object_free(error); throw Gfal::CoreException(scope, globus_errno, errbuff); } } void gfal_globus_check_result(GQuark scope, globus_result_t res) { if (res != GLOBUS_SUCCESS) { globus_object_t * error = globus_error_get(res); // get error from result code if (error == NULL) throw Gfal::CoreException(scope, EINVAL, "Unknown error: unable to map result code to globus error"); gfal_globus_check_error(scope, error); } } GridFTPSession* GridFTPFactory::get_new_handle(const std::string &baseurl) { bool gridftp_v2 = gfal2_get_opt_boolean_with_default(gfal2_context, GRIDFTP_CONFIG_GROUP, GRIDFTP_CONFIG_V2, true); bool ipv6 = gfal2_get_opt_boolean_with_default(gfal2_context, GRIDFTP_CONFIG_GROUP, GRIDFTP_CONFIG_IPV6, false); bool delay_passv = gfal2_get_opt_boolean_with_default(gfal2_context, GRIDFTP_CONFIG_GROUP, GRIDFTP_CONFIG_DELAY_PASSV, true); bool dcau = gfal2_get_opt_boolean_with_default(gfal2_context, GRIDFTP_CONFIG_GROUP, GRIDFTP_CONFIG_DCAU, false); std::unique_ptr session(new GridFTPSession(gfal2_context, baseurl)); session->set_gridftpv2(gridftp_v2); session->set_dcau(dcau); session->set_ipv6(ipv6); session->set_delayed_pass(delay_passv); return session.release(); } GridFTPSession* GridFTPFactory::get_session(const std::string &url) { gchar *ucert = NULL, *ukey = NULL; gchar *user = NULL, *passwd = NULL; std::string baseurl = gfal_gridftp_get_credentials(gfal2_context, url, &ucert, &ukey, &user, &passwd); GridFTPSession* session = NULL; if ((session = get_recycled_handle(baseurl)) == NULL) { session = get_new_handle(baseurl); gfal_globus_set_credentials(ucert, ukey, user, passwd, &session->cred_id, &session->operation_attr_ftp); } else if (session->baseurl != baseurl) { gfal_globus_set_credentials(ucert, ukey, user, passwd, &session->cred_id, &session->operation_attr_ftp); session->baseurl = baseurl; } g_free(ucert); g_free(ukey); g_free(user); g_free(passwd); return session; } void GridFTPFactory::release_session(GridFTPSession* session) { session_reuse = gfal2_get_opt_boolean_with_default(gfal2_context, GRIDFTP_CONFIG_GROUP, GRIDFTP_CONFIG_SESSION_REUSE, FALSE); if (session_reuse) { recycle_session(session); } else { gfal2_log(G_LOG_LEVEL_DEBUG, "destroy gridftp session for %s ...", session->baseurl.c_str()); delete session; } } static void gfal_globus_done_callback(void* user_args, globus_object_t *globus_error) { GridFTPRequestState* state = (GridFTPRequestState*) user_args; globus_mutex_lock(&state->mutex); if (globus_error != GLOBUS_SUCCESS) { char *err_buffer; int err_code = gfal_globus_error_convert(globus_error, &err_buffer); char err_static[2048]; g_strlcpy(err_static, err_buffer, sizeof(err_static)); g_free(err_buffer); state->error = new Gfal::CoreException(GFAL_GLOBUS_DONE_SCOPE, err_code, err_static); // Log complete error dump char *chain = globus_error_print_chain(globus_error); if (chain != NULL) { gfal2_log(G_LOG_LEVEL_DEBUG, chain); globus_free(chain); } } state->done = true; globus_cond_signal(&state->cond); globus_mutex_unlock(&state->mutex); } // gridftp callback generic implementation void globus_ftp_client_done_callback(void * user_arg, globus_ftp_client_handle_t * handle, globus_object_t * error) { gfal2_log(G_LOG_LEVEL_DEBUG, " gridFTP operation done"); gfal_globus_done_callback(user_arg, error); } // gass operation callback implementation void globus_gass_client_done_callback(void * callback_arg, globus_gass_copy_handle_t * handle, globus_object_t * error) { gfal2_log(G_LOG_LEVEL_DEBUG, "gass operation done"); gfal_globus_done_callback(callback_arg, error); } GridFTPRequestState::GridFTPRequestState(GridFTPSessionHandler* s, GridFTPRequestType request_type) : handler(s), request_type(request_type), error(NULL), done(false) { int global_ns_timeout = gfal2_get_opt_integer_with_default( s->get_factory()->get_gfal2_context(), CORE_CONFIG_GROUP, CORE_CONFIG_NAMESPACE_TIMEOUT, 300); this->default_timeout = gfal2_get_opt_integer_with_default( s->get_factory()->get_gfal2_context(), GRIDFTP_CONFIG_GROUP, GRIDFTP_CONFIG_OP_TIMEOUT, global_ns_timeout); globus_mutex_init(&mutex, NULL); globus_cond_init(&cond, NULL); } GridFTPRequestState::~GridFTPRequestState() { if (!done) { this->cancel(GFAL_GRIDFTP_SCOPE_REQ_STATE, "GridFTPRequestState destructor called before the operation finished!", ECANCELED); } globus_mutex_destroy(&mutex); globus_cond_destroy(&cond); delete error; } static void gridftp_cancel(gfal2_context_t context, void* userdata) { GridFTPRequestState* state = (GridFTPRequestState*)userdata; state->cancel(gfal_cancel_quark(), "Operation canceled from gfal2_cancel", ECANCELED); } static int callback_cond_wait(GridFTPRequestState* req, time_t timeout) { globus_abstime_t timeout_expires; GlobusTimeAbstimeGetCurrent(timeout_expires); timeout_expires.tv_sec += timeout; globus_mutex_lock(&req->mutex); int wait_ret = 0; while (!req->done && wait_ret != ETIMEDOUT) { wait_ret = globus_cond_timedwait(&req->cond, &req->mutex, &timeout_expires); } globus_mutex_unlock(&req->mutex); return wait_ret; } void GridFTPRequestState::wait(GQuark scope, time_t timeout) { if (timeout < 0) timeout = default_timeout; gfal2_log(G_LOG_LEVEL_DEBUG, " [GridFTP_Request_state::wait_callback] setup gsiftp timeout to %lld seconds", (long long) timeout); gfal_cancel_token_t cancel_token; cancel_token = gfal2_register_cancel_callback(handler->get_factory()->get_gfal2_context(), gridftp_cancel, this); int wait_ret = callback_cond_wait(this, timeout); gfal2_remove_cancel_callback(handler->get_factory()->get_gfal2_context(), cancel_token); // Operation expired, so cancel and raise an error if (wait_ret == ETIMEDOUT) { gfal2_log(G_LOG_LEVEL_DEBUG, " [GridFTP_Request_state::wait_callback] Operation timeout of %d seconds expired, canceling...", timeout); gridftp_cancel(handler->get_factory()->get_gfal2_context(), this); // Wait again for the callback, ignoring timeout this time callback_cond_wait(this, timeout); throw Gfal::CoreException(scope, ETIMEDOUT, "Operation timed out"); } if (error) { if (error->domain() != 0) throw Gfal::CoreException(scope, error->code(), error->what()); else throw *error; } } void GridFTPRequestState::cancel(GQuark scope, const std::string& msg, int errcode) { if (request_type == GRIDFTP_REQUEST_FTP) { globus_ftp_client_abort(handler->get_ftp_client_handle()); } else { globus_gass_copy_cancel(handler->get_gass_copy_handle(), globus_gass_client_done_callback, this); } error = new Gfal::CoreException(scope, errcode, msg); } GridFTPStreamState::GridFTPStreamState(GridFTPSessionHandler * s): GridFTPRequestState(s), offset(0), buffer_size(0), eof(false), expect_eof(false) { } GridFTPStreamState::~GridFTPStreamState() { } static void gfal_stream_done_callback_err_handling(GridFTPStreamState* state, globus_ftp_client_handle_t *handle, globus_object_t *globus_error, globus_byte_t *buffer, globus_size_t length, globus_off_t offset, globus_bool_t eof) { if (globus_error != GLOBUS_SUCCESS) { char *err_buffer; int err_code = gfal_globus_error_convert(globus_error, &err_buffer); char err_static[2048]; g_strlcpy(err_static, err_buffer, sizeof(err_static)); g_free(err_buffer); state->error = new Gfal::CoreException(GFAL_GLOBUS_DONE_SCOPE, err_code, err_static); } state->offset += length; state->eof = eof; } static void gfal_griftp_stream_read_done_callback(void *user_arg, globus_ftp_client_handle_t *handle, globus_object_t *error, globus_byte_t *buffer, globus_size_t length, globus_off_t offset, globus_bool_t eof) { GridFTPStreamState* state = static_cast(user_arg); globus_mutex_lock(&state->mutex); gfal_stream_done_callback_err_handling(state, handle, error, buffer, length, offset, eof); if (!state->expect_eof || eof) { state->done = true; globus_cond_signal(&state->cond); } else { // It may happen that the buffer size and the requested size are of // the same size, and there is enough data to fill it. // If that's the case, a second callback will be done with EOF, and we need // to get it, or waiting for the operation completion will block forever globus_ftp_client_register_read( handle, buffer, state->buffer_size, gfal_griftp_stream_read_done_callback, state); } globus_mutex_unlock(&state->mutex); } static void gfal_griftp_stream_write_done_callback(void *user_arg, globus_ftp_client_handle_t *handle, globus_object_t *error, globus_byte_t *buffer, globus_size_t length, globus_off_t offset, globus_bool_t eof) { GridFTPStreamState* state = static_cast(user_arg); globus_mutex_lock(&state->mutex); gfal_stream_done_callback_err_handling(state, handle, error, buffer, length, offset, eof); state->done = true; globus_cond_signal(&state->cond); globus_mutex_unlock(&state->mutex); } ssize_t gridftp_read_stream(GQuark scope, GridFTPStreamState* stream, void* buffer, size_t s_read, bool expect_eof) { gfal2_log(G_LOG_LEVEL_DEBUG, " -> [gridftp_read_stream]"); off_t initial_offset = stream->offset; if (stream->eof) return 0; stream->done = false; stream->buffer_size = s_read; stream->expect_eof = expect_eof; globus_result_t res = globus_ftp_client_register_read( stream->handler->get_ftp_client_handle(), (globus_byte_t*) buffer, s_read, gfal_griftp_stream_read_done_callback, stream); gfal_globus_check_result(scope, res); stream->wait(scope); return stream->offset - initial_offset; } ssize_t gridftp_write_stream(GQuark scope, GridFTPStreamState* stream, const void* buffer, size_t s_write, bool eof) { gfal2_log(G_LOG_LEVEL_DEBUG, " -> [gridftp_write_stream]"); off_t initial_offset = stream->offset; stream->done = false; globus_result_t res = globus_ftp_client_register_write( stream->handler->get_ftp_client_handle(), (globus_byte_t*) buffer, s_write, initial_offset, eof, gfal_griftp_stream_write_done_callback, stream ); gfal_globus_check_result(scope, res); stream->wait(scope); return stream->offset - initial_offset; } gfal2-v2.23.0/src/plugins/gridftp/gridftpwrapper.h000066400000000000000000000147251465240014500221110ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #ifndef GRIDFTPWRAPPER_H #define GRIDFTPWRAPPER_H #include #include #include #include #include #include #include #include #include #include // Forward declarations class GridFTPFactory; class GridFTPSessionHandler; class GridFTPSessionHandler; class GridFTPRequestState; class GridFTPStreamState; enum GridFTPRequestStatus { GRIDFTP_REQUEST_NOT_LAUNCHED, GRIDFTP_REQUEST_RUNNING, GRIDFTP_REQUEST_FINISHED, }; enum GridFTPRequestType { GRIDFTP_REQUEST_GASS, GRIDFTP_REQUEST_FTP }; class GridFTPRequestState { public: GridFTPRequestState(GridFTPSessionHandler * s, GridFTPRequestType request_type = GRIDFTP_REQUEST_FTP); virtual ~GridFTPRequestState(); void wait(GQuark scope, time_t timeout = -1); void cancel(GQuark scope, const std::string& msg, int errcode); GridFTPSessionHandler* handler; GridFTPRequestType request_type; globus_mutex_t mutex; globus_cond_t cond; Gfal::CoreException* error; bool done; time_t default_timeout; }; class GridFTPStreamState: public GridFTPRequestState { public: off_t offset; // file offset in the stream globus_size_t buffer_size; bool eof; // end of file reached bool expect_eof; // true for partial reads, false for streamed reads GridFTPStreamState(GridFTPSessionHandler * s); virtual ~GridFTPStreamState(); }; struct GassCopyAttrHandler { GassCopyAttrHandler(globus_ftp_client_operationattr_t* ftp_operation_attr); ~GassCopyAttrHandler(); globus_gass_copy_attr_t attr_gass; globus_ftp_client_operationattr_t operation_attr_ftp_for_gass; gss_cred_id_t cred_id; }; class GridFTPSession { public: GridFTPSession(gfal2_context_t context, const std::string& baseurl); ~GridFTPSession(); std::string baseurl; gss_cred_id_t cred_id; globus_ftp_client_handle_t handle_ftp; globus_ftp_client_plugin_t debug_ftp_plugin; globus_ftp_client_handleattr_t attr_handle; globus_ftp_client_operationattr_t operation_attr_ftp; globus_gass_copy_handle_t gass_handle; globus_gass_copy_handleattr_t gass_handle_attr; globus_ftp_control_dcau_t dcau_control; globus_ftp_client_features_t ftp_features; // options globus_ftp_control_parallelism_t parallelism; globus_ftp_control_mode_t mode; globus_ftp_control_tcpbuffer_t tcp_buffer_size; // client plugins globus_ftp_client_plugin_t pasv_plugin; // pointers to the gfal2 context and transfer params, if relevant gfal2_context_t context; gfalt_params_t params; // Handy option setters void set_gridftpv2(bool v2); void set_ipv6(bool ipv6); void set_udt(bool udt); void set_dcau(bool dcau); void set_delayed_pass(bool enable); void set_nb_streams(unsigned int nbstreams); void set_tcp_buffer_size(guint64 tcp_buffer_size); void set_user_agent(gfal2_context_t context); }; class GridFTPSessionHandler { public: GridFTPSessionHandler(GridFTPFactory* f, const std::string &uri); ~GridFTPSessionHandler(); globus_ftp_client_handle_t* get_ftp_client_handle(); globus_gass_copy_handle_t* get_gass_copy_handle(); globus_ftp_client_operationattr_t* get_ftp_client_operationattr(); globus_gass_copy_handleattr_t* get_gass_copy_handleattr(); globus_ftp_client_handleattr_t* get_ftp_client_handleattr(); globus_ftp_client_features_t* get_ftp_features(); GridFTPFactory* get_factory(); GridFTPSession* session; private: GridFTPFactory* factory; }; class GridFTPFactory { public: GridFTPFactory(gfal2_context_t handle); ~GridFTPFactory(); /** Get a suitable session, new or reused **/ GridFTPSession* get_session(const std::string &url); /** Release the session, and close it if should not be reused **/ void release_session(GridFTPSession* h); gfal2_context_t get_gfal2_context(); private: gfal2_context_t gfal2_context; // session re-use management bool session_reuse; unsigned int size_cache; // session cache std::multimap session_cache; globus_mutex_t mux_cache; void recycle_session(GridFTPSession* sess); void clear_cache(); GridFTPSession* get_recycled_handle(const std::string &baseurl); GridFTPSession* get_new_handle(const std::string &baseurl); }; void globus_ftp_client_done_callback(void * user_arg, globus_ftp_client_handle_t * handle, globus_object_t * error); void globus_gass_client_done_callback( void * callback_arg, globus_gass_copy_handle_t * handle, globus_object_t * error); // do atomic read operation from globus async call ssize_t gridftp_read_stream(GQuark scope, GridFTPStreamState* stream, void* buffer, size_t s_read, bool expect_eof); // do atomic write operation from globus async call ssize_t gridftp_write_stream(GQuark scope, GridFTPStreamState* stream, const void* buffer, size_t s_write, bool eof); // return 0 if no error, or return errno and set the error string properly // error allocation is dynamic int gfal_globus_error_convert(globus_object_t * error, char ** str_error); // throw Glib::Error if error associated with this result void gfal_globus_check_result(GQuark scope, globus_result_t res); void gfal_globus_set_credentials(const char* ucert, const char* ukey, const char *user, const char *passwd, gss_cred_id_t *cred_id, globus_ftp_client_operationattr_t* opattr); // Obtain credentials for the given URL and return the matching prefix std::string gfal_gridftp_get_credentials(gfal2_context_t context, const std::string &url, gchar **ucert, gchar **ukey, gchar **user, gchar **passwd); #endif /* GRIDFTPWRAPPER_H */ gfal2-v2.23.0/src/plugins/http/000077500000000000000000000000001465240014500162075ustar00rootroot00000000000000gfal2-v2.23.0/src/plugins/http/CMakeLists.txt000066400000000000000000000031431465240014500207500ustar00rootroot00000000000000cmake_minimum_required (VERSION 2.6) if (PLUGIN_HTTP) include (CheckLibraryExists) find_package(Davix REQUIRED) find_package(Cryptopp REQUIRED) # Includes include_directories (${CMAKE_CURRENT_BINARY_DIR}) include_directories(${DAVIX_INCLUDE_DIR}) include_directories(${JSONC_INCLUDE_DIRS}) include_directories(${CRYPTOPP_INCLUDE_DIRS}) # definitions add_definitions (${DAVIX_CFLAGS}) # Build plugin file (GLOB src_http "gfal_http_*.cpp") add_library (plugin_http MODULE ${src_http}) add_library (plugin_http_static STATIC ${src_http}) target_link_libraries(plugin_http gfal2 gfal2_transfer ${DAVIX_LIBRARIES} ${JSONC_LIBRARIES} ${CRYPTOPP_LIBRARIES} ) target_link_libraries(plugin_http_static gfal2 gfal2_transfer ${DAVIX_LIBRARIES} ${JSONC_LIBRARIES} ${CRYPTOPP_LIBRARIES} ) set_target_properties(plugin_http PROPERTIES CLEAN_DIRECT_OUTPUT 1 OUTPUT_NAME "gfal_plugin_http" LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/plugins) # Install install(TARGETS plugin_http LIBRARY DESTINATION ${PLUGIN_INSTALL_DIR}) install(FILES "README_PLUGIN_HTTP" DESTINATION ${DOC_INSTALL_DIR}) # install http configuration files LIST(APPEND http_conf_file "${CMAKE_SOURCE_DIR}/dist/etc/gfal2.d/http_plugin.conf") install(FILES ${http_conf_file} DESTINATION ${SYSCONF_INSTALL_DIR}/gfal2.d/) endif (PLUGIN_HTTP) gfal2-v2.23.0/src/plugins/http/README_PLUGIN_HTTP000066400000000000000000000000771465240014500210100ustar00rootroot00000000000000DAV Plugin: - provides backends for HTTP/S and DAV/S servers. gfal2-v2.23.0/src/plugins/http/gfal_http_copy.cpp000066400000000000000000001060751465240014500217260ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2015 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include #include #include #include #include #include #include #include "gfal_http_plugin.h" using CopyMode = HttpCopyMode::CopyMode; struct PerfCallbackData { gfalt_params_t params; std::string source; std::string destination; PerfCallbackData(gfalt_params_t params, const std::string& src, const std::string& dst): params(params), source(src), destination(dst) { } }; static void extract_query_parameter(const char* url, const char *key, char *value, size_t val_size) { value[0] = '\0'; const char *str = strchr(url, '?'); if (str == NULL) { return; } size_t key_len = strlen(key); char **args = g_strsplit(str + 1, "&", 0); int i; for (i = 0; args[i] != NULL; ++i) { if (strncmp(args[i], key, key_len) == 0) { char *p = strchr(args[i], '='); if (p) { g_strlcpy(value, p + 1, val_size); break; } } } g_strfreev(args); } void HttpCopyMode::next() { if (copyMode == CopyMode::PULL) { copyMode = CopyMode::PUSH; } else if (copyMode == CopyMode::PUSH && streamingEnabled) { copyMode = CopyMode::STREAM; } else { copyMode = CopyMode::NONE; } } bool HttpCopyMode::end() const { return (copyMode == CopyMode::NONE); } const char* HttpCopyMode::str() const { return CopyModeToStr(copyMode); } CopyMode HttpCopyMode::CopyModeFromQueryArguments(const char* surl) { char buffer[64] = {0}; extract_query_parameter(surl, "copy_mode", buffer, sizeof(buffer)); if (buffer[0] != '\0') { if (!strcmp(buffer, "pull")) { return CopyMode::PULL; } else if (!strcmp(buffer, "push")) { return CopyMode::PUSH; } } return CopyMode::NONE; } CopyMode HttpCopyMode::CopyModeFromStr(const char* copyModeStr) { if (copyModeStr != NULL) { if (!strcmp(copyModeStr, GFAL_TRANSFER_TYPE_PULL)) { return CopyMode::PULL; } else if (!strcmp(copyModeStr, GFAL_TRANSFER_TYPE_PUSH)) { return CopyMode::PUSH; } else if (!strcmp(copyModeStr, GFAL_TRANSFER_TYPE_STREAMED)) { return CopyMode::STREAM; } } return CopyMode::NONE; } const char* HttpCopyMode::CopyModeToStr(CopyMode copyMode) { if (copyMode == CopyMode::PULL) { return GFAL_TRANSFER_TYPE_PULL; } else if (copyMode == CopyMode::PUSH) { return GFAL_TRANSFER_TYPE_PUSH; } else if (copyMode == CopyMode::STREAM) { return GFAL_TRANSFER_TYPE_STREAMED; } else { return "None"; } } HttpCopyMode HttpCopyMode::ConstructCopyMode(gfal2_context_t context, const char* src, const char* dst) { // If source is not HTTP or ThirdPartyCopy is disabled, go straight to streaming if (!is_http_scheme(src) || !is_http_3rdcopy_enabled(context, src, dst)) { return HttpCopyMode{CopyMode::STREAM, true, true}; } bool streamingEnabled = is_http_streaming_enabled(context, src, dst); CopyMode copyMode = HttpCopyMode::CopyModeFromQueryArguments(src); if (copyMode == CopyMode::NONE) { copyMode = HttpCopyMode::CopyModeFromQueryArguments(dst); } if (copyMode != CopyMode::NONE) { // Disable fallback when copy mode extracted from query arguments GError* error = NULL; gfal2_set_opt_boolean(context, "HTTP PLUGIN", "ENABLE_REMOTE_COPY", TRUE, &error); gfal2_set_opt_boolean(context, "HTTP PLUGIN", "ENABLE_FALLBACK_TPC_COPY", FALSE, &error); gfal2_log(G_LOG_LEVEL_INFO, "Extracted copy mode from query arguments: %s", HttpCopyMode::CopyModeToStr(copyMode)); g_clear_error(&error); return HttpCopyMode{copyMode, (copyMode == CopyMode::STREAM), streamingEnabled}; } copyMode = HttpCopyMode::CopyModeFromStr(get_se_custom_opt_string(context, src, "DEFAULT_COPY_MODE")); if (copyMode == CopyMode::NONE) { copyMode = HttpCopyMode::CopyModeFromStr(get_se_custom_opt_string(context, dst, "DEFAULT_COPY_MODE")); } if (copyMode != CopyMode::NONE) { gfal2_log(G_LOG_LEVEL_INFO, "Using storage specific copy mode configuration: %s", HttpCopyMode::CopyModeToStr(copyMode)); } else { copyMode = HttpCopyMode::CopyModeFromStr(gfal2_get_opt_string_with_default(context, "HTTP PLUGIN", "DEFAULT_COPY_MODE", GFAL_TRANSFER_TYPE_PULL)); if (copyMode == CopyMode::NONE) { copyMode = CopyMode::PULL; gfal2_log(G_LOG_LEVEL_WARNING, "Invalid Gfal2 configuration for 'DEFAULT_COPY_MODE'. Using default copy mode: %s", HttpCopyMode::CopyModeToStr(copyMode)); } } return HttpCopyMode{copyMode, (copyMode == CopyMode::STREAM), streamingEnabled}; }; bool is_http_scheme(const char* url) { const char *schemes[] = {"http:", "https:", "dav:", "davs:", "s3:", "s3s:", "gcloud:", "gclouds:", "swift:", "swifts:", "cs3:", "cs3s:", NULL}; const char *colon = strchr(url, ':'); if (!colon) return false; size_t scheme_len = colon - url + 1; for (size_t i = 0; schemes[i] != NULL; ++i) { if (strncmp(url, schemes[i], scheme_len) == 0) { return true; } } return false; } bool is_http_3rdcopy_enabled(gfal2_context_t context, const char* src, const char* dst) { int src_remote_copy = get_se_custom_opt_boolean(context, src, "ENABLE_REMOTE_COPY"); int dst_remote_copy = get_se_custom_opt_boolean(context, dst, "ENABLE_REMOTE_COPY"); if (src_remote_copy > -1 || dst_remote_copy > -1) { return src_remote_copy && dst_remote_copy; } return gfal2_get_opt_boolean_with_default(context, "HTTP PLUGIN", "ENABLE_REMOTE_COPY", TRUE); } bool is_http_streaming_enabled(gfal2_context_t context, const char* src, const char* dst) { int src_streaming = get_se_custom_opt_boolean(context, src, "ENABLE_STREAM_COPY"); int dst_streaming = get_se_custom_opt_boolean(context, dst, "ENABLE_STREAM_COPY"); if (src_streaming > -1 || dst_streaming > -1) { return src_streaming && dst_streaming; } return gfal2_get_opt_boolean_with_default(context, "HTTP PLUGIN", "ENABLE_STREAM_COPY", TRUE); } bool is_http_3rdcopy_fallback_enabled(gfal2_context_t context, const char* src, const char* dst) { int src_streaming = get_se_custom_opt_boolean(context, src, "ENABLE_FALLBACK_TPC_COPY"); int dst_streaming = get_se_custom_opt_boolean(context, dst, "ENABLE_FALLBACK_TPC_COPY"); if (src_streaming > -1 || dst_streaming > -1) { return src_streaming && dst_streaming; } return gfal2_get_opt_boolean_with_default(context, "HTTP PLUGIN", "ENABLE_FALLBACK_TPC_COPY", TRUE); } static gboolean gfal_http_copy_should_fallback(int error_code) { switch (error_code) { case ECANCELED: case EPERM: case ENOENT: case EACCES: return false; default: return true; } } static int gfal_http_exists(plugin_handle plugin_data, const char* url, GError** err) { GError *nestedError = NULL; int access_stat = gfal_http_access(plugin_data, url, F_OK, &nestedError); if (access_stat == 0) { return 1; } else if (nestedError->code == ENOENT) { g_error_free(nestedError); return 0; } else { gfalt_propagate_prefixed_error(err, nestedError, __func__, "", ""); return -1; } } static int gfal_http_copy_overwrite(plugin_handle plugin_data, gfalt_params_t params, const char *dst, GError** err) { GError *nestedError = NULL; int exists = gfal_http_exists(plugin_data, dst, &nestedError); if (exists > 0) { if (!gfalt_get_replace_existing_file(params,NULL)) { gfalt_set_error(err, http_plugin_domain, EEXIST, __func__, GFALT_ERROR_DESTINATION, GFALT_ERROR_EXISTS, "The destination file exists and overwrite is not enabled"); return -1; } gfal_http_unlinkG(plugin_data, dst, &nestedError); if (nestedError) { gfalt_propagate_prefixed_error(err, nestedError, __func__, GFALT_ERROR_DESTINATION, GFALT_ERROR_OVERWRITE); return -1; } gfal2_log(G_LOG_LEVEL_DEBUG, "File %s deleted with success (overwrite set)", dst); plugin_trigger_event(params, http_plugin_domain, GFAL_EVENT_DESTINATION, GFAL_EVENT_OVERWRITE_DESTINATION, "Deleted %s", dst); } else if (exists == 0) { gfal2_log(G_LOG_LEVEL_DEBUG, "Destination does not exist"); } else if (exists < 0) { gfalt_propagate_prefixed_error(err, nestedError, __func__, GFALT_ERROR_DESTINATION, GFALT_ERROR_OVERWRITE); return -1; } return 0; } static char* gfal_http_get_parent(const char* url) { char *parent = g_strdup(url); char *slash = strrchr(parent, '/'); if (slash) { *slash = '\0'; } else { g_free(parent); parent = NULL; } return parent; } static int gfal_http_copy_make_parent(plugin_handle plugin_data, gfalt_params_t params, gfal2_context_t context, const char* dst, GError** err) { GError *nestedError = NULL; if (!gfalt_get_create_parent_dir(params, NULL)) return 0; char *parent = gfal_http_get_parent(dst); if (!parent) { gfalt_set_error(err, http_plugin_domain, EINVAL, __func__, GFALT_ERROR_DESTINATION, GFALT_ERROR_PARENT, "Could not get the parent directory of %s", dst); return -1; } int exists = gfal_http_exists(plugin_data, parent, &nestedError); // Error if (exists < 0) { gfalt_propagate_prefixed_error(err, nestedError, __func__, GFALT_ERROR_DESTINATION, GFALT_ERROR_PARENT); return -1; } // Does exist else if (exists == 1) { return 0; } // Does not exist else { gfal2_mkdir_rec(context, parent, 0755, &nestedError); if (nestedError) { gfalt_propagate_prefixed_error(err, nestedError, __func__, GFALT_ERROR_DESTINATION, GFALT_ERROR_PARENT); return -1; } gfal2_log(G_LOG_LEVEL_DEBUG, "[%s] Created parent directory %s", __func__, parent); return 0; } } static bool gfal_http_cancellationcopy_callback(void* data) { return gfal2_is_canceled(*static_cast(data)); } static void gfal_http_3rdcopy_perfcallback(const Davix::PerformanceData& perfData, void* data) { PerfCallbackData* pdata = static_cast(data); static int ipevent_sent = false; if (pdata) { _gfalt_transfer_status status; status.average_baudrate = static_cast(perfData.avgTransfer()); status.bytes_transfered = static_cast(perfData.totalTransferred()); status.instant_baudrate = static_cast(perfData.diffTransfer()); status.transfer_time = perfData.absElapsed(); if ((!ipevent_sent) && (perfData.ipflag != Davix::undefined)) { GQuark ipevent = g_quark_from_static_string("IP:UNDEFINED"); if (perfData.ipflag == Davix::IPv6) { ipevent = GFAL_EVENT_IPV6; } else if (perfData.ipflag == Davix::IPv4) { ipevent = GFAL_EVENT_IPV4; } plugin_trigger_event(pdata->params, http_plugin_domain, GFAL_EVENT_DESTINATION, ipevent, "TRUE"); ipevent_sent = true; } plugin_trigger_monitor(pdata->params, &status, pdata->source.c_str(), pdata->destination.c_str()); } } /// Utility function to clean destination file after failed copy operation. /// Clean-up is performed if the copy error is something else than EEXIST /// Triggers an event when attempts to unlink destination file. /// If unlink fails reports in the event the correspondent errno. /// If unlink succeeds or if it fails because the file does not exist, the event reports 0 as the status code. static void gfal_http_copy_cleanup(plugin_handle plugin_data, const gfalt_params_t& params, const char* dst, GError** err) { GError *unlink_err = NULL; if ((*err)->code != EEXIST) { int status = 0; if (gfal_http_unlinkG(plugin_data, dst, &unlink_err) != 0) { if (unlink_err->code != ENOENT) { gfal2_log(G_LOG_LEVEL_WARNING, "When trying to clean the destination: %s", unlink_err->message); status = unlink_err->code; } g_error_free(unlink_err); } else { gfal2_log(G_LOG_LEVEL_DEBUG, "Destination file removed"); } plugin_trigger_event(params, http_plugin_domain, GFAL_EVENT_DESTINATION, GFAL_EVENT_CLEANUP, "%d", status); } else { gfal2_log(G_LOG_LEVEL_DEBUG, "The transfer failed because the file exists. Do not clean!"); } } // Only http or https // See DMC-473 and DMC-1010 static std::string get_canonical_uri(const std::string& original) { std::string scheme; char last_scheme; if ((original.compare(0, 2, "s3") == 0) || (original.compare(0, 6, "gcloud") == 0 ) || (original.compare(0, 5, "swift") == 0) || (original.compare(0, 3, "cs3") == 0)) { return original; } size_t plus_pos = original.find('+'); size_t colon_pos = original.find(':'); if (plus_pos < colon_pos) last_scheme = original[plus_pos - 1]; else last_scheme = original[colon_pos - 1]; if (last_scheme == 's') scheme = "https"; else scheme = "http"; std::string canonical = (scheme + original.substr(colon_pos)); return canonical; } struct HttpTransferHosts { std::string sourceHost; std::string destHost; }; static void set_archive_metadata_header(Davix::RequestParams& req_params, CopyMode mode, const std::string& metadata) { std::string encoded_metadata; // Base64 encode the header const bool noNewLineInBase64Output = false; CryptoPP::StringSource ss1(metadata, true, new CryptoPP::Base64Encoder( new CryptoPP::StringSink(encoded_metadata), noNewLineInBase64Output)); if (mode == CopyMode::PUSH) { req_params.addHeader("TransferHeaderArchiveMetadata", encoded_metadata); } else { req_params.addHeader("ArchiveMetadata", encoded_metadata); } } static int gfal_http_third_party_copy(gfal2_context_t context, GfalHttpPluginData* davix, const char* src, const char* dst, CopyMode mode, gfalt_params_t params, HttpTransferHosts& transferHosts, GError** err) { gfal2_log(G_LOG_LEVEL_MESSAGE, "Performing a HTTP third party copy"); PerfCallbackData perfCallbackData( params, src, dst ); Davix::RequestParams req_params; davix->get_tpc_params(&req_params, Davix::Uri(src), Davix::Uri(dst), params, mode == CopyMode::PUSH); if (mode == CopyMode::PUSH) { req_params.setCopyMode(Davix::CopyMode::Push); } else if (mode == CopyMode::PULL) { req_params.setCopyMode(Davix::CopyMode::Pull); } else { gfal2_set_error(err, http_plugin_domain, EIO, __func__, "gfal_http_third_party_copy invalid copy mode"); return -1; } // dCache requires RequireChecksumVerification to be explicitly false if no checksums // are to be used gfalt_checksum_mode_t checksum_mode = gfalt_get_checksum_mode(params, err); if (*err) { g_clear_error(err); } else { if (mode == CopyMode::PUSH) { if ((checksum_mode & GFALT_CHECKSUM_SOURCE) || (checksum_mode == GFALT_CHECKSUM_NONE)) { req_params.addHeader("RequireChecksumVerification", "false"); } } else if (mode == CopyMode::PULL) { if ((checksum_mode & GFALT_CHECKSUM_TARGET) || (checksum_mode == GFALT_CHECKSUM_NONE)) { req_params.addHeader("RequireChecksumVerification", "false"); } } } // Set archive metadata header const char* const metadata = gfalt_get_archive_metadata(params, NULL); if (metadata != NULL && metadata[0] != '\0') { set_archive_metadata_header(req_params, mode, metadata); } // Set SciTag header guint scitag = gfalt_get_scitag(params, NULL); if (scitag != 0) { req_params.addHeader("SciTag", std::to_string(scitag)); } // add timeout struct timespec opTimeout; opTimeout.tv_sec = gfalt_get_timeout(params, NULL); req_params.setOperationTimeout(&opTimeout); // DMC-1348: Use resolved URLs for data operations std::string resolved_src = davix->resolved_url(src); std::string resolved_dst = davix->resolved_url(dst); Davix::Uri src_uri(get_canonical_uri(resolved_src)); Davix::Uri dst_uri(get_canonical_uri(resolved_dst)); Davix::DavixCopy copy(davix->context, &req_params); copy.setPerformanceCallback(gfal_http_3rdcopy_perfcallback, &perfCallbackData); copy.setCancellationCallback(gfal_http_cancellationcopy_callback, &context); Davix::DavixError* davError = NULL; copy.copy(src_uri, dst_uri, gfalt_get_nbstreams(params, NULL), &davError); if (davError != NULL) { davix2gliberr(davError, err, __func__); Davix::DavixError::clearError(&davError); } else { auto _transferHost = [](const std::string& initialHost, const std::string& transferHost) -> std::string { if (initialHost == transferHost) { return ""; } return transferHost; }; transferHosts.sourceHost = _transferHost(src_uri.getHost(), copy.getTransferSourceHost()); transferHosts.destHost = _transferHost(dst_uri.getHost(), copy.getTransferDestinationHost()); } return *err == NULL ? 0 : -1; } struct HttpStreamProvider { const char *source, *destination; gfal2_context_t context; gfalt_params_t params; int source_fd; time_t start, last_update; dav_ssize_t read_instant; _gfalt_transfer_status perf; GError* stream_err; HttpStreamProvider(const char *source, const char *destination, gfal2_context_t context, int source_fd, gfalt_params_t params) : source(source), destination(destination), context(context), params(params), source_fd(source_fd), start(time(NULL)), last_update(start), read_instant(0), stream_err(NULL) { memset(&perf, 0, sizeof(perf)); } }; static dav_ssize_t gfal_http_streamed_provider(void *userdata, void *buffer, dav_size_t buflen) { GError* error = NULL; HttpStreamProvider* data = static_cast(userdata); dav_ssize_t ret = 0; time_t now = time(NULL); if (buflen == 0) { data->perf.bytes_transfered = data->read_instant = 0; data->perf.average_baudrate = 0; data->perf.instant_baudrate = 0; data->start = data->last_update = now; if (gfal2_lseek(data->context, data->source_fd, 0, SEEK_SET, &error) < 0) ret = -1; } else { ret = gfal2_read(data->context, data->source_fd, buffer, buflen, &error); if (ret > 0) data->read_instant += ret; if (now - data->last_update >= 5) { data->perf.bytes_transfered += data->read_instant; data->perf.transfer_time = now - data->start; data->perf.average_baudrate = data->perf.bytes_transfered / data->perf.transfer_time; data->perf.instant_baudrate = data->read_instant / (now - data->last_update); data->last_update = now; data->read_instant = 0; plugin_trigger_monitor(data->params, &data->perf, data->source, data->destination); } } if (error) { gfal2_propagate_prefixed_error(&data->stream_err, error, __func__); } return ret; } static int gfal_http_streamed_copy(gfal2_context_t context, GfalHttpPluginData* davix, const char* src, const char* dst, gfalt_checksum_mode_t checksum_mode, const char *checksum_type, const char *user_checksum, gfalt_params_t params, GError** err) { gfal2_log(G_LOG_LEVEL_MESSAGE, "Performing a HTTP streamed copy"); GError *nested_err = NULL; struct stat src_stat; if (gfal2_stat(context, src, &src_stat, &nested_err) != 0) { gfal2_propagate_prefixed_error(err, nested_err, __func__); return -1; } // Must reset the HTTP OPERATION_TIMEOUT to the transfer timeout bool reset_operation_timeout = is_http_scheme(src); int transfer_timeout = static_cast(gfalt_get_timeout(params, NULL)); int previous_timeout = 0; if (reset_operation_timeout) { previous_timeout = davix->get_operation_timeout(); davix->set_operation_timeout(transfer_timeout); gfal2_log(G_LOG_LEVEL_DEBUG, "Source HTTP Open transfer timeout=%d", transfer_timeout); } int source_fd = gfal2_open(context, src, O_RDONLY, &nested_err); if (reset_operation_timeout) { davix->set_operation_timeout(previous_timeout); } if (source_fd < 0) { gfal2_propagate_prefixed_error(err, nested_err, __func__); return -1; } Davix::Uri dst_uri(dst); Davix::RequestParams req_params; davix->get_params(&req_params, dst_uri, GfalHttpPluginData::OP::WRITE); // Add timeout struct timespec opTimeout{transfer_timeout}; req_params.setOperationTimeout(&opTimeout); // Set MD5 header on the PUT if (checksum_mode & GFALT_CHECKSUM_TARGET && strcasecmp(checksum_type, "md5") == 0 && user_checksum[0]) { req_params.addHeader("Content-MD5", user_checksum); } // Set archive metadata header const char* const metadata = gfalt_get_archive_metadata(params, NULL); if (metadata != NULL && metadata[0] != '\0') { set_archive_metadata_header(req_params, CopyMode::STREAM, metadata); } if (dst_uri.getProtocol() == "s3" || dst_uri.getProtocol() == "s3s") req_params.setProtocol(Davix::RequestProtocol::AwsS3); else if (dst_uri.getProtocol() == "gcloud" || dst_uri.getProtocol() == "gclouds") req_params.setProtocol(Davix::RequestProtocol::Gcloud); else if (dst_uri.getProtocol() == "swift" || dst_uri.getProtocol() == "swifts") req_params.setProtocol(Davix::RequestProtocol::Swift); else if (dst_uri.getProtocol() == "cs3" || dst_uri.getProtocol() == "cs3s") req_params.setProtocol(Davix::RequestProtocol::CS3); // DMC-1348: Use resolved URLs for data operations std::string resolved_dst = davix->resolved_url(dst); Davix::DavFile dest(davix->context, req_params, resolved_dst); HttpStreamProvider provider(src, dst, context, source_fd, params); try { dest.put(&req_params, std::bind(&gfal_http_streamed_provider,&provider, std::placeholders::_1, std::placeholders::_2), src_stat.st_size); } catch (Davix::DavixException& ex) { GError* tmp_err = NULL; // Propagate the source error first, then the destination error if (provider.stream_err != NULL) { tmp_err = provider.stream_err; } else { tmp_err = g_error_new(http_plugin_domain, ex.code(), "%s", ex.what()); } gfal2_set_error(err, http_plugin_domain, tmp_err->code, __func__, "%s (%s)", tmp_err->message, (provider.stream_err) ? "source" : "destination"); g_clear_error(&tmp_err); } gfal2_close(context, source_fd, &nested_err); // Throw away this error if (nested_err) g_error_free(nested_err); return *err == NULL ? 0 : -1; } void strip_3rd_from_url(const char* url_full, char* url, size_t url_size) { const char* colon = strchr(url_full, ':'); const char* plus = strchr(url_full, '+'); if (!plus || !colon || plus > colon) { g_strlcpy(url, url_full, url_size); } else { size_t len = plus - url_full + 1; if (len >= url_size) len = url_size; g_strlcpy(url, url_full, len); g_strlcat(url, colon, url_size); gfal2_log(G_LOG_LEVEL_WARNING, "+3rd schemes deprecated"); } } int gfal_http_copy(plugin_handle plugin_data, gfal2_context_t context, gfalt_params_t params, const char* src_full, const char* dst_full, GError** err) { GError* nested_error = NULL; GfalHttpPluginData* davix = gfal_http_get_plugin_context(plugin_data); plugin_trigger_event(params, http_plugin_domain, GFAL_EVENT_NONE, GFAL_EVENT_PREPARE_ENTER, "%s => %s", src_full, dst_full); // Determine if urls are 3rd party, and strip the +3rd if they are char src[GFAL_URL_MAX_LEN], dst[GFAL_URL_MAX_LEN]; strip_3rd_from_url(src_full, src, sizeof(src)); strip_3rd_from_url(dst_full, dst, sizeof(dst)); // Resolve DNS alias for later usage in the HTTP copy davix->resolve_and_store_url(src); davix->resolve_and_store_url(dst); gfal2_log(G_LOG_LEVEL_INFO, "Will use copy source: %s", davix->resolved_url(src).c_str()); gfal2_log(G_LOG_LEVEL_INFO, "Will use copy destination: %s", davix->resolved_url(dst).c_str()); // Get user defined checksum gfalt_checksum_mode_t checksum_mode = GFALT_CHECKSUM_NONE; char checksum_type[1024] = {0}; char user_checksum[1024] = {0}; char src_checksum[1024] = {0}; if (!gfalt_get_strict_copy_mode(params, NULL)) { checksum_mode = gfalt_get_checksum(params, checksum_type, sizeof(checksum_type), user_checksum, sizeof(user_checksum), NULL); } // Source checksum if (checksum_mode & GFALT_CHECKSUM_SOURCE) { plugin_trigger_event(params, http_plugin_domain, GFAL_EVENT_SOURCE, GFAL_EVENT_CHECKSUM_ENTER, ""); gfal2_checksum(context, src, checksum_type, 0, 0, src_checksum, sizeof(src_checksum), &nested_error); if (nested_error) { if (nested_error->code == ENOSYS || nested_error->code == ENOTSUP) { gfal2_log(G_LOG_LEVEL_WARNING, "Checksum type %s not supported by source. Skip source check.", checksum_type); g_clear_error (&nested_error); src_checksum[0] = '\0'; } else { gfalt_propagate_prefixed_error(err, nested_error, __func__, GFALT_ERROR_SOURCE, GFALT_ERROR_CHECKSUM); return -1; } } else if (user_checksum[0]) { if (gfal_compare_checksums(src_checksum, user_checksum, sizeof(src_checksum)) != 0) { gfalt_set_error(err, http_plugin_domain, EIO, __func__, GFALT_ERROR_SOURCE, GFALT_ERROR_CHECKSUM_MISMATCH, "Source and user-defined %s do not match (%s != %s)", checksum_type, src_checksum, user_checksum); return -1; } } plugin_trigger_event(params, http_plugin_domain, GFAL_EVENT_SOURCE, GFAL_EVENT_CHECKSUM_EXIT, ""); } // When this flag is not set, the plugin should handle overwriting, // parent directory creation,... if (!gfalt_get_strict_copy_mode(params, NULL)) { if (gfal_http_copy_overwrite(plugin_data, params, dst, &nested_error) != 0 || gfal_http_copy_make_parent(plugin_data, params, context, dst, &nested_error) != 0) { gfal2_propagate_prefixed_error(err, nested_error, __func__); return -1; } } plugin_trigger_event(params, http_plugin_domain, GFAL_EVENT_NONE, GFAL_EVENT_PREPARE_EXIT, "%s => %s", src_full, dst_full); int ret = 0; std::list attempted_mode; HttpTransferHosts transferHosts; HttpCopyMode copyMode = HttpCopyMode::ConstructCopyMode(context, src, dst); plugin_trigger_event(params, http_plugin_domain, GFAL_EVENT_NONE, GFAL_EVENT_TRANSFER_ENTER, "%s => %s", src_full, dst_full); do { // Perform the copy, going through the different fallback copy modes gfal2_log(G_LOG_LEVEL_MESSAGE, "Trying copying with mode %s", copyMode.str()); plugin_trigger_event(params, http_plugin_domain, GFAL_EVENT_NONE, GFAL_EVENT_TRANSFER_TYPE, "%s", copyMode.str()); g_clear_error(&nested_error); if (copyMode.value() == CopyMode::STREAM) { if (copyMode.isStreamingEnabled()) { ret = gfal_http_streamed_copy(context, davix, src, dst, checksum_mode, checksum_type, user_checksum, params, &nested_error); } else if (copyMode.isStreamingOnly()) { gfal2_set_error(&nested_error, http_plugin_domain, EINVAL, __func__, "STREAMED DISABLED Only streamed copy possible but streaming is disabled"); gfal2_log(G_LOG_LEVEL_WARNING, "%s", nested_error->message); ret = -1; break; } } else { ret = gfal_http_third_party_copy(context, davix, src, dst, copyMode.value(), params, transferHosts, &nested_error); } if (ret == 0) { gfal2_log(G_LOG_LEVEL_MESSAGE, "Copy succeeded using mode %s", copyMode.str()); break; } else { gfal2_log(G_LOG_LEVEL_WARNING, "Copy failed with mode %s: %s", copyMode.str(), nested_error->message); // Delete any potential destination file if (gfalt_get_transfer_cleanup(params, &nested_error)) { gfal_http_copy_cleanup(plugin_data, params, dst, &nested_error); } else { gfal2_log(G_LOG_LEVEL_INFO, "Gfal http copy clean-up disabled"); } } attempted_mode.emplace_back(copyMode.str()); copyMode.next(); } while (!copyMode.end() && is_http_3rdcopy_fallback_enabled(context, src, dst) && gfal_http_copy_should_fallback(nested_error->code)); if (ret == 0) { std::ostringstream msg; msg << src; if (!transferHosts.sourceHost.empty()) { msg << " (" << transferHosts.sourceHost << ")"; } msg << " => " << dst; if (!transferHosts.destHost.empty()) { msg << " (" << transferHosts.destHost << ")"; } plugin_trigger_event(params, http_plugin_domain, GFAL_EVENT_NONE,GFAL_EVENT_TRANSFER_EXIT, "%s", msg.str().c_str()); } else { if (!attempted_mode.empty()) { std::ostringstream errmsg; errmsg << "Copy failed ("; for (auto mode = attempted_mode.begin(); mode != attempted_mode.end(); mode++) { if (mode != attempted_mode.begin()) { errmsg << ", "; } errmsg << (*mode); } errmsg << "). Last attempt: "; g_prefix_error(&nested_error, "ERROR: %s", errmsg.str().c_str()); } plugin_trigger_event(params, http_plugin_domain, GFAL_EVENT_NONE, GFAL_EVENT_TRANSFER_EXIT, "%s", nested_error->message); gfalt_propagate_prefixed_error(err, nested_error, __func__, GFALT_ERROR_TRANSFER, NULL); return -1; } // Destination checksum validation if (checksum_mode & GFALT_CHECKSUM_TARGET) { char dst_checksum[1024]; plugin_trigger_event(params, http_plugin_domain, GFAL_EVENT_DESTINATION, GFAL_EVENT_CHECKSUM_ENTER, ""); gfal2_checksum(context, dst, checksum_type, 0, 0, dst_checksum, sizeof(dst_checksum), &nested_error); if (nested_error) { gfalt_propagate_prefixed_error(err, nested_error, __func__, GFALT_ERROR_DESTINATION, GFALT_ERROR_CHECKSUM); return -1; } if (src_checksum[0]) { if (gfal_compare_checksums(src_checksum, dst_checksum, sizeof(src_checksum)) != 0) { gfalt_set_error(err, http_plugin_domain, EIO, __func__, GFALT_ERROR_DESTINATION, GFALT_ERROR_CHECKSUM_MISMATCH, "Source and destination %s do not match (%s != %s)", checksum_type, src_checksum, dst_checksum); return -1; } } else if (user_checksum[0]) { if (gfal_compare_checksums(user_checksum, dst_checksum, sizeof(user_checksum)) != 0) { gfalt_set_error(err, http_plugin_domain, EIO, __func__, GFALT_ERROR_DESTINATION, GFALT_ERROR_CHECKSUM_MISMATCH, "User-defined and destination %s do not match (%s != %s)", checksum_type, user_checksum, dst_checksum); return -1; } } plugin_trigger_event(params, http_plugin_domain, GFAL_EVENT_DESTINATION, GFAL_EVENT_CHECKSUM_EXIT, ""); } // Evict source file if configured if (gfalt_get_use_evict(params, NULL)) { GError* tmp_err; const char* request_id = gfalt_get_stage_request_id(params, NULL); ret = gfal_http_release_file(plugin_data, src, request_id, &tmp_err); gfal2_log(G_LOG_LEVEL_INFO, "Eviction request exited with status code: %d", ret); if (ret < 0) { gfal2_log(G_LOG_LEVEL_INFO, "Eviction request failed: %s", tmp_err->message); g_error_free(tmp_err); } plugin_trigger_event(params, http_plugin_domain, GFAL_EVENT_SOURCE, GFAL_EVENT_EVICT, "%d", ret); } return 0; } int gfal_http_copy_check(plugin_handle plugin_data, gfal2_context_t context, const char* src, const char* dst, gfal_url2_check check) { if (check != GFAL_FILE_COPY) return 0; // This plugin handles everything that writes into an http endpoint // It will try to decide if it is better to do a third party copy, or a streamed copy later on return (is_http_scheme(dst) && ((strncmp(src, "file://", 7) == 0) || is_http_scheme(src))); } gfal2-v2.23.0/src/plugins/http/gfal_http_plugin.cpp000066400000000000000000001310051465240014500222410ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2015 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "gfal_http_plugin.h" #include "gfal_http_plugin_token.h" #include "uri/gfal2_parsing.h" #include "network/gfal2_network.h" #include #include #include #include #include #include #include #include using namespace Davix; static const char* http_module_name = "http_plugin"; GQuark http_plugin_domain = g_quark_from_static_string(http_module_name); const char* gfal_http_get_name(void) { return GFAL2_PLUGIN_VERSIONED("http", VERSION);; } // The active storage in TPC needs gridsite delegation (avoid if not using TLS) static bool delegation_required(const Davix::Uri& uri) { bool needs_delegation = false; if ((uri.getProtocol().compare(0, 5, "https") == 0) || (uri.getProtocol().compare(0, 4, "davs") == 0)) { needs_delegation = true; } return needs_delegation; } static int get_corresponding_davix_log_level() { int davix_log_level = DAVIX_LOG_CRITICAL; GLogLevelFlags gfal2_log_level = gfal2_log_get_level(); if (gfal2_log_level & G_LOG_LEVEL_DEBUG) davix_log_level = DAVIX_LOG_TRACE; else if (gfal2_log_level & G_LOG_LEVEL_INFO) davix_log_level = DAVIX_LOG_VERBOSE; return davix_log_level; } static std::string construct_config_group_from_url(const char* surl) { Davix::Uri uri(surl); if (uri.getStatus() != Davix::StatusCode::OK) { return ""; } std::string prot = uri.getProtocol(); if (prot.back() == 's') { prot.pop_back(); } std::string group = prot + ":" + uri.getHost(); std::transform(group.begin(), group.end(), group.begin(), ::toupper); return group; } /// Get custom Storage Element configuration option (boolean value) int get_se_custom_opt_boolean(const gfal2_context_t& context, const char* surl, const char* key) { std::string group = construct_config_group_from_url(surl); if (group.empty()) { return -1; } GError* error = NULL; gboolean value = gfal2_get_opt_boolean(context, group.c_str(), key, &error); if (error != NULL) { g_error_free(error); return -1; } return value; } /// Get custom Storage Element configuration option (string value) char* get_se_custom_opt_string(const gfal2_context_t& context, const char* surl, const char* key) { std::string group = construct_config_group_from_url(surl); if (group.empty()) { return NULL; } GError* error = NULL; gchar* value = gfal2_get_opt_string(context, group.c_str(), key, &error); if (error != NULL) { g_error_free(error); return NULL; } return value; } /// Get custom HTTP headers for Storage Element group char** get_se_custom_headers_list(const gfal2_context_t& context, const Davix::Uri& uri) { if (uri.getStatus() != Davix::StatusCode::OK) { return NULL; } std::string prot = uri.getProtocol(); if (prot.back() == 's') { prot.pop_back(); } std::string group = prot + ":" + uri.getHost(); std::transform(group.begin(), group.end(), group.begin(), ::toupper); gsize headers_length = 0; char** headers = gfal2_get_opt_string_list_with_default(context, group.c_str(), "HEADERS", &headers_length, NULL); if (headers == NULL) { headers = gfal2_get_opt_string_list_with_default(context, "HTTP PLUGIN", "HEADERS", &headers_length, NULL); } return headers; } /// Specific function for the retrieve-bearer-token configuration option bool get_retrieve_bearer_token_config(const gfal2_context_t& context, const char* surl, bool default_value) { int retrieve_token = get_se_custom_opt_boolean(context, surl, "RETRIEVE_BEARER_TOKEN"); if (retrieve_token == -1) { return gfal2_get_opt_boolean_with_default(context, "HTTP PLUGIN", "RETRIEVE_BEARER_TOKEN", default_value); } return static_cast(retrieve_token); } static bool isCloudStorage(const Davix::Uri& uri) { return (uri.getProtocol().rfind("s3", 0) == 0 || uri.getProtocol().rfind("gcloud", 0) == 0 || uri.getProtocol().rfind("swift", 0) == 0 || uri.getProtocol().rfind("cs3", 0) == 0); } static bool allowsBearerTokenRetrieve(const Davix::Uri& uri, const GfalHttpPluginData::OP& operation) { return ((uri.getProtocol().rfind("https", 0) == 0) || (uri.getProtocol().rfind("davs", 0) == 0)) && (operation != GfalHttpPluginData::OP::TAPE); } bool GfalHttpPluginData::writeFlagFromOperation(const OP& operation) { return (operation == OP::MKCOL) || (operation == OP::WRITE) || (operation == OP::WRITE_PASV); } bool GfalHttpPluginData::searchFlagFromOperation(const OP& operation) { return (operation == OP::MKCOL) || (operation == OP::HEAD); } bool GfalHttpPluginData::needsTransferHeader(const OP& operation) { return (operation == OP::READ_PASV) || (operation == OP::WRITE_PASV); } static bool isS3SignedURL(const Davix::Uri& url) { return (url.queryParamExists("AWSAccessKeyId") && url.queryParamExists("Signature")) || (url.queryParamExists("X-Amz-Credential") && url.queryParamExists("X-Amz-Signature")); } // Retrieve x509 certificate pair from credential mapping/config static bool gfal_http_get_x509_cert_pair(gfal2_context_t handle, const Davix::Uri& uri, std::string& cert, std::string& key) { bool certificate_pair = false; GError* error = NULL; // Extract certificate from credential map to give priority to user-set pair gchar *cert_p = gfal2_cred_get(handle, GFAL_CRED_X509_CERT, uri.getString().c_str(), NULL, &error); g_clear_error(&error); gchar *key_p = gfal2_cred_get(handle, GFAL_CRED_X509_KEY, uri.getString().c_str(), NULL, &error); g_clear_error(&error); if (cert_p) { cert.assign(cert_p); key = (key_p != NULL) ? std::string(key_p) : cert; certificate_pair = true; } g_free(cert_p); g_free(key_p); return certificate_pair; } char* GfalHttpPluginData::find_se_token(const Davix::Uri& uri, const OP& operation) { using credTuple = std::pair; bool write_access = writeFlagFromOperation(operation); bool extended_search = searchFlagFromOperation(operation); // Helper function to find a token in the Gfal HTTP internal token map auto find_in_token_map = [&](const char* token, const char* token_path, bool write_access) -> bool { auto it = token_map.find(token); if (it == token_map.end()) { gfal2_log(G_LOG_LEVEL_DEBUG, "(SEToken) Retrieved token not in token access map (path=%s) (assuming user-set)", token_path); return true; } if (it->second || (write_access == it->second)) { gfal2_log(G_LOG_LEVEL_DEBUG, "(SEToken) Found token in credential_map[%s] (access=%s) (needed=%s)", token_path, it->second ? "write" : "read", write_access ? "write" : "read"); return true; } return false; }; // Helper function to extract all credentials of type "BEARER" auto cred_map_callback = [](const char* url_prefix, const gfal2_cred_t* cred, void* user_data) { auto cred_list = static_cast*>(user_data); if (strcmp(cred->type, GFAL_CRED_BEARER) == 0) { cred_list->emplace_back(url_prefix, cred->value); } }; // Helper function to check whether a path contains the search path auto path_contains = [](const std::string& path, const std::string& search) -> bool { size_t pos = path.find(search); if (pos == 0) { return (path.size() == search.size()) || (path.at(search.size() - 1) == '/') || (path.at(search.size()) == '/'); } return false; }; std::list cred_list; gfal2_cred_foreach(handle, cred_map_callback, &cred_list); for (auto cred = cred_list.begin(); cred != cred_list.end(); cred++) { if (path_contains(uri.getString(), cred->first) || (extended_search && path_contains(cred->first, uri.getString()))) { if (find_in_token_map(cred->second.c_str(), cred->first.c_str(), write_access)) { return g_strdup(cred->second.c_str()); } } } // Search token for the full host (backwards compatibility with FTS) GError* error = NULL; char* token = gfal2_cred_get(handle, GFAL_CRED_BEARER, uri.getHost().c_str(), NULL, &error); g_clear_error(&error); return token; } char* GfalHttpPluginData::retrieve_and_store_se_token(const Davix::Uri& uri, const OP& operation, unsigned validity) { bool retrieve_token = get_retrieve_bearer_token_config(handle, uri.getString().c_str(), false); GError* error = NULL; char* token = NULL; if (!retrieve_token || !allowsBearerTokenRetrieve(uri, operation)) { return NULL; } Davix::RequestParams params = reference_params; get_params_internal(params, uri); get_certificate(params, uri); bool write_access = writeFlagFromOperation(operation); // Get storage server std::stringstream endpoint; endpoint << uri.getProtocol() << "://" << uri.getHost(); if (uri.getPort()) { endpoint << ":" << uri.getPort(); } std::string storage = endpoint.str(); std::unique_ptr retriever_chain; retriever_chain.reset(new MacaroonRetriever()); retriever_chain->add(new MacaroonRetriever(storage)); TokenRetriever* retriever = retriever_chain.get(); while (retriever != NULL) { try { gfal_http_token_t http_token = retriever->retrieve_token(uri, params, write_access, validity); token = strdup(http_token.token.c_str()); break; } catch (const Gfal::CoreException& e) { gfal2_log(G_LOG_LEVEL_INFO, "(SEToken) Error during token retrieval: %s", e.what()); retriever = retriever->next(); } } if (!token) { gfal2_log(G_LOG_LEVEL_WARNING, "(SEToken) Could not retrieve any token for %s", uri.getString().c_str()); return NULL; } // Tokens are treated as opaque, therefor they are cached in the TokenAccessMap // together with write access and validity info gfal2_cred_t* token_cred = gfal2_cred_new(GFAL_CRED_BEARER, token); if (gfal2_cred_set(handle, uri.getString().c_str(), token_cred, &error) < 0) { gfal2_log(G_LOG_LEVEL_DEBUG, "(SEToken) Failed to set bearer token in credential_map[%s] due to error: %s", uri.getString().c_str(), error->message); g_clear_error(&error); } else { gfal2_log(G_LOG_LEVEL_DEBUG, "(SEToken) Set bearer token in credential_map[%s] (access=%s) (validity=%u)", uri.getString().c_str(), write_access ? "write" : "read" , validity); token_map[token] = write_access; } gfal2_cred_free(token_cred); return token; } GfalHttpPluginData::tape_endpoint_info_t GfalHttpPluginData::retrieve_and_store_tape_endpoint(const std::string& endpoint, GError** err) { // Construct and send "GET /.well-known/wlcg-tape-rest-api" request std::string config_endpoint = endpoint + "/.well-known/wlcg-tape-rest-api"; Davix::DavixError* reqerr = NULL; Davix::Uri config_uri(config_endpoint); Davix::RequestParams params; GetRequest request(context, config_uri, &reqerr); get_params(¶ms, config_uri, GfalHttpPluginData::OP::TAPE); request.setParameters(params); if (request.executeRequest(&reqerr)) { gfal2_set_error(err, http_plugin_domain, davix2errno(reqerr->getStatus()), __func__, "[Tape REST API] Failed to query /.well-known/wlcg-tape-rest-api: %s", reqerr->getErrMsg().c_str()); return tape_endpoint_info{}; } if (request.getRequestCode() != 200) { gfal2_set_error(err, http_plugin_domain, EINVAL, __func__, "[Tape REST API] Failed to query /.well-known/wlcg-tape-rest-api: %s: %s", reqerr->getErrMsg().c_str(), request.getAnswerContent()); return tape_endpoint_info{}; } struct json_object* json_response = json_tokener_parse(request.getAnswerContent()); if (!json_response) { gfal2_set_error(err, http_plugin_domain, ENOMSG, __func__, "[Tape REST API] Malformed served response from /.well-known/wlcg-tape-rest-api"); return tape_endpoint_info{}; } // Check if "sitename" attribute exists struct json_object* sitename_obj = 0; bool foundSitename = json_object_object_get_ex(json_response, "sitename", &sitename_obj); if (!foundSitename) { gfal2_set_error(err, http_plugin_domain, ENOMSG, __func__, "[Tape REST API] No sitename in response from /.well-known/wlcg-tape-rest-api"); return tape_endpoint_info{}; } std::string sitename = json_object_get_string(sitename_obj); // Check if "endpoints" attribute exists struct json_object* endpoints = 0; bool foundEndpoints = json_object_object_get_ex(json_response, "endpoints", &endpoints); if (!foundEndpoints) { gfal2_set_error(err, http_plugin_domain, ENOMSG, __func__, "[Tape REST API] No endpoints in response from /.well-known/wlcg-tape-rest-api"); return tape_endpoint_info{}; } // Helper function to parse version field from /.well-known endpoint // Expects version in the format v0,v1 etc. Returns -1 if it fails to find the correct version number auto parseVersion = [](std::string version) { std::transform(version.begin(), version.end(), version.begin(), tolower); if (!version.empty() && version[0] == 'v') { version.erase(0, 1); } if (!version.empty() && !isdigit(version[0])) { return -1; } return std::atoi(version.c_str()); }; // Iterate over the endpoints list and find v0 or v1 const int len = json_object_array_length(endpoints); int maxVersion = 0; std::string tape_endpoint_uri; std::string tape_endpoint_version; for (int i = 0; i < len; ++i) { json_object *endpoint_obj = json_object_array_get_idx(endpoints, i); if (endpoint_obj == NULL) { continue; } // Check if "version" attribute exists struct json_object* version_obj = 0; bool foundVersion = json_object_object_get_ex(endpoint_obj, "version", &version_obj); if (foundVersion) { std::string version_str = json_object_get_string(version_obj); int parsedVersion = parseVersion(version_str); // Check if "uri" attribute exists struct json_object *uri_obj = 0; bool foundUri = json_object_object_get_ex(endpoint_obj, "uri", &uri_obj); if (foundUri) { if (parsedVersion >= maxVersion && parsedVersion <= 1) { tape_endpoint_uri = json_object_get_string(uri_obj); tape_endpoint_version = version_str; maxVersion = parsedVersion; } } } } // Free the JSON object json_object_put(json_response); if (tape_endpoint_uri.empty()) { gfal2_set_error(err, http_plugin_domain, ENOMSG, __func__, "[Tape REST API] Failed to find v0 or v1 metadata endpoint in response" " from /.well-known/wlcg-tape-rest-api"); return tape_endpoint_info{}; } tape_endpoint_map[endpoint] = tape_endpoint_info{sitename, tape_endpoint_uri, tape_endpoint_version}; return tape_endpoint_map[endpoint]; } std::string gfal_http_discover_tape_endpoint(GfalHttpPluginData* davix, const char* url, const char* method, GError** err) { Davix::Uri uri(url); if (uri.getStatus() != StatusCode::OK) { gfal2_set_error(err, http_plugin_domain, EINVAL, __func__, "Invalid URL: %s", url); return NULL; } // Construct remote storage endpoint std::stringstream endpoint; endpoint << uri.getProtocol() << "://" << uri.getHost(); if (uri.getPort()) { endpoint << ":" << uri.getPort(); } auto it = davix->tape_endpoint_map.find(endpoint.str()); if (it == davix->tape_endpoint_map.end()) { davix->retrieve_and_store_tape_endpoint(endpoint.str(), err); if (*err != NULL) { return ""; } it = davix->tape_endpoint_map.find(endpoint.str()); } std::stringstream tape_endpoint; tape_endpoint << it->second.uri; if (tape_endpoint.str().back() != '/') { tape_endpoint << "/"; } if (method[0] == '/') { tape_endpoint.seekp(-1, std::ios_base::end); } tape_endpoint << method; return tape_endpoint.str(); } bool GfalHttpPluginData::get_token(Davix::RequestParams& params, const Davix::Uri& uri, const OP& operation, unsigned validity) { if (isS3SignedURL(uri)) { return false; } gchar* token = find_se_token(uri, operation); if (!token) { // Attempt to obtain SE-issued token token = retrieve_and_store_se_token(uri, operation, validity); } if (!token) { return false; } std::stringstream ss; ss << "Bearer " << token; gfal2_log(G_LOG_LEVEL_INFO, "Using bearer token for HTTPS request authorization%s", needsTransferHeader(operation) ? " (passive TPC)" : ""); if (needsTransferHeader(operation)) { params.addHeader("TransferHeaderAuthorization", ss.str()); // Disable credential delegation if we have a valid token for the destination params.addHeader("Credential", "none"); } else { params.addHeader("Authorization", ss.str()); } g_free(token); return true; } void GfalHttpPluginData::get_certificate(Davix::RequestParams& params, const Davix::Uri& uri) { DavixError* daverr = NULL; std::string cert, key; bool certificate_pair = gfal_http_get_x509_cert_pair(handle, uri, cert, key); if (certificate_pair) { gfal2_log(G_LOG_LEVEL_DEBUG, "Using client X509 for HTTPS session authorization"); X509Credential cred; if (cred.loadFromFilePEM(key, cert, "", &daverr) < 0 ) { gfal2_log(G_LOG_LEVEL_WARNING, "Could not load the user credentials: %s", daverr->getErrMsg().c_str()); DavixError::clearError(&daverr); } else { params.setClientCertX509(cred); } } } /// AWS authorization static void gfal_http_get_aws_keys(gfal2_context_t handle, const std::string& group, gchar** secret_key, gchar** access_key, gchar** token, gchar** region) { *secret_key = (*secret_key) ? *secret_key : gfal2_get_opt_string(handle, group.c_str(), "SECRET_KEY", NULL); *access_key = (*access_key) ? *access_key : gfal2_get_opt_string(handle, group.c_str(), "ACCESS_KEY", NULL); *token = (*token) ? *token : gfal2_get_opt_string(handle, group.c_str(), "TOKEN", NULL); *region = (*region) ? *region : gfal2_get_opt_string(handle, group.c_str(), "REGION", NULL); // For retro-compatibility if (!*access_key || !*secret_key) { *access_key = gfal2_get_opt_string(handle, group.c_str(), "ACCESS_TOKEN", NULL); *secret_key = gfal2_get_opt_string(handle, group.c_str(), "ACCESS_TOKEN_SECRET", NULL); } } void GfalHttpPluginData::get_aws_params(Davix::RequestParams& params, const Davix::Uri& uri) { bool alternate_url; gchar *access_key = NULL, *secret_key = NULL, *token = NULL, *region = NULL; bool access_pair_set = false, token_set = false, region_set = false; bool alternate_set = false; std::list group_labels; std::string host = uri.getHost(); // Add S3:HOST group label std::string group_label = std::string("S3:") + host; std::transform(group_label.begin(), group_label.end(), group_label.begin(), ::toupper); group_labels.push_back(group_label); // Add S3:(no-bucket)HOST group label size_t pos = host.find("."); if (pos != std::string::npos) { group_label = std::string("S3:") + host.substr(pos + 1); std::transform(group_label.begin(), group_label.end(), group_label.begin(), ::toupper); group_labels.push_back(group_label); } // ADD S3 group label group_labels.push_back("S3"); // Extract data from the config options // Order: Most specific group --> most generic group // Mechanism: Once a property is set, it will not be overwritten by later groups std::list::const_iterator it; for (it = group_labels.begin(); it != group_labels.end(); it++) { gfal_http_get_aws_keys(handle, *it, &secret_key, &access_key, &token, ®ion); if (!access_pair_set && secret_key && access_key) { gfal2_log(G_LOG_LEVEL_DEBUG, "Setting S3 key pair [%s]", it->c_str()); params.setAwsAuthorizationKeys(secret_key, access_key); access_pair_set = true; } if (!token_set && token) { gfal2_log(G_LOG_LEVEL_DEBUG, "Using short-lived access token [%s]", it->c_str()); params.setAwsToken(token); token_set = true; } if (!region_set && region) { gfal2_log(G_LOG_LEVEL_DEBUG, "Using region %s [%s]", region, it->c_str()); params.setAwsRegion(region); region_set = true; } if (!alternate_set) { GError *alternate_err = NULL; alternate_url = gfal2_get_opt_boolean(handle, it->c_str(), "ALTERNATE", &alternate_err); if (!alternate_err) { gfal2_log(G_LOG_LEVEL_DEBUG, "Setting S3 alternate URL to: %s [%s]", (alternate_url) ? "true" : "false", it->c_str()); params.setAwsAlternate(alternate_url); alternate_set = true; } g_clear_error(&alternate_err); } } g_free(access_key); g_free(secret_key); g_free(token); g_free(region); } /// Swift authorization static void gfal_http_get_swift_credentials(gfal2_context_t handle, const std::string& group, gchar** os_token, gchar** os_project_id, gchar** swift_account) { *os_token = (*os_token) ? *os_token : gfal2_get_opt_string(handle, group.c_str(), "OS_TOKEN", NULL); *os_project_id = (*os_project_id) ? *os_project_id : gfal2_get_opt_string(handle, group.c_str(), "OS_PROJECT_ID", NULL); *swift_account = (*swift_account) ? *swift_account : gfal2_get_opt_string(handle, group.c_str(), "SWIFT_ACCOUNT", NULL); } void GfalHttpPluginData::get_swift_params(Davix::RequestParams& params, const Davix::Uri& uri) { gchar *os_token = NULL, *os_project_id = NULL, *swift_account = NULL; bool token_set = false, project_id_set = false, swift_account_set = false; std::list group_labels; std::string host = uri.getHost(); // Add SWIFT:HOST group label std::string group_label = std::string("SWIFT:") + host; std::transform(group_label.begin(), group_label.end(), group_label.begin(), ::toupper); group_labels.push_back(group_label); // ADD SWIFT group label group_labels.push_back("SWIFT"); // Extract data from the config options // Order: Most specific group --> most generic group // Mechanism: Once a property is set, it will not be overwritten by later groups std::list::const_iterator it; for (it = group_labels.begin(); it != group_labels.end(); it++) { gfal_http_get_swift_credentials(handle, *it, &os_token, &os_project_id, &swift_account); if (!token_set && os_token) { gfal2_log(G_LOG_LEVEL_DEBUG, "Setting OS token [%s]", it->c_str()); params.setOSToken(os_token); token_set = true; } if (!project_id_set && os_project_id) { gfal2_log(G_LOG_LEVEL_DEBUG, "Setting OS project id [%s]", it->c_str()); params.setOSProjectID(os_project_id); project_id_set = true; } if (!swift_account_set && swift_account) { gfal2_log(G_LOG_LEVEL_DEBUG, "Using Swift account %s [%s]", swift_account, it->c_str()); params.setSwiftAccount(swift_account); swift_account_set = true; } } g_free(os_token); g_free(os_project_id); g_free(swift_account); } void GfalHttpPluginData::get_gcloud_credentials(Davix::RequestParams& params, const Davix::Uri& uri) { gchar *gcloud_json_file, *gcloud_json_string; std::string group_label("GCLOUD"); gcloud_json_file = gfal2_get_opt_string(handle, group_label.c_str(), "JSON_AUTH_FILE", NULL); gcloud_json_string = gfal2_get_opt_string(handle, group_label.c_str(), "JSON_AUTH_STRING", NULL); gcloud::CredentialProvider provider; if (gcloud_json_file) { gfal2_log(G_LOG_LEVEL_DEBUG, "Using gcloud json credential file"); params.setGcloudCredentials(provider.fromFile(std::string(gcloud_json_file))); } else if (gcloud_json_string) { gfal2_log(G_LOG_LEVEL_DEBUG, "Using gcloud json credential string"); params.setGcloudCredentials(provider.fromJSONString(std::string(gcloud_json_string))); } g_free(gcloud_json_file); g_free(gcloud_json_string); } void GfalHttpPluginData::get_reva_credentials(Davix::RequestParams ¶ms, const Davix::Uri &uri, const OP& operation) { // Reva is capable to handle bearer tokens, therefore we do here what is done for the typical case of bearer tokens // that are valid across sites (Reva would map them to local identities), i.e. use the token given in the GFAL context. // Note that a missing token will not raise exceptions but will obviously make the request fail. std::string header = gfal2_get_opt_string_with_default(handle, GFAL_CRED_BEARER, "TOKEN", ""); if (header == "") { return; } header = "Bearer " + header; if (needsTransferHeader(operation)) { // Reva is the active side of the transfer, set the corresponding TPC header params.addHeader("TransferHeaderAuthorization", header); } else { // Reva is the passive side of the transfer params.addHeader("Authorization", header); } } void GfalHttpPluginData::get_credentials(Davix::RequestParams& params, const Davix::Uri& uri, const OP& operation, unsigned token_validity) { // Setup GSI in case the storage endpoint tries to fall back to GridSite delegation. // That does mean that we might contact the endpoint with both X509 and token auth, // but seems to be an acceptable compromise get_certificate(params, uri); // Explicit request for S3 or GCloud if (uri.getProtocol().compare(0, 2, "s3") == 0) { get_aws_params(params, uri); } else if (uri.getProtocol().compare(0, 6, "gcloud") == 0) { get_gcloud_credentials(params, uri); } else if (uri.getProtocol().compare(0, 5, "swift") == 0) { get_swift_params(params, uri); }else if (uri.getProtocol().compare(0, 3, "cs3") == 0) { get_reva_credentials(params, uri, operation); } // Use bearer token (other authentication mechanism should be disabled) // Not the case for the moment, as certificates are still used (but should be unset in the future) else if (!get_token(params, uri, operation, token_validity)) { // Utilize AWS or GCLOUD tokens if no bearer token is available (to be reviewed) get_aws_params(params, uri); get_gcloud_credentials(params, uri); get_swift_params(params, uri); } } void GfalHttpPluginData::get_params_internal(Davix::RequestParams& params, const Davix::Uri& uri) { if (uri.getProtocol().compare(0, 4, "http") == 0) { params.setProtocol(Davix::RequestProtocol::Http); } else if (uri.getProtocol().compare(0, 3, "dav") == 0) { params.setProtocol(Davix::RequestProtocol::Webdav); } else if (uri.getProtocol().compare(0, 2, "s3") == 0) { params.setProtocol(Davix::RequestProtocol::AwsS3); } else if (uri.getProtocol().compare(0, 6, "gcloud") == 0) { params.setProtocol(Davix::RequestProtocol::Gcloud); } else if (uri.getProtocol().compare(0, 5, "swift") == 0) { params.setProtocol(Davix::RequestProtocol::Swift); } else if (uri.getProtocol().compare(0, 3, "cs3") == 0) { params.setProtocol(Davix::RequestProtocol::CS3); }else { params.setProtocol(Davix::RequestProtocol::Auto); } // Insecure flag gboolean insecure_mode = gfal2_get_opt_boolean_with_default(handle, "HTTP PLUGIN", "INSECURE", FALSE); if (insecure_mode) { params.setSSLCAcheck(false); } // Metalink mode gboolean metalink = gfal2_get_opt_boolean_with_default(handle, "HTTP PLUGIN", "METALINK", FALSE); params.setMetalinkMode((metalink) ? Davix::MetalinkMode::Auto : Davix::MetalinkMode::Disable); if (isCloudStorage(uri)) { params.setMetalinkMode(Davix::MetalinkMode::Disable); } // Keep alive gboolean keep_alive = gfal2_get_opt_boolean_with_default(handle, "HTTP PLUGIN", "KEEP_ALIVE", TRUE); params.setKeepAlive(keep_alive); // Reset here the verbosity level int davix_level = gfal2_get_opt_integer_with_default(handle, "HTTP PLUGIN", "LOG_LEVEL", 0); if (!davix_level) davix_level = get_corresponding_davix_log_level(); davix_set_log_level(davix_level); // Reset scope mask int davix_scope_mask = Davix::getLogScope() & ~(DAVIX_LOG_SSL | DAVIX_LOG_SENSITIVE | DAVIX_LOG_BODY); if (gfal2_get_opt_boolean_with_default(handle, "HTTP PLUGIN", "LOG_SENSITIVE", false)) { davix_scope_mask |= (DAVIX_LOG_SSL | DAVIX_LOG_SENSITIVE); } if (gfal2_get_opt_boolean_with_default(handle, "HTTP PLUGIN", "LOG_CONTENT", false)) { davix_scope_mask |= DAVIX_LOG_BODY; } Davix::setLogScope(davix_scope_mask); // Avoid retries params.setOperationRetry(0); // User agent const char *agent, *version; gfal2_get_user_agent(handle, &agent, &version); std::ostringstream user_agent; if (agent) { user_agent << agent << "/" << version << " "; } user_agent << "gfal2/" << gfal2_version(); params.setUserAgent(user_agent.str()); // Client information char* client_info = gfal2_get_client_info_string(handle); if (client_info) { params.addHeader("ClientInfo", client_info); } g_free(client_info); // Custom headers by SE char** headers = get_se_custom_headers_list(handle, uri); if (headers) { for (char **hi = headers; *hi != NULL; ++hi) { char **kv = g_strsplit(*hi, ":", 2); g_strstrip(kv[0]); g_strstrip(kv[1]); params.addHeader(kv[0], kv[1]); g_strfreev(kv); } g_strfreev(headers); } // Operation timeout struct timespec opTimeout{get_operation_timeout()}; params.setOperationTimeout(&opTimeout); } void GfalHttpPluginData::get_tpc_params(Davix::RequestParams* req_params, const Davix::Uri& src_uri, const Davix::Uri& dst_uri, gfalt_params_t transfer_params, bool push_mode) { *req_params = reference_params; bool do_delegation = false; // Token validity unsigned token_timeout = ((unsigned) (2 * gfalt_get_timeout(transfer_params, NULL)) / 60) + 10; if (push_mode) { get_params_internal(*req_params, src_uri); get_credentials(*req_params, src_uri, OP::READ, token_timeout); get_credentials(*req_params, dst_uri, OP::WRITE_PASV, token_timeout); do_delegation = delegation_required(dst_uri); } else { // Pull mode get_params_internal(*req_params, dst_uri); get_credentials(*req_params, src_uri, OP::READ_PASV, token_timeout); get_credentials(*req_params, dst_uri, OP::WRITE, token_timeout); do_delegation = delegation_required(src_uri); } // The TPC request should be explicit in terms of how the active endpoint should manage credentials, // as it can be ambiguous from the request (i.e., client X509 authenticated by Macaroon present or // Macaroon present at an endpoint that supports OIDC). // If a token is present for the inactive endpoint, then we set `Credential: none` earlier; hence, // if that header is missing, we explicitly chose `gridsite` here. We should first check if source/dest // needs delegation if (do_delegation) { const HeaderVec &headers = req_params->getHeaders(); bool set_credential = false; for (HeaderVec::const_iterator iter = headers.begin(); iter != headers.end(); iter++) { if (!strcasecmp(iter->first.c_str(), "Credential")) { set_credential = true; } } if (!set_credential) { req_params->addHeader("Credential", "gridsite"); } } else { req_params->addHeader("Credential", "none"); req_params->addHeader("X-No-Delegate", "true"); } } void GfalHttpPluginData::get_params(Davix::RequestParams* req_params, const Davix::Uri& uri, const OP& operation) { *req_params = reference_params; get_params_internal(*req_params, uri); get_credentials(*req_params, uri, operation); } int GfalHttpPluginData::get_operation_timeout() const { int global_timeout = gfal2_get_opt_integer_with_default(handle, CORE_CONFIG_GROUP, CORE_CONFIG_NAMESPACE_TIMEOUT, 300); return gfal2_get_opt_integer_with_default(handle, "HTTP PLUGIN", HTTP_CONFIG_OP_TIMEOUT, global_timeout); } void GfalHttpPluginData::set_operation_timeout(int timeout) { gfal2_set_opt_integer(handle, "HTTP PLUGIN", HTTP_CONFIG_OP_TIMEOUT, timeout, NULL); } void GfalHttpPluginData::resolve_and_store_url(const char* url) { gboolean resolve_dns = gfal2_get_opt_boolean_with_default(handle, CORE_CONFIG_GROUP, RESOLVE_DNS, FALSE); if (resolve_dns && is_http_scheme(url)) { char *url_tmp = resolve_dns_helper(url, "Resolved url"); if (url_tmp) { resolution_map[url] = url_tmp; free(url_tmp); } } } std::string GfalHttpPluginData::resolved_url(const std::string& url) { auto resolved = resolution_map.find(url); if (resolved != resolution_map.end()) { return resolved->second; } return url; } static void log_davix2gfal(void* userdata, int msg_level, const char* msg) { GLogLevelFlags gfal_level = G_LOG_LEVEL_MESSAGE; switch (msg_level) { case DAVIX_LOG_TRACE: case DAVIX_LOG_DEBUG: gfal_level = G_LOG_LEVEL_DEBUG; break; default: gfal_level = G_LOG_LEVEL_INFO; } gchar *escaped_msg = gfal2_utf8escape_string(msg, strlen(msg), "\n\r\t\\"); gfal2_log(gfal_level, "Davix: %s", escaped_msg); g_free(escaped_msg); } GfalHttpPluginData::GfalHttpPluginData(gfal2_context_t handle): context(), posix(&context), handle(handle), reference_params(), token_map(), tape_endpoint_map() { davix_set_log_handler(log_davix2gfal, NULL); int davix_level = gfal2_get_opt_integer_with_default(handle, "HTTP PLUGIN", "LOG_LEVEL", 0); if (!davix_level) davix_level = get_corresponding_davix_log_level(); davix_set_log_level(davix_level); Davix::setLogScope(Davix::getLogScope() & ~(DAVIX_LOG_SSL | DAVIX_LOG_SENSITIVE)); reference_params.setTransparentRedirectionSupport(true); reference_params.setUserAgent("gfal2::http"); context.loadModule("grid"); // TODO: read token issuer value // if (!issuer.empty()) { // token_retriever_chain.reset(new SciTokensRetriever(issuer)); // token_retriever_chain->add(new MacaroonRetriever()); // } token_retriever_chain.reset(new MacaroonRetriever()); } GfalHttpPluginData* gfal_http_get_plugin_context(gpointer ptr) { return static_cast(ptr); } void gfal_http_context_delete(gpointer plugin_data){ GfalHttpPluginData* data = static_cast(plugin_data); delete data; } void gfal_http_delete(plugin_handle plugin_data) { gfal_http_context_delete(plugin_data); } static gboolean gfal_http_check_url(plugin_handle plugin_data, const char* url, plugin_mode operation, GError** err) { switch(operation){ case GFAL_PLUGIN_QOS_CHECK_CLASSES: case GFAL_PLUGIN_CHECK_FILE_QOS: case GFAL_PLUGIN_CHECK_QOS_AVAILABLE_TRANSITIONS: case GFAL_PLUGIN_CHECK_TARGET_QOS: case GFAL_PLUGIN_CHANGE_OBJECT_QOS: return true; case GFAL_PLUGIN_ACCESS: case GFAL_PLUGIN_OPEN: case GFAL_PLUGIN_STAT: case GFAL_PLUGIN_MKDIR: case GFAL_PLUGIN_OPENDIR: case GFAL_PLUGIN_RMDIR: case GFAL_PLUGIN_UNLINK: case GFAL_PLUGIN_CHECKSUM: case GFAL_PLUGIN_RENAME: case GFAL_PLUGIN_TOKEN: return (strncmp("http:", url, 5) == 0 || strncmp("https:", url, 6) == 0 || strncmp("dav:", url, 4) == 0 || strncmp("davs:", url, 5) == 0 || strncmp("s3:", url, 3) == 0 || strncmp("s3s:", url, 4) == 0 || strncmp("gcloud:", url, 7) == 0 || strncmp("gclouds:", url, 8) == 0 || strncmp("swift:", url, 6) == 0 || strncmp("swifts:", url, 7) == 0 || strncmp("http+3rd:", url, 9) == 0 || strncmp("https+3rd:", url, 10) == 0 || strncmp("dav+3rd:", url, 8) == 0 || strncmp("davs+3rd:", url, 9) == 0 || strncmp("cs3:", url, 4) == 0 || strncmp("cs3s:", url, 5) == 0); case GFAL_PLUGIN_GETXATTR: case GFAL_PLUGIN_SETXATTR: case GFAL_PLUGIN_LISTXATTR: case GFAL_PLUGIN_BRING_ONLINE: case GFAL_PLUGIN_ARCHIVE: return (strncmp("http:", url, 5) == 0 || strncmp("https:", url, 6) == 0 || strncmp("dav:", url, 4) == 0 || strncmp("davs:", url, 5) == 0); default: return false; } } int davix2errno(StatusCode::Code code) { int errcode; switch (code) { case StatusCode::OK: case StatusCode::PartialDone: errcode = 0; break; case StatusCode::WebDavPropertiesParsingError: case StatusCode::UriParsingError: errcode = EIO; break; case StatusCode::SessionCreationError: errcode = EPERM; break; case StatusCode::NameResolutionFailure: errcode = EHOSTUNREACH; break; case StatusCode::ConnectionProblem: errcode = EHOSTDOWN; break; case StatusCode::OperationNonSupported: case StatusCode::RedirectionNeeded: errcode = ENOSYS; break; case StatusCode::ConnectionTimeout: case StatusCode::OperationTimeout: errcode = ETIMEDOUT; break; case StatusCode::PermissionRefused: errcode = EPERM; break; case StatusCode::IsADirectory: errcode = EISDIR; break; case StatusCode::IsNotADirectory: errcode = ENOTDIR; break; case StatusCode::InvalidFileHandle: errcode = EBADF; break; case StatusCode::AuthentificationError: case StatusCode::LoginPasswordError: case StatusCode::CredentialNotFound: case StatusCode::CredDecryptionError: case StatusCode::SSLError: case StatusCode::DelegationError: errcode = EACCES; break; case StatusCode::FileNotFound: errcode = ENOENT; break; case StatusCode::FileExist: errcode = EEXIST; break; case StatusCode::Canceled: errcode = ECANCELED; break; default: errcode = EIO; break; } return errcode; } void davix2gliberr(const DavixError* daverr, GError** err, const gchar* function) { const char *str = daverr->getErrMsg().c_str(); size_t str_len = daverr->getErrMsg().length(); gchar *escaped_str = gfal2_utf8escape_string(str, str_len, NULL); gfal2_set_error(err, http_plugin_domain, davix2errno(daverr->getStatus()), function, "%s", escaped_str); g_free(escaped_str); } static int http2errno(int http) { if (http < 400) return 0; switch (http) { case 400: case 406: return EINVAL; case 401: case 402: case 403: return EACCES; case 404: case 410: return ENOENT; case 405: return EPERM; case 409: return EEXIST; case 501: return ENOSYS; default: if (http >= 400 && http < 500) { return EINVAL; } else { #ifdef ECOMM return ECOMM; #else return EIO; #endif } } } void http2gliberr(GError** err, int http, const char* func, const char* msg) { int errn = http2errno(http); char buffer[512] = {0}; strerror_r(errn, buffer, sizeof(buffer)); gfal2_set_error(err, http_plugin_domain, errn, func, "%s: %s (HTTP %d)", msg, buffer, http); } /// Init function extern "C" gfal_plugin_interface gfal_plugin_init(gfal2_context_t handle, GError** err) { gfal_plugin_interface http_plugin; *err = NULL; memset(&http_plugin, 0, sizeof(http_plugin)); // Bind metadata http_plugin.check_plugin_url = &gfal_http_check_url; http_plugin.getName = &gfal_http_get_name; http_plugin.priority = GFAL_PLUGIN_PRIORITY_DATA ; http_plugin.plugin_data = new GfalHttpPluginData(handle); http_plugin.plugin_delete = &gfal_http_delete; http_plugin.statG = &gfal_http_stat; http_plugin.accessG = &gfal_http_access; http_plugin.mkdirpG = &gfal_http_mkdirpG; http_plugin.unlinkG = &gfal_http_unlinkG; http_plugin.rmdirG = &gfal_http_rmdirG; http_plugin.renameG = &gfal_http_rename; http_plugin.opendirG = &gfal_http_opendir; http_plugin.readdirG = &gfal_http_readdir; http_plugin.readdirppG = &gfal_http_readdirpp; http_plugin.closedirG = &gfal_http_closedir; // Bind IO http_plugin.openG = &gfal_http_fopen; http_plugin.readG = &gfal_http_fread; http_plugin.writeG = &gfal_http_fwrite; http_plugin.lseekG = &gfal_http_fseek; http_plugin.closeG = &gfal_http_fclose; // Extended attributes http_plugin.getxattrG = &gfal_http_getxattrG; http_plugin.listxattrG = &gfal_http_listxattrG; http_plugin.setxattrG = &gfal_http_setxattrG; // Checksum http_plugin.checksum_calcG = &gfal_http_checksum; // Bind 3rd party copy http_plugin.check_plugin_url_transfer = gfal_http_copy_check; http_plugin.copy_file = gfal_http_copy; // QoS http_plugin.check_qos_classes = &gfal_http_check_classes; http_plugin.check_file_qos = &gfal_http_check_file_qos; http_plugin.check_qos_available_transitions = &gfal_http_check_qos_available_transitions; http_plugin.check_target_qos = &gfal_http_check_target_qos; http_plugin.change_object_qos = &gfal_http_change_object_qos; // Token http_plugin.token_retrieve = &gfal_http_token_retrieve; // Tape http_plugin.bring_online_v2 = &gfal_http_bring_online_v2; http_plugin.bring_online_list_v2 = &gfal_http_bring_online_list_v2; http_plugin.bring_online = &gfal_http_bring_online; http_plugin.bring_online_list = &gfal_http_bring_online_list; http_plugin.release_file = &gfal_http_release_file; http_plugin.release_file_list = &gfal_http_release_file_list; http_plugin.archive_poll = &gfal_http_archive_poll; http_plugin.archive_poll_list = &gfal_http_archive_poll_list; http_plugin.bring_online_poll = &gfal_http_bring_online_poll; http_plugin.bring_online_poll_list = &gfal_http_bring_online_poll_list; http_plugin.abort_files = &gfal_http_abort_files; return http_plugin; } gfal2-v2.23.0/src/plugins/http/gfal_http_plugin.h000066400000000000000000000356431465240014500217210ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #ifndef _GFAL_HTTP_PLUGIN_H #define _GFAL_HTTP_PLUGIN_H #include #include #include #include "gfal_http_plugin_token.h" #include "gfal_http_plugin_copy.h" #define HTTP_CONFIG_OP_TIMEOUT "OPERATION_TIMEOUT" class GfalHttpPluginData { public: GfalHttpPluginData(gfal2_context_t); Davix::Context context; Davix::DavPosix posix; gfal2_context_t handle; enum class OP { READ, HEAD, WRITE, MKCOL, TAPE, READ_PASV, WRITE_PASV }; // Set up the Davix request parameters for a given URL // @param operation the HTTP operation to be performed void get_params(Davix::RequestParams*, const Davix::Uri& uri, const OP& operation = OP::READ); // Put together parameters for the TPC, which may depend on both URLs in the transfer // Further, the request headers depend on the transfer mode that will be used. void get_tpc_params(Davix::RequestParams*, const Davix::Uri& src_uri, const Davix::Uri& dst_uri, gfalt_params_t transfer_params, bool push_mode); // Resolve the DNS alias URL and store it in the internal DNS Alias map void resolve_and_store_url(const char* url); // Returns the resolved URL, if it exists, or the original URL // This function uses the internal DNS Alias map std::string resolved_url(const std::string& url); int get_operation_timeout() const; void set_operation_timeout(int timeout); friend ssize_t gfal_http_token_retrieve(plugin_handle plugin_data, const char* url, const char* issuer, gboolean write_access, unsigned validity, const char* const* activities, char* buff, size_t s_buff, GError** err); friend int gfal_http_mkdirpG(plugin_handle plugin_data, const char *url, mode_t mode, gboolean rec_flag, GError **err); friend int gfal_http_rename(plugin_handle plugin_data, const char* oldurl, const char* newurl, GError** err); friend class TokenMapTest; friend std::string gfal_http_discover_tape_endpoint(GfalHttpPluginData* davix, const char* url, const char* method, GError** err); friend ssize_t gfal_http_getxattr_internal(plugin_handle plugin_data, const char* url, const char *key, char* buff, size_t s_buff, GError** err); private: /// Tape REST API endpoint info struct typedef struct tape_endpoint_info { std::string sitename; std::string uri; std::string version; tape_endpoint_info() = default; } tape_endpoint_info_t; typedef std::map TokenAccessMap; typedef std::map TapeEndpointMap; typedef std::map DNSResolutionMap; /// baseline Davix Request Parameters Davix::RequestParams reference_params; /// map a token with read/write access flag TokenAccessMap token_map; /// token retriever object (can be chained) std::unique_ptr token_retriever_chain; /// map a url with a tape endpoint info struct TapeEndpointMap tape_endpoint_map; /// map an initial DNS alias URL to it's resolved URL DNSResolutionMap resolution_map; // Set up general request parameters void get_params_internal(Davix::RequestParams& params, const Davix::Uri& uri); // Obtain credentials for a given Uri and set those credentials in the Davix request parameters. // @param operation the HTTP operation to be performed // @param token_validity requested lifetime of the token in minutes void get_credentials(Davix::RequestParams& params, const Davix::Uri& uri, const OP& operation, unsigned token_validity = 180); // Obtain token credentials // @param operation the HTTP operation to be performed // @param validity requested lifetime of the token in seconds // @return true if a bearer for the provided Uri was set in the request params bool get_token(Davix::RequestParams& params, const Davix::Uri& uri, const OP& operation, unsigned validity); // Find SE-issued token in the Gfal2 credential map based on the path. // The found token provides either an exact path match or a parent directory path. // When the "extended_search" is requested, the found token may be a subpath of the search path. // @param operation the HTTP operation to be performed. Read/write access and extended search is inferred // @return the SE-issued token or null char* find_se_token(const Davix::Uri& uri, const OP& operation); // Attempt to obtain a SE-issued token (by exchanging x509 certificate) // @param operation the HTTP operation to be performed. Read/write access is inferred // @param validity lifetime of the token in seconds // @return the SE-issued token or null char* retrieve_and_store_se_token(const Davix::Uri& uri, const OP& operation, unsigned validity); // Discover tape endpoint and cache it // @param endpoint the SE, defined as protocol://host // @param err error handle // @return the tape endpoint tape_endpoint_info_t retrieve_and_store_tape_endpoint(const std::string& endpoint, GError** err); // Obtain request parameters + credentials for an AWS endpoint void get_aws_params(Davix::RequestParams& params, const Davix::Uri& uri); // Obtain GCloud endpoint credentials void get_gcloud_credentials(Davix::RequestParams& params, const Davix::Uri& uri); // Obtain Reva endpoint credentials void get_reva_credentials(Davix::RequestParams ¶ms, const Davix::Uri &uri, const OP& operation); // Obtain certificate credentials void get_certificate(Davix::RequestParams& params, const Davix::Uri& uri); // Obtain request parameters + credentials for a Swift endpoint void get_swift_params(Davix::RequestParams ¶ms, const Davix::Uri &uri); bool writeFlagFromOperation(const OP& operation); bool searchFlagFromOperation(const OP& operation); bool needsTransferHeader(const OP& operation); }; const char* gfal_http_get_name(void); GfalHttpPluginData* gfal_http_get_plugin_context(gpointer plugin_data); void gfal_http_context_delete(gpointer plugin_data); void gfal_http_delete(plugin_handle plugin_data); extern GQuark http_plugin_domain; // Initializes a GError from a DavixError void davix2gliberr(const Davix::DavixError* daverr, GError** err, const gchar* function); // Initializes a GError from an HTTP code void http2gliberr(GError** err, int http, const char* func, const char* msg); // Returns errno from Davix StatusCode int davix2errno(Davix::StatusCode::Code code); // Returns whether the URL is HTTP scheme bool is_http_scheme(const char* url); // Returns whether HTTP remote copy is enabled for the involved Storage Endpoints bool is_http_3rdcopy_enabled(gfal2_context_t context, const char* src, const char* dst); // Returns whether HTTP streaming is enabled for the involved Storage Endpoints bool is_http_streaming_enabled(gfal2_context_t context, const char* src, const char* dst); // Returns whether HTTP 3rd party copy fallback is enabled for the involved Storage Endpoints bool is_http_3rdcopy_fallback_enabled(gfal2_context_t context, const char* src, const char* dst); // Removes +3rd from the url, if there void strip_3rd_from_url(const char* url_full, char* url, size_t url_size); // Get custom Storage Element configuration option (boolean value) int get_se_custom_opt_boolean(const gfal2_context_t& context, const char* surl, const char* key); // Get custom Storage Element configuration option (string value) char* get_se_custom_opt_string(const gfal2_context_t& context, const char* surl, const char* key); // Get custom HTTP headers for Storage Element group char** get_se_custom_headers_list(const gfal2_context_t& context, const Davix::Uri& uri); // Specific function for the retrieve-bearer-token configuration option bool get_retrieve_bearer_token_config(const gfal2_context_t& context, const char* surl, bool default_value); // Find tape endpoint for a given method std::string gfal_http_discover_tape_endpoint(GfalHttpPluginData* davix, const char* url, const char* method, GError** err); // METADATA OPERATIONS int gfal_http_stat(plugin_handle plugin_data, const char* url, struct stat* buf, GError** err); int gfal_http_rename(plugin_handle plugin_data, const char* oldurl, const char* newurl, GError** err); int gfal_http_access(plugin_handle plugin_data, const char* url, int mode, GError** err); int gfal_http_mkdirpG(plugin_handle plugin_data, const char* url, mode_t mode, gboolean rec_flag, GError** err); int gfal_http_rmdirG(plugin_handle plugin_data, const char* url, GError** err); int gfal_http_unlinkG(plugin_handle plugin_data, const char* url, GError** err); gfal_file_handle gfal_http_opendir(plugin_handle plugin_data, const char* url, GError** err); struct dirent* gfal_http_readdir(plugin_handle plugin_data, gfal_file_handle dir_desc, GError** err); struct dirent* gfal_http_readdirpp(plugin_handle plugin_data, gfal_file_handle dir_desc, struct stat* st, GError** err); int gfal_http_closedir(plugin_handle plugin_data, gfal_file_handle dir_desc, GError** err); // IO gfal_file_handle gfal_http_fopen(plugin_handle plugin_data, const char* url, int flag, mode_t mode, GError** err); ssize_t gfal_http_fread(plugin_handle, gfal_file_handle fd, void* buff, size_t count, GError** err); ssize_t gfal_http_fwrite(plugin_handle, gfal_file_handle fd, const void* buff, size_t count, GError** err); int gfal_http_fclose(plugin_handle, gfal_file_handle fd, GError ** err); off_t gfal_http_fseek(plugin_handle, gfal_file_handle fd, off_t offset, int whence, GError** err); // Checksum int gfal_http_checksum(plugin_handle data, const char* url, const char* check_type, char * checksum_buffer, size_t buffer_length, off_t start_offset, size_t data_length, GError ** err); // Extended attributes ssize_t gfal_http_getxattrG(plugin_handle plugin_data, const char* url, const char* key, void* buff, size_t s_buff, GError** err); ssize_t gfal_http_listxattrG(plugin_handle plugin_data, const char* url, char* list, size_t s_list, GError** err); int gfal_http_setxattrG(plugin_handle plugin_data, const char* url, const char* key, const void* buff , size_t s_buff, int flags, GError** err); // Copy operation int gfal_http_copy(plugin_handle plugin_data, gfal2_context_t context, gfalt_params_t params, const char* src, const char* dst, GError** err); int gfal_http_copy_check(plugin_handle plugin_data, gfal2_context_t context, const char* src, const char* dst, gfal_url2_check check); // QoS ssize_t gfal_http_check_classes(plugin_handle plugin_data, const char* url, const char* type, char* buff, size_t s_buff, GError** err); ssize_t gfal_http_check_file_qos(plugin_handle plugin_data, const char* url, char* buff, size_t s_buff, GError** err); ssize_t gfal_http_check_qos_available_transitions(plugin_handle plugin_data, const char* qos_class_url, char* buff, size_t s_buff, GError** err); ssize_t gfal_http_check_target_qos(plugin_handle plugin_data, const char* url, char* buff, size_t s_buff, GError** err); int gfal_http_change_object_qos(plugin_handle plugin_data, const char* url, const char* target_qos, GError** err); bool http_cdmi_code_is_valid(int code); // Token ssize_t gfal_http_token_retrieve(plugin_handle plugin_data, const char* url, const char* issuer, gboolean write_access, unsigned validity, const char* const* activities, char* buff, size_t s_buff, GError** err); // Tape Operations int gfal_http_release_file(plugin_handle plugin_data, const char* url, const char* request_id, GError** err); int gfal_http_release_file_list(plugin_handle plugin_data, int nbfiles, const char* const* urls, const char* request_id, GError** errors); int gfal_http_archive_poll(plugin_handle plugin_data, const char* url, GError** err); int gfal_http_archive_poll_list(plugin_handle plugin_data, int nbfiles, const char* const* urls, GError** errors); int gfal_http_bring_online_poll(plugin_handle plugin_data, const char* url, const char* token, GError** err); int gfal_http_bring_online_poll_list(plugin_handle plugin_data, int nbfiles, const char* const* urls, const char* token, GError** errors); int gfal_http_abort_files(plugin_handle handle, int nbfiles, const char* const* uris, const char* token, GError** errors); int gfal_http_bring_online(plugin_handle plugin_data, const char* url, time_t pintime, time_t timeout, char* token, size_t tsize, int async, GError** err); int gfal_http_bring_online_v2(plugin_handle plugin_data, const char* url, const char* metadata, time_t pintime, time_t timeout, char* token, size_t tsize, int async, GError** err); int gfal_http_bring_online_list(plugin_handle plugin_data, int nbfiles, const char* const* urls, time_t pintime, time_t timeout, char* token, size_t tsize, int async, GError** errors); int gfal_http_bring_online_list_v2(plugin_handle plugin_data, int nbfiles, const char* const* urls, const char* const* metadata, time_t pintime, time_t timeout, char* token, size_t tsize, int async, GError** errors); // Get the extended attribute in "key" from a remote storage endpoint ssize_t gfal_http_getxattr_internal(plugin_handle plugin_data, const char* url, const char *key, char* buff, size_t s_buff, GError** err); // Get "user.status" extended attribute ssize_t gfal_http_status_getxattr(plugin_handle plugin_data, const char* url, char* buff, size_t s_buff, GError** err); #endif //_GFAL_HTTP_PLUGIN_H gfal2-v2.23.0/src/plugins/http/gfal_http_plugin_copy.h000066400000000000000000000041641465240014500227450ustar00rootroot00000000000000/* * Copyright (c) CERN 2023 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #pragma once #ifndef _GFAL_HTTP_PLUGIN_COPY_H #define _GFAL_HTTP_PLUGIN_COPY_H /** * Gfal2 HTTP Plugin abstraction for the copy mode, * which can be one of "pull", "push" and "stream". * * There is a fallback mechanism from pull --> push --> stream. * * A factory method is provided to construct the HttpCopyMode object * based on the Gfal2 configuration and the source and destination endpoints. */ class HttpCopyMode { public: enum class CopyMode { PULL, PUSH, STREAM, NONE }; void next(); bool end() const; const char* str() const; inline CopyMode value() const { return copyMode; }; inline bool isStreamingOnly() const { return streamingOnly; }; inline bool isStreamingEnabled() const { return streamingEnabled; }; static HttpCopyMode ConstructCopyMode(gfal2_context_t context, const char* src, const char* dst); static CopyMode CopyModeFromQueryArguments(const char* surl); static CopyMode CopyModeFromStr(const char* copyModeStr); static const char* CopyModeToStr(CopyMode copyMode); private: HttpCopyMode(CopyMode copyMode, bool streamingOnly, bool streamingEnabled) : copyMode(copyMode), streamingOnly(streamingOnly), streamingEnabled(streamingEnabled) {} CopyMode copyMode; bool streamingOnly; bool streamingEnabled; }; #endif //_GFAL_HTTP_PLUGIN_COPY_H gfal2-v2.23.0/src/plugins/http/gfal_http_plugin_io.cpp000066400000000000000000000111741465240014500227340ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2015 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include "gfal_http_plugin.h" struct GfalHTTPFD { Davix::RequestParams req_params; DAVIX_FD* davix_fd; }; gfal_file_handle gfal_http_fopen(plugin_handle plugin_data, const char* url, int flag, mode_t mode, GError** err) { char stripped_url[GFAL_URL_MAX_LEN]; strip_3rd_from_url(url, stripped_url, sizeof(stripped_url)); GfalHttpPluginData* davix = gfal_http_get_plugin_context(plugin_data); Davix::DavixError* daverr = NULL; GfalHTTPFD* fd = new GfalHTTPFD(); GfalHttpPluginData::OP operation = (flag & O_WRONLY) ? GfalHttpPluginData::OP::WRITE : GfalHttpPluginData::OP::READ; davix->get_params(&fd->req_params, Davix::Uri(stripped_url), operation); if (strncmp("s3:", url, 3) == 0 || strncmp("s3s:", url, 4) == 0) { fd->req_params.setProtocol(Davix::RequestProtocol::AwsS3); } else if (strncmp("gcloud:", url, 7) == 0 || strncmp("gclouds:", url, 8) == 0) { fd->req_params.setProtocol(Davix::RequestProtocol::Gcloud); } else if (strncmp("swift:", url, 6) == 0 || strncmp("swifts:", url, 7) == 0) { fd->req_params.setProtocol(Davix::RequestProtocol::Swift); } else if (strncmp("cs3:", url, 4) == 0 || strncmp("cs3s:", url, 5) == 0) { fd->req_params.setProtocol(Davix::RequestProtocol::CS3); } // DMC-1348: Use resolved URLs for data operations std::string resolved_url = davix->resolved_url(stripped_url); fd->davix_fd = davix->posix.open(&fd->req_params, resolved_url, flag, &daverr); if (fd->davix_fd == NULL) { davix2gliberr(daverr, err, __func__); Davix::DavixError::clearError(&daverr); delete fd; return NULL; } return gfal_file_handle_new(gfal_http_get_name(), fd); } ssize_t gfal_http_fread(plugin_handle plugin_data, gfal_file_handle fd, void* buff, size_t count, GError** err) { GfalHttpPluginData* davix = gfal_http_get_plugin_context(plugin_data); Davix::DavixError* daverr = NULL; GfalHTTPFD* dfd = (GfalHTTPFD*) gfal_file_handle_get_fdesc(fd); ssize_t reads = davix->posix.read(dfd->davix_fd, buff, count, &daverr); if (reads < 0) { davix2gliberr(daverr, err, __func__); Davix::DavixError::clearError(&daverr); } return reads; } ssize_t gfal_http_fwrite(plugin_handle plugin_data, gfal_file_handle fd, const void* buff, size_t count, GError** err) { GfalHttpPluginData* davix = gfal_http_get_plugin_context(plugin_data); Davix::DavixError* daverr = NULL; GfalHTTPFD* dfd = (GfalHTTPFD*) gfal_file_handle_get_fdesc(fd); ssize_t writes = davix->posix.write(dfd->davix_fd, buff, count, &daverr); if (writes < 0) { davix2gliberr(daverr, err, __func__); Davix::DavixError::clearError(&daverr); } return writes; } int gfal_http_fclose(plugin_handle plugin_data, gfal_file_handle fd, GError ** err) { GfalHttpPluginData* davix = gfal_http_get_plugin_context(plugin_data); Davix::DavixError* daverr = NULL; GfalHTTPFD* dfd = (GfalHTTPFD*) gfal_file_handle_get_fdesc(fd); int ret = 0; if (davix->posix.close(dfd->davix_fd, &daverr) != 0) { davix2gliberr(daverr, err, __func__); Davix::DavixError::clearError(&daverr); ret = -1; } gfal_file_handle_delete(fd); return ret; } off_t gfal_http_fseek(plugin_handle plugin_data, gfal_file_handle fd, off_t offset, int whence, GError** err) { GfalHttpPluginData* davix = gfal_http_get_plugin_context(plugin_data); Davix::DavixError* daverr = NULL; GfalHTTPFD* dfd = (GfalHTTPFD*) gfal_file_handle_get_fdesc(fd); off_t newOffset = static_cast(davix->posix.lseek64(dfd->davix_fd, static_cast(offset), whence, &daverr)); if (newOffset < 0) { davix2gliberr(daverr, err, __func__); Davix::DavixError::clearError(&daverr); } return newOffset; } gfal2-v2.23.0/src/plugins/http/gfal_http_plugin_metadata.cpp000066400000000000000000000333041465240014500241040ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2015 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include "gfal_http_plugin.h" int gfal_http_stat(plugin_handle plugin_data, const char* url, struct stat* buf, GError** err) { Davix::StatInfo info; char stripped_url[GFAL_URL_MAX_LEN]; strip_3rd_from_url(url, stripped_url, sizeof(stripped_url)); if(buf==NULL){ gfal2_set_error(err, http_plugin_domain, EINVAL, __func__, "Invalid stat argument"); return -1; } GfalHttpPluginData* davix = gfal_http_get_plugin_context(plugin_data); Davix::DavixError* daverr = NULL; Davix::RequestParams req_params; davix->get_params(&req_params, Davix::Uri(stripped_url), GfalHttpPluginData::OP::HEAD); // Attempt stat over WebDav first, then fallback to HTTP if (req_params.getProtocol() == Davix::RequestProtocol::Http) { gfal2_log(G_LOG_LEVEL_DEBUG, "Identified stat over HTTP protocol. Attempting stat over WebDav first"); req_params.setProtocol(Davix::RequestProtocol::Webdav); Davix::StatInfo statInfo; if (davix->posix.stat64(&req_params, stripped_url, &statInfo, &daverr) != 0) { gfal2_log(G_LOG_LEVEL_MESSAGE, "Stat over WebDav failed with error: %s. Will fallback to HTTP protocol", daverr->getErrMsg().c_str()); Davix::DavixError::clearError(&daverr); req_params.setProtocol(Davix::RequestProtocol::Http); } else { statInfo.toPosixStat(*buf); return 0; } } if (davix->posix.stat64(&req_params, stripped_url, &info, &daverr) != 0) { davix2gliberr(daverr, err, __func__); Davix::DavixError::clearError(&daverr); return -1; } info.toPosixStat(*buf); return 0; } int gfal_http_mkdirpG(plugin_handle plugin_data, const char* url, mode_t mode, gboolean rec_flag, GError** err) { char stripped_url[GFAL_URL_MAX_LEN]; strip_3rd_from_url(url, stripped_url, sizeof(stripped_url)); GfalHttpPluginData* davix = gfal_http_get_plugin_context(plugin_data); Davix::DavixError* daverr = NULL; Davix::Uri uri(stripped_url); Davix::RequestParams req_params; bool retrieve_token = get_retrieve_bearer_token_config(davix->handle, uri.getString().c_str(), false); if (retrieve_token) { gchar *token = davix->find_se_token(uri, GfalHttpPluginData::OP::MKCOL); if (!token) { std::string reserved(stripped_url); if (reserved.back() != '/') { reserved.push_back('/'); } reserved += "gfal2_mkdir.reserved"; davix->retrieve_and_store_se_token(Davix::Uri(reserved), GfalHttpPluginData::OP::MKCOL, 60); } g_free(token); } davix->get_params(&req_params, uri, GfalHttpPluginData::OP::MKCOL); if (davix->posix.mkdir(&req_params, stripped_url, mode, &daverr) != 0) { davix2gliberr(daverr, err, __func__); Davix::DavixError::clearError(&daverr); return -1; } return 0; } int gfal_http_unlinkG(plugin_handle plugin_data, const char* url, GError** err) { char stripped_url[GFAL_URL_MAX_LEN]; strip_3rd_from_url(url, stripped_url, sizeof(stripped_url)); GfalHttpPluginData* davix = gfal_http_get_plugin_context(plugin_data); Davix::DavixError* daverr = NULL; Davix::RequestParams req_params; davix->get_params(&req_params, Davix::Uri(stripped_url), GfalHttpPluginData::OP::WRITE); req_params.setMetalinkMode(Davix::MetalinkMode::Disable); if (davix->posix.unlink(&req_params, stripped_url, &daverr) != 0) { davix2gliberr(daverr, err, __func__); Davix::DavixError::clearError(&daverr); return -1; } return 0; } int gfal_http_rmdirG(plugin_handle plugin_data, const char* url, GError** err) { char stripped_url[GFAL_URL_MAX_LEN]; strip_3rd_from_url(url, stripped_url, sizeof(stripped_url)); struct stat st; if (gfal_http_stat(plugin_data, stripped_url, &st, err) != 0) { return -1; } if (!S_ISDIR(st.st_mode)) { gfal2_set_error(err, http_plugin_domain, ENOTDIR, __func__, "Can not rmdir a file"); return -1; } // Enforce rmdir path to always end in '/' size_t len = strlen(stripped_url); if (GFAL_URL_MAX_LEN > len && stripped_url[len - 1] != '/') { stripped_url[len] = '/'; stripped_url[len + 1] = '\0'; } GfalHttpPluginData* davix = gfal_http_get_plugin_context(plugin_data); Davix::DavixError* daverr = NULL; Davix::RequestParams req_params; davix->get_params(&req_params, Davix::Uri(stripped_url), GfalHttpPluginData::OP::WRITE); if (davix->posix.rmdir(&req_params, stripped_url, &daverr) != 0) { davix2gliberr(daverr, err, __func__); Davix::DavixError::clearError(&daverr); return -1; } return 0; } int gfal_http_rename(plugin_handle plugin_data, const char* oldurl, const char* newurl, GError** err) { char stripped_old[GFAL_URL_MAX_LEN]; char stripped_new[GFAL_URL_MAX_LEN]; strip_3rd_from_url(oldurl, stripped_old, sizeof(stripped_old)); strip_3rd_from_url(newurl, stripped_new, sizeof(stripped_new)); GfalHttpPluginData* davix = gfal_http_get_plugin_context(plugin_data); Davix::DavixError* daverr = NULL; Davix::Uri uri(stripped_old); Davix::RequestParams req_params; bool retrieve_token = get_retrieve_bearer_token_config(davix->handle, uri.getString().c_str(), false); if (retrieve_token) { // Find the common base directory std::string oldpath = uri.getPath(); std::string newpath = Davix::Uri(stripped_new).getPath(); size_t common_base_idx = 0; size_t idx = 0; while ((idx < oldpath.size()) && (idx < newpath.size()) && (oldpath[idx] == newpath[idx])) { if (oldpath[idx] == '/') { common_base_idx = idx; } idx++; } uri.setPath(oldpath.substr(0, common_base_idx + 1)); gchar *token = davix->find_se_token(uri, GfalHttpPluginData::OP::WRITE); if (!token) { davix->retrieve_and_store_se_token(uri, GfalHttpPluginData::OP::WRITE, 60); } g_free(token); } davix->get_params(&req_params, uri, GfalHttpPluginData::OP::WRITE); if (davix->posix.rename(&req_params, stripped_old, stripped_new, &daverr) != 0) { davix2gliberr(daverr, err, __func__); Davix::DavixError::clearError(&daverr); return -1; } return 0; } int gfal_http_access(plugin_handle plugin_data, const char* url, int mode, GError** err) { struct stat buf; GError* tmp_err = NULL; memset(&buf, 0, sizeof(buf)); if (gfal_http_stat(plugin_data, url, &buf, &tmp_err) != 0) { gfal2_propagate_prefixed_error(err, tmp_err, __func__); return -1; } uid_t real_uid = getuid(); gid_t real_gid = getgid(); int ngroups = getgroups(0, NULL); if (ngroups < 0) { gfal2_set_error(err, http_plugin_domain, errno, __func__, "Could not get the groups of the current user"); return -1; } gid_t additional_gids[ngroups]; getgroups(ngroups, additional_gids); if (real_uid == buf.st_uid) mode <<= 6; else if (real_gid == buf.st_gid) mode <<= 3; else { for (int i = 0; i < ngroups; ++i) { if (additional_gids[i] == buf.st_gid) { mode <<= 3; break; } } } if ((mode & buf.st_mode) != static_cast(mode)) { gfal2_set_error(err, http_plugin_domain, EACCES, __func__, "Does not have enough permissions on '%s'", url); return -1; } else { return 0; } } gfal_file_handle gfal_http_opendir(plugin_handle plugin_data, const char* url, GError** err) { char stripped_url[GFAL_URL_MAX_LEN]; strip_3rd_from_url(url, stripped_url, sizeof(stripped_url)); GfalHttpPluginData* davix = gfal_http_get_plugin_context(plugin_data); Davix::DavixError* daverr = NULL; Davix::RequestParams req_params; davix->get_params(&req_params, Davix::Uri(stripped_url)); DAVIX_DIR* dir = davix->posix.opendirpp(&req_params, stripped_url, &daverr); if (dir == NULL) { davix2gliberr(daverr, err, __func__); Davix::DavixError::clearError(&daverr); return NULL; } return gfal_file_handle_new2(gfal_http_get_name(), dir, NULL, url); } struct dirent* gfal_http_readdir(plugin_handle plugin_data, gfal_file_handle dir_desc, GError** err) { GfalHttpPluginData* davix = gfal_http_get_plugin_context(plugin_data); Davix::DavixError* daverr = NULL; daverr = NULL; struct stat _; struct dirent* de = davix->posix.readdirpp((DAVIX_DIR*)gfal_file_handle_get_fdesc(dir_desc), &_, &daverr); if (de == NULL && daverr != NULL) { davix2gliberr(daverr, err, __func__); Davix::DavixError::clearError(&daverr); } return de; } struct dirent* gfal_http_readdirpp(plugin_handle plugin_data, gfal_file_handle dir_desc, struct stat* st, GError** err) { GfalHttpPluginData* davix = gfal_http_get_plugin_context(plugin_data); Davix::DavixError* daverr = NULL; daverr = NULL; struct dirent* de = davix->posix.readdirpp((DAVIX_DIR*)gfal_file_handle_get_fdesc(dir_desc), st, &daverr); if (de == NULL && daverr != NULL) { davix2gliberr(daverr, err, __func__); Davix::DavixError::clearError(&daverr); } return de; } int gfal_http_closedir(plugin_handle plugin_data, gfal_file_handle dir_desc, GError** err) { GfalHttpPluginData* davix = gfal_http_get_plugin_context(plugin_data); Davix::DavixError* daverr = NULL; int ret = 0; if (davix->posix.closedir((DAVIX_DIR*) gfal_file_handle_get_fdesc(dir_desc), &daverr) != 0) { davix2gliberr(daverr, err, __func__); Davix::DavixError::clearError(&daverr); ret = -1; } gfal_file_handle_delete(dir_desc); return ret; } int gfal_http_checksum(plugin_handle plugin_data, const char* url, const char* check_type, char * checksum_buffer, size_t buffer_length, off_t start_offset, size_t data_length, GError ** err) { char stripped_url[GFAL_URL_MAX_LEN]; strip_3rd_from_url(url, stripped_url, sizeof(stripped_url)); GfalHttpPluginData* davix = gfal_http_get_plugin_context(plugin_data); Davix::DavixError* daverr = NULL; std::string buffer_chk, algo_chk(check_type); if (start_offset != 0 || data_length != 0) { gfal2_set_error(err, http_plugin_domain, ENOTSUP, __func__, "HTTP does not support partial checksums"); return -1; } Davix::RequestParams req_params; davix->get_params(&req_params, Davix::Uri(stripped_url)); // Override timeout with checksum timeout struct timespec opTimeout; opTimeout.tv_sec = gfal2_get_opt_integer_with_default(davix->handle, CORE_CONFIG_GROUP, CORE_CONFIG_CHECKSUM_TIMEOUT, 300); req_params.setOperationTimeout(&opTimeout); // thois is needed by DPM + DOME as it implements a queue for checksum calculation req_params.setAcceptedRetry(100); req_params.setAcceptedRetryDelay(15); Davix::File f(davix->context, Davix::Uri(stripped_url)); if(f.checksum(&req_params, buffer_chk, check_type, &daverr) <0 ){ davix2gliberr(daverr, err, __func__); Davix::DavixError::clearError(&daverr); return -1; } g_strlcpy(checksum_buffer, buffer_chk.c_str(), buffer_length); return 0; } ssize_t gfal_http_getxattrG(plugin_handle plugin_data, const char* url, const char* key, void* buff, size_t s_buff, GError** err) { if (strcmp(key, GFAL_XATTR_STATUS) == 0) { return gfal_http_status_getxattr(plugin_data, url, (char *) buff, s_buff, err); } else if ((strcmp(key, GFAL_XATTR_TAPE_API_VERSION) == 0) || (strcmp(key, GFAL_XATTR_TAPE_API_URI) == 0) || (strcmp(key, GFAL_XATTR_TAPE_API_SITENAME) == 0)) { return gfal_http_getxattr_internal(plugin_data, url, key, (char*)buff, s_buff, err); } else { gfal2_set_error(err, http_plugin_domain, ENODATA, __func__, "Failed to get the xattr \"%s\" (No data available)", key); return -1; } } ssize_t gfal_http_listxattrG(plugin_handle plugin_data, const char* url, char* list, size_t s_list, GError** err) { static const char props[] = "taperestapi.version\0taperestapi.uri\0taperestapi.sitename"; static const size_t proplen = sizeof(props); size_t len = proplen > s_list ? s_list : proplen; memcpy(list, props, len); return len; } int gfal_http_setxattrG(plugin_handle plugin_data, const char* url, const char* key, const void* buff, size_t s_buff, int flags, GError** err) { gfal2_set_error(err, http_plugin_domain, ENOSYS, __func__, "Can not set extended attributes"); return -1; } gfal2-v2.23.0/src/plugins/http/gfal_http_plugin_tape.cpp000066400000000000000000001002011465240014500232440ustar00rootroot00000000000000/* * Copyright (c) CERN 2022 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include "uri/gfal2_parsing.h" #include "gfal_http_plugin.h" using namespace Davix; // Wrapper function to provide string functionality on top of Gfal2 util functionality static std::string collapse_slashes(const std::string& path) { char* collapsed_ptr = gfal2_path_collapse_slashes(path.c_str()); std::string collapsed(collapsed_ptr); g_free(collapsed_ptr); return collapsed; } namespace tape_rest_api { // Struct containing file locality on the remote storage endpoint // (Storage endpoint must support Tape REST API functionality) struct file_locality_s { bool on_disk; bool on_tape; }; typedef struct file_locality_s file_locality_t; // Construct a request body that consists in a list if file paths from the given set of URLs std::string list_files_body(int nbfiles, const char *const *urls) { std::stringstream body; body << "{\"paths\": ["; for (int i = 0; i < nbfiles; i++) { if (i != 0) { body << ", "; } body << "\"" << collapse_slashes(Davix::Uri(urls[i]).getPath()) << "\""; } body << "]}"; return body.str(); } // Construct targetedMetadata json std::string stage_request_body(int disk_lifetime, int nbfiles, const char *const *urls, const char *const *metadata) { std::stringstream body; body << "{\"files\": ["; for (int i = 0; i < nbfiles; i++) { if (i != 0) { body << ", "; } body << "{\"path\": " << "\"" << collapse_slashes(Davix::Uri(urls[i]).getPath()) << "\""; if ((metadata[i] != NULL) && (metadata[i][0] != '\0')) { body << ", \"targetedMetadata\": " << metadata[i]; } body << "}"; } body << "]}"; return body.str(); } // Parse metadata and return 0 if a valid JSON is found. If metadata is not a valid JSON return -1 int metadata_format_checker(int nbfiles, const char *const *metadata_list, GError** err) { struct json_object* json_metadata = 0; for (int i = 0; i < nbfiles; i++) { if ((metadata_list[i] != NULL) && (metadata_list[i][0] != '\0')) { json_metadata = json_tokener_parse(metadata_list[i]); if (!json_metadata) { gfal2_set_error(err, http_plugin_domain, EINVAL, __func__, "Invalid metadata format: %s", metadata_list[i]); return -1; } } //Free the JSON object json_object_put(json_metadata); } return 0; } void copyErrors(GError* tmp_err, int n, GError** errors) { for (int i = 0; i < n; i++) { errors[i] = g_error_copy(tmp_err); } // Frees tmp_err g_error_free(tmp_err); } // Run through the full response and identify the JSON item corresponding to our path // Very inefficient O(N^2) complexity but avoids complicated data structures (tape polling is time permissive) struct json_object* polling_get_item_by_path(struct json_object* response, const std::string& surl) { const int len = json_object_array_length(response); for (int i = 0; i < len; i++) { auto item = json_object_array_get_idx(response, i); if (item != NULL) { struct json_object* item_path = 0; json_object_object_get_ex(item, "path", &item_path); std::string path = item_path ? json_object_get_string(item_path) : ""; if (!path.empty()) { std::string cpath = collapse_slashes(path); std::string csurl = collapse_slashes(surl); if (cpath == csurl) { return item; } } } } return NULL; } std::string get_archiveinfo(plugin_handle plugin_data, int nbfiles, const char* const* urls, GError** err) { GError* tmp_err = NULL; GfalHttpPluginData* davix = gfal_http_get_plugin_context(plugin_data); std::string tapeEndpoint = gfal_http_discover_tape_endpoint(davix, urls[0],"/archiveinfo", &tmp_err); if (tmp_err != NULL) { *err = g_error_copy(tmp_err); g_error_free(tmp_err); return ""; } // Construct and send "POST /archiveinfo" request Davix::DavixError* reqerr = NULL; Davix::Uri uri(tapeEndpoint); Davix::RequestParams params; PostRequest request(davix->context, uri, &reqerr); davix->get_params(¶ms, uri, GfalHttpPluginData::OP::TAPE); params.addHeader("Content-Type", "application/json"); request.setParameters(params); request.setRequestBody(tape_rest_api::list_files_body(nbfiles, urls)); if (request.executeRequest(&reqerr)) { gfal2_set_error(err, http_plugin_domain, davix2errno(reqerr->getStatus()), __func__, "[Tape REST API] Archive polling call failed: %s", reqerr->getErrMsg().c_str()); Davix::DavixError::clearError(&reqerr); return ""; } if (request.getRequestCode() != 200) { gfal2_set_error(err, http_plugin_domain, EINVAL, __func__, "[Tape REST API] Archive polling call failed: %s: %s", reqerr->getErrMsg().c_str(), request.getAnswerContent()); Davix::DavixError::clearError(&reqerr); return ""; } std::string content = request.getAnswerContent(); if (content.empty()) { gfal2_set_error(err, http_plugin_domain, ENOMSG, __func__, "[Tape REST API] Response with no data"); return ""; } return content; } // Get locality field from "/archiveinfo" response // On failed request, sets the "err" object // On successful request, but "error" field is present in the response: // - sets the "err" object, if bypass_archive_error not true // - returns file locality, if bypass_archive_error is true file_locality_t get_file_locality(struct json_object* file, const std::string& path, GError** err, bool bypass_archive_error = false) { file_locality_t locality{false, false}; if (file == NULL) { gfal2_set_error(err, http_plugin_domain, ENOMSG, __func__, "[Tape REST API] Missing response item for path=%s", path.c_str()); return locality; } // Check if "error" attribute exists struct json_object *file_error_text = 0; bool foundError = json_object_object_get_ex(file, "error", &file_error_text); if (foundError && !bypass_archive_error) { std::string error_text = json_object_get_string(file_error_text); gfal2_set_error(err, http_plugin_domain, ENOMSG, __func__, "[Tape REST API] %s", error_text.c_str()); return locality; } // Retrieve "locality" attribute struct json_object *file_locality = 0; bool localityExist = json_object_object_get_ex(file, "locality", &file_locality); if (!localityExist) { gfal2_set_error(err, http_plugin_domain, ENOMSG, __func__, "[Tape REST API] Locality attribute missing"); return locality; } std::string locality_text = json_object_get_string(file_locality); if (locality_text == "TAPE") { locality.on_tape = true; } else if (locality_text == "DISK") { locality.on_disk = true; } else if (locality_text == "DISK_AND_TAPE") { locality.on_disk = true; locality.on_tape = true; } else if (locality_text == "LOST") { gfal2_set_error(err, http_plugin_domain, ENOENT, __func__, "[Tape REST API] File locality reported as LOST (path=%s)", path.c_str()); } else if (locality_text == "NONE") { gfal2_set_error(err, http_plugin_domain, EPERM, __func__, "[Tape REST API] File locality reported as NONE (path=%s)", path.c_str()); } else if (locality_text == "UNAVAILABLE") { gfal2_set_error(err, http_plugin_domain, EAGAIN, __func__, "[Tape REST API] File locality reported as UNAVAILABLE (path=%s)", path.c_str()); }else { gfal2_set_error(err, http_plugin_domain, ENOMSG, __func__, "[Tape REST API] File locality reported as \"%s\" (path=%s)", locality_text.c_str(), path.c_str()); } return locality; } } ssize_t gfal_http_getxattr_internal(plugin_handle plugin_data, const char* url, const char *key, char* buff, size_t s_buff, GError** err) { GError* tmp_err = NULL; GfalHttpPluginData* davix = gfal_http_get_plugin_context(plugin_data); Davix::Uri uri(url); if (uri.getStatus() != StatusCode::OK) { gfal2_set_error(err, http_plugin_domain, EINVAL, __func__, "Invalid URL: %s", url); return -1; } // Construct remote storage endpoint std::stringstream endpoint; endpoint << uri.getProtocol() << "://" << uri.getHost(); if (uri.getPort()) { endpoint << ":" << uri.getPort(); } auto it = davix->tape_endpoint_map.find(endpoint.str()); if (it == davix->tape_endpoint_map.end()) { davix->retrieve_and_store_tape_endpoint(endpoint.str(), &tmp_err); if (tmp_err != NULL) { *err = g_error_copy(tmp_err); g_clear_error(&tmp_err); return -1; } it = davix->tape_endpoint_map.find(endpoint.str()); } if (strcmp(key, GFAL_XATTR_TAPE_API_VERSION) == 0) { strncpy(buff, it->second.version.c_str(), s_buff); } else if (strcmp(key, GFAL_XATTR_TAPE_API_URI) == 0) { strncpy(buff, it->second.uri.c_str(), s_buff); } else if (strcmp(key, GFAL_XATTR_TAPE_API_SITENAME) == 0) { strncpy(buff, it->second.sitename.c_str(), s_buff); } else { gfal2_set_error(err, http_plugin_domain, ENODATA, __func__, "Failed to get the xattr \"%s\" (No data available)", key); return -1; } return strnlen(buff, s_buff); } int gfal_http_bring_online(plugin_handle plugin_data, const char* url, time_t pintime, time_t timeout, char* token, size_t tsize, int async, GError** err) { GError* errors[1] = {NULL}; const char* const urls[1] = {url}; const char* const metadata_list[1] = {0}; int ret = gfal_http_bring_online_list_v2(plugin_data, 1, urls, metadata_list, pintime, timeout,token, tsize, async, err); if (errors[0] != NULL) { *err = errors[0]; } return ret; } int gfal_http_bring_online_list(plugin_handle plugin_data, int nbfiles, const char* const* urls, time_t pintime, time_t timeout, char* token, size_t tsize, int async, GError** errors) { const char* metadata_list[nbfiles]; for (int i = 0; i < nbfiles; ++i) { metadata_list[i] = {0}; } int ret = gfal_http_bring_online_list_v2(plugin_data, nbfiles, urls, metadata_list, pintime, timeout, token, tsize, async, errors); return ret; } int gfal_http_bring_online_v2(plugin_handle plugin_data, const char* url, const char* metadata, time_t pintime, time_t timeout, char* token, size_t tsize, int async, GError** err) { GError* errors[1] = {NULL}; const char* const urls[1] = {url}; const char* const metadata_list[1] = {metadata}; int ret = gfal_http_bring_online_list_v2(plugin_data, 1, urls, metadata_list, pintime, timeout,token, tsize, async, err); if (errors[0] != NULL) { *err = errors[0]; } return ret; } int gfal_http_bring_online_list_v2(plugin_handle plugin_data, int nbfiles, const char* const* urls, const char* const* metadata, time_t pintime, time_t timeout, char* token, size_t tsize, int async, GError** errors) { if (nbfiles <= 0) { return -1; } GError* tmp_err = NULL; // Check if all the metadata is in a valid JSON format if (tape_rest_api::metadata_format_checker(nbfiles, metadata, &tmp_err)) { tape_rest_api::copyErrors(tmp_err, nbfiles, errors); return -1; } // Find out Tape Rest API endpoint GfalHttpPluginData* davix = gfal_http_get_plugin_context(plugin_data); std::string tapeEndpoint = gfal_http_discover_tape_endpoint(davix, urls[0], "/stage", &tmp_err); if (tmp_err != NULL) { tape_rest_api::copyErrors(tmp_err, nbfiles, errors); return -1; } // Construct and send "POST /stage/" request Davix::DavixError* reqerr = NULL; Davix::Uri uri(tapeEndpoint); Davix::RequestParams params; PostRequest request(davix->context, uri, &reqerr); davix->get_params(¶ms, uri, GfalHttpPluginData::OP::TAPE); params.addHeader("Content-Type", "application/json"); request.setParameters(params); request.setRequestBody(tape_rest_api::stage_request_body(pintime, nbfiles, urls, metadata)); if (request.executeRequest(&reqerr)) { gfal2_set_error(&tmp_err, http_plugin_domain, davix2errno(reqerr->getStatus()), __func__, "[Tape REST API] Stage call failed: %s", reqerr->getErrMsg().c_str()); tape_rest_api::copyErrors(tmp_err, nbfiles, errors); Davix::DavixError::clearError(&reqerr); return -1; } if (request.getRequestCode() != 201) { gfal2_set_error(&tmp_err, http_plugin_domain, EINVAL, __func__, "[Tape REST API] Stage call failed: %s: %s", reqerr->getErrMsg().c_str(), request.getAnswerContent()); tape_rest_api::copyErrors(tmp_err, nbfiles, errors); Davix::DavixError::clearError(&reqerr); return -1; } std::string content = std::string(request.getAnswerContent()); if (content.empty()) { gfal2_set_error(&tmp_err, http_plugin_domain, ENOMSG, __func__, "[Tape REST API] Response with no data"); tape_rest_api::copyErrors(tmp_err, nbfiles, errors); return -1; } struct json_object* json_response = json_tokener_parse(content.c_str()); if (!json_response) { gfal2_set_error(&tmp_err, http_plugin_domain, ENOMSG, __func__, "[Tape REST API] Malformed served response"); tape_rest_api::copyErrors(tmp_err, nbfiles, errors); return -1; } // Check if "requestId" attribute exists struct json_object* id = 0; bool foundId = json_object_object_get_ex(json_response, "requestId", &id); if (!foundId) { gfal2_set_error(&tmp_err, http_plugin_domain, ENOMSG, __func__, "[Tape REST API] requestID attribute missing"); tape_rest_api::copyErrors(tmp_err, nbfiles, errors); return -1; } std::string reqid = json_object_get_string(id); // Copy request id to token buffer g_strlcpy(token, reqid.c_str(), tsize); // Free the top JSON object json_object_put(json_response); return 0; } int gfal_http_abort_files(plugin_handle plugin_data, int nbfiles, const char* const* urls, const char* token, GError** errors) { if (nbfiles <= 0) { return -1; } GError* tmp_err = NULL; if (!token || strlen(token) == 0) { gfal2_set_error(&tmp_err, http_plugin_domain, EINVAL, __func__, "The request ID was not provided"); tape_rest_api::copyErrors(tmp_err, nbfiles, errors); return -1; } std::stringstream method; method << "/stage/" << token << "/cancel"; // Find out Tape Rest API endpoint GfalHttpPluginData* davix = gfal_http_get_plugin_context(plugin_data); std::string tapeEndpoint = gfal_http_discover_tape_endpoint(davix, urls[0], method.str().c_str(), &tmp_err); if (tmp_err != NULL) { tape_rest_api::copyErrors(tmp_err, nbfiles, errors); return -1; } // Construct and send "POST /stage//cancel" request Davix::DavixError* reqerr = NULL; Davix::Uri uri(tapeEndpoint); Davix::RequestParams params; PostRequest request(davix->context, uri, &reqerr); davix->get_params(¶ms, uri, GfalHttpPluginData::OP::TAPE); params.addHeader("Content-Type", "application/json"); request.setParameters(params); request.setRequestBody(tape_rest_api::list_files_body(nbfiles, urls)); if (request.executeRequest(&reqerr)) { gfal2_set_error(&tmp_err, http_plugin_domain, davix2errno(reqerr->getStatus()), __func__, "[Tape REST API] Cancel call failed: %s", reqerr->getErrMsg().c_str()); tape_rest_api::copyErrors(tmp_err, nbfiles, errors); Davix::DavixError::clearError(&reqerr); return -1; } if (request.getRequestCode() != 200) { gfal2_set_error(&tmp_err, http_plugin_domain, EINVAL, __func__, "[Tape REST API] Stage call failed: %s: %s", reqerr->getErrMsg().c_str(), request.getAnswerContent()); tape_rest_api::copyErrors(tmp_err, nbfiles, errors); Davix::DavixError::clearError(&reqerr); return -1; } return 0; } int gfal_http_bring_online_poll(plugin_handle plugin_data, const char* url, const char* token, GError** err) { GError* errors[1] = {NULL}; const char* const urls[1] = {url}; int ret = gfal_http_bring_online_poll_list(plugin_data, 1, urls, token, err); if (errors[0] != NULL) { *err = errors[0]; } return ret; } int gfal_http_bring_online_poll_list(plugin_handle plugin_data, int nbfiles, const char* const* urls, const char* token, GError** errors) { if (nbfiles <= 0) { return -1; } GError* tmp_err = NULL; if (!token || strlen(token) == 0) { gfal2_set_error(&tmp_err, http_plugin_domain, EINVAL, __func__, "The request ID was not provided"); tape_rest_api::copyErrors(tmp_err, nbfiles, errors); return -1; } std::stringstream method; method << "/stage/" << token; // Find out Tape Rest API endpoint GfalHttpPluginData* davix = gfal_http_get_plugin_context(plugin_data); std::string tapeEndpoint = gfal_http_discover_tape_endpoint(davix, urls[0], method.str().c_str(), &tmp_err); if (tmp_err != NULL) { tape_rest_api::copyErrors(tmp_err, nbfiles, errors); return -1; } // Construct and send "GET /stage/" request Davix::DavixError* reqerr = NULL; Davix::Uri uri(tapeEndpoint); Davix::RequestParams params; GetRequest request(davix->context, uri, &reqerr); davix->get_params(¶ms, uri, GfalHttpPluginData::OP::TAPE); request.setParameters(params); if (request.executeRequest(&reqerr)) { gfal2_set_error(&tmp_err, http_plugin_domain, davix2errno(reqerr->getStatus()), __func__, "[Tape REST API] Stage pooling call failed: %s", reqerr->getErrMsg().c_str()); tape_rest_api::copyErrors(tmp_err, nbfiles, errors); Davix::DavixError::clearError(&reqerr); return -1; } if (request.getRequestCode() != 200) { gfal2_set_error(&tmp_err, http_plugin_domain, EINVAL, __func__, "[Tape REST API] Stage call failed: %s: %s)", reqerr->getErrMsg().c_str(), request.getAnswerContent()); tape_rest_api::copyErrors(tmp_err, nbfiles, errors); Davix::DavixError::clearError(&reqerr); return -1; } std::string content = std::string(request.getAnswerContent()); if (content.empty()) { gfal2_set_error(&tmp_err, http_plugin_domain, ENOMSG, __func__, "[Tape REST API] Response with no data"); tape_rest_api::copyErrors(tmp_err, nbfiles, errors); return -1; } struct json_object* json_response = json_tokener_parse(content.c_str()); if (!json_response) { gfal2_set_error(&tmp_err, http_plugin_domain, ENOMSG, __func__, "[Tape REST API] Malformed served response"); tape_rest_api::copyErrors(tmp_err, nbfiles, errors); return -1; } // Check if "id" attribute exists struct json_object* id = 0; bool foundId = json_object_object_get_ex(json_response, "id", &id); std::string reqid = foundId ? json_object_get_string(id) : ""; // Check if "request_id" attribute matches if (reqid.empty()) { gfal2_set_error(&tmp_err, http_plugin_domain, ENOMSG, __func__, "[Tape REST API] Request ID missing from polling response (expected id=%s)", token); tape_rest_api::copyErrors(tmp_err, nbfiles, errors); return -1; } if (reqid != token) { gfal2_set_error(&tmp_err, http_plugin_domain, ENOMSG, __func__, "[Tape REST API] Request ID mismatch. Expected id=%s but received id=%s", token, reqid.c_str()); tape_rest_api::copyErrors(tmp_err, nbfiles, errors); return -1; } // Check if "files" attribute exists struct json_object* files = 0; bool foundFiles = json_object_object_get_ex(json_response, "files", &files); if (!foundFiles) { gfal2_set_error(&tmp_err, http_plugin_domain, ENOMSG, __func__, "[Tape REST API] Files attribute missing from server poll response"); tape_rest_api::copyErrors(tmp_err, nbfiles, errors); return -1; } // Iterate over the "files" list int online_count = 0; int error_count = 0; for (int i = 0; i < nbfiles; ++i) { std::string path = Davix::Uri(urls[i]).getPath(); struct json_object* file = tape_rest_api::polling_get_item_by_path(files, path); if (file == NULL) { error_count++; gfal2_set_error(&errors[i], http_plugin_domain, ENOMSG, __func__, "[Tape REST API] Missing response item for path=%s", path.c_str()); continue; } // Check if "error" attribute exists struct json_object* file_error_text = 0; bool foundError = json_object_object_get_ex(file, "error", &file_error_text); if (foundError) { error_count++; std::string error_text = json_object_get_string(file_error_text); gfal2_set_error(&errors[i], http_plugin_domain, ENOMSG, __func__, "[Tape REST API] %s", error_text.c_str()); continue; } // Retrieve "onDisk" attribute struct json_object* file_on_disk = 0; bool foundOnDisk = json_object_object_get_ex(file, "onDisk", &file_on_disk); if (foundOnDisk) { std::string disk = json_object_get_string(file_on_disk); std::transform(disk.begin(), disk.end(), disk.begin(), tolower); if (disk == "true") { online_count++; continue; } else { gfal2_set_error(&errors[i], http_plugin_domain, EAGAIN, __func__, "[Tape REST API] File %s is not yet on disk", path.c_str()); continue; } } // Retrieve "state" attribute struct json_object* file_state = 0; bool foundState = json_object_object_get_ex(file, "state", &file_state); if (!foundState) { error_count++; gfal2_set_error(&errors[i], http_plugin_domain, ENOMSG, __func__, "[Tape REST API] State and onDisk attributes missing"); continue; } std::string state = json_object_get_string(file_state); if (state == "COMPLETED") { online_count++; } else if (state == "STARTED" || state == "SUBMITTED") { gfal2_set_error(&errors[i], http_plugin_domain, EAGAIN, __func__, "[Tape REST API] File %s is not yet on disk", path.c_str()); } else if (state == "CANCELED") { gfal2_set_error(&errors[i], http_plugin_domain, ECANCELED, __func__, "[Tape REST API] Staging operation cancelled. File=%s", path.c_str()); error_count++; } else if (state == "FAILED") { gfal2_set_error(&errors[i], http_plugin_domain, ENOENT, __func__, "[Tape REST API] Staging operation failed for file=%s", path.c_str()); error_count++; } else { gfal2_set_error(&errors[i], http_plugin_domain, ENOENT, __func__, "[Tape REST API] Unrecognized staging status. File=%s status=%s", path.c_str(), state.c_str()); error_count++; } } // Free the top JSON object json_object_put(json_response); // All files are on disk: return 1 if (online_count == nbfiles) { return 1; } // All files encountered errors: return -1 if (error_count == nbfiles) { return -1; } // Some files are on disk, others encountered errors if (online_count + error_count == nbfiles) { return 2; } // Staging still in process: return 0 return 0; } ssize_t gfal_http_status_getxattr(plugin_handle plugin_data, const char* url, char* buff, size_t s_buff, GError** err) { GError* tmp_err = NULL; const char* const urls[1] = {url}; std::string content = tape_rest_api::get_archiveinfo(plugin_data, 1, urls, &tmp_err); if (tmp_err != NULL) { *err = g_error_copy(tmp_err); g_error_free(tmp_err); return -1; } struct json_object* json_response = json_tokener_parse(content.c_str()); if (!json_response) { gfal2_set_error(err, http_plugin_domain, ENOMSG, __func__, "[Tape REST API] Malformed server response"); return -1; } std::string path = Uri(url).getPath(); struct json_object* file = tape_rest_api::polling_get_item_by_path(json_response, path); tape_rest_api::file_locality_t locality = tape_rest_api::get_file_locality(file, path, &tmp_err, true); // Free the top JSON object json_object_put(json_response); if (tmp_err != NULL) { *err = g_error_copy(tmp_err); g_clear_error(&tmp_err); return -1; } if (locality.on_tape && locality.on_disk) { strncpy(buff, GFAL_XATTR_STATUS_NEARLINE_ONLINE, s_buff); gfal2_log(G_LOG_LEVEL_DEBUG, GFAL_XATTR_STATUS_NEARLINE_ONLINE); } else if (locality.on_tape) { strncpy(buff, GFAL_XATTR_STATUS_NEARLINE, s_buff); gfal2_log(G_LOG_LEVEL_DEBUG, GFAL_XATTR_STATUS_NEARLINE); } else if (locality.on_disk) { strncpy(buff, GFAL_XATTR_STATUS_ONLINE, s_buff); gfal2_log(G_LOG_LEVEL_DEBUG, GFAL_XATTR_STATUS_ONLINE); } else { strncpy(buff, GFAL_XATTR_STATUS_UNKNOWN, s_buff); gfal2_log(G_LOG_LEVEL_DEBUG, GFAL_XATTR_STATUS_UNKNOWN); } return strnlen(buff, s_buff); } int gfal_http_archive_poll(plugin_handle plugin_data, const char* url, GError** err) { GError* errors[1] = {NULL}; const char* const urls[1] = {url}; int ret = gfal_http_archive_poll_list(plugin_data, 1, urls, err); if (errors[0] != NULL) { *err = errors[0]; } return ret; } int gfal_http_archive_poll_list(plugin_handle plugin_data, int nbfiles, const char* const* urls, GError** errors) { if (nbfiles <= 0) { return -1; } GError* tmp_err = NULL; std::string content = tape_rest_api::get_archiveinfo(plugin_data, nbfiles, urls, &tmp_err); if (tmp_err != NULL) { tape_rest_api::copyErrors(tmp_err, nbfiles, errors); return -1; } struct json_object* json_response = json_tokener_parse(content.c_str()); if (!json_response) { gfal2_set_error(&tmp_err, http_plugin_domain, ENOMSG, __func__, "[Tape REST API] Malformed server response"); tape_rest_api::copyErrors(tmp_err, nbfiles, errors); return -1; } // Iterate over the file list int ontape_count = 0; int error_count = 0; for (int i = 0; i < nbfiles; ++i) { std::string path = Davix::Uri(urls[i]).getPath(); struct json_object* file = tape_rest_api::polling_get_item_by_path(json_response, path); auto locality = tape_rest_api::get_file_locality(file, path, &tmp_err); if (tmp_err != NULL) { errors[i] = g_error_copy(tmp_err); g_clear_error(&tmp_err); error_count++; continue; } if (locality.on_tape) { ontape_count++; } else { gfal2_set_error(&errors[i], http_plugin_domain, EAGAIN, __func__, "[Tape REST API] File %s is not yet archived", path.c_str()); } } // Free the top JSON object json_object_put(json_response); // All files are on tape: return 1 if (ontape_count == nbfiles) { return 1; } // All files encountered errors: return -1 if (error_count == nbfiles) { return -1; } // Some files are on tape, others encountered errors if (ontape_count + error_count == nbfiles) { return 2; } // Archiving in process: return 0 return 0; } int gfal_http_release_file(plugin_handle plugin_data, const char* url, const char* request_id, GError** err) { GError* errors[1] = {NULL}; const char* const urls[1] = {url}; int ret = gfal_http_release_file_list(plugin_data, 1, urls, request_id, errors); if (errors[0] != NULL) { *err = errors[0]; } return ret; } int gfal_http_release_file_list(plugin_handle plugin_data, int nbfiles, const char* const* urls, const char* request_id, GError** errors) { if (nbfiles <= 0) { return -1; } GError* tmp_err = NULL; std::stringstream method; method << "/release/" << ((request_id && strlen(request_id) > 0) ? request_id : "gfal2-placeholder-id"); // Find out Tape REST API endpoint GfalHttpPluginData* davix = gfal_http_get_plugin_context(plugin_data); std::string tapeEndpoint = gfal_http_discover_tape_endpoint(davix, urls[0], method.str().c_str(), &tmp_err); if (tmp_err != NULL) { tape_rest_api::copyErrors(tmp_err, nbfiles, errors); return -1; } // Construct and send "POST /release" request Davix::DavixError* reqerr = NULL; Davix::Uri uri(tapeEndpoint); Davix::RequestParams params; PostRequest request(davix->context, uri, &reqerr); davix->get_params(¶ms, uri, GfalHttpPluginData::OP::TAPE); params.addHeader("Content-Type", "application/json"); request.setParameters(params); request.setRequestBody(tape_rest_api::list_files_body(nbfiles, urls)); if (request.executeRequest(&reqerr)) { gfal2_set_error(&tmp_err, http_plugin_domain, davix2errno(reqerr->getStatus()), __func__, "[Tape REST API] Release call failed: %s", reqerr->getErrMsg().c_str()); tape_rest_api::copyErrors(tmp_err, nbfiles, errors); Davix::DavixError::clearError(&reqerr); return -1; } if (request.getRequestCode() != 200) { gfal2_set_error(&tmp_err, http_plugin_domain, EINVAL, __func__, "[Tape REST API] Release call failed: %s: %s", reqerr->getErrMsg().c_str(), request.getAnswerContent()); tape_rest_api::copyErrors(tmp_err, nbfiles, errors); Davix::DavixError::clearError(&reqerr); return -1; } return 0; } gfal2-v2.23.0/src/plugins/http/gfal_http_plugin_token.cpp000066400000000000000000000366431465240014500234550ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2021 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include "json.h" #include "gfal_http_plugin.h" #include "exceptions/gfalcoreexception.hpp" #ifndef ENODATA #define ENODATA ENOATTR #endif /* * NOTE: * The token interaction in this unit file is largely based * on the x509-scitokens-issuer project (https://github.com/scitokens/x509-scitokens-issuer). * * As SE-issued tokens are to become an integral part of the Gfal2 HTTP plugin, * the goal was to move the token-related client interaction here. */ using namespace Davix; ssize_t gfal_http_token_retrieve(plugin_handle plugin_data, const char* url, const char* issuer, gboolean write_access, unsigned validity, const char* const* activities, char* buff, size_t s_buff, GError** err) { GfalHttpPluginData* davix = gfal_http_get_plugin_context(plugin_data); TokenRetriever* retriever_chain = NULL; ssize_t ret = -1; // Emulate GfalHttpPluginData::get_params(..) without the get_credentials(..) part Davix::RequestParams params = davix->reference_params; davix->get_params_internal(params, Davix::Uri(url)); if (issuer && *issuer) { retriever_chain = new SciTokensRetriever(issuer); retriever_chain->add(new MacaroonRetriever(issuer)); } else { retriever_chain = new MacaroonRetriever(); } std::string token; std::string last_emsg; TokenRetriever* retriever = retriever_chain; while (retriever != NULL) { try { gfal_http_token_t http_token = retriever->retrieve_token(Davix::Uri(url), params, write_access, validity, activities); token = http_token.token; break; } catch (const Gfal::CoreException& e) { gfal2_log(G_LOG_LEVEL_INFO, "(SEToken) Error during token retrieval: %s", e.what()); last_emsg = e.what_str(); retriever = retriever->next(); } } if (token.empty()) { gfal2_set_error(err, http_plugin_domain, ENODATA, __func__, "Could not retrieve token for %s [last failed attempt: %s]", url, last_emsg.c_str()); } else if (token.size() >= s_buff) { gfal2_set_error(err, http_plugin_domain, ENOMEM, __func__, "response larger than allocated buffer size [%zd]", s_buff); } else { std::strcpy(buff, token.c_str()); ret = token.size() + 1; } delete retriever_chain; return ret; } // -------------------------------------------------------- // General Token Retrieval implementation // -------------------------------------------------------- TokenRetriever::TokenRetriever(std::string label, std::string issuer): label(std::move(label)), issuer(std::move(issuer)), context(), discovery_fallback(false), token_key("access_token") { context.loadModule("grid"); } TokenRetriever* TokenRetriever::add(TokenRetriever* elem) { _next.reset(elem); return _next.get(); } TokenRetriever* TokenRetriever::next() { return _next.get(); } Uri TokenRetriever::format_protocol(const Uri& _url) { Uri url(_url); if (url.getStatus() != Davix::StatusCode::OK) { std::stringstream errmsg; errmsg << "Failed to parse url '" << url.getString() << "'"; throw Gfal::CoreException(http_plugin_domain, EINVAL, errmsg.str()); } if (url.getProtocol() == "davs") { url.setProtocol("https"); } if (url.getProtocol() != "https") { throw Gfal::CoreException(http_plugin_domain, EINVAL, "Token request must be done over HTTPs"); } return url; } std::string TokenRetriever::get_token_endpoint(RequestParams& params) { Uri url = format_protocol(Uri(issuer)); std::string oauth_endpoint = _metadata_endpoint(url); std::string endpoint = _endpoint_discovery(oauth_endpoint, params); if (!endpoint.empty() || !discovery_fallback) { return endpoint; } std::string config_url = issuer; if (config_url[config_url.size() -1 ] != '/') { config_url += "/"; } config_url += ".well-known/openid-configuration"; return _endpoint_discovery(config_url, params); } std::string TokenRetriever::_metadata_endpoint(const Uri& url) { std::stringstream endpoint; endpoint << url.getProtocol() << "://" << url.getHost(); if (url.getPort()) { endpoint << ":" << url.getPort(); } endpoint << "/.well-known/oauth-authorization-server"; if (url.getPath() != "/") { endpoint << url.getPath(); } return endpoint.str(); } std::string TokenRetriever::_endpoint_discovery(const std::string& metadata_url, RequestParams& params) { DavixError* err = NULL; GetRequest request(context, metadata_url, &err); request.setParameters(params); std::string response = perform_request(request, "Token endpoint discovery"); return parse_json_response(response, "token_endpoint"); } std::string TokenRetriever::parse_json_response(const std::string& response, const std::string& key) { json_object* json_response; json_object* json_item; if (response.empty()) { throw Gfal::CoreException(http_plugin_domain, EINVAL, "Response with no data"); } json_response = json_tokener_parse(response.c_str()); if (!json_response) { throw Gfal::CoreException(http_plugin_domain, EINVAL, "Response was not valid JSON"); } if (!json_object_object_get_ex(json_response, key.c_str(), &json_item)) { std::stringstream errmsg; errmsg << "Response did not include '" << key << "' key"; json_object_put(json_response); throw Gfal::CoreException(http_plugin_domain, EINVAL, errmsg.str()); } const char* value = json_object_get_string(json_item); if (!value) { std::stringstream errmsg; errmsg << "Key '" << key << "' was not a string"; json_object_put(json_response); throw Gfal::CoreException(http_plugin_domain, EINVAL, errmsg.str()); } std::string result(value); json_object_put(json_response); if (result.empty()) { std::stringstream errmsg; errmsg << "Extracted value for key '" << key << "' is empty"; throw Gfal::CoreException(http_plugin_domain, EINVAL, errmsg.str()); } return result; } std::string TokenRetriever::perform_request(HttpRequest& request, std::string description) { DavixError* err = NULL; if (description.empty()) { description = label; } if (request.executeRequest(&err)) { std::stringstream errmsg; errmsg << description << " request failed: " << err->getErrMsg(); throw Gfal::CoreException(http_plugin_domain, davix2errno(err->getStatus()), errmsg.str()); } if (request.getRequestCode() != 200) { std::stringstream errmsg; errmsg << description << " request failed with status code: " << request.getRequestCode(); throw Gfal::CoreException(http_plugin_domain, davix2errno(err->getStatus()), errmsg.str()); } return std::string(request.getAnswerContent()); } gfal_http_token_t TokenRetriever::retrieve_token(const Uri& _url, const RequestParams& _params, bool write_access, unsigned validity, const char* const* activities) { Uri url = format_protocol(_url); RequestParams params(_params); params.setProtocol(RequestProtocol::Http); std::string path = url.getPath(); std::string endpoint; try { endpoint = (!issuer.empty()) ? get_token_endpoint(params) : ""; } catch (const Gfal::CoreException& e) { gfal2_log(G_LOG_LEVEL_DEBUG, "(SEToken) Error during issuer endpoint discovery: %s", e.what()); } // Let sub-classes validate token endpoint if (!validate_endpoint(endpoint, url)) { throw Gfal::CoreException(http_plugin_domain, EINVAL, "Invalid or empty token issuer endpoint"); } DavixError* err = NULL; PostRequest request(context, endpoint, &err); request.setParameters(params); // Let sub-classes prepare the request prepare_request(request, path, write_access, validity, activities); std::string response = perform_request(request); std::string stoken = parse_json_response(response, token_key); gfal_http_token_t token = {stoken, validity, write_access}; return token; } // -------------------------------------------------------- // Macaroon-specific Retrieval implementation // -------------------------------------------------------- MacaroonRetriever::MacaroonRetriever(): MacaroonRetriever("") { } MacaroonRetriever::MacaroonRetriever(std::string issuer): TokenRetriever("Macaroon", std::move(issuer)), is_oauth(false) { discovery_fallback = true; } bool MacaroonRetriever::validate_endpoint(std::string& endpoint, const Uri& url) { is_oauth = !endpoint.empty(); if (endpoint.empty()) { endpoint = url.getString(); } return true; } void MacaroonRetriever::prepare_request(HttpRequest& request, const std::string& path, bool write_access, unsigned validity, const char* const* activities) { std::vector v_activities = _activities(write_access, activities); if (is_oauth) { request.addHeaderField("Content-Type", "application/x-www-form-urlencoded"); request.addHeaderField("Accept", "application/json"); request.setRequestBody(oauth_request_content(path, validity, v_activities)); } else { request.addHeaderField("Content-Type", "application/macaroon-request"); request.setRequestBody(macaroon_request_content(validity, v_activities)); } token_key = (is_oauth) ? "access_token" : "macaroon"; } std::string MacaroonRetriever::perform_request(HttpRequest& request, std::string description) { std::vector buffer(MacaroonRetriever::RESPONSE_MAX_SIZE); DavixError* err = NULL; description = (is_oauth) ? "Token" : "Macaroon"; if (request.beginRequest(&err)) { std::stringstream errmsg; errmsg << description << " request failed: " << err->getErrMsg(); throw Gfal::CoreException(http_plugin_domain, davix2errno(err->getStatus()), errmsg.str()); } dav_ssize_t response_size = request.getAnswerSize(); if (response_size >= MacaroonRetriever::RESPONSE_MAX_SIZE) { std::stringstream errmsg; errmsg << description << " response exceeds maximum size: " << response_size << " bytes (max size = " << MacaroonRetriever::RESPONSE_MAX_SIZE << ")"; throw Gfal::CoreException(http_plugin_domain, EINVAL, errmsg.str()); } // StoRM has an interesting bug where an unknown/unhandled POST is treated like a corresponding GET, // meaning it would respond to the macaroon request with the entire file itself. // To protect against this, we read out at most 1MB. dav_ssize_t segment_size = request.readSegment(&buffer[0], MacaroonRetriever::RESPONSE_MAX_SIZE, &err); if (segment_size < 0) { std::stringstream errmsg; errmsg << "Reading body of " << description << " request failed: " << err->getErrMsg(); throw Gfal::CoreException(http_plugin_domain, davix2errno(err->getStatus()), errmsg.str()); } if (segment_size >= MacaroonRetriever::RESPONSE_MAX_SIZE) { std::stringstream errmsg; errmsg << description << " response exceeds maximum size: " << segment_size << " bytes (max size = " << MacaroonRetriever::RESPONSE_MAX_SIZE << ")"; throw Gfal::CoreException(http_plugin_domain, EINVAL, errmsg.str()); } if (request.getRequestCode() != 200) { std::stringstream errmsg; errmsg << description << " request failed with status code " << request.getRequestCode(); throw Gfal::CoreException(http_plugin_domain, davix2errno(err->getStatus()), errmsg.str()); } return std::string(&buffer[0], segment_size); } std::string MacaroonRetriever::macaroon_request_content(unsigned validity, const std::vector& activities) { std::stringstream content; content << "{\"caveats\": [\"activity:"; std::vector::const_iterator it; for (it = activities.begin(); it != activities.end(); it++) { if (it != activities.begin()) { content << ","; } content << (*it); } content << "\"], \"validity\": \"PT" << validity << "M\"}"; return content.str(); } std::string MacaroonRetriever::oauth_request_content(const std::string& path, unsigned validity, const std::vector& activities) { std::stringstream scopes; std::vector::const_iterator it; for (it = activities.begin(); it != activities.end(); it++) { if (it != activities.begin()) { scopes << " "; } scopes << (*it) << ":" << path; } std::stringstream content; content << "grant_type=client_credentials&expire_in=" << (validity * 60); content << "&scopes=" << Uri::queryParamEscape(scopes.str()); return content.str(); } std::vector MacaroonRetriever::_activities(bool write_access, const char* const* activities) { std::vector v_activities; // User-provided activities if (activities && activities[0]) { for (int idx = 0; activities[idx]; idx++) { v_activities.emplace_back(activities[idx]); } return v_activities; } // Construct activities based on read/write flag v_activities.emplace_back("LIST"); v_activities.emplace_back("DOWNLOAD"); if (write_access) { v_activities.emplace_back("MANAGE"); v_activities.emplace_back("UPLOAD"); v_activities.emplace_back("DELETE"); } return v_activities; } // -------------------------------------------------------- // SciTokens-specific Retrieval implementation // -------------------------------------------------------- SciTokensRetriever::SciTokensRetriever(std::string issuer): TokenRetriever("SciTokens", std::move(issuer)) { } bool SciTokensRetriever::validate_endpoint(std::string& endpoint, const Davix::Uri& uri) { return !endpoint.empty(); } void SciTokensRetriever::prepare_request(HttpRequest& request, const std::string& path, bool write_access, unsigned validity, const char* const* activities) { request.addHeaderField("Accept", "application/json"); request.addHeaderField("Content-Type", "application/x-www-form-urlencoded"); request.setRequestBody("grant_type=client_credentials"); } gfal2-v2.23.0/src/plugins/http/gfal_http_plugin_token.h000066400000000000000000000171071465240014500231140ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2021 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #ifndef _GFAL_HTTP_PLUGIN_TOKEN_H #define _GFAL_HTTP_PLUGIN_TOKEN_H #include #include "gfal_http_plugin.h" /** * Gfal2 HTTP plugin abstraction for a bearer token. * Holds the token itself, validity and write access. * * The tokens are associated to a resource (URL) in the token internal map. * When the same resource is requested again, given the validity * and permissions match, the token retrieved from the map can be reused. */ struct gfal_http_token_s { std::string token; unsigned validity; bool write_access; }; typedef struct gfal_http_token_s gfal_http_token_t; class TokenRetriever { public: TokenRetriever(std::string label, std::string issuer); virtual ~TokenRetriever() = default; TokenRetriever* add(TokenRetriever* elem); TokenRetriever* next(); /** * Performs necessary steps to retrieve a SE-issued token for a given URL. * Token-specific steps are defined by sub-classes. * * @param _url path to the resource * @param _params Davix request parameters * @param write_access token needs write permissions * @params validity token validity expressed in minutes * @return Gfal2 HTTP token structure */ gfal_http_token_t retrieve_token(const Davix::Uri& _url, const Davix::RequestParams& _params, bool write_access, unsigned validity, const char* const* activities = NULL); /// Short class description const std::string label; /// Token issuer endpoint const std::string issuer; protected: /** * Template method to allow validating the token issuer endpoint. * @param endpoint the token issuer endpoint * @params uri the resource URL * @return true if valid endpoint, false otherwise */ virtual bool validate_endpoint(std::string& endpoint, const Davix::Uri& uri) = 0; /** * Template method to prepare the token request. * @param request Davix HTTP request * @param path the resource to access * @param write_access flag to signal write access needed * @params validity token validity expressed in minutes * @params activities activities for access request. * If null, they will be created according to write access */ virtual void prepare_request(Davix::HttpRequest& request, const std::string& path, bool write_access, unsigned validity, const char* const* activities = NULL) = 0; /** * Perform an HTTP request. * @param request Davix Request object * @param description the request description * @return the response body */ virtual std::string perform_request(Davix::HttpRequest& request, std::string description = ""); /** * Parse the JSON response and extract the given key. * @param response the server response * @param key the key to extract * @return extracted key value */ std::string parse_json_response(const std::string& response, const std::string& key); /// Separate Davix context object Davix::Context context; /// Endpoint discovery fallback flag bool discovery_fallback; /// Key used to extract the token from JSON response std::string token_key; /// Pointer to allow chaining token retrievers std::unique_ptr _next; private: /** * Checks that the protocol allows token retrieval (https:// or davs://). * @param _url input URL * @return HTTPS schema URL */ Davix::Uri format_protocol(const Davix::Uri& _url); /** * Identify the token issuer endpoint. * @param params Davix request parameters * @return the token issuer endpoint */ std::string get_token_endpoint(Davix::RequestParams& params); // Convenience function to build OAuth authorization endpoint std::string _metadata_endpoint(const Davix::Uri& url); // Query the metadata URL for the token issuer endpoint std::string _endpoint_discovery(const std::string& metadata_url, Davix::RequestParams& params); }; class MacaroonRetriever: public TokenRetriever { public: MacaroonRetriever(); MacaroonRetriever(std::string issuer); /// Maximum size in bytes accepted for token response static constexpr dav_ssize_t RESPONSE_MAX_SIZE {1024 * 1024}; protected: /** * Returns true in all cases. * For an empty endpoint, query the resource host directly. */ bool validate_endpoint(std::string& endpoint, const Davix::Uri& uri) override; /** * Macaroon specific request headers and content. * @param request Davix HTTP request * @param path the resource to access * @param write_access flag to signal write access needed * @param validity token validity expressed in minutes */ void prepare_request(Davix::HttpRequest& request, const std::string& path, bool write_access, unsigned validity, const char* const* activities) override; /** * Override for the base class perform HTTP request. * For macaroons, a special request is needed to accommodate a bug encountered in STORM. * @param request Davix Request object * @param description the request description * @return the response body */ std::string perform_request(Davix::HttpRequest& request, std::string description = "") override; private: // Create macaroon request content for the given validity and activities std::string macaroon_request_content(unsigned validity, const std::vector& activities); // Create oauth-type request content for the given path, validity and activities std::string oauth_request_content(const std::string& path, unsigned validity, const std::vector& activities); // Utility function to generate activities container // based on a list of activities or read/write access flag std::vector _activities(bool write_access, const char* const* activities); /// OAuth macaroon issuer flag bool is_oauth; }; class SciTokensRetriever: public TokenRetriever { public: SciTokensRetriever(std::string issuer); protected: /** * Returns true if endpoint is not empty. */ bool validate_endpoint(std::string& endpoint, const Davix::Uri& uri) override; /** * SciTokens specific request headers and content. * @param request Davix HTTP request * @param path the resource to access * @param write_access flag to signal write access needed * @param validity token validity expressed in minutes */ void prepare_request(Davix::HttpRequest& request, const std::string& path, bool write_access, unsigned validity, const char* const* activities) override; }; #endif //_GFAL_HTTP_PLUGIN_TOKEN_H gfal2-v2.23.0/src/plugins/http/gfal_http_qos.cpp000066400000000000000000000254161465240014500215550ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2015 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include #include #include #include #include "gfal_http_plugin.h" #pragma GCC diagnostic ignored "-Wdeprecated-declarations" using namespace Davix; ssize_t gfal_http_check_classes(plugin_handle plugin_data, const char* url, const char* type, char* buff, size_t s_buff, GError** err) { ssize_t ret = -1; if (type != NULL && (strcmp(type, "dataobject") == 0 || strcmp(type, "container") == 0 )) { GfalHttpPluginData* davix = gfal_http_get_plugin_context(plugin_data); DavixError* dav_err = NULL; Context c; std::string uri(url); uri += "/cdmi_capabilities/"; uri += type; HttpRequest r(c, uri, &dav_err); Davix::RequestParams req_params; davix->get_params(&req_params, Davix::Uri(url)); r.setParameters(req_params); if (!dav_err) { r.executeRequest(&dav_err); } if (dav_err) { std::cerr << " error in request of getting available QoS classes: " << dav_err->getErrMsg() << std::endl; davix2gliberr(dav_err, err, __func__); Davix::DavixError::clearError(&dav_err); } else { std::vector body = r.getAnswerContentVec(); std::string response(body.begin(), body.end()); json_object* info = json_tokener_parse(response.c_str()); std::string classes = json_object_get_string(json_object_object_get(info, "children")); // Remove all extra chars and create a comma separated string to return classes.erase(std::remove(classes.begin(), classes.end(), '['), classes.end()); classes.erase(std::remove(classes.begin(), classes.end(), ']'), classes.end()); classes.erase(std::remove(classes.begin(), classes.end(), ' '), classes.end()); classes.erase(std::remove(classes.begin(), classes.end(), '"'), classes.end()); // Adding prefix of cdmi capabilitiesURI std::string stype(type); std::string prefix = "/cdmi_capabilities/" + stype + "/"; std::istringstream iss(classes); std::string classToken; classes = ""; while (std::getline(iss, classToken, ',')) { classes += prefix + classToken + ","; } // Remove final comma classes.erase(classes.size() - 1); if (classes.size() < s_buff) { std::strcpy(buff, classes.c_str()); ret = classes.size() + 1; } else { gfal2_set_error(err, http_plugin_domain, ENOMEM, __func__, "response larger than allocated buffer size [%zd]", s_buff); } } } else { gfal2_set_error(err, http_plugin_domain, EINVAL, __func__, "type argument should be either dataobject or container"); } return ret; } ssize_t gfal_http_check_file_qos(plugin_handle plugin_data, const char* url, char* buff, size_t s_buff, GError** err) { GfalHttpPluginData* davix = gfal_http_get_plugin_context(plugin_data); DavixError* dav_err = NULL; ssize_t ret = -1; Context c; std::string uri(url); HttpRequest r(c, uri, &dav_err); Davix::RequestParams req_params; davix->get_params(&req_params, Davix::Uri(url)); r.setParameters(req_params); if (!dav_err) { r.executeRequest(&dav_err); } if (dav_err) { std::cerr << " error in request of checking file QoS: " << dav_err->getErrMsg() << std::endl; davix2gliberr(dav_err, err, __func__); Davix::DavixError::clearError(&dav_err); } else { std::vector body = r.getAnswerContentVec(); std::string response(body.begin(), body.end()); json_object *info = json_tokener_parse(response.c_str()); std::string qos_class = json_object_get_string(json_object_object_get(info, "capabilitiesURI")); qos_class.erase(std::remove(qos_class.begin(), qos_class.end(), '"'), qos_class.end()); if (qos_class.size() < s_buff) { std::strcpy(buff, qos_class.c_str()); ret = qos_class.size() + 1; } else { gfal2_set_error(err, http_plugin_domain, ENOMEM, __func__, "response larger than allocated buffer size [%zd]", s_buff); } } return ret; } ssize_t gfal_http_check_qos_available_transitions(plugin_handle plugin_data, const char* qos_class_url, char* buff, size_t s_buff, GError** err) { GfalHttpPluginData* davix = gfal_http_get_plugin_context(plugin_data); DavixError* dav_err = NULL; ssize_t ret = -1; Context c; std::string uri(qos_class_url); HttpRequest r(c, uri, &dav_err); Davix::RequestParams req_params; davix->get_params(&req_params, Davix::Uri(qos_class_url)); r.setParameters(req_params); if (!dav_err) { r.executeRequest(&dav_err); } if (dav_err) { std::cerr << " error in request of checking file QoS: " << dav_err->getErrMsg() << std::endl; davix2gliberr(dav_err, err, __func__); Davix::DavixError::clearError(&dav_err); } else { std::vector body = r.getAnswerContentVec(); std::string response(body.begin(), body.end()); json_object *info = json_tokener_parse(response.c_str()); json_object *metadata = json_object_object_get(info, "metadata"); std::string transitions = json_object_get_string(json_object_object_get(metadata, "cdmi_capabilities_allowed")); // Remove all extra chars and create a comma separated string to return transitions.erase(std::remove(transitions.begin(), transitions.end(), '['), transitions.end()); transitions.erase(std::remove(transitions.begin(), transitions.end(), ']'), transitions.end()); transitions.erase(std::remove(transitions.begin(), transitions.end(), ' '), transitions.end()); transitions.erase(std::remove(transitions.begin(), transitions.end(), '"'), transitions.end()); transitions.erase(std::remove(transitions.begin(), transitions.end(), '\\'), transitions.end()); if (transitions.size() < s_buff) { std::strcpy(buff, transitions.c_str()); ret = transitions.size() + 1; } else { gfal2_set_error(err, http_plugin_domain, ENOMEM, __func__, "response larger than allocated buffer size [%zd]", s_buff); } } return ret; } /* TODO: Have all get requests being done in a separate method, at the moment there is an issue with the second level json reading of gfal_http_check_qos_available_transitions when all tests are running std::string response = execute_get_request_to_cdmi(plugin_data, qosClassUrl); std::string execute_get_request_to_cdmi(plugin_handle plugin_data, const char *url) { GfalHttpPluginData* davix = gfal_http_get_plugin_context(plugin_data); Context c; DavixError* tmp_err=NULL; std::string uri(url); HttpRequest r(c, uri, &tmp_err); Davix::RequestParams req_params; davix->get_params(&req_params, Davix::Uri(url), false); r.setParameters(req_params); if(!tmp_err) r.executeRequest(&tmp_err); if(tmp_err){ std::cerr << " ERROR on get Request to the CDMI server: " << tmp_err->getErrMsg() << std::endl; } else { std::vector body = r.getAnswerContentVec(); std::string response(body.begin(), body.end()); return response; } return ""; }*/ ssize_t gfal_http_check_target_qos(plugin_handle plugin_data, const char* url, char* buff, size_t s_buff, GError** err) { GfalHttpPluginData* davix = gfal_http_get_plugin_context(plugin_data); DavixError* dav_err = NULL; ssize_t ret = -1; Context c; std::string uri(url); HttpRequest r(c, uri, &dav_err); Davix::RequestParams req_params; davix->get_params(&req_params, Davix::Uri(url)); r.setParameters(req_params); if (!dav_err) { r.executeRequest(&dav_err); } if (dav_err) { std::cerr << " error in request of checking file QoS: " << dav_err->getErrMsg() << std::endl; davix2gliberr(dav_err, err, __func__); Davix::DavixError::clearError(&dav_err); } else { std::vector body = r.getAnswerContentVec(); std::string response(body.begin(), body.end()); json_object *info = json_tokener_parse(response.c_str()); json_object *metadata = json_object_object_get(info, "metadata"); json_object *target = json_object_object_get(metadata, "cdmi_capabilities_target"); std::string target_qos = ""; if (target != NULL) { target_qos = json_object_get_string(target); // Remove all extra chars target_qos.erase(std::remove(target_qos.begin(), target_qos.end(), '['), target_qos.end()); target_qos.erase(std::remove(target_qos.begin(), target_qos.end(), ']'), target_qos.end()); target_qos.erase(std::remove(target_qos.begin(), target_qos.end(), ' '), target_qos.end()); target_qos.erase(std::remove(target_qos.begin(), target_qos.end(), '"'), target_qos.end()); target_qos.erase(std::remove(target_qos.begin(), target_qos.end(), '\\'), target_qos.end()); } if (target_qos.size() < s_buff) { std::strcpy(buff, target_qos.c_str()); ret = target_qos.size() + 1; } else { gfal2_set_error(err, http_plugin_domain, ENOMEM, __func__, "response larger than allocated buffer size [%zd]", s_buff); } } return ret; } int gfal_http_change_object_qos(plugin_handle plugin_data, const char* url, const char* target_qos, GError** err) { GfalHttpPluginData* davix = gfal_http_get_plugin_context(plugin_data); DavixError* dav_err = NULL; Context c; std::string uri(url); std::stringstream body; body << "{\"capabilitiesURI\":\"" << target_qos << "\"}"; PutRequest pr(c, uri, &dav_err); Davix::RequestParams req_params; davix->get_params(&req_params, Davix::Uri(url)); req_params.addHeader("Content-Type", "application/cdmi-object"); pr.setParameters(req_params); pr.setRequestBody(body.str()); if (!dav_err) { pr.executeRequest(&dav_err); } if (dav_err || !http_cdmi_code_is_valid(pr.getRequestCode())) { if (dav_err) { std::cerr << " error in request of changing file QoS: " << dav_err->getErrMsg() << std::endl; davix2gliberr(dav_err, err, __func__); Davix::DavixError::clearError(&dav_err); } else { std::cerr << " error in request of changing file QoS " << std::endl; } return -1; } return 0; } bool http_cdmi_code_is_valid(int code) { /* Should expect 204 as per CDMI document page 8 for a PUT request */ switch (code) { case 200: /* OK */ case 201: /* OK */ case 202: /* Accepted */ case 204: /* No Content */ return true; default: return false; } } gfal2-v2.23.0/src/plugins/lfc/000077500000000000000000000000001465240014500157745ustar00rootroot00000000000000gfal2-v2.23.0/src/plugins/lfc/CMakeLists.txt000066400000000000000000000021331465240014500205330ustar00rootroot00000000000000cmake_minimum_required (VERSION 2.6) if (PLUGIN_LFC) find_package (LFC REQUIRED) find_package (UUID REQUIRED) add_definitions (-D_REENTRANT) include_directories(${LFC_INCLUDE_DIR} ${LCGDM_INCLUDE_DIR} ${UUID_PKG_INCLUDE_DIR}) file (GLOB src_lfc "*.c*") add_library (plugin_lfc MODULE ${src_lfc}) target_link_libraries(plugin_lfc gfal2 gfal2_transfer ${LFC_LIBRARIES} ${UUID_PKG_LIBRARIES} ) set_target_properties(plugin_lfc PROPERTIES CLEAN_DIRECT_OUTPUT 1 OUTPUT_NAME "gfal_plugin_lfc" LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/plugins) install (TARGETS plugin_lfc LIBRARY DESTINATION ${PLUGIN_INSTALL_DIR}) install (FILES "README_PLUGIN_LFC" DESTINATION ${DOC_INSTALL_DIR}) # install lfc configuration files LIST(APPEND lfc_conf_file "${CMAKE_SOURCE_DIR}/dist/etc/gfal2.d/lfc_plugin.conf") install(FILES ${lfc_conf_file} DESTINATION ${SYSCONF_INSTALL_DIR}/gfal2.d/) endif (PLUGIN_LFC) gfal2-v2.23.0/src/plugins/lfc/README_PLUGIN_LFC000066400000000000000000000002531465240014500204160ustar00rootroot00000000000000 gfal2 lfc plugin : - features : * meta-data management * redirection to dcap / rfio for the data access Note this plugin can *not* be used in a thread safe manner. gfal2-v2.23.0/src/plugins/lfc/gfal_lfc.c000066400000000000000000001122101465240014500176720ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #ifndef ENOATTR #define ENOATTR ENODATA #endif #if defined __APPLE__ #include #else #if defined __GLIBC_PREREQ && __GLIBC_PREREQ(2,27) #include #else #include #endif #endif #include "gfal_lfc.h" #include "gfal_lfc_open.h" static gboolean init_thread = FALSE; pthread_mutex_t m_lfcinit = PTHREAD_MUTEX_INITIALIZER; typedef struct _lfc_opendir_handle { char url[GFAL_URL_MAX_LEN]; struct dirent current_dir; } *lfc_opendir_handle; static char *file_xattr[] = { GFAL_XATTR_GUID, GFAL_XATTR_REPLICA, GFAL_XATTR_COMMENT, GFAL_XATTR_CHKSUM_TYPE, GFAL_XATTR_CHKSUM_VALUE, NULL }; /* * just return the name of the layer */ const char *lfc_getName() { return GFAL2_PLUGIN_VERSIONED("lfc", VERSION); } // LFC plugin GQuark GQuark gfal2_get_plugin_lfc_quark() { return g_quark_from_static_string(GFAL2_QUARK_PLUGINS "::LFC"); } /* * convert the lfn url for internal usage * result must be free */ static inline char *lfc_urlconverter(const char *lfn_url, const char *prefix) { const int pref_len = strlen(prefix); const int strsize = strnlen(lfn_url, GFAL_URL_MAX_LEN - 1); const int res_len = strsize - pref_len; char *p, *pdest, *porg; p = pdest = g_malloc(sizeof(char) * (res_len + 1)); porg = (char *) lfn_url + pref_len; while ((pdest - p) < res_len && (porg - lfn_url) < strsize) { // remove double sep, remove end sep if ((*porg == '/' && *(porg + 1) == '/') == FALSE && (*porg == '/' && *(porg + 1) == '\0') == FALSE) { *pdest = *porg; ++pdest; } ++porg; } *(pdest) = '\0'; return p; } /* * convert the lfc type url "lfc://" to an url * result must be free */ static inline int lfc_full_urlconverter(const char *lfn_url, char **host, char **path, GError **err) { GError *tmp_err = NULL; int res = -1; const int pref_len = (sizeof(GFAL_LFC_PREFIX2) - 1); const int strsize = strnlen(lfn_url, GFAL_URL_MAX_LEN - 1); const int res_len = strsize - pref_len; char *p_org, *p_end, *p; p = (char *) lfn_url + pref_len; p_end = (char *) lfn_url + strsize; if (res_len > 0) { while (p < p_end && *p == '/') ++p; p_org = p; while (p < p_end && *p != '/') ++p; if (p_org < p && p < p_end) { if (host) { *host = g_strndup(p_org, p - p_org); } if (path) { *path = g_strndup(p, p_end - p); } res = 0; } } if (res != 0) { gfal2_set_error(&tmp_err, gfal2_get_plugin_lfc_quark(), EINVAL, __func__, "Invalid lfc:// url"); } return res; } /// manage convertion for all lfc url type : lfc://, lfn://, guid: /// return 0 if success, or -1 if bad url int url_converter(plugin_handle handle, const char *url, char **host, char **path, GError **err) { GError *tmp_err = NULL; int res = -1; if (strnlen(url, 5) != 5) { // bad string size, return empty string gfal2_log(G_LOG_LEVEL_WARNING, "lfc url converter -> bad url size"); return res; } if (strncmp(url, "lfn", 3) == 0) { if (path) { *path = lfc_urlconverter(url, GFAL_LFC_PREFIX); } if (host) { *host = g_strdup(lfc_plugin_get_lfc_env((struct lfc_ops *) handle, "LFC_HOST")); } res = 0; } else if (strncmp(url, "lfc", 3) == 0) { res = lfc_full_urlconverter(url, host, path, &tmp_err); } else { char buff_lfn[GFAL_URL_MAX_LEN]; res = gfal_convert_guid_to_lfn_r(handle, url + GFAL_LFC_GUID_PREFIX_LEN, buff_lfn, GFAL_URL_MAX_LEN, &tmp_err); if (path) { *path = g_strdup(buff_lfn); } } G_RETURN_ERR(res, tmp_err, err); } /* * Deleter to unload the lfc part * */ static void lfc_destroyG(plugin_handle handle) { struct lfc_ops *ops = (struct lfc_ops *) handle; if (ops) { gsimplecache_delete(ops->cache_stat); regfree(&(ops->rex)); free(ops); } } /* * Implementation of the chmod function with the LFC plugin * return 0 or the errno if error, or set GError if serious error */ int lfc_chmodG(plugin_handle handle, const char *path, mode_t mode, GError **err) { g_return_val_err_if_fail(handle && path, -1, err, "[lfc_chmodG] Invalid valid value in handle/path "); GError *tmp_err = NULL; struct lfc_ops *ops = (struct lfc_ops *) handle; int ret = -1; char *url_path = NULL, *url_host = NULL; if ((ret = url_converter(handle, path, &url_host, &url_path, &tmp_err)) == 0) { ret = lfc_configure_environment(ops, url_host, path, &tmp_err); if (!tmp_err) { gfal_auto_maintain_session(ops, &tmp_err); ret = ops->chmod(url_path, mode); if (ret < 0) { const int myerrno = gfal_lfc_get_errno(ops); gfal2_set_error(&tmp_err, gfal2_get_plugin_lfc_quark(), myerrno, __func__, "Errno reported from lfc : %s ", gfal_lfc_get_strerror(ops)); } else { errno = 0; gsimplecache_remove_kstr(ops->cache_stat, url_path); } } } g_free(url_path); g_free(url_host); lfc_unset_environment(ops); G_RETURN_ERR(ret, tmp_err, err); } /* * * implementation of the access call with the lfc plugin * return 0 or -1 if error and report GError** with error code and message */ int lfc_accessG(plugin_handle handle, const char *lfn, int mode, GError **err) { g_return_val_err_if_fail(handle && lfn, -1, err, "[lfc_accessG] Invalid value in arguments handle or/and path"); GError *tmp_err = NULL; int ret = -1; struct lfc_ops *ops = (struct lfc_ops *) handle; char *url_path = NULL, *url_host = NULL; if ((ret = url_converter(handle, lfn, &url_host, &url_path, &tmp_err)) == 0) { ret = lfc_configure_environment(ops, url_host, lfn, &tmp_err); if (!tmp_err) { gfal_auto_maintain_session(ops, &tmp_err); ret = ops->access(url_path, mode); if (ret < 0) { int sav_errno = gfal_lfc_get_errno(ops); gfal2_set_error(&tmp_err, gfal2_get_plugin_lfc_quark(), sav_errno, __func__, "lfc access error, file : %s, error : %s", lfn, gfal_lfc_get_strerror(ops)); } else errno = 0; } } g_free(url_path); g_free(url_host); lfc_unset_environment(ops); G_RETURN_ERR(ret, tmp_err, err); } /* * Implementation of the rename call for the lfc plugin * return 0 if success else -1 if error and set GError * * */ int lfc_renameG(plugin_handle handle, const char *oldpath, const char *newpath, GError **err) { g_return_val_err_if_fail(handle && oldpath && newpath, -1, err, "[lfc_renameG] Invalid value in args handle/oldpath/newpath"); struct lfc_ops *ops = (struct lfc_ops *) handle; GError *tmp_err = NULL; int ret = -1; char *source_url_path = NULL, *source_url_host = NULL; char *dest_url_path = NULL, *dest_url_host = NULL; if ((ret = url_converter(handle, oldpath, &source_url_host, &source_url_path, &tmp_err)) == 0 && (ret = url_converter(handle, newpath, &dest_url_host, &dest_url_path, &tmp_err)) == 0) { ret = lfc_configure_environment(ops, source_url_host, oldpath, &tmp_err); if (!tmp_err) { gfal_auto_maintain_session(ops, &tmp_err); ret = ops->rename(source_url_path, dest_url_path); if (ret < 0) { int sav_errno = gfal_lfc_get_errno(ops); gfal2_set_error(&tmp_err, gfal2_get_plugin_lfc_quark(), sav_errno, __func__, "Error report from LFC : %s", gfal_lfc_get_strerror(ops)); } else { gsimplecache_remove_kstr(ops->cache_stat, source_url_path); } } } g_free(source_url_path); g_free(source_url_host); g_free(dest_url_path); g_free(dest_url_host); lfc_unset_environment(ops); G_RETURN_ERR(ret, tmp_err, err); } /* * Implementation of the symlinkG call for the lfc plugin * return 0 if success else -1 if error and set GError * * */ int lfc_symlinkG(plugin_handle handle, const char *oldpath, const char *newpath, GError **err) { g_return_val_err_if_fail(handle && oldpath && newpath, -1, err, "[lfc_symlinkG] Invalid value in args handle/oldpath/newpath"); struct lfc_ops *ops = (struct lfc_ops *) handle; GError *tmp_err = NULL; int ret = -1; char *url_path = NULL, *url_host = NULL; char *link_url_path = NULL, *link_url_host = NULL; if ((ret = url_converter(handle, oldpath, &url_host, &url_path, &tmp_err)) == 0 && (ret = url_converter(handle, newpath, &link_url_host, &link_url_path, &tmp_err)) == 0) { ret = lfc_configure_environment(ops, url_host, oldpath, &tmp_err); if (!tmp_err) { gfal_auto_maintain_session(ops, &tmp_err); ret = ops->symlink(url_path, link_url_path); if (ret < 0) { int sav_errno = gfal_lfc_get_errno(ops); gfal2_set_error(&tmp_err, gfal2_get_plugin_lfc_quark(), sav_errno, __func__, "Error report from LFC : %s", gfal_lfc_get_strerror(ops)); } } } g_free(url_path); g_free(url_host); lfc_unset_environment(ops); G_RETURN_ERR(ret, tmp_err, err); } /* * execute a posix stat request on the lfc * return 0 and set struct if correct answer, else return negative value and set GError * */ int lfc_statG(plugin_handle handle, const char *path, struct stat *st, GError **err) { g_return_val_err_if_fail(handle && path && st, -1, err, "[lfc_statG] Invalid value in args handle/path/stat"); GError *tmp_err = NULL; int ret = -1; struct lfc_ops *ops = (struct lfc_ops *) handle; char *url_path = NULL, *url_host = NULL; if ((ret = url_converter(handle, path, &url_host, &url_path, &tmp_err)) == 0) { ret = lfc_configure_environment(ops, url_host, path, &tmp_err); if (!tmp_err) { gfal_auto_maintain_session(ops, &tmp_err); struct lfc_filestatg statbuf; ret = gfal_lfc_statg(ops, url_path, &statbuf, &tmp_err); if (ret == 0) { ret = gfal_lfc_convert_statg(st, &statbuf, err); errno = 0; } } } g_free(url_path); g_free(url_host); lfc_unset_environment(ops); G_RETURN_ERR(ret, tmp_err, err); } /* * execute a posix lstat request on the lfc ( stat request with link information) * return 0 if success and set the struct buf else return negative value and set GError */ static int lfc_lstatG(plugin_handle handle, const char *path, struct stat *st, GError **err) { g_return_val_err_if_fail(handle && path && st, -1, err, "[lfc_lstatG] Invalid value in args handle/path/stat"); GError *tmp_err = NULL; struct lfc_ops *ops = (struct lfc_ops *) handle; int ret = -1; char *url_path = NULL, *url_host = NULL; if ((ret = url_converter(handle, path, &url_host, &url_path, &tmp_err)) == 0) { ret = lfc_configure_environment(ops, url_host, path, &tmp_err); if (!tmp_err) { struct lfc_filestat statbuf; if ((ret = gsimplecache_take_one_kstr(ops->cache_stat, url_path, st)) == 0) { // take the version of the buffer gfal2_log(G_LOG_LEVEL_DEBUG, " lfc_lstatG -> value taken from cache"); } else { gfal2_log(G_LOG_LEVEL_DEBUG, " lfc_lstatG -> value not in cache, do normal call"); gfal_auto_maintain_session(ops, &tmp_err); if (!tmp_err) { ret = ops->lstat(url_path, &statbuf); if (ret != 0) { int sav_errno = gfal_lfc_get_errno(ops); gfal2_set_error(&tmp_err, gfal2_get_plugin_lfc_quark(), sav_errno, __func__, "Error report from LFC : %s", gfal_lfc_get_strerror(ops)); } else { ret = gfal_lfc_convert_lstat(st, &statbuf, err); errno = 0; } } } } } g_free(url_path); g_free(url_host); lfc_unset_environment(ops); G_RETURN_ERR(ret, tmp_err, err); } /* * Execute a posix mkdir on the lfc * return 0 on success else -1 and err is set with the correct value * */ static int lfc_mkdirpG(plugin_handle handle, const char *path, mode_t mode, gboolean pflag, GError **err) { g_return_val_err_if_fail(handle && path, -1, err, "[lfc_mkdirpG] Invalid value in args handle/path"); GError *tmp_err = NULL; int ret = -1; struct lfc_ops *ops = (struct lfc_ops *) handle; char *url_path = NULL, *url_host = NULL; if ((ret = url_converter(handle, path, &url_host, &url_path, &tmp_err)) == 0) { ret = lfc_configure_environment(ops, url_host, path, &tmp_err); if (!tmp_err) { gfal_auto_maintain_session(ops, &tmp_err); ret = gfal_lfc_ifce_mkdirpG(ops, url_path, mode, pflag, &tmp_err); } } g_free(url_path); g_free(url_host); lfc_unset_environment(ops); G_RETURN_ERR(ret, tmp_err, err); } /* * Execute a rmdir on the lfc * return 0 on success else -1 and err is set with the correct value * */ static int lfc_rmdirG(plugin_handle handle, const char *path, GError **err) { g_return_val_err_if_fail(handle && path, -1, err, "[lfc_rmdirG] Invalid value in args handle/path"); GError *tmp_err = NULL; int ret = -1; struct lfc_ops *ops = (struct lfc_ops *) handle; char *url_path = NULL, *url_host = NULL; if ((ret = url_converter(handle, path, &url_host, &url_path, &tmp_err)) == 0) { ret = lfc_configure_environment(ops, url_host, path, &tmp_err); if (!tmp_err) { ret = ops->rmdir(url_path); if (ret < 0) { int sav_errno = gfal_lfc_get_errno(ops); sav_errno = (sav_errno == EEXIST) ? ENOTEMPTY : sav_errno; // convert wrong reponse code gfal2_set_error(err, gfal2_get_plugin_lfc_quark(), sav_errno, __func__, "Error report from LFC %s", gfal_lfc_get_strerror(ops)); } } } g_free(url_path); g_free(url_host); lfc_unset_environment(ops); G_RETURN_ERR(ret, tmp_err, err); } /* * execute an opendir func to the lfc * */ static gfal_file_handle lfc_opendirG(plugin_handle handle, const char *path, GError **err) { g_return_val_err_if_fail(handle && path, NULL, err, "[lfc_rmdirG] Invalid value in args handle/path"); GError *tmp_err = NULL; struct lfc_ops *ops = (struct lfc_ops *) handle; lfc_opendir_handle oh = NULL; DIR *d = NULL; char *url_path = NULL, *url_host = NULL; if (url_converter(handle, path, &url_host, &url_path, &tmp_err) == 0) { lfc_configure_environment(ops, url_host, path, &tmp_err); if (!tmp_err) { gfal_auto_maintain_session(ops, &tmp_err); d = (DIR *) ops->opendirg(url_path, NULL); if (d == NULL) { int sav_errno = gfal_lfc_get_errno(ops); gfal2_set_error(err, gfal2_get_plugin_lfc_quark(), sav_errno, __func__, "Error report from LFC %s", gfal_lfc_get_strerror(ops)); } else { oh = g_new0(struct _lfc_opendir_handle, 1); g_strlcpy(oh->url, url_path, GFAL_URL_MAX_LEN); } } } g_free(url_path); g_free(url_host); if (d == NULL) { lfc_unset_environment(ops); } G_RETURN_ERR(((d) ? (gfal_file_handle_new2(lfc_getName(), (gpointer) d, (gpointer) oh, path)) : NULL), tmp_err, err); } static struct dirent *lfc_convert_dirent_struct(struct lfc_ops *ops, struct dirent *dir, struct stat *st, struct Cns_direnstat *filestat, const char *url) { struct stat st2; if (st == NULL) { st = &st2; } if (filestat == NULL) { return NULL; } GSimpleCache *cache = ops->cache_stat; char fullurl[GFAL_URL_MAX_LEN]; g_strlcpy(fullurl, url, GFAL_URL_MAX_LEN); g_strlcat(fullurl, "/", GFAL_URL_MAX_LEN); g_strlcat(fullurl, filestat->d_name, GFAL_URL_MAX_LEN); memset(st, 0, sizeof(struct stat)); st->st_mode = (mode_t) filestat->filemode; st->st_nlink = (nlink_t) filestat->nlink; st->st_uid = (uid_t) filestat->uid; st->st_gid = (gid_t) filestat->gid; st->st_size = (off_t) filestat->filesize; st->st_atime = (time_t) filestat->atime; st->st_ctime = (time_t) filestat->ctime; st->st_mtime = (time_t) filestat->mtime; gsimplecache_add_item_kstr(cache, fullurl, (void *) st); #if defined(SOLARIS) || defined(__linux__) dir->d_off += 1; #endif g_strlcpy(dir->d_name, filestat->d_name, NAME_MAX); return dir; } /* * Execute a readdirpp func on the lfc * */ static struct dirent *lfc_readdirppG(plugin_handle handle, gfal_file_handle fh, struct stat *st, GError **err) { g_return_val_err_if_fail(handle && fh, NULL, err, "[lfc_rmdirG] Invalid value in args handle/path"); GError *tmp_err = NULL; int sav_errno = 0; struct lfc_ops *ops = (struct lfc_ops *) handle; gfal_auto_maintain_session(ops, &tmp_err); gfal_lfc_reset_errno(ops); lfc_opendir_handle oh = (lfc_opendir_handle) gfal_file_handle_get_user_data(fh); struct dirent *ret; lfc_DIR *lfc_dir = (lfc_DIR *) gfal_file_handle_get_fdesc(fh); ret = lfc_convert_dirent_struct(ops, ((struct dirent *) &oh->current_dir), st, ops->readdirx(lfc_dir), oh->url); if (ret == NULL && (sav_errno = gfal_lfc_get_errno(ops))) { gfal2_set_error(err, gfal2_get_plugin_lfc_quark(), sav_errno, __func__, "Error report from LFC %s", gfal_lfc_get_strerror(ops)); } return ret; } /* * Execute a readdir func on the lfc * */ static struct dirent *lfc_readdirG(plugin_handle handle, gfal_file_handle fh, GError **err) { return lfc_readdirppG(handle, fh, NULL, err); } /* * execute an closedir func on the lfc * */ static int lfc_closedirG(plugin_handle handle, gfal_file_handle fh, GError **err) { g_return_val_err_if_fail(handle && fh, -1, err, "[lfc_rmdirG] Invalid value in args handle/path"); struct lfc_ops *ops = (struct lfc_ops *) handle; int ret = ops->closedir(gfal_file_handle_get_fdesc(fh)); if (ret != 0) { int sav_errno = gfal_lfc_get_errno(ops); gfal2_set_error(err, gfal2_get_plugin_lfc_quark(), sav_errno, __func__, "Error report from LFC %s", gfal_lfc_get_strerror(ops)); } else { g_free(gfal_file_handle_get_user_data(fh)); gfal_file_handle_delete(fh); } lfc_unset_environment(ops); return ret; } static int lfc_checksumG(plugin_handle handle, const char *path, const char *check_type, char *checksum_buffer, size_t buffer_length, off_t start_offset, size_t data_length, GError **err) { g_return_val_err_if_fail(handle && path && checksum_buffer, -1, err, "[lfc_checksumG] Invalid value in args handle/path/stat"); struct lfc_ops *ops = (struct lfc_ops *) handle; GError *tmp_err = NULL; ssize_t ret = -1; gfal_auto_maintain_session(ops, &tmp_err); char *url_path = NULL, *url_host = NULL; if ((ret = url_converter(handle, path, &url_host, &url_path, &tmp_err)) == 0) { ret = lfc_configure_environment(ops, url_host, path, &tmp_err); if (!tmp_err) { lfc_checksum checksum_st; ret = gfal_lfc_getChecksum(ops, url_path, &checksum_st, &tmp_err); if (ret == 0) { g_strlcpy(checksum_buffer, checksum_st.value, buffer_length); } } } g_free(url_path); g_free(url_host); lfc_unset_environment(ops); G_RETURN_ERR(ret, tmp_err, err); } static int lfc_checksumTypeG(plugin_handle handle, const char *path, char *checksum_buffer, size_t buffer_length, GError **err) { g_return_val_err_if_fail(handle && path && checksum_buffer, -1, err, "[lfc_checksumG] Invalid value in args handle/path/stat"); struct lfc_ops *ops = (struct lfc_ops *) handle; GError *tmp_err = NULL; ssize_t ret = -1; gfal_auto_maintain_session(ops, &tmp_err); char *url_path = NULL, *url_host = NULL; if ((ret = url_converter(handle, path, &url_host, &url_path, &tmp_err)) == 0) { ret = lfc_configure_environment(ops, url_host, path, &tmp_err); if (!tmp_err) { lfc_checksum checksum_st; ret = gfal_lfc_getChecksum(ops, url_path, &checksum_st, &tmp_err); if (ret == 0) { g_strlcpy(checksum_buffer, checksum_st.type, buffer_length); } } } g_free(url_path); g_free(url_host); lfc_unset_environment(ops); G_RETURN_ERR(ret, tmp_err, err); } /* * resolve the lfc link to the surls */ char **lfc_getSURLG(plugin_handle handle, const char *path, GError **err) { g_return_val_err_if_fail(handle && path, NULL, err, "[lfc_getSURLG] Invalid value in args handle/path"); GError *tmp_err = NULL; char **resu = NULL; struct lfc_ops *ops = (struct lfc_ops *) handle; char *url_path = NULL, *url_host = NULL; if ((url_converter(handle, path, &url_host, &url_path, &tmp_err)) == 0) { (void) lfc_configure_environment(ops, url_host, path, &tmp_err); if (!tmp_err) { resu = gfal_lfc_getSURL(ops, url_path, &tmp_err); } } g_free(url_path); g_free(url_host); lfc_unset_environment(ops); G_RETURN_ERR(resu, tmp_err, err); } /* * lfc getxattr for the path -> surls resolution * */ ssize_t lfc_getxattr_getsurl(plugin_handle handle, const char *path, void *buff, size_t size, GError **err) { GError *tmp_err = NULL; ssize_t res = -1; char **tmp_ret = lfc_getSURLG(handle, path, &tmp_err); if (tmp_ret != NULL) { res = g_strv_catbuff(tmp_ret, buff, size); g_strfreev(tmp_ret); } G_RETURN_ERR(res, tmp_err, err); } /* * lfc getxattr for the path -> guid resolution * */ ssize_t lfc_getxattr_getguid(plugin_handle handle, const char *path, void *buff, size_t size, GError **err) { GError *tmp_err = NULL; ssize_t res = -1; struct lfc_ops *ops = (struct lfc_ops *) handle; char *url_path = NULL, *url_host = NULL; if ((res = url_converter(handle, path, &url_host, &url_path, &tmp_err)) == 0) { res = lfc_configure_environment(ops, url_host, path, &tmp_err); if (!tmp_err) { if (size == 0 || buff == NULL) { // just return the size of a guid res = sizeof(char) * 36; // strng uuid are 36 bytes long } else { struct lfc_filestatg statbuf; int tmp_ret = gfal_lfc_statg(ops, url_path, &statbuf, &tmp_err); if (tmp_ret == 0) { res = strnlen(statbuf.guid, GFAL_URL_MAX_LEN); g_strlcpy(buff, statbuf.guid, size); errno = 0; } } } } g_free(url_path); g_free(url_host); lfc_unset_environment(ops); G_RETURN_ERR(res, tmp_err, err); } /* * lfc getxattr for path -> comment resolution * * */ ssize_t lfc_getxattr_comment(plugin_handle handle, const char *path, void *buff, size_t size, GError **err) { GError *tmp_err = NULL; ssize_t res = -1; struct lfc_ops *ops = (struct lfc_ops *) handle; char *url_path = NULL, *url_host = NULL; if ((res = url_converter(handle, path, &url_host, &url_path, &tmp_err)) == 0) { res = lfc_configure_environment(ops, url_host, path, &tmp_err); if (!tmp_err) { res = gfal_lfc_getComment(ops, url_path, buff, size, &tmp_err); } } g_free(url_path); g_free(url_host); lfc_unset_environment(ops); G_RETURN_ERR(res, tmp_err, err); } /* * lfc getxattr implem * */ ssize_t lfc_getxattrG(plugin_handle handle, const char *path, const char *name, void *buff, size_t size, GError **err) { GError *tmp_err = NULL; ssize_t res = -1; struct lfc_ops *ops = (struct lfc_ops *) handle; gfal_auto_maintain_session(ops, &tmp_err); if (strncmp(name, GFAL_XATTR_GUID, LFC_MAX_XATTR_LEN) == 0) { res = lfc_getxattr_getguid(handle, path, buff, size, &tmp_err); } else if (strncmp(name, GFAL_XATTR_REPLICA, LFC_MAX_XATTR_LEN) == 0) { res = lfc_getxattr_getsurl(handle, path, buff, size, &tmp_err); } else if (strncmp(name, GFAL_XATTR_COMMENT, LFC_MAX_XATTR_LEN) == 0) { res = lfc_getxattr_comment(handle, path, buff, size, &tmp_err); } else if (strncmp(name, GFAL_XATTR_CHKSUM_TYPE, LFC_MAX_XATTR_LEN) == 0) { res = lfc_checksumTypeG(handle, path, buff, size, &tmp_err); } else if (strncmp(name, GFAL_XATTR_CHKSUM_VALUE, LFC_MAX_XATTR_LEN) == 0) { res = lfc_checksumG(handle, path, "", buff, size, 0, 0, &tmp_err); } else { gfal2_set_error(&tmp_err, gfal2_get_plugin_lfc_quark(), ENOATTR, __func__, "axttr not found"); res = -1; } G_RETURN_ERR(res, tmp_err, err); } /* * lfc getxattr implem * */ ssize_t lfc_listxattrG(plugin_handle handle, const char *path, char *list, size_t size, GError **err) { ssize_t res = 0; char **p = file_xattr; char *plist = list; GError *tmp_err = NULL; struct stat st; if (lfc_lstatG(handle, path, &st, &tmp_err) < 0) { res = -1; } else { if (!S_ISDIR(st.st_mode)) { while (*p != NULL) { const int size_str = strlen(*p) + 1; if (size > res && size - res >= size_str) { plist = mempcpy(plist, *p, size_str * sizeof(char)); } res += size_str; p++; } } else { mempcpy(list, GFAL_XATTR_COMMENT, size); res = 1; } } G_RETURN_ERR(res, tmp_err, err); } /* * setxattr function special for comments * */ int lfc_setxattr_comment(plugin_handle handle, const char *path, const char *name, const void *value, size_t size, int flags, GError **err) { GError *tmp_err = NULL; struct lfc_ops *ops = (struct lfc_ops *) handle; int res = -1; char *url_path = NULL, *url_host = NULL; if ((res = url_converter(handle, path, &url_host, &url_path, &tmp_err)) == 0) { res = lfc_configure_environment(ops, url_host, path, &tmp_err); if (!tmp_err) { res = gfal_lfc_setComment(ops, url_path, value, size, &tmp_err); } } g_free(url_path); g_free(url_host); lfc_unset_environment(ops); return res; } /* * setxattr for replicas */ int lfc_setxattr_replica(plugin_handle handle, const char *path, const char *name, const void *value, size_t size, int flags, GError **err) { const char *sfn = (const char *) value; if (size == 0) { gfal2_set_error(err, gfal2_get_plugin_lfc_quark(), EINVAL, __func__, "Missing value"); return -1; } struct lfc_ops *ops = (struct lfc_ops *) handle; if (sfn[0] == '+') { int ret = -1; gfalt_params_t params = gfalt_params_handle_new(err); if (!*err) { ret = gfal_lfc_register(handle, ops->handle, params, sfn + 1, path, err); gfalt_params_handle_delete(params, err); if (*err) { ret = -1; } } return ret; } else if (sfn[0] == '-') { return gfal_lfc_unregister(handle, path, sfn + 1, err); } else { gfal2_set_error(err, gfal2_get_plugin_lfc_quark(), EINVAL, __func__, "user.replica only accepts additions (+) or deletions (-)"); return -1; } } /* * lfc setxattr implem * */ int lfc_setxattrG(plugin_handle handle, const char *path, const char *name, const void *value, size_t size, int flags, GError **err) { g_return_val_err_if_fail(path && name, -1, err, "invalid name/path"); int res = -1; GError *tmp_err = NULL; if (strcmp(name, GFAL_XATTR_COMMENT) == 0) { res = lfc_setxattr_comment(handle, path, name, value, size, flags, err); } else if (strcmp(name, GFAL_XATTR_REPLICA) == 0) { res = lfc_setxattr_replica(handle, path, name, value, size, flags, err); } else { gfal2_set_error(&tmp_err, gfal2_get_plugin_lfc_quark(), ENOATTR, __func__, "unable to set this attribute on this file"); } G_RETURN_ERR(res, tmp_err, err); } /* * Convert a guid to a plugin url if possible * return the link in a plugin's url string or err and NULL if not found */ char *lfc_resolve_guid(plugin_handle handle, const char *guid, GError **err) { g_return_val_err_if_fail(handle && guid, NULL, err, "[lfc_resolve_guid] Invalid args in handle and/or guid "); GError *tmp_err = NULL; char *url_path = NULL, *url_host = NULL, *res = NULL; struct lfc_ops *ops = (struct lfc_ops *) handle; if (url_converter(handle, guid, &url_host, &url_path, &tmp_err) == 0) { lfc_configure_environment(ops, url_host, guid, &tmp_err); if (!tmp_err) { res = url_path; } } g_free(url_host); lfc_unset_environment(ops); G_RETURN_ERR(res, tmp_err, err); } static int lfc_unlinkG(plugin_handle handle, const char *path, GError **err) { g_return_val_err_if_fail(path, -1, err, "[lfc_unlink] Invalid value in args handle/path/stat"); GError *tmp_err = NULL; struct lfc_ops *ops = (struct lfc_ops *) handle; char *url_path = NULL, *url_host = NULL; int ret = -1; if ((ret = url_converter(handle, path, &url_host, &url_path, &tmp_err)) == 0) { ret = lfc_configure_environment(ops, url_host, path, &tmp_err); if (!tmp_err) { int nreplies = 0; int *replies = NULL; ret = ops->delfilesbyname(1, (const char **) (&url_path), 1, &nreplies, &replies); if (ret != 0 || (nreplies && replies[0] != 0)) { int sav_errno = gfal_lfc_get_errno(ops); if (sav_errno != 0) { gfal2_set_error(&tmp_err, gfal2_get_plugin_lfc_quark(), sav_errno, __func__, "Error report from LFC : %s", gfal_lfc_get_strerror(ops)); } else { gfal2_set_error(&tmp_err, gfal2_get_plugin_lfc_quark(), replies[0], __func__, "Error report from LFC : %s", ops->sstrerror(replies[0])); ret = -1; } } else { gsimplecache_remove_kstr(ops->cache_stat, url_path); // remove the key associated in the buffer errno = 0; } free(replies); } } g_free(url_path); g_free(url_host); lfc_unset_environment(ops); G_RETURN_ERR(ret, tmp_err, err); } /* * execute a posix readlink request on the lfc * return size of the buffer if success and set the struct buf else return negative value and set GError */ static ssize_t lfc_readlinkG(plugin_handle handle, const char *path, char *buff, size_t buffsiz, GError **err) { g_return_val_err_if_fail(handle && path && buff, -1, err, "[lfc_readlinkG] Invalid value in args handle/path/stat"); struct lfc_ops *ops = (struct lfc_ops *) handle; GError *tmp_err = NULL; ssize_t ret = -1; char res_buff[LFC_BUFF_SIZE]; gfal_auto_maintain_session(ops, &tmp_err); char *url_path = NULL, *url_host = NULL; if ((ret = url_converter(handle, path, &url_host, &url_path, &tmp_err)) == 0) { ret = lfc_configure_environment(ops, url_host, path, &tmp_err); if (!tmp_err) { ret = ops->readlink(url_path, res_buff, LFC_BUFF_SIZE); if (ret == -1) { int sav_errno = gfal_lfc_get_errno(ops); gfal2_set_error(err, gfal2_get_plugin_lfc_quark(), sav_errno, __func__, "Error report from LFC : %s", gfal_lfc_get_strerror(ops)); } else { errno = 0; if (buffsiz > 0) { memcpy(buff, GFAL_LFC_PREFIX, MIN(buffsiz, GFAL_LFC_PREFIX_LEN)); } if (buffsiz - GFAL_LFC_PREFIX_LEN > 0) { memcpy(buff + GFAL_LFC_PREFIX_LEN, res_buff, MIN(ret, buffsiz - GFAL_LFC_PREFIX_LEN)); } ret += GFAL_LFC_PREFIX_LEN; } } } g_free(url_path); g_free(url_host); lfc_unset_environment(ops); G_RETURN_ERR(ret, tmp_err, err); } static void internal_stat_copy(gpointer original, gpointer copy) { memcpy(copy, original, sizeof(struct stat)); } /* * Map function for the lfc interface * this function provide the generic PLUGIN interface for the LFC plugin. * lfc_initG do : liblfc shared library load, sym resolve, endpoint check, and plugin function map. * * */ gfal_plugin_interface gfal_plugin_init(gfal2_context_t handle, GError **err) { pthread_mutex_lock(&m_lfcinit); gfal_plugin_interface lfc_plugin; GError *tmp_err = NULL; memset(&lfc_plugin, 0, sizeof(gfal_plugin_interface)); // clear the plugin struct lfc_ops *ops = gfal_load_lfc(GFAL_LFC_LIBRARY_NAME, &tmp_err); // load library if (ops == NULL) { gfal2_propagate_prefixed_error(err, tmp_err, __func__); pthread_mutex_unlock(&m_lfcinit); return lfc_plugin; } ops->lfc_endpoint_predefined = (char *) g_getenv(LFC_ENV_VAR_HOST); ops->lfc_conn_retry = (char *) g_getenv(LFC_ENV_VAR_CONRETRY); ops->lfc_conn_try_int = (char *) g_getenv(LFC_ENV_VAR_CONRETRYINT); ops->lfc_conn_timeout = (char *) g_getenv(LFC_ENV_VAR_CONNTIMEOUT); ops->handle = handle; ops->cache_stat = gsimplecache_new(5000, &internal_stat_copy, sizeof(struct stat)); gfal_lfc_regex_compile(&(ops->rex), err); lfc_plugin.plugin_data = (void *) ops; lfc_plugin.priority = GFAL_PLUGIN_PRIORITY_CATALOG; lfc_plugin.check_plugin_url = &gfal_lfc_check_lfn_url; lfc_plugin.plugin_delete = &lfc_destroyG; lfc_plugin.accessG = &lfc_accessG; lfc_plugin.chmodG = &lfc_chmodG; lfc_plugin.renameG = &lfc_renameG; lfc_plugin.statG = &lfc_statG; lfc_plugin.lstatG = &lfc_lstatG; lfc_plugin.mkdirpG = &lfc_mkdirpG; lfc_plugin.rmdirG = &lfc_rmdirG; lfc_plugin.opendirG = &lfc_opendirG; lfc_plugin.closedirG = &lfc_closedirG; lfc_plugin.readdirG = &lfc_readdirG; lfc_plugin.getName = &lfc_getName; lfc_plugin.openG = &lfc_openG; lfc_plugin.symlinkG = &lfc_symlinkG; lfc_plugin.getxattrG = &lfc_getxattrG; lfc_plugin.setxattrG = &lfc_setxattrG; lfc_plugin.listxattrG = &lfc_listxattrG; lfc_plugin.readlinkG = &lfc_readlinkG; lfc_plugin.unlinkG = &lfc_unlinkG; lfc_plugin.readdirppG = &lfc_readdirppG; lfc_plugin.checksum_calcG = &lfc_checksumG; // Copy (as register) lfc_plugin.check_plugin_url_transfer = gfal_lfc_register_check; lfc_plugin.copy_file = gfal_lfc_register; if (init_thread == FALSE) { // initiate Cthread system ops->Cthread_init(); // must be called one time for DPM thread safety init_thread = TRUE; } pthread_mutex_unlock(&m_lfcinit); return lfc_plugin; } /* * parse a guid to check the validity */ gboolean gfal_checker_guid(const char *guid, GError **err) { g_return_val_err_if_fail(guid != NULL, FALSE, err, "[gfal_checker_guid] check URL failed : guid is empty"); const size_t sguid = strnlen(guid, GFAL_URL_MAX_LEN); return (sguid < GFAL_URL_MAX_LEN && sguid > 5 && strncmp(guid, "guid:", 5) == 0); } /* * Check if the passed url and operation is compatible with lfc * * */ gboolean gfal_lfc_check_lfn_url(plugin_handle handle, const char *url, plugin_mode mode, GError **err) { struct lfc_ops *ops = (struct lfc_ops *) handle; int ret; switch (mode) { case GFAL_PLUGIN_RESOLVE_GUID: return TRUE; case GFAL_PLUGIN_ACCESS: case GFAL_PLUGIN_CHMOD: case GFAL_PLUGIN_STAT: case GFAL_PLUGIN_LSTAT: case GFAL_PLUGIN_OPEN: case GFAL_PLUGIN_GETXATTR: case GFAL_PLUGIN_LISTXATTR: case GFAL_PLUGIN_SETXATTR: case GFAL_PLUGIN_UNLINK: case GFAL_PLUGIN_CHECKSUM: ret = regexec(&(ops->rex), url, 0, NULL, 0); return (!ret || gfal_checker_guid(url, err)) ? TRUE : FALSE; case GFAL_PLUGIN_RENAME: case GFAL_PLUGIN_MKDIR: case GFAL_PLUGIN_RMDIR: case GFAL_PLUGIN_OPENDIR: case GFAL_PLUGIN_SYMLINK: case GFAL_PLUGIN_READLINK: ret = regexec(&(ops->rex), url, 0, NULL, 0); return (!ret) ? TRUE : FALSE; default: return FALSE; } } gfal2-v2.23.0/src/plugins/lfc/gfal_lfc.h000066400000000000000000000033561465240014500177110ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #pragma once #define GFAL_LFC_LIBRARY_NAME "liblfc.so.1" #define GFAL_LFC_PREFIX "lfn:" #define GFAL_LFC_PREFIX2 "lfc://" #define GFAL_LFC_GUID_PREFIX "guid:" #define GFAL_LFC_GUID_PREFIX_LEN 5 #define GFAL_LFC_PREFIX_LEN 4 #define LFC_XATTR_GUID "user.guid" #define LFC_XATTR_SURLS "user.replicas" #define LFC_MAX_XATTR_LEN 2048 #define LFC_BUFF_SIZE 2048 #define LFC_PARAMETER_NAMESPACE "lfc" #define LFC_PARAMETER_HOST "host" #include #include // LFC plugin GQuark GQuark gfal2_get_plugin_lfc_quark(); // protos gboolean gfal_checker_guid(const char* guid, GError** err); gfal_plugin_interface lfc_initG(gfal2_context_t, GError**); gboolean gfal_lfc_check_lfn_url(plugin_handle handle, const char* lfn_url, plugin_mode mode, GError** err); char ** lfc_getSURLG(plugin_handle handle, const char * path, GError** err); void lfc_set_session_timeout(int timeout); int url_converter(plugin_handle handle, const char * url, char** host, char** path, GError** err); gfal2-v2.23.0/src/plugins/lfc/gfal_lfc_open.c000066400000000000000000000033151465240014500207200ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include "gfal_lfc.h" #include "lfc_ifce_ng.h" #ifndef ECOMM #define ECOMM EIO #endif /* * open function for the srm plugin */ gfal_file_handle lfc_openG(plugin_handle ch, const char *path, int flag, mode_t mode, GError **err) { gfal2_context_t handle = ((struct lfc_ops *) ch)->handle; GError *tmp_err = NULL; gfal_file_handle res = NULL; gfal2_log(G_LOG_LEVEL_DEBUG, " %s ->", __func__); char **surls = lfc_getSURLG(ch, path, &tmp_err); if (surls != 0 && tmp_err == NULL) { char **p = surls; while (*p != NULL) { gfal2_log(G_LOG_LEVEL_MESSAGE, " LFC resolution %s -> %s ", path, *p); res = gfal_plugin_openG(handle, *p, flag, mode, &tmp_err); if (res || (tmp_err && tmp_err->code != ECOMM)) { break; } p++; } } g_strfreev(surls); if (tmp_err) { gfal2_propagate_prefixed_error(err, tmp_err, __func__); } return res; } gfal2-v2.23.0/src/plugins/lfc/gfal_lfc_open.h000066400000000000000000000017751465240014500207350ustar00rootroot00000000000000#pragma once /* * Copyright @ Members of the EMI Collaboration, 2010. * See www.eu-emi.eu for details on the copyright holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ /* * file gfal_common_lfc_open.c brief header for lfc implementation for open/read/write/close author Adrien Devresse date 06/07/2011 */ #include #include #include "gfal_lfc.h" #include "lfc_ifce_ng.h" gfal_file_handle lfc_openG(plugin_handle ch, const char* path, int flag, mode_t mode, GError** err); gfal2-v2.23.0/src/plugins/lfc/lfc_ifce_ng.c000066400000000000000000000537501465240014500203700ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #define GFAL_LFN_MAX_LEN 2048 #include #ifndef ECOMM #define ECOMM EIO #endif #include #include #include #include #include #include #include #include #include #include #include "gfal_lfc.h" #include "lfc_ifce_ng.h" int _Cthread_addcid(char *, int, char *, int, Cth_pid_t *, unsigned, void *(*)(void *), int); // hack in order to use the internal CThread API ( DPNS limitation ) static __thread int _local_thread_init = FALSE; static volatile time_t session_timestamp = 0; static long session_duration = 20; static pthread_mutex_t m_session = PTHREAD_MUTEX_INITIALIZER; int gfal_lfc_regex_compile(regex_t *rex, GError **err) { int ret = regcomp(rex, "^(lfn:/|lfc://)([:alnum:]|-|/|.|_)+", REG_ICASE | REG_EXTENDED); g_return_val_err_if_fail(ret == 0, -1, err, "[gfal_lfc_check_lfn_url] fail to compile regex, report this bug"); return ret; } static void lfc_plugin_set_lfc_env(struct lfc_ops *ops, const char *var_name, const char *var_value) { if (ops->set_env) { // if new lfc library with set_env call, set at runtime ops->set_env(var_name, var_value, TRUE); } else { // else set it as env var !! NOT THREAD SAFE g_setenv(var_name, var_value, TRUE); } } const char *lfc_plugin_get_lfc_env(struct lfc_ops *ops, const char *var_name) { if (ops->get_env) { return ops->get_env(var_name); } else { return g_getenv(var_name); } } int lfc_configure_environment(struct lfc_ops *ops, const char *host, const char *url, GError **err) { gfal_lfc_init_thread(ops); GError *tmp_err = NULL; const char *tab_envar[] = {ops->lfc_endpoint_predefined, ops->lfc_conn_timeout, ops->lfc_conn_retry, ops->lfc_conn_try_int}; const char *tab_envar_name[] = {LFC_ENV_VAR_HOST, LFC_ENV_VAR_CONNTIMEOUT, LFC_ENV_VAR_CONRETRY, LFC_ENV_VAR_CONRETRYINT}; const int tab_type[] = {0, 1, 1, 1}; const char *tab_override[] = {host, NULL, NULL, NULL, NULL}; const int n_var = 4; const char *plugin_group = LFC_ENV_VAR_GROUP_PLUGIN; int i, ret; ret = 0; for (i = 0; i < n_var; ++i) { if (tab_envar[i] == NULL) { switch (tab_type[i]) { case 0: { char *v1 = NULL; char *value = NULL; if (tab_override[i] != NULL) { value = (char *) tab_override[i]; } else { value = v1 = gfal2_get_opt_string(ops->handle, plugin_group, tab_envar_name[i], &tmp_err); } if (!tmp_err) { gfal2_log(G_LOG_LEVEL_DEBUG, "lfc plugin : setup env var value %s to %s", tab_envar_name[i], value); lfc_plugin_set_lfc_env(ops, tab_envar_name[i], value); g_free(v1); } else { ret = -1; } break; } case 1: { int v2; v2 = gfal2_get_opt_integer(ops->handle, plugin_group, tab_envar_name[i], &tmp_err); if (!tmp_err) { char v_str[20]; snprintf(v_str, 20, "%d", v2); gfal2_log(G_LOG_LEVEL_DEBUG, "lfc plugin : setup env var value %s to %d", tab_envar_name[i], v2); lfc_plugin_set_lfc_env(ops, tab_envar_name[i], v_str); } else { ret = -1; } break; } default: ret = -1; gfal2_set_error(&tmp_err, gfal2_get_plugin_lfc_quark(), EINVAL, __func__, "Invalid value %s in configuration file ", tab_envar_name[i]); } } if (tmp_err) { break; } } // Set credentials gchar *ucert = gfal2_cred_get(ops->handle, GFAL_CRED_X509_CERT, url, NULL, err); if (*err) { return -1; } gchar *ukey = gfal2_cred_get(ops->handle, GFAL_CRED_X509_KEY, url, NULL, err); if (*err) { return -1; } ops->env_user_cert = getenv("X509_USER_CERT"); ops->env_user_key = getenv("X509_USER_KEY"); ops->env_user_proxy = getenv("X509_USER_PROXY"); if (ucert && ukey) { gfal2_log(G_LOG_LEVEL_DEBUG, "lfc plugin : using certificate %s", ucert); gfal2_log(G_LOG_LEVEL_DEBUG, "lfc plugin : using private key %s", ukey); setenv("X509_USER_CERT", ucert, 1); setenv("X509_USER_KEY", ukey, 1); } else if (ucert) { gfal2_log(G_LOG_LEVEL_DEBUG, "lfc plugin : using proxy %s", ucert); setenv("X509_USER_PROXY", ucert, 1); } g_free(ucert); g_free(ukey); G_RETURN_ERR(ret, tmp_err, err); } void lfc_unset_environment(struct lfc_ops *ops) { if (ops->env_user_cert) { setenv("X509_USER_CERT", ops->env_user_cert, 1); } else { unsetenv("X509_USER_CERT"); } if (ops->env_user_key) { setenv("X509_USER_KEY", ops->env_user_key, 1); } else { unsetenv("X509_USER_KEY"); } if (ops->env_user_proxy) { setenv("X509_USER_PROXY", ops->env_user_proxy, 1); } else { unsetenv("X509_USER_PROXY"); } } // Routine for internal lfc hack, need to be call for the thread safety void gfal_lfc_init_thread(struct lfc_ops *ops) { if (_local_thread_init == FALSE) { Cth_pid_t th = pthread_self(); ops->_Cthread_addcid(NULL, 0, NULL, 0, &th, 0, NULL, 0); _local_thread_init = TRUE; } } // WARNING : ALL SESSIONS and transactions for LFC disabled // REASONS : unstable, can just fail forever in case of connexion drop int gfal_lfc_startSession(struct lfc_ops *ops, GError **err) { /*if (ops->startsess (ops->lfc_endpoint, "gfal2 auto-session") < 0){ int sav_errno = gfal_lfc_get_errno(ops); gfal2_set_error(err, gfal2_get_plugin_lfc_quark(), sav_errno, __func__, "Error while start session with lfc, lfc_endpoint: %s, Error : %s ", ops->lfc_endpoint, gfal_lfc_get_strerror(ops)); return -1; }*/ return 0; } /* --> session reused disabled -> bugged in liblfc static int gfal_lfc_endSession(struct lfc_ops* ops, GError ** err){ if (ops->endsess() < 0){ int sav_errno = gfal_lfc_get_errno(ops); gfal2_set_error(err, gfal2_get_plugin_lfc_quark(), sav_errno, __func__, "Error while start transaction with lfc, Error : %s ", gfal_lfc_get_strerror(ops)); return -1; } return 0; }*/ // session re-use void gfal_auto_maintain_session(struct lfc_ops *ops, GError **err) { /*time_t current = time(NULL); --> disabled, thread safety problem in liblfc inthe current state if(session_timestamp < current){ pthread_mutex_lock(&m_session); if(session_timestamp < current){ gfal_lfc_endSession(ops, NULL); gfal_lfc_startSession(ops, err); current = time(NULL); session_timestamp = current + session_duration; } pthread_mutex_unlock(&m_session); }*/ } void lfc_set_session_timeout(int timeout) { pthread_mutex_lock(&m_session); session_duration = timeout; pthread_mutex_unlock(&m_session); } static int gfal_lfc_startTransaction(struct lfc_ops *ops, GError **err) { /*if (ops->starttrans(ops->lfc_endpoint, "gfal2 auto-trans")){ int sav_errno = gfal_lfc_get_errno(ops); gfal2_set_error(err, gfal2_get_plugin_lfc_quark() ,sav_errno, __func__, "Error while start transaction with lfc, lfc_endpoint: %s, Error : %s ", ops->lfc_endpoint, gfal_lfc_get_strerror(ops)); return -1; }*/ return 0; } static int gfal_lfc_endTransaction(struct lfc_ops *ops, GError **err) { if (ops->endtrans() < 0) { int sav_errno = gfal_lfc_get_errno(ops); gfal2_set_error(err, gfal2_get_plugin_lfc_quark(), sav_errno, __func__, "Error while start transaction with lfc, Error : %s ", gfal_lfc_get_strerror(ops)); return -1; } return 0; } static int gfal_lfc_abortTransaction(struct lfc_ops *ops, GError **err) { if (ops->aborttrans() < 0) { int sav_errno = gfal_lfc_get_errno(ops); gfal2_set_error(err, gfal2_get_plugin_lfc_quark(), sav_errno, __func__, "Error while abort transaction with lfc, Error : %s ", gfal_lfc_get_strerror(ops)); return -1; } return 0; } int gfal_convert_guid_to_lfn_r(plugin_handle handle, const char *guid, char *buff_lfn, size_t sbuff_lfn, GError **err) { int ret; int size = 0; struct lfc_ops *ops = (struct lfc_ops *) handle; struct lfc_linkinfo *links = NULL; if (ops->getlinks(NULL, guid, &size, &links) < 0) { int sav_errno = gfal_lfc_get_errno(ops); gfal2_set_error(err, gfal2_get_plugin_lfc_quark(), sav_errno, __func__, "Error while getlinks() with lfclib, guid : %s, Error : %s ", guid, gfal_lfc_get_strerror(ops)); ret = -1; } else { if (!links || strnlen(links[0].path, GFAL_LFN_MAX_LEN) >= GFAL_LFN_MAX_LEN) { gfal2_set_error(err, gfal2_get_plugin_lfc_quark(), EINVAL, __func__, "Error no links associated with this guid or corrupted one : %s", guid); ret = -1; } else { g_strlcpy(buff_lfn, links[0].path, sbuff_lfn); ret = 0; } } free(links); return ret; } /* * load the shared library and link the symbol for the LFC usage * @param name : name of the library * @param err: error report */ struct lfc_ops *gfal_load_lfc(const char *name, GError **err) { struct lfc_ops *lfc_sym = NULL; GError *tmp_err = NULL; lfc_sym = calloc(1, sizeof(struct lfc_ops)); // static resolution lfc_sym->addreplica = &lfc_addreplica; lfc_sym->get_serrno = &C__serrno; lfc_sym->sstrerror = &sstrerror; lfc_sym->creatg = &lfc_creatg; lfc_sym->delreplica = &lfc_delreplica; lfc_sym->delfilesbyname = &lfc_delfilesbyname; lfc_sym->aborttrans = &lfc_aborttrans; lfc_sym->endtrans = &lfc_endtrans; lfc_sym->getpath = &lfc_getpath; lfc_sym->getlinks = &lfc_getlinks; lfc_sym->getreplica = &lfc_getreplica; lfc_sym->lstat = &lfc_lstat; lfc_sym->mkdirg = &lfc_mkdirg; lfc_sym->seterrbuf = &lfc_seterrbuf; lfc_sym->setfsizeg = &lfc_setfsizeg; lfc_sym->setfsize = &lfc_setfsize; lfc_sym->starttrans = &lfc_starttrans; lfc_sym->statg = &lfc_statg; lfc_sym->statr = &lfc_statr; lfc_sym->symlink = &lfc_symlink; lfc_sym->unlink = &lfc_unlink; lfc_sym->access = &lfc_access; lfc_sym->chmod = &lfc_chmod; lfc_sym->rename = &lfc_rename; lfc_sym->opendirg = &lfc_opendirg; lfc_sym->rmdir = &lfc_rmdir; lfc_sym->startsess = &lfc_startsess; lfc_sym->endsess = &lfc_endsess; lfc_sym->closedir = &lfc_closedir; lfc_sym->readdir = &lfc_readdir; lfc_sym->Cthread_init = &Cthread_init; lfc_sym->readlink = &lfc_readlink; lfc_sym->readdirx = &lfc_readdirx; lfc_sym->getcomment = &lfc_getcomment; lfc_sym->setcomment = &lfc_setcomment; lfc_sym->_Cthread_addcid = &_Cthread_addcid; // dyn resolution void *lib_handle = dlopen("liblfc.so.1", RTLD_LAZY); if (lib_handle) { lfc_sym->set_env = dlsym(lib_handle, "lfc_setenv"); lfc_sym->get_env = dlsym(lib_handle, "lfc_getenv"); dlclose(lib_handle); } else { lfc_sym->set_env = NULL; } errno = 0; G_RETURN_ERR(lfc_sym, tmp_err, err); } /* * convert a internal lfc statg to a POSIX lfc stat * struct must be already allocated */ int gfal_lfc_convert_statg(struct stat *output, struct lfc_filestatg *input, GError **err) { g_return_val_err_if_fail(output && input, -1, err, "[gfal_lfc_convert_statg] Invalid args statg/stat"); output->st_mode = input->filemode; output->st_nlink = input->nlink; output->st_uid = input->uid; output->st_gid = input->gid; output->st_size = input->filesize; output->st_atime = input->atime; output->st_ctime = input->ctime; output->st_mtime = input->mtime; return 0; } /* * convert a internal lfc lstat to a POSIX lfc stat * struct must be already allocated */ int gfal_lfc_convert_lstat(struct stat *output, struct lfc_filestat *input, GError **err) { g_return_val_err_if_fail(output && input, -1, err, "[gfal_lfc_convert_lstat] Invalid args statg/stat"); output->st_mode = input->filemode; output->st_nlink = input->nlink; output->st_uid = input->uid; output->st_gid = input->gid; output->st_size = input->filesize; output->st_atime = input->atime; output->st_ctime = input->ctime; output->st_mtime = input->mtime; return 0; } /* * basic wrapper mkdir to the lfc api */ static int gfal_lfc_mkdir(struct lfc_ops *ops, const char *path, mode_t mode, GError **err) { char struid[37]; gfal_generate_guidG(struid, NULL); if (ops->mkdirg(path, struid, mode)) { int sav_errno = gfal_lfc_get_errno(ops); gfal2_set_error(err, gfal2_get_plugin_lfc_quark(), sav_errno, __func__, "Error while mkdir call in the lfc %s", strerror(sav_errno)); return -1; } return 0; } /* * Begin a recursive call on mkdir to create a full tree path * @warning not safe, please ensure that string begin by "/" * */ int gfal_lfc_mkdir_rec(struct lfc_ops *ops, char *browser_path, const char *full_path, mode_t mode, GError **err) { int ret = -1; char *next_sep = strchr(browser_path, G_DIR_SEPARATOR); if (next_sep == NULL || *(next_sep + 1) == '\0') { // last folder return gfal_lfc_mkdir(ops, full_path, mode, err); } else { const int path_size = next_sep - full_path; GError *tmp_err = NULL; char path[path_size + 1]; *((char *) mempcpy(path, full_path, path_size)) = '\0'; ret = gfal_lfc_mkdir(ops, path, (0700 | mode), &tmp_err); if (ret == 0 || tmp_err->code == EEXIST || tmp_err->code == EACCES) { g_clear_error(&tmp_err); return gfal_lfc_mkdir_rec(ops, next_sep + 1, full_path, mode, err); } g_propagate_error(err, tmp_err); return ret; } } /* * Implementation of mkdir -p call on the lfc * * */ int gfal_lfc_ifce_mkdirpG(struct lfc_ops *ops, const char *path, mode_t mode, gboolean pflag, GError **err) { g_return_val_err_if_fail(ops && path, -1, err, "[gfal_lfc_ifce_mkdirpG] Invalid args in ops or/and path"); int ret; GError *tmp_err = NULL; ret = gfal_lfc_startTransaction(ops, &tmp_err); if (ret >= 0) { ret = gfal_lfc_mkdir(ops, path, mode, &tmp_err); // try to create the directory, suppose the non-recursive case if (tmp_err && tmp_err->code == ENOENT && pflag) { // failure on the simple call, try to do a recursive create errno = 0; g_clear_error(&tmp_err); ret = gfal_lfc_mkdir_rec(ops, (char *) path + 1, path, mode, &tmp_err); } if (ret == 0) { ret = gfal_lfc_endTransaction(ops, &tmp_err); } else { gfal_lfc_abortTransaction(ops, NULL); } } if (tmp_err) { gfal2_propagate_prefixed_error(err, tmp_err, __func__); } else errno = 0; return ret; } /* * return a list of surls from a getreplica request * */ char **gfal_lfc_getSURL(struct lfc_ops *ops, const char *path, GError **err) { struct lfc_filereplica *list = NULL; char **replicas = NULL; int size = 0, i; if (ops->getreplica(path, NULL, NULL, &size, &list) < 0) { int myerrno = gfal_lfc_get_errno(ops); gfal2_set_error(err, gfal2_get_plugin_lfc_quark(), myerrno, __func__, "error reported from lfc : %s", gfal_lfc_get_strerror(ops)); return NULL; } replicas = malloc(sizeof(char *) * (size + 1)); replicas[size] = NULL; for (i = 0; i < size; ++i) { replicas[i] = strndup(list[i].sfn, GFAL_URL_MAX_LEN); } free(list); return replicas; } /* * return the comment associated with this path * follow the xattr behavior, if buff==NULL, return only the appropriate buffer size for the call * @return the size of the comment or -1 if error */ int gfal_lfc_getComment(struct lfc_ops *ops, const char *lfn, char *buff, size_t s_buff, GError **err) { g_return_val_err_if_fail(lfn, -1, err, "bad path"); const size_t req_size = CA_MAXCOMMENTLEN + 1; char local_buff[CA_MAXCOMMENTLEN + 1]; int ret, resu_len; if (buff == NULL || s_buff == 0) { return req_size; } else { ret = ops->getcomment(lfn, local_buff); if (ret < 0) { const int sav_errno = gfal_lfc_get_errno(ops); if (sav_errno == ENOENT) { // no comments is define or not file exist, can be ambigous resu_len = 0; *buff = '\0'; ret = 0; } else { gfal2_set_error(err, gfal2_get_plugin_lfc_quark(), sav_errno, __func__, "Error report from LFC : %s", gfal_lfc_get_strerror(ops)); } } else { resu_len = strnlen(local_buff, MIN(s_buff, req_size)); *((char *) mempcpy(buff, local_buff, resu_len)) = '\0'; } return (ret == 0) ? (resu_len) : -1; } } int gfal_lfc_setComment(struct lfc_ops *ops, const char *lfn, const char *buff, size_t s_buff, GError **err) { g_return_val_err_if_fail(lfn, -1, err, "bad path"); int res = -1; char internal_buff[GFAL_URL_MAX_LEN]; GError *tmp_err = NULL; if (s_buff > 0 && buff != NULL) { *((char *) mempcpy(internal_buff, buff, MIN(s_buff, GFAL_URL_MAX_LEN - 1))) = '\0'; if ((res = ops->setcomment(lfn, internal_buff)) != 0) { const int sav_errno = gfal_lfc_get_errno(ops); gfal2_set_error(err, gfal2_get_plugin_lfc_quark(), sav_errno, __func__, "Error report from LFC : %s", gfal_lfc_get_strerror(ops)); } } else { gfal2_set_error(&tmp_err, gfal2_get_plugin_lfc_quark(), EINVAL, __func__, "sizeof the buffer incorrect"); } return res; } /* * provide the checksum for a given file * @return the size of the checksum string or -1 if error */ int gfal_lfc_getChecksum(struct lfc_ops *ops, const char *lfn, lfc_checksum *checksum, GError **err) { g_return_val_err_if_fail(ops && checksum, -1, err, " inval args") GError *tmp_err = NULL; struct lfc_filestatg statbuf; memset(&statbuf, 0, sizeof(struct lfc_filestatg)); int ret; if ((ret = gfal_lfc_statg(ops, lfn, &statbuf, &tmp_err)) == 0) { *((char *) mempcpy(checksum->type, statbuf.csumtype, sizeof(char) * 3)) = '\0'; *((char *) mempcpy(checksum->value, statbuf.csumvalue, sizeof(char) * 33)) = '\0'; } if (tmp_err) { gfal2_propagate_prefixed_error(err, tmp_err, __func__); } return ret; } int gfal_lfc_statg(struct lfc_ops *ops, const char *lfn, struct lfc_filestatg *statbuf, GError **err) { int ret = ops->statg(lfn, NULL, statbuf); if (ret != 0) { int sav_errno = gfal_lfc_get_errno(ops); gfal2_set_error(err, gfal2_get_plugin_lfc_quark(), sav_errno, __func__, "Error report from LFC : %s", gfal_lfc_get_strerror(ops)); } return ret; } ssize_t g_strv_catbuff(char **strv, char *buff, size_t size) { if (strv == NULL || buff == NULL) { return -1; } memset(buff, 0, size); const size_t sbuff = g_strv_length(strv); ssize_t resu = 0; size_t i; char *p = buff; for (i = 0; i < sbuff; ++i) { const size_t s_str = strnlen(strv[i], GFAL_URL_MAX_LEN); resu += s_str + 1; if (size) { *(p = (char *) mempcpy(p, strv[i], MIN(size, s_str))) = '\n'; ++p; } size = (size >= s_str + 1) ? (size - s_str - 1) : 0; } buff[resu - 1] = '\0'; return resu; } int gfal_lfc_get_errno(struct lfc_ops *ops) { int lfc_error; #if defined(_REENTRANT) || defined(_THREAD_SAFE) || (defined(_WIN32) && (defined(_MT) || defined(_DLL))) lfc_error = *(ops->get_serrno()); #else lfc_error = *ops->get_serrno ; #endif switch (lfc_error) { case ESEC_BAD_CREDENTIALS: lfc_error = EPERM; break; default: lfc_error = (lfc_error < 1000) ? lfc_error : ECOMM; } return lfc_error; } void gfal_lfc_reset_errno(struct lfc_ops *ops) { int *lfc_error; #if defined(_REENTRANT) || defined(_THREAD_SAFE) || (defined(_WIN32) && (defined(_MT) || defined(_DLL))) lfc_error = ops->get_serrno(); #else lfc_error = ops->get_serrno; #endif *lfc_error = 0; } char *gfal_lfc_get_strerror(struct lfc_ops *ops) { #if defined(_REENTRANT) || defined(_THREAD_SAFE) || (defined(_WIN32) && (defined(_MT) || defined(_DLL))) return ops->sstrerror(*(ops->get_serrno())); #else return ops->sstrerror (*ops->get_serrno); #endif } /* * @brief generate an uiid string * Generate a uuid string and copy it in the buf, * @warning buff must be > uuid size ( 37 bytes ) * * */ void gfal_generate_guidG(char *buf, GError **err) { uuid_t myuid; uuid_generate_random(myuid); uuid_unparse(myuid, buf); uuid_clear(myuid); } gfal2-v2.23.0/src/plugins/lfc/lfc_ifce_ng.h000066400000000000000000000140261465240014500203660ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #pragma once #define GFAL_MAX_LFCHOST_LEN 1024 #define NSTYPE_LFC #include #include #include #include #include #include #include #define LFC_ENV_VAR_HOST "LFC_HOST" #define LFC_GROUP_CONFIG_VAR "LFC PLUGIN" #define LFC_HOST_CONFIG_VAR LFC_ENV_VAR_HOST #define LFC_ENV_VAR_CONNTIMEOUT "LFC_CONNTIMEOUT" #define LFC_ENV_VAR_CONRETRY "LFC_CONRETRY" #define LFC_ENV_VAR_CONRETRYINT "LFC_CONRETRYINT" #define LFC_ENV_VAR_GROUP_PLUGIN "LFC PLUGIN" typedef struct _lfc_checksum{ char type[255]; char value[GFAL_URL_MAX_LEN]; } lfc_checksum; struct lfc_ops { char* lfc_endpoint_predefined; // default lfc env var char* lfc_conn_retry; char* lfc_conn_try_int; char* lfc_conn_timeout; regex_t rex; // regular expression compiled gfal2_context_t handle; GSimpleCache* cache_stat; // Store X509_USER_* environment prior to setenv calls so it can be restored char *env_user_cert, *env_user_key, *env_user_proxy; #if defined(_REENTRANT) || defined(_THREAD_SAFE) || (defined(_WIN32) && (defined(_MT) || defined(_DLL))) int* (*get_serrno)(void); #else int value_serrno; #endif char*(*sstrerror)(int); int (*addreplica)(const char *, struct lfc_fileid *, const char *, const char *, const char, const char, const char *, const char *); int (*creatg)(const char *, const char *, mode_t); int (*delreplica)(const char *, struct lfc_fileid *, const char *); int (*delfilesbyname)(int nbfiles, const char **paths, int force, int *nbstatuses, int **statuses); int (*aborttrans)(); int (*endtrans)(); int (*getpath)(char *, u_signed64, char *); int (*getlinks)(const char *, const char *, int *, struct lfc_linkinfo **); int (*getreplica)(const char *, const char *, const char *, int *, struct lfc_filereplica **); int (*setcomment) (const char * path, char * comment ); int (*getcomment) (const char * path, char * comment); int (*lstat)(const char *, struct lfc_filestat *); int (*readlink)(const char *, char *, size_t); int (*mkdirg)(const char *, const char *, mode_t); int (*seterrbuf)(char *, int); int (*setfsizeg)(const char *, u_signed64, const char *, char *); int (*setfsize)(const char *, struct lfc_fileid *, u_signed64); int (*starttrans)(char *, char *); int (*statg)(const char *, const char *, struct lfc_filestatg *); int (*statr)(const char *, struct lfc_filestatg *); int (*symlink)(const char *, const char *); int (*unlink)(const char *); int (*access)(const char *, int); int (*chmod)(const char *, mode_t); int (*closedir)(lfc_DIR*); int (*rename)(const char *, const char *); lfc_DIR *(*opendirg)(const char *, const char *); struct dirent* (*readdir)(lfc_DIR *); struct lfc_direnstat* (*readdirx)(lfc_DIR *dirp); int (*rmdir)(const char *); int (*startsess) (char *, char *); int (*endsess) (); int (*Cthread_init)(); int (*_Cthread_addcid)(char *, int, char *, int, Cth_pid_t *, unsigned, void *(*)(void *), int); int (*set_env)(const char*, const char*, int); const char* (*get_env)(const char*); }; int lfc_configure_environment(struct lfc_ops * ops, const char* host, const char *url, GError** err); void lfc_unset_environment(struct lfc_ops *ops); const char* lfc_plugin_get_lfc_env(struct lfc_ops* ops, const char* var_name); int gfal_lfc_get_errno(struct lfc_ops* ops); void gfal_lfc_reset_errno(struct lfc_ops* ops); int gfal_lfc_regex_compile(regex_t* rex, GError** err); char* gfal_lfc_get_strerror(struct lfc_ops* ops); char* gfal_convert_guid_to_lfn(plugin_handle handle, char* guid, GError ** err); int gfal_convert_guid_to_lfn_r(plugin_handle handle, const char* guid, char* buff_lfn, size_t sbuff_lfn, GError ** err); int gfal_lfc_statg(struct lfc_ops* ops, const char*, struct lfc_filestatg* resu, GError** err); int gfal_lfc_getComment(struct lfc_ops *ops, const char* lfn, char* buff, size_t s_buff, GError** err); int gfal_lfc_setComment(struct lfc_ops * ops, const char* lfn, const char* buff, size_t s_buff, GError** err); int gfal_lfc_getChecksum(struct lfc_ops* ops, const char* lfn, lfc_checksum* checksum, GError** err); int gfal_lfc_convert_statg(struct stat* output, struct lfc_filestatg* input, GError** err); int gfal_lfc_ifce_mkdirpG(struct lfc_ops* ops,const char* path, mode_t mode, gboolean pflag, GError** err); char ** gfal_lfc_getSURL(struct lfc_ops* ops, const char* path, GError** err); void gfal_lfc_init_thread(struct lfc_ops* ops); int gfal_lfc_startSession(struct lfc_ops* ops, GError ** err); void gfal_auto_maintain_session(struct lfc_ops* ops, GError ** err); ssize_t g_strv_catbuff(char** strv, char* buff, size_t size); int gfal_lfc_convert_lstat(struct stat* output, struct lfc_filestat* input, GError** err); void gfal_generate_guidG(char* buf, GError** err); struct lfc_ops* gfal_load_lfc(const char* name, GError** err); int gfal_lfc_register_check(plugin_handle plugin_data, gfal2_context_t context, const char* src, const char* dst, gfal_url2_check check); int gfal_lfc_register(plugin_handle plugin_data, gfal2_context_t context, gfalt_params_t params, const char* src, const char* dst, GError**); int gfal_lfc_unregister(plugin_handle handle, const char* path, const char* sfn, GError** err); gfal2-v2.23.0/src/plugins/lfc/lfc_register.c000066400000000000000000000260731465240014500206200ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include "gfal_lfc.h" #include "lfc_ifce_ng.h" struct size_and_checksum { u_signed64 filesize; char csumtype[3]; char csumvalue[33]; }; /** * Check URLs */ int gfal_lfc_register_check(plugin_handle handle, gfal2_context_t context, const char *src_url, const char *dst_url, gfal_url2_check check) { struct lfc_ops *ops = (struct lfc_ops *) handle; if (check != GFAL_FILE_COPY) { return 0; } return regexec(&(ops->rex), dst_url, 0, NULL, 0) == 0; } /** * Create path in the LFC, including the creation of the directories */ static int _lfc_touch(struct lfc_ops *ops, const char *path, const char *guid, struct size_and_checksum *info, GError **error) { char *last_slash = NULL; int ret_status = 0; gfal2_log(G_LOG_LEVEL_DEBUG, "lfc register: trying to create %s", path); last_slash = strrchr(path, '/'); if (last_slash != NULL) { size_t dir_len = last_slash - path + 1; char *dir = g_malloc0(dir_len); g_strlcpy(dir, path, dir_len); gfal2_log(G_LOG_LEVEL_DEBUG, "lfc register: checking parent directory %s", dir); if (ops->access(dir, F_OK) != 0) { gfal2_log(G_LOG_LEVEL_DEBUG, "lfc register: parent directory does not exist, creating", dir); ret_status = gfal_lfc_ifce_mkdirpG(ops, dir, 0755, TRUE, error); } g_free(dir); } if (ret_status == 0) { gfal2_log(G_LOG_LEVEL_DEBUG, "lfc register: creating the file"); ret_status = ops->creatg(path, guid, 0644); if (ret_status != 0) { gfal2_set_error(error, gfal2_get_plugins_quark(), errno, __func__, "Could not create the file: %s", gfal_lfc_get_strerror(ops)); } else { ret_status = ops->setfsizeg(guid, info->filesize, info->csumtype, info->csumvalue); if (ret_status != 0) { gfal2_set_error(error, gfal2_get_plugins_quark(), errno, __func__, "Could not set file size and checksum: %s", gfal_lfc_get_strerror(ops)); } } } return ret_status; } /** * Set host to point to the host part of the url */ static int _get_host(const char *url, char **host, GError **error) { regex_t regex; size_t ngroups = 4; regmatch_t matches[ngroups]; int ret = 0; regcomp(®ex, "(.+://([a-zA-Z0-9\\.-]+))(:[0-9]+)?/.+", REG_EXTENDED); ret = regexec(®ex, url, ngroups, matches, 0); if (ret == 0) { size_t host_len = matches[2].rm_eo - matches[2].rm_so; *host = g_malloc0(host_len + 1); g_strlcpy(*host, url + matches[2].rm_so, host_len); } else { char err_buffer[64]; regerror(ret, ®ex, err_buffer, sizeof(err_buffer)); gfal2_set_error(error, gfal2_get_plugins_quark(), EINVAL, __func__, "The source is not a valid url: %s (%s)", url, err_buffer); ret = -1; } regfree(®ex); return ret; } /** * Full checksum type from an LFC-like checksum type */ static const char *_full_checksum_type(const char *short_name) { if (strcmp("AD", short_name) == 0) { return "ADLER32"; } else if (strcmp("MD", short_name) == 0) { return "MD5"; } else { return "CS"; } } /** * Get checksum and file size from the source replica */ int _get_replica_info(gfal2_context_t context, struct size_and_checksum *info, const char *replica_url, GError **error) { memset(info, 0, sizeof(*info)); struct stat replica_stat; if (gfal2_stat(context, replica_url, &replica_stat, error) != 0) { return -1; } info->filesize = replica_stat.st_size; const char *lfc_checksums[] = {"AD", "MD", "CS", NULL}; unsigned i; for (i = 0; lfc_checksums[i] != NULL; ++i) { if (gfal2_checksum(context, replica_url, _full_checksum_type(lfc_checksums[i]), 0, 0, info->csumvalue, sizeof(info->csumvalue), NULL) == 0) { memcpy(info->csumtype, lfc_checksums[i], sizeof(info->csumtype)); gfal2_log(G_LOG_LEVEL_DEBUG, "found checksum %s:%s for the replica", info->csumtype, info->csumvalue); break; } } return 0; } /** * Checks that the new replica matches the information in the catalog */ int _validate_new_replica(gfal2_context_t context, struct lfc_filestatg *statg, struct size_and_checksum *replica_info, GError **error) { if (replica_info->filesize != statg->filesize) { gfal2_set_error(error, gfal2_get_plugin_lfc_quark(), EINVAL, __func__, "Replica file size (%lld) and LFC file size (%lld) do not match", replica_info->filesize, statg->filesize); return -1; } else { gfal2_log(G_LOG_LEVEL_DEBUG, "lfc register: file size match"); } if (statg->csumvalue[0] != '\0' && replica_info->csumvalue[0] != '\0' && strncmp(replica_info->csumtype, statg->csumtype, sizeof(statg->csumtype)) == 0) { if (strncmp(replica_info->csumvalue, statg->csumvalue, sizeof(statg->csumvalue)) != 0) { gfal2_set_error(error, gfal2_get_plugin_lfc_quark(), EINVAL, __func__, "Replica checksum (%s) and LFC checksum (%s) do not match", replica_info->csumvalue, statg->csumvalue); return -1; } else { gfal2_log(G_LOG_LEVEL_DEBUG, "lfc register: checksum match"); } } else { gfal2_log(G_LOG_LEVEL_WARNING, "lfc register: no checksum available to do the validation"); } return 0; } /** * src_url can be anything * dst_url must be an LFC url */ int gfal_lfc_register(plugin_handle handle, gfal2_context_t context, gfalt_params_t params, const char *src_url, const char *dst_url, GError **error) { struct lfc_ops *ops = (struct lfc_ops *) handle; char *lfc_host = NULL; char *lfc_path = NULL; char *src_host = NULL; int ret_status = 0; int lfc_errno = 0; gboolean file_existed = FALSE; GError *tmp_err = NULL; // Get URL components ret_status = url_converter(handle, dst_url, &lfc_host, &lfc_path, &tmp_err); if (ret_status != 0) { goto register_end; } ret_status = _get_host(src_url, &src_host, &tmp_err); if (ret_status != 0) { goto register_end; } gfal2_log(G_LOG_LEVEL_DEBUG, "lfc register: %s -> %s:%s", src_url, lfc_host, lfc_path); // Information about the replica struct size_and_checksum replica_info; ret_status = _get_replica_info(context, &replica_info, src_url, &tmp_err); if (ret_status != 0) { goto register_end; } // Set up LFC environment ret_status = lfc_configure_environment(ops, lfc_host, dst_url, &tmp_err); if (ret_status != 0) { goto register_end; } // Stat LFC entry struct lfc_filestatg statg; ret_status = ops->statg(lfc_path, NULL, &statg); lfc_errno = gfal_lfc_get_errno(ops); // File exists, validate the incoming replica if (ret_status == 0) { gfal2_log(G_LOG_LEVEL_DEBUG, "lfc register: lfc exists, validate"); file_existed = TRUE; ret_status = _validate_new_replica(context, &statg, &replica_info, &tmp_err); } // File do not exist, try to create else if (lfc_errno == ENOENT) { gfal_generate_guidG(statg.guid, NULL); ret_status = _lfc_touch(ops, lfc_path, statg.guid, &replica_info, &tmp_err); } // Failure else { ret_status = -1; gfal2_set_error(error, gfal2_get_plugin_lfc_quark(), errno, __func__, "Failed to stat the file: %s (%d)", gfal_lfc_get_strerror(ops), lfc_errno); } if (ret_status != 0) { goto register_end; } struct lfc_fileid unique_id = {{0}, 0}; unique_id.fileid = statg.fileid; g_strlcpy(unique_id.server, lfc_host, sizeof(unique_id.server)); ret_status = ops->addreplica(statg.guid, file_existed ? &unique_id : NULL, src_host, src_url, '-', 'P', NULL, NULL); if (ret_status != 0) { int err_code = gfal_lfc_get_errno(ops); if (err_code != EEXIST) { gfal2_set_error(error, gfal2_get_plugin_lfc_quark(), err_code, __func__, "Could not register the replica : %s ", gfal_lfc_get_strerror(ops)); } else { gfal2_log(G_LOG_LEVEL_MESSAGE, "lfc register: the replica is already registered, that is ok"); ret_status = 0; } } else { gfal2_log(G_LOG_LEVEL_DEBUG, "lfc register: done"); } register_end: if (tmp_err) { gfal2_propagate_prefixed_error(error, tmp_err, __func__); } g_free(lfc_host); g_free(lfc_path); g_free(src_host); lfc_unset_environment(ops); return ret_status; } /** * Unregister a replica */ int gfal_lfc_unregister(plugin_handle handle, const char *url, const char *sfn, GError **error) { struct lfc_ops *ops = (struct lfc_ops *) handle; int ret_status, lfc_errno; char *lfc_host = NULL; char *lfc_path = NULL; GError *tmp_err = NULL; ret_status = url_converter(handle, url, &lfc_host, &lfc_path, &tmp_err); if (ret_status < 0) { goto unregister_end; } ret_status = lfc_configure_environment(ops, lfc_host, url, &tmp_err); if (ret_status != 0) { goto unregister_end; } struct lfc_filestatg statg; ret_status = ops->statg(lfc_path, NULL, &statg); if (ret_status != 0) { lfc_errno = gfal_lfc_get_errno(ops); gfal2_set_error(error, gfal2_get_plugin_lfc_quark(), lfc_errno, __func__, "Could not stat the file: %s (%d)", gfal_lfc_get_strerror(ops), lfc_errno); goto unregister_end; } gfal2_log(G_LOG_LEVEL_DEBUG, "lfc unregister: the replica is to be unregistered (file id %d)", statg.fileid); struct lfc_fileid file_id = {{0}, 0}; file_id.fileid = statg.fileid; ret_status = ops->delreplica(NULL, &file_id, sfn); if (ret_status < 0) { lfc_errno = gfal_lfc_get_errno(ops); gfal2_set_error(error, gfal2_get_plugin_lfc_quark(), lfc_errno, __func__, "Could not register the replica : %s (%d) ", gfal_lfc_get_strerror(ops), lfc_errno); } gfal2_log(G_LOG_LEVEL_DEBUG, "lfc unregister: replica %s unregistered", sfn); unregister_end: g_free(lfc_host); g_free(lfc_path); if (tmp_err) { gfal2_propagate_prefixed_error(error, tmp_err, __func__); } lfc_unset_environment(ops); return ret_status; } gfal2-v2.23.0/src/plugins/mock/000077500000000000000000000000001465240014500161615ustar00rootroot00000000000000gfal2-v2.23.0/src/plugins/mock/CMakeLists.txt000066400000000000000000000016061465240014500207240ustar00rootroot00000000000000cmake_minimum_required (VERSION 2.6) if (PLUGIN_MOCK) file (GLOB src_file "*.c*") add_library (plugin_mock MODULE ${src_file}) target_link_libraries (plugin_mock gfal2 gfal2_transfer uuid) set_target_properties(plugin_mock PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/src CLEAN_DIRECT_OUTPUT 1 OUTPUT_NAME "gfal_plugin_mock" LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/plugins) install(TARGETS plugin_mock LIBRARY DESTINATION ${PLUGIN_INSTALL_DIR}) install(FILES "README_PLUGIN_MOCK" DESTINATION ${DOC_INSTALL_DIR}) # install mock configuration files LIST(APPEND mock_conf_file "${CMAKE_SOURCE_DIR}/dist/etc/gfal2.d/mock_plugin.conf") install(FILES ${mock_conf_file} DESTINATION ${SYSCONF_INSTALL_DIR}/gfal2.d/) endif (PLUGIN_MOCK) gfal2-v2.23.0/src/plugins/mock/README_PLUGIN_MOCK000066400000000000000000000027741465240014500207420ustar00rootroot00000000000000Mock plugin =========== Provides a fake protocol that can be customized by the caller via query arguments. It can be used for testing. Supported arguments: - list For directories, a comma separated list of items as name:mode(octal):size(decimal) - size File size, in bytes - size_pre File size, in bytes, for stats previous to a copy - size_post File size, in bytes, for stats following a copy - checksum Checksum value - time Time that a copy will take. To be specified on the destination URL. - errno Trigger an error with this errno number - transfer_errno Trigger an error with this errno number *during the transfer* - staging_time Staging total time - staging_errno Fail the staging with this error number - release_errno Fail the release with this error number - signal Raise the signal specified as an integer Also, if the string MOCK_LOAD_TIME_SIGNAL is found on any parameter for the current process (obtained reading /proc/self/cmdline), the following digits will be used to raise a signal at instantiation time. By default, signals are disabled. They have to be enabled setting SIGNALS to 1. Examples -------- Fail with a ENOENT gfal-stat mock://host/path?errno=2 Stat a regular file with a size of 1000 bytes gfal-stat mock://host/path?size=1000 Trigger a copy that will take 5 seconds gfal-copy "mock://host/path?size=1000" "mock://host/path2?errno=2&size_pre=0&size_post=1000&time=5" Trigger a segfault gfal-ls "mock://host/path?signal=11" gfal2-v2.23.0/src/plugins/mock/gfal_mock_archiving.c000066400000000000000000000055401465240014500223050ustar00rootroot00000000000000/* * Copyright (c) CERN 2022 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "gfal_mock_plugin.h" static GHashTable *archiving_end_table; __attribute__((constructor)) static void init() { archiving_end_table = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, g_free); } int gfal_plugin_mock_archive_poll(plugin_handle plugin_data, const char* url, GError** err) { // Archiving errno char arg_buffer[64]; gfal_plugin_mock_get_value(url, "archiving_errno", arg_buffer, sizeof(arg_buffer)); int archiving_error = gfal_plugin_mock_get_int_from_str(arg_buffer); // Look up in the hash table to see if a poll task is already in progress gboolean archiving_started = g_hash_table_contains(archiving_end_table, url); if (!archiving_started) { // Get archiving time from url gfal_plugin_mock_get_value(url, "archiving_time", arg_buffer, sizeof(arg_buffer)); time_t *archiving_end = g_new0(time_t, 1); *archiving_end = time(NULL) + gfal_plugin_mock_get_int_from_str(arg_buffer); g_hash_table_insert(archiving_end_table, g_strdup(url), archiving_end); } // Get archiving time from hash table in memory time_t *archiving_end = g_hash_table_lookup(archiving_end_table, url); if (archiving_end == NULL || *archiving_end <= time(NULL)) { if (archiving_error) { gfal_plugin_mock_report_error(strerror(archiving_error), archiving_error, err); g_hash_table_remove(archiving_end_table, url); return -1; } g_hash_table_remove(archiving_end_table, url); return 1; } else { gfal_plugin_mock_report_error("Not ready", EAGAIN, err); } return 0; } int gfal_plugin_mock_archive_poll_list(plugin_handle plugin_data, int nbfiles, const char* const* urls, GError** errors) { int terminal_count = 0, r, i, error_count = 0; for (i = 0; i < nbfiles; ++i) { r = gfal_plugin_mock_archive_poll(plugin_data, urls[i], &(errors[i])); if (r > 0) { ++terminal_count; } else if (r < 0) { ++terminal_count; ++error_count; } } if (terminal_count == nbfiles) return 1; return 0; } gfal2-v2.23.0/src/plugins/mock/gfal_mock_dir.c000066400000000000000000000065541465240014500211170ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "gfal_mock_plugin.h" #include typedef struct { struct stat st; struct dirent de; } MockPluginDirEntry; typedef struct { GSList *list; GSList *item; } MockPluginDirectory; gfal_file_handle gfal_plugin_mock_opendir(plugin_handle plugin_data, const char *url, GError **err) { struct stat st; gfal_plugin_mock_stat(plugin_data, url, &st, err); if (*err) { return NULL; } if (!S_ISDIR(st.st_mode)) { gfal_plugin_mock_report_error(strerror(ENOTDIR), ENOTDIR, err); return NULL; } char file_list[1024]; gfal_plugin_mock_get_value(url, "list", file_list, sizeof(file_list)); MockPluginDirectory *dir = g_malloc0(sizeof(MockPluginDirectory)); dir->list = NULL; // Populate list char *saveptr = NULL, *p; p = strtok_r(file_list, ",", &saveptr); while (p) { MockPluginDirEntry *entry = g_malloc0(sizeof(MockPluginDirEntry)); char *sep = strchr(p, ':'); if (!sep) { g_strlcpy(entry->de.d_name, p, sizeof(entry->de.d_name)); } else { g_strlcpy(entry->de.d_name, p, sep - p + 1); entry->st.st_mode = strtol(sep + 1, &sep, 8); if (!(entry->st.st_mode & S_IFMT)) { entry->st.st_mode |= S_IFREG; } if (sep) { entry->st.st_size = strtol(sep + 1, &sep, 10); } } entry->de.d_reclen = strnlen(entry->de.d_name, 256); dir->list = g_slist_append(dir->list, entry); p = strtok_r(NULL, ",", &saveptr); } dir->item = dir->list; return gfal_file_handle_new2(gfal_mock_plugin_getName(), dir, NULL, url); } int gfal_plugin_mock_closedir(plugin_handle plugin_data, gfal_file_handle dir_desc, GError **err) { MockPluginDirectory *dir = gfal_file_handle_get_fdesc(dir_desc); g_slist_foreach(dir->list, (GFunc) g_free, NULL); g_slist_free(dir->list); g_free(dir); gfal_file_handle_delete(dir_desc); return 0; } struct dirent *gfal_plugin_mock_readdirpp(plugin_handle plugin_data, gfal_file_handle dir_desc, struct stat *st, GError **err) { MockPluginDirectory *dir = gfal_file_handle_get_fdesc(dir_desc); if (!dir->item) { return NULL; } MockPluginDirEntry *entry = (MockPluginDirEntry *) (dir->item->data); dir->item = g_slist_next(dir->item); memcpy(st, &entry->st, sizeof(struct stat)); return &entry->de; } struct dirent *gfal_plugin_mock_readdir(plugin_handle plugin_data, gfal_file_handle dir_desc, GError **err) { struct stat st; return gfal_plugin_mock_readdirpp(plugin_data, dir_desc, &st, err); } gfal2-v2.23.0/src/plugins/mock/gfal_mock_file.c000066400000000000000000000103111465240014500212420ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "gfal_mock_plugin.h" #include #include #ifndef EBADFD #define EBADFD EBADF #endif typedef struct { const char *url; int fd; off_t size; off_t offset; } MockFile; gfal_file_handle gfal_plugin_mock_open(plugin_handle plugin_data, const char *url, int flag, mode_t mode, GError **err) { struct stat st; int ret = gfal_plugin_mock_stat(plugin_data, url, &st, err); if (ret < 0) { return NULL; } char arg_buffer[64] = {0}; gfal_plugin_mock_get_value(url, "open_errno", arg_buffer, sizeof(arg_buffer)); int errcode = gfal_plugin_mock_get_int_from_str(arg_buffer); if (errcode > 0) { gfal_plugin_mock_report_error(strerror(errcode), errcode, err); return NULL; } MockFile *fd = g_malloc(sizeof(MockFile)); fd->url = url; fd->size = st.st_size; fd->offset = 0; if (flag == O_RDONLY) { fd->fd = open("/dev/urandom", O_RDONLY); } else if (flag == O_WRONLY) { fd->fd = open("/dev/null", O_WRONLY); } else { gfal_plugin_mock_report_error("Mock plugin does not support read and write", ENOSYS, err); return NULL; } if (fd->fd < 0) { gfal_plugin_mock_report_error("Could not open the file!", errno, err); return NULL; } return gfal_file_handle_new2(gfal_mock_plugin_getName(), fd, NULL, url); } ssize_t gfal_plugin_mock_read(plugin_handle plugin_data, gfal_file_handle fd, void *buff, size_t count, GError **err) { MockFile *mfd = gfal_file_handle_get_fdesc(fd); char arg_buffer[64] = {0}; gfal_plugin_mock_get_value(mfd->url, "read_wait", arg_buffer, sizeof(arg_buffer)); int wait = gfal_plugin_mock_get_int_from_str(arg_buffer); if (wait > 0) { sleep(wait); } gfal_plugin_mock_get_value(mfd->url, "read_errno", arg_buffer, sizeof(arg_buffer)); int errcode = gfal_plugin_mock_get_int_from_str(arg_buffer); if (errcode > 0) { gfal_plugin_mock_report_error(strerror(errcode), errcode, err); return -1; } off_t remaining = mfd->size - mfd->offset; if (count > remaining) { count = remaining; } if (count < 0) { gfal_plugin_mock_report_error("Reading passed end of file", EBADFD, err); return -1; } off_t nread = read(mfd->fd, buff, count); if (nread < 0) { gfal_plugin_mock_report_error("Failed to read file", errno, err); return -1; } mfd->offset += nread; return nread; } ssize_t gfal_plugin_mock_write(plugin_handle plugin_data, gfal_file_handle fd, const void *buff, size_t count, GError **err) { MockFile *mfd = gfal_file_handle_get_fdesc(fd); off_t nwrite = write(mfd->fd, buff, count); if (nwrite < 0) { gfal_plugin_mock_report_error("Failed to write file", errno, err); } mfd->offset += nwrite; return nwrite; } int gfal_plugin_mock_close(plugin_handle plugin_data, gfal_file_handle fd, GError **err) { MockFile *mfd = gfal_file_handle_get_fdesc(fd); close(mfd->fd); g_free(mfd); return 0; } off_t gfal_plugin_mock_seek(plugin_handle plugin_data, gfal_file_handle fd, off_t offset, int whence, GError **err) { MockFile *mfd = gfal_file_handle_get_fdesc(fd); switch (whence) { case SEEK_SET: mfd->offset = offset; break; case SEEK_END: mfd->offset = mfd->size + offset; break; case SEEK_CUR: mfd->offset += offset; } return mfd->offset; } gfal2-v2.23.0/src/plugins/mock/gfal_mock_metadata.c000066400000000000000000000134231465240014500221120ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "gfal_mock_plugin.h" #include int gfal_plugin_mock_stat(plugin_handle plugin_data, const char *path, struct stat *buf, GError **err) { MockPluginData *mdata = plugin_data; char arg_buffer[64] = {0}; int errcode = 0; int signum = 0; unsigned long long size = 0, wait_time = 0; // Is fts_url_copy calling us? const char *agent, *version; gfal2_get_user_agent(mdata->handle, &agent, &version); int is_url_copy = (agent && strncmp(agent, "fts_url_copy", 12) == 0); // Wait a bit? gfal_plugin_mock_get_value(path, "wait", arg_buffer, sizeof(arg_buffer)); wait_time = gfal_plugin_mock_get_int_from_str(arg_buffer); if (wait_time > 0) { sleep(wait_time); } // Trigger signal gfal_plugin_mock_get_value(path, "signal", arg_buffer, sizeof(arg_buffer)); signum = gfal_plugin_mock_get_int_from_str(arg_buffer); if (signum > 0 && mdata->enable_signals) { sleep(1); raise(signum); } // Check errno first gfal_plugin_mock_get_value(path, "errno", arg_buffer, sizeof(arg_buffer)); errcode = gfal_plugin_mock_get_int_from_str(arg_buffer); if (errcode > 0) { gfal_plugin_mock_report_error(strerror(errcode), errcode, err); return -1; } // Default size first gfal_plugin_mock_get_value(path, "size", arg_buffer, sizeof(arg_buffer)); size = gfal_plugin_mock_get_unsigned_int_from_str(arg_buffer); // Try specific stage then if (is_url_copy) { switch (mdata->stat_stage) { case STAT_DESTINATION_BEFORE_TRANSFER: mdata->stat_stage = STAT_DESTINATION_AFTER_TRANSFER; gfal_plugin_mock_get_value(path, "size_pre", arg_buffer, sizeof(arg_buffer)); size = gfal_plugin_mock_get_unsigned_int_from_str(arg_buffer); // If this size were <= 0, consider it a ENOENT if (size <= 0) { gfal_plugin_mock_report_error(strerror(ENOENT), ENOENT, err); return -1; } break; case STAT_DESTINATION_AFTER_TRANSFER: mdata->stat_stage = STAT_SOURCE; gfal_plugin_mock_get_value(path, "size_post", arg_buffer, sizeof(arg_buffer)); size = gfal_plugin_mock_get_unsigned_int_from_str(arg_buffer); break; case STAT_SOURCE: mdata->stat_stage = STAT_DESTINATION_BEFORE_TRANSFER; break; } } // Set the struct memset(buf, 0x00, sizeof(*buf)); buf->st_size = size; buf->st_mode = 0755; arg_buffer[0] = '\0'; gfal_plugin_mock_get_value(path, "list", arg_buffer, sizeof(arg_buffer)); if (arg_buffer[0]) { buf->st_mode |= S_IFDIR; } else { buf->st_mode |= S_IFREG; } return 0; } int gfal_plugin_mock_unlink(plugin_handle plugin_data, const char *url, GError **err) { struct stat buf; if (gfal_plugin_mock_stat(plugin_data, url, &buf, err) < 0) return -1; return 0; } int gfal_mock_checksumG(plugin_handle plugin_data, const char* url, const char* check_type, char * checksum_buffer, size_t buffer_length, off_t start_offset, size_t data_length, GError ** err) { char arg_buffer[GFAL_URL_MAX_LEN] = {0}; int errcode = 0; // Check errno first gfal_plugin_mock_get_value(url, "errno", arg_buffer, sizeof(arg_buffer)); errcode = gfal_plugin_mock_get_int_from_str(arg_buffer); if (errcode > 0) { gfal_plugin_mock_report_error(strerror(errcode), errcode, err); return -1; } gfal_plugin_mock_get_value(url, "checksum", arg_buffer, sizeof(arg_buffer)); g_strlcpy(checksum_buffer, arg_buffer, buffer_length); return 0; } ssize_t gfal_mock_getxattrG(plugin_handle plugin_data, const char* url, const char* key, void* buff, size_t s_buff, GError** err) { char arg_buffer[GFAL_URL_MAX_LEN] = {0}; int errcode = 0; int emsg_size; char* emsg = NULL; // Check errno first gfal_plugin_mock_get_value(url, "errno", arg_buffer, sizeof(arg_buffer)); errcode = gfal_plugin_mock_get_int_from_str(arg_buffer); if (errcode > 0) { gfal_plugin_mock_report_error(strerror(errcode), errcode, err); return -1; } if ((strncmp(key, GFAL_XATTR_STATUS, sizeof(GFAL_XATTR_STATUS)) == 0) || (strncmp(key, GFAL_XATTR_REPLICA, sizeof(GFAL_XATTR_REPLICA)) == 0) || (strncmp(key, GFAL_XATTR_GUID, sizeof(GFAL_XATTR_GUID)) == 0) || (strncmp(key, GFAL_XATTR_COMMENT, sizeof(GFAL_XATTR_COMMENT)) == 0) || (strncmp(key, GFAL_XATTR_SPACETOKEN, sizeof(GFAL_XATTR_SPACETOKEN)) == 0)) { gfal_plugin_mock_get_value(url, key, arg_buffer, sizeof(arg_buffer)); g_strlcpy(buff, arg_buffer, s_buff); } if (arg_buffer[0] == '\0') { emsg_size = 26 + strlen(key); emsg = malloc(emsg_size); snprintf(emsg, emsg_size, "Failed to retrieve xattr %s", key); gfal_plugin_mock_report_error(emsg, ENOATTR, err); free(emsg); return -1; } return strlen(buff); } gfal2-v2.23.0/src/plugins/mock/gfal_mock_plugin.h000066400000000000000000000120431465240014500216320ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #ifndef GFAL_MOCK_PLUGIN_H #define GFAL_MOCK_PLUGIN_H #include // Types typedef enum { STAT_SOURCE = 0, STAT_DESTINATION_BEFORE_TRANSFER, STAT_DESTINATION_AFTER_TRANSFER } StatStage; typedef struct { gfal2_context_t handle; StatStage stat_stage; char enable_signals; } MockPluginData; // Helpers const char *gfal_mock_plugin_getName(); GQuark gfal2_get_plugin_mock_quark(); void gfal_plugin_mock_report_error(const char *msg, int errn, GError **err); void gfal_plugin_mock_get_value(const char *url, const char *key, char *value, size_t val_size); long long gfal_plugin_mock_get_int_from_str(const char* buff); unsigned long long gfal_plugin_mock_get_unsigned_int_from_str(const char* buff); // Metadata operations int gfal_plugin_mock_stat(plugin_handle plugin_data, const char *path, struct stat *buf, GError **err); int gfal_plugin_mock_unlink(plugin_handle plugin_data, const char *url, GError **err); int gfal_mock_checksumG(plugin_handle plugin_data, const char* url, const char* check_type, char * checksum_buffer, size_t buffer_length, off_t start_offset, size_t data_length, GError ** err); ssize_t gfal_mock_getxattrG(plugin_handle plugin_data, const char* url, const char* key, void* buff, size_t s_buff, GError** err); // Directory operations gfal_file_handle gfal_plugin_mock_opendir(plugin_handle plugin_data, const char *url, GError **err); int gfal_plugin_mock_closedir(plugin_handle plugin_data, gfal_file_handle dir_desc, GError **err); struct dirent *gfal_plugin_mock_readdirpp(plugin_handle plugin_data, gfal_file_handle dir_desc, struct stat *st, GError **err); struct dirent *gfal_plugin_mock_readdir(plugin_handle plugin_data, gfal_file_handle dir_desc, GError **err); // IO operations gfal_file_handle gfal_plugin_mock_open(plugin_handle plugin_data, const char *url, int flag, mode_t mode, GError **); ssize_t gfal_plugin_mock_read(plugin_handle, gfal_file_handle fd, void *buff, size_t count, GError **); ssize_t gfal_plugin_mock_write(plugin_handle, gfal_file_handle fd, const void *buff, size_t count, GError **); int gfal_plugin_mock_close(plugin_handle, gfal_file_handle fd, GError **); off_t gfal_plugin_mock_seek(plugin_handle, gfal_file_handle fd, off_t offset, int whence, GError **err); // Staging int gfal_plugin_mock_bring_online(plugin_handle plugin_data, const char *url, time_t pintime, time_t timeout, char *token, size_t tsize, int async, GError **err); int gfal_plugin_mock_bring_online_v2(plugin_handle plugin_data, const char *url, const char *metadata, time_t pintime, time_t timeout, char *token, size_t tsize, int async, GError **err); int gfal_plugin_mock_bring_online_poll(plugin_handle plugin_data, const char *url, const char *token, GError **err); int gfal_plugin_mock_release_file(plugin_handle plugin_data, const char *url, const char *token, GError **err); int gfal_plugin_mock_archive_poll(plugin_handle plugin_data, const char* url, GError** err); int gfal_plugin_mock_bring_online_list(plugin_handle plugin_data, int nbfiles, const char *const *urls, time_t pintime, time_t timeout, char *token, size_t tsize, int async, GError **err); int gfal_plugin_mock_bring_online_list_v2(plugin_handle plugin_data, int nbfiles, const char *const *urls, const char *const *metadata, time_t pintime, time_t timeout, char *token, size_t tsize, int async, GError **err); int gfal_plugin_mock_bring_online_poll_list(plugin_handle plugin_data, int nbfiles, const char *const *urls, const char *token, GError **err); int gfal_plugin_mock_release_file_list(plugin_handle plugin_data, int nbfiles, const char *const *urls, const char *token, GError **err); int gfal_plugin_mock_abort_file_list(plugin_handle plugin_data, int nbfiles, const char *const *uris, const char *token, GError **err); int gfal_plugin_mock_archive_poll_list(plugin_handle plugin_data, int nbfiles, const char* const* urls, GError** errors); // Copy int gfal_plugin_mock_filecopy(plugin_handle plugin_data, gfal2_context_t context, gfalt_params_t params, const char *src, const char *dst, GError **err); #endif // GFAL_MOCK_PLUGIN_H gfal2-v2.23.0/src/plugins/mock/gfal_mock_plugin_main.c000066400000000000000000000153071465240014500226370ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include "gfal_mock_plugin.h" GQuark gfal2_get_plugin_mock_quark() { return g_quark_from_static_string(GFAL2_QUARK_PLUGINS "::FILE"); } static gboolean is_mock_uri(const char *src) { return strncmp(src, "mock:", 5) == 0; } const char *gfal_mock_plugin_getName() { return GFAL2_PLUGIN_VERSIONED("mock", VERSION); } static gboolean gfal_mock_check_url(plugin_handle handle, const char *url, plugin_mode mode, GError **err) { g_return_val_err_if_fail(url != NULL, EINVAL, err, "[gfal_lfile_path_checker] Invalid url "); switch (mode) { //case GFAL_PLUGIN_ACCESS: //case GFAL_PLUGIN_MKDIR: case GFAL_PLUGIN_STAT: case GFAL_PLUGIN_LSTAT: //case GFAL_PLUGIN_RMDIR: case GFAL_PLUGIN_OPENDIR: //case GFAL_PLUGIN_OPEN: //case GFAL_PLUGIN_CHMOD: case GFAL_PLUGIN_UNLINK: case GFAL_PLUGIN_GETXATTR: //case GFAL_PLUGIN_LISTXATTR: //case GFAL_PLUGIN_SETXATTR: //case GFAL_PLUGIN_RENAME: //case GFAL_PLUGIN_SYMLINK: case GFAL_PLUGIN_CHECKSUM: case GFAL_PLUGIN_BRING_ONLINE: case GFAL_PLUGIN_ARCHIVE: case GFAL_PLUGIN_OPEN: return is_mock_uri(url); default: return FALSE; } } void gfal_plugin_mock_report_error(const char *msg, int errn, GError **err) { g_set_error(err, gfal2_get_plugin_mock_quark(), errn, "%s", msg); } void gfal_plugin_mock_get_value(const char *url, const char *key, char *value, size_t val_size) { // make sure it's an empty C-string value[0] = '\0'; char *str = strchr(url, '?'); if (str == NULL) { return; } size_t key_len = strlen(key); char **args = g_strsplit(str + 1, "&", 0); int i; for (i = 0; args[i] != NULL; ++i) { if (strncmp(args[i], key, key_len) == 0) { char *p = strchr(args[i], '='); if (p) { g_strlcpy(value, p + 1, val_size); break; } } } g_strfreev(args); } long long gfal_plugin_mock_get_int_from_str(const char *buff) { if (buff == 0 || buff[0] == '\0') return 0; return atoll(buff); } unsigned long long gfal_plugin_mock_get_unsigned_int_from_str(const char *buff) { if (buff == 0 || buff[0] == '\0') return 0; char* pEnd; return strtoull(buff, &pEnd, 10); } gboolean gfal_plugin_mock_check_url_transfer(plugin_handle handle, gfal2_context_t ctx, const char *src, const char *dst, gfal_url2_check type) { gboolean res = FALSE; if (src != NULL && dst != NULL) { if (type == GFAL_FILE_COPY && is_mock_uri(src) && is_mock_uri(dst)) { res = TRUE; } } return res; } void gfal_plugin_mock_delete(plugin_handle plugin_data) { free(plugin_data); } /* * This is a super ugly hack to allow gfal2 mock plugin to kill self * on instantiation time. It peeks the arguments looking for a "magic" * string */ static void gfal_mock_seppuku_hook() { static const char mock_load_time_signal_str[] = "MOCK_LOAD_TIME_SIGNAL"; FILE *fd = fopen("/proc/self/cmdline", "r"); if (!fd) { return; } char *value = NULL; size_t len = 0; const char *signal_str = NULL; while (getdelim(&value, &len, '\0', fd) != -1) { signal_str = strstr(value, mock_load_time_signal_str); if (signal_str) { break; } } fclose(fd); if (signal_str) { signal_str += sizeof(mock_load_time_signal_str) - 1; int signal_number = gfal_plugin_mock_get_int_from_str(signal_str); raise(signal_number); } free(value); } /* * Init function, called before all **/ gfal_plugin_interface gfal_plugin_init(gfal2_context_t handle, GError **err) { gfal_plugin_interface mock_plugin; memset(&mock_plugin, 0, sizeof(gfal_plugin_interface)); MockPluginData *mdata = calloc(1, sizeof(MockPluginData)); mdata->handle = handle; mdata->enable_signals = gfal2_get_opt_boolean_with_default(handle, "MOCK PLUGIN", "SIGNALS", FALSE); if (mdata->enable_signals) { gfal_mock_seppuku_hook(); } mock_plugin.plugin_data = mdata; mock_plugin.plugin_delete = gfal_plugin_mock_delete; mock_plugin.check_plugin_url = &gfal_mock_check_url; mock_plugin.getName = &gfal_mock_plugin_getName; mock_plugin.statG = &gfal_plugin_mock_stat; mock_plugin.lstatG = &gfal_plugin_mock_stat; mock_plugin.unlinkG = &gfal_plugin_mock_unlink; mock_plugin.getxattrG = &gfal_mock_getxattrG; mock_plugin.checksum_calcG = &gfal_mock_checksumG; mock_plugin.bring_online = gfal_plugin_mock_bring_online; mock_plugin.bring_online_v2 = gfal_plugin_mock_bring_online_v2; mock_plugin.bring_online_poll = gfal_plugin_mock_bring_online_poll; mock_plugin.release_file = gfal_plugin_mock_release_file; mock_plugin.archive_poll = gfal_plugin_mock_archive_poll; mock_plugin.bring_online_list = gfal_plugin_mock_bring_online_list; mock_plugin.bring_online_list_v2 = gfal_plugin_mock_bring_online_list_v2; mock_plugin.bring_online_poll_list = gfal_plugin_mock_bring_online_poll_list; mock_plugin.release_file_list = gfal_plugin_mock_release_file_list; mock_plugin.abort_files = gfal_plugin_mock_abort_file_list; mock_plugin.archive_poll_list = &gfal_plugin_mock_archive_poll_list; mock_plugin.check_plugin_url_transfer = &gfal_plugin_mock_check_url_transfer; mock_plugin.copy_file = &gfal_plugin_mock_filecopy; mock_plugin.opendirG = gfal_plugin_mock_opendir; mock_plugin.readdirG = gfal_plugin_mock_readdir; mock_plugin.readdirppG = gfal_plugin_mock_readdirpp; mock_plugin.closedirG = gfal_plugin_mock_closedir; mock_plugin.openG = gfal_plugin_mock_open; mock_plugin.closeG = gfal_plugin_mock_close; mock_plugin.readG = gfal_plugin_mock_read; mock_plugin.writeG = gfal_plugin_mock_write; mock_plugin.lseekG = gfal_plugin_mock_seek; return mock_plugin; } gfal2-v2.23.0/src/plugins/mock/gfal_mock_staging.c000066400000000000000000000134451465240014500217720ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "gfal_mock_plugin.h" #include #include static GHashTable *staging_end_table; __attribute__((constructor)) static void init() { staging_end_table = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, g_free); } int gfal_plugin_mock_bring_online(plugin_handle plugin_data, const char *url, time_t pintime, time_t timeout, char *token, size_t tsize, int async, GError **err) { char arg_buffer[64]; // Bring online errno gfal_plugin_mock_get_value(url, "staging_errno", arg_buffer, sizeof(arg_buffer)); int staging_errno = gfal_plugin_mock_get_int_from_str(arg_buffer); // Polling time gfal_plugin_mock_get_value(url, "staging_time", arg_buffer, sizeof(arg_buffer)); time_t *staging_end = g_new0(time_t, 1); *staging_end = time(NULL) + gfal_plugin_mock_get_int_from_str(arg_buffer); g_hash_table_insert(staging_end_table, g_strdup(url), staging_end); // Fake token if (tsize > 36) { uuid_t uuid; uuid_generate_random(uuid); uuid_unparse(uuid, token); } else { g_strlcpy(token, "mock-token", tsize); } // Now, if remaining is <= 0, or blocking call, we are done if (*staging_end <= time(NULL) || !async) { if (staging_errno) { gfal_plugin_mock_report_error(strerror(staging_errno), staging_errno, err); return -1; } return 1; } return 0; } int gfal_plugin_mock_bring_online_v2(plugin_handle plugin_data, const char *url, const char *metadata, time_t pintime, time_t timeout, char *token, size_t tsize, int async, GError **err) { return gfal_plugin_mock_bring_online(plugin_data, url, pintime, timeout, token, tsize, async, err); } int gfal_plugin_mock_bring_online_poll(plugin_handle plugin_data, const char *url, const char *token, GError **err) { char arg_buffer[64]; gfal_plugin_mock_get_value(url, "staging_errno", arg_buffer, sizeof(arg_buffer)); int staging_errno = gfal_plugin_mock_get_int_from_str(arg_buffer); time_t *staging_end = g_hash_table_lookup(staging_end_table, url); if (staging_end == NULL || *staging_end <= time(NULL)) { if (staging_errno) { gfal_plugin_mock_report_error(strerror(staging_errno), staging_errno, err); return -1; } return 1; } else { gfal_plugin_mock_report_error("Not ready", EAGAIN, err); } return 0; } int gfal_plugin_mock_release_file(plugin_handle plugin_data, const char *url, const char *token, GError **err) { char arg_buffer[64]; gfal_plugin_mock_get_value(url, "release_errno", arg_buffer, sizeof(arg_buffer)); int release_errno = gfal_plugin_mock_get_int_from_str(arg_buffer); if (release_errno) { gfal_plugin_mock_report_error(strerror(release_errno), release_errno, err); return -1; } return 0; } int gfal_plugin_mock_bring_online_list(plugin_handle plugin_data, int nbfiles, const char *const *urls, time_t pintime, time_t timeout, char *token, size_t tsize, int async, GError **err) { int terminal_count = 0, r, i; for (i = 0; i < nbfiles; ++i) { r = gfal_plugin_mock_bring_online(plugin_data, urls[i], pintime, timeout, token, tsize, async, &(err[i])); if (r > 0) ++terminal_count; } if (terminal_count == nbfiles) return 1; return 0; } int gfal_plugin_mock_bring_online_list_v2(plugin_handle plugin_data, int nbfiles, const char *const *urls, const char *const *metadata, time_t pintime, time_t timeout, char *token, size_t tsize, int async, GError **err) { return gfal_plugin_mock_bring_online_list(plugin_data, nbfiles, urls, pintime, timeout, token, tsize, async, err); } int gfal_plugin_mock_bring_online_poll_list(plugin_handle plugin_data, int nbfiles, const char *const *urls, const char *token, GError **err) { int terminal_count = 0, r, i, error_count = 0; for (i = 0; i < nbfiles; ++i) { r = gfal_plugin_mock_bring_online_poll(plugin_data, urls[i], token, &(err[i])); if (r > 0) { ++terminal_count; } else if (r < 0) { ++terminal_count; ++error_count; } } if (terminal_count == nbfiles) return 1; return 0; } int gfal_plugin_mock_release_file_list(plugin_handle plugin_data, int nbfiles, const char *const *urls, const char *token, GError **err) { int i; for (i = 0; i < nbfiles; ++i) { gfal_plugin_mock_release_file(plugin_data, urls[i], token, &(err[i])); } return 1; } int gfal_plugin_mock_abort_file_list(plugin_handle plugin_data, int nbfiles, const char *const *uris, const char *token, GError **err) { MockPluginData *mdata = plugin_data; // Just make sure the pointers are at least valid, so access them int token_len = strlen(token); int i = 0, total_len = 0; for (i = 0; i < nbfiles; ++i) { total_len += strlen(uris[i]); } gfal2_log(G_LOG_LEVEL_DEBUG, "Counter to avoid optimizing away: %d (state %d)", token_len + total_len, mdata->stat_stage); return 0; } gfal2-v2.23.0/src/plugins/mock/gfal_mock_transfer.c000066400000000000000000000116351465240014500221610ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "gfal_mock_plugin.h" #include static gboolean gfal_plugin_mock_checksum_verify(const char *chk1, const char *chk2) { // if no checksum was defined return if (!chk1 || !chk2 || !*chk1 || !*chk2) return TRUE; return strcmp(chk1, chk2) == 0; } static void gfal_mock_cancel_transfer(gfal2_context_t context, void *userdata) { int *seconds = (int *) userdata; *seconds = -10; } int gfal_plugin_mock_filecopy(plugin_handle plugin_data, gfal2_context_t context, gfalt_params_t params, const char *src, const char *dst, GError **err) { char checksum_type[GFAL_URL_MAX_LEN] = {0}; char checksum_usr[GFAL_URL_MAX_LEN] = {0}; char checksum_src[GFAL_URL_MAX_LEN] = {0}; gfalt_checksum_mode_t checksum_method = gfalt_get_checksum(params, checksum_type, sizeof(checksum_type), checksum_usr, sizeof(checksum_usr), NULL); // validate source checksum if (checksum_method & GFALT_CHECKSUM_SOURCE) { gfal_plugin_mock_get_value(src, "checksum", checksum_src, sizeof(checksum_src)); if (!gfal_plugin_mock_checksum_verify(checksum_usr, checksum_src)) { gfal_plugin_mock_report_error("User and source checksums do not match", EIO, err); return -1; } } // transfer duration int seconds = 0; // check if the duration is specified in destination char time_dst[GFAL_URL_MAX_LEN] = {0}; gfal_plugin_mock_get_value(dst, "time", time_dst, sizeof(time_dst)); if (time_dst[0] != '\0') { // get the value from destination seconds = atoi(time_dst); } else { // get the range from configuration file int max = gfal2_get_opt_integer_with_default(context, "MOCK PLUGIN", "MAX_TRANSFER_TIME", 100); int min = gfal2_get_opt_integer_with_default(context, "MOCK PLUGIN", "MIN_TRANSFER_TIME", 10); // determine the duration if (max == min) seconds = max; else seconds = rand() % (max - min) + min; } // Trigger an error on the transfer? char transfer_errno_buffer[64] = {0}; gfal_plugin_mock_get_value(dst, "transfer_errno", transfer_errno_buffer, sizeof(transfer_errno_buffer)); int transfer_errno = gfal_plugin_mock_get_int_from_str(transfer_errno_buffer); // mock transfer duration gfal_cancel_token_t cancel_token; cancel_token = gfal2_register_cancel_callback(context, gfal_mock_cancel_transfer, &seconds); plugin_trigger_event(params, gfal2_get_plugin_mock_quark(), GFAL_EVENT_NONE, GFAL_EVENT_TRANSFER_ENTER, "Mock copy start, sleep %d", seconds); plugin_trigger_event(params, gfal2_get_plugin_mock_quark(), GFAL_EVENT_NONE, GFAL_EVENT_TRANSFER_TYPE, "mock"); while (seconds > 0) { sleep(1); --seconds; // Fail here if (transfer_errno) { gfal_plugin_mock_report_error(strerror(transfer_errno), transfer_errno, err); break; } } plugin_trigger_event(params, gfal2_get_plugin_mock_quark(), GFAL_EVENT_NONE, GFAL_EVENT_TRANSFER_EXIT, "Mock copy start, sleep %d", seconds); gfal2_remove_cancel_callback(context, cancel_token); // Canceled? if (seconds < 0) { gfal_plugin_mock_report_error("Transfer canceled", ECANCELED, err); return -1; } // Jump over to the destination stat MockPluginData *mdata = plugin_data; mdata->stat_stage = STAT_DESTINATION_AFTER_TRANSFER; // validate destination checksum if (!*err && (checksum_method & GFALT_CHECKSUM_TARGET)) { char checksum_dst[GFAL_URL_MAX_LEN] = {0}; gfal_plugin_mock_get_value(dst, "checksum", checksum_dst, sizeof(checksum_dst)); if (checksum_method & GFALT_CHECKSUM_SOURCE) { if (!gfal_plugin_mock_checksum_verify(checksum_src, checksum_dst)) { gfal_plugin_mock_report_error("Source and destination checksums do not match", EIO, err); } } else { if (!gfal_plugin_mock_checksum_verify(checksum_usr, checksum_dst)) { gfal_plugin_mock_report_error("User and destination checksums do not match", EIO, err); } } } if (*err) return -1; return 0; } gfal2-v2.23.0/src/plugins/rfio/000077500000000000000000000000001465240014500161675ustar00rootroot00000000000000gfal2-v2.23.0/src/plugins/rfio/CMakeLists.txt000066400000000000000000000017271465240014500207360ustar00rootroot00000000000000cmake_minimum_required (VERSION 2.6) if (PLUGIN_RFIO) file (GLOB src_rfio "*.c*") find_package (DPM REQUIRED) include_directories (${DPM_INCLUDE_DIR}) add_definitions(${DPM_CFLAGS}) add_library (plugin_rfio MODULE ${src_rfio}) target_link_libraries (plugin_rfio gfal2 ${DPM_LIBRARIES}) set_target_properties(plugin_rfio PROPERTIES CLEAN_DIRECT_OUTPUT 1 OUTPUT_NAME "gfal_plugin_rfio" LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/plugins) install(TARGETS plugin_rfio LIBRARY DESTINATION ${PLUGIN_INSTALL_DIR}) install(FILES "README_PLUGIN_RFIO" DESTINATION ${DOC_INSTALL_DIR}) # install rfio configuration files LIST(APPEND rfio_conf_file "${CMAKE_SOURCE_DIR}/dist/etc/gfal2.d/rfio_plugin.conf") install(FILES ${rfio_conf_file} DESTINATION ${SYSCONF_INSTALL_DIR}/gfal2.d/) endif (PLUGIN_RFIO) gfal2-v2.23.0/src/plugins/rfio/README_PLUGIN_RFIO000066400000000000000000000001001465240014500207330ustar00rootroot00000000000000 gfal2 rfio plugin : - features : * data access functions gfal2-v2.23.0/src/plugins/rfio/gfal_rfio_plugin_bindings.c000066400000000000000000000113251465240014500235200ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include "gfal_rfio_plugin_bindings.h" #include "gfal_rfio_plugin_main.h" #include "gfal_rfio_plugin_layer.h" #ifndef ECOMM #define ECOMM EIO #endif static void rfio_report_error(gfal_plugin_rfio_handle h, const char * func_name, GError** err){ char buff_error[2048]= {0}; int status = h->rf->geterror(); status = (status > 1000)?ECOMM:status; h->rf->serror_r(buff_error, 2048); gfal2_set_error(err, gfal2_get_plugin_rfio_quark(), status, func_name, "Error reported by the external library rfio : %s", buff_error); } gfal_file_handle gfal_rfio_openG(plugin_handle handle , const char* path, int flag, mode_t mode, GError** err){ gfal_plugin_rfio_handle h = (gfal_plugin_rfio_handle) handle; gfal_file_handle ret = NULL; gfal2_log(G_LOG_LEVEL_DEBUG, " %s -> ",__func__); int fd= h->rf->open(path, flag, mode); if(fd <= 0) rfio_report_error(h, __func__, err); else ret = gfal_file_handle_new(gfal_rfio_getName(), GINT_TO_POINTER(fd)); return ret; } ssize_t gfal_rfio_readG(plugin_handle handle , gfal_file_handle fd, void* buff, size_t s_buff, GError** err){ gfal_plugin_rfio_handle h = (gfal_plugin_rfio_handle) handle; int ret = h->rf->read(GPOINTER_TO_INT(gfal_file_handle_get_fdesc(fd)), buff, s_buff); if(ret <0) rfio_report_error(h, __func__, err); else errno =0; return ret; } off_t gfal_rfio_lseekG(plugin_handle handle , gfal_file_handle fd, off_t offset, int whence, GError** err){ gfal_plugin_rfio_handle h = (gfal_plugin_rfio_handle) handle; off_t ret = h->rf->lseek(GPOINTER_TO_INT(gfal_file_handle_get_fdesc(fd)), offset, (int) whence); if(ret == ((off_t)0)-1) rfio_report_error(h, __func__, err); else errno =0; return (int)ret; } ssize_t gfal_rfio_writeG(plugin_handle handle , gfal_file_handle fd, const void* buff, size_t s_buff, GError** err){ gfal_plugin_rfio_handle h = (gfal_plugin_rfio_handle) handle; int ret = h->rf->write(GPOINTER_TO_INT(gfal_file_handle_get_fdesc(fd)), (void*) buff, s_buff); if(ret <0) rfio_report_error(h, __func__, err); else errno =0; return ret; } int gfal_rfio_closeG(plugin_handle handle, gfal_file_handle fd, GError ** err){ gfal_plugin_rfio_handle h = (gfal_plugin_rfio_handle) handle; int ret= h->rf->close(GPOINTER_TO_INT(gfal_file_handle_get_fdesc(fd))); if(ret != 0){ rfio_report_error(h, __func__, err); }else gfal_file_handle_delete(fd); return ret; } int gfal_rfio_statG(plugin_handle handle, const char* name, struct stat* buff, GError ** err){ gfal_plugin_rfio_handle h = (gfal_plugin_rfio_handle) handle; int ret= h->rf->stat(name, buff); if(ret != 0){ rfio_report_error(h, __func__, err); } return ret; } int gfal_rfio_lstatG(plugin_handle handle, const char* name, struct stat* buff, GError ** err){ gfal_plugin_rfio_handle h = (gfal_plugin_rfio_handle) handle; int ret= h->rf->lstat(name, buff); if(ret != 0){ rfio_report_error(h, __func__, err); } return ret; } gfal_file_handle gfal_rfio_opendirG(plugin_handle handle, const char* name, GError ** err){ gfal_plugin_rfio_handle h = (gfal_plugin_rfio_handle) handle; DIR * ret = h->rf->opendir(name); if(ret == NULL){ rfio_report_error(h, __func__, err); return NULL; } return gfal_file_handle_new(gfal_rfio_getName(), (gpointer) ret); } struct dirent* gfal_rfio_readdirG(plugin_handle handle, gfal_file_handle fh , GError** err){ gfal_plugin_rfio_handle h = (gfal_plugin_rfio_handle) handle; struct dirent* dir = h->rf->readdir(gfal_file_handle_get_fdesc(fh)); if(dir == NULL && h->rf->geterror() != 0){ rfio_report_error(h, __func__, err); return NULL; } return dir; } int gfal_rfio_closedirG(plugin_handle handle, gfal_file_handle fh, GError** err){ gfal_plugin_rfio_handle h = (gfal_plugin_rfio_handle) handle; int ret = h->rf->closedir(gfal_file_handle_get_fdesc(fh)); if(ret != 0){ rfio_report_error(h, __func__, err); } gfal_file_handle_delete(fh); return ret; } const char* gfal_rfio_getName(){ return GFAL2_PLUGIN_VERSIONED("rfio", VERSION); } gfal2-v2.23.0/src/plugins/rfio/gfal_rfio_plugin_bindings.h000066400000000000000000000034721465240014500235310ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include gfal_file_handle gfal_rfio_openG(plugin_handle ch , const char* path, int flag, mode_t mode, GError**); int gfal_rfio_closeG(plugin_handle handle, gfal_file_handle fd, GError ** err); ssize_t gfal_rfio_writeG(plugin_handle handle , gfal_file_handle fd, const void* buff, size_t s_buff, GError** err); ssize_t gfal_rfio_readG(plugin_handle handle , gfal_file_handle fd, void* buff, size_t s_buff, GError** err); off_t gfal_rfio_lseekG(plugin_handle handle , gfal_file_handle fd, off_t offset, int whence, GError** err); int gfal_rfio_statG(plugin_handle handle, const char* name, struct stat* buff, GError ** err); int gfal_rfio_lstatG(plugin_handle handle, const char* name, struct stat* buff, GError ** err); gfal_file_handle gfal_rfio_opendirG(plugin_handle handle, const char* name, GError ** err); struct dirent* gfal_rfio_readdirG(plugin_handle handle, gfal_file_handle fh , GError** err); int gfal_rfio_closedirG(plugin_handle handle, gfal_file_handle fh, GError** err); const char* gfal_rfio_getName(); gfal2-v2.23.0/src/plugins/rfio/gfal_rfio_plugin_layer.c000066400000000000000000000107121465240014500230360ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include #include "gfal_rfio_plugin_layer.h" #include "gfal_rfio_plugin_main.h" static char* libdpm_name= "libdpm.so.1"; static char* libcastor_name= "libshift.so.2.1"; struct rfio_proto_ops * gfal_rfio_internal_loader_base(GError** err){ void *dlhandle=NULL; struct rfio_proto_ops * pops = NULL; GError* tmp_err=NULL; char *p; char* libname=NULL; p = getenv ("LCG_RFIO_TYPE"); if (p && strcmp (p, "dpm") == 0) { libname = libdpm_name; } else if (p && strcmp (p, "castor") == 0) { libname= libcastor_name; } if( libname != NULL){ gfal2_log(G_LOG_LEVEL_DEBUG, " lib rfio defined in LCG_RFIO_TYPE : %s", libname); if( (dlhandle = dlopen(libname, RTLD_LAZY)) == NULL){ gfal2_set_error(&tmp_err, gfal2_get_plugin_rfio_quark(), EPROTONOSUPPORT, __func__, " library %s for the rfio_plugin cannot be loaded properly, failure : %s ", libname, dlerror()); } }else{ gfal2_log(G_LOG_LEVEL_DEBUG, "lib rfio is not defined in LCG_RFIO_TYPE, try to found it "); char* tab_lib[] = { libdpm_name, libcastor_name, NULL}; char** p = tab_lib; while(*p != NULL){ if((dlhandle = dlopen (*p, RTLD_LAZY)) != NULL){ gfal2_log(G_LOG_LEVEL_DEBUG, "rfio library %s found! configured to use it", *p); break; } p++; } if(!dlhandle){ gfal2_set_error(&tmp_err, gfal2_get_plugin_rfio_quark(), EPROTONOSUPPORT, __func__, "Unable to find %s or %s, failure : %s ", libcastor_name, libdpm_name, dlerror()); } } if(dlhandle){ pops = g_new0(struct rfio_proto_ops, 1); pops->geterror = (int (*) ()) dlsym (dlhandle, "rfio_serrno"); pops->serror_r = (char* (*) (char*, size_t)) dlsym(dlhandle, "rfio_serror_r"); pops->access = (int (*) (const char *, int)) dlsym (dlhandle, "rfio_access"); pops->chmod = (int (*) (const char *, mode_t)) dlsym (dlhandle, "rfio_chmod"); pops->close = (int (*) (int)) dlsym (dlhandle, "rfio_close"); pops->closedir = (int (*) (DIR *)) dlsym (dlhandle, "rfio_closedir"); pops->lseek = (off_t (*) (int, off_t, int)) dlsym (dlhandle, "rfio_lseek"); pops->lseek64 = (off64_t (*) (int, off64_t, int)) dlsym (dlhandle, "rfio_lseek64"); pops->lstat = (int (*) (const char *, struct stat *)) dlsym (dlhandle, "rfio_lstat"); pops->lstat64 = (int (*) (const char *, struct stat64 *)) dlsym (dlhandle, "rfio_lstat64"); pops->mkdir = (int (*) (const char *, mode_t)) dlsym (dlhandle, "rfio_mkdir"); pops->open = (int (*) (const char *, int, ...)) dlsym (dlhandle, "rfio_open"); pops->opendir = (DIR * (*) (const char *)) dlsym (dlhandle, "rfio_opendir"); pops->read = (ssize_t (*) (int, void *, size_t)) dlsym (dlhandle, "rfio_read"); pops->readdir = (struct dirent * (*) (DIR *)) dlsym (dlhandle, "rfio_readdir"); pops->readdir64 = (struct dirent64 * (*) (DIR *)) dlsym (dlhandle, "rfio_readdir64"); pops->rename = (int (*) (const char *, const char *)) dlsym (dlhandle, "rfio_rename"); pops->rmdir = (int (*) (const char *)) dlsym (dlhandle, "rfio_rmdir"); pops->setfilchg = (ssize_t (*) (int, const void *, size_t)) dlsym (dlhandle, "rfio_HsmIf_FirstWrite"); pops->stat = (int (*) (const char *, struct stat *)) dlsym (dlhandle, "rfio_stat"); pops->stat64 = (int (*) (const char *, struct stat64 *)) dlsym (dlhandle, "rfio_stat64"); pops->unlink = (int (*) (const char *)) dlsym (dlhandle, "rfio_unlink"); pops->write = (ssize_t (*) (int, const void *, size_t)) dlsym (dlhandle, "rfio_write"); } if(tmp_err) gfal2_propagate_prefixed_error(err, tmp_err, __func__); return pops; } struct rfio_proto_ops * (*gfal_rfio_internal_loader)(GError** err)= &gfal_rfio_internal_loader_base; gfal2-v2.23.0/src/plugins/rfio/gfal_rfio_plugin_layer.h000066400000000000000000000040121465240014500230370ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include extern struct rfio_proto_ops * (*gfal_rfio_internal_loader)(GError** err); struct rfio_proto_ops { int (*geterror)(); int (*access)(const char *, int); int (*chmod)(const char *, mode_t); int (*close)(int); int (*closedir)(DIR *); char* (*serror_r)(char* buff, size_t buff_size); off_t (*lseek)(int, off_t, int); #if ! defined(linux) || defined(_LARGEFILE64_SOURCE) off64_t (*lseek64)(int, off64_t, int); #endif int (*lstat)(const char *, struct stat *); #if ! defined(linux) || defined(_LARGEFILE64_SOURCE) int (*lstat64)(const char *, struct stat64 *); #endif int (*mkdir)(const char *, mode_t); int (*open)(const char *, int, ...); DIR *(*opendir)(const char *); ssize_t (*read)(int, void *, size_t); struct dirent *(*readdir)(DIR *); #if ! defined(linux) || defined(_LARGEFILE64_SOURCE) struct dirent64 *(*readdir64)(DIR *); #endif int (*rename)(const char *, const char *); int (*rmdir)(const char *); ssize_t (*setfilchg)(int, const void *, size_t); int (*stat)(const char *, struct stat *); #if ! defined(linux) || defined(_LARGEFILE64_SOURCE) int (*stat64)(const char *, struct stat64 *); #endif int (*unlink)(const char *); ssize_t (*write)(int, const void *, size_t); }; gfal2-v2.23.0/src/plugins/rfio/gfal_rfio_plugin_main.c000066400000000000000000000074131465240014500226520ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include "gfal_rfio_plugin_layer.h" #include "gfal_rfio_plugin_main.h" #include "gfal_rfio_plugin_bindings.h" gboolean gfal_rfio_check_url(plugin_handle, const char* url, plugin_mode mode, GError** err); gboolean gfal_rfio_internal_check_url(gfal_plugin_rfio_handle rh, const char* surl, GError** err); const char* gfal_rfio_getName(); void gfal_rfio_destroyG(plugin_handle handle); int gfal_rfio_regex_compile(regex_t * rex, GError** err){ int ret = regcomp(rex, "^rfio://([:alnum:]|-|/|.|_)+$",REG_ICASE | REG_EXTENDED); g_return_val_err_if_fail(ret==0,-1,err,"[gfal_rfio_internal_check_url] fail to compile regex, report this bug"); return ret; } // RFIO plugin GQuark GQuark gfal2_get_plugin_rfio_quark(){ return g_quark_from_static_string(GFAL2_QUARK_PLUGINS "::RFIO"); } /* * Init function, called before all * */ gfal_plugin_interface gfal_plugin_init(gfal2_context_t handle, GError** err){ gfal_plugin_interface rfio_plugin; GError* tmp_err=NULL; memset(&rfio_plugin,0,sizeof(gfal_plugin_interface)); // clear the plugin gfal_plugin_rfio_handle h = g_new(struct _gfal_plugin_rfio_handle,1); h->handle = handle; h->rf = gfal_rfio_internal_loader(&tmp_err); gfal_rfio_regex_compile(&h->rex, err); rfio_plugin.plugin_data = (void*) h; rfio_plugin.check_plugin_url = &gfal_rfio_check_url; rfio_plugin.getName= &gfal_rfio_getName; rfio_plugin.plugin_delete= &gfal_rfio_destroyG; rfio_plugin.openG= &gfal_rfio_openG; rfio_plugin.closeG= &gfal_rfio_closeG; rfio_plugin.readG= &gfal_rfio_readG; rfio_plugin.writeG= &gfal_rfio_writeG; rfio_plugin.lseekG = &gfal_rfio_lseekG; rfio_plugin.statG = &gfal_rfio_statG; rfio_plugin.lstatG= &gfal_rfio_lstatG; rfio_plugin.opendirG = &gfal_rfio_opendirG; rfio_plugin.readdirG = &gfal_rfio_readdirG; rfio_plugin.closedirG = &gfal_rfio_closedirG; if(tmp_err) gfal2_propagate_prefixed_error(err, tmp_err, __func__); return rfio_plugin; } gboolean gfal_rfio_internal_check_url(gfal_plugin_rfio_handle rh, const char* surl, GError** err){ if(surl == NULL || strnlen(surl, GFAL_URL_MAX_LEN) == GFAL_URL_MAX_LEN){ gfal2_set_error(err, gfal2_get_plugin_rfio_quark(), EINVAL, __func__, "Invalid surl, surl too long or NULL"); return FALSE; } int ret= regexec(&rh->rex,surl,0,NULL,0); return (ret==0)?TRUE:FALSE; } /* * Check the rfio url in the gfal module way * */ gboolean gfal_rfio_check_url(plugin_handle ch, const char* url, plugin_mode mode, GError** err){ int ret; GError* tmp_err=NULL; gfal_plugin_rfio_handle rh = (gfal_plugin_rfio_handle) ch; switch(mode){ case GFAL_PLUGIN_OPEN: case GFAL_PLUGIN_STAT: case GFAL_PLUGIN_LSTAT: case GFAL_PLUGIN_OPENDIR: ret = gfal_rfio_internal_check_url(rh, url, &tmp_err); break; default: ret = FALSE; break; } if(tmp_err) gfal2_propagate_prefixed_error(err, tmp_err, __func__); return ret; } void gfal_rfio_destroyG(plugin_handle handle){ gfal_plugin_rfio_handle h = (gfal_plugin_rfio_handle) handle; g_free(h->rf); regfree(&h->rex); g_free(h); } gfal2-v2.23.0/src/plugins/rfio/gfal_rfio_plugin_main.h000066400000000000000000000022041465240014500226500ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include typedef struct _gfal_plugin_rfio_handle{ gfal2_context_t handle; struct rfio_proto_ops* rf; regex_t rex; }* gfal_plugin_rfio_handle; gboolean gfal_rfio_check_url(plugin_handle, const char* url, plugin_mode mode, GError** err); // LFC plugin GQuark GQuark gfal2_get_plugin_rfio_quark(); gfal_plugin_interface gfal_plugin_init(gfal2_context_t context, GError** err); gfal2-v2.23.0/src/plugins/sftp/000077500000000000000000000000001465240014500162045ustar00rootroot00000000000000gfal2-v2.23.0/src/plugins/sftp/CMakeLists.txt000066400000000000000000000016301465240014500207440ustar00rootroot00000000000000cmake_minimum_required (VERSION 2.6) if (PLUGIN_SFTP) find_package (LIBSSH2 REQUIRED) include_directories(${LIBSSH2_INCLUDE_DIR}) file (GLOB src_sftp "*.c*") add_library (plugin_sftp MODULE ${src_sftp}) target_link_libraries(plugin_sftp gfal2 gfal2_transfer ${LIBSSH2_LIBRARIES} ${UUID_PKG_LIBRARIES} ) set_target_properties(plugin_sftp PROPERTIES CLEAN_DIRECT_OUTPUT 1 OUTPUT_NAME "gfal_plugin_sftp" LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/plugins ) install (TARGETS plugin_sftp LIBRARY DESTINATION ${PLUGIN_INSTALL_DIR} ) install (FILES "README_PLUGIN_SFTP" DESTINATION ${DOC_INSTALL_DIR} ) list (APPEND sftp_conf_file "${CMAKE_SOURCE_DIR}/dist/etc/gfal2.d/sftp_plugin.conf") install(FILES ${sftp_conf_file} DESTINATION ${SYSCONF_INSTALL_DIR}/gfal2.d/ ) endif (PLUGIN_SFTP) gfal2-v2.23.0/src/plugins/sftp/README_PLUGIN_SFTP000066400000000000000000000001671465240014500210020ustar00rootroot00000000000000SFTP Plugin =========== Provides access via SFTP (SSH) Examples -------- gfal-stat sftp://arioch.cern.ch/etc/passwd gfal2-v2.23.0/src/plugins/sftp/gfal_sftp_connection.c000066400000000000000000000273121465240014500225410ustar00rootroot00000000000000/* * Copyright (c) CERN 2016 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "gfal_sftp_connection.h" #include #include #include #include // libssh2_session_handshake introduced with 1.2.8 #if LIBSSH2_VERSION_NUM < 0x010208 # define libssh2_session_handshake libssh2_session_startup #endif void gfal_plugin_sftp_translate_error(const char *func, gfal_sftp_handle_t *handle, GError **err) { char *msg; int len; int ssh_errn = libssh2_session_last_error(handle->ssh_session, &msg, &len, 0); int errn = EIO; switch (ssh_errn) { case LIBSSH2_ERROR_TIMEOUT: case LIBSSH2_ERROR_SOCKET_TIMEOUT: errn = ETIMEDOUT; break; case LIBSSH2_ERROR_SOCKET_DISCONNECT: errn = ECONNRESET; break; case LIBSSH2_ERROR_PROTO: errn = EPROTO; break; #ifdef LIBSSH2_ERROR_AUTHENTICATION_FAILED case LIBSSH2_ERROR_AUTHENTICATION_FAILED: #endif case LIBSSH2_ERROR_PUBLICKEY_UNVERIFIED: case LIBSSH2_ERROR_CHANNEL_REQUEST_DENIED: case LIBSSH2_ERROR_REQUEST_DENIED: errn = EACCES; break; case LIBSSH2_ERROR_METHOD_NOT_SUPPORTED: errn = ENOSYS; break; case LIBSSH2_ERROR_INVAL: errn = EINVAL; break; case LIBSSH2_ERROR_EAGAIN: errn = EAGAIN; break; case LIBSSH2_ERROR_SFTP_PROTOCOL: errn = libssh2_sftp_last_error(handle->sftp_session); break; } gfal2_set_error(err, gfal2_get_plugin_sftp_quark(), errn, func, "%s", msg); } static int gfal_sftp_socket(gfal2_uri *parsed, GError **err) { struct addrinfo hints, *addresses = NULL; memset(&hints, 0, sizeof(hints)); hints.ai_family = PF_UNSPEC; hints.ai_socktype = SOCK_STREAM; hints.ai_flags |= AI_CANONNAME; int rc = getaddrinfo(parsed->host, NULL, &hints, &addresses); if (rc != 0) { gfal2_set_error(err, gfal2_get_plugin_sftp_quark(), EREMOTE, __func__, "Could not resolve host"); return -1; } int port = htons(parsed->port ? parsed->port : 22); struct addrinfo *i; struct sockaddr_in *ipv4 = NULL; struct sockaddr_in6 *ipv6 = NULL; for (i = addresses; i != NULL; i = i->ai_next) { switch (i->ai_family) { case AF_INET: ipv4 = (struct sockaddr_in *) i->ai_addr; ipv4->sin_port = port; break; case AF_INET6: ipv6 = (struct sockaddr_in6 *) i->ai_addr; ipv6->sin6_port = port; break; } } // TODO: Configuration for IPv4 or 6 char addrstr[100] = {0}; struct sockaddr *addr = NULL; if (ipv4) { addr = (struct sockaddr *) ipv4; inet_ntop(AF_INET, &ipv4->sin_addr, addrstr, sizeof(addrstr)); } else if (ipv6) { addr = (struct sockaddr *) ipv6; inet_ntop(AF_INET6, &ipv6->sin6_addr, addrstr, sizeof(addrstr)); } else { freeaddrinfo(addresses); gfal2_set_error(err, gfal2_get_plugin_sftp_quark(), EHOSTUNREACH, __func__, "Could not find an IPv4 or IPv6"); return -1; } gfal2_log(G_LOG_LEVEL_DEBUG, "Connect to %s:%d", addrstr, port); int sock = socket(addr->sa_family, SOCK_STREAM, 0); if (sock < 0) { freeaddrinfo(addresses); gfal2_set_error(err, gfal2_get_plugin_sftp_quark(), errno, __func__, "Could not create the socket"); return -1; } rc = connect(sock, addr, sizeof(*addr)); freeaddrinfo(addresses); if (rc < 0) { close(sock); gfal2_set_error(err, gfal2_get_plugin_sftp_quark(), errno, __func__, "Could not connect"); return -1; } return sock; } static void gfal_sftp_get_authn_params(gfal_sftp_context_t *data, gfal2_uri *parsed, char **user, char **passwd, char **privkey, char **passphrase) { *user = *passwd = *privkey = NULL; char *config_user = gfal2_get_opt_string_with_default(data->gfal2_context, "SFTP PLUGIN", "USER", NULL); char *config_passwd = gfal2_get_opt_string_with_default(data->gfal2_context, "SFTP PLUGIN", "PASSWORD", NULL); // User and password if (parsed->userinfo) { char *separator = strchr(parsed->userinfo, ':'); if (!separator) { *user = g_strdup(parsed->userinfo); } else { *user = g_strndup(parsed->userinfo, separator - parsed->userinfo); *passwd = g_strdup(separator + 1); } } else if (config_user) { *user = g_strdup(config_user); *passwd = g_strdup(config_passwd); } else { struct passwd *me_info = getpwuid(getuid()); if (me_info) { *user = g_strdup(me_info->pw_name); } } // key *privkey = gfal2_get_opt_string_with_default(data->gfal2_context, "SFTP PLUGIN", "PRIVKEY", NULL); *passphrase = gfal2_get_opt_string_with_default(data->gfal2_context, "SFTP PLUGIN", "PASSPHRASE", NULL); if (!*privkey && getenv("HOME")) { *privkey = g_strconcat(getenv("HOME"), "/.ssh/id_rsa", NULL); } g_free(config_user); g_free(config_passwd); } static int gfal_sftp_authn(gfal_sftp_context_t *data, gfal2_uri *parsed, gfal_sftp_handle_t *handle, GError **err) { char *user, *passwd, *privkey, *passphrase; gfal_sftp_get_authn_params(data, parsed, &user, &passwd, &privkey, &passphrase); gfal2_log(G_LOG_LEVEL_DEBUG, "User %s, key %s", user, privkey); const char *userauthlist = libssh2_userauth_list(handle->ssh_session, user, strlen(user)); gfal2_log(G_LOG_LEVEL_DEBUG, "Supported authn methods: %s", userauthlist); const char *auth_method = userauthlist; int authenticated = 0; while (auth_method) { if (strncmp(auth_method, "publickey", 9) == 0) { gfal2_log(G_LOG_LEVEL_DEBUG, "Trying publickey"); if (libssh2_userauth_publickey_fromfile(handle->ssh_session, user, passwd, privkey, passphrase) == 0) { authenticated = 1; } } else if (strncmp(auth_method, "password", 8) == 0) { gfal2_log(G_LOG_LEVEL_DEBUG, "Trying password"); if (libssh2_userauth_password(handle->ssh_session, user, passwd) == 0) { authenticated = 1; } } if (authenticated) { break; } auth_method = strchr(auth_method, ','); if (auth_method) { ++auth_method; } } g_free(user); g_free(passwd); g_free(privkey); if (!authenticated) { gfal2_set_error(err, gfal2_get_plugin_sftp_quark(), EACCES, __func__, "All supported authentication methods failed"); return -1; } return 0; } static gfal_sftp_handle_t *gfal_sftp_new_handle(gfal_sftp_context_t *data, gfal2_uri *parsed, GError **err) { int rc; gfal_sftp_handle_t *handle = g_malloc(sizeof(gfal_sftp_handle_t)); handle->host = g_strdup(parsed->host); handle->port = parsed->port; handle->sock = gfal_sftp_socket(parsed, err); if (handle->sock < 0) { goto get_handle_failure; } gfal2_log(G_LOG_LEVEL_DEBUG, "Connected to remote"); handle->ssh_session = libssh2_session_init(); if (!handle->ssh_session) { gfal2_set_error(err, gfal2_get_plugin_sftp_quark(), ECONNABORTED, __func__, "Failed to get a session"); goto get_handle_failure; } rc = libssh2_session_handshake(handle->ssh_session, handle->sock); if (rc != 0) { goto get_handle_failure_ssh; } rc = gfal_sftp_authn(data, parsed, handle, err); if (rc != 0) { goto get_handle_failure; } gfal2_log(G_LOG_LEVEL_DEBUG, "Authenticated with remote"); handle->sftp_session = libssh2_sftp_init(handle->ssh_session); if (!handle->sftp_session) { goto get_handle_failure_ssh; } gfal2_log(G_LOG_LEVEL_DEBUG, "SFTP initialized"); libssh2_session_set_blocking(handle->ssh_session, 1); return handle; get_handle_failure_ssh: gfal_plugin_sftp_translate_error(__func__, handle, err); get_handle_failure: g_free(handle); return NULL; } static void gfal_sftp_destroy_handle(gfal_sftp_handle_t *handle, gpointer user_data) { close(handle->sock); libssh2_sftp_shutdown(handle->sftp_session); libssh2_session_disconnect(handle->ssh_session, ""); libssh2_session_free(handle->ssh_session); g_free(handle); } gfal_sftp_handle_t *gfal_sftp_connect(gfal_sftp_context_t *context, const char *url, GError **err) { gfal2_uri *parsed = gfal2_parse_uri(url, err); if (!parsed) { return NULL; } gfal_sftp_handle_t *handle = gfal_sftp_cache_pop(context->cache, parsed->host, parsed->port); if (!handle) { gfal2_log(G_LOG_LEVEL_DEBUG, "Creating new SFTP handle"); handle = gfal_sftp_new_handle(context, parsed, err); } else { #if LIBSSH2_VERSION_NUM >= 0x010205 int seconds = 10; gfal2_log(G_LOG_LEVEL_DEBUG, "Reusing SFTP handle from cache for %s:%d", handle->host, handle->port); int rc = libssh2_keepalive_send(handle->ssh_session, &seconds); if (rc < 0) { gfal2_log(G_LOG_LEVEL_DEBUG, "Recycled SFTP handle failed to send keepalive. Discard and reconnect"); gfal_sftp_destroy_handle(handle, NULL); handle = gfal_sftp_new_handle(context, parsed, err); } #endif } if (handle) { handle->path = g_strdup(parsed->path); } gfal2_free_uri(parsed); return handle; } void gfal_sftp_release(gfal_sftp_context_t *context, gfal_sftp_handle_t *handle) { gfal2_log(G_LOG_LEVEL_DEBUG, "Pushing SFTP handle into cache for %s:%d", handle->host, handle->port); gfal_sftp_cache_push(context->cache, handle); } static void gfal_sftpdestroy_key(gpointer p) { g_string_free((GString*)p, TRUE); } GHashTable *gfal_sftp_cache_new() { return g_hash_table_new_full((GHashFunc) g_string_hash, (GEqualFunc) g_string_equal, gfal_sftpdestroy_key, NULL); } gfal_sftp_handle_t *gfal_sftp_cache_pop(GHashTable *cache, const char *host, int port) { GString *key = g_string_new(NULL); g_string_printf(key, "%s:%d", host, port); GSList *list = (GSList*)g_hash_table_lookup(cache, key); if (!list) { g_string_free(key, TRUE); return NULL; } gfal_sftp_handle_t *handle = (gfal_sftp_handle_t*)list->data; list = g_slist_delete_link(list, list); // g_hash_table_insert acquires ownership of key g_hash_table_insert(cache, key, list); return handle; } void gfal_sftp_cache_push(GHashTable *cache, gfal_sftp_handle_t *handle) { GString *key = g_string_new(NULL); g_string_printf(key, "%s:%d", handle->host, handle->port); GSList *list = (GSList*)g_hash_table_lookup(cache, key); list = g_slist_prepend(list, handle); // g_hash_table_insert acquires ownership of key g_hash_table_insert(cache, key, list); } void gfal_sftp_destroy_cache_entry(gpointer key, gpointer value, gpointer user_data) { g_slist_foreach((GSList*)value, (GFunc)gfal_sftp_destroy_handle, NULL); g_slist_free((GSList*)value); } void gfal_sftp_cache_destroy(GHashTable *cache) { g_hash_table_foreach(cache, gfal_sftp_destroy_cache_entry, NULL); g_hash_table_destroy(cache); } gfal2-v2.23.0/src/plugins/sftp/gfal_sftp_connection.h000066400000000000000000000055131465240014500225450ustar00rootroot00000000000000/* * Copyright (c) CERN 2016 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #ifndef GFAL_SFTP_CONNECTION_H #define GFAL_SFTP_CONNECTION_H #include "gfal_sftp_plugin.h" /// Wraps a connection plus a session to a remote SSH server struct gfal_sftp_handle_s { int sock; LIBSSH2_SESSION *ssh_session; LIBSSH2_SFTP *sftp_session; const char *host; int port; const char *path; }; typedef struct gfal_sftp_handle_s gfal_sftp_handle_t; /// SSH session cache struct gfal_sftp_handle_cache_s { GHashTable *caches; }; typedef struct gfal_sftp_handle_cache_s gfal_sftp_handle_cache_t; /// Plugin internal data struct gfal_sftp_context_s { gfal2_context_t gfal2_context; GHashTable *cache; }; typedef struct gfal_sftp_context_s gfal_sftp_context_t; /// Translates an SSH error to a GError /// @param func The function that caused the error /// @param handle The handle used when the error happened. The error will be extracted from here. /// @param[out] err This GError will be filled up with the error message and code void gfal_plugin_sftp_translate_error(const char *func, gfal_sftp_handle_t *handle, GError **err); /// Returns a new handle wrapping a connection to the remote endpoint /// @param context The SFTP context /// @param url Full URL (sftp://host:port/path) to which to connect /// @param[out] err Any error will be put here /// @return NULL on error gfal_sftp_handle_t *gfal_sftp_connect(gfal_sftp_context_t *context, const char *url, GError **err); /// Releases a handle /// @param context The SFTP context /// @param handle The handle we are done with void gfal_sftp_release(gfal_sftp_context_t *context, gfal_sftp_handle_t *handle); /// Creates a new GHashTable that models a connection cache GHashTable *gfal_sftp_cache_new(); /// Gets a handle from the cache /// @param cache An initialized cache /// @param host The remote host /// @param port The remote port /// @return NULL if there is no cached entry gfal_sftp_handle_t *gfal_sftp_cache_pop(GHashTable *cache, const char *host, int port); /// Puts a handle back to the cache /// @param handle The handle to release void gfal_sftp_cache_push(GHashTable *cache, gfal_sftp_handle_t *handle); /// Frees memory and closes connections void gfal_sftp_cache_destroy(GHashTable *cache); #endif // GFAL_SFTP_CONNECTION_H gfal2-v2.23.0/src/plugins/sftp/gfal_sftp_dir.c000066400000000000000000000052601465240014500211560ustar00rootroot00000000000000/* * Copyright (c) CERN 2016 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "gfal_sftp_plugin.h" #include "gfal_sftp_connection.h" struct gfal_sftp_dir_s { gfal_sftp_handle_t *sftp_handle; LIBSSH2_SFTP_HANDLE *dir_handle; struct dirent dent; }; typedef struct gfal_sftp_dir_s gfal_sftp_dir_t; gfal_file_handle gfal_sftp_opendir(plugin_handle plugin_data, const char *url, GError **err) { gfal_sftp_context_t *data = (gfal_sftp_context_t*)plugin_data; gfal_sftp_handle_t *sftp_handle = gfal_sftp_connect(data, url, err); if (!sftp_handle) { return NULL; } gfal_sftp_dir_t *dir = g_malloc(sizeof(gfal_sftp_dir_t)); dir->sftp_handle = sftp_handle; dir->dir_handle = libssh2_sftp_opendir(sftp_handle->sftp_session, sftp_handle->path); if (!dir->dir_handle) { gfal_plugin_sftp_translate_error(__func__, sftp_handle, err); g_free(dir); gfal_sftp_release(data, sftp_handle); return NULL; } return gfal_file_handle_new2(gfal_sftp_plugin_get_name(), dir, NULL, url); } int gfal_sftp_closedir(plugin_handle plugin_data, gfal_file_handle dir_desc, GError **err) { gfal_sftp_context_t *data = (gfal_sftp_context_t*)plugin_data; gfal_sftp_dir_t *dir = gfal_file_handle_get_fdesc(dir_desc); libssh2_sftp_closedir(dir->dir_handle); gfal_sftp_release(data, dir->sftp_handle); g_free(dir); gfal_file_handle_delete(dir_desc); return 0; } struct dirent *gfal_sftp_readdirpp(plugin_handle plugin_data, gfal_file_handle dir_desc, struct stat *st, GError **err) { gfal_sftp_dir_t *dir = gfal_file_handle_get_fdesc(dir_desc); LIBSSH2_SFTP_ATTRIBUTES attrs; int rc = libssh2_sftp_readdir(dir->dir_handle, dir->dent.d_name, sizeof(dir->dent.d_name), &attrs); if (rc < 0) { gfal_plugin_sftp_translate_error(__func__, dir->sftp_handle, err); return NULL; } if (rc == 0) { return NULL; } gfal_sftp_fill_stat(st, &attrs); return &dir->dent; } struct dirent *gfal_sftp_readdir(plugin_handle plugin_data, gfal_file_handle dir_desc, GError **err) { struct stat st; return gfal_sftp_readdirpp(plugin_data, dir_desc, &st, err); } gfal2-v2.23.0/src/plugins/sftp/gfal_sftp_io.c000066400000000000000000000113321465240014500210040ustar00rootroot00000000000000/* * Copyright (c) CERN 2016 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "gfal_sftp_plugin.h" #include "gfal_sftp_connection.h" // These methods were introduced with version 1.0 #if LIBSSH2_VERSION_MAJOR < 1 # define libssh2_sftp_tell64 libssh2_sftp_tell # define libssh2_sftp_seek64 libssh2_sftp_seek #endif struct gfal_sftp_file_s { gfal_sftp_handle_t *sftp_handle; LIBSSH2_SFTP_HANDLE *file_handle; }; typedef struct gfal_sftp_file_s gfal_sftp_file_t; static unsigned long gfal_sftp_std2ssh2_open_flags(int flag) { unsigned long ssh2_flags = 0; if (flag & O_RDONLY || flag & O_RDWR) { ssh2_flags |= LIBSSH2_FXF_READ; } if (flag & O_WRONLY || flag & O_RDWR) { ssh2_flags |= LIBSSH2_FXF_WRITE; } if (flag & O_APPEND) { ssh2_flags |= LIBSSH2_FXF_APPEND; } if (flag & O_TRUNC) { ssh2_flags |= LIBSSH2_FXF_TRUNC; } if (flag & O_CREAT) { ssh2_flags |= LIBSSH2_FXF_CREAT; } if (flag & O_EXCL) { ssh2_flags |= LIBSSH2_FXF_EXCL; } return ssh2_flags; } gfal_file_handle gfal_sftp_open(plugin_handle plugin_data, const char *url, int flag, mode_t mode, GError **err) { gfal_sftp_context_t *data = (gfal_sftp_context_t*)plugin_data; gfal_sftp_handle_t *sftp_handle = gfal_sftp_connect(data, url, err); if (!sftp_handle) { return NULL; } gfal_sftp_file_t *fd = g_malloc(sizeof(gfal_sftp_file_t)); fd->sftp_handle = sftp_handle; fd->file_handle = libssh2_sftp_open(sftp_handle->sftp_session, sftp_handle->path, gfal_sftp_std2ssh2_open_flags(flag), mode); if (!fd->file_handle) { gfal_plugin_sftp_translate_error(__func__, sftp_handle, err); g_free(fd); gfal_sftp_release(data, sftp_handle); return NULL; } return gfal_file_handle_new2(gfal_sftp_plugin_get_name(), fd, NULL, url); } int gfal_sftp_close(plugin_handle plugin_data, gfal_file_handle fd, GError **err) { gfal_sftp_context_t *data = (gfal_sftp_context_t*)plugin_data; gfal_sftp_file_t *ssh_fd = gfal_file_handle_get_fdesc(fd); libssh2_sftp_close(ssh_fd->file_handle); gfal_sftp_release(data, ssh_fd->sftp_handle); g_free(ssh_fd); gfal_file_handle_delete(fd); return 0; } ssize_t gfal_sftp_read(plugin_handle plugin_data, gfal_file_handle fd, void *buff, size_t count, GError **err) { gfal_sftp_file_t *ssh_fd = gfal_file_handle_get_fdesc(fd); ssize_t read = 0; // libssh2 may need to read in chunks char *buffer = (char*)buff; do { ssize_t rc = libssh2_sftp_read(ssh_fd->file_handle, buffer + read, count - read); if (rc < 0) { gfal_plugin_sftp_translate_error(__func__, ssh_fd->sftp_handle, err); return rc; } else if (rc == 0) { break; } read += rc; } while (read < count); return read; } ssize_t gfal_sftp_write(plugin_handle plugin_data, gfal_file_handle fd, const void *buff, size_t count, GError **err) { gfal_sftp_file_t *ssh_fd = gfal_file_handle_get_fdesc(fd); ssize_t rc = libssh2_sftp_write(ssh_fd->file_handle, buff, count); if (rc < 0) { gfal_plugin_sftp_translate_error(__func__, ssh_fd->sftp_handle, err); } // normally, one would expect to return the value from libssh2_sftp_write, but as it happens it may be // shorter, but still the data is cached on the pipe. // See https://www.libssh2.org/libssh2_sftp_write.html return count; } off_t gfal_sftp_seek(plugin_handle plugin_data, gfal_file_handle fd, off_t offset, int whence, GError **err) { gfal_sftp_file_t *ssh_fd = gfal_file_handle_get_fdesc(fd); off_t absolute = 0; LIBSSH2_SFTP_ATTRIBUTES attrs; switch (whence) { case SEEK_SET: absolute = offset; break; case SEEK_CUR: absolute = libssh2_sftp_tell64(ssh_fd->file_handle) + offset; break; case SEEK_END: if (libssh2_sftp_fstat(ssh_fd->file_handle, &attrs) < 0) { gfal_plugin_sftp_translate_error(__func__, ssh_fd->sftp_handle, err); return -1; } absolute = attrs.filesize + offset; } libssh2_sftp_seek64(ssh_fd->file_handle, absolute); return absolute; } gfal2-v2.23.0/src/plugins/sftp/gfal_sftp_metadata.c000066400000000000000000000152301465240014500221560ustar00rootroot00000000000000/* * Copyright (c) CERN 2016 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include "gfal_sftp_plugin.h" #include "gfal_sftp_connection.h" void gfal_sftp_fill_stat(struct stat *st, LIBSSH2_SFTP_ATTRIBUTES *attrs) { if (attrs->flags & LIBSSH2_SFTP_ATTR_SIZE) { st->st_size = attrs->filesize; } if (attrs->flags & LIBSSH2_SFTP_ATTR_UIDGID) { st->st_uid = attrs->uid; st->st_gid = attrs->gid; } if (attrs->flags & LIBSSH2_SFTP_ATTR_PERMISSIONS) { st->st_mode = attrs->permissions; } if (attrs->flags & LIBSSH2_SFTP_ATTR_ACMODTIME) { st->st_atime = attrs->atime; st->st_mtime = attrs->mtime; } } int gfal_sftp_stat(plugin_handle plugin_data, const char *url, struct stat *buf, GError **err) { gfal_sftp_context_t *data = (gfal_sftp_context_t*)plugin_data; gfal_sftp_handle_t *sftp_handle = gfal_sftp_connect(data, url, err); if (!sftp_handle) { return -1; } LIBSSH2_SFTP_ATTRIBUTES attrs; int rc = libssh2_sftp_stat(sftp_handle->sftp_session, sftp_handle->path, &attrs); if (rc < 0) { gfal_plugin_sftp_translate_error(__func__, sftp_handle, err); } else { gfal_sftp_fill_stat(buf, &attrs); } gfal_sftp_release(data, sftp_handle); return rc; } int gfal_sftp_unlink(plugin_handle plugin_data, const char *url, GError **err) { gfal_sftp_context_t *data = (gfal_sftp_context_t*)plugin_data; gfal_sftp_handle_t *sftp_handle = gfal_sftp_connect(data, url, err); if (!sftp_handle) { return -1; } int rc = libssh2_sftp_unlink(sftp_handle->sftp_session, sftp_handle->path); if (rc < 0) { gfal_plugin_sftp_translate_error(__func__, sftp_handle, err); } gfal_sftp_release(data, sftp_handle); return rc; } int gfal_sftp_rename(plugin_handle plugin_data, const char *oldurl, const char *urlnew, GError **err) { gfal_sftp_context_t *data = (gfal_sftp_context_t*)plugin_data; gfal_sftp_handle_t *sftp_handle = gfal_sftp_connect(data, oldurl, err); if (!sftp_handle) { return -1; } int rc = -1; gfal2_uri *new_parsed = gfal2_parse_uri(urlnew, err); if (new_parsed) { rc = libssh2_sftp_rename(sftp_handle->sftp_session, sftp_handle->path, new_parsed->path); if (rc < 0) { gfal_plugin_sftp_translate_error(__func__, sftp_handle, err); if ((*err)->code == 4) { (*err)->code = EISDIR; } } } gfal2_free_uri(new_parsed); gfal_sftp_release(data, sftp_handle); return rc; } int gfal_sftp_mkdir(plugin_handle plugin_data, const char *url, mode_t mode, gboolean rec_flag, GError **err) { gfal_sftp_context_t *data = (gfal_sftp_context_t*)plugin_data; gfal_sftp_handle_t *sftp_handle = gfal_sftp_connect(data, url, err); if (!sftp_handle) { return -1; } int rc = libssh2_sftp_mkdir(sftp_handle->sftp_session, sftp_handle->path, mode); if (rc < 0) { gfal_plugin_sftp_translate_error(__func__, sftp_handle, err); if ((*err)->code == 4) { (*err)->code = EEXIST; } } gfal_sftp_release(data, sftp_handle); return rc; } int gfal_sftp_rmdir(plugin_handle plugin_data, const char *url, GError **err) { LIBSSH2_SFTP_ATTRIBUTES attrs; gfal_sftp_context_t *data = (gfal_sftp_context_t*)plugin_data; gfal_sftp_handle_t *sftp_handle = gfal_sftp_connect(data, url, err); if (!sftp_handle) { return -1; } int rc = libssh2_sftp_rmdir(sftp_handle->sftp_session, sftp_handle->path); if (rc < 0) { gfal_plugin_sftp_translate_error(__func__, sftp_handle, err); // Need to patch some error codes switch ((*err)->code) { case 4: (*err)->code = ENOTEMPTY; break; case 3: (*err)->code = EACCES; break; case 2: // Some times return ENOENT when actually it exists, but it is a file if (libssh2_sftp_stat(sftp_handle->sftp_session, sftp_handle->path, &attrs) == 0) { (*err)->code = ENOTDIR; } break; } } gfal_sftp_release(data, sftp_handle); return rc; } int gfal_sftp_symlink(plugin_handle plugin_data, const char *oldurl, const char *urlnew, GError **err) { gfal_sftp_context_t *data = (gfal_sftp_context_t*)plugin_data; gfal_sftp_handle_t *sftp_handle = gfal_sftp_connect(data, oldurl, err); if (!sftp_handle) { return -1; } int rc = -1; gfal2_uri *new_parsed = gfal2_parse_uri(urlnew, err); if (new_parsed) { rc = libssh2_sftp_symlink(sftp_handle->sftp_session, sftp_handle->path, new_parsed->path); if (rc < 0) { gfal_plugin_sftp_translate_error(__func__, sftp_handle, err); } } gfal2_free_uri(new_parsed); gfal_sftp_release(data, sftp_handle); return rc; } ssize_t gfal_sftp_readlink(plugin_handle plugin_data, const char *url, char *buff, size_t buffsiz, GError **err) { gfal_sftp_context_t *data = (gfal_sftp_context_t*)plugin_data; gfal_sftp_handle_t *sftp_handle = gfal_sftp_connect(data, url, err); if (!sftp_handle) { return -1; } int rc = libssh2_sftp_readlink(sftp_handle->sftp_session, sftp_handle->path, buff, buffsiz); if (rc < 0) { gfal_plugin_sftp_translate_error(__func__, sftp_handle, err); } gfal_sftp_release(data, sftp_handle); return rc; } int gfal_sftp_chmod(plugin_handle plugin_data, const char * url, mode_t mode, GError** err) { gfal_sftp_context_t *data = (gfal_sftp_context_t*)plugin_data; gfal_sftp_handle_t *sftp_handle = gfal_sftp_connect(data, url, err); if (!sftp_handle) { return -1; } LIBSSH2_SFTP_ATTRIBUTES attrs = {0}; attrs.flags = LIBSSH2_SFTP_ATTR_PERMISSIONS; attrs.permissions = mode; int rc = libssh2_sftp_stat_ex(sftp_handle->sftp_session, sftp_handle->path, strlen(sftp_handle->path), LIBSSH2_SFTP_SETSTAT, &attrs); if (rc < 0) { gfal_plugin_sftp_translate_error(__func__, sftp_handle, err); } gfal_sftp_release(data, sftp_handle); return rc; } gfal2-v2.23.0/src/plugins/sftp/gfal_sftp_plugin.c000066400000000000000000000061171465240014500217000ustar00rootroot00000000000000/* * Copyright (c) CERN 2016 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "gfal_sftp_plugin.h" GQuark gfal2_get_plugin_sftp_quark() { return g_quark_from_static_string(GFAL2_QUARK_PLUGINS "::SFTP"); } static gboolean is_sftp_uri(const char *src) { return strncmp(src, "sftp:", 5) == 0; } const char *gfal_sftp_plugin_get_name() { return GFAL2_PLUGIN_VERSIONED("sftp", VERSION); } static gboolean gfal_sftp_check_url(plugin_handle handle, const char *url, plugin_mode mode, GError **err) { g_return_val_err_if_fail(url != NULL, EINVAL, err, "[gfal_sftp_check_url] Invalid url "); switch (mode) { case GFAL_PLUGIN_STAT: case GFAL_PLUGIN_LSTAT: case GFAL_PLUGIN_OPENDIR: case GFAL_PLUGIN_RENAME: case GFAL_PLUGIN_UNLINK: case GFAL_PLUGIN_SYMLINK: case GFAL_PLUGIN_READLINK: case GFAL_PLUGIN_MKDIR: case GFAL_PLUGIN_RMDIR: case GFAL_PLUGIN_CHMOD: case GFAL_PLUGIN_OPEN: return is_sftp_uri(url); default: return FALSE; } } static void gfal_plugin_sftp_delete(plugin_handle plugin_data) { gfal_sftp_context_t *data = (gfal_sftp_context_t*)plugin_data; gfal_sftp_cache_destroy(data->cache); free(data); } gfal_plugin_interface gfal_plugin_init(gfal2_context_t context, GError **err) { gfal_plugin_interface sftp_plugin; memset(&sftp_plugin, 0, sizeof(gfal_plugin_interface)); gfal_sftp_context_t *data = g_malloc(sizeof(gfal_sftp_context_t)); data->gfal2_context = context; data->cache = gfal_sftp_cache_new(); sftp_plugin.plugin_data = data; sftp_plugin.plugin_delete = gfal_plugin_sftp_delete; sftp_plugin.check_plugin_url = &gfal_sftp_check_url; sftp_plugin.getName = &gfal_sftp_plugin_get_name; sftp_plugin.statG = &gfal_sftp_stat; sftp_plugin.lstatG = &gfal_sftp_stat; sftp_plugin.opendirG = gfal_sftp_opendir; sftp_plugin.readdirG = gfal_sftp_readdir; sftp_plugin.readdirppG = gfal_sftp_readdirpp; sftp_plugin.closedirG = gfal_sftp_closedir; sftp_plugin.renameG = &gfal_sftp_rename; sftp_plugin.unlinkG = &gfal_sftp_unlink; sftp_plugin.mkdirpG = &gfal_sftp_mkdir; sftp_plugin.rmdirG = &gfal_sftp_rmdir; sftp_plugin.symlinkG = &gfal_sftp_symlink; sftp_plugin.readlinkG = &gfal_sftp_readlink; sftp_plugin.chmodG = &gfal_sftp_chmod; sftp_plugin.openG = gfal_sftp_open; sftp_plugin.closeG = gfal_sftp_close; sftp_plugin.readG = gfal_sftp_read; sftp_plugin.writeG = gfal_sftp_write; sftp_plugin.lseekG = gfal_sftp_seek; return sftp_plugin; } gfal2-v2.23.0/src/plugins/sftp/gfal_sftp_plugin.h000066400000000000000000000054321465240014500217040ustar00rootroot00000000000000/* * Copyright (c) CERN 2016 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #ifndef GFAL_SFTP_PLUGIN_H #define GFAL_SFTP_PLUGIN_H #include #include #include #include "gfal_sftp_connection.h" GQuark gfal2_get_plugin_sftp_quark(); // Get the plugin name const char *gfal_sftp_plugin_get_name(); // Metadata operations void gfal_sftp_fill_stat(struct stat *st, LIBSSH2_SFTP_ATTRIBUTES *attrs); int gfal_sftp_stat(plugin_handle plugin_data, const char *url, struct stat *buf, GError **err); int gfal_sftp_unlink(plugin_handle plugin_data, const char *url, GError **err); int gfal_sftp_rename(plugin_handle plugin_data, const char *oldurl, const char *urlnew, GError **err); int gfal_sftp_mkdir(plugin_handle plugin_data, const char *url, mode_t mode, gboolean rec_flag, GError **err); int gfal_sftp_rmdir(plugin_handle plugin_data, const char *url, GError **err); int gfal_sftp_symlink(plugin_handle plugin_data, const char *oldurl, const char *urlnew, GError **err); ssize_t gfal_sftp_readlink(plugin_handle plugin_data, const char *url, char *buff, size_t buffsiz, GError **err); int gfal_sftp_chmod(plugin_handle plugin_data, const char * url, mode_t mode, GError** err); // Directory operations gfal_file_handle gfal_sftp_opendir(plugin_handle plugin_data, const char *url, GError **err); int gfal_sftp_closedir(plugin_handle plugin_data, gfal_file_handle dir_desc, GError **err); struct dirent *gfal_sftp_readdirpp(plugin_handle plugin_data, gfal_file_handle dir_desc, struct stat *st, GError **err); struct dirent *gfal_sftp_readdir(plugin_handle plugin_data, gfal_file_handle dir_desc, GError **err); // IO operations gfal_file_handle gfal_sftp_open(plugin_handle plugin_data, const char *url, int flag, mode_t mode, GError **err); ssize_t gfal_sftp_read(plugin_handle plugin_data, gfal_file_handle fd, void *buff, size_t count, GError **err); ssize_t gfal_sftp_write(plugin_handle plugin_data, gfal_file_handle fd, const void *buff, size_t count, GError **err); int gfal_sftp_close(plugin_handle plugin_data, gfal_file_handle fd, GError **err); off_t gfal_sftp_seek(plugin_handle plugin_data, gfal_file_handle fd, off_t offset, int whence, GError **err); #endif // GFAL_SFTP_PLUGIN_H gfal2-v2.23.0/src/plugins/srm/000077500000000000000000000000001465240014500160315ustar00rootroot00000000000000gfal2-v2.23.0/src/plugins/srm/CMakeLists.txt000066400000000000000000000024501465240014500205720ustar00rootroot00000000000000cmake_minimum_required (VERSION 2.6) if (PLUGIN_SRM) file (GLOB src_srm "*.c*") find_package (SRM_IFCE REQUIRED) find_package (Globus_COMMON) find_package (Globus_GSSAPI_GSI REQUIRED) find_package (Globus_GSS_ASSIST REQUIRED) add_definitions (${SRM_IFCE_CFLAGS} ${GLOBUS_GSSAPI_GSI_CFLAGS}) include_directories (${SRM_IFCE_INCLUDE_DIR} ${GLOBUS_GSSAPI_GSI_INCLUDE_DIRS}) add_library(plugin_srm MODULE ${src_srm}) target_link_libraries(plugin_srm gfal2 gfal2_transfer ${SRM_IFCE_LIBRARIES} ${GLOBUS_COMMON_LIBRARIES} ${GLOBUS_GSSAPI_GSI_LIBRARIES} ${GLOBUS_GSS_ASSIST_LIBRARIES} ) set_target_properties(plugin_srm PROPERTIES CLEAN_DIRECT_OUTPUT 1 OUTPUT_NAME "gfal_plugin_srm" LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/plugins) install(TARGETS plugin_srm LIBRARY DESTINATION ${PLUGIN_INSTALL_DIR}) install(FILES "README_PLUGIN_SRM" DESTINATION ${DOC_INSTALL_DIR}) # install srm configuration files list (APPEND srm_conf_file "${CMAKE_SOURCE_DIR}/dist/etc/gfal2.d/srm_plugin.conf") install(FILES ${srm_conf_file} DESTINATION ${SYSCONF_INSTALL_DIR}/gfal2.d/) endif (PLUGIN_SRM) gfal2-v2.23.0/src/plugins/srm/README_PLUGIN_SRM000066400000000000000000000004621465240014500205120ustar00rootroot00000000000000 gfal2 srm plugin : - features : * third party transfer copy * all the posix related call excepted rename - warnings : * SRM uses SOAP, it inherits of the advantages and the inconvenients of this technology. SRM is portable, but can be slow for little files/ entry or for big directory listing gfal2-v2.23.0/src/plugins/srm/gfal_srm.c000066400000000000000000000237731465240014500200030ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include "gfal_srm.h" #include "gfal_srm_namespace.h" #include "gfal_srm_bringonline.h" #include "gfal_srm_archive.h" #include "gfal_srm_internal_layer.h" #include "gfal_srm_opendir.h" #include "gfal_srm_open.h" #include "gfal_srm_copy.h" #include "gfal_srm_url_check.h" #include "gfal_srm_internal_ls.h" #include #include /* * Set up globus (see LCGUTIL-429) */ __attribute__((constructor)) static void globus_setup(void) { if (!getenv("GLOBUS_THREAD_MODEL")) globus_thread_set_model("pthread"); globus_module_activate(GLOBUS_GSI_GSS_ASSIST_MODULE); globus_module_activate(GLOBUS_GSI_GSSAPI_MODULE); } /* * * list of the turls supported protocols */ static char *srm_turls_sup_protocols_default[] = {"rfio", "gsidcap", "dcap", "kdcap", "gsiftp", NULL}; GQuark gfal2_get_plugin_srm_quark() { return g_quark_from_static_string(GFAL2_QUARK_PLUGINS "::SRM"); } /* * list of protocols supporting third party transfer */ char *srm_turls_thirdparty_protocols_default[] = {"gsiftp", NULL}; char **srm_get_turls_sup_protocol(gfal2_context_t context) { gsize len; return gfal2_get_opt_string_list_with_default(context, srm_config_group, srm_config_turl_protocols, &len, srm_turls_sup_protocols_default); } char **srm_get_3rdparty_turls_sup_protocol(gfal2_context_t context) { gsize len; return gfal2_get_opt_string_list_with_default(context, srm_config_group, srm_config_3rd_party_turl_protocols, &len, srm_turls_thirdparty_protocols_default); } /* * * srm plugin id */ const char *gfal_srm_getName() { return GFAL2_PLUGIN_VERSIONED("srm", VERSION); } int gfal_checker_compile(gfal_srmv2_opt *opts, GError **err) { int ret = regcomp(&opts->rexurl, "^srm://([:alnum:]|-|/|.|_)+$", REG_ICASE | REG_EXTENDED); g_return_val_err_if_fail(ret == 0, -1, err, "[gfal_surl_checker_] fail to compile regex for srm checking, report this bug"); ret = regcomp(&(opts->rex_full), "^srm://([:alnum:]|-|/|.|_)+:[0-9]+/([:alnum:]|-|/|.|_)+?SFN=", REG_ICASE | REG_EXTENDED); g_return_val_err_if_fail(ret == 0, -1, err, "[gfal_surl_checker_] fail to compile regex for the full SURL srm checking, report this bug"); return ret; } /* * parse a surl to check the validity */ int gfal_surl_checker(plugin_handle ch, const char *surl, GError **err) { gfal_srmv2_opt *opts = (gfal_srmv2_opt *) ch; if (surl == NULL || strnlen(surl, GFAL_URL_MAX_LEN) == GFAL_URL_MAX_LEN) { gfal2_set_error(err, gfal2_get_plugin_srm_quark(), EINVAL, __func__, "Invalid surl, surl too long or NULL"); return -1; } return regexec(&opts->rexurl, surl, 0, NULL, 0); } /* * convenience func for a group of surls */ gboolean gfal_srm_surl_group_checker(gfal_srmv2_opt *opts, char **surls, GError **err) { GError *tmp_err = NULL; if (surls == NULL) { gfal2_set_error(err, gfal2_get_plugin_srm_quark(), EINVAL, __func__, "Invalid argument surls "); return FALSE; } while (*surls != NULL) { if (gfal_surl_checker(opts, *surls, &tmp_err) != 0) { gfal2_propagate_prefixed_error(err, tmp_err, __func__); return FALSE; } surls++; } return TRUE; } /* * url checker for the srm module, surl part * * */ static gboolean gfal_srm_check_url(plugin_handle handle, const char *url, plugin_mode mode, GError **err) { switch (mode) { case GFAL_PLUGIN_ACCESS: case GFAL_PLUGIN_MKDIR: case GFAL_PLUGIN_STAT: case GFAL_PLUGIN_LSTAT: case GFAL_PLUGIN_RMDIR: case GFAL_PLUGIN_OPENDIR: case GFAL_PLUGIN_OPEN: case GFAL_PLUGIN_CHMOD: case GFAL_PLUGIN_UNLINK: case GFAL_PLUGIN_GETXATTR: case GFAL_PLUGIN_LISTXATTR: case GFAL_PLUGIN_CHECKSUM: case GFAL_PLUGIN_MKDIR_REC: case GFAL_PLUGIN_BRING_ONLINE: case GFAL_PLUGIN_RENAME: case GFAL_PLUGIN_ARCHIVE: return (gfal_surl_checker(handle, url, err) == 0); default: return FALSE; } } /* * destroyer function, call when the module is unload * */ void gfal_srm_destroyG(plugin_handle ch) { gfal_srmv2_opt *opts = (gfal_srmv2_opt *) ch; regfree(&opts->rexurl); regfree(&opts->rex_full); g_static_rec_mutex_free(&opts->srm_context_mutex); srm_context_free(opts->srm_context); gsimplecache_delete(opts->cache); free(opts); } static void srm_internal_copy_stat(gpointer origin, gpointer copy) { memcpy(copy, origin, sizeof(struct extended_stat)); } /* * Init an opts struct with the default parameters * */ void gfal_srm_opt_initG(gfal_srmv2_opt *opts, gfal2_context_t handle) { memset(opts, 0, sizeof(gfal_srmv2_opt)); gfal_checker_compile(opts, NULL); opts->srm_proto_type = PROTO_SRMv2; opts->handle = handle; opts->cache = gsimplecache_new(5000, &srm_internal_copy_stat, sizeof(struct extended_stat)); g_static_rec_mutex_init(&opts->srm_context_mutex); } /* * Init function, called before all * */ gfal_plugin_interface gfal_plugin_init(gfal2_context_t handle, GError **err) { gfal_plugin_interface srm_plugin; memset(&srm_plugin, 0, sizeof(gfal_plugin_interface)); // clear the plugin gfal_srmv2_opt *opts = g_new0(struct _gfal_srmv2_opt, 1); // define the srmv2 option struct and clear it gfal_srm_opt_initG(opts, handle); srm_plugin.plugin_data = (void *) opts; srm_plugin.check_plugin_url = &gfal_srm_check_url; srm_plugin.plugin_delete = &gfal_srm_destroyG; srm_plugin.accessG = &gfal_srm_accessG; srm_plugin.mkdirpG = &gfal_srm_mkdirG; srm_plugin.statG = &gfal_srm_statG; srm_plugin.lstatG = &gfal_srm_statG; // no management for symlink in srm protocol/srm-ifce, just map to stat srm_plugin.rmdirG = &gfal_srm_rmdirG; srm_plugin.opendirG = &gfal_srm_opendirG; srm_plugin.readdirG = &gfal_srm_readdirG; srm_plugin.readdirppG = &gfal_srm_readdirppG; srm_plugin.closedirG = &gfal_srm_closedirG; srm_plugin.getName = &gfal_srm_getName; srm_plugin.openG = &gfal_srm_openG; srm_plugin.closeG = &gfal_srm_closeG; srm_plugin.readG = &gfal_srm_readG; srm_plugin.preadG = &gfal_srm_preadG; srm_plugin.writeG = &gfal_srm_writeG; srm_plugin.chmodG = &gfal_srm_chmodG; srm_plugin.lseekG = &gfal_srm_lseekG; srm_plugin.unlinkG = &gfal_srm_unlinkG; srm_plugin.getxattrG = &gfal_srm_getxattrG; srm_plugin.listxattrG = &gfal_srm_listxattrG; srm_plugin.checksum_calcG = &gfal_srm_checksumG; srm_plugin.copy_file = &srm_plugin_filecopy; srm_plugin.check_plugin_url_transfer = &plugin_url_check2; srm_plugin.bring_online = &gfal_srmv2_bring_onlineG; srm_plugin.bring_online_v2 = &gfal_srmv2_bring_online_v2G; srm_plugin.bring_online_poll = &gfal_srmv2_bring_online_pollG; srm_plugin.release_file = &gfal_srmv2_release_fileG; srm_plugin.bring_online_list = &gfal_srmv2_bring_online_listG; srm_plugin.bring_online_list_v2 = &gfal_srmv2_bring_online_list_v2G; srm_plugin.bring_online_poll_list = &gfal_srmv2_bring_online_poll_listG; srm_plugin.release_file_list = &gfal_srmv2_release_file_listG; srm_plugin.abort_files = &gfal_srm2_abort_filesG; srm_plugin.renameG = &gfal_srm_renameG; srm_plugin.unlink_listG = &gfal_srm_unlink_listG; srm_plugin.archive_poll = &gfal_srm_archive_pollG; srm_plugin.archive_poll_list = &gfal_srm_archive_poll_listG; return srm_plugin; } /* * Construct a key for the cache system from a url and a prefix * */ char *gfal_srm_construct_key(const char *url, const char *prefix, char *buff, const size_t s_buff) { g_strlcpy(buff, prefix, s_buff); g_strlcat(buff, url, s_buff); char *p2 = buff + strlen(prefix) + strlen(GFAL_PREFIX_SRM) + 2; while (*p2 != '\0') { //remove the duplicate // if (*p2 == '/' && *(p2 + 1) == '/') { memmove(p2, p2 + 1, strlen(p2 + 1) + 1); } else p2++; } return buff; } /* * brief accessor for the default storage type definition * */ void gfal_set_default_storageG(gfal_srmv2_opt *opts, enum gfal_srm_proto proto) { opts->srm_proto_type = proto; } int gfal_srm_convert_filestatuses_to_GError(struct srmv2_filestatus *statuses, int n, GError **err) { g_return_val_err_if_fail(statuses && n, -1, err, "[gfal_srm_convert_filestatuses_to_GError] args invalids"); int i; int ret = 0; for (i = 0; i < n; ++i) { if (statuses[i].status != 0) { gfal2_set_error(err, gfal2_get_plugin_srm_quark(), statuses[i].status, __func__, "Error on the surl %s while putdone : %s", statuses[i].surl, statuses[i].explanation); ret = -1; } } return ret; } void gfal_srm_report_error(char *errbuff, GError **err) { int errcode = (errno != ECOMM && errno != 0) ? errno : ECOMM; gfal2_set_error(err, gfal2_get_plugin_srm_quark(), errcode, __func__, "srm-ifce err: %s, err: %s", strerror(errcode), errbuff); } gboolean gfal_srm_check_cancel(gfal2_context_t context, GError **err) { if (gfal2_is_canceled(context)) { gfal2_set_error(err, gfal2_get_plugin_srm_quark(), ECANCELED, __func__, "SRM operation canceled"); return TRUE; } return FALSE; } gfal2-v2.23.0/src/plugins/srm/gfal_srm.h000066400000000000000000000100631465240014500177740ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #pragma once #include #include #include #include #define GFAL_PREFIX_SRM "srm://" #define GFAL_PREFIX_SRM_LEN 6 #define GFAL_ENDPOINT_DEFAULT_PREFIX "httpg://" #define GFAL_ENDPOINT_DEFAULT_PREFIX_LEN 8 #define GFAL_DEFAULT_SERVICE_ENDPOINT_SUFFIX "/srm/managerv2" #define SRM_XATTR_GETURL "user.replicas" #define GFAL_SRM_LSTAT_PREFIX "lstat_" //typedef struct srm_spacemd gfal_spacemd; enum status_type {DEFAULT_STATUS = 0, MD_STATUS, PIN_STATUS}; enum se_type {TYPE_NONE = 0, TYPE_SRM, TYPE_SRMv2, TYPE_SE}; enum gfal_srm_proto {PROTO_SRM=0, PROTO_SRMv2, PROTO_ERROR_UNKNOWN}; // SRM plugin GQuark GQuark gfal2_get_plugin_srm_quark(); /* * the state of the last request -> depreciated * needed to get the response */ typedef struct _gfal_request_state{ char * srmv2_token; struct srmv2_filestatus * srmv2_statuses; struct srmv2_pinfilestatus *srmv2_pinstatuses; enum gfal_srm_proto current_request_proto; char * request_endpoint; gboolean finished; // finished or not int number; // number of files in request } gfal_request_state; /* * @struct structure for the srmv2 */ typedef struct srm_context* srm_context_t; typedef struct _gfal_srmv2_opt{ enum gfal_srm_proto srm_proto_type; // default protocol version regex_t rexurl; regex_t rex_full; gfal2_context_t handle; GSimpleCache* cache; char srm_ifce_error_buffer[GFAL_ERRMSG_LEN]; GStaticRecMutex srm_context_mutex; // Avoid same context being used from more than one thread at the time srm_context_t srm_context; // Used to know if the srm context must be cleaned char x509_ucert[GFAL_URL_MAX_LEN], x509_ukey[GFAL_URL_MAX_LEN]; char endpoint[GFAL_URL_MAX_LEN]; } gfal_srmv2_opt; typedef struct _gfal_srm_result{ char turl[GFAL_URL_MAX_LEN+1]; // turl associated with the request ( main result ) char *reqtoken; // token of the request ( common to all result of a request ) int err_code; // errcode, !=0 if error char err_str[GFAL_ERRMSG_LEN+1]; // explanation about the error } gfal_srm_result; typedef struct _gfal_srm_params{ char** protocols; // optional protocols list for manual set enum gfal_srm_proto proto_version; // default protocol version char * spacetokendesc; // optional spacetokens desc for srmv2 //int desiredpintime; // optional desired default endpoint size_t file_size; }* gfal_srm_params_t; typedef void* srm_request_handle; // default set of protocols for TURL in case of remote IO char** srm_get_turls_sup_protocol(gfal2_context_t context); // default set of protocols for TURL in case of third party transfer char** srm_get_3rdparty_turls_sup_protocol(gfal2_context_t context); const char* gfal_srm_getName(); gfal_plugin_interface gfal_srm_initG(gfal2_context_t handle, GError** err); void gfal_srm_destroyG(plugin_handle ch); void gfal_srm_opt_initG(gfal_srmv2_opt* opts, gfal2_context_t handle); char* gfal_srm_construct_key(const char* url, const char* prefix, char* buff, const size_t s_buff); void gfal_set_default_storageG(gfal_srmv2_opt* opts, enum gfal_srm_proto proto); int gfal_srm_convert_filestatuses_to_GError(struct srmv2_filestatus* statuses, int n, GError** err); gboolean gfal_srm_check_cancel(gfal2_context_t context, GError** err); int gfal_surl_checker(plugin_handle ch, const char* surl, GError** err); gfal2-v2.23.0/src/plugins/srm/gfal_srm_access.c000066400000000000000000000064131465240014500213140ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include "gfal_srm_namespace.h" #include "gfal_srm_internal_layer.h" static int gfal_access_srmv2_internal(srm_context_t context, const char *surl, int mode, GError **err) { GError *tmp_err = NULL; struct srm_checkpermission_input checkpermission_input; struct srmv2_filestatus *resu; int ret = -1; char *tab_surl[] = {(char *) surl, NULL}; checkpermission_input.nbfiles = 1; checkpermission_input.amode = mode; checkpermission_input.surls = tab_surl; ret = gfal_srm_external_call.srm_check_permission(context, &checkpermission_input, &resu); if (ret != 1) { gfal_srm_report_error(context->errbuf, &tmp_err); gfal2_propagate_prefixed_error(err, tmp_err, __func__); return -1; } if (resu[0].status) { if (strnlen(resu[0].surl, GFAL_URL_MAX_LEN) >= GFAL_URL_MAX_LEN || strnlen(resu[0].explanation, GFAL_URL_MAX_LEN) >= GFAL_URL_MAX_LEN) { gfal2_set_error(&tmp_err, gfal2_get_plugin_srm_quark(), resu[0].status, __func__, "Memory corruption in the libgfal_srm_ifce answer, fatal"); } else { gfal2_set_error(&tmp_err, gfal2_get_plugin_srm_quark(), resu[0].status, __func__, "Error %d : %s , file %s: %s", resu[0].status, strerror(resu[0].status), resu[0].surl, resu[0].explanation); } ret = -1; } // resu[0].status == 0 is success else { ret = 0; errno = 0; } gfal_srm_external_call.srm_srmv2_filestatus_delete(resu, 1); G_RETURN_ERR(ret, tmp_err, err); } /* * @brief access method for SRMv2 * check the right for a given SRM url, work only for SRMv2, V1 deprecated. * @param ch the handle of the plugin * @param surl srm url of a given file * @param mode access mode to check * @param err : GError error reprot system * @warning : not safe, surl must be verified */ int gfal_srm_accessG(plugin_handle ch, const char *surl, int mode, GError **err) { g_return_val_err_if_fail(ch && surl, EINVAL, err, "[gfal_srm_accessG] Invalid value handle and/or surl"); GError *tmp_err = NULL; gfal_srmv2_opt *opts = (gfal_srmv2_opt *) ch; int ret = -1; gfal_srm_easy_t easy = gfal_srm_ifce_easy_context(opts, surl, &tmp_err); if (easy != NULL) { ret = gfal_access_srmv2_internal(easy->srm_context, easy->path, mode, &tmp_err); } gfal_srm_ifce_easy_context_release(opts, easy); if (ret != 0) gfal2_propagate_prefixed_error(err, tmp_err, __func__); return ret; } gfal2-v2.23.0/src/plugins/srm/gfal_srm_archive.c000066400000000000000000000072361465240014500215000ustar00rootroot00000000000000/* * Copyright (c) CERN 2020 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "gfal_srm.h" #include "gfal_srm_archive.h" #include "gfal_srm_namespace.h" int gfal_srm_archive_pollG(plugin_handle ch, const char* surl, GError** err) { GError* tmp_err = NULL; char buffer[1024]; int poll_result = 0; int ret = -1; g_return_val_err_if_fail(ch && surl, EINVAL, err, "[gfal_srm_archive_pollG] Invalid value handle and/or surl"); gfal2_log(G_LOG_LEVEL_DEBUG, " gfal_srm_archive_pollG ->"); ret = gfal_srm_status_getxattrG(ch, surl, GFAL_XATTR_STATUS, buffer, sizeof(buffer), &tmp_err); if (ret > 0 && strlen(buffer) && tmp_err == NULL) { gfal2_log(G_LOG_LEVEL_DEBUG, "GFAL_XATTR_STATUS response: %s", buffer); if ((strncmp(buffer, GFAL_XATTR_STATUS_NEARLINE, sizeof(GFAL_XATTR_STATUS_NEARLINE)) == 0) || (strncmp(buffer, GFAL_XATTR_STATUS_NEARLINE_ONLINE, sizeof(GFAL_XATTR_STATUS_NEARLINE_ONLINE)) == 0)) { poll_result = 1; } else { gfal2_set_error(&tmp_err, gfal2_get_plugin_srm_quark(), EAGAIN, __func__, "File %s is not yet archived", surl); } } else if (ret == -1 || tmp_err) { poll_result = -1; if (!tmp_err) { gfal2_set_error(&tmp_err, gfal2_get_plugin_srm_quark(), EINVAL, __func__, "polling failed but error not set by getxattr"); } } gfal2_log(G_LOG_LEVEL_DEBUG, " gfal_srm_archive_pollG <-"); G_RETURN_ERR(poll_result, tmp_err, err); } int gfal_srm_archive_poll_listG(plugin_handle ch, int nbfiles, const char* const* surls, GError** errors) { int error_count = 0; int ontape_count = 0; int ret = -1; int i; if (nbfiles <= 0) { return 1; } if (!(ch && surls)) { for (i = 0 ; i < nbfiles; i++) { gfal2_set_error(&errors[i], gfal2_get_plugin_srm_quark(), EINVAL, __func__, "Invalid value handle and/or surls array"); } return -1; } gfal2_log(G_LOG_LEVEL_DEBUG, " gfal_srm_archive_poll_listG ->"); for (i = 0; i < nbfiles; i++) { if (!surls[i]) { gfal2_set_error(&errors[i], gfal2_get_plugin_srm_quark(), EINVAL, __func__, "Invalid surl value"); error_count++; continue; } ret = gfal_srm_archive_pollG(ch, surls[i], &errors[i]); if (errors[i] && errors[i]->code != EAGAIN) { error_count++; } else if (ret == 1) { ontape_count++; } } gfal2_log(G_LOG_LEVEL_DEBUG, " Archive polling: nbfiles=%d ontape_count=%d error_count=%d", nbfiles, ontape_count, error_count); gfal2_log(G_LOG_LEVEL_DEBUG, " gfal_srm_archive_poll_listG <-"); // All files are on tape: return 1 if (ontape_count == nbfiles) { return 1; } // All files encountered errors: return -1 if (error_count == nbfiles) { return -1; } // Some files are on tape, others encountered errors if (ontape_count + error_count == nbfiles) { return 2; } // Archiving in process: return 0 return 0; } gfal2-v2.23.0/src/plugins/srm/gfal_srm_archive.h000066400000000000000000000015031465240014500214740ustar00rootroot00000000000000/* * Copyright (c) CERN 2020 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #pragma once #include #include "gfal_srm.h" int gfal_srm_archive_pollG(plugin_handle ch, const char* surl, GError** err); int gfal_srm_archive_poll_listG(plugin_handle ch, int nbfiles, const char* const* surls, GError** errors); gfal2-v2.23.0/src/plugins/srm/gfal_srm_bringonline.c000066400000000000000000000371761465240014500223730ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include "gfal_srm.h" #include "gfal_srm_internal_layer.h" #include "gfal_srm_request.h" #include "gfal_srm_url_check.h" // Sadly, this is quite inefficient, but has to be done since there might be duplicated // surls in the initial request (plus, the response order doesn't have to be the same // as when requested) static int gfal_srmv2_bring_online_internal_status_index( int nresponses, struct srm_bringonline_output *output, const char *surl) { int i; for (i = 0; i < nresponses; ++i) { if (gfal2_srm_surl_cmp(output->filestatuses[i].surl, surl) == 0) { return i; } } return -1; } static int gfal_srmv2_bring_online_internal(srm_context_t context, gfal_srmv2_opt *opts, int nbfiles, const char *const *surl, time_t pintime, time_t timeout, char *token, size_t tsize, int async, GError **errors) { struct srm_bringonline_input input; struct srm_bringonline_output output; gfal_srm_params_t params = gfal_srm_params_new(opts); int i; memset(&output, 0, sizeof(output)); srm_set_desired_request_time(context, timeout); input.nbfiles = nbfiles; input.surls = (char **) surl; input.desiredpintime = pintime; input.protocols = gfal_srm_params_get_protocols(params); input.spacetokendesc = gfal_srm_params_get_spacetoken(params); if (input.spacetokendesc) gfal2_log(G_LOG_LEVEL_DEBUG, "Bringonline with spacetoken %s", input.spacetokendesc); int nresponses; if (async) nresponses = gfal_srm_external_call.srm_bring_online_async(context, &input, &output); else nresponses = gfal_srm_external_call.srm_bring_online(context, &input, &output); if (nresponses < 0) { GError *tmp_err = NULL; gfal_srm_report_error(context->errbuf, &tmp_err); for (i = 0; i < nbfiles; ++i) { errors[i] = g_error_copy(tmp_err); } g_error_free(tmp_err); return -1; } if (nresponses != nbfiles) { gfal2_log(G_LOG_LEVEL_DEBUG, "%d files in the request, %d in the response", nbfiles, nresponses); } if (output.token) g_strlcpy(token, output.token, tsize); else token[0] = '\0'; gfal2_log(G_LOG_LEVEL_MESSAGE, "Got BRINGONLINE token %s", token); int nterminal = 0; for (i = 0; i < nbfiles; ++i) { int status_index = gfal_srmv2_bring_online_internal_status_index(nresponses, &output, surl[i]); if (status_index >= 0) { switch (output.filestatuses[status_index].status) { case 0: ++nterminal; break; case EAGAIN: break; default: gfal2_set_error(&errors[i], gfal2_get_plugin_srm_quark(), output.filestatuses[status_index].status, __func__, "error on the bring online request: %s ", output.filestatuses[status_index].explanation); ++nterminal; break; } } else { gfal2_set_error(&errors[i], gfal2_get_plugin_srm_quark(), EPROTO, __func__, "missing surl on the response: %s", surl[i]); ++nterminal; } } gfal_srm_external_call.srm_srmv2_pinfilestatus_delete(output.filestatuses, nresponses); gfal_srm_external_call.srm_srm2__TReturnStatus_delete(output.retstatus); free(output.token); gfal_srm_params_free(params); // Return 1 if all are already terminal return nterminal == nbfiles; } int gfal_srmv2_bring_onlineG(plugin_handle ch, const char *surl, time_t pintime, time_t timeout, char *token, size_t tsize, int async, GError **err) { g_return_val_err_if_fail(ch && surl && token, EINVAL, err, "[gfal_srmv2_bring_onlineG] Invalid value handle and/or surl"); GError *tmp_err = NULL; gfal_srmv2_opt *opts = (gfal_srmv2_opt *) ch; int ret = -1; gfal_srm_easy_t easy = gfal_srm_ifce_easy_context(opts, surl, &tmp_err); if (easy != NULL) { ret = gfal_srmv2_bring_online_internal(easy->srm_context, opts, 1, (const char *const *) &easy->path, pintime, timeout, token, tsize, async, &tmp_err); } gfal_srm_ifce_easy_context_release(opts, easy); if (tmp_err) { gfal2_propagate_prefixed_error(err, tmp_err, __func__); return -1; } return ret; } int gfal_srmv2_bring_online_v2G(plugin_handle ch, const char *surl, const char *metadata, time_t pintime, time_t timeout, char *token, size_t tsize, int async, GError **err) { return gfal_srmv2_bring_onlineG(ch, surl, pintime, timeout, token, tsize, async, err); } int gfal_srmv2_bring_online_listG(plugin_handle ch, int nbfiles, const char *const *surls, time_t pintime, time_t timeout, char *token, size_t tsize, int async, GError **errors) { int i; GError *tmp_err = NULL; gfal_srmv2_opt *opts = (gfal_srmv2_opt *) ch; gfal_srm_easy_t easy = gfal_srm_ifce_easy_context(opts, *surls, &tmp_err); if (easy == NULL) { for (i = 0; i < nbfiles; ++i) { errors[i] = g_error_copy(tmp_err); } g_error_free(tmp_err); return -1; } char *decoded[nbfiles]; for (i = 0; i < nbfiles; ++i) { decoded[i] = gfal2_srm_get_decoded_path(surls[i]); } int ret = gfal_srmv2_bring_online_internal(easy->srm_context, opts, nbfiles, (const char *const *) decoded, pintime, timeout, token, tsize, async, errors); gfal_srm_ifce_easy_context_release(opts, easy); for (i = 0; i < nbfiles; ++i) { g_free(decoded[i]); } return ret; } int gfal_srmv2_bring_online_list_v2G(plugin_handle ch, int nbfiles, const char *const *surls, const char *const *metadata, time_t pintime, time_t timeout, char *token, size_t tsize, int async, GError **errors) { return gfal_srmv2_bring_online_listG(ch, nbfiles, surls, pintime, timeout, token, tsize, async, errors); } static int gfal_srmv2_bring_online_poll_internal(srm_context_t context, int nbfiles, const char *const *surls, const char *token, GError **errors) { struct srm_bringonline_input input; struct srm_bringonline_output output; int i; memset(&input, 0, sizeof(input)); memset(&output, 0, sizeof(output)); input.nbfiles = nbfiles; input.surls = (char **) surls; output.token = (char *) token; int nresponses = gfal_srm_external_call.srm_bring_online_status(context, &input, &output); if (nresponses < 0) { GError *tmp_err = NULL; gfal_srm_report_error(context->errbuf, &tmp_err); for (i = 0; i < nbfiles; ++i) { errors[i] = g_error_copy(tmp_err); } g_error_free(tmp_err); return -1; } int nterminal = 0; for (i = 0; i < nbfiles; ++i) { int status_index = gfal_srmv2_bring_online_internal_status_index(nresponses, &output, surls[i]); if (status_index >= 0) { switch (output.filestatuses[status_index].status) { case 0: ++nterminal; break; case EAGAIN: gfal2_set_error(&errors[i], gfal2_get_plugin_srm_quark(), EAGAIN, __func__, "still queued: %s ", output.filestatuses[i].explanation); break; default: gfal2_set_error(&errors[i], gfal2_get_plugin_srm_quark(), output.filestatuses[status_index].status, __func__, "error on the bring online request: %s ", output.filestatuses[status_index].explanation); ++nterminal; break; } } else { gfal2_set_error(&errors[i], gfal2_get_plugin_srm_quark(), EPROTO, __func__, "missing surl on the response: %s", surls[i]); ++nterminal; } } gfal_srm_external_call.srm_srmv2_pinfilestatus_delete(output.filestatuses, nresponses); gfal_srm_external_call.srm_srm2__TReturnStatus_delete(output.retstatus); // Return will be 1 if all files are terminal return nterminal == nbfiles; } int gfal_srmv2_bring_online_pollG(plugin_handle ch, const char *surl, const char *token, GError **err) { g_return_val_err_if_fail(ch && surl && token, EINVAL, err, "[gfal_srmv2_bring_online_pollG] Invalid value handle and, surl or token"); GError *tmp_err = NULL; gfal_srmv2_opt *opts = (gfal_srmv2_opt *) ch; int ret = -1; gfal_srm_easy_t easy = gfal_srm_ifce_easy_context(opts, surl, &tmp_err); if (easy != NULL) { ret = gfal_srmv2_bring_online_poll_internal(easy->srm_context, 1, (const char *const *) &easy->path, token, &tmp_err); } gfal_srm_ifce_easy_context_release(opts, easy); if (tmp_err) { gfal2_propagate_prefixed_error(err, tmp_err, __func__); return -1; } return ret; } int gfal_srmv2_bring_online_poll_listG(plugin_handle ch, int nbfiles, const char *const *surls, const char *token, GError **errors) { int i; GError *tmp_err = NULL; gfal_srmv2_opt *opts = (gfal_srmv2_opt *) ch; gfal_srm_easy_t easy = gfal_srm_ifce_easy_context(opts, *surls, &tmp_err); if (easy == NULL) { for (i = 0; i < nbfiles; ++i) { errors[i] = g_error_copy(tmp_err); } g_error_free(tmp_err); return -1; } char *decoded[nbfiles]; for (i = 0; i < nbfiles; ++i) { decoded[i] = gfal2_srm_get_decoded_path(surls[i]); } int ret = gfal_srmv2_bring_online_poll_internal(easy->srm_context, nbfiles, (const char *const *) decoded, token, errors); gfal_srm_ifce_easy_context_release(opts, easy); for (i = 0; i < nbfiles; ++i) { g_free(decoded[i]); } return ret; } static int gfal_srmv2_release_file_internal(srm_context_t context, gfal_srmv2_opt *opts, int nbfiles, const char *const *surl, const char *token, GError **errors) { struct srm_releasefiles_input input; struct srmv2_filestatus *statuses; int i; if (token) gfal2_log(G_LOG_LEVEL_MESSAGE, "Released file with token %s", token); else gfal2_log(G_LOG_LEVEL_MESSAGE, "Released file without token"); // Perform input.nbfiles = nbfiles; input.reqtoken = NULL; input.surls = (char **) surl; if (token) input.reqtoken = (char *) token; int ret = gfal_srm_external_call.srm_release_files(context, &input, &statuses); if (ret < 0) { GError *tmp_err = NULL; gfal_srm_report_error(context->errbuf, &tmp_err); for (i = 0; i < nbfiles; ++i) { errors[i] = g_error_copy(tmp_err); } g_error_free(tmp_err); return -1; } for (i = 0; i < nbfiles; ++i) { if (statuses[i].status != 0) { gfal2_set_error(&errors[i], gfal2_get_plugin_srm_quark(), statuses[i].status, __func__, "error on the release request : %s ", statuses[0].explanation); } } gfal_srm_external_call.srm_srmv2_filestatus_delete(statuses, 1); return 0; } int gfal_srmv2_release_fileG(plugin_handle ch, const char *surl, const char *token, GError **err) { g_return_val_err_if_fail(ch && surl && token, EINVAL, err, "[gfal_srmv2_release_fileG] Invalid value handle, surl or token"); GError *tmp_err = NULL; gfal_srmv2_opt *opts = (gfal_srmv2_opt *) ch; int ret = -1; gfal_srm_easy_t easy = gfal_srm_ifce_easy_context(opts, surl, &tmp_err); if (easy != NULL) { ret = gfal_srmv2_release_file_internal(easy->srm_context, opts, 1, (const char *const *) &easy->path, token, &tmp_err); } gfal_srm_ifce_easy_context_release(opts, easy); if (tmp_err) { gfal2_propagate_prefixed_error(err, tmp_err, __func__); return -1; } return ret; } int gfal_srmv2_release_file_listG(plugin_handle ch, int nbfiles, const char *const *surls, const char *token, GError **errors) { int i; GError *tmp_err = NULL; gfal_srmv2_opt *opts = (gfal_srmv2_opt *) ch; gfal_srm_easy_t easy = gfal_srm_ifce_easy_context(opts, surls[0], &tmp_err); if (easy == NULL) { for (i = 0; i < nbfiles; ++i) { errors[i] = g_error_copy(tmp_err); } g_error_free(tmp_err); return -1; } char *decoded[nbfiles]; for (i = 0; i < nbfiles; ++i) { decoded[i] = gfal2_srm_get_decoded_path(surls[i]); } int ret = gfal_srmv2_release_file_internal(easy->srm_context, opts, nbfiles, (const char *const *) decoded, token, errors); gfal_srm_ifce_easy_context_release(opts, easy); for (i = 0; i < nbfiles; ++i) { g_free(decoded[i]); } return ret; } static int gfal_srmv2_abort_files_internal(srm_context_t context, gfal_srmv2_opt *opts, int nbfiles, const char *const *surl, const char *token, GError **errors) { struct srm_abort_files_input input; struct srmv2_filestatus *statuses; GError *tmp_err = NULL; int i; if (token) gfal2_log(G_LOG_LEVEL_MESSAGE, "Abort file with token %s", token); else gfal2_log(G_LOG_LEVEL_MESSAGE, "Abort file without token"); // Perform input.nbfiles = nbfiles; input.reqtoken = NULL; input.surls = (char **) surl; if (token) input.reqtoken = (char *) token; int ret = gfal_srm_external_call.srm_abort_files(context, &input, &statuses); if (ret < 0) { gfal_srm_report_error(context->errbuf, &tmp_err); for (i = 0; i < nbfiles; ++i) { errors[i] = g_error_copy(tmp_err); } g_error_free(tmp_err); } else { ret = 0; for (i = 0; i < nbfiles; ++i) { if (statuses[i].status != 0) { gfal2_set_error(&(errors[i]), gfal2_get_plugin_srm_quark(), statuses[i].status, __func__, "error on the abort request : %s ", statuses[i].explanation); ret -= 1; } } gfal_srm_external_call.srm_srmv2_filestatus_delete(statuses, 1); } return ret; } int gfal_srm2_abort_filesG(plugin_handle ch, int nbfiles, const char *const *surls, const char *token, GError **errors) { int i; GError *tmp_err = NULL; gfal_srmv2_opt *opts = (gfal_srmv2_opt *) ch; gfal_srm_easy_t easy = gfal_srm_ifce_easy_context(opts, surls[0], &tmp_err); if (easy == NULL) { for (i = 0; i < nbfiles; ++i) { errors[i] = g_error_copy(tmp_err); } g_error_free(tmp_err); return -1; } char *decoded[nbfiles]; for (i = 0; i < nbfiles; ++i) { decoded[i] = gfal2_srm_get_decoded_path(surls[i]); } int ret = gfal_srmv2_abort_files_internal(easy->srm_context, opts, nbfiles, (const char *const *) decoded, token, errors); gfal_srm_ifce_easy_context_release(opts, easy); for (i = 0; i < nbfiles; ++i) { g_free(decoded[i]); } return ret; } gfal2-v2.23.0/src/plugins/srm/gfal_srm_bringonline.h000066400000000000000000000045721465240014500223720ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #pragma once #include #include "gfal_srm.h" int gfal_srmv2_bring_onlineG(plugin_handle ch, const char *surl, time_t pintime, time_t timeout, char *token, size_t tsize, int async, GError **err); int gfal_srmv2_bring_online_v2G(plugin_handle ch, const char *surl, const char *metadata, time_t pintime, time_t timeout, char *token, size_t tsize, int async, GError **err); int gfal_srmv2_bring_online_pollG(plugin_handle ch, const char *surl, const char *token, GError **err); int gfal_srmv2_release_fileG(plugin_handle ch, const char *surl, const char *token, GError **err); int gfal_srmv2_bring_online_listG(plugin_handle ch, int nbfiles, const char *const *surls, time_t pintime, time_t timeout, char *token, size_t tsize, int async, GError **err); int gfal_srmv2_bring_online_list_v2G(plugin_handle ch, int nbfiles, const char *const *surls, const char *const *metadata, time_t pintime, time_t timeout, char *token, size_t tsize, int async, GError **err); int gfal_srmv2_bring_online_poll_listG(plugin_handle ch, int nbfiles, const char *const *surls, const char *token, GError **err); int gfal_srmv2_release_file_listG(plugin_handle ch, int nbfiles, const char *const *surls, const char *token, GError **err); int gfal_srm2_abort_filesG(plugin_handle ch, int nbfiles, const char *const *surls, const char *token, GError **err); gfal2-v2.23.0/src/plugins/srm/gfal_srm_checksum.c000066400000000000000000000146431465240014500216610ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "gfal_srm_internal_layer.h" #include "gfal_srm_getput.h" #include "gfal_srm_url_check.h" static int gfal_checksumG_srmv2_internal(srm_context_t context, const char *surl, char *buf_checksum, size_t s_checksum, char *buf_chktype, size_t s_chktype, GError **err) { g_return_val_err_if_fail(context && surl && buf_checksum && buf_chktype, -1, err, "[gfal_checksumG_srmv2_internal] Invalid input parameters : endpoint, surl, checksum, checksum_type"); GError *tmp_err = NULL; struct srm_ls_input input; struct srm_ls_output output; struct srmv2_mdfilestatus *srmv2_mdstatuses = NULL; int ret = -1; char *tab_surl[] = {(char *) surl, NULL}; input.nbfiles = 1; input.surls = tab_surl; input.numlevels = 0; input.offset = 0; input.count = 0; ret = gfal_srm_external_call.srm_ls(context, &input, &output); if (ret >= 0) { srmv2_mdstatuses = output.statuses; if (srmv2_mdstatuses->status != 0) { gfal2_set_error(&tmp_err, gfal2_get_plugin_srm_quark(), srmv2_mdstatuses->status, __func__, "Error reported from srm_ifce : %d %s", srmv2_mdstatuses->status, srmv2_mdstatuses->explanation); ret = -1; } else { if (srmv2_mdstatuses->checksum && srmv2_mdstatuses->checksumtype) { g_strlcpy(buf_checksum, srmv2_mdstatuses->checksum, s_checksum); g_strlcpy(buf_chktype, srmv2_mdstatuses->checksumtype, s_chktype); } else { if (s_checksum > 0) *buf_checksum = '\0'; if (s_chktype > 0) *buf_chktype = '\0'; } ret = 0; } } else { gfal_srm_report_error(context->errbuf, &tmp_err); ret = -1; } gfal_srm_external_call.srm_srmv2_mdfilestatus_delete(srmv2_mdstatuses, 1); gfal_srm_external_call.srm_srm2__TReturnStatus_delete(output.retstatus); G_RETURN_ERR(ret, tmp_err, err); } /* * get checksum from a remote SRM URL * * */ static int gfal_srm_cheksumG_internal(plugin_handle ch, const char *surl, char *buf_checksum, size_t s_checksum, char *buf_chktype, size_t s_chktype, GError **err) { g_return_val_err_if_fail(ch && surl, EINVAL, err, "[gfal_srm_cheksumG_internal] Invalid value handle, surl or buffers"); GError *tmp_err = NULL; gfal_srmv2_opt *opts = (gfal_srmv2_opt *) ch; int ret = -1; gfal_srm_easy_t easy = gfal_srm_ifce_easy_context(opts, surl, &tmp_err); if (easy != NULL) { ret = gfal_checksumG_srmv2_internal(easy->srm_context, easy->path, buf_checksum, s_checksum, buf_chktype, s_chktype, &tmp_err); } gfal_srm_ifce_easy_context_release(opts, easy); if (ret != 0) gfal2_propagate_prefixed_error(err, tmp_err, __func__); return ret; } int gfal_srm_checksumG_fallback(plugin_handle handle, const char *url, const char *check_type, char *checksum_buffer, size_t buffer_length, off_t start_offset, size_t data_length, gboolean turl_fallback, GError **err) { gfal2_log(G_LOG_LEVEL_DEBUG, " [gfal_srm_checksumG] ->"); gfal2_log(G_LOG_LEVEL_DEBUG, "[gfal_srm_checksumG] try to get checksum %s for %s", check_type, url); char buffer_type[GFAL_URL_MAX_LEN] = {0}; GError *tmp_err = NULL; gfal_srmv2_opt *opts = (gfal_srmv2_opt *) handle; const gboolean srm_url = srm_check_url(url); int res = -1; // try SRM checksum only if full file checksum is requested if (srm_url && start_offset == 0 && data_length == 0) { res = gfal_srm_cheksumG_internal(handle, url, checksum_buffer, buffer_length, buffer_type, GFAL_URL_MAX_LEN, &tmp_err); } // Make sure the returned type matches the requested one if (res == 0) { gfal2_log(G_LOG_LEVEL_DEBUG, "registered checksum type %s", buffer_type); if (strncasecmp(check_type, buffer_type, GFAL_URL_MAX_LEN) != 0) { // does not match the correct type // this can be because checksum is populated on DPM server, cause the first gsiftp checksum calculation res = -1; // cancel result } } // If we got no error, but neither a valid checksum, // fallback into the turl if (res != 0 && !tmp_err && turl_fallback) { gfal2_log(G_LOG_LEVEL_DEBUG, "\t\tNo valid SRM checksum, fallback to the TURL checksum"); char buff_turl[GFAL_URL_MAX_LEN]; char *res_turl; if (srm_url) { // SRM URL do TURL resolution if ((res = gfal_srm_getTURL_checksum(handle, url, buff_turl, GFAL_URL_MAX_LEN, &tmp_err)) >= 0) { res_turl = buff_turl; } else { res = -1; } } else { // native protocol -> act like this res_turl = (char *) url; res = 0; } if (res == 0) { gfal2_log(G_LOG_LEVEL_DEBUG, "\t\t\tExecute checksum on turl %s", res_turl); res = gfal2_checksum(opts->handle, res_turl, check_type, 0, 0, checksum_buffer, buffer_length, &tmp_err); } } // If no fallback, then return an empty value else if (!turl_fallback && (tmp_err || res != 0)) { res = 0; memset(checksum_buffer, '\0', buffer_length); } G_RETURN_ERR(res, tmp_err, err); } // Wrapper for gfal_srm_checksumG_fallback so it matches // the expected gfal2 signature int gfal_srm_checksumG(plugin_handle handle, const char *url, const char *check_type, char *checksum_buffer, size_t buffer_length, off_t start_offset, size_t data_length, GError **err) { return gfal_srm_checksumG_fallback(handle, url, check_type, checksum_buffer, buffer_length, start_offset, data_length, TRUE, err); } gfal2-v2.23.0/src/plugins/srm/gfal_srm_chmod.c000066400000000000000000000061601465240014500211440ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include "gfal_srm.h" #include "gfal_srm_internal_layer.h" #include "gfal_srm_namespace.h" #include "gfal_srm_endpoint.h" #include "gfal_srm_internal_ls.h" /* * * convert a mode_t to a TPermissionMode, right dec and mask are used to get the good oct (mode & mask) >> right_dec * WARNING : hard conversion mode, subject to problem if the TPermissionMode declaration begin to change ! */ static TPermissionMode gfal_srmv2_mode_t_to_TPermissionMode(mode_t mode, mode_t mask, mode_t right_dec) { return ((mode & mask) >> right_dec); } /* * Do a translation of a chmod right to a srm right * */ static void gfal_srmv2_configure_set_permission(const char *surl, mode_t mode, struct srm_setpermission_input *perms_input) { memset(perms_input, 0, sizeof(struct srm_setpermission_input)); perms_input->surl = (char *) surl; perms_input->permission_type = SRM_PERMISSION_CHANGE; perms_input->owner_permission = gfal_srmv2_mode_t_to_TPermissionMode(mode, 00700, 6); perms_input->other_permission = gfal_srmv2_mode_t_to_TPermissionMode(mode, 007, 0); } static int gfal_srmv2_chmod_internal(srm_context_t context, const char *path, mode_t mode, GError **err) { g_return_val_err_if_fail(context && path, -1, err, "[gfal_srmv2_chmod_internal] invalid args "); GError *tmp_err = NULL; int ret = 0; struct srm_setpermission_input perms_input; // set the structures datafields gfal_srmv2_configure_set_permission(path, mode, &perms_input); if ((ret = gfal_srm_external_call.srm_setpermission(context, &perms_input)) < 0) { gfal_srm_report_error(context->errbuf, &tmp_err); } else { ret = 0; } G_RETURN_ERR(ret, tmp_err, err); } int gfal_srm_chmodG(plugin_handle ch, const char *path, mode_t mode, GError **err) { g_return_val_err_if_fail(ch && path, EINVAL, err, "[gfal_srm_chmodG] Invalid value handle and/or surl"); GError *tmp_err = NULL; gfal_srmv2_opt *opts = (gfal_srmv2_opt *) ch; int ret = -1; gfal_srm_easy_t easy = gfal_srm_ifce_easy_context(opts, path, &tmp_err); if (easy != NULL) { gfal_srm_cache_stat_remove(ch, path); ret = gfal_srmv2_chmod_internal(easy->srm_context, easy->path, mode, &tmp_err); } gfal_srm_ifce_easy_context_release(opts, easy); if (ret != 0) gfal2_propagate_prefixed_error(err, tmp_err, __func__); return ret; } gfal2-v2.23.0/src/plugins/srm/gfal_srm_copy.c000066400000000000000000000575011465240014500210310ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include "gfal_srm_getput.h" #include "gfal_srm_namespace.h" #include "gfal_srm_url_check.h" #include "gfal_srm_internal_layer.h" #include "gfal_srm_bringonline.h" GQuark srm_domain() { return g_quark_from_static_string("SRM"); } static GQuark gfal2_get_srm_get_quark() { return g_quark_from_static_string("SRM:GET"); } static GQuark gfal2_get_srm_put_quark() { return g_quark_from_static_string("SRM:PUT"); } int srm_plugin_delete_existing_copy(plugin_handle handle, gfalt_params_t params, const char *surl, GError **err) { GError *tmp_err = NULL; int res = 0; const gboolean replace = gfalt_get_replace_existing_file(params, NULL); if (replace) { gfal2_log(G_LOG_LEVEL_DEBUG, "Trying to delete %s", surl); res = gfal_srm_unlinkG(handle, surl, &tmp_err); if (res == 0) { gfal2_log(G_LOG_LEVEL_DEBUG, "%s deleted with success", surl); plugin_trigger_event(params, srm_domain(), GFAL_EVENT_DESTINATION, GFAL_EVENT_OVERWRITE_DESTINATION, "Deleted %s", surl); } else if (tmp_err->code == ENOENT) { gfal2_log(G_LOG_LEVEL_MESSAGE, "%s doesn't exist, carry on", surl); g_clear_error(&tmp_err); tmp_err = NULL; res = 0; } // Workaround for BeStMan, which returns EINVAL instead of ENOENT else if (tmp_err->code == EINVAL) { gfal2_log(G_LOG_LEVEL_MESSAGE, "Got EINVAL removing %s. Assuming ENOENT (for BeStMan storages)", surl); g_clear_error(&tmp_err); tmp_err = NULL; res = 0; } } if (tmp_err) gfal2_propagate_prefixed_error(err, tmp_err, __func__); return res; } // create the parent directory // return 0 if nothing or not requested // return 1 if creation has been done // return < 0 in case of error int srm_plugin_create_parent_copy(plugin_handle handle, gfalt_params_t params, const char *surl, GError **err) { GError *tmp_err = NULL; int res = -1; const gboolean create_parent = gfalt_get_create_parent_dir(params, NULL); if (create_parent) { char *path_dir = g_strdup(surl); char *p = path_dir + strlen(path_dir) - 1; while (*p == '/') { // remote trailing / *p = '\0'; p--; } const unsigned int pref_len = GFAL_PREFIX_SRM_LEN; while (*p != '/' && (path_dir + pref_len) < p) p--; if ((path_dir + pref_len) < p) { *p = '\0'; gfal2_log(G_LOG_LEVEL_DEBUG, " try to create parent dir : %s for %s", path_dir, surl); res = gfal_srm_mkdir_recG(handle, path_dir, 0755, &tmp_err); if (res == 0) gfal2_log(G_LOG_LEVEL_DEBUG, "parent path %s created with success", path_dir); } else { gfal2_set_error(&tmp_err, gfal2_get_plugin_srm_quark(), EINVAL, __func__, "Invalid srm url %s", surl); res = -1; } g_free(path_dir); } else { res = 0; } if (tmp_err) gfal2_propagate_prefixed_error(err, tmp_err, __func__); return res; } static int srm_plugin_prepare_dest_put(plugin_handle handle, gfal2_context_t context, gfalt_params_t params, const char *surl, GError **err) { GError *tmp_err = NULL; int res; res = srm_plugin_delete_existing_copy(handle, params, surl, &tmp_err); if (res == 0) { res = srm_plugin_create_parent_copy(handle, params, surl, &tmp_err); if (res < 0) gfalt_propagate_prefixed_error(err, tmp_err, __func__, GFALT_ERROR_DESTINATION, GFALT_ERROR_PARENT); } else { gfalt_propagate_prefixed_error(err, tmp_err, __func__, GFALT_ERROR_DESTINATION, GFALT_ERROR_OVERWRITE); } return res; } static int srm_resolve_get_turl(plugin_handle handle, gfalt_params_t params, const char *surl, const char *other_surl, char *turl, size_t turl_size, char *token, size_t token_size, GError **err) { GError *tmp_err = NULL; int res = 0; if (srm_check_url(surl)) { gfal2_log(G_LOG_LEVEL_DEBUG, "\t\tGET surl -> turl resolution start"); res = gfal_srm_get_rd3_turl(handle, params, surl, other_surl, turl, turl_size, token, token_size, &tmp_err); if (res >= 0) { gfal2_log(G_LOG_LEVEL_DEBUG, "\t\tGET surl -> turl resolution finished: %s -> %s (%s)", surl, turl, token); plugin_trigger_event(params, gfal2_get_plugin_srm_quark(), GFAL_EVENT_SOURCE, gfal2_get_srm_get_quark(), "Got TURL %s => %s", surl, turl); } } else { g_strlcpy(turl, surl, turl_size); gfal2_log(G_LOG_LEVEL_DEBUG, "\t\tNo SRM resolution needed on %s", surl); token[0] = '\0'; } if (tmp_err) gfalt_propagate_prefixed_error(err, tmp_err, __func__, GFALT_ERROR_SOURCE, "SRM_GET_TURL"); return res; } // = 0 on success // < 0 on failure // > 0 if surl is not an srm endpoint static int srm_resolve_put_turl(plugin_handle handle, gfal2_context_t context, gfalt_params_t params, const char *surl, const char *other_surl, off_t file_size_surl, char *turl, size_t turl_size, char *token, size_t token_size, GError **err) { GError *tmp_err = NULL; int res = 0; if (srm_check_url(surl)) { gfal2_log(G_LOG_LEVEL_DEBUG, "\t\tPUT surl -> turl resolution start "); res = srm_plugin_prepare_dest_put(handle, context, params, surl, &tmp_err); if (res == 0) { res = gfal_srm_put_rd3_turl(handle, params, surl, other_surl, file_size_surl, turl, turl_size, token, token_size, &tmp_err); if (res >= 0) { gfal2_log(G_LOG_LEVEL_DEBUG, "\t\tPUT surl -> turl resolution ended : %s -> %s (%s)", surl, turl, token); plugin_trigger_event(params, gfal2_get_plugin_srm_quark(), GFAL_EVENT_DESTINATION, gfal2_get_srm_put_quark(), "Got TURL %s => %s", surl, turl); } else { gfalt_propagate_prefixed_error(err, tmp_err, __func__, GFALT_ERROR_DESTINATION, "SRM_PUT_TURL"); return res; } } } else { g_strlcpy(turl, surl, turl_size); gfal2_log(G_LOG_LEVEL_DEBUG, "\t\tNo SRM resolution needed on %s", surl); token[0] = '\0'; res = 1; } if (tmp_err != NULL) gfal2_propagate_prefixed_error(err, tmp_err, __func__); return res; } static int srm_get_checksum_config(gfal2_context_t context, gfalt_params_t params, gfalt_checksum_mode_t *mode, char *algorithm, size_t algorithm_size, char *user_checksum, size_t user_checksum_size, GError **err) { *mode = gfalt_get_checksum(params, algorithm, algorithm_size, user_checksum, user_checksum_size, NULL); gboolean allow_empty_source = gfal2_get_opt_boolean(context, "SRM PLUGIN", "ALLOW_EMPTY_SOURCE_CHECKSUM", NULL); if (allow_empty_source) { *mode = *mode & ~GFALT_CHECKSUM_SOURCE; } if (algorithm[0] == '\0') { const char *configured; configured = gfal2_get_opt_string(context, srm_config_group, srm_config_transfer_checksum, err); if (configured != NULL) g_strlcpy(algorithm, configured, algorithm_size); } if (*err == NULL) { gfal2_log(G_LOG_LEVEL_DEBUG, "\t\tChecksum check enabled: %d", *mode); if (*mode) { gfal2_log(G_LOG_LEVEL_DEBUG, "\t\tVerify source checksum: %d", (*mode & GFALT_CHECKSUM_SOURCE)); gfal2_log(G_LOG_LEVEL_DEBUG, "\t\tChecksum algorithm: %s", algorithm); gfal2_log(G_LOG_LEVEL_DEBUG, "\t\tUser defined checksum: %s", user_checksum); } return 0; } else { return -1; } } static int srm_validate_source_checksum(plugin_handle handle, gfal2_context_t context, gfalt_params_t params, const char *src, gfalt_checksum_mode_t checksum_mode, const char *checksum_algorithm, const char *checksum_user, char *checksum_source, size_t checksum_source_size, GError **err) { GError *tmp_err = NULL; plugin_trigger_event(params, srm_domain(), GFAL_EVENT_SOURCE, GFAL_EVENT_CHECKSUM_ENTER, ""); int ret = 0; gboolean turl_fallback = (checksum_mode & GFALT_CHECKSUM_SOURCE); ret = gfal_srm_checksumG_fallback(handle, src, checksum_algorithm, checksum_source, checksum_source_size, 0, 0, turl_fallback, &tmp_err); if (ret != 0) { gfalt_propagate_prefixed_error(err, tmp_err, __func__, GFALT_ERROR_SOURCE, GFALT_ERROR_CHECKSUM); } else if (checksum_mode & GFALT_CHECKSUM_SOURCE && checksum_user[0]) { if (gfal_compare_checksums(checksum_source, checksum_user, checksum_source_size) != 0) { gfalt_set_error(err, gfal2_get_plugin_srm_quark(), EIO, __func__, GFALT_ERROR_SOURCE, GFALT_ERROR_CHECKSUM_MISMATCH, "User defined checksum and source checksum do not match %s != %s", checksum_user, checksum_source); ret = -1; } } plugin_trigger_event(params, srm_domain(), GFAL_EVENT_SOURCE, GFAL_EVENT_CHECKSUM_EXIT, ""); return ret; } static int srm_validate_destination_checksum(plugin_handle handle, gfal2_context_t context, gfalt_params_t params, const char *dst, const char *checksum_algorithm, const char *checksum_user, const char *checksum_source, GError **err) { GError *tmp_err = NULL; plugin_trigger_event(params, srm_domain(), GFAL_EVENT_DESTINATION, GFAL_EVENT_CHECKSUM_ENTER, ""); int ret = 0; char checksum_destination[GFAL_URL_MAX_LEN] = {0}; ret = gfal_srm_checksumG_fallback(handle, dst, checksum_algorithm, checksum_destination, sizeof(checksum_destination), 0, 0, TRUE, &tmp_err); if (ret == 0) { if (checksum_destination[0] != '\0') { if (checksum_source[0] != '\0' && gfal_compare_checksums(checksum_source, checksum_destination, sizeof(checksum_destination)) != 0) { gfalt_set_error(err, gfal2_get_plugin_srm_quark(), EIO, __func__, GFALT_ERROR_TRANSFER, GFALT_ERROR_CHECKSUM_MISMATCH, "Source and destination checksums do not match %s != %s", checksum_source, checksum_destination); ret = -1; } else if (checksum_user[0] != '\0' && gfal_compare_checksums(checksum_user, checksum_destination, sizeof(checksum_destination)) != 0) { gfalt_set_error(err, gfal2_get_plugin_srm_quark(), EIO, __func__, GFALT_ERROR_TRANSFER, GFALT_ERROR_CHECKSUM_MISMATCH, "User defined checksum and destination checksums do not match %s != %s", checksum_user, checksum_destination); ret = -1; } } else { gfalt_set_error(err, gfal2_get_plugin_srm_quark(), EINVAL, __func__, GFALT_ERROR_DESTINATION, GFALT_ERROR_CHECKSUM, "Empty destination checksum"); ret = -1; } } else { gfalt_propagate_prefixed_error(err, tmp_err, __func__, GFALT_ERROR_DESTINATION, GFALT_ERROR_CHECKSUM); } plugin_trigger_event(params, srm_domain(), GFAL_EVENT_DESTINATION, GFAL_EVENT_CHECKSUM_EXIT, ""); return ret; } static void srm_force_unlink(plugin_handle handle, gfal2_context_t context, const char *surl, GError **err) { GError *unlink_err = NULL; gfal_srm_unlinkG(handle, surl, &unlink_err); if (unlink_err != NULL) { if (unlink_err->code != ENOENT) { gfal2_log(G_LOG_LEVEL_WARNING, "Got an error when removing the destination surl: %s", unlink_err->message); } else { gfal2_log(G_LOG_LEVEL_DEBUG, "Destination surl did not exist after abort"); } g_error_free(unlink_err); } else { gfal2_log(G_LOG_LEVEL_MESSAGE, "Successfully removed destination surl after abort: %s", surl); } } static void srm_rollback_put(plugin_handle handle, gfal2_context_t context, gfalt_params_t params, const char *surl, const char *token, gboolean transfer_finished, GError **err) { gfal2_log(G_LOG_LEVEL_MESSAGE, "Rolling back PUT"); GError *abort_error = NULL; // If the transfer finished, or the destination is not an SRM unlink the destination and trigger an event // If unlink fails reports in the event the correspondent errno. // If unlink succeeds or if it fails because the file does not exist, the event reports 0 as the status code. if ((*err && (*err)->code != EEXIST) && (transfer_finished || !srm_check_url(surl))) { int status = 0; if (gfal2_unlink(context, surl, &abort_error)) { if (abort_error->code != ENOENT) { gfal2_log(G_LOG_LEVEL_WARNING, "When trying to clean the destination: %s", abort_error->message); status = abort_error->code; } } else { gfal2_log(G_LOG_LEVEL_INFO, "Destination file removed"); } plugin_trigger_event(params, srm_domain(), GFAL_EVENT_DESTINATION, GFAL_EVENT_CLEANUP, "%d", status); // It may not be there, so be gentle if (abort_error != NULL) g_error_free(abort_error); } // If the transfer is not finished, abort it else if (!transfer_finished && token[0] != '\0') { srm_abort_request_plugin(handle, surl, token, &abort_error); if (abort_error != NULL) { if (*err == NULL) { *err = abort_error; } else { gfal2_log(G_LOG_LEVEL_WARNING, "Got an error when canceling the PUT request: %s", abort_error->message); g_error_free(abort_error); } } // Some endpoints may not remove the file after an abort (i.e. Castor), // so do it ourselves if it is still there (see LCGUTIL-358) srm_force_unlink(handle, context, surl, err); } } static void srm_release_get(plugin_handle handle, const char *surl, const char *token, GError **err) { gfal2_log(G_LOG_LEVEL_MESSAGE, "Rolling back GET"); GError *release_error = NULL; gfal_srmv2_release_fileG(handle, surl, token, &release_error); if (release_error != NULL) { gfal2_log(G_LOG_LEVEL_WARNING, "Got an error when releasing the source file: %s", release_error->message); gfal2_log(G_LOG_LEVEL_WARNING, "It will be ignored!"); } } static int srm_resolve_turls(plugin_handle handle, gfal2_context_t context, gfalt_params_t params, const char *source, char *turl_source, char *token_source, const char *dest, char *turl_destination, char *token_destination, GError **err) { GError *tmp_err = NULL; char buffer[1024]; struct stat stat_source; memset(&stat_source, 0, sizeof(stat_source)); if (gfal2_stat(context, source, &stat_source, &tmp_err) != 0) { stat_source.st_size = 0; gfal2_log(G_LOG_LEVEL_DEBUG, "Fail to stat src SRM url %s to determine file size, try with file_size=0, error %s", source, tmp_err->message); g_clear_error(&tmp_err); tmp_err = NULL; } //check if the source file is online in case the SRM_COPY_FAIL_NEARLINE is set gboolean fail_nearline = gfal2_get_opt_boolean_with_default(context, "SRM PLUGIN", "COPY_FAIL_NEARLINE", FALSE); if (fail_nearline && srm_check_url(source)) { gfal2_log(G_LOG_LEVEL_DEBUG, "Copy-fail-nearline: querying status first"); ssize_t ret = gfal2_getxattr(context, source, GFAL_XATTR_STATUS, buffer, sizeof(buffer), &tmp_err); if (ret > 0 && strlen(buffer) > 0 && tmp_err == NULL) { if (strncmp(buffer, GFAL_XATTR_STATUS_NEARLINE, sizeof(GFAL_XATTR_STATUS_NEARLINE)) == 0) { gfal2_log(G_LOG_LEVEL_DEBUG, "Copy-fail-nearline: The source file is not ONLINE"); gfalt_set_error(&tmp_err, gfal2_get_plugin_srm_quark(), EINVAL, __func__, GFALT_ERROR_SOURCE, "SRM_GET_TURL", "The source file is not ONLINE"); gfal2_propagate_prefixed_error(err, tmp_err, __func__); return -1; } } else { if (tmp_err == NULL) { gfalt_set_error(&tmp_err, gfal2_get_plugin_srm_quark(), EINVAL, __func__, GFALT_ERROR_SOURCE, "SRM_GET_TURL", "Error while checking if the source file is ONLINE"); } gfal2_propagate_prefixed_error(err, tmp_err, __func__); return -1; } } srm_resolve_get_turl(handle, params, source, dest, turl_source, GFAL_URL_MAX_LEN, token_source, GFAL_URL_MAX_LEN, &tmp_err); if (tmp_err != NULL) { gfal2_propagate_prefixed_error(err, tmp_err, __func__); return -1; } srm_resolve_put_turl(handle, context, params, dest, source, stat_source.st_size, turl_destination, GFAL_URL_MAX_LEN, token_destination, GFAL_URL_MAX_LEN, &tmp_err); if (tmp_err != NULL) { gfal2_propagate_prefixed_error(err, tmp_err, __func__); return -1; } return 0; } static int srm_do_transfer(plugin_handle handle, gfal2_context_t context, gfalt_params_t params, const char *destination, const char *token_destination, const char *turl_source, const char *turl_destination, GError **err) { GError *tmp_err = NULL; gfalt_params_t params_turl; params_turl = gfalt_params_handle_copy(params, NULL); // checksum check done here! if (gfalt_set_checksum(params_turl, GFALT_CHECKSUM_NONE, NULL, NULL, err) < 0) { return -1; } if (srm_check_url(destination)) { // srm destination gfalt_set_replace_existing_file(params_turl, FALSE, NULL); gfalt_set_strict_copy_mode(params_turl, TRUE, NULL); } gfalt_copy_file(context, params_turl, turl_source, turl_destination, &tmp_err); if (tmp_err != NULL) // We assume the underlying copy tagged properly gfal2_propagate_prefixed_error(err, tmp_err, __func__); gfalt_params_handle_delete(params_turl, NULL); if (*err == NULL) { plugin_trigger_event(params, srm_domain(), GFAL_EVENT_DESTINATION, GFAL_EVENT_CLOSE_ENTER, "%s", destination); if (srm_check_url(destination)) { if (gfal_srm_putdone(handle, destination, token_destination, &tmp_err) < 0) gfalt_propagate_prefixed_error(err, tmp_err, __func__, GFALT_ERROR_DESTINATION, "SRM_PUTDONE"); } plugin_trigger_event(params, srm_domain(), GFAL_EVENT_DESTINATION, GFAL_EVENT_CLOSE_EXIT, "%s", destination); } return *err == NULL ? 0 : -1; } static int srm_cleanup_copy(plugin_handle handle, gfal2_context_t context, gfalt_params_t params, const char *source, const char *destination, const char *token_source, const char *token_destination, gboolean transfer_finished, GError **err) { if (*err != NULL) { srm_rollback_put(handle, context, params, destination, token_destination, transfer_finished, err); } if (token_source[0] != '\0') { srm_release_get(handle, source, token_source, err); } return 0; } static void castor_gridftp_session_hack(plugin_handle handle, gfal2_context_t context, const char *src, const char *dst) { int src_is_castor = is_castor_endpoint(handle, src); int dst_is_castor = is_castor_endpoint(handle, dst); if (src_is_castor || dst_is_castor) { gfal2_log(G_LOG_LEVEL_MESSAGE, "Found a Castor endpoint, or could not determine version! Disabling GridFTP session reuse and stat on open"); gfal2_set_opt_boolean(context, "GRIDFTP PLUGIN", "SESSION_REUSE", FALSE, NULL); gfal2_set_opt_boolean(context, "GRIDFTP PLUGIN", "STAT_ON_OPEN", FALSE, NULL); } else { gfal2_log(G_LOG_LEVEL_DEBUG, "No Castor endpoint. Honor configuration for SESSION_REUSE"); } } int srm_plugin_filecopy(plugin_handle handle, gfal2_context_t context, gfalt_params_t params, const char *source, const char *dest, GError **err) { GError *nested_error = NULL; char checksum_algorithm[64] = {0}; char checksum_user[GFAL_URL_MAX_LEN] = {0}; char checksum_source[GFAL_URL_MAX_LEN] = {0}; char turl_source[GFAL_URL_MAX_LEN] = {0}; char token_source[GFAL_URL_MAX_LEN] = {0}; char turl_destination[GFAL_URL_MAX_LEN] = {0}; char token_destination[GFAL_URL_MAX_LEN] = {0}; gfalt_checksum_mode_t checksum_mode; gboolean transfer_finished = FALSE; // Check if any of the endpoints is castor // In that case, disable GridFTP session reuse (see LCGUTIL-448) // Reason is: Castor does not allow doing multiple operations within the same connection castor_gridftp_session_hack(handle, context, source, dest); plugin_trigger_event(params, srm_domain(), GFAL_EVENT_NONE, GFAL_EVENT_PREPARE_ENTER, ""); srm_get_checksum_config(context, params, &checksum_mode, checksum_algorithm, sizeof(checksum_algorithm), checksum_user, sizeof(checksum_user), &nested_error); if (nested_error != NULL) goto copy_finalize; // Source checksum validation if (checksum_mode) { srm_validate_source_checksum(handle, context, params, source, checksum_mode, checksum_algorithm, checksum_user, checksum_source, sizeof(checksum_source), &nested_error); if (nested_error != NULL) goto copy_finalize; } // Resolve turls srm_resolve_turls(handle, context, params, source, turl_source, token_source, dest, turl_destination, token_destination, &nested_error); if (nested_error != NULL) goto copy_finalize; plugin_trigger_event(params, srm_domain(), GFAL_EVENT_NONE, GFAL_EVENT_PREPARE_EXIT, ""); if (gfal_srm_check_cancel(context, &nested_error)) goto copy_finalize; // Transfer srm_do_transfer(handle, context, params, dest, token_destination, turl_source, turl_destination, &nested_error); if (nested_error != NULL) goto copy_finalize; transfer_finished = TRUE; // Destination checksum validation if (checksum_mode & GFALT_CHECKSUM_TARGET) { srm_validate_destination_checksum(handle, context, params, dest, checksum_algorithm, checksum_user, checksum_source, &nested_error); } // Cleanup and propagate error if needed copy_finalize: if (nested_error) { gfal2_log(G_LOG_LEVEL_WARNING, "Transfer failed with: %s", nested_error->message); } if (gfalt_get_transfer_cleanup(params, &nested_error)) { srm_cleanup_copy(handle, context, params, source, dest, token_source, token_destination, transfer_finished, &nested_error); } else { gfal2_log(G_LOG_LEVEL_INFO, "Gfal srm copy clean-up disabled"); } if (nested_error != NULL) gfal2_propagate_prefixed_error(err, nested_error, __func__); else *err = NULL; return (*err == NULL) ? 0 : -1; } gfal2-v2.23.0/src/plugins/srm/gfal_srm_copy.h000066400000000000000000000025001465240014500210230ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #pragma once #ifndef _GFAL2_COMMON_SRM_COPY_ #define _GFAL2_COMMON_SRM_COPY_ /** * initialize a file copy from the given source to the given dest with the parameters params * srm implementation of the plugin filecopy * @param handle : srm plugin handle * @param params :tranfer parameters * @param status : transfer status handle * @param src : source url * @param dst : destination url * @param err : error report system */ int srm_plugin_filecopy(plugin_handle handle, gfal2_context_t context, gfalt_params_t params, const char *src, const char *dst, GError **err); #endif gfal2-v2.23.0/src/plugins/srm/gfal_srm_endpoint.c000066400000000000000000000237621465240014500217010ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include #include #include "gfal_srm_url_check.h" #include "gfal_srm_internal_layer.h" static enum gfal_srm_proto gfal_proto_list_prefG[] = {PROTO_SRMv2, PROTO_SRM, PROTO_ERROR_UNKNOWN}; // construct a default service endpoint format, Guessing that the service endpoint follows the default DPM/dCache convention static int gfal_srm_guess_service_endpoint(gfal_srmv2_opt *opts, const char *surl, char *buff_endpoint, size_t s_buff, enum gfal_srm_proto *srm_type, GError **err) { GError *tmp_err = NULL; gfal2_uri *parsed = gfal2_parse_uri(surl, &tmp_err); if (!parsed) { gfal2_propagate_prefixed_error(err, tmp_err, __func__); return -1; } if (parsed->port) { snprintf(buff_endpoint, s_buff, "%s%s:%d%s", GFAL_ENDPOINT_DEFAULT_PREFIX, parsed->host, parsed->port, GFAL_DEFAULT_SERVICE_ENDPOINT_SUFFIX ); } else { snprintf(buff_endpoint, s_buff, "%s%s%s", GFAL_ENDPOINT_DEFAULT_PREFIX, parsed->host, GFAL_DEFAULT_SERVICE_ENDPOINT_SUFFIX ); } *srm_type = opts->srm_proto_type; gfal2_free_uri(parsed); return 0; } /* * return TRUE if a full endpoint is contained in surl else FALSE * */ static gboolean gfal_check_fullendpoint_in_surlG(gfal_srmv2_opt *opts, const char *surl, GError **err) { const int ret = regexec(&(opts->rex_full), surl, 0, NULL, 0); return (ret == 0) ? TRUE : FALSE; } /* * @brief create a full endpoint from a "full-surl" * */ static int gfal_get_fullendpointG(const char *surl, char *buff_endpoint, size_t s_buff, GError **err) { char *sfn = strstr(surl, "?SFN="); g_return_val_err_if_fail(sfn, -1, err, "[gfal_get_fullendpoint] full surl must contain ?SFN= and a valid prefix, fatal error"); // Endpoint length not counting the scheme size_t endpoint_len = (sfn - surl) - GFAL_PREFIX_SRM_LEN; // Endpoint length counting the httpg scheme size_t full_endpoint_len = endpoint_len + GFAL_ENDPOINT_DEFAULT_PREFIX_LEN; // buff_endpoint must have enough space to allocate the full_endpoint_len + '\0' if (s_buff > full_endpoint_len) { memcpy(buff_endpoint, GFAL_ENDPOINT_DEFAULT_PREFIX, GFAL_ENDPOINT_DEFAULT_PREFIX_LEN); g_strlcpy(buff_endpoint + GFAL_ENDPOINT_DEFAULT_PREFIX_LEN, surl + GFAL_PREFIX_SRM_LEN, endpoint_len + 1); return 0; } gfal2_set_error(err, gfal2_get_plugin_srm_quark(), ENOBUFS, __func__, "buffer too small"); return -1; } /* * map a bdii se protocol type to a gfal protocol type */ static enum gfal_srm_proto gfal_convert_proto_from_bdii(const char *se_type_bdii) { enum gfal_srm_proto resu; if (strcmp(se_type_bdii, "srm_v1") == 0) { resu = PROTO_SRM; } else if (strcmp(se_type_bdii, "srm_v2") == 0) { resu = PROTO_SRMv2; } else { resu = PROTO_ERROR_UNKNOWN; } return resu; } /* * select the best protocol choice and the best endpoint choice from a list of protocol and endpoints obtained by the bdii * */ static int gfal_select_best_protocol_and_endpointG(gfal_srmv2_opt *opts, char **tab_se_type, char **tab_endpoint, char *buff_endpoint, size_t s_buff, enum gfal_srm_proto *srm_type, GError **err) { g_return_val_err_if_fail(opts && buff_endpoint && s_buff && srm_type && tab_se_type && tab_endpoint, -1, err, "[gfal_select_best_protocol_and_endpoint] Invalid value"); char **pse = tab_se_type; enum gfal_srm_proto *p_pref = &(opts->srm_proto_type); while (*p_pref != PROTO_ERROR_UNKNOWN) { while (*pse != NULL && *tab_endpoint != NULL) { if (*p_pref == gfal_convert_proto_from_bdii(*pse)) { // test if the response is the actual preferred response g_strlcpy(buff_endpoint, *tab_endpoint, s_buff); *srm_type = *p_pref; return 0; } tab_endpoint++; pse++; } if (p_pref == &(opts->srm_proto_type)) // switch desired proto to the list if the default choice is not in the list p_pref = gfal_proto_list_prefG; else p_pref++; } gfal2_set_error(err, gfal2_get_plugin_srm_quark(), EINVAL, __func__, "cannot obtain a valid protocol from the bdii response, fatal error"); return -2; } /* * get endpoint from the bdii system only * 0 == success * < 0 error * > 0 : bddi disabled * * */ static int gfal_get_endpoint_and_setype_from_bdiiG(gfal_srmv2_opt *opts, const char *surl, char *buff_endpoint, size_t s_buff, enum gfal_srm_proto *srm_type, GError **err) { g_return_val_err_if_fail(opts && buff_endpoint && srm_type && surl && s_buff, -1, err, "[gfal_get_endpoint_and_setype_from_bdiiG] invalid parameters"); char **tab_endpoint = NULL; char **tab_se_type = NULL; int ret = -1; GError *tmp_err = NULL; gfal2_uri *parsed = gfal2_parse_uri(surl, &tmp_err); if (parsed != NULL) { // get the hostname // questioning the bdii if ((ret = gfal_mds_get_se_types_and_endpoints(opts->handle, parsed->host, &tab_se_type, &tab_endpoint, &tmp_err)) == 0) { ret = gfal_select_best_protocol_and_endpointG(opts, tab_se_type, tab_endpoint, buff_endpoint, GFAL_URL_MAX_LEN, srm_type, &tmp_err); // map the response if correct g_strfreev(tab_endpoint); g_strfreev(tab_se_type); } gfal2_free_uri(parsed); } G_RETURN_ERR(ret, tmp_err, err); } int gfal_srm_determine_endpoint(gfal_srmv2_opt *opts, const char *surl, char *buff_endpoint, size_t s_buff, enum gfal_srm_proto *srm_type, GError **err) { g_return_val_err_if_fail( opts && buff_endpoint && srm_type && surl && s_buff, -1, err, "[gfal_srm_determine_endpoint] invalid value in params"); // check params GError *tmp_err = NULL; int ret = -1; gboolean isFullEndpoint = gfal_check_fullendpoint_in_surlG(opts, surl, &tmp_err); // check if a full endpoint exist if (!tmp_err) { if (isFullEndpoint == TRUE) { // if full endpoint contained in url, get it and set type to default type if (gfal_get_fullendpointG(surl, buff_endpoint, s_buff, &tmp_err) == 0) { *srm_type = opts->srm_proto_type; ret = 0; gfal2_log(G_LOG_LEVEL_DEBUG, "Service endpoint resolution, resolved from FULL SURL %s -> %s", surl, buff_endpoint); } } else { if (gfal_get_nobdiiG(opts->handle) == TRUE || ((ret = gfal_get_endpoint_and_setype_from_bdiiG(opts, surl, buff_endpoint, s_buff, srm_type, &tmp_err)) != 0)) { if (tmp_err) { gfal2_log(G_LOG_LEVEL_WARNING, "Error while bdii SRM service resolution : %s, fallback on the default service path." "This can lead to wrong service path, you should use FULL SURL format or register your endpoint into the BDII", tmp_err->message); g_clear_error(&tmp_err); } else { gfal2_log(G_LOG_LEVEL_WARNING, "BDII usage disabled, fallback on the default service path." "This can lead to wrong service path, you should use FULL SURL format or register your endpoint into the BDII"); } ret = gfal_srm_guess_service_endpoint(opts, surl, buff_endpoint, s_buff, srm_type, &tmp_err); if (ret == 0) { gfal2_log(G_LOG_LEVEL_DEBUG, "Service endpoint resolution, set to default path %s -> %s", surl, buff_endpoint); } } else { gfal2_log(G_LOG_LEVEL_DEBUG, "Service endpoint resolution, resolved from BDII %s -> %s", surl, buff_endpoint); } } } G_RETURN_ERR(ret, tmp_err, err); } int is_castor_endpoint(plugin_handle handle, const char *surl) { gfal_srmv2_opt *opts = (gfal_srmv2_opt *) handle; if (!srm_check_url(surl)) { gfal2_log(G_LOG_LEVEL_DEBUG, "Endpoint not SRM: %s", surl); return 0; } GError *tmp_err = NULL; gfal_srm_easy_t easy = gfal_srm_ifce_easy_context(opts, surl, &tmp_err); if (tmp_err) g_error_free(tmp_err); if (!easy) { gfal2_log(G_LOG_LEVEL_WARNING, "Could not get a context for %s", surl); return -1; } struct srm_xping_output output; if (gfal_srm_external_call.srm_xping(easy->srm_context, &output) < 0) { gfal2_log(G_LOG_LEVEL_WARNING, "Failed to ping %s", surl); gfal_srm_ifce_easy_context_release(opts, easy); return -1; } int i, is_castor = 0; for (i = 0; i < output.n_extra && !is_castor; ++i) { if (strcmp(output.extra[i].key, "backend_type") == 0) { gfal2_log(G_LOG_LEVEL_MESSAGE, "Endpoint of type %s: %s", output.extra[i].value, surl); is_castor = (strcasecmp(output.extra[i].value, "CASTOR") == 0); } } srm_xping_output_free(output); gfal_srm_ifce_easy_context_release(opts, easy); return is_castor; } gfal2-v2.23.0/src/plugins/srm/gfal_srm_endpoint.h000066400000000000000000000026151465240014500217000ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #pragma once #include "gfal_srm.h" #include /** * Extract endpoint and srm_type from a surl * determine the best endpoint associated with the surl and the param of the actual handle (no bdii check or not) * see the diagram in doc/diagrams/surls_get_endpoint_activity_diagram.svg for more informations * @return return 0 with endpoint and types set if success else -1 and set Error */ int gfal_srm_determine_endpoint(gfal_srmv2_opt *opts, const char *surl, char *buff_endpoint, size_t s_buff, enum gfal_srm_proto *srm_type, GError **err); /** Returns 1 if the surl is a castor endpoint */ int is_castor_endpoint(plugin_handle handle, const char *surl); gfal2-v2.23.0/src/plugins/srm/gfal_srm_getput.c000066400000000000000000000467031465240014500213710ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include "gfal_srm.h" #include "gfal_srm_request.h" #include "gfal_srm_internal_layer.h" #include "gfal_srm_endpoint.h" #include "gfal_srm_getput.h" // Make sure the TURL returned by the endpoint is one of the requested protocols static int validate_turls(int n_results, gfal_srm_result **resu, gfal_srm_params_t params, GError **tmp_err) { int failed = 0; int n_protocols = g_strv_length(params->protocols); int i, j; for (i = 0; i < n_results && !failed; ++i) { const char *turl = (*resu)[i].turl; if (turl[0] == '/') { failed = -1; gfal2_set_error(tmp_err, gfal2_get_plugin_srm_quark(), EBADMSG, __func__, "A turl can not start with /"); break; } // If error is set, skip the check if ((*resu)[i].err_code != 0) continue; // Check the turl protocol is in the request list int matching_protocol = 0; for (j = 0; j < n_protocols; ++j) { size_t proto_len = strlen(params->protocols[j]); if (strncmp(params->protocols[j], turl, proto_len) == 0 && turl[proto_len] == ':') { matching_protocol = 1; break; } } // If no matching protocol, fail already if (!matching_protocol) { failed = -1; gfal2_set_error(tmp_err, gfal2_get_plugin_srm_quark(), EBADMSG, __func__, "The SRM endpoint returned a protocol that wasn't requested: %s", turl); } } // Didn't match, so free and set an error if (failed) { free(*resu); *resu = NULL; } return failed; } static int gfal_srm_convert_filestatuses_to_srm_result(struct srmv2_pinfilestatus *statuses, char *reqtoken, int n, gfal_srm_result **resu, GError **err) { g_return_val_err_if_fail(statuses && n && resu, -1, err, "[gfal_srm_convert_filestatuses_to_srm_result] args invalids"); *resu = calloc(n, sizeof(gfal_srm_result)); int i = 0; for (i = 0; i < n; ++i) { if (statuses[i].turl) g_strlcpy((*resu)[i].turl, statuses[i].turl, GFAL_URL_MAX_LEN); if (statuses[i].explanation) g_strlcpy((*resu)[i].err_str, statuses[i].explanation, GFAL_URL_MAX_LEN); (*resu)[i].err_code = statuses[i].status; (*resu)[i].reqtoken = g_strdup(reqtoken); } return 0; } int gfal_srmv2_get_global(gfal_srmv2_opt *opts, gfal_srm_params_t params, srm_context_t context, struct srm_preparetoget_input *input, gfal_srm_result **resu, GError **err) { g_return_val_err_if_fail(opts != NULL && input != NULL && resu != NULL, -1, err, "[gfal_srmv2_get_global] tab null "); GError *tmp_err = NULL; int ret = 0; struct srm_preparetoget_output preparetoget_output; memset(&preparetoget_output, 0, sizeof(preparetoget_output)); ret = gfal_srm_external_call.srm_prepare_to_get(context, input, &preparetoget_output); if (ret < 0) { gfal_srm_report_error(context->errbuf, &tmp_err); } else { gfal2_log(G_LOG_LEVEL_MESSAGE, "Got GET token for %s: %s", input->surls[0], preparetoget_output.token); gfal_srm_convert_filestatuses_to_srm_result(preparetoget_output.filestatuses, preparetoget_output.token, ret, resu, &tmp_err); } if (preparetoget_output.filestatuses != NULL) gfal_srm_external_call.srm_srmv2_pinfilestatus_delete(preparetoget_output.filestatuses, ret); if (preparetoget_output.retstatus != NULL) gfal_srm_external_call.srm_srm2__TReturnStatus_delete(preparetoget_output.retstatus); free(preparetoget_output.token); G_RETURN_ERR(ret, tmp_err, err); } int gfal_srmv2_put_global(gfal_srmv2_opt *opts, gfal_srm_params_t params, srm_context_t context, struct srm_preparetoput_input *input, gfal_srm_result **resu, GError **err) { g_return_val_err_if_fail(opts != NULL && input != NULL && resu != NULL, -1, err, "[gfal_srmv2_put_global] tab null "); GError *tmp_err = NULL; int ret = 0; struct srm_preparetoput_output preparetoput_output; memset(&preparetoput_output, 0, sizeof(preparetoput_output)); ret = gfal_srm_external_call.srm_prepare_to_put(context, input, &preparetoput_output); if (ret < 0) { gfal_srm_report_error(context->errbuf, &tmp_err); } else { gfal2_log(G_LOG_LEVEL_MESSAGE, "Got PUT token for %s: %s", input->surls[0], preparetoput_output.token); gfal_srm_convert_filestatuses_to_srm_result(preparetoput_output.filestatuses, preparetoput_output.token, ret, resu, &tmp_err); } if (preparetoput_output.filestatuses != NULL) gfal_srm_external_call.srm_srmv2_pinfilestatus_delete(preparetoput_output.filestatuses, ret); if (preparetoput_output.retstatus != NULL) gfal_srm_external_call.srm_srm2__TReturnStatus_delete(preparetoput_output.retstatus); free(preparetoput_output.token); G_RETURN_ERR(ret, tmp_err, err); } // @brief execute a srmv2 request sync "GET" on the srm_ifce int gfal_srm_getTURLS_srmv2_internal(srm_context_t context, gfal_srmv2_opt *opts, gfal_srm_params_t params, char *surl, gfal_srm_result **resu, GError **err) { g_return_val_err_if_fail(surl != NULL, -1, err, "[gfal_srmv2_getasync] tab null"); GError *tmp_err = NULL; int ret = -1; struct srm_preparetoget_input preparetoget_input; // set the structures datafields preparetoget_input.desiredpintime = 0; preparetoget_input.nbfiles = 1; preparetoget_input.protocols = gfal_srm_params_get_protocols(params); preparetoget_input.spacetokendesc = gfal_srm_params_get_spacetoken(params); preparetoget_input.surls = &surl; ret = gfal_srmv2_get_global(opts, params, context, &preparetoget_input, resu, &tmp_err); G_RETURN_ERR(ret, tmp_err, err); } // execute a srmv2 request sync "PUT" on the srm_ifce int gfal_srm_putTURLS_srmv2_internal(srm_context_t context, gfal_srmv2_opt *opts, gfal_srm_params_t params, char *surl, gfal_srm_result **resu, GError **err) { g_return_val_err_if_fail(surl != NULL, -1, err, "[gfal_srm_putTURLS_srmv2_internal] GList passed null"); GError *tmp_err = NULL; int ret = -1; struct srm_preparetoput_input preparetoput_input; SRM_LONG64 filesize = params->file_size; // set the structures datafields preparetoput_input.desiredpintime = 0; preparetoput_input.nbfiles = 1; preparetoput_input.protocols = gfal_srm_params_get_protocols(params); preparetoput_input.spacetokendesc = gfal_srm_params_get_spacetoken(params); preparetoput_input.surls = &surl; preparetoput_input.filesizes = &filesize; ret = gfal_srmv2_put_global(opts, params, context, &preparetoput_input, resu, &tmp_err); G_RETURN_ERR(ret, tmp_err, err); } // Internal function of gfal_srm_getTurls without argument check for internal usage static int gfal_srm_mTURLS_internal(gfal_srmv2_opt *opts, gfal_srm_params_t params, srm_req_type req_type, const char *surl, gfal_srm_result **resu, GError **err) { GError *tmp_err = NULL; int ret = -1; gfal_srm_easy_t easy = gfal_srm_ifce_easy_context(opts, surl, &tmp_err); if (easy != NULL) { if (req_type == SRM_GET) ret = gfal_srm_getTURLS_srmv2_internal(easy->srm_context, opts, params, easy->path, resu, &tmp_err); else ret = gfal_srm_putTURLS_srmv2_internal(easy->srm_context, opts, params, easy->path, resu, &tmp_err); } gfal_srm_ifce_easy_context_release(opts, easy); if (ret < 0) { gfal2_propagate_prefixed_error(err, tmp_err, __func__); } else { if (validate_turls(1, resu, params, &tmp_err)) { gfal2_propagate_prefixed_error(err, tmp_err, __func__); ret = -1; } } return ret; } // simple wrapper to getTURLs for the gfal_module layer int gfal_srm_getTURLS_plugin(plugin_handle ch, const char *surl, char *buff_turl, int size_turl, char **reqtoken, GError **err) { gfal_srmv2_opt *opts = (gfal_srmv2_opt *) ch; gfal_srm_result *resu = NULL; GError *tmp_err = NULL; int ret = -1; gfal_srm_params_t params = gfal_srm_params_new(opts); if (params != NULL) { ret = gfal_srm_mTURLS_internal(opts, params, SRM_GET, surl, &resu, &tmp_err); if (ret > 0) { if (resu->err_code == 0) { g_strlcpy(buff_turl, resu->turl, size_turl); if (reqtoken) *reqtoken = resu->reqtoken; ret = 0; } else { gfal2_set_error(&tmp_err, gfal2_get_plugin_srm_quark(), resu->err_code, __func__, "error on the turl request : %s ", resu->err_str); ret = -1; g_free(resu->reqtoken); } free(resu); } gfal_srm_params_free(params); } G_RETURN_ERR(ret, tmp_err, err); } // special call for TURL resolution for checksum fallback solution int gfal_srm_getTURL_checksum(plugin_handle ch, const char *surl, char *buff_turl, int size_turl, GError **err) { gfal_srmv2_opt *opts = (gfal_srmv2_opt *) ch; gfal_srm_result *resu = NULL; char **sup_protocols = NULL; GError *tmp_err = NULL; int ret = -1; gfal_srm_params_t params = gfal_srm_params_new(opts); if (params != NULL) { sup_protocols = srm_get_3rdparty_turls_sup_protocol(opts->handle); gfal_srm_params_set_protocols(params, sup_protocols); ret = gfal_srm_mTURLS_internal(opts, params, SRM_GET, surl, &resu, &tmp_err); if (ret > 0) { if (resu->err_code == 0) { g_strlcpy(buff_turl, resu->turl, size_turl); ret = 0; } else { gfal2_set_error(&tmp_err, gfal2_get_plugin_srm_quark(), resu->err_code, __func__, "error on the turl %s request : %s ", resu->turl, resu->err_str); ret = -1; } free(resu); } gfal_srm_params_free(params); } G_RETURN_ERR(ret, tmp_err, err); } // execute a get for thirdparty transfer turl int gfal_srm_get_rd3_turl(plugin_handle ch, gfalt_params_t p, const char *surl, const char *other_surl, char *buff_turl, int size_turl, char *reqtoken, size_t size_reqtoken, GError **err) { gfal_srmv2_opt *opts = (gfal_srmv2_opt *) ch; gfal_srm_result *resu = NULL; char **sup_protocols = NULL; GError *tmp_err = NULL; int ret = -1; gfal_srm_params_t params = gfal_srm_params_new(opts); if (params != NULL) { gfal_srm_params_set_spacetoken(params, gfalt_get_src_spacetoken(p, NULL)); sup_protocols = srm_get_3rdparty_turls_sup_protocol(opts->handle); reorder_rd3_sup_protocols(sup_protocols, other_surl); gfal_srm_params_set_protocols(params, sup_protocols); ret = gfal_srm_mTURLS_internal(opts, params, SRM_GET, surl, &resu, &tmp_err); if (ret >= 0) { if (resu->err_code == 0) { g_strlcpy(buff_turl, resu->turl, size_turl); if (reqtoken) g_strlcpy(reqtoken, resu->reqtoken, size_reqtoken); ret = 0; } else { gfal2_set_error(&tmp_err, gfal2_get_plugin_srm_quark(), resu->err_code, __func__, "error on the turl %s request : %s ", resu->turl, resu->err_str); ret = -1; } free(resu); } gfal_srm_params_free(params); } G_RETURN_ERR(ret, tmp_err, err); } // execute a put for thirdparty transfer turl int gfal_srm_put_rd3_turl(plugin_handle ch, gfalt_params_t p, const char *surl, const char *other_surl, size_t surl_file_size, char *buff_turl, int size_turl, char *reqtoken, size_t size_reqtoken, GError **err) { gfal_srmv2_opt *opts = (gfal_srmv2_opt *) ch; gfal_srm_result *resu = NULL; char **sup_protocols = NULL; GError *tmp_err = NULL; int ret = -1; gfal_srm_params_t params = gfal_srm_params_new(opts); if (params != NULL) { gfal_srm_params_set_spacetoken(params, gfalt_get_dst_spacetoken(p, NULL)); sup_protocols = srm_get_3rdparty_turls_sup_protocol(opts->handle); reorder_rd3_sup_protocols(sup_protocols, other_surl); gfal_srm_params_set_protocols(params, sup_protocols); gfal_srm_params_set_size(params, surl_file_size); ret = gfal_srm_mTURLS_internal(opts, params, SRM_PUT, surl, &resu, &tmp_err); if (ret >= 0) { if (resu->err_code == 0) { g_strlcpy(buff_turl, resu->turl, size_turl); if (reqtoken) g_strlcpy(reqtoken, resu->reqtoken, size_reqtoken); ret = 0; free(resu); } else { gfal2_set_error(&tmp_err, gfal2_get_plugin_srm_quark(), resu->err_code, __func__, "error on the turl %s request : %s ", resu->turl, resu->err_str); ret = -1; } } gfal_srm_params_free(params); } G_RETURN_ERR(ret, tmp_err, err); } // simple wrapper to putTURLs for the gfal_module layer int gfal_srm_putTURLS_plugin(plugin_handle ch, const char *surl, char *buff_turl, int size_turl, char **reqtoken, GError **err) { gfal_srmv2_opt *opts = (gfal_srmv2_opt *) ch; gfal_srm_result *resu = NULL; GError *tmp_err = NULL; int ret = -1; gfal_srm_params_t params = gfal_srm_params_new(opts); if (params != NULL) { ret = gfal_srm_mTURLS_internal(opts, params, SRM_PUT, surl, &resu, &tmp_err); if (ret > 0) { if (resu->err_code == 0) { g_strlcpy(buff_turl, resu->turl, size_turl); if (reqtoken) *reqtoken = resu->reqtoken; ret = 0; } else { gfal2_set_error(&tmp_err, gfal2_get_plugin_srm_quark(), resu->err_code, __func__, "error on the turl request : %s ", resu->err_str); ret = -1; } free(resu); } gfal_srm_params_free(params); } G_RETURN_ERR(ret, tmp_err, err); } static void gfal_log_3rd_sup_protocols(const char *msg, char **protocols) { size_t n_protocols = g_strv_length(protocols); GString *msgline = g_string_new(msg); int i; for (i = 0; i < n_protocols; ++i) { if (i > 0) { g_string_append_c(msgline, ';'); } g_string_append(msgline, protocols[i]); } gfal2_log(G_LOG_LEVEL_DEBUG, "%s", msgline->str); g_string_free(msgline, TRUE); } // The other_surl protocol is prioritized to the first position of the supported protocols int reorder_rd3_sup_protocols(char **sup_protocols, const char *other_surl) { size_t n_protocols = g_strv_length(sup_protocols); size_t compare_surl_len = strlen(other_surl); char *compare_surl = other_surl; char *swap; int j; gfal_log_3rd_sup_protocols("\t\tInitial TURLs: ", sup_protocols); // Treat "davs://" and "https:// as the same protocol if (strncmp(compare_surl, "davs", 4) == 0) { compare_surl_len += 1; compare_surl = malloc(compare_surl_len + 1); snprintf(compare_surl, compare_surl_len + 1, "https%s", other_surl + 4); } // Check the compare_surl protocol is in the request list for (j = 0; j < n_protocols; ++j) { size_t proto_len = strlen(sup_protocols[j]); if ((compare_surl_len > proto_len) && (compare_surl[proto_len] == ':') && (strncmp(sup_protocols[j], compare_surl, proto_len) == 0)) { swap = sup_protocols[0]; sup_protocols[0] = sup_protocols[j]; sup_protocols[j] = swap; break; } } if (compare_surl != other_surl) { free(compare_surl); } gfal_log_3rd_sup_protocols("\t\tReordered TURLs: ", sup_protocols); return 0; } // execute a srm put done on the specified surl and token, return 0 if success else -1 and errno is set static int gfal_srm_putdone_srmv2_internal(srm_context_t context, char *surl, const char *token, GError **err) { g_return_val_err_if_fail(surl != NULL, -1, err, "[gfal_srm_putdone_srmv2_internal] invalid args "); GError *tmp_err = NULL; int ret = 0; struct srm_putdone_input putdone_input; struct srmv2_filestatus *statuses; // set the structures datafields putdone_input.nbfiles = 1; putdone_input.reqtoken = (char *) token; putdone_input.surls = &surl; gfal2_log(G_LOG_LEVEL_DEBUG, " [gfal_srm_putdone_srmv2_internal] start srm put done on %s", surl); ret = gfal_srm_external_call.srm_put_done(context, &putdone_input, &statuses); if (ret < 0) { gfal2_set_error(&tmp_err, gfal2_get_plugin_srm_quark(), errno, __func__, "call to srm_ifce error: %s", context->errbuf); } else { ret = gfal_srm_convert_filestatuses_to_GError(statuses, ret, &tmp_err); gfal_srm_external_call.srm_srmv2_filestatus_delete(statuses, 1); } G_RETURN_ERR(ret, tmp_err, err); } int gfal_srm_putdone(gfal_srmv2_opt *opts, const char *surl, const char *token, GError **err) { GError *tmp_err = NULL; int ret = -1; gfal2_log(G_LOG_LEVEL_DEBUG, " -> [gfal_srm_putdone] "); gfal_srm_easy_t easy = gfal_srm_ifce_easy_context(opts, surl, &tmp_err); if (easy != NULL) { ret = gfal_srm_putdone_srmv2_internal(easy->srm_context, easy->path, token, &tmp_err); } gfal_srm_ifce_easy_context_release(opts, easy); if (ret < 0) gfal2_propagate_prefixed_error(err, tmp_err, __func__); return ret; } static int srmv2_abort_request_internal(srm_context_t context, const char *surl, const char *req_token, GError **err) { GError *tmp_err = NULL; int ret = gfal_srm_external_call.srm_abort_request(context, (char *) req_token); if (ret < 0) { gfal2_set_error(&tmp_err, gfal2_get_plugin_srm_quark(), errno, __func__, "SRMv2 abort request error : %s", context->errbuf); } G_RETURN_ERR(ret, tmp_err, err); } int srm_abort_request_plugin(plugin_handle *handle, const char *surl, const char *reqtoken, GError **err) { g_return_val_err_if_fail(handle != NULL && reqtoken != NULL, -1, err, "[srm_abort_request_plugin] invalid values for token/handle"); GError *tmp_err = NULL; gfal_srmv2_opt *opts = (gfal_srmv2_opt *) handle; int ret = -1; gfal2_log(G_LOG_LEVEL_DEBUG, " -> [srm_abort_request] "); gfal_srm_easy_t easy = gfal_srm_ifce_easy_context(opts, surl, &tmp_err); if (easy != NULL) { ret = srmv2_abort_request_internal(easy->srm_context, easy->path, reqtoken, &tmp_err); } gfal_srm_ifce_easy_context_release(opts, easy); gfal2_log(G_LOG_LEVEL_DEBUG, " [srm_abort_request] <-"); if (ret < 0) gfal2_propagate_prefixed_error(err, tmp_err, __func__); return ret; } gfal2-v2.23.0/src/plugins/srm/gfal_srm_getput.h000066400000000000000000000026461465240014500213740ustar00rootroot00000000000000/* * Copyright @ Members of the EMI Collaboration, 2010. * See www.eu-emi.eu for details on the copyright holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include "gfal_srm.h" int gfal_srm_put_rd3_turl(plugin_handle ch, gfalt_params_t p, const char *surl, const char *other_surl, size_t surl_file_size, char *buff_turl, int size_turl, char *reqtoken, size_t size_reqtoken, GError **err); int gfal_srm_get_rd3_turl(plugin_handle ch, gfalt_params_t params, const char *surl, const char *other_surl, char *buff_turl, int size_turl, char *reqtoken, size_t size_reqtoken, GError **err); int gfal_srm_getTURL_checksum(plugin_handle ch, const char *surl, char *buff_turl, int size_turl, GError **err); int reorder_rd3_sup_protocols(char **sup_protocols, const char *other_surl); int srm_abort_request_plugin(plugin_handle *handle, const char *surl, const char *reqtoken, GError **err); gfal2-v2.23.0/src/plugins/srm/gfal_srm_getxattr.c000066400000000000000000000127601465240014500217170ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include "gfal_srm.h" #include "gfal_srm_internal_layer.h" #include "gfal_srm_namespace.h" #include "gfal_srm_space.h" #define GFAL_XATTR_SRM_TYPE "srm.type" static char *srm_listxattr[] = {SRM_XATTR_GETURL, GFAL_XATTR_STATUS, GFAL_XATTR_SRM_TYPE, GFAL_XATTR_SPACETOKEN, NULL}; static ssize_t gfal_srm_get_endpoint_type_xattrG(plugin_handle handle, const char *path, const char *name, void *buff, size_t s_buff, GError **err) { GError *tmp_err = NULL; gfal_srm_easy_t easy = gfal_srm_ifce_easy_context(handle, path, &tmp_err); if (!easy) { gfal2_propagate_prefixed_error(err, tmp_err, __func__); return -1; } struct srm_xping_output output; if (gfal_srm_external_call.srm_xping(easy->srm_context, &output) < 0) { gfal2_set_error(err, gfal2_get_plugin_srm_quark(), errno, __func__, "Could not get the storage type"); return -1; } memset(buff, 0, s_buff); int i; for (i = 0; i < output.n_extra; ++i) { if (strcmp(output.extra[i].key, "backend_type") == 0) { g_strlcpy(buff, output.extra[i].value, s_buff); break; } } srm_xping_output_free(output); gfal_srm_ifce_easy_context_release(handle, easy); return strnlen(buff, s_buff); } ssize_t gfal_srm_geturl_getxattrG(plugin_handle handle, const char *path, const char *name, void *buff, size_t s_buff, GError **err) { GError *tmp_err = NULL; char tmp_buff[1024]; ssize_t ret = -1; if (s_buff == 0 || buff == NULL) return GFAL_URL_MAX_LEN; // If XATTR_FAIL_NEARLINE is set, obtain TURLs only if the file is ONLINE gfal_srmv2_opt *opts = (gfal_srmv2_opt *) handle; gfal2_context_t context = opts->handle; gboolean fail_nearline = gfal2_get_opt_boolean_with_default(context, "SRM PLUGIN", "XATTR_FAIL_NEARLINE", FALSE); if (fail_nearline) { gfal2_log(G_LOG_LEVEL_DEBUG, "XAttr-fail-nearline: querying status first"); ret = gfal_srm_status_getxattrG(handle, path, GFAL_XATTR_STATUS, tmp_buff, sizeof(tmp_buff), &tmp_err); if (ret > 0 && strlen(tmp_buff) && tmp_err == NULL) { if (strncmp(tmp_buff, GFAL_XATTR_STATUS_NEARLINE, sizeof(GFAL_XATTR_STATUS_NEARLINE)) == 0) { gfal2_log(G_LOG_LEVEL_DEBUG, "XAttr-fail-nearline: source file is not ONLINE"); gfal2_set_error(&tmp_err, gfal2_get_plugin_srm_quark(), EINVAL, __func__, "The source file is not ONLINE"); gfal2_propagate_prefixed_error(err, tmp_err, __func__); return -1; } } else { if (tmp_err == NULL) { gfal2_set_error(&tmp_err, gfal2_get_plugin_srm_quark(), EINVAL, __func__, "Error while checking if the source file is ONLINE"); } gfal2_propagate_prefixed_error(err, tmp_err, __func__); return -1; } } ret = gfal_srm_getTURLS_plugin(handle, path, buff, s_buff, NULL, &tmp_err); if (ret >= 0) { ret = strnlen(buff, s_buff) * sizeof(char); } G_RETURN_ERR(ret, tmp_err, err); } /* * implementation of the getxattr for turl resolution, pin management and spacetoken set/get * * */ ssize_t gfal_srm_getxattrG(plugin_handle handle, const char *path, const char *name, void *buff, size_t s_buff, GError **err) { GError *tmp_err = NULL; ssize_t ret = -1; gfal2_log(G_LOG_LEVEL_DEBUG, " gfal_srm_getxattrG ->"); if (strcmp(name, SRM_XATTR_GETURL) == 0) { ret = gfal_srm_geturl_getxattrG(handle, path, name, buff, s_buff, &tmp_err); } else if (strcmp(name, GFAL_XATTR_STATUS) == 0) { ret = gfal_srm_status_getxattrG(handle, path, name, buff, s_buff, &tmp_err); } else if (strcmp(name, GFAL_XATTR_SRM_TYPE) == 0) { ret = gfal_srm_get_endpoint_type_xattrG(handle, path, name, buff, s_buff, err); } else if (strncmp(name, GFAL_XATTR_SPACETOKEN, 10) == 0) { return gfal_srm_space_getxattrG(handle, path, name, buff, s_buff, err); } else { gfal2_set_error(&tmp_err, gfal2_get_plugin_srm_quark(), ENOATTR, __func__, "not an existing extended attribute"); } gfal2_log(G_LOG_LEVEL_DEBUG, " gfal_srm_getxattrG <- "); G_RETURN_ERR(ret, tmp_err, err); } /* * lfc getxattr implem * */ ssize_t gfal_srm_listxattrG(plugin_handle handle, const char *path, char *list, size_t size, GError **err) { ssize_t res = 0; char **p = srm_listxattr; char *plist = list; while (*p != NULL) { const int size_str = strlen(*p) + 1; if (size > res && size - res >= size_str) { memcpy(plist, *p, size_str); plist += size_str; } res += size_str; p++; } return res; } gfal2-v2.23.0/src/plugins/srm/gfal_srm_getxattr_status.c000066400000000000000000000066151465240014500233240ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include "gfal_srm.h" #include "gfal_srm_internal_layer.h" #include "gfal_srm_namespace.h" #include "gfal_srm_endpoint.h" #include "gfal_srm_internal_ls.h" void gfal_srm_status_copy(TFileLocality loc, char *buff, size_t s_buff) { char *org_string; switch (loc) { case GFAL_LOCALITY_ONLINE_: org_string = GFAL_XATTR_STATUS_ONLINE; break; case GFAL_LOCALITY_LOST: org_string = GFAL_XATTR_STATUS_LOST; break; case GFAL_LOCALITY_NEARLINE_: org_string = GFAL_XATTR_STATUS_NEARLINE; break; case GFAL_LOCALITY_UNAVAILABLE: org_string = GFAL_XATTR_STATUS_UNAVAILABLE; break; case GFAL_LOCALITY_ONLINE_USCOREAND_USCORENEARLINE: org_string = GFAL_XATTR_STATUS_NEARLINE_ONLINE; break; case GFAL_LOCALITY_NONE_: org_string = GFAL_XATTR_STATUS_NONE; break; default: org_string = GFAL_XATTR_STATUS_UNKNOWN; break; } g_strlcpy(buff, org_string, s_buff); } ssize_t gfal_srm_status_internal(gfal_srmv2_opt *opts, srm_context_t context, const char *path, void *buff, size_t s_buff, GError **err) { GError *tmp_err = NULL; ssize_t ret = -1; struct extended_stat buf; char key_buff[GFAL_URL_MAX_LEN]; gfal_srm_construct_key(path, GFAL_SRM_LSTAT_PREFIX, key_buff, GFAL_URL_MAX_LEN); if (gsimplecache_take_one_kstr(opts->cache, key_buff, &buf) == 0) { gfal2_log(G_LOG_LEVEL_DEBUG, " gfal_srm_status_internal -> value taken from the cache"); ret = 0; } else { ret = gfal_statG_srmv2__generic_internal(context, &buf.stat, &buf.locality, path, &tmp_err); } if (ret >= 0) { gfal_srm_status_copy(buf.locality, (char *) buff, s_buff); ret = strnlen(buff, s_buff); } G_RETURN_ERR(ret, tmp_err, err); } /* * main implementation of the srm status -> getxattr */ ssize_t gfal_srm_status_getxattrG(plugin_handle handle, const char *surl, const char *name, void *buff, size_t s_buff, GError **err) { g_return_val_err_if_fail(handle && surl, EINVAL, err, "[gfal_srm_status_getxattrG] Invalid value handle and/or surl"); GError *tmp_err = NULL; gfal_srmv2_opt *opts = (gfal_srmv2_opt *) handle; int ret = -1; gfal_srm_easy_t easy = gfal_srm_ifce_easy_context(opts, surl, &tmp_err); if (easy != NULL) { ret = gfal_srm_status_internal(opts, easy->srm_context, easy->path, buff, s_buff, &tmp_err); } gfal_srm_ifce_easy_context_release(opts, easy); if (ret < 0) gfal2_propagate_prefixed_error(err, tmp_err, __func__); return ret; } gfal2-v2.23.0/src/plugins/srm/gfal_srm_internal_layer.c000066400000000000000000000216121465240014500230610ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ const char *srm_config_group = "SRM PLUGIN"; const char *srm_config_transfer_checksum = "COPY_CHECKSUM_TYPE"; const char *srm_ops_timeout_key = "OPERATION_TIMEOUT"; const char *srm_conn_timeout_key = "CONN_TIMEOUT"; const char *srm_desired_request_lifetime = "REQUEST_LIFETIME"; const char *srm_config_turl_protocols = "TURL_PROTOCOLS"; const char *srm_config_3rd_party_turl_protocols = "TURL_3RD_PARTY_PROTOCOLS"; const char *srm_config_keep_alive = "KEEP_ALIVE"; const char *srm_spacetokendesc = "SPACETOKENDESC"; #include "gfal_srm_internal_layer.h" #include "gfal_srm_url_check.h" // hotfix for the old srm lib void disable_srm_srmv2_pinfilestatus_delete(struct srmv2_pinfilestatus *srmv2_pinstatuses, int n) { } void disable_srm_srmv2_mdfilestatus_delete(struct srmv2_mdfilestatus *mdfilestatus, int n) { } void disable_srm_srmv2_filestatus_delete(struct srmv2_filestatus *srmv2_statuses, int n) { } void disable_srm_srm2__TReturnStatus_delete(struct srm2__TReturnStatus *status) { } struct _gfal_srm_external_call gfal_srm_external_call = { .srm_ls = &srm_ls, .srm_rmdir = &srm_rmdir, .srm_mkdir = &srm_mkdir, .srm_getpermission = &srm_getpermission, .srm_check_permission = &srm_check_permission, .srm_srmv2_pinfilestatus_delete = &srm_srmv2_pinfilestatus_delete, .srm_srmv2_mdfilestatus_delete = &srm_srmv2_mdfilestatus_delete, .srm_srmv2_filestatus_delete = &srm_srmv2_filestatus_delete, .srm_srm2__TReturnStatus_delete = &srm_srm2__TReturnStatus_delete, .srm_prepare_to_get= &srm_prepare_to_get, .srm_prepare_to_put= &srm_prepare_to_put, .srm_put_done = &srm_put_done, .srm_setpermission= &srm_setpermission, .srm_rm = &srm_rm, .srm_set_timeout_connect = &srm_set_timeout_connect, .srm_bring_online = &srm_bring_online, .srm_bring_online_async = &srm_bring_online_async, .srm_bring_online_status = &srm_status_of_bring_online_async, .srm_release_files = &srm_release_files, .srm_mv = &srm_mv, .srm_abort_request = &srm_abort_request, .srm_getspacetokens = &srm_getspacetokens, .srm_getspacemd = &srm_getspacemd, .srm_abort_files = &srm_abort_files, .srm_xping = &srm_xping }; static srm_context_t gfal_srm_ifce_context_setup(gfal2_context_t handle, const char *endpoint, const char *ucert, const char *ukey, char *errbuff, size_t s_errbuff, GError **err) { gint timeout; srm_context_t context = NULL; GError *tmp_err = NULL; const gboolean keep_alive = gfal2_get_opt_boolean_with_default(handle, srm_config_group, srm_config_keep_alive, FALSE); gfal2_log(G_LOG_LEVEL_DEBUG, " SRM connection keep-alive %d", keep_alive); context = srm_context_new2(endpoint, errbuff, s_errbuff, gfal2_log_get_level() >= G_LOG_LEVEL_DEBUG, keep_alive); if (context != NULL) { gint global_timeout = gfal2_get_opt_integer_with_default(handle, CORE_CONFIG_GROUP, CORE_CONFIG_NAMESPACE_TIMEOUT, 180); timeout = gfal2_get_opt_integer_with_default(handle, srm_config_group, srm_ops_timeout_key, global_timeout); gfal2_log(G_LOG_LEVEL_DEBUG, " SRM operation timeout %d", timeout); context->timeout = timeout; context->timeout_ops = timeout; timeout = gfal2_get_opt_integer_with_default(handle, srm_config_group, srm_conn_timeout_key, 60); gfal2_log(G_LOG_LEVEL_DEBUG, " SRM connection timeout %d", timeout); context->timeout_conn = timeout; if (ucert) { gfal2_log(G_LOG_LEVEL_DEBUG, " SRM using certificate %s", ucert); if (ukey) gfal2_log(G_LOG_LEVEL_DEBUG, " SRM using private key %s", ukey); srm_set_credentials(context, ucert, ukey); } // User agent const char *agent, *agent_version; gfal2_get_user_agent(handle, &agent, &agent_version); if (agent) { srm_set_user_agent(context, "%s/%s gfal2/%s", agent, agent_version, gfal2_version()); } else { srm_set_user_agent(context, "gfal2/%s", gfal2_version()); } // Client information char *client_info = gfal2_get_client_info_string(handle); if (client_info) { srm_set_http_header(context, "ClientInfo", client_info); } g_free(client_info); } else { gfal2_set_error(&tmp_err, gfal2_get_plugin_srm_quark(), EINVAL, __func__, "Impossible to create srm context"); } G_RETURN_ERR(context, tmp_err, err); } static int is_same_context(gfal_srmv2_opt *opts, const char *endpoint, const char *ucert, const char *ukey) { if (strcmp(opts->endpoint, endpoint) != 0) return 0; if ((ucert && strcmp(opts->x509_ucert, ucert) != 0) || (!ucert && opts->x509_ucert[0])) return 0; if ((ukey && strcmp(opts->x509_ukey, ukey) != 0) || (!ukey && opts->x509_ukey[0])) return 0; return 1; } gfal_srm_easy_t gfal_srm_ifce_easy_context(gfal_srmv2_opt *opts, const char *surl, GError **err) { GError *nested_error = NULL; char full_endpoint[GFAL_URL_MAX_LEN]; const char *baseurl; enum gfal_srm_proto srm_types; if (gfal_srm_determine_endpoint(opts, surl, full_endpoint, sizeof(full_endpoint), &srm_types, &nested_error) < 0) { gfal2_propagate_prefixed_error(err, nested_error, __func__); return NULL; } gchar *ucert = gfal2_cred_get(opts->handle, GFAL_CRED_X509_CERT, surl, &baseurl, err); if (*err) { return NULL; } gchar *ukey = gfal2_cred_get(opts->handle, GFAL_CRED_X509_KEY, surl, &baseurl, err); if (*err) { return NULL; } if (!baseurl || !*baseurl) { baseurl = full_endpoint; } g_static_rec_mutex_lock(&opts->srm_context_mutex); // Try with existing one if (opts->srm_context) { if (is_same_context(opts, baseurl, ucert, ukey)) { gfal2_log(G_LOG_LEVEL_DEBUG, "SRM context recycled for %s", baseurl); } else { gfal2_log(G_LOG_LEVEL_DEBUG, "SRM context invalidated for %s", baseurl); srm_context_free(opts->srm_context); opts->srm_context = NULL; } } else { gfal2_log(G_LOG_LEVEL_DEBUG, "SRM context not available"); } // Instantiate if we haven't got any if (opts->srm_context == NULL) { switch (srm_types) { case PROTO_SRMv2: opts->srm_context = gfal_srm_ifce_context_setup(opts->handle, full_endpoint, ucert, ukey, opts->srm_ifce_error_buffer, sizeof(opts->srm_ifce_error_buffer), &nested_error); if (nested_error) gfal2_propagate_prefixed_error(err, nested_error, __func__); break; case PROTO_SRM: gfal2_set_error(err, gfal2_get_plugin_srm_quark(), EPROTONOSUPPORT, __func__, "SRM v1 is not supported, failure"); return NULL; default: gfal2_set_error(err, gfal2_get_plugin_srm_quark(), EPROTONOSUPPORT, __func__, "Unknown version of the protocol SRM, failure"); return NULL; } } // Configure gfal_srm_easy_t easy = g_malloc0(sizeof(struct gfal_srm_easy)); easy->path = gfal2_srm_get_decoded_path(surl); if (opts->srm_context) { g_strlcpy(opts->endpoint, full_endpoint, GFAL_URL_MAX_LEN); if (ucert) { g_strlcpy(opts->x509_ucert, ucert, GFAL_URL_MAX_LEN); } if (ukey) { g_strlcpy(opts->x509_ukey, ukey, sizeof(opts->x509_ukey)); } time_t request_lifetime = gfal2_get_opt_integer_with_default(opts->handle, srm_config_group, srm_desired_request_lifetime, 3600); srm_set_desired_request_time(opts->srm_context, request_lifetime); } else { g_static_rec_mutex_unlock(&opts->srm_context_mutex); } g_free(ucert); g_free(ukey); easy->srm_context = opts->srm_context; return easy; } void gfal_srm_ifce_easy_context_release(gfal_srmv2_opt *opts, gfal_srm_easy_t easy) { if (opts) { g_static_rec_mutex_unlock(&opts->srm_context_mutex); } if (easy) { g_free(easy->path); g_free(easy); } } gfal2-v2.23.0/src/plugins/srm/gfal_srm_internal_layer.h000066400000000000000000000122541465240014500230700ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #pragma once #include #include #include #include "gfal_srm_endpoint.h" #include "gfal_srm.h" #include "gfal_srm_bringonline.h" extern const char *srm_config_group; extern const char *srm_config_transfer_checksum; extern const char *srm_config_turl_protocols; extern const char *srm_config_3rd_party_turl_protocols; extern const char *srm_spacetokendesc; // request type for surl <-> turl translation typedef enum _srm_req_type { SRM_GET, SRM_PUT } srm_req_type; struct gfal_srm_easy { srm_context_t srm_context; char *path; }; typedef struct gfal_srm_easy *gfal_srm_easy_t; /* * structure for mock abylity in the srm part * */ struct _gfal_srm_external_call { int (*srm_ls)(struct srm_context *context, struct srm_ls_input *input, struct srm_ls_output *output); int (*srm_rm)(struct srm_context *context, struct srm_rm_input *input, struct srm_rm_output *output); int (*srm_rmdir)(struct srm_context *context, struct srm_rmdir_input *input, struct srm_rmdir_output *output); int (*srm_mkdir)(struct srm_context *context, struct srm_mkdir_input *input); int (*srm_getpermission)(struct srm_context *context, struct srm_getpermission_input *input, struct srm_getpermission_output *output); int (*srm_check_permission)(struct srm_context *context, struct srm_checkpermission_input *input, struct srmv2_filestatus **statuses); int (*srm_prepare_to_get)(struct srm_context *context, struct srm_preparetoget_input *input, struct srm_preparetoget_output *output); void (*srm_srmv2_pinfilestatus_delete)(struct srmv2_pinfilestatus *srmv2_pinstatuses, int n); void (*srm_srmv2_mdfilestatus_delete)(struct srmv2_mdfilestatus *mdfilestatus, int n); void (*srm_srmv2_filestatus_delete)(struct srmv2_filestatus *srmv2_statuses, int n); void (*srm_srm2__TReturnStatus_delete)(struct srm2__TReturnStatus *status); int (*srm_prepare_to_put)(struct srm_context *context, struct srm_preparetoput_input *input, struct srm_preparetoput_output *output); int (*srm_put_done)(struct srm_context *context, struct srm_putdone_input *input, struct srmv2_filestatus **statuses); int (*srm_setpermission)(struct srm_context *context, struct srm_setpermission_input *input); void (*srm_set_timeout_connect)(int); int (*srm_bring_online)(struct srm_context *context, struct srm_bringonline_input *input, struct srm_bringonline_output *output); int (*srm_bring_online_async)(struct srm_context *context, struct srm_bringonline_input *input, struct srm_bringonline_output *output); int (*srm_bring_online_status)(struct srm_context *context, struct srm_bringonline_input *input, struct srm_bringonline_output *output); int (*srm_release_files)(struct srm_context *context, struct srm_releasefiles_input *input, struct srmv2_filestatus **statuses); int (*srm_mv)(struct srm_context *context, struct srm_mv_input *input); int (*srm_abort_request)(struct srm_context *context, char *reqtoken); // Space methods int (*srm_getspacetokens)(struct srm_context *context, struct srm_getspacetokens_input *input, struct srm_getspacetokens_output *output); int (*srm_getspacemd)(struct srm_context *context, struct srm_getspacemd_input *input, struct srm_spacemd **spaces); int (*srm_abort_files)(struct srm_context *context, struct srm_abort_files_input *input, struct srmv2_filestatus **statuses); // Ping int (*srm_xping)(struct srm_context *context, struct srm_xping_output *output); }; extern struct _gfal_srm_external_call gfal_srm_external_call; int gfal_check_fullendpoint_in_surl(const char *surl, GError **err); gboolean gfal_srm_surl_group_checker(gfal_srmv2_opt *opts, char **surls, GError **err); int gfal_srm_getTURLS_plugin(plugin_handle ch, const char *surl, char *buff_turl, int size_turl, char **reqtoken, GError **err); int gfal_srm_putTURLS_plugin(plugin_handle ch, const char *surl, char *buff_turl, int size_turl, char **reqtoken, GError **err); int gfal_srm_putdone(gfal_srmv2_opt *opts, const char *surl, const char *token, GError **err); void gfal_srm_report_error(char *errbuff, GError **err); gfal_srm_easy_t gfal_srm_ifce_easy_context(gfal_srmv2_opt *opts, const char *surl, GError **err); void gfal_srm_ifce_easy_context_release(gfal_srmv2_opt *opts, gfal_srm_easy_t easy); gfal2-v2.23.0/src/plugins/srm/gfal_srm_internal_ls.c000066400000000000000000000110321465240014500223560ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "gfal_srm_internal_ls.h" #include "gfal_srm_endpoint.h" /* * clear memory used by the internal srm_ifce items * required by the old design of srm_ifce * * */ void gfal_srm_ls_memory_management(struct srm_ls_input *input, struct srm_ls_output *output) { if (input) { // nothing } if (output) { gfal_srm_external_call.srm_srmv2_mdfilestatus_delete(output->statuses, 1); gfal_srm_external_call.srm_srm2__TReturnStatus_delete(output->retstatus); } } /* * concentrate the srm_ls logical in one point for stat, readdir, status, and access * * */ static int gfal_srm_ls_internal(srm_context_t context, struct srm_ls_input *input, struct srm_ls_output *output, GError **err) { GError *tmp_err = NULL; int ret = -1; if ((ret = gfal_srm_external_call.srm_ls(context, input, output)) < 0) { gfal_srm_report_error(context->errbuf, &tmp_err); ret = -1; } G_RETURN_ERR(ret, tmp_err, err); } static time_t gfal_srm_local_timezone_offset(void) { time_t epoch = 0; return -mktime(gmtime(&epoch)); } static void gfal_srm_adjust_time(struct stat *buf) { tzset(); time_t tz_offset = gfal_srm_local_timezone_offset(); // Do not apply offset if the value is 0 if (buf->st_ctime != 0) buf->st_ctime += tz_offset; if (buf->st_atime != 0) buf->st_atime += tz_offset; if (buf->st_mtime != 0) buf->st_mtime += tz_offset; } int gfal_statG_srmv2__generic_internal(srm_context_t context, struct stat *buf, TFileLocality *loc, const char *surl, GError **err) { g_return_val_err_if_fail(context && surl && buf #if !defined(_DARWIN_FEATURE_ONLY_64_BIT_INODE) && (sizeof(struct stat) == sizeof(struct stat64)) #endif , -1, err, "[gfal_statG_srmv2_generic_internal] Invalid args handle/endpoint or invalid stat struct size"); GError *tmp_err = NULL; struct srm_ls_input input; struct srm_ls_output output; struct srmv2_mdfilestatus *srmv2_mdstatuses = NULL; const int nb_request = 1; int ret = -1; char *tab_surl[] = {(char *) surl, NULL}; input.nbfiles = nb_request; input.surls = tab_surl; input.numlevels = 0; input.offset = 0; input.count = 0; ret = gfal_srm_ls_internal(context, &input, &output, &tmp_err); if (ret >= 0) { srmv2_mdstatuses = output.statuses; if (srmv2_mdstatuses->status != 0) { gfal2_set_error(&tmp_err, gfal2_get_plugin_srm_quark(), srmv2_mdstatuses->status, __func__, "Error reported from srm_ifce : %d %s", srmv2_mdstatuses->status, srmv2_mdstatuses->explanation); ret = -1; } else { memcpy(buf, &(srmv2_mdstatuses->stat), sizeof(struct stat)); if (loc) *loc = srmv2_mdstatuses->locality; errno = 0; ret = 0; // SRM returns the time in UTC gfal_srm_adjust_time(buf); } } gfal_srm_ls_memory_management(&input, &output); G_RETURN_ERR(ret, tmp_err, err); } int gfal_srm_cache_stat_add(plugin_handle ch, const char *surl, const struct stat *value, const TFileLocality *loc) { char buff_key[GFAL_URL_MAX_LEN]; gfal_srmv2_opt *opts = (gfal_srmv2_opt *) ch; gfal_srm_construct_key(surl, GFAL_SRM_LSTAT_PREFIX, buff_key, GFAL_URL_MAX_LEN); struct extended_stat xstat; xstat.stat = *value; xstat.locality = *loc; gsimplecache_add_item_kstr(opts->cache, buff_key, &xstat); return 0; } void gfal_srm_cache_stat_remove(plugin_handle ch, const char *surl) { char buff_key[GFAL_URL_MAX_LEN]; gfal_srmv2_opt *opts = (gfal_srmv2_opt *) ch; gfal_srm_construct_key(surl, GFAL_SRM_LSTAT_PREFIX, buff_key, GFAL_URL_MAX_LEN); gsimplecache_remove_kstr(opts->cache, buff_key); } gfal2-v2.23.0/src/plugins/srm/gfal_srm_internal_ls.h000066400000000000000000000022651465240014500223730ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "gfal_srm.h" #include "gfal_srm_internal_layer.h" struct extended_stat { struct stat stat; TFileLocality locality; }; int gfal_statG_srmv2__generic_internal(srm_context_t context, struct stat *buf, TFileLocality *loc, const char *surl, GError **err); int gfal_srm_cache_stat_add(plugin_handle ch, const char *surl, const struct stat *value, const TFileLocality *loc); void gfal_srm_cache_stat_remove(plugin_handle ch, const char *surl); gfal2-v2.23.0/src/plugins/srm/gfal_srm_mkdir.c000066400000000000000000000100561465240014500211570ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "gfal_srm.h" #include "gfal_srm_namespace.h" #include "gfal_srm_internal_layer.h" #include "gfal_srm_endpoint.h" static int gfal_mkdir_srmv2_internal(srm_context_t context, const char *path, mode_t mode, GError **err) { struct srm_mkdir_input mkdir_input; int res = -1; GError *tmp_err = NULL; errno = 0; mkdir_input.dir_name = (char *) path; res = gfal_srm_external_call.srm_mkdir(context, &mkdir_input); if (res < 0) { gfal_srm_report_error(context->errbuf, &tmp_err); res = -1; } G_RETURN_ERR(res, tmp_err, err); } int gfal_srm_mkdir_recG(plugin_handle ch, const char *surl, mode_t mode, GError **err) { g_return_val_err_if_fail(ch && surl, EINVAL, err, "[gfal_srm_mkdir_recG] Invalid value handle and/or surl"); GError *tmp_err = NULL; gfal_srmv2_opt *opts = (gfal_srmv2_opt *) ch; int ret = -1; gfal2_log(G_LOG_LEVEL_DEBUG, " -> [gfal_srm_mkdir_recG] "); gfal_srm_easy_t easy = gfal_srm_ifce_easy_context(opts, surl, &tmp_err); if (easy != NULL) { gfal2_log(G_LOG_LEVEL_DEBUG, " [gfal_srm_mkdir_recG] try to create directory %s", surl); struct stat st; ret = gfal_statG_srmv2_internal(easy->srm_context, &st, NULL, easy->path, &tmp_err); if (ret == 0) { if (!S_ISDIR(st.st_mode)) { gfal2_set_error(&tmp_err, gfal2_get_plugin_srm_quark(), ENOTDIR, __func__, "%s it is a file", surl); ret = -1; } else { ret = 0; } } else { g_clear_error(&tmp_err); ret = gfal_mkdir_srmv2_internal(easy->srm_context, easy->path, mode, &tmp_err); if (ret < 0 && tmp_err->code == EEXIST) { ret = 0; g_clear_error(&tmp_err); } } } gfal_srm_ifce_easy_context_release(opts, easy); gfal2_log(G_LOG_LEVEL_DEBUG, " [gfal_srm_mkdir_recG] <-"); G_RETURN_ERR(ret, tmp_err, err); } int gfal_srm_mkdirG(plugin_handle ch, const char *surl, mode_t mode, gboolean pflag, GError **err) { g_return_val_err_if_fail(ch && surl, EINVAL, err, "[gfal_srm_mkdirG] Invalid value handle and/or surl"); GError *tmp_err = NULL; gfal_srmv2_opt *opts = (gfal_srmv2_opt *) ch; int ret = -1; if (pflag) { // pflag set : behavior similar to mkdir -p requested ret = gfal_srm_mkdir_recG(ch, surl, mode, &tmp_err); } else { gfal2_log(G_LOG_LEVEL_DEBUG, " -> [gfal_srm_mkdirG] "); gfal_srm_easy_t easy = gfal_srm_ifce_easy_context(opts, surl, &tmp_err); if (easy != NULL) { gfal2_log(G_LOG_LEVEL_DEBUG, " [gfal_srm_mkdirG] try to create directory %s", surl); struct stat st; ret = gfal_statG_srmv2_internal(easy->srm_context, &st, NULL, easy->path, &tmp_err); if (ret == 0) { gfal2_set_error(&tmp_err, gfal2_get_plugin_srm_quark(), EEXIST, __func__, "directory already exist"); ret = -1; } else { g_clear_error(&tmp_err); ret = gfal_mkdir_srmv2_internal(easy->srm_context, easy->path, mode, &tmp_err); } } gfal_srm_ifce_easy_context_release(opts, easy); gfal2_log(G_LOG_LEVEL_DEBUG, " [gfal_srm_mkdirG] <-"); } G_RETURN_ERR(ret, tmp_err, err); } gfal2-v2.23.0/src/plugins/srm/gfal_srm_namespace.h000066400000000000000000000052441465240014500220150ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #pragma once #include #include #include "gfal_srm.h" int gfal_srm_accessG(plugin_handle handle, const char* surl, int mode, GError** err); int gfal_srm_checksumG_fallback(plugin_handle handle, const char* url, const char* check_type, char * checksum_buffer, size_t buffer_length, off_t start_offset, size_t data_length, gboolean turl_fallback, GError ** err); int gfal_srm_checksumG(plugin_handle handle, const char* url, const char* check_type, char * checksum_buffer, size_t buffer_length, off_t start_offset, size_t data_length, GError ** err); int gfal_srm_chmodG(plugin_handle, const char *, mode_t, GError** err); ssize_t gfal_srm_getxattrG(plugin_handle handle, const char* path, const char* name, void* buff, size_t s_buff, GError** err); ssize_t gfal_srm_status_getxattrG(plugin_handle handle, const char* path, const char* name, void* buff, size_t s_buff, GError** err); ssize_t gfal_srm_listxattrG(plugin_handle handle, const char* path, char* list, size_t size, GError** err); int gfal_srm_mkdir_recG(plugin_handle ch, const char* surl, mode_t mode, GError** err); int gfal_srm_mkdirG(plugin_handle handle, const char* surl, mode_t mode, gboolean pflag, GError** err); int gfal_srm_renameG(plugin_handle plugin_data, const char * oldurl, const char * urlnew, GError** err); int gfal_srm_unlinkG(plugin_handle ch, const char * path, GError** err); int gfal_srm_unlink_listG(plugin_handle ch, int nbfiles, const char* const* paths, GError** err); int gfal_srm_rmdirG(plugin_handle handle, const char* surl, GError** err); int gfal_srm_statG(plugin_handle handle, const char* surl, struct stat* buf, GError** err); int gfal_statG_srmv2_internal(srm_context_t context, struct stat* buf, TFileLocality* loc, const char* surl, GError** err); gfal2-v2.23.0/src/plugins/srm/gfal_srm_open.c000066400000000000000000000136071465240014500210170ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include "gfal_srm_internal_layer.h" #include "gfal_srm.h" typedef struct _gfal_srm_handle_open { gfal_file_handle internal_handle; char surl[GFAL_URL_MAX_LEN]; srm_req_type req_type; char *reqtoken; } *gfal_srm_handle_open; static gfal_file_handle gfal_srm_file_handle_create(gfal_file_handle fh, char *surl, char *reqtoken, srm_req_type req_type) { if (fh == NULL) return NULL; gfal_srm_handle_open sh = g_new(struct _gfal_srm_handle_open, 1); sh->internal_handle = fh; g_strlcpy(sh->surl, surl, GFAL_URL_MAX_LEN); sh->reqtoken = reqtoken; sh->req_type = req_type; return gfal_file_handle_new(gfal_srm_getName(), sh); } static gfal_file_handle gfal_srm_file_handle_map(gfal_file_handle fh) { gfal_srm_handle_open fd = gfal_file_handle_get_fdesc(fh); return fd->internal_handle; } static void gfal_srm_file_handle_delete(gfal_file_handle fh) { g_free(gfal_file_handle_get_fdesc(fh)); gfal_file_handle_delete(fh); } /* * open function for the srm plugin */ gfal_file_handle gfal_srm_openG(plugin_handle ch, const char *path, int flag, mode_t mode, GError **err) { gfal_file_handle ret = NULL; gfal_srmv2_opt *opts = (gfal_srmv2_opt *) ch; GError *tmp_err = NULL; char *p = (char *) path; char turl[GFAL_URL_MAX_LEN]; char *reqtoken = NULL; srm_req_type req_type; int tmp_ret; gfal2_log(G_LOG_LEVEL_DEBUG, " %s ->", __func__); // If endpoint is Castor, disable stat on open for GridFTP if (is_castor_endpoint(ch, path)) { gfal2_set_opt_boolean(opts->handle, "GRIDFTP PLUGIN", "SESSION_REUSE", FALSE, NULL); gfal2_set_opt_boolean(opts->handle, "GRIDFTP PLUGIN", "STAT_ON_OPEN", FALSE, NULL); } if (flag & O_CREAT) { // create turl if file is not existing else get one for this file gfal2_log(G_LOG_LEVEL_DEBUG, " SRM PUT mode", __func__); tmp_ret = gfal_srm_putTURLS_plugin(ch, p, turl, GFAL_URL_MAX_LEN, &reqtoken, &tmp_err); req_type = SRM_PUT; } else { gfal2_log(G_LOG_LEVEL_DEBUG, " SRM GET mode", __func__); tmp_ret = gfal_srm_getTURLS_plugin(ch, p, turl, GFAL_URL_MAX_LEN, &reqtoken, &tmp_err); req_type = SRM_GET; } if (tmp_ret == 0) { gfal2_log(G_LOG_LEVEL_DEBUG, " SRM RESOLUTION : %s -> %s ", path, turl); ret = gfal_plugin_openG(opts->handle, turl, flag, mode, &tmp_err); ret = gfal_srm_file_handle_create(ret, p, g_strdup(reqtoken), req_type); } g_free(reqtoken); if (tmp_err) { gfal2_propagate_prefixed_error(err, tmp_err, __func__); } return ret; } /* * read function for the srm plugin */ ssize_t gfal_srm_readG(plugin_handle ch, gfal_file_handle fd, void *buff, size_t count, GError **err) { gfal_srmv2_opt *opts = (gfal_srmv2_opt *) ch; GError *tmp_err = NULL; int ret = gfal_plugin_readG(opts->handle, gfal_srm_file_handle_map(fd), buff, count, &tmp_err); if (tmp_err) gfal2_propagate_prefixed_error(err, tmp_err, __func__); return ret; } /* * pread function for the srm plugin */ ssize_t gfal_srm_preadG(plugin_handle ch, gfal_file_handle fd, void *buff, size_t count, off_t offset, GError **err) { gfal_srmv2_opt *opts = (gfal_srmv2_opt *) ch; GError *tmp_err = NULL; int ret = gfal_plugin_preadG(opts->handle, gfal_srm_file_handle_map(fd), buff, count, offset, &tmp_err); if (tmp_err) gfal2_propagate_prefixed_error(err, tmp_err, __func__); return ret; } /* * write function for the srm plugin */ ssize_t gfal_srm_writeG(plugin_handle ch, gfal_file_handle fd, const void *buff, size_t count, GError **err) { gfal_srmv2_opt *opts = (gfal_srmv2_opt *) ch; GError *tmp_err = NULL; int ret = gfal_plugin_writeG(opts->handle, gfal_srm_file_handle_map(fd), (void *) buff, count, &tmp_err); if (tmp_err) gfal2_propagate_prefixed_error(err, tmp_err, __func__); return ret; } /* * lseek function for the srm plugin */ off_t gfal_srm_lseekG(plugin_handle ch, gfal_file_handle fd, off_t offset, int whence, GError **err) { gfal_srmv2_opt *opts = (gfal_srmv2_opt *) ch; GError *tmp_err = NULL; int ret = gfal_plugin_lseekG(opts->handle, gfal_srm_file_handle_map(fd), offset, whence, &tmp_err); if (tmp_err) gfal2_propagate_prefixed_error(err, tmp_err, __func__); return ret; } int gfal_srm_closeG(plugin_handle ch, gfal_file_handle fh, GError **err) { gfal_srmv2_opt *opts = (gfal_srmv2_opt *) ch; GError *tmp_err = NULL; int ret = gfal_plugin_closeG(opts->handle, gfal_srm_file_handle_map(fh), &tmp_err); if (ret == 0) { gfal_srm_handle_open sh = (gfal_srm_handle_open) gfal_file_handle_get_fdesc(fh); if (sh->req_type == SRM_PUT) ret = gfal_srm_putdone(opts, sh->surl, sh->reqtoken, &tmp_err); else ret = gfal_srmv2_release_fileG(ch, sh->surl, sh->reqtoken, &tmp_err); g_free(sh->reqtoken); gfal_srm_file_handle_delete(fh); } if (tmp_err) gfal2_propagate_prefixed_error(err, tmp_err, __func__); return ret; } gfal2-v2.23.0/src/plugins/srm/gfal_srm_open.h000066400000000000000000000026631465240014500210240ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #pragma once #include #include #include "gfal_srm.h" gfal_file_handle gfal_srm_openG(plugin_handle, const char *path, int flag, mode_t mode, GError **); ssize_t gfal_srm_readG(plugin_handle, gfal_file_handle fd, void *buff, size_t count, GError **); ssize_t gfal_srm_preadG(plugin_handle ch, gfal_file_handle fd, void *buff, size_t count, off_t offset, GError **err); ssize_t gfal_srm_writeG(plugin_handle, gfal_file_handle fd, const void *buff, size_t count, GError **); int gfal_srm_closeG(plugin_handle, gfal_file_handle fd, GError **); /* * lseek function for the srm plugin */ off_t gfal_srm_lseekG(plugin_handle ch, gfal_file_handle fd, off_t offset, int whence, GError **err); gfal2-v2.23.0/src/plugins/srm/gfal_srm_opendir.c000066400000000000000000000114261465240014500215130ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include "gfal_srm.h" #include "gfal_srm_opendir.h" #include "gfal_srm_namespace.h" #include "gfal_srm_endpoint.h" #include "gfal_srm_internal_layer.h" // Replaces the first occurence of ';' with '/0', so // we have a valid surl. Returns a pointer // to the beginning of the additional parameters, if any static char *_strip_parameters(char *surl) { char *first_semicolon = strchr(surl, ';'); if (first_semicolon) { *first_semicolon = '\0'; return first_semicolon + 1; } else { return NULL; } } // Parse any possible additional parameters passed in the URL and set the // corresponding values in h // parameters should be already in the form of key=value;key=value static void _parse_opendir_parameters(char *parameters, gfal_srm_opendir_handle h) { if (parameters) { char *saveptr = NULL; char *pair = strtok_r(parameters, ";", &saveptr); if (pair) { do { char *key = pair; char *value = strchr(pair, '='); if (value) { *value = '\0'; ++value; if (strcasecmp("offset", key) == 0) { h->chunk_offset = atoi(value); } else if (strcasecmp("count", key) == 0) { h->chunk_size = atoi(value); } } } while ((pair = strtok_r(NULL, ";", &saveptr))); } } else { h->chunk_offset = 0; h->chunk_size = 0; } if (h->chunk_offset || h->chunk_size) { h->is_chunked_listing = 1; } } static gfal_file_handle gfal_srm_opendir_internal(gfal_srm_easy_t easy, GError **err) { // As extra parameters may be passed separated with ';', // we need to remove those from the surl, and then process them char *real_path = g_strdup(easy->path); char *parameters = _strip_parameters(real_path); GError *tmp_err = NULL; gfal_file_handle resu = NULL; struct stat st; int exist = gfal_statG_srmv2_internal(easy->srm_context, &st, NULL, real_path, &tmp_err); if (exist == 0) { if (S_ISDIR(st.st_mode)) { gfal_srm_opendir_handle h = g_new0(struct _gfal_srm_opendir_handle, 1); h->easy = easy; char *p = stpncpy(h->surl, real_path, GFAL_URL_MAX_LEN); // remove trailing '/' for (--p; *p == '/'; --p) { *p = '\0'; } _parse_opendir_parameters(parameters, h); resu = gfal_file_handle_new2(gfal_srm_getName(), (gpointer) h, NULL, real_path); } else { gfal2_set_error(&tmp_err, gfal2_get_plugin_srm_quark(), ENOTDIR, __func__, "srm-plugin: %s is not a directory, impossible to list content", easy->path); } } if (tmp_err) gfal2_propagate_prefixed_error(err, tmp_err, __func__); g_free(real_path); return resu; } gfal_file_handle gfal_srm_opendirG(plugin_handle handle, const char *surl, GError **err) { g_return_val_err_if_fail(handle && surl, NULL, err, "[gfal_srm_opendirG] Invalid args"); gfal_srmv2_opt *opts = (gfal_srmv2_opt *) handle; gfal_file_handle resu = NULL; GError *tmp_err = NULL; gfal_srm_easy_t easy = gfal_srm_ifce_easy_context(opts, surl, &tmp_err); if (easy) { resu = gfal_srm_opendir_internal(easy, &tmp_err); } if (tmp_err) { gfal2_propagate_prefixed_error(err, tmp_err, __func__); } return resu; } int gfal_srm_closedirG(plugin_handle handle, gfal_file_handle fh, GError **err) { g_return_val_err_if_fail(handle && fh, -1, err, "[gfal_srm_opendirG] Invalid args"); gfal_srmv2_opt *opts = (gfal_srmv2_opt *) handle; gfal_srm_opendir_handle oh = (gfal_srm_opendir_handle)gfal_file_handle_get_fdesc(fh); gfal_srm_external_call.srm_srmv2_mdfilestatus_delete(oh->srm_file_statuses, 1); gfal_srm_ifce_easy_context_release(opts, oh->easy); g_free(oh); gfal_file_handle_delete(fh); return 0; } gfal2-v2.23.0/src/plugins/srm/gfal_srm_opendir.h000066400000000000000000000037071465240014500215230ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #pragma once #include "gfal_srm_internal_layer.h" #include #include #include #include #include typedef struct _gfal_srm_opendir_handle { gfal_srm_easy_t easy; // SURL we are listing char surl[GFAL_URL_MAX_LEN]; // Buffer where to store read entries, and returned by readdir calls struct dirent dirent_buffer; // Will be set to 1 by gfal_srm_readdirppG if the directory was too big // and we decided to read in chunks int is_chunked_listing; // These two are used internally in chunk listing, to keep track int chunk_offset; int chunk_size; // Array of file statuses as returned by srm-ifce struct srmv2_mdfilestatus *srm_file_statuses; // Array position inside srm_file_statuses while iterating int response_index; } *gfal_srm_opendir_handle; gfal_file_handle gfal_srm_opendirG(plugin_handle handle, const char *path, GError **err); int gfal_srm_closedirG(plugin_handle handle, gfal_file_handle fh, GError **err); struct dirent *gfal_srm_readdirG(plugin_handle handle, gfal_file_handle fh, GError **err); struct dirent *gfal_srm_readdirppG(plugin_handle ch, gfal_file_handle fh, struct stat *st, GError **err); gfal2-v2.23.0/src/plugins/srm/gfal_srm_readdir.c000066400000000000000000000202551465240014500214650ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #define GFAL_FILENAME_MAX FILENAME_MAX #include #include #include "gfal_srm_namespace.h" #include "gfal_srm_opendir.h" #include "gfal_srm_internal_layer.h" #include "gfal_srm_internal_ls.h" #if defined(_DARWIN_FEATURE_ONLY_64_BIT_INODE) # define stat64 stat #endif /** * Casts a 64 bits stat into whatever stat type we are using in this compilation unit. * Intended for cases where we compile using 32 bits size types, which I don't believe to be * the case any more, but just in case... */ static void gfal_srm_stat64_to_stat(const struct stat64 *st64, struct stat *st) { if (sizeof(struct stat64) == sizeof(struct stat)) memcpy(st, st64, sizeof(*st)); else { st->st_dev = (dev_t) st64->st_dev; st->st_ino = (ino_t) st64->st_ino; st->st_mode = (mode_t) st64->st_mode; st->st_nlink = (nlink_t) st64->st_nlink; st->st_uid = (uid_t) st64->st_uid; st->st_gid = (gid_t) st64->st_gid; st->st_rdev = (dev_t) st64->st_rdev; st->st_size = (off_t) st64->st_size; st->st_blksize = (blkcnt_t) st64->st_blksize; st->st_blocks = (blkcnt_t) st64->st_blocks; st->st_atime = (time_t) st64->st_atime; st->st_mtime = (time_t) st64->st_mtime; st->st_ctime = (time_t) st64->st_ctime; } } /** * Converts a SRM status into a dirent + struct stat * Returns, for convenience, the same pointer passed as dir_ent */ static struct dirent *gfal_srm_readdir_convert_result(plugin_handle ch, const char *parent_surl, const struct srmv2_mdfilestatus *srm_status, struct dirent *dir_ent, struct stat *st, GError **err) { char buff_surlfull[GFAL_URL_MAX_LEN]; char *p = strrchr(srm_status->surl, '/'); if (p != NULL) { g_strlcpy(buff_surlfull, parent_surl, GFAL_URL_MAX_LEN); g_strlcat(buff_surlfull, p, GFAL_URL_MAX_LEN); dir_ent->d_reclen = g_strlcpy(dir_ent->d_name, p + 1, GFAL_URL_MAX_LEN); } else { g_strlcpy(buff_surlfull, parent_surl, GFAL_URL_MAX_LEN); g_strlcat(buff_surlfull, "/", GFAL_URL_MAX_LEN); g_strlcat(buff_surlfull, p, GFAL_URL_MAX_LEN); dir_ent->d_reclen = g_strlcpy(dir_ent->d_name, srm_status->surl, GFAL_URL_MAX_LEN); } if (S_ISDIR(srm_status->stat.st_mode)) dir_ent->d_type = DT_DIR; else if (S_ISLNK(srm_status->stat.st_mode)) dir_ent->d_type = DT_LNK; else dir_ent->d_type = DT_REG; gfal_srm_stat64_to_stat(&srm_status->stat, st); // Stores cache information gfal_srm_cache_stat_add(ch, buff_surlfull, st, &srm_status->locality); return dir_ent; } /** * Wraps the actual call to srm-ifce */ static int gfal_srm_readdir_internal(plugin_handle ch, gfal_srm_opendir_handle oh, GError **err) { g_return_val_err_if_fail(ch && oh, -1, err, "[gfal_srmv2_opendir_internal] invaldi args"); GError *tmp_err = NULL; int resu = -1; struct srm_ls_input input; struct srm_ls_output output; struct srmv2_mdfilestatus *srmv2_mdstatuses = NULL; int ret = -1; memset(&input, 0, sizeof(input)); memset(&output, 0, sizeof(output)); char *tab_surl[] = {oh->surl, NULL}; input.nbfiles = 1; input.surls = tab_surl; input.numlevels = 1; input.count = oh->chunk_size; // Mind that srm_ls may - or may not - modify the value pointed by input.offset int offset_buffer = oh->chunk_offset; input.offset = &offset_buffer; oh->response_index = 0; ret = gfal_srm_external_call.srm_ls(oh->easy->srm_context, &input, &output); if (ret >= 0) { srmv2_mdstatuses = output.statuses; if (srmv2_mdstatuses[0].status != 0) { gfal2_set_error(err, gfal2_get_plugin_srm_quark(), srmv2_mdstatuses->status, __func__, "Error reported from srm_ifce : %d %s", srmv2_mdstatuses->status, srmv2_mdstatuses->explanation); resu = -1; } else { oh->srm_file_statuses = &srmv2_mdstatuses[0]; resu = 0; } } else { gfal_srm_report_error(oh->easy->srm_context->errbuf, &tmp_err); resu = -1; } gfal_srm_external_call.srm_srm2__TReturnStatus_delete(output.retstatus); G_RETURN_ERR(resu, tmp_err, err); } /** * Wraps the SRM request. * Request each chunks, then iterates through the responses as readdir is called */ static struct dirent *gfal_srm_readdir_pipeline(plugin_handle ch, gfal_srm_opendir_handle oh, struct stat *st, GError **err) { GError *tmp_err = NULL; // Nothing yet, so get the bulk if (oh->srm_file_statuses == NULL) { gfal_srm_readdir_internal(ch, oh, &tmp_err); if (tmp_err) { gfal2_propagate_prefixed_error(err, tmp_err, __func__); return NULL; } } // Empty directory if (oh->srm_file_statuses == NULL || oh->srm_file_statuses->nbsubpaths == 0) { return NULL; } // Done here if (oh->response_index >= oh->srm_file_statuses->nbsubpaths) { return NULL; } // Iterate and return statuses struct dirent *ret = gfal_srm_readdir_convert_result(ch, oh->surl, &oh->srm_file_statuses->subpaths[oh->response_index], &oh->dirent_buffer, st, &tmp_err); oh->response_index++; // If chunk listing, and the index passed the last entry in the buffer, // release, and prepare next bulk if (oh->is_chunked_listing && oh->response_index >= oh->chunk_size) { oh->chunk_offset += oh->chunk_size; gfal_srm_external_call.srm_srmv2_mdfilestatus_delete(oh->srm_file_statuses, 1); oh->srm_file_statuses = NULL; } return ret; } /** * Only read. * SRM listing returns the file stat anyway, so wrap Read + Stat and discard the stat */ struct dirent *gfal_srm_readdirG(plugin_handle ch, gfal_file_handle fh, GError **err) { g_return_val_err_if_fail(ch && fh, NULL, err, "[gfal_srm_readdirG] Invalid args"); struct stat _; // Ignore this return gfal_srm_readdirppG(ch, fh, &_, err); } /** * Read + Stat */ struct dirent *gfal_srm_readdirppG(plugin_handle ch, gfal_file_handle fh, struct stat *st, GError **err) { g_return_val_err_if_fail(ch && fh, NULL, err, "[gfal_srm_readdirppG] Invalid args"); GError *tmp_err = NULL; struct dirent *ret = NULL; gfal_srm_opendir_handle oh = (gfal_srm_opendir_handle)gfal_file_handle_get_fdesc(fh); ret = gfal_srm_readdir_pipeline(ch, oh, st, &tmp_err); // Directory too big, so prepare to read in chunks and delegate if (tmp_err && tmp_err->code == EFBIG) { // If we already tried, abort! if (oh->is_chunked_listing) { gfal2_propagate_prefixed_error_extended(err, tmp_err, __func__, "EFBIG received when already trying chunk listing"); return NULL; } // Prepare for chunk listing, and re-issue g_clear_error(&tmp_err); oh->is_chunked_listing = 1; oh->chunk_offset = 0; oh->chunk_size = 1000; oh->response_index = 0; gfal2_log(G_LOG_LEVEL_WARNING, "EFBIG while listing SRM directory, trying with chunk listing of size %d", oh->chunk_size); ret = gfal_srm_readdir_pipeline(ch, oh, st, &tmp_err); if (tmp_err) gfal2_propagate_prefixed_error_extended(err, tmp_err, __func__, "Failed when attempting chunk listing"); } // Just an error else if (tmp_err) { gfal2_propagate_prefixed_error(err, tmp_err, __func__); } return ret; } gfal2-v2.23.0/src/plugins/srm/gfal_srm_rename.c000066400000000000000000000043011465240014500213140ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "gfal_srm.h" #include "gfal_srm_endpoint.h" #include "gfal_srm_internal_ls.h" #include "gfal_srm_namespace.h" #include "gfal_srm_url_check.h" #include static int gfal_srm_rename_internal_srmv2(srm_context_t context, const char *src, const char *dst, GError **err) { GError *tmp_err = NULL; int ret = -1; struct srm_mv_input input; input.from = (char *) src; input.to = (char *) dst; ret = gfal_srm_external_call.srm_mv(context, &input); if (ret != 0) { gfal_srm_report_error(context->errbuf, &tmp_err); ret = -1; } G_RETURN_ERR(ret, tmp_err, err); } int gfal_srm_renameG(plugin_handle plugin_data, const char *oldurl, const char *urlnew, GError **err) { g_return_val_err_if_fail(plugin_data && oldurl && urlnew, EINVAL, err, "[gfal_srm_renameG] Invalid value handle and/or surl"); GError *tmp_err = NULL; gfal_srmv2_opt *opts = (gfal_srmv2_opt *) plugin_data; int ret = -1; gfal_srm_easy_t easy = gfal_srm_ifce_easy_context(opts, oldurl, &tmp_err); if (easy != NULL) { gfal_srm_cache_stat_remove(plugin_data, oldurl); char *decodednew = gfal2_srm_get_decoded_path(urlnew); ret = gfal_srm_rename_internal_srmv2(easy->srm_context, easy->path, decodednew, &tmp_err); g_free(decodednew); } gfal_srm_ifce_easy_context_release(opts, easy); if (ret != 0) gfal2_propagate_prefixed_error(err, tmp_err, __func__); return ret; } gfal2-v2.23.0/src/plugins/srm/gfal_srm_request.c000066400000000000000000000044231465240014500215420ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "gfal_srm_request.h" #include "gfal_srm_internal_layer.h" static gchar *get_spacetoken_from_config(gfal_srmv2_opt *handle) { GError *error = NULL; gchar *stoken = gfal2_get_opt_string(handle->handle, srm_config_group, srm_spacetokendesc, &error); if (error) g_error_free(error); // Do not really care about this return stoken; } gfal_srm_params_t gfal_srm_params_new(gfal_srmv2_opt *handle) { gfal_srm_params_t res = g_new0(struct _gfal_srm_params, 1); res->protocols = srm_get_turls_sup_protocol(handle->handle); res->proto_version = handle->srm_proto_type; res->spacetokendesc = get_spacetoken_from_config(handle); res->file_size = 0; return res; } void gfal_srm_params_free(gfal_srm_params_t params) { if (params) { g_free(params->spacetokendesc); g_strfreev(params->protocols); g_free(params); } } char **gfal_srm_params_get_protocols(gfal_srm_params_t params) { return params->protocols; } gchar *gfal_srm_params_get_spacetoken(gfal_srm_params_t params) { return params->spacetokendesc; } void gfal_srm_params_set_spacetoken(gfal_srm_params_t params, const char *spacetoken) { g_free(params->spacetokendesc); params->spacetokendesc = g_strdup(spacetoken); } void gfal_srm_params_set_protocols(gfal_srm_params_t params, char **protocols) { if (params->protocols) g_strfreev(params->protocols); params->protocols = protocols; } void gfal_srm_params_set_size(gfal_srm_params_t params, size_t file_size) { params->file_size = file_size; } gfal2-v2.23.0/src/plugins/srm/gfal_srm_request.h000066400000000000000000000026601465240014500215500ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #pragma once #ifndef GFAL_SRM_REQUEST_H #define GFAL_SRM_REQUEST_H #include #include "gfal_srm.h" /* * Next gen request srm system for gfal2 * Come with srm-ifce 2.0 * */ gfal_srm_params_t gfal_srm_params_new(gfal_srmv2_opt *handle); void gfal_srm_params_free(gfal_srm_params_t params); char **gfal_srm_params_get_protocols(gfal_srm_params_t params); void gfal_srm_params_set_protocols(gfal_srm_params_t params, char **protocols); gchar *gfal_srm_params_get_spacetoken(gfal_srm_params_t params); void gfal_srm_params_set_spacetoken(gfal_srm_params_t params, const char *spacetoken); void gfal_srm_params_set_size(gfal_srm_params_t params, size_t file_size); #endif /* GFAL_SRM_REQUEST_H */ gfal2-v2.23.0/src/plugins/srm/gfal_srm_rm.c000066400000000000000000000112311465240014500204630ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "gfal_srm.h" #include "gfal_srm_internal_layer.h" #include "gfal_srm_internal_ls.h" #include "gfal_srm_url_check.h" static int gfal_srm_rm_srmv2_isdir(srm_context_t context, const char *surl) { struct srm_ls_input input; struct srm_ls_output output; input.nbfiles = 1; input.surls = (char **) (&surl); input.numlevels = 0; input.offset = 0; input.count = 0; if (gfal_srm_external_call.srm_ls(context, &input, &output) < 0) return 0; int isdir = S_ISDIR(output.statuses->stat.st_mode); gfal_srm_external_call.srm_srmv2_mdfilestatus_delete(output.statuses, 1); gfal_srm_external_call.srm_srm2__TReturnStatus_delete(output.retstatus); return isdir; } static int gfal_srm_rm_srmv2_internal(srm_context_t context, int nbfiles, const char *const *surls, GError **errors) { struct srm_rm_input input; struct srm_rm_output output; int ret = -1, i; input.nbfiles = nbfiles; input.surls = (char **) (surls); ret = gfal_srm_external_call.srm_rm(context, &input, &output); if (ret == nbfiles) { ret = 0; struct srmv2_filestatus *statuses = output.statuses; for (i = 0; i < nbfiles; ++i) { int err_code = statuses[i].status; if (err_code != 0) { ret -= 1; // DPM returns an EINVAL when srm_rm is called over a directory // Check if the file is actually a directory, and override the return // code with EISDIR in that case if (err_code == EINVAL && gfal_srm_rm_srmv2_isdir(context, surls[i])) err_code = EISDIR; if (statuses[i].explanation) gfal2_set_error(&errors[i], gfal2_get_plugin_srm_quark(), err_code, __func__, "error reported from srm_ifce, %s", statuses[i].explanation); else gfal2_set_error(&errors[i], gfal2_get_plugin_srm_quark(), err_code, __func__, "error reported from srm_ifce, without explanation!"); } } gfal_srm_external_call.srm_srm2__TReturnStatus_delete( output.retstatus); gfal_srm_external_call.srm_srmv2_filestatus_delete(output.statuses, nbfiles); } else { gfal_srm_report_error(context->errbuf, &errors[0]); for (i = 1; i < nbfiles; ++i) errors[i] = g_error_copy(errors[0]); ret = -1; } return ret; } /** * * bindings of the unlink plugin call */ int gfal_srm_unlink_listG(plugin_handle ch, int nbfiles, const char *const *surls, GError **err) { GError *tmp_err = NULL; int ret = -1, i; if (!err) return -1; if (!ch || nbfiles < 0 || surls == NULL || *surls == NULL) { gfal2_set_error(&tmp_err, gfal2_get_plugin_srm_quark(), EINVAL, __func__, "incorrect args"); } else { gfal_srmv2_opt *opts = (gfal_srmv2_opt *) ch; gfal_srm_easy_t easy = gfal_srm_ifce_easy_context(opts, surls[0], &tmp_err); if (easy) { char *decoded[nbfiles]; for (i = 0; i < nbfiles; ++i) { gfal_srm_cache_stat_remove(ch, surls[i]); decoded[i] = gfal2_srm_get_decoded_path(surls[i]); } ret = gfal_srm_rm_srmv2_internal(easy->srm_context, nbfiles, (const char *const *) decoded, err); for (i = 0; i < nbfiles; ++i) { g_free(decoded[i]); } } gfal_srm_ifce_easy_context_release(opts, easy); } if (tmp_err) { for (i = 1; i < nbfiles; ++i) err[i] = g_error_copy(err[0]); } return ret; } int gfal_srm_unlinkG(plugin_handle ch, const char *path, GError **err) { int ret; GError *tmp_err = NULL; const char *paths[1] = {path}; ret = gfal_srm_unlink_listG(ch, 1, paths, &tmp_err); if (ret != 0) gfal2_propagate_prefixed_error(err, tmp_err, __func__); return ret; } gfal2-v2.23.0/src/plugins/srm/gfal_srm_rmdir.c000066400000000000000000000061401465240014500211650ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "gfal_srm.h" #include "gfal_srm_namespace.h" #include "gfal_srm_internal_layer.h" #include "gfal_srm_endpoint.h" #include "gfal_srm_internal_ls.h" static int gfal_srmv2_rmdir_internal(srm_context_t context, const char *surl, GError **err) { struct srm_rmdir_input rmdir_input; struct srm_rmdir_output rmdir_output; GError *tmp_err = NULL; int ret = -1; rmdir_input.recursive = 0; rmdir_input.surl = (char *) surl; if (gfal_srm_external_call.srm_rmdir(context, &rmdir_input, &rmdir_output) >= 0) { const int sav_errno = rmdir_output.statuses[0].status; if (sav_errno) { gfal2_set_error(&tmp_err, gfal2_get_plugin_srm_quark(), sav_errno, __func__, "Error report from the srm_ifce %s ", strerror(sav_errno)); ret = -1; } else { ret = 0; } gfal_srm_external_call.srm_srmv2_filestatus_delete(rmdir_output.statuses, 1); gfal_srm_external_call.srm_srm2__TReturnStatus_delete(rmdir_output.retstatus); } else { gfal_srm_report_error(context->errbuf, &tmp_err); ret = -1; } G_RETURN_ERR(ret, tmp_err, err); } int gfal_srm_rmdirG(plugin_handle ch, const char *surl, GError **err) { g_return_val_err_if_fail(ch && surl, EINVAL, err, "[gfal_srm_rmdirG] Invalid value handle and/or surl"); GError *tmp_err = NULL; gfal_srmv2_opt *opts = (gfal_srmv2_opt *) ch; int ret = -1; gfal_srm_easy_t easy = gfal_srm_ifce_easy_context(opts, surl, &tmp_err); if (easy != NULL) { struct stat st; gfal2_log(G_LOG_LEVEL_DEBUG, " [gfal_srm_rmdirG] try to delete directory %s", surl); ret = gfal_statG_srmv2_internal(easy->srm_context, &st, NULL, easy->path, &tmp_err); if (ret == 0) { if (S_ISDIR(st.st_mode)) { gfal_srm_cache_stat_remove(ch, surl); // invalidate cache entry ret = gfal_srmv2_rmdir_internal(easy->srm_context, easy->path, &tmp_err); } else { ret = -1; gfal2_set_error(&tmp_err, gfal2_get_plugin_srm_quark(), ENOTDIR, __func__, "This file is not a directory, impossible to use rmdir on it"); } } } gfal_srm_ifce_easy_context_release(opts, easy); if (ret != 0) gfal2_propagate_prefixed_error(err, tmp_err, __func__); return ret; } gfal2-v2.23.0/src/plugins/srm/gfal_srm_space.c000066400000000000000000000165441465240014500211540ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include "gfal_srm_space.h" #include "space/gfal2_space.h" static void json_putc(char *buff, size_t s_buff, char c, size_t *offset) { if (*offset < s_buff) buff[(*offset)++] = c; } static void json_puts(char *buff, size_t s_buff, const char *str, size_t *offset) { json_putc(buff, s_buff, '"', offset); const char *p = str; while (*p != '\0') { if (*p == '\\') { json_putc(buff, s_buff, '\\', offset); json_putc(buff, s_buff, '\\', offset); } else if (*p == '"') { json_putc(buff, s_buff, '\\', offset); json_putc(buff, s_buff, '"', offset); } else { json_putc(buff, s_buff, *p, offset); } ++p; } json_putc(buff, s_buff, '"', offset); } static ssize_t gfal_srm_space_list(srm_context_t context, char *buff, size_t s_buff, GError **err) { GError *tmp_err = NULL; struct srm_getspacetokens_input input; struct srm_getspacetokens_output output; ssize_t ret_size = 0; input.spacetokendesc = NULL; if (gfal_srm_external_call.srm_getspacetokens(context, &input, &output) < 0) { gfal_srm_report_error(context->errbuf, &tmp_err); if (tmp_err->code == EINVAL && !strstr(tmp_err->message, "[EINVAL] Invalid arguments")) { // This means there is no space token that belongs to the user, so we can just return empty g_error_free(tmp_err); tmp_err = NULL; } else { ret_size = -1; } } else { int i; size_t offset = 0; json_putc(buff, s_buff, '[', &offset); for (i = 0; i < output.nbtokens; ++i) { json_puts(buff, s_buff, output.spacetokens[i], &offset); json_putc(buff, s_buff, ',', &offset); } if (buff[offset - 1] == ',') --offset; // Strip last comma json_putc(buff, s_buff, ']', &offset); json_putc(buff, s_buff, '\0', &offset); ret_size = offset; } if (tmp_err != NULL) gfal2_propagate_prefixed_error(err, tmp_err, __func__); return ret_size; } static ssize_t gfal_srm_space_token_info(srm_context_t context, const char *token, char *buff, size_t s_buff, GError **err) { GError *tmp_err = NULL; struct srm_getspacemd_input input; struct srm_spacemd *spaces = NULL; ssize_t ret_size = 0; char *spacetokens[] = {(char *) token, NULL}; input.nbtokens = 1; input.spacetokens = spacetokens; if (gfal_srm_external_call.srm_getspacemd(context, &input, &spaces) < 0) { gfal_srm_report_error(context->errbuf, &tmp_err); ret_size = -1; } else { struct space_report report = {0}; uint64_t guaranteed = (uint64_t)spaces[0].guaranteedsize; report.spacetoken = spaces[0].spacetoken; report.owner = spaces[0].owner; report.total = (uint64_t)spaces[0].totalsize; report.largest_chunk = &guaranteed; report.free = (uint64_t)spaces[0].unusedsize; report.lifetime_assigned = &spaces[0].lifetimeassigned; report.lifetime_left = &spaces[0].lifetimeleft; report.retention = (enum space_retention_policy)spaces[0].retentionpolicy; report.latency = (enum space_latency)spaces[0].accesslatency; ret_size = gfal2_space_generate_json(&report, buff, s_buff); } if (tmp_err != NULL) gfal2_propagate_prefixed_error(err, tmp_err, __func__); return ret_size; } static ssize_t gfal_srm_space_token_descr_info(srm_context_t context, const char *token_desc, char *buff, size_t s_buff, GError **err) { GError *tmp_err = NULL; struct srm_getspacetokens_input input; struct srm_getspacetokens_output output; ssize_t ret_size = 0; input.spacetokendesc = (char *) token_desc; if (gfal_srm_external_call.srm_getspacetokens(context, &input, &output) < 0) { gfal_srm_report_error(context->errbuf, &tmp_err); ret_size = -1; } else { int i; size_t offset = 0; json_putc(buff, s_buff, '[', &offset); for (i = 0; i < output.nbtokens; ++i) { ssize_t s = gfal_srm_space_token_info(context, output.spacetokens[i], buff + offset, s_buff - offset, &tmp_err); if (s < 0) { ret_size = -1; break; } offset += s; json_putc(buff, s_buff, ',', &offset); } if (ret_size >= 0) { if (buff[offset - 1] == ',') --offset; // Strip last comma json_putc(buff, s_buff, ']', &offset); json_putc(buff, s_buff, '\0', &offset); ret_size = offset; } } if (tmp_err != NULL) gfal2_propagate_prefixed_error(err, tmp_err, __func__); return ret_size; } static ssize_t gfal_srm_space_property(srm_context_t context, const char *name, char *buff, size_t s_buff, GError **err) { if (name[0] == '\0') { return gfal_srm_space_list(context, buff, s_buff, err); } else if (strncmp(name, "token?", 6) == 0) { return gfal_srm_space_token_info(context, name + 6, buff, s_buff, err); } else if (strncmp(name, "description?", 12) == 0) { return gfal_srm_space_token_descr_info(context, name + 12, buff, s_buff, err); } else { gfal2_set_error(err, gfal2_get_plugin_srm_quark(), ENOATTR, __func__, "Unknown space token attribute %s", name); return -1; } } ssize_t gfal_srm_space_getxattrG(plugin_handle handle, const char *path, const char *name, void *buff, size_t s_buff, GError **err) { if (strncmp(name, GFAL_XATTR_SPACETOKEN, 10) != 0) { gfal2_set_error(err, gfal2_get_plugin_srm_quark(), ENOATTR, __func__, "Unknown attribute %s", name); return -1; } const char *subprop_name = name + 10; if (*subprop_name == '.') { ++subprop_name; } else if (subprop_name[0] != '\0') { gfal2_set_error(err, gfal2_get_plugin_srm_quark(), ENOATTR, __func__, "Unknown space token attribute %s", name); return -1; } GError *nested_error = NULL; ssize_t ret_size = 0; gfal_srmv2_opt *opts = (gfal_srmv2_opt *) handle; gfal_srm_easy_t easy = gfal_srm_ifce_easy_context(opts, path, &nested_error); if (easy) { ret_size = gfal_srm_space_property(easy->srm_context, subprop_name, (char *) buff, s_buff, &nested_error); } gfal_srm_ifce_easy_context_release(opts, easy); if (nested_error != NULL) gfal2_propagate_prefixed_error(err, nested_error, __func__); return ret_size; } gfal2-v2.23.0/src/plugins/srm/gfal_srm_space.h000066400000000000000000000016671465240014500211610ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #pragma once #include #include "gfal_srm_internal_layer.h" ssize_t gfal_srm_space_getxattrG(plugin_handle handle, const char *path, const char *name, void *buff, size_t s_buff, GError **err); gfal2-v2.23.0/src/plugins/srm/gfal_srm_stat.c000066400000000000000000000052621465240014500210270ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "gfal_srm.h" #include "gfal_srm_internal_ls.h" #include "gfal_srm_namespace.h" #include "gfal_srm_internal_layer.h" #include "gfal_srm_endpoint.h" int gfal_statG_srmv2_internal(srm_context_t context, struct stat *buf, TFileLocality *loc, const char *surl, GError **err) { return gfal_statG_srmv2__generic_internal(context, buf, loc, surl, err); } /* * stat call, for the srm interface stat and lstat are the same call !! the default behavior is similar to stat by default and ignore links * * */ int gfal_srm_statG(plugin_handle ch, const char *surl, struct stat *buf, GError **err) { g_return_val_err_if_fail(ch && surl && buf, -1, err, "[gfal_srm_statG] Invalid args in handle/surl/buf"); GError *tmp_err = NULL; int ret = -1; char key_buff[GFAL_URL_MAX_LEN]; gfal_srmv2_opt *opts = (gfal_srmv2_opt *) ch; TFileLocality loc; struct extended_stat xstat; // Try cache first gfal_srm_construct_key(surl, GFAL_SRM_LSTAT_PREFIX, key_buff, GFAL_URL_MAX_LEN); if (gsimplecache_take_one_kstr(opts->cache, key_buff, &xstat) == 0) { gfal2_log(G_LOG_LEVEL_DEBUG, " srm_statG -> value taken from the cache"); ret = 0; *buf = xstat.stat; } // Ask server otherwise else { gfal_srm_easy_t easy = gfal_srm_ifce_easy_context(opts, surl, &tmp_err); if (easy != NULL) { gfal2_log(G_LOG_LEVEL_DEBUG, " [gfal_srm_statG] try to stat file %s", surl); ret = gfal_statG_srmv2_internal(easy->srm_context, buf, &loc, easy->path, &tmp_err); if (ret == 0) { gfal2_log(G_LOG_LEVEL_DEBUG, " [gfal_srm_statG] store %s stat info in cache", surl); gfal_srm_cache_stat_add(ch, surl, buf, &loc); } } else { ret = -1; } gfal_srm_ifce_easy_context_release(opts, easy); } if (tmp_err) gfal2_propagate_prefixed_error(err, tmp_err, __func__); return ret; } gfal2-v2.23.0/src/plugins/srm/gfal_srm_url_check.c000066400000000000000000000067021465240014500220130ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include "gfal_srm_url_check.h" const char *surl_prefix = GFAL_PREFIX_SRM; gboolean srm_check_url(const char *surl) { gboolean res = FALSE; const size_t prefix_len = strlen(surl_prefix); size_t surl_len = strnlen(surl, GFAL_URL_MAX_LEN); if ((surl_len < GFAL_URL_MAX_LEN) && (strncmp(surl, surl_prefix, prefix_len) == 0)) { res = TRUE; } return res; } static gboolean srm_has_schema(const char *surl) { return strchr(surl, ':') != NULL; } gboolean plugin_url_check2(plugin_handle handle, gfal2_context_t context, const char *src, const char *dst, gfal_url2_check type) { g_return_val_if_fail(handle != NULL && src != NULL && dst != NULL, FALSE); gboolean src_srm = srm_check_url(src); gboolean dst_srm = srm_check_url(dst); gboolean src_valid_url = src_srm || srm_has_schema(src); gboolean dst_valid_url = dst_srm || srm_has_schema(dst); return (type == GFAL_FILE_COPY && ((src_srm && dst_valid_url) || (dst_srm && src_valid_url))); } static char *gfal2_srm_surl_find_path(gfal2_uri *parsed) { char *path; if (parsed->query != NULL && (path = strstr(parsed->query, "SFN=")) != NULL) { path += 4; } else { path = parsed->path; } return path; } char *gfal2_srm_get_decoded_path(const char *surl) { GError *err = NULL; gfal2_uri *parsed = gfal2_parse_uri(surl, &err); if (err != NULL) { g_clear_error(&err); return g_strdup(surl); } char *path = gfal2_srm_surl_find_path(parsed); gfal2_urldecode(path); char *decoded = g_strconcat("srm://", parsed->host, path, NULL); gfal2_free_uri(parsed); return decoded; } int gfal2_srm_surl_cmp(const char *surl1, const char *surl2) { int cmp; GError *error = NULL; gfal2_uri *parsed1 = NULL, *parsed2 = NULL; // Parse urls parsed1 = gfal2_parse_uri(surl1, &error); if (error) goto srm_surl_cmp_fallback; parsed2 = gfal2_parse_uri(surl2, &error); if (error) goto srm_surl_cmp_fallback; // If hosts are different, surls are different if (strcmp(parsed1->host, parsed2->host) != 0 || parsed1->port != parsed2->port) { cmp = -1; goto srm_surl_cmp_done; } // If no SFN is found, the path is as-is // Otherwise, the path is whatever is found in the SFN const char *sfn1 = gfal2_srm_surl_find_path(parsed1); const char *sfn2 = gfal2_srm_surl_find_path(parsed2); cmp = strcmp(sfn1, sfn2); goto srm_surl_cmp_done; // Fallback to raw strcmp srm_surl_cmp_fallback: g_error_free(error); cmp = strcmp(surl1, surl2); srm_surl_cmp_done: gfal2_free_uri(parsed1); gfal2_free_uri(parsed2); return cmp; } gfal2-v2.23.0/src/plugins/srm/gfal_srm_url_check.h000066400000000000000000000032161465240014500220150ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #pragma once #ifndef GFAL_COMMON_SRM_URL_CHECK_H #define GFAL_COMMON_SRM_URL_CHECK_H #include "gfal_srm.h" gboolean srm_check_url(const char *surl); /* * * implementation of the plugi_url_transfer_check for the srm plugin * Check if the srm plugin is able to handle a given type of URL. * * */ gboolean plugin_url_check2(plugin_handle handle, gfal2_context_t context, const char *src, const char *dst, gfal_url2_check type); /** * Comparing two SURLs is not a trivial strcmp, since a SURL like * srm://host:port/endpoint?SFN=blah * must be considered equivalent to * srm://host:port/blah **/ int gfal2_srm_surl_cmp(const char *surl1, const char *surl2); /** * Returns srm://host//url-decoded path from the surl, regardless of wether the path * is as SFN=/ form, or directly. * The return value needs to be g_free */ char *gfal2_srm_get_decoded_path(const char *surl); #endif /* GFAL_COMMON_SRM_URL_CHECK_H */ gfal2-v2.23.0/src/plugins/xrootd/000077500000000000000000000000001465240014500165475ustar00rootroot00000000000000gfal2-v2.23.0/src/plugins/xrootd/CMakeLists.txt000066400000000000000000000025721465240014500213150ustar00rootroot00000000000000cmake_minimum_required (VERSION 2.6) if (PLUGIN_XROOTD) find_package(XROOTD REQUIRED) include_directories( ${XROOTD_INCLUDE_DIR} ${JSONC_INCLUDE_DIRS} ${UUID_INCLUDE_DIRS} ) file (GLOB src_xrootd "*.cpp") add_library (plugin_xrootd MODULE ${src_xrootd}) add_library (plugin_xrootd_static STATIC ${src_xrootd}) target_link_libraries (plugin_xrootd ${GFAL2_PKG_LIBRARIES} gfal2_transfer ${XROOTD_LIBRARIES} ${JSONC_LIBRARIES} ${UUID_LIBRARIES} ) target_link_libraries(plugin_xrootd_static ${GFAL2_PKG_LIBRARIES} gfal2_transfer ${XROOTD_LIBRARIES} ${JSONC_LIBRARIES} ${UUID_LIBRARIES} ) set_target_properties(plugin_xrootd PROPERTIES CLEAN_DIRECT_OUTPUT 1 OUTPUT_NAME "gfal_plugin_xrootd" LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/plugins) install(TARGETS plugin_xrootd LIBRARY DESTINATION ${PLUGIN_INSTALL_DIR}) # install xrootd configuration files list (APPEND xrootd_conf_file "${CMAKE_SOURCE_DIR}/dist/etc/gfal2.d/xrootd_plugin.conf") install(FILES ${xrootd_conf_file} DESTINATION ${SYSCONF_INSTALL_DIR}/gfal2.d/) # readme install(FILES "README_PLUGIN_XROOTD" DESTINATION ${DOC_INSTALL_DIR}) endif () gfal2-v2.23.0/src/plugins/xrootd/README_PLUGIN_XROOTD000066400000000000000000000003451465240014500216060ustar00rootroot00000000000000Plugin for GFAL2 to access data through xrootd. URLs that begin with root:// will use this plugin. All GFAL2 functions are supported in this plugin except pread/pwrite and readlink/symlink (symlinks are not supported in xrootd). gfal2-v2.23.0/src/plugins/xrootd/gfal_xrootd_plugin_3rd_copy.cpp000066400000000000000000000355761465240014500247630ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2015 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include "gfal_xrootd_plugin_interface.h" #include "gfal_xrootd_plugin_utils.h" #include "uri/gfal2_parsing.h" #undef TRUE #undef FALSE #include #include #include #include class CopyFeedback: public XrdCl::CopyProgressHandler { public: CopyFeedback(gfal2_context_t context, gfalt_params_t p, bool isThirdParty) : context(context), params(p), startTime(0), isThirdParty(isThirdParty) { memset(&status, 0x00, sizeof(status)); } virtual ~CopyFeedback() { } void BeginJob(uint16_t jobNum, uint16_t jobTotal, const XrdCl::URL *source, const XrdCl::URL *destination) { this->startTime = time(NULL); this->source = source->GetURL(); this->destination = destination->GetURL(); plugin_trigger_event(this->params, xrootd_domain, GFAL_EVENT_NONE, GFAL_EVENT_TRANSFER_ENTER, "%s => %s", this->source.c_str(), this->destination.c_str()); if (this->isThirdParty) { plugin_trigger_event(params, xrootd_domain, GFAL_EVENT_NONE, GFAL_EVENT_TRANSFER_TYPE, GFAL_TRANSFER_TYPE_PULL); } else { plugin_trigger_event(params, xrootd_domain, GFAL_EVENT_NONE, GFAL_EVENT_TRANSFER_TYPE, GFAL_TRANSFER_TYPE_STREAMED); } } void EndJob(uint16_t jobNum, const XrdCl::PropertyList* result) { std::ostringstream msg; msg << "Job finished"; if (result->HasProperty("status")) { XrdCl::XRootDStatus status; result->Get("status", status); msg << ", " << status.ToStr(); } if (result->HasProperty("realTarget")) { std::string value; result->Get("realTarget", value); msg << ", Real target: " << value; } plugin_trigger_event(this->params, xrootd_domain, GFAL_EVENT_NONE, GFAL_EVENT_TRANSFER_EXIT, "%s", msg.str().c_str()); } void JobProgress(uint16_t jobNum, uint64_t bytesProcessed, uint64_t bytesTotal) { time_t now = time(NULL); time_t elapsed = now - this->startTime; this->status.status = 0; this->status.bytes_transfered = bytesProcessed; this->status.transfer_time = elapsed; if (elapsed > 0) this->status.average_baudrate = bytesProcessed / elapsed; this->status.instant_baudrate = this->status.average_baudrate; plugin_trigger_monitor(this->params, &this->status, this->source.c_str(), this->destination.c_str()); } bool ShouldCancel(uint16_t jobNum) { return gfal2_is_canceled(this->context); } private: gfal2_context_t context; gfalt_params_t params; _gfalt_transfer_status status; time_t startTime; std::string source, destination; bool isThirdParty; }; static void xrootd2gliberr(GError** err, const char* func, const char* format, const XrdCl::XRootDStatus& status) { std::string statusStr = status.ToStr(); const char *str = statusStr.c_str(); size_t str_len = statusStr.length(); gchar *escaped_str = gfal2_utf8escape_string(str, str_len, "\n\r\t\\"); gfal2_set_error(err, xrootd_domain, xrootd_status_to_posix_errno(status), func, format, escaped_str); g_free(escaped_str); } int gfal_xrootd_3rdcopy_check(plugin_handle plugin_data, gfal2_context_t context, const char* src, const char* dst, gfal_url2_check check) { if (check != GFAL_FILE_COPY && check != GFAL_BULK_COPY) return 0; bool src_is_root = strncmp(src, "root://", 7) == 0 || strncmp(src, "roots://", 8) == 0 || strncmp(src, "xroot://", 8) == 0 || strncmp(src, "xroots://", 9) == 0; bool dst_is_root = strncmp(dst, "root://", 7) == 0 || strncmp(dst, "roots://", 8) == 0 || strncmp(dst, "xroot://", 8) == 0 || strncmp(dst, "xroots://", 9) == 0; bool src_is_file = strncmp(src, "file://", 7) == 0; bool dst_is_file = strncmp(dst, "file://", 7) == 0; if (src_is_root) { return dst_is_root || dst_is_file; } else if (dst_is_root) { return src_is_root || src_is_file; } return false; } static void gfal_xrootd_3rd_init_url(gfal2_context_t context, XrdCl::URL& xurl, const char* url, const char* token) { xurl.FromString(prepare_url(context, url)); if (token) { XrdCl::URL::ParamsMap params; params.insert(std::make_pair("svcClass", token)); xurl.SetParams(params); } } /// Clean dst, update err if failed during cleanup with something else than ENOENT, /// returns always -1 for convenience /// Triggers an event when attempts to unlink destination file. /// If unlink fails reports in the event the correspondent errno. /// If unlink succeeds or if it fails because the file does not exist, the event reports 0 as the status code. static int gfal_xrootd_copy_cleanup(plugin_handle plugin_data, const gfalt_params_t& params, const char* dst, GError** err) { GError *unlink_err = NULL; if ((*err)->code != EEXIST) { int status = 0; if (gfal_xrootd_unlinkG(plugin_data, dst, &unlink_err) != 0) { if (unlink_err->code != ENOENT) { gfal2_log(G_LOG_LEVEL_WARNING, "When trying to clean the destination: %s", unlink_err->message); status = unlink_err->code; } g_error_free(unlink_err); } else { gfal2_log(G_LOG_LEVEL_INFO, "Destination file removed"); } plugin_trigger_event(params, xrootd_domain, GFAL_EVENT_DESTINATION, GFAL_EVENT_CLEANUP, "%d", status); } else { gfal2_log(G_LOG_LEVEL_DEBUG, "The transfer failed because the file exists. Do not clean!"); } return -1; } int gfal_xrootd_3rd_copy_bulk(plugin_handle plugin_data, gfal2_context_t context, gfalt_params_t params, size_t nbfiles, const char* const * srcs, const char* const * dsts, const char* const * checksums, GError** op_error, GError*** file_errors) { GError* internalError = NULL; char _checksumType[64] = { 0 }; char _checksumValue[512] = { 0 }; bool isThirdParty = false; gfalt_checksum_mode_t checksumMode = gfalt_get_checksum(params, _checksumType, sizeof(_checksumType), _checksumValue, sizeof(_checksumValue), NULL); XrdCl::CopyProcess copy_process; std::vector results; for (size_t i = 0; i < nbfiles; ++i) { results.push_back(XrdCl::PropertyList()); } const char* src_spacetoken = gfalt_get_src_spacetoken(params, NULL); const char* dst_spacetoken = gfalt_get_dst_spacetoken(params, NULL); for (size_t i = 0; i < nbfiles; ++i) { XrdCl::URL source_url, dest_url; gfal_xrootd_3rd_init_url(context, source_url, srcs[i], src_spacetoken); gfal_xrootd_3rd_init_url(context, dest_url, dsts[i], dst_spacetoken); XrdCl::PropertyList job; job.Set("source", source_url.GetURL()); job.Set("target", dest_url.GetURL()); job.Set("force", gfalt_get_replace_existing_file(params, NULL)); job.Set("makeDir", gfalt_get_create_parent_dir(params, NULL)); job.Set("SubStreamsPerChannel", gfalt_get_nbstreams(params, NULL)); job.Set("posc", true); if ((source_url.GetProtocol() == "root") && (dest_url.GetProtocol() == "root")) { job.Set("thirdParty", "only"); isThirdParty = true; job.Set("delegate", gfalt_get_use_proxy_delegation(params, NULL)); } else { job.Set("thirdParty", "first"); } job.Set("tpcTimeout", gfalt_get_timeout(params, NULL)); gfal2_log(G_LOG_LEVEL_DEBUG, "Copy job: tpc_only=%d tpc_delegation=%d tpc_timeout=%d", isThirdParty, gfalt_get_use_proxy_delegation(params, NULL), gfalt_get_timeout(params, NULL)); if (checksumMode) { char checksumType[64] = { 0 }; char checksumValue[512] = { 0 }; char **chks = g_strsplit(checksums[i], ":", 2); char *s = chks[1]; while (*s && *s == '0') s++; strncpy(checksumType, chks[0], sizeof(chks[0])); strncpy(checksumValue, s, sizeof(s)); checksumType[63] = checksumValue[511] = '\0'; g_strfreev(chks); if (!checksumType[0]) { char* defaultChecksumType = gfal2_get_opt_string(context, XROOTD_CONFIG_GROUP, XROOTD_DEFAULT_CHECKSUM, &internalError); if (internalError) { gfal2_set_error(op_error, xrootd_domain, internalError->code, __func__, "%s", internalError->message); g_error_free(internalError); return -1; } g_strlcpy(checksumType, defaultChecksumType, sizeof(checksumType)); g_free(defaultChecksumType); } std::string sChecksumType = predefined_checksum_type_to_lower(checksumType); std::string sChecksumValue(checksumValue); std::transform(sChecksumValue.begin(), sChecksumValue.end(), sChecksumValue.begin(), ::tolower); std::string sChecksumMode = "none"; switch (checksumMode) { case GFALT_CHECKSUM_BOTH: sChecksumMode = "end2end"; break; case GFALT_CHECKSUM_TARGET: sChecksumMode = "target"; break; case GFALT_CHECKSUM_SOURCE: sChecksumMode = "source"; break; } gfal2_log(G_LOG_LEVEL_DEBUG, "Predefined Checksum Mode: %s", sChecksumMode.c_str()); gfal2_log(G_LOG_LEVEL_DEBUG, "Predefined Checksum Type: %s", sChecksumType.c_str()); gfal2_log(G_LOG_LEVEL_DEBUG, "Predefined Checksum Value: %s", sChecksumValue.c_str()); job.Set("checkSumMode", sChecksumMode); job.Set("checkSumType", sChecksumType); job.Set("checkSumPreset", sChecksumValue); } copy_process.AddJob(job, &(results[i])); } // Configuration job int parallel = gfal2_get_opt_integer_with_default(context, XROOTD_CONFIG_GROUP, XROOTD_PARALLEL_COPIES, 20); XrdCl::PropertyList config_job; config_job.Set("jobType", "configuration"); config_job.Set("parallel", parallel); copy_process.AddJob(config_job, NULL); XrdCl::XRootDStatus status = copy_process.Prepare(); if (!status.IsOK()) { xrootd2gliberr(op_error, __func__, "Error on XrdCl::CopyProcess::Prepare(): %s", status); return -1; } CopyFeedback feedback(context, params, isThirdParty); status = copy_process.Run(&feedback); // On bulk operations, even if there is one single failure we will get it // here, so ignore! if (nbfiles == 1 && !status.IsOK()) { xrootd2gliberr(op_error, __func__, "Error on XrdCl::CopyProcess::Run(): %s", status); GError* nested_error = NULL; if (gfalt_get_transfer_cleanup(params, &nested_error)) { return gfal_xrootd_copy_cleanup(plugin_data, params, dsts[0],op_error); } gfal2_log(G_LOG_LEVEL_INFO, "Gfal xrootd copy clean-up disabled"); return -1; } // For bulk operations, here we do get the actual status per file int n_failed = 0; int n_evict = 0; *file_errors = g_new0(GError*, nbfiles); const char* files_to_evict[nbfiles]; for (size_t i = 0; i < nbfiles; ++i) { status = results[i].Get("status"); if (!status.IsOK()) { xrootd2gliberr(&((*file_errors)[i]), __func__, "Error on XrdCl::CopyProcess::Run(): %s", status); GError* nested_error = NULL; if (gfalt_get_transfer_cleanup(params, &nested_error)) { gfal_xrootd_copy_cleanup(plugin_data, params, dsts[i],file_errors[i]); } else { gfal2_log(G_LOG_LEVEL_INFO, "Gfal xrootd copy clean-up disabled: " "file \"%s\" has not been removed !", srcs[i]); } ++n_failed; } else { files_to_evict[n_evict] = srcs[i]; ++n_evict; } } // Evict source files if flag is set to true if (gfalt_get_use_evict(params, NULL)) { std::vector errors(n_evict, NULL); int ret = gfal_xrootd_release_file_list(plugin_data, n_evict, files_to_evict, "", errors.data()); //No token is needed for evict operation in xrootd gfal2_log(G_LOG_LEVEL_DEBUG, "Eviction request exited with status code: %d", ret); if (ret < 0) { gfal2_log(G_LOG_LEVEL_INFO, "Eviction request failed in one or more files"); for(int i = 0; i < n_evict; ++i) { g_error_free(errors[i]); } } plugin_trigger_event(params, xrootd_domain, GFAL_EVENT_SOURCE, GFAL_EVENT_EVICT, "%d", ret); } return -n_failed; } int gfal_xrootd_3rd_copy(plugin_handle plugin_data, gfal2_context_t context, gfalt_params_t params, const char* src, const char* dst, GError** err) { GError* op_error = NULL; GError** file_error = NULL; char checksumType[64] = { 0 }; char checksumValue[512] = { 0 }; gfalt_get_checksum(params, checksumType, sizeof(checksumType), checksumValue, sizeof(checksumValue), NULL); char *checksumConcat[1]; checksumConcat[0] = g_strdup_printf("%s:%s", checksumType, checksumValue); int ret = gfal_xrootd_3rd_copy_bulk(plugin_data, context, params, 1, &src, &dst, checksumConcat, &op_error, &file_error); g_free(checksumConcat[0]); if (ret < 0) { if (op_error) { gfal2_propagate_prefixed_error(err, op_error, __func__); } else if (file_error) { gfal2_propagate_prefixed_error(err, file_error[0], __func__); g_free(file_error); } } return ret; } gfal2-v2.23.0/src/plugins/xrootd/gfal_xrootd_plugin_archive.cpp000066400000000000000000000161121465240014500246430ustar00rootroot00000000000000/* * Copyright (c) CERN 2020 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include "gfal_xrootd_plugin_interface.h" #include "gfal_xrootd_plugin_utils.h" #include #include #include #include int gfal_xrootd_archive_poll(plugin_handle plugin_data, const char* url, GError** err) { GError *errors[1] = {NULL}; const char* const urls[1] = {url}; int ret = gfal_xrootd_archive_poll_list(plugin_data, 1, urls, errors); if (errors[0] != NULL) { *err = errors[0]; } return ret; } int gfal_xrootd_archive_poll_list(plugin_handle plugin_data, int nbfiles, const char* const* urls, GError** errors) { if (nbfiles <= 0) { return 1; } gfal2_context_t context = (gfal2_context_t) plugin_data; XrdCl::URL endpoint(prepare_url(context, urls[0])); endpoint.SetPath(std::string()); XrdCl::FileSystem fs(endpoint); // Generate a dummy prepare id uuid_t uuid; char suuid[128]; uuid_generate_random(uuid); uuid_unparse(uuid, suuid); std::string str_query_args(suuid); std::set paths; for (int i = 0; i < nbfiles; i++) { XrdCl::URL url(prepare_url(context, urls[i])); std::string path = url.GetPath(); collapse_slashes(path); paths.insert(path); str_query_args += '\n'; str_query_args += path; } XrdCl::Buffer* resp = 0; XrdCl::Buffer query_args; query_args.FromString(str_query_args); gfal2_log(G_LOG_LEVEL_DEBUG, "Issuing query prepare: %s", query_args.ToString().c_str()); XrdCl::XRootDStatus st = fs.Query(XrdCl::QueryCode::Prepare, query_args, resp); if (!st.IsOK()) { gfal2_log(G_LOG_LEVEL_WARNING, "Query prepare failed: %s", st.ToString().c_str()); for (int i = 0; i < nbfiles; i++) { gfal2_set_error(&errors[i], xrootd_domain, xrootd_status_to_posix_errno(st, true), __func__, "%s", st.ToString().c_str()); } return -1; } std::string jsonresp = resp->ToString(); delete resp; struct json_object* parsed_json = json_tokener_parse(jsonresp.c_str()); if (!parsed_json) { for (int i = 0; i < nbfiles; i++) { gfal2_set_error(&errors[i], xrootd_domain, ENOMSG, __func__, "Response from server is an invalid JSON: %s", jsonresp.c_str()); } return -1; } struct json_object* request_id; json_object_object_get_ex(parsed_json, "request_id", &request_id); std::string str_request_id = request_id ? json_object_get_string(request_id) : ""; if (str_request_id.empty() || str_request_id != suuid) { for (int i = 0; i < nbfiles; i++) { gfal2_set_error(&errors[i], xrootd_domain, ENOMSG, __func__, "%s", "Request ID mismatch."); } return -1; } int ontape_count = 0; int error_count = 0; // Iterate over the file list struct json_object* responses = 0; json_object_object_get_ex(parsed_json, "responses", &responses); int size = responses ? json_object_array_length(responses) : 0; if (size != nbfiles) { for (int i = 0; i < nbfiles; i++) { gfal2_set_error(&errors[i], xrootd_domain, ENOMSG, __func__, "Number of files in the request doest not match!"); } return -1; } for (int i = 0 ; i < size; i++) { struct json_object* fileobj = json_object_array_get_idx(responses, i); if (!fileobj) { error_count++; gfal2_set_error(&errors[i], xrootd_domain, ENOMSG, __func__, "Failed to parse responses JSON from server: %s", jsonresp.c_str()); continue; } // Retrieve "error_text" attribute // Note: if it exists, this text should be appended to all other error messages struct json_object* fileobj_error_text = 0; json_object_object_get_ex(fileobj, "error_text", &fileobj_error_text); std::string error_text; if (!fileobj_error_text) { error_count++; gfal2_set_error(&errors[i], xrootd_domain, ENOMSG, __func__, "Error attribute missing."); continue; } else { error_text = json_object_get_string(fileobj_error_text); } // Retrieve "path" attribute struct json_object* fileobj_path = 0; json_object_object_get_ex(fileobj, "path", &fileobj_path); std::string path = fileobj_path ? json_object_get_string(fileobj_path) : ""; collapse_slashes(path); if (path.empty() || !paths.count(path)) { error_count++; gfal2_xrootd_poll_set_error(&errors[i], ENOMSG, __func__, error_text.c_str(), "Wrong path: %s", path.c_str()); continue; } // Retrieve "path_exists" attribute struct json_object* fileobj_exists = 0; json_object_object_get_ex(fileobj, "path_exists", &fileobj_exists); bool path_exists = json_obj_to_bool(fileobj_exists); if (!path_exists) { error_count++; gfal2_xrootd_poll_set_error(&errors[i], ENOENT, __func__, error_text.c_str(), "File does not exist: %s", path.c_str()); continue; } // Retrieve "ontape" attribute struct json_object* fileobj_ontape = 0; json_object_object_get_ex(fileobj, "on_tape", &fileobj_ontape); bool ontape = json_obj_to_bool(fileobj_ontape); if (ontape) { ontape_count++; continue; } if (!error_text.empty()) { error_count++; gfal2_set_error(&errors[i], xrootd_domain, ENOMSG, __func__, "%s", error_text.c_str()); continue; } // In case of no errors but the file is not yet archived, set EAGAIN if (!ontape) { gfal2_set_error(&errors[i], xrootd_domain, EAGAIN, __func__, "File %s is not yet archived", path.c_str()); } } // Free the top JSON object json_object_put(parsed_json); // All files are on tape: return 1 if (ontape_count == nbfiles) { return 1; } // All files encountered errors: return -1 if (error_count == nbfiles) { return -1; } // Some files are on tape, others encountered errors if (ontape_count + error_count == nbfiles) { return 2; } // Archiving in process: return 0 return 0; } gfal2-v2.23.0/src/plugins/xrootd/gfal_xrootd_plugin_bringonline.cpp000066400000000000000000000334341465240014500255360ustar00rootroot00000000000000/* * Copyright (c) CERN 2017 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include "gfal_xrootd_plugin_interface.h" #include "gfal_xrootd_plugin_utils.h" #include #include #include #include #include #include int gfal_xrootd_bring_online_list(plugin_handle plugin_data, int nbfiles, const char* const* urls, time_t pintime, time_t timeout, char* token, size_t tsize, int async, GError** err) { if (nbfiles <= 0) { return 1; } gfal2_context_t context = (gfal2_context_t)plugin_data; XrdCl::URL endpoint(prepare_url(context, urls[0])); endpoint.SetPath(std::string()); XrdCl::FileSystem fs(endpoint); std::vector fileList; for (int i = 0; i < nbfiles; ++i) { XrdCl::URL file(prepare_url(context, urls[i])); fileList.emplace_back(file.GetPathWithParams()); } XrdCl::Buffer *responsePtr = 0; XrdCl::Status st = fs.Prepare(fileList, XrdCl::PrepareFlags::Flags::Stage, 0, responsePtr, timeout); if (!st.IsOK()) { GError *tmp_err = NULL; gfal2_set_error(&tmp_err, xrootd_domain, xrootd_status_to_posix_errno(st), __func__, "Bringonline request failed. One or more files failed with: %s", st.ToString().c_str()); for (int i = 0; i < nbfiles; ++i) { err[i] = g_error_copy(tmp_err); } g_error_free(tmp_err); delete responsePtr; return -1; } if (responsePtr && responsePtr->GetBuffer()) { copy_to_cstring(token, tsize, responsePtr->GetBuffer(), responsePtr->GetSize()); } else { gfal2_log(G_LOG_LEVEL_DEBUG, "Empty response from the server"); delete responsePtr; return -1; } delete responsePtr; return 0; } int gfal_xrootd_bring_online_list_v2(plugin_handle plugin_data, int nbfiles, const char* const* urls, const char* const* metadata, time_t pintime, time_t timeout, char* token, size_t tsize, int async, GError** err) { return gfal_xrootd_bring_online_list(plugin_data, nbfiles, urls, pintime, timeout, token, tsize, async, err); } int gfal_xrootd_bring_online_poll_list(plugin_handle plugin_data, int nbfiles, const char* const* urls, const char* token, GError** err) { if (nbfiles <= 0) { return 1; } gfal2_context_t context = (gfal2_context_t)plugin_data; XrdCl::URL endpoint(prepare_url(context, urls[0])); endpoint.SetPath(std::string()); XrdCl::FileSystem fs(endpoint); std::set paths; std::string strarg = token; for( int i = 0; i < nbfiles; ++i ) { strarg += '\n'; XrdCl::URL url( prepare_url( context, urls[i] ) ); std::string path = url.GetPath(); // collapse redundant slashes collapse_slashes( path ); strarg += path; paths.insert( path ); } XrdCl::Buffer arg; arg.FromString( strarg ); XrdCl::Buffer *resp = 0; gfal2_log( G_LOG_LEVEL_DEBUG, "Issuing query prepare." ); XrdCl::XRootDStatus st = fs.Query( XrdCl::QueryCode::Prepare, arg, resp ); if( !st.IsOK() ) { gfal2_log( G_LOG_LEVEL_WARNING, "Query prepare failed: %s", st.ToString().c_str() ); for( int i = 0; i < nbfiles; ++i ) gfal2_set_error(&err[i], xrootd_domain, xrootd_status_to_posix_errno(st, true), __func__, "%s", st.ToString().c_str() ); return -1; } std::string jsonresp = resp->ToString(); delete resp; struct json_object *parsed_json = json_tokener_parse( jsonresp.c_str() ); if( !parsed_json ) { for( int i = 0; i < nbfiles; ++i ) gfal2_set_error(&err[i], xrootd_domain, ENOMSG, __func__, "Response from server is an invalid JSON: %s", jsonresp.c_str()); return -1; } struct json_object *request_id; json_object_object_get_ex( parsed_json, "request_id", &request_id ); std::string reqid = request_id ? json_object_get_string( request_id ) : ""; if( reqid.empty() || reqid != token ) { for( int i = 0; i < nbfiles; ++i ) gfal2_set_error( &err[i], xrootd_domain, ENOMSG, __func__, "%s", "Request ID mismatch." ); return -1; } int onlinecnt = 0; int errorcnt = 0; // now iterate over the file list struct json_object *responses = 0; json_object_object_get_ex( parsed_json, "responses", &responses ); int size = responses ? json_object_array_length( responses ) : 0; if( size != nbfiles ) { for( int i = 0; i < nbfiles; ++i ) gfal2_set_error( &err[i], xrootd_domain, ENOMSG, __func__, "Number of files in the request does not match!" ); return -1; } for( int i = 0; i < size; ++i ) { // get the i-th object in the array struct json_object *arrobj = json_object_array_get_idx( responses, i ); if( !arrobj ) { ++errorcnt; gfal2_set_error(&err[i], xrootd_domain, ENOMSG, __func__, "Failed to parse responses JSON from server: %s", jsonresp.c_str()); continue; } // get the error_text attribute // Note: if it exists, this text should be appended to all other error messages struct json_object *arrobj_error_text = 0; json_object_object_get_ex( arrobj, "error_text", &arrobj_error_text ); std::string error_text; if( !arrobj_error_text ) { ++errorcnt; gfal2_set_error( &err[i], xrootd_domain, ENOMSG, __func__, "Error attribute missing." ); continue; } else { error_text = json_object_get_string( arrobj_error_text ); } // get the path attribute struct json_object *arrobj_path = 0; json_object_object_get_ex( arrobj, "path", &arrobj_path ); std::string path = arrobj_path ? json_object_get_string( arrobj_path ) : ""; // collapse redundant slashes collapse_slashes( path ); if( path.empty() || !paths.count( path ) ) { // it's not our file, this is an error ++errorcnt; gfal2_xrootd_poll_set_error( &err[i], ENOMSG, __func__, error_text.c_str(), "Wrong path: %s", path.c_str() ); continue; } // get the path_exists attribute // Note: CTA changed from "exists" to "path_exists". // Keep a fallback to "exists" for the time being. struct json_object *arrobj_exists = 0; json_object_object_get_ex( arrobj, "path_exists", &arrobj_exists ); bool path_exists = json_obj_to_bool(arrobj_exists); if( !path_exists ) { // Try "exists" fallback json_object_object_get_ex( arrobj, "exists", &arrobj_exists ); bool exists = json_obj_to_bool(arrobj_exists); if ( !exists ) { ++errorcnt; gfal2_xrootd_poll_set_error( &err[i], ENOENT, __func__, error_text.c_str(), "File does not exist: %s", path.c_str() ); continue; } } // get the online attribute struct json_object *arrobj_online = 0; json_object_object_get_ex( arrobj, "online", &arrobj_online ); bool online = json_obj_to_bool(arrobj_online); if( online ) { ++onlinecnt; continue; } // get the requested attribute struct json_object *arrobj_requested = 0; json_object_object_get_ex( arrobj, "requested", &arrobj_requested ); bool requested = json_obj_to_bool(arrobj_requested); if( !requested ) { ++errorcnt; gfal2_xrootd_poll_set_error( &err[i], ENOMSG, __func__, error_text.c_str(), "File is not being brought online: %s", path.c_str() ); continue; } // get the has_reqid attribute struct json_object *arrobj_has_reqid = 0; json_object_object_get_ex( arrobj, "has_reqid", &arrobj_has_reqid ); bool has_reqid = json_obj_to_bool(arrobj_has_reqid); if( !has_reqid ) { ++errorcnt; gfal2_xrootd_poll_set_error( &err[i], ENOMSG, __func__, error_text.c_str(), "File (%s) is not included in the bring online request: %s", path.c_str(), token ); continue; } // get the req_time attribute struct json_object *arrobj_req_time = 0; json_object_object_get_ex( arrobj, "req_time", &arrobj_req_time ); std::string req_time = arrobj_req_time ? json_object_get_string( arrobj_req_time ) : ""; if( !req_time.empty() ) gfal2_log( G_LOG_LEVEL_DEBUG, "File (%s) has been requested at: %s", path.c_str(), req_time.c_str() ); else { ++errorcnt; gfal2_xrootd_poll_set_error( &err[i], ENOMSG, __func__, error_text.c_str(), "Bring-online timestamp missing." ); continue; } if( !error_text.empty() ) { ++errorcnt; gfal2_set_error( &err[i], xrootd_domain, ENOMSG, __func__, "%s", error_text.c_str() ); continue; } // if there is no error but the file is not online set EAGAIN if( !online ) gfal2_set_error( &err[i], xrootd_domain, EAGAIN, __func__, "File (%s) is not yet online.", path.c_str() ); } // Free the top JSON object json_object_put(parsed_json); // if all files are online return 1 if( onlinecnt == nbfiles ) return 1; // if there were errors return -1 if( errorcnt == nbfiles ) return -1; // Some files are online, others encountered errors if( (errorcnt + onlinecnt) == nbfiles ) return 2; // otherwise 0 means user still needs to wait return 0; } int gfal_xrootd_release_file_list(plugin_handle plugin_data, int nbfiles, const char* const* urls, const char* token, GError** err) { gfal2_context_t context = (gfal2_context_t)plugin_data; XrdCl::URL endpoint(prepare_url(context, urls[0])); endpoint.SetPath(std::string()); XrdCl::FileSystem fs(endpoint); std::vector fileList; for(int i = 0; i < nbfiles; ++i) { XrdCl::URL file(prepare_url(context, urls[i])); fileList.emplace_back(file.GetPathWithParams()); } XrdCl::Buffer *responsePtr = 0; XrdCl::Status st = fs.Prepare(fileList, XrdCl::PrepareFlags::Flags::Evict, 0, responsePtr, 30); if (!st.IsOK()) { GError *tmp_err = NULL; gfal2_set_error(&tmp_err, xrootd_domain, xrootd_status_to_posix_errno(st), __func__, "%s", st.ToString().c_str()); for (int i = 0; i < nbfiles; ++i) { err[i] = g_error_copy(tmp_err); } g_error_free(tmp_err); delete responsePtr; return -1; } delete responsePtr; return 0; } int gfal_xrootd_bring_online(plugin_handle plugin_data, const char* url, time_t pintime, time_t timeout, char* token, size_t tsize, int async, GError** err) { GError *errors[1] = {NULL}; const char* const urls[1] = {url}; int ret = gfal_xrootd_bring_online_list(plugin_data, 1, urls, pintime, timeout, token, tsize, async, errors); if (errors[0] != NULL) { *err = errors[0]; } return ret; } int gfal_xrootd_bring_online_v2(plugin_handle plugin_data, const char* url, const char* metadata, time_t pintime, time_t timeout, char* token, size_t tsize, int async, GError** err) { return gfal_xrootd_bring_online(plugin_data, url, pintime, timeout, token, tsize, async, err); } int gfal_xrootd_bring_online_poll(plugin_handle plugin_data, const char* url, const char* token, GError** err) { GError *errors[1] = {NULL}; const char* const urls[1] = {url}; int ret = gfal_xrootd_bring_online_poll_list(plugin_data, 1, urls, token, errors); if (errors[0] != NULL) { *err = errors[0]; } return ret; } int gfal_xrootd_release_file(plugin_handle plugin_data, const char* url, const char* token, GError** err) { GError *errors[1] = {NULL}; const char* const urls[1] = {url}; int ret = gfal_xrootd_release_file_list(plugin_data, 1, urls, token, errors); if (errors[0] != NULL) { *err = errors[0]; } return ret; } int gfal_xrootd_abort_files(plugin_handle plugin_data, int nbfiles, const char* const* urls, const char* token, GError** err) { if (nbfiles <= 0) { return 1; } gfal2_context_t context = (gfal2_context_t)plugin_data; XrdCl::URL endpoint(prepare_url(context, urls[0])); endpoint.SetPath(std::string()); XrdCl::FileSystem fs(endpoint); std::vector fileList; fileList.emplace_back(token); for (int i = 0; i < nbfiles; ++i) { XrdCl::URL file(prepare_url(context, urls[i])); fileList.emplace_back(file.GetPathWithParams()); } XrdCl::Buffer *reponsePtr = 0; XrdCl::Status st = fs.Prepare(fileList, XrdCl::PrepareFlags::Flags::Cancel, 0, reponsePtr); std::unique_ptr response(reponsePtr); if (!st.IsOK()) { GError *tmp_err = NULL; gfal2_set_error(&tmp_err, xrootd_domain, xrootd_status_to_posix_errno(st), __func__, "%s", st.ToString().c_str()); for (int i = 0; i < nbfiles; ++i) { err[i] = g_error_copy(tmp_err); } g_error_free(tmp_err); return -1; } return 0; } gfal2-v2.23.0/src/plugins/xrootd/gfal_xrootd_plugin_interface.cpp000066400000000000000000000454651465240014500251770ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2015 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include // This header provides all the required functions except chmod #include // This header is required for chmod #include // For directory listing #include #include // For setting the log level #include #include // TRUE and FALSE are defined in Glib and xrootd headers #ifdef TRUE #undef TRUE #endif #ifdef FALSE #undef FALSE #endif #include #include "gfal_xrootd_plugin_interface.h" #include "gfal_xrootd_plugin_utils.h" void set_xrootd_log_level() { // Note: xrootd lib logs to stderr if (gfal2_log_get_level() & G_LOG_LEVEL_DEBUG) XrdCl::DefaultEnv::SetLogLevel( "Debug" ); else if (gfal2_log_get_level() & G_LOG_LEVEL_INFO) XrdCl::DefaultEnv::SetLogLevel( "Info" ); else if (gfal2_log_get_level() & G_LOG_LEVEL_MESSAGE) XrdCl::DefaultEnv::SetLogLevel( "Info" ); else if (gfal2_log_get_level() & G_LOG_LEVEL_WARNING) XrdCl::DefaultEnv::SetLogLevel( "Warning" ); else XrdCl::DefaultEnv::SetLogLevel( "Error" ); } int gfal_xrootd_statG(plugin_handle handle, const char* path, struct stat* buff, GError ** err) { std::string sanitizedUrl = prepare_url((gfal2_context_t) handle, path); // reset stat fields reset_stat(*buff); if (XrdPosixXrootd::Stat(sanitizedUrl.c_str(), buff) != 0) { gfal2_xrootd_set_error(err, errno, __func__, "Failed to stat file"); return -1; } return 0; } gfal_file_handle gfal_xrootd_openG(plugin_handle handle, const char *path, int flag, mode_t mode, GError ** err) { std::string sanitizedUrl = prepare_url((gfal2_context_t) handle, path); int *fd = new int; *fd = XrdPosixXrootd::Open(sanitizedUrl.c_str(), flag, mode); if (*fd == -1) { gfal2_xrootd_set_error(err, errno, __func__, "Failed to open file"); delete fd; return NULL; } return gfal_file_handle_new(gfal_xrootd_getName(), (gpointer) fd); } ssize_t gfal_xrootd_readG(plugin_handle handle, gfal_file_handle fd, void *buff, size_t count, GError ** err) { int * fdesc = (int*) (gfal_file_handle_get_fdesc(fd)); if (!fdesc) { gfal2_xrootd_set_error(err, errno, __func__, "Bad file handle"); return -1; } ssize_t l = XrdPosixXrootd::Read(*fdesc, buff, count); if (l < 0) { gfal2_xrootd_set_error(err, errno, __func__, "Failed while reading from file"); return -1; } return l; } ssize_t gfal_xrootd_writeG(plugin_handle handle, gfal_file_handle fd, const void *buff, size_t count, GError ** err) { int * fdesc = (int*) (gfal_file_handle_get_fdesc(fd)); if (!fdesc) { gfal2_xrootd_set_error(err, errno, __func__, "Bad file handle"); return -1; } ssize_t l = XrdPosixXrootd::Write(*fdesc, buff, count); if (l < 0) { gfal2_xrootd_set_error(err, errno, __func__, "Failed while writing to file"); return -1; } return l; } off_t gfal_xrootd_lseekG(plugin_handle handle, gfal_file_handle fd, off_t offset, int whence, GError **err) { int * fdesc = (int*) (gfal_file_handle_get_fdesc(fd)); if (!fdesc) { gfal2_xrootd_set_error(err, errno, __func__, "Bad file handle"); return -1; } off_t l = XrdPosixXrootd::Lseek(*fdesc, offset, whence); if (l < 0) { gfal2_xrootd_set_error(err, errno, __func__, "Failed to seek within file"); return -1; } return l; } int gfal_xrootd_closeG(plugin_handle handle, gfal_file_handle fd, GError ** err) { int r = 0; int * fdesc = (int*) (gfal_file_handle_get_fdesc(fd)); if (fdesc) { r = XrdPosixXrootd::Close(*fdesc); if (r != 0) { gfal2_xrootd_set_error(err, errno, __func__, "Failed to close file"); } delete (int*) (gfal_file_handle_get_fdesc(fd)); } gfal_file_handle_delete(fd); return r; } int gfal_xrootd_mkdirpG(plugin_handle handle, const char *url, mode_t mode, gboolean pflag, GError **err) { std::string sanitizedUrl = prepare_url((gfal2_context_t) handle, url); // EOS returns success for mkdir for a directory that exists struct stat buf; if (XrdPosixXrootd::Stat(sanitizedUrl.c_str(), &buf) == 0) { errno = EEXIST; gfal2_xrootd_set_error(err, errno, __func__, "Failed to create directory %s", url); return -1; } if (XrdPosixXrootd::Mkdir(sanitizedUrl.c_str(), mode) != 0) { if (errno == ECANCELED) { errno = EEXIST; } gfal2_xrootd_set_error(err, errno, __func__, "Failed to create directory %s", url); return -1; } return 0; } int gfal_xrootd_chmodG(plugin_handle handle, const char *url, mode_t mode, GError **err) { std::string sanitizedUrl = prepare_url((gfal2_context_t) handle, url); set_xrootd_log_level(); XrdCl::URL xrdcl_url( sanitizedUrl ); XrdCl::FileSystem fs( xrdcl_url ); XrdCl::Access::Mode xrdcl_mode = file_mode_to_xrdcl_access( mode ); XrdCl::XRootDStatus status = fs.ChMod( xrdcl_url.GetPath(), xrdcl_mode ); if (!status.IsOK()) { gfal2_xrootd_set_error(err, xrootd_status_to_posix_errno(status), __func__, status.ToStr().c_str()); return -1; } return 0; } int gfal_xrootd_unlinkG(plugin_handle handle, const char *url, GError **err) { std::string sanitizedUrl = prepare_url((gfal2_context_t) handle, url); if (XrdPosixXrootd::Unlink(sanitizedUrl.c_str()) != 0) { gfal2_xrootd_set_error(err, errno, __func__, "Failed to delete file"); return -1; } return 0; } int gfal_xrootd_rmdirG(plugin_handle handle, const char *url, GError **err) { std::string sanitizedUrl = prepare_url((gfal2_context_t) handle, url); if (XrdPosixXrootd::Rmdir(sanitizedUrl.c_str()) != 0) { struct stat buf; // Need some errno massaging because of EOS if (errno == EEXIST) { errno = ENOTEMPTY; } else if (errno == EIO) { if (XrdPosixXrootd::Stat(sanitizedUrl.c_str(), &buf) == 0 && S_ISDIR(buf.st_mode)) { errno = ENOTEMPTY; } else { errno = ENOTDIR; } } else if (errno == ENOENT) { if (XrdPosixXrootd::Stat(sanitizedUrl.c_str(), &buf) == 0) { errno = ENOTDIR; } } gfal2_xrootd_set_error(err, errno, __func__, "Failed to delete directory"); return -1; } return 0; } int gfal_xrootd_accessG(plugin_handle handle, const char *url, int mode, GError **err) { std::string sanitizedUrl = prepare_url((gfal2_context_t) handle, url); if (XrdPosixXrootd::Access(sanitizedUrl.c_str(), mode) != 0) { gfal2_xrootd_set_error(err, errno, __func__, "Failed to access file or directory"); return -1; } return 0; } int gfal_xrootd_renameG(plugin_handle handle, const char *oldurl, const char *urlnew, GError **err) { std::string oldSanitizedUrl = prepare_url((gfal2_context_t) handle, oldurl); std::string newSanitizedUrl = prepare_url((gfal2_context_t) handle, urlnew); if (XrdPosixXrootd::Rename(oldSanitizedUrl.c_str(), newSanitizedUrl.c_str()) != 0) { gfal2_xrootd_set_error(err, errno, __func__, "Failed to rename file or directory"); // EOS may return EEXIST when new is a directory if (*err && (*err)->code == EEXIST) { struct stat buf; if (XrdPosixXrootd::Stat(newSanitizedUrl.c_str(), &buf) == 0 && S_ISDIR(buf.st_mode)) { (*err)->code = EISDIR; } } return -1; } return 0; } void StatInfo2Xattr(const XrdCl::StatInfo* stinfo, char * buff) { bool ontape = false; bool ondisk = false; if (stinfo->TestFlags(XrdCl::StatInfo::BackUpExists)) ontape = true; if (!stinfo->TestFlags(XrdCl::StatInfo::Offline)) ondisk= true; if (ontape && ondisk) { strcpy(buff,GFAL_XATTR_STATUS_NEARLINE_ONLINE); gfal2_log(G_LOG_LEVEL_DEBUG, GFAL_XATTR_STATUS_NEARLINE_ONLINE); } else if (ontape) { strcpy(buff,GFAL_XATTR_STATUS_NEARLINE); gfal2_log(G_LOG_LEVEL_DEBUG, GFAL_XATTR_STATUS_NEARLINE); } else if (ondisk) { strcpy(buff,GFAL_XATTR_STATUS_ONLINE); gfal2_log(G_LOG_LEVEL_DEBUG, GFAL_XATTR_STATUS_ONLINE); } else { strcpy(buff,GFAL_XATTR_STATUS_UNKNOWN); gfal2_log(G_LOG_LEVEL_DEBUG, GFAL_XATTR_STATUS_UNKNOWN); } } // Callback class for directory listing class DirListHandler: public XrdCl::ResponseHandler { private: XrdCl::URL url; XrdCl::FileSystem fs; std::list entries; struct dirent dbuffer; std::mutex mutex; std::condition_variable cv; bool done; public: int errcode; std::string errstr; DirListHandler(const XrdCl::URL& url): url(url), fs(url), done(false), errcode(0) { memset(&dbuffer, 0, sizeof(dbuffer)); } int List() { XrdCl::XRootDStatus status = fs.DirList(url.GetPath(), XrdCl::DirListFlags::Stat, this); if (!status.IsOK()) { errcode = status.code; errstr = status.ToString(); return -1; } return 0; } // AFAIK, this is called only once void HandleResponse(XrdCl::XRootDStatus* status, XrdCl::AnyObject* response) { std::lock_guard lock(mutex); if (status->IsOK()) { XrdCl::DirectoryList* list; response->Get(list); if (list) { XrdCl::DirectoryList::ConstIterator i; for (i = list->Begin(); i != list->End(); ++i) { entries.push_back(*i); } } } else { errcode = status->code; errstr = status->ToString(); } done = true; cv.notify_all(); } void StatInfo2Stat(const XrdCl::StatInfo* stinfo, struct stat* st) { st->st_size = stinfo->GetSize(); st->st_mtime = stinfo->GetModTime(); st->st_mode = 0; if (stinfo->TestFlags(XrdCl::StatInfo::IsDir)) st->st_mode |= S_IFDIR; if (stinfo->TestFlags(XrdCl::StatInfo::IsReadable)) st->st_mode |= (S_IRUSR | S_IRGRP | S_IROTH); if (stinfo->TestFlags(XrdCl::StatInfo::IsWritable)) st->st_mode |= (S_IWUSR | S_IWGRP | S_IWOTH); if (stinfo->TestFlags(XrdCl::StatInfo::XBitSet)) st->st_mode |= (S_IXUSR | S_IXGRP | S_IXOTH); } struct dirent* Get(struct stat* st = NULL) { if (!done) { std::unique_lock lock(mutex); cv.wait_for(lock, std::chrono::seconds(60)); if (!done) { return NULL; } } if (entries.empty()) return NULL; XrdCl::DirectoryList::ListEntry* entry = entries.front(); entries.pop_front(); XrdCl::StatInfo* stinfo = entry->GetStatInfo(); g_strlcpy(dbuffer.d_name, entry->GetName().c_str(), sizeof(dbuffer.d_name)); dbuffer.d_reclen = strnlen(dbuffer.d_name, sizeof(dbuffer.d_reclen)); if (stinfo && stinfo->TestFlags(XrdCl::StatInfo::IsDir)) dbuffer.d_type = DT_DIR; else dbuffer.d_type = DT_REG; if (st != NULL) { if (stinfo != NULL) { StatInfo2Stat(stinfo, st); } else { stinfo = new XrdCl::StatInfo(); std::string fullPath = url.GetPath() + "/" + dbuffer.d_name; XrdCl::XRootDStatus status = this->fs.Stat(fullPath, stinfo); if (!status.IsOK()) { errcode = status.code; errstr = status.ToString(); return NULL; } StatInfo2Stat(stinfo, st); delete stinfo; } } delete entry; return &dbuffer; } }; gfal_file_handle gfal_xrootd_opendirG(plugin_handle handle, const char* url, GError** err) { std::string sanitizedUrl = prepare_url((gfal2_context_t) handle, url); XrdCl::URL parsed(sanitizedUrl); // Need to do stat first so we can fail syncrhonously for some errors! struct stat st; if (XrdPosixXrootd::Stat(sanitizedUrl.c_str(), &st) != 0) { gfal2_xrootd_set_error(err, errno, __func__, "Failed to stat file"); return NULL; } if (!S_ISDIR(st.st_mode)) { gfal2_xrootd_set_error(err, ENOTDIR, __func__, "Not a directory"); return NULL; } DirListHandler* handler = new DirListHandler(parsed); if (handler->List() != 0) { gfal2_xrootd_set_error(err, handler->errcode, __func__, "Failed to open dir: %s", handler->errstr.c_str()); return NULL; } return gfal_file_handle_new2(gfal_xrootd_getName(), (gpointer) handler, NULL, url); } struct dirent* gfal_xrootd_readdirG(plugin_handle plugin_data, gfal_file_handle dir_desc, GError** err) { DirListHandler* handler = (DirListHandler*)(gfal_file_handle_get_fdesc(dir_desc)); if (!handler) { gfal2_xrootd_set_error(err, errno, __func__, "Bad dir handle"); return NULL; } dirent* entry = handler->Get(); if (!entry && handler->errcode != 0) { gfal2_xrootd_set_error(err, handler->errcode, __func__, "Failed reading directory: %s", handler->errstr.c_str()); return NULL; } return entry; } struct dirent* gfal_xrootd_readdirppG(plugin_handle plugin_data, gfal_file_handle dir_desc, struct stat* st, GError** err) { DirListHandler* handler = (DirListHandler*)(gfal_file_handle_get_fdesc(dir_desc)); if (!handler) { gfal2_xrootd_set_error(err, errno, __func__, "Bad dir handle"); return NULL; } dirent* entry = handler->Get(st); if (!entry && handler->errcode != 0) { gfal2_xrootd_set_error(err, handler->errcode, __func__, "Failed reading directory: %s", handler->errstr.c_str()); return NULL; } return entry; } int gfal_xrootd_closedirG(plugin_handle plugin_data, gfal_file_handle dir_desc, GError** err) { // Free all objects associated with this client DirListHandler* handler = (DirListHandler*)(gfal_file_handle_get_fdesc(dir_desc)); if (handler) { delete handler; } gfal_file_handle_delete(dir_desc); return 0; } int gfal_xrootd_checksumG(plugin_handle plugin_data, const char* url, const char* check_type, char * checksum_buffer, size_t buffer_length, off_t start_offset, size_t data_length, GError ** err) { std::string sanitizedUrl = prepare_url((gfal2_context_t) plugin_data, url); std::string lowerChecksumType = predefined_checksum_type_to_lower(check_type); if (start_offset != 0 || data_length != 0) { gfal2_xrootd_set_error(err, ENOTSUP, __func__, "XROOTD does not support partial checksums"); return -1; } if (sanitizedUrl.find("?") == std::string::npos) { sanitizedUrl += "?"; } else { sanitizedUrl += "&"; } sanitizedUrl += "cks.type="; sanitizedUrl += lowerChecksumType; time_t mTime; if (XrdPosixXrootd::QueryChksum(sanitizedUrl.c_str(), mTime, checksum_buffer, buffer_length) < 0) { gfal2_xrootd_set_error(err, errno, __func__, "Could not get the checksum"); return -1; } // Note that the returned value is "type value" char* space = ::index(checksum_buffer, ' '); if (!space) { gfal2_xrootd_set_error(err, errno, __func__, "Could not get the checksum (Wrong format)"); return -1; } *space = '\0'; if (strncasecmp(checksum_buffer, lowerChecksumType.c_str(), lowerChecksumType.length()) != 0) { gfal2_xrootd_set_error(err, errno, __func__, "Got '%s' while expecting '%s'", checksum_buffer, lowerChecksumType.c_str()); return -1; } g_strlcpy(checksum_buffer, space + 1, buffer_length); return 0; } ssize_t gfal_xrootd_getxattrG(plugin_handle plugin_data, const char* url, const char* key, void* buff, size_t s_buff, GError** err) { ssize_t len = 0; if (strcmp(key, GFAL_XATTR_SPACETOKEN) == 0) { len = gfal_xrootd_space_reporting(plugin_data, url, key, buff, s_buff, err); } else if (strcmp(key, GFAL_XATTR_STATUS) == 0) { std::string sanitizedUrl = prepare_url((gfal2_context_t) plugin_data, url); XrdCl::URL parsed(sanitizedUrl); XrdCl::FileSystem fs(parsed); XrdCl::StatInfo * info; XrdCl::XRootDStatus st = fs.Stat( parsed.GetPath(), info ); if (!st.IsOK()) { errno = ENOENT; gfal2_xrootd_set_error(err, errno, __func__, "Failed to get the xattr \"%s\"", key); return -1; } StatInfo2Xattr(info,(char*)buff); len = strnlen((char*)buff, s_buff); delete info; } else { std::string sanitizedUrl = prepare_url((gfal2_context_t) plugin_data, url); memset(buff, 0x00, s_buff); len = XrdPosixXrootd::Getxattr(sanitizedUrl.c_str(), key, buff, s_buff); if (len < 0) { gfal2_xrootd_set_error(err, errno, __func__, "Failed to get the xattr \"%s\"", key); } } return len; } ssize_t gfal_xrootd_listxattrG(plugin_handle plugin_data, const char* url, char* list, size_t s_list, GError** err) { static const char props[] = "xroot.cksum\0xroot.space\0xroot.xattr\0spacetoken"; static const size_t proplen = sizeof(props); size_t len = proplen > s_list ? s_list : proplen; memcpy(list, props, len); return len; } int gfal_xrootd_setxattrG(plugin_handle plugin_data, const char* url, const char* key, const void* buff , size_t s_buff, int flags, GError** err) { gfal2_xrootd_set_error(err, ENOSYS, __func__, "Can not set extended attributes"); return -1; } const char* gfal_xrootd_getName() { return GFAL2_PLUGIN_VERSIONED("xrootd", VERSION); } gfal2-v2.23.0/src/plugins/xrootd/gfal_xrootd_plugin_interface.h000066400000000000000000000136241465240014500246340ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #ifndef GFAL_XROOTD_PLUGIN_INTERFACE_H_ #define GFAL_XROOTD_PLUGIN_INTERFACE_H_ #include #define XROOTD_CONFIG_GROUP "XROOTD PLUGIN" #define XROOTD_DEFAULT_CHECKSUM "COPY_CHECKSUM_TYPE" #define XROOTD_CHECKSUM_MODE "COPY_CHECKSUM_MODE" #define XROOTD_PARALLEL_COPIES "PARALLEL_COPIES" #define XROOTD_NORMALIZE_PATH "NORMALIZE_PATH" extern "C" { int gfal_xrootd_statG(plugin_handle handle, const char* name, struct stat* buff, GError ** err); gfal_file_handle gfal_xrootd_openG(plugin_handle handle, const char *path, int flag, mode_t mode, GError ** err); ssize_t gfal_xrootd_readG(plugin_handle handle, gfal_file_handle fd, void *buff, size_t count, GError ** err); ssize_t gfal_xrootd_writeG(plugin_handle handle, gfal_file_handle fd, const void *buff, size_t count, GError ** err); off_t gfal_xrootd_lseekG(plugin_handle handle, gfal_file_handle fd, off_t offset, int whence, GError **err); int gfal_xrootd_closeG(plugin_handle handle, gfal_file_handle fd, GError ** err); int gfal_xrootd_mkdirpG(plugin_handle plugin_data, const char *url, mode_t mode, gboolean pflag, GError **err); int gfal_xrootd_chmodG(plugin_handle plugin_data, const char *url, mode_t mode, GError **err); int gfal_xrootd_unlinkG(plugin_handle plugin_data, const char *url, GError **err); int gfal_xrootd_rmdirG(plugin_handle plugin_data, const char *url, GError **err); int gfal_xrootd_accessG(plugin_handle plugin_data, const char *url, int mode, GError **err); int gfal_xrootd_renameG(plugin_handle plugin_data, const char *oldurl, const char *urlnew, GError **err); gfal_file_handle gfal_xrootd_opendirG(plugin_handle plugin_data, const char* url, GError** err); struct dirent* gfal_xrootd_readdirG(plugin_handle plugin_data, gfal_file_handle dir_desc, GError** err); struct dirent* gfal_xrootd_readdirppG(plugin_handle plugin_data, gfal_file_handle dir_desc, struct stat* st, GError** err); int gfal_xrootd_closedirG(plugin_handle plugin_data, gfal_file_handle dir_desc, GError** err); int gfal_xrootd_checksumG(plugin_handle data, const char* url, const char* check_type, char * checksum_buffer, size_t buffer_length, off_t start_offset, size_t data_length, GError ** err); ssize_t gfal_xrootd_getxattrG(plugin_handle plugin_data, const char* url, const char* key, void* buff, size_t s_buff, GError** err); ssize_t gfal_xrootd_listxattrG(plugin_handle plugin_data, const char* url, char* list, size_t s_list, GError** err); int gfal_xrootd_setxattrG(plugin_handle plugin_data, const char* url, const char* key, const void* buff , size_t s_buff, int flags, GError** err); int gfal_xrootd_3rdcopy_check(plugin_handle plugin_data, gfal2_context_t context, const char* src, const char* dst, gfal_url2_check check); int gfal_xrootd_3rd_copy(plugin_handle plugin_data, gfal2_context_t context, gfalt_params_t params, const char* src, const char* dst, GError** err); int gfal_xrootd_3rd_copy_bulk(plugin_handle plugin_data, gfal2_context_t context, gfalt_params_t params, size_t nbfiles, const char* const * srcs, const char* const * dsts, const char* const * checksums, GError** op_error, GError*** file_errors); ssize_t gfal_xrootd_space_reporting(plugin_handle plugin_data, const char *sanitizedUrl, const char *key, void *buff, size_t s_buf, GError **err); int gfal_xrootd_bring_online(plugin_handle plugin_data, const char* url, time_t pintime, time_t timeout, char* token, size_t tsize, int async, GError** err); int gfal_xrootd_bring_online_poll(plugin_handle plugin_data, const char* url, const char* token, GError** err); int gfal_xrootd_bring_online_v2(plugin_handle plugin_data, const char* url, const char* metadata, time_t pintime, time_t timeout, char* token, size_t tsize, int async, GError** err); int gfal_xrootd_release_file(plugin_handle plugin_data, const char* url, const char* token, GError** err); int gfal_xrootd_bring_online_list(plugin_handle plugin_data, int nbfiles, const char* const* urls, time_t pintime, time_t timeout, char* token, size_t tsize, int async, GError** err); int gfal_xrootd_bring_online_list_v2(plugin_handle plugin_data, int nbfiles, const char* const* urls, const char* const* metadata, time_t pintime, time_t timeout, char* token, size_t tsize, int async, GError** err); int gfal_xrootd_bring_online_poll_list(plugin_handle plugin_data, int nbfiles, const char* const* urls, const char* token, GError** err); int gfal_xrootd_release_file_list(plugin_handle plugin_data, int nbfiles, const char* const* urls, const char* token, GError** err); int gfal_xrootd_abort_files(plugin_handle plugin_data, int nbfiles, const char* const* urls, const char* token, GError** err); int gfal_xrootd_archive_poll(plugin_handle plugin_data, const char* url, GError** err); int gfal_xrootd_archive_poll_list(plugin_handle plugin_data, int nbfiles, const char* const* urls, GError** errors); const char* gfal_xrootd_getName(); void set_xrootd_log_level(); } #endif /* GFAL_XROOTD_PLUGIN_INTERFACE_H_ */ gfal2-v2.23.0/src/plugins/xrootd/gfal_xrootd_plugin_main.cpp000066400000000000000000000112451465240014500241500ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2015 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include "gfal_xrootd_plugin_interface.h" #include extern "C" { gboolean gfal_xrootd_check_url(plugin_handle ch, const char* url, plugin_mode mode, GError** err); gfal_plugin_interface gfal_plugin_init(gfal2_context_t handle, GError** err) { static XrdPosixXrootd singleXroot; gfal_plugin_interface xrootd_plugin; memset(&xrootd_plugin, 0, sizeof(gfal_plugin_interface)); // clear the plugin // set xrootd log level //set_xrootd_log_level(); xrootd_plugin.plugin_data = handle; xrootd_plugin.getName = &gfal_xrootd_getName; xrootd_plugin.check_plugin_url = &gfal_xrootd_check_url; xrootd_plugin.openG = &gfal_xrootd_openG; xrootd_plugin.closeG = &gfal_xrootd_closeG; xrootd_plugin.readG = &gfal_xrootd_readG; xrootd_plugin.writeG = &gfal_xrootd_writeG; xrootd_plugin.lseekG = &gfal_xrootd_lseekG; xrootd_plugin.statG = &gfal_xrootd_statG; xrootd_plugin.lstatG = &gfal_xrootd_statG; xrootd_plugin.preadG = NULL; // &gfal_xrootd_preadG; xrootd_plugin.pwriteG = NULL; // &gfal_xrootd_pwriteG; xrootd_plugin.mkdirpG = &gfal_xrootd_mkdirpG; xrootd_plugin.chmodG = &gfal_xrootd_chmodG; xrootd_plugin.unlinkG = &gfal_xrootd_unlinkG; xrootd_plugin.rmdirG = &gfal_xrootd_rmdirG; xrootd_plugin.accessG = &gfal_xrootd_accessG; xrootd_plugin.renameG = &gfal_xrootd_renameG; xrootd_plugin.opendirG = &gfal_xrootd_opendirG; xrootd_plugin.readdirG = &gfal_xrootd_readdirG; xrootd_plugin.readdirppG = &gfal_xrootd_readdirppG; xrootd_plugin.closedirG = &gfal_xrootd_closedirG; xrootd_plugin.getxattrG = &gfal_xrootd_getxattrG; xrootd_plugin.listxattrG = &gfal_xrootd_listxattrG; xrootd_plugin.setxattrG = &gfal_xrootd_setxattrG; xrootd_plugin.readlinkG = NULL; // symlinks not supported on xrootd xrootd_plugin.symlinkG = NULL; // symlinks not supported on xrootd xrootd_plugin.checksum_calcG = &gfal_xrootd_checksumG; xrootd_plugin.check_plugin_url_transfer = &gfal_xrootd_3rdcopy_check; xrootd_plugin.copy_file = &gfal_xrootd_3rd_copy; xrootd_plugin.copy_bulk = &gfal_xrootd_3rd_copy_bulk; xrootd_plugin.bring_online = &gfal_xrootd_bring_online; xrootd_plugin.bring_online_list = &gfal_xrootd_bring_online_list; xrootd_plugin.bring_online_v2 = &gfal_xrootd_bring_online_v2; xrootd_plugin.bring_online_list_v2 = &gfal_xrootd_bring_online_list_v2; xrootd_plugin.bring_online_poll = &gfal_xrootd_bring_online_poll; xrootd_plugin.bring_online_poll_list = &gfal_xrootd_bring_online_poll_list; xrootd_plugin.release_file = &gfal_xrootd_release_file; xrootd_plugin.release_file_list = &gfal_xrootd_release_file_list; xrootd_plugin.abort_files = &gfal_xrootd_abort_files; xrootd_plugin.archive_poll = &gfal_xrootd_archive_poll; xrootd_plugin.archive_poll_list = &gfal_xrootd_archive_poll_list; return xrootd_plugin; } gboolean gfal_xrootd_check_url(plugin_handle ch, const char* url, plugin_mode mode, GError** err) { if (strncmp(url, "root://", 7) != 0 && strncmp(url, "roots://", 8) != 0 && strncmp(url, "xroot://", 8) != 0 && strncmp(url, "xroots://", 9) != 0) return FALSE; int ret; switch (mode) { case GFAL_PLUGIN_STAT: case GFAL_PLUGIN_LSTAT: case GFAL_PLUGIN_OPEN: case GFAL_PLUGIN_MKDIR: case GFAL_PLUGIN_CHMOD: case GFAL_PLUGIN_UNLINK: case GFAL_PLUGIN_RMDIR: case GFAL_PLUGIN_ACCESS: case GFAL_PLUGIN_RENAME: case GFAL_PLUGIN_OPENDIR: case GFAL_PLUGIN_CHECKSUM: case GFAL_PLUGIN_GETXATTR: case GFAL_PLUGIN_SETXATTR: case GFAL_PLUGIN_LISTXATTR: case GFAL_PLUGIN_BRING_ONLINE: case GFAL_PLUGIN_ARCHIVE: ret = TRUE; break; default: ret = FALSE; break; } return ret; } } // extern "C" gfal2-v2.23.0/src/plugins/xrootd/gfal_xrootd_plugin_main.h000066400000000000000000000020431465240014500236110ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #ifndef GFAL_XROOTD_PLUGIN_MAIN_H_ #define GFAL_XROOTD_PLUGIN_MAIN_H_ #include #ifdef __cplusplus extern "C" { #endif // Entry point of the plugin gfal_plugin_interface gfal_plugin_init(gfal_handle context, GError** err); #ifdef __cplusplus } #endif #endif /* GFAL_XROOTD_PLUGIN_MAIN_H_ */ gfal2-v2.23.0/src/plugins/xrootd/gfal_xrootd_plugin_space.cpp000066400000000000000000000035611465240014500243210ustar00rootroot00000000000000/* * Copyright (c) CERN 2016 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "gfal_xrootd_plugin_interface.h" #include "gfal_xrootd_plugin_utils.h" #include #include #include "space/gfal2_space.h" ssize_t gfal_xrootd_space_reporting(plugin_handle plugin_data, const char *url, const char *key, void *buff, size_t s_buf, GError **err) { std::string sanitizedUrl = prepare_url((gfal2_context_t) plugin_data, url); XrdCl::FileSystem fs(sanitizedUrl); XrdCl::FileSystemUtils::SpaceInfo *space = NULL; XrdCl::URL xurl(sanitizedUrl); XrdCl::XRootDStatus status = XrdCl::FileSystemUtils::GetSpaceInfo(space, &fs, xurl.GetPath()); if (!status.IsOK()) { gfal2_set_error(err, xrootd_domain, EIO, __func__, "Failed to get the space information: %s", status.GetErrorMessage().c_str()); return -1; } struct space_report report = {0}; report.used = space->GetUsed(); report.free = space->GetFree(); report.total = space->GetTotal(); uint64_t chunk = space->GetLargestFreeChunk(); report.largest_chunk = &chunk; delete space; return gfal2_space_generate_json(&report, (char*)buff, s_buf); } gfal2-v2.23.0/src/plugins/xrootd/gfal_xrootd_plugin_utils.cpp000066400000000000000000000273641465240014500243750ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2015 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include #include #include #include #include "gfal_xrootd_plugin_utils.h" #include "gfal_xrootd_plugin_interface.h" GQuark xrootd_domain = g_quark_from_static_string("xroot"); XrdCl::Access::Mode file_mode_to_xrdcl_access( mode_t mode ) { XrdCl::Access::Mode xrdcl_mode = XrdCl::Access::None; if( mode & S_IRUSR ) xrdcl_mode |= XrdCl::Access::UR; if( mode & S_IWUSR ) xrdcl_mode |= XrdCl::Access::UW; if( mode & S_IXUSR ) xrdcl_mode |= XrdCl::Access::UX; if( mode & S_IRGRP ) xrdcl_mode |= XrdCl::Access::GR; if( mode & S_IWGRP ) xrdcl_mode |= XrdCl::Access::GW; if( mode & S_IXGRP ) xrdcl_mode |= XrdCl::Access::GX; if( mode & S_IROTH ) xrdcl_mode |= XrdCl::Access::OR; if( mode & S_IWOTH ) xrdcl_mode |= XrdCl::Access::OW; if( mode & S_IXOTH ) xrdcl_mode |= XrdCl::Access::OX; return xrdcl_mode; } void reset_stat(struct stat& st) { st.st_mode = 0; st.st_atime = 0; st.st_ctime = 0; st.st_mtime = 0; st.st_blksize = 0; st.st_blocks = 0; st.st_dev = 0; st.st_gid = 0; st.st_ino = 0; st.st_nlink = 0; st.st_rdev = 0; st.st_size = 0; st.st_uid = 0; } static std::string query_args(gfal2_context_t context, const char *url) { bool prev_args = false; GError *error = NULL; gchar *ucert = gfal2_cred_get(context, GFAL_CRED_X509_CERT, url, NULL, &error); g_clear_error(&error); gchar *ukey = gfal2_cred_get(context, GFAL_CRED_X509_KEY, url, NULL, &error); g_clear_error(&error); std::ostringstream args; if (ucert) { if (!ukey) { ukey = ucert; } // Certificate == key, assume proxy if (strcmp(ucert, ukey) == 0) { args << "xrd.gsiusrpxy=" << ucert; prev_args = true; } else { args << "xrd.gsiusrcrt=" << ucert << '&' << "xrd.gsiusrkey=" << ukey; prev_args = true; } g_free(ucert); if (ucert != ukey) g_free(ukey); } // Additional keys gsize keyCount; gchar **keys = gfal2_get_opt_keys(context, "XROOTD PLUGIN", &keyCount, &error); if (keys != NULL) { for (gsize keyIndex =0; keyIndex < keyCount; ++keyIndex) { if (g_str_has_prefix(keys[keyIndex], "XRD.")) { gchar *lowercase = g_utf8_strdown(keys[keyIndex], -1); gchar *value = gfal2_get_opt_string_with_default(context, "XROOTD PLUGIN", keys[keyIndex], ""); // Note: When value is a list, value may have ';', which should be replaced with ',' for (gsize i = 0; value[i] != '\0'; ++i) { if (value[i] == ';') { value[i] = ','; } } if (prev_args) { args << "&"; } args << lowercase << "=" << value; prev_args = true; g_free(lowercase); g_free(value); } } } g_clear_error(&error); g_strfreev(keys); return args.str(); } /** * Add to the URL query any argument required to make the configuration effective */ static void fill_query(gfal_handle_ *context, const char *url, gfal2_uri *uri) { std::string args = query_args(context, url); if (!args.empty()) { if (uri->query != NULL) { char *p = uri->query; uri->query = g_strconcat(uri->query, "&", args.c_str(), NULL); g_free(p); } else { uri->query = g_strdup(args.c_str()); } } gfal2_log(G_LOG_LEVEL_DEBUG, "Xrootd Query URI: %s", uri->query); } /** * Normalize the URL, so the path starts with the right number of slashes */ static void normalize_path(gfal2_uri *uri) { if (uri->path == NULL) { uri->path = g_strdup("///"); } else if (strncmp(uri->path, "///", 3) == 0) { // pass } else if (strncmp(uri->path, "//", 2) == 0) { char *p = uri->path; uri->path = g_strconcat("/", uri->path, NULL); g_free(p); } else { char *p = uri->path; uri->path = g_strconcat("//", uri->path, NULL); g_free(p); } } std::string prepare_url(gfal2_context_t context, const char *url) { GError *error = NULL; gfal2_uri *uri = gfal2_parse_uri(url, &error); if (error != NULL) { g_clear_error(&error); return url; } gboolean normalize = gfal2_get_opt_boolean_with_default(context, XROOTD_CONFIG_GROUP, XROOTD_NORMALIZE_PATH, TRUE); if (normalize) { normalize_path(uri); } fill_query(context, url, uri); // Url-decode path // XRootD and SRM expect the path to be url-decoded, while dav and gsiftp expect the opposite gfal2_urldecode(uri->path); char *new_url = gfal2_join_uri(uri); std::string sanitized(new_url); gfal2_free_uri(uri); g_free(new_url); return sanitized; } std::string predefined_checksum_type_to_lower(const std::string& type) { std::string lowerForm(type); std::transform(lowerForm.begin(), lowerForm.end(), lowerForm.begin(), ::tolower); if (lowerForm == "adler32" || lowerForm == "crc32" || lowerForm == "md5") return lowerForm; else return type; } bool json_obj_to_bool(struct json_object *boolobj) { if( !boolobj ) return false; static const std::string str_true( "true" ); std::string str_bool = json_object_get_string( boolobj ); std::transform( str_bool.begin(), str_bool.end(), str_bool.begin(), tolower ); return ( str_bool == str_true ); } void collapse_slashes(std::string& path) { std::string::iterator itr = path.begin(), store = path.begin(); ++itr; while( itr != path.end() ) { if( *store != '/' || *itr != '/' ) { ++store; *store = *itr; } ++itr; } size_t size = store - path.begin() + 1; if( path.size() != size ) path.resize( size ); } // Mask network error codes as ECOMM #ifndef ECOMM #define ECOMM EIO #endif static int mask_network_errno(int errc) { switch (errc) { case EHOSTUNREACH: case ENOTSOCK: case ETIMEDOUT: case ENOTCONN: case ECONNRESET: case ECONNREFUSED: case ENETRESET: case ECONNABORTED: return ECOMM; default: return errc; } } // Copied from xrootd/src/XrdPosix/XrdPosixMap.cc #ifndef ENOSR #define ENOSR ENOSPC #endif #ifndef ECHRNG #define ECHRNG EINVAL #endif static int map_status_code_to_errno(int code) { switch (code) { case XrdCl::errRetry: return EAGAIN; // Cl:001 case XrdCl::errInvalidOp: return EOPNOTSUPP; // Cl:003 case XrdCl::errConfig: return ENOEXEC; // Cl:006 case XrdCl::errInvalidArgs: return EINVAL; // Cl:009 case XrdCl::errInProgress: return EINPROGRESS; // Cl:010 case XrdCl::errNotSupported: return ENOTSUP; // Cl:013 case XrdCl::errDataError: return EDOM; // Cl:014 case XrdCl::errNotImplemented: return ENOSYS; // Cl:015 case XrdCl::errNoMoreReplicas: return ENOSR; // Cl:016 case XrdCl::errInvalidAddr: return EHOSTUNREACH; // Cl:101 case XrdCl::errSocketError: return ENOTSOCK; // Cl:102 case XrdCl::errSocketTimeout: return ETIMEDOUT; // Cl:103 case XrdCl::errSocketDisconnected: return ENOTCONN; // Cl:104 case XrdCl::errStreamDisconnect: return ECONNRESET; // Cl:107 case XrdCl::errConnectionError: return ECONNREFUSED; // Cl:108 case XrdCl::errInvalidSession: return ECHRNG; // Cl:109 case XrdCl::errTlsError: return ENETRESET; // Cl:110 case XrdCl::errInvalidMessage: return EPROTO; // Cl:201 case XrdCl::errHandShakeFailed: return EPROTO; // Cl:202 case XrdCl::errLoginFailed: return ECONNABORTED; // Cl:203 case XrdCl::errAuthFailed: return EAUTH; // Cl:204 case XrdCl::errQueryNotSupported: return ENOTSUP; // Cl:205 case XrdCl::errOperationExpired: return ESTALE; // Cl:206 case XrdCl::errOperationInterrupted: return EINTR; // Cl:207 case XrdCl::errNoMoreFreeSIDs: return ENOSR; // Cl:301 case XrdCl::errInvalidRedirectURL: return ESPIPE; // Cl:302 case XrdCl::errInvalidResponse: return EBADMSG; // Cl:303 case XrdCl::errNotFound: return EIDRM; // Cl:304 case XrdCl::errCheckSumError: return EILSEQ; // Cl:305 case XrdCl::errRedirectLimit: return ELOOP; // Cl:306 default: return ENOMSG; } } int xrootd_status_to_posix_errno(const XrdCl::XRootDStatus& status, bool query_prepare) { int ret; if (status.IsOK()) { return 0; } if (status.code == XrdCl::errErrorResponse) { ret = XProtocol::toErrno(status.errNo); } else { ret = (status.errNo ? status.errNo : map_status_code_to_errno(status.code)); } if (query_prepare) { ret = mask_network_errno(ret); } return ret; } void gfal2_xrootd_set_error(GError **err, int errcode, const char *func, const char *desc, ...) { char error_string[64]; char *error_string_ptr; #if ((_POSIX_C_SOURCE >= 200112L || _XOPEN_SOURCE >= 600) && ! _GNU_SOURCE) || defined(__APPLE__) strerror_r(errcode, error_string, sizeof(error_string)); error_string_ptr = error_string; #else error_string_ptr = strerror_r(errcode, error_string, sizeof(error_string)); #endif char err_msg[256]; va_list args; va_start(args, desc); vsnprintf(err_msg, sizeof(err_msg), desc, args); va_end(args); char buffer[512]; snprintf(buffer, sizeof(buffer), "%s (%s)", err_msg, error_string_ptr); gfal2_set_error(err, xrootd_domain, errno, func, "%s", buffer); } void gfal2_xrootd_poll_set_error(GError **err, int errcode, const char *func, const char *err_reason, const char *format, ...) { char err_msg[256]; va_list args; va_start(args, format); vsnprintf(err_msg, sizeof(err_msg), format, args); va_end(args); char buffer[512]; if (err_reason != NULL) { snprintf(buffer, sizeof(buffer), "%s (reason: %s)", err_msg, err_reason); } else { snprintf(buffer, sizeof(buffer), "%s", err_msg); } gfal2_set_error(err, xrootd_domain, errcode, func, "%s", buffer); } void copy_to_cstring(char* dest, size_t dest_size, const char* source, size_t source_size) { size_t copy_size = std::min(source_size, dest_size); // Use memcpy because we have no guarantee that the source buffer is null terminated memcpy(dest, source, copy_size); if (source_size < dest_size) { dest[copy_size] = '\0'; } else { dest[dest_size - 1] = '\0'; } } gfal2-v2.23.0/src/plugins/xrootd/gfal_xrootd_plugin_utils.h000066400000000000000000000047171465240014500240370ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #ifndef GFAL_XROOTD_PLUGIN_UTILS_H_ #define GFAL_XROOTD_PLUGIN_UTILS_H_ #include #include #include #include #include extern GQuark xrootd_domain; /// Convert file mode_t to ints XrdCl::Access::Mode file_mode_to_xrdcl_access( mode_t mode ); /// Initialize all stat fields to zero void reset_stat(struct stat& st); /// Return the same URL, but making sure the path is always relative /// and adding the user credentials appended as keywords std::string prepare_url(gfal2_context_t context, const char *url); /// If the checksum type is one of the predefined ones, always lowercase /// @note adler32, crc32, md5 std::string predefined_checksum_type_to_lower(const std::string& type); /// Parse a JSON object into a boolean value bool json_obj_to_bool(struct json_object *boolobj); /// Collapse multiple consecutive slashes into a single one void collapse_slashes(std::string& path); /// Map an xrootd status code to a posix errno /// @note for a query prepare, mask network errors as ECOMM int xrootd_status_to_posix_errno(const XrdCl::XRootDStatus& status, bool query_prepare = false); /// Set error code with errno description void gfal2_xrootd_set_error(GError **err, int errcode, const char *func, const char *desc, ...); /// Set error code during polling, providing additional reason void gfal2_xrootd_poll_set_error(GError **err, int errcode, const char *func, const char *err_reasno, const char *format, ...); /// Copy contents of source buffer to dest buffer and always terminate dest buffer with null terminator void copy_to_cstring(char* dest, size_t dest_size, const char* source, size_t source_size); #endif /* GFAL_XROOTD_PLUGIN_UTILS_H_ */ gfal2-v2.23.0/src/utils/000077500000000000000000000000001465240014500147075ustar00rootroot00000000000000gfal2-v2.23.0/src/utils/CMakeLists.txt000066400000000000000000000042531465240014500174530ustar00rootroot00000000000000cmake_minimum_required (VERSION 2.6) # Dependencies if (IS_IFCE) list (APPEND gfal2_utils_definitions "-DMDS_BDII_EXTERNAL=1") set (is_ifce_link "is_ifce") list (APPEND src_mds "${CMAKE_CURRENT_SOURCE_DIR}/mds/gfal_mds.c") else (IS_IFCE) list (APPEND gfal2_utils_definitions "-DMDS_BDII_EXTERNAL=0") find_library(LDAP_LIBRARY NAMES ldap_r ldap) list (APPEND is_ifce_link ${LDAP_LIBRARY} "lber") list (APPEND src_mds "${CMAKE_CURRENT_SOURCE_DIR}/mds/gfal_mds.c" "${CMAKE_CURRENT_SOURCE_DIR}/mds/gfal_mds_internal.c" "${CMAKE_CURRENT_SOURCE_DIR}/mds/gfal_mds_ldap_internal_layer.c" ) endif (IS_IFCE) find_package (PugiXML) if (NOT PUGIXML_FOUND) message ("PugiXML not found. MDS Cache disabled") list (APPEND gfal2_utils_definitions "-DMDS_WITHOUT_CACHE=1") set (mds_cache_link "") else (NOT PUGIXML_FOUND) list (APPEND src_mds "${CMAKE_CURRENT_SOURCE_DIR}/mds/gfal_mds_cache.cpp" ) set (mds_cache_link "${PUGIXML_LIBRARIES}") endif (NOT PUGIXML_FOUND) # Link list (APPEND gfal2_utils_libraries ${is_ifce_link} ${mds_cache_link} ${JSONC_LIBRARIES} ) # Sources file (GLOB src_exceptions "${CMAKE_CURRENT_SOURCE_DIR}/exceptions/*.c*") file (GLOB src_gsimplecache "${CMAKE_CURRENT_SOURCE_DIR}/gsimplecache/*.c*") file (GLOB src_uri "${CMAKE_CURRENT_SOURCE_DIR}/uri/*.c*") file (GLOB src_checksums "${CMAKE_CURRENT_SOURCE_DIR}/checksums/*.c*") file (GLOB src_space "${CMAKE_CURRENT_SOURCE_DIR}/space/*.c*") file (GLOB src_network "${CMAKE_CURRENT_SOURCE_DIR}/network/*.c*") list (APPEND gfal2_utils_src ${src_exceptions}) list (APPEND gfal2_utils_c_src ${src_gsimplecache} ${src_uri} ${src_checksums} ${src_mds} ${src_space} ${src_network} ) set (gfal2_utils_c_src ${gfal2_utils_c_src} PARENT_SCOPE) set (gfal2_utils_src ${gfal2_utils_src} PARENT_SCOPE) set (gfal2_utils_libraries ${gfal2_utils_libraries} PARENT_SCOPE) set (gfal2_utils_definitions ${gfal2_utils_definitions} PARENT_SCOPE) set (gfal2_utils_includes ${JSONC_INCLUDE_DIRS} PARENT_SCOPE) # Install public headers install (FILES "uri/gfal2_uri.h" DESTINATION ${INCLUDE_INSTALL_DIR}/gfal2/utils) gfal2-v2.23.0/src/utils/checksums/000077500000000000000000000000001465240014500166745ustar00rootroot00000000000000gfal2-v2.23.0/src/utils/checksums/checksums.c000066400000000000000000000225431465240014500210330ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include "checksums.h" static const char* _no_zeros(const char* str) { const char* p = str; while (*p == '0') ++p; return p; } int gfal_compare_checksums(const char* chk1, const char* chk2, size_t len) { const char* c1 = _no_zeros(chk1); const char* c2 = _no_zeros(chk2); return strncasecmp(c1, c2, len); } // ---------------------------------------------------------------------------------------------------- /* * Taken from public domain md5 implementation, converted to ASL 2.0 license for gfal2 * * Homepage: * http://openwall.info/wiki/people/solar/software/public-domain-source-code/md5 * **/ /* * The basic MD5 functions. * * F and G are optimized compared to their RFC 1321 definitions for * architectures that lack an AND-NOT instruction, just like in Colin Plumb's * implementation. */ #define F(x, y, z) ((z) ^ ((x) & ((y) ^ (z)))) #define G(x, y, z) ((y) ^ ((z) & ((x) ^ (y)))) #define H(x, y, z) (((x) ^ (y)) ^ (z)) #define H2(x, y, z) ((x) ^ ((y) ^ (z))) #define I(x, y, z) ((y) ^ ((x) | ~(z))) /* * The MD5 transformation for all four rounds. */ #define STEP(f, a, b, c, d, x, t, s) \ (a) += f((b), (c), (d)) + (x) + (t); \ (a) = (((a) << (s)) | (((a) & 0xffffffff) >> (32 - (s)))); \ (a) += (b); /* * SET reads 4 input bytes in little-endian byte order and stores them * in a properly aligned word in host byte order. * * The check for little-endian architectures that tolerate unaligned * memory accesses is just an optimization. Nothing will break if it * doesn't work. */ #if defined(__i386__) || defined(__x86_64__) || defined(__vax__) #define SET(n) \ (*(gfal2_md5_u32plus *)&ptr[(n) * 4]) #define GET(n) \ SET(n) #else #define SET(n) \ (ctx->block[(n)] = \ (gfal2_md5_u32plus)ptr[(n) * 4] | \ ((gfal2_md5_u32plus)ptr[(n) * 4 + 1] << 8) | \ ((gfal2_md5_u32plus)ptr[(n) * 4 + 2] << 16) | \ ((gfal2_md5_u32plus)ptr[(n) * 4 + 3] << 24)) #define GET(n) \ (ctx->block[(n)]) #endif /* * This processes one or more 64-byte data blocks, but does NOT update * the bit counters. There are no alignment requirements. */ static const void *body(GFAL_MD5_CTX *ctx, const void *data, unsigned long size) { const unsigned char *ptr; gfal2_md5_u32plus a, b, c, d; gfal2_md5_u32plus saved_a, saved_b, saved_c, saved_d; ptr = (const unsigned char *)data; a = ctx->a; b = ctx->b; c = ctx->c; d = ctx->d; do { saved_a = a; saved_b = b; saved_c = c; saved_d = d; /* Round 1 */ STEP(F, a, b, c, d, SET(0), 0xd76aa478, 7) STEP(F, d, a, b, c, SET(1), 0xe8c7b756, 12) STEP(F, c, d, a, b, SET(2), 0x242070db, 17) STEP(F, b, c, d, a, SET(3), 0xc1bdceee, 22) STEP(F, a, b, c, d, SET(4), 0xf57c0faf, 7) STEP(F, d, a, b, c, SET(5), 0x4787c62a, 12) STEP(F, c, d, a, b, SET(6), 0xa8304613, 17) STEP(F, b, c, d, a, SET(7), 0xfd469501, 22) STEP(F, a, b, c, d, SET(8), 0x698098d8, 7) STEP(F, d, a, b, c, SET(9), 0x8b44f7af, 12) STEP(F, c, d, a, b, SET(10), 0xffff5bb1, 17) STEP(F, b, c, d, a, SET(11), 0x895cd7be, 22) STEP(F, a, b, c, d, SET(12), 0x6b901122, 7) STEP(F, d, a, b, c, SET(13), 0xfd987193, 12) STEP(F, c, d, a, b, SET(14), 0xa679438e, 17) STEP(F, b, c, d, a, SET(15), 0x49b40821, 22) /* Round 2 */ STEP(G, a, b, c, d, GET(1), 0xf61e2562, 5) STEP(G, d, a, b, c, GET(6), 0xc040b340, 9) STEP(G, c, d, a, b, GET(11), 0x265e5a51, 14) STEP(G, b, c, d, a, GET(0), 0xe9b6c7aa, 20) STEP(G, a, b, c, d, GET(5), 0xd62f105d, 5) STEP(G, d, a, b, c, GET(10), 0x02441453, 9) STEP(G, c, d, a, b, GET(15), 0xd8a1e681, 14) STEP(G, b, c, d, a, GET(4), 0xe7d3fbc8, 20) STEP(G, a, b, c, d, GET(9), 0x21e1cde6, 5) STEP(G, d, a, b, c, GET(14), 0xc33707d6, 9) STEP(G, c, d, a, b, GET(3), 0xf4d50d87, 14) STEP(G, b, c, d, a, GET(8), 0x455a14ed, 20) STEP(G, a, b, c, d, GET(13), 0xa9e3e905, 5) STEP(G, d, a, b, c, GET(2), 0xfcefa3f8, 9) STEP(G, c, d, a, b, GET(7), 0x676f02d9, 14) STEP(G, b, c, d, a, GET(12), 0x8d2a4c8a, 20) /* Round 3 */ STEP(H, a, b, c, d, GET(5), 0xfffa3942, 4) STEP(H2, d, a, b, c, GET(8), 0x8771f681, 11) STEP(H, c, d, a, b, GET(11), 0x6d9d6122, 16) STEP(H2, b, c, d, a, GET(14), 0xfde5380c, 23) STEP(H, a, b, c, d, GET(1), 0xa4beea44, 4) STEP(H2, d, a, b, c, GET(4), 0x4bdecfa9, 11) STEP(H, c, d, a, b, GET(7), 0xf6bb4b60, 16) STEP(H2, b, c, d, a, GET(10), 0xbebfbc70, 23) STEP(H, a, b, c, d, GET(13), 0x289b7ec6, 4) STEP(H2, d, a, b, c, GET(0), 0xeaa127fa, 11) STEP(H, c, d, a, b, GET(3), 0xd4ef3085, 16) STEP(H2, b, c, d, a, GET(6), 0x04881d05, 23) STEP(H, a, b, c, d, GET(9), 0xd9d4d039, 4) STEP(H2, d, a, b, c, GET(12), 0xe6db99e5, 11) STEP(H, c, d, a, b, GET(15), 0x1fa27cf8, 16) STEP(H2, b, c, d, a, GET(2), 0xc4ac5665, 23) /* Round 4 */ STEP(I, a, b, c, d, GET(0), 0xf4292244, 6) STEP(I, d, a, b, c, GET(7), 0x432aff97, 10) STEP(I, c, d, a, b, GET(14), 0xab9423a7, 15) STEP(I, b, c, d, a, GET(5), 0xfc93a039, 21) STEP(I, a, b, c, d, GET(12), 0x655b59c3, 6) STEP(I, d, a, b, c, GET(3), 0x8f0ccc92, 10) STEP(I, c, d, a, b, GET(10), 0xffeff47d, 15) STEP(I, b, c, d, a, GET(1), 0x85845dd1, 21) STEP(I, a, b, c, d, GET(8), 0x6fa87e4f, 6) STEP(I, d, a, b, c, GET(15), 0xfe2ce6e0, 10) STEP(I, c, d, a, b, GET(6), 0xa3014314, 15) STEP(I, b, c, d, a, GET(13), 0x4e0811a1, 21) STEP(I, a, b, c, d, GET(4), 0xf7537e82, 6) STEP(I, d, a, b, c, GET(11), 0xbd3af235, 10) STEP(I, c, d, a, b, GET(2), 0x2ad7d2bb, 15) STEP(I, b, c, d, a, GET(9), 0xeb86d391, 21) a += saved_a; b += saved_b; c += saved_c; d += saved_d; ptr += 64; } while (size -= 64); ctx->a = a; ctx->b = b; ctx->c = c; ctx->d = d; return ptr; } void gfal2_md5_init(GFAL_MD5_CTX *ctx) { ctx->a = 0x67452301; ctx->b = 0xefcdab89; ctx->c = 0x98badcfe; ctx->d = 0x10325476; ctx->lo = 0; ctx->hi = 0; } void gfal2_md5_update(GFAL_MD5_CTX *ctx, const void *data, unsigned long size) { gfal2_md5_u32plus saved_lo; unsigned long used, available; saved_lo = ctx->lo; if ((ctx->lo = (saved_lo + size) & 0x1fffffff) < saved_lo) ctx->hi++; ctx->hi += size >> 29; used = saved_lo & 0x3f; if (used) { available = 64 - used; if (size < available) { memcpy(&ctx->buffer[used], data, size); return; } memcpy(&ctx->buffer[used], data, available); data = (const unsigned char *)data + available; size -= available; body(ctx, ctx->buffer, 64); } if (size >= 64) { data = body(ctx, data, size & ~(unsigned long)0x3f); size &= 0x3f; } memcpy(ctx->buffer, data, size); } void gfal2_md5_final(unsigned char *result, GFAL_MD5_CTX *ctx) { unsigned long used, available; used = ctx->lo & 0x3f; ctx->buffer[used++] = 0x80; available = 64 - used; if (available < 8) { memset(&ctx->buffer[used], 0, available); body(ctx, ctx->buffer, 64); used = 0; available = 64; } memset(&ctx->buffer[used], 0, available - 8); ctx->lo <<= 3; ctx->buffer[56] = ctx->lo; ctx->buffer[57] = ctx->lo >> 8; ctx->buffer[58] = ctx->lo >> 16; ctx->buffer[59] = ctx->lo >> 24; ctx->buffer[60] = ctx->hi; ctx->buffer[61] = ctx->hi >> 8; ctx->buffer[62] = ctx->hi >> 16; ctx->buffer[63] = ctx->hi >> 24; body(ctx, ctx->buffer, 64); result[0] = ctx->a; result[1] = ctx->a >> 8; result[2] = ctx->a >> 16; result[3] = ctx->a >> 24; result[4] = ctx->b; result[5] = ctx->b >> 8; result[6] = ctx->b >> 16; result[7] = ctx->b >> 24; result[8] = ctx->c; result[9] = ctx->c >> 8; result[10] = ctx->c >> 16; result[11] = ctx->c >> 24; result[12] = ctx->d; result[13] = ctx->d >> 8; result[14] = ctx->d >> 16; result[15] = ctx->d >> 24; memset(ctx, 0, sizeof(*ctx)); } void gfal2_md5_to_hex_string(const unsigned char *bytes, char *hex, size_t hex_size) { int i; const char hex_str[] = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'}; char *p = hex; for (i = 0; i < 16 && i < hex_size; i++) { *p = hex_str[((bytes[i] >> 4) & 0x0f)]; p++; *p = hex_str[bytes[i] & 0x0f]; p++; } *p = '\0'; } gfal2-v2.23.0/src/utils/checksums/checksums.h000066400000000000000000000030621465240014500210330ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #pragma once #include #ifdef __cplusplus extern "C" { #endif /** * Compares the checksum 1 with the checksum 2 not taking into * account case and heading zeros. * The return value is the same as for strncasecmp */ int gfal_compare_checksums(const char* chk1, const char* chk2, size_t len); // md5 checksum calculation typedef unsigned int gfal2_md5_u32plus; typedef struct { gfal2_md5_u32plus lo, hi; gfal2_md5_u32plus a, b, c, d; unsigned char buffer[64]; gfal2_md5_u32plus block[16]; } GFAL_MD5_CTX; void gfal2_md5_init(GFAL_MD5_CTX *ctx); void gfal2_md5_update(GFAL_MD5_CTX *ctx, const void *data, unsigned long size); void gfal2_md5_final(unsigned char *result, GFAL_MD5_CTX *ctx); void gfal2_md5_to_hex_string(const unsigned char *bytes, char *hex, size_t hex_size); #ifdef __cplusplus } #endif gfal2-v2.23.0/src/utils/exceptions/000077500000000000000000000000001465240014500170705ustar00rootroot00000000000000gfal2-v2.23.0/src/utils/exceptions/cpp_to_gerror.hpp000066400000000000000000000031731465240014500224510ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2015 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #pragma once #ifndef CPP_TO_GERROR_HPP #define CPP_TO_GERROR_HPP #include #include #include namespace Gfal{ #define CPP_GERROR_TRY do{ \ try{ #define CPP_GERROR_CATCH(my_err_caught) } \ catch (Gfal::TransferException & e) {\ gfalt_set_error(my_err_caught, e.domain(), e.code(), __func__, e.side.c_str(), e.note.c_str(), "%s", e.what()); \ }catch(const Gfal::CoreException& e){ \ gfal2_set_error(my_err_caught, e.domain(), e.code(), __func__, "%s", e.what()); \ }catch(std::exception & e){ \ gfal2_set_error(my_err_caught, gfal2_get_core_quark(), EPROTONOSUPPORT, __func__, "%s", e.what()); \ }catch(...){ \ gfal2_set_error(my_err_caught, gfal2_get_core_quark(), EIO, __func__, "Undefined Exception caught: Bug found !! "); \ } \ }while(0) } #endif /* CPP_TO_GERROR_HPP */ gfal2-v2.23.0/src/utils/exceptions/gerror_to_cpp.cpp000066400000000000000000000017061465240014500224440ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2015 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "gerror_to_cpp.h" #include #include void Gfal::gerror_to_cpp(GError ** err){ g_return_if_fail(err != NULL); if(*err != NULL){ throw Gfal::CoreException(*err); } } gfal2-v2.23.0/src/utils/exceptions/gerror_to_cpp.h000066400000000000000000000016141465240014500221070ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #pragma once #ifndef GERROR_TO_CPP_H #define GERROR_TO_CPP_H #include namespace Gfal{ void gerror_to_cpp(GError ** err); } #endif /* GERROR_TO_CPP_H */ gfal2-v2.23.0/src/utils/exceptions/gfalcoreexception.cpp000066400000000000000000000026031465240014500232760ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2015 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "gfalcoreexception.hpp" Gfal::CoreException::CoreException(GQuark scope, int code, const std::string& msg): _scope(scope), _msg(msg), _code(code) { } Gfal::CoreException::CoreException(const GError* error): _scope(error->domain), _msg(error->message), _code(error->code) { } Gfal::CoreException::~CoreException() throw () { } GQuark Gfal::CoreException::domain() const throw () { return _scope; } const char* Gfal::CoreException::what() const throw () { return _msg.c_str(); } const std::string& Gfal::CoreException::what_str() const throw () { return _msg; } int Gfal::CoreException::code() const throw () { return _code; } gfal2-v2.23.0/src/utils/exceptions/gfalcoreexception.hpp000066400000000000000000000033501465240014500233030ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2015 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #pragma once #ifndef GFALCOREEXCEPTION_H #define GFALCOREEXCEPTION_H #include #include #include namespace Gfal{ class CoreException: public std::exception { public: CoreException(GQuark scope, int code, const std::string & msg); CoreException(const GError* error); virtual ~CoreException() throw(); virtual GQuark domain() const throw (); virtual const char* what() const throw(); virtual const std::string& what_str() const throw(); virtual int code() const throw(); private: GQuark _scope; std::string _msg; int _code; }; class TransferException: public CoreException { public: std::string side; std::string note; TransferException(GQuark scope, int code, const std::string & msg, const std::string & side, const std::string & note = std::string()): CoreException(scope, code, msg), side(side), note(note) { } virtual ~TransferException() throw() { } }; } #endif /* GFALCOREEXCEPTION_H */ gfal2-v2.23.0/src/utils/gsimplecache/000077500000000000000000000000001465240014500173335ustar00rootroot00000000000000gfal2-v2.23.0/src/utils/gsimplecache/gcachemain.c000066400000000000000000000107171465240014500215640ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include #include #include "gcachemain.h" static const guint64 max_list_len = MAX_LIST_LEN; typedef struct _Internal_item{ int ref_count; char item[]; } Internal_item; struct _GSimpleCache_Handle{ GHashTable* table; GSimpleCache_CopyConstructor do_copy; size_t size_item; size_t max_number_item; pthread_mutex_t mux; }; static void gsimplecache_destroy_item_internal(gpointer a){ Internal_item* i = (Internal_item*) a; g_free(i); } static gboolean hash_strings_are_equals(gconstpointer a, gconstpointer b){ return (strcmp((char*) a, (char*) b)== 0); } /** * Construct a new cache with a capacity of max_size bytes * */ GSimpleCache* gsimplecache_new(guint64 max_number_item, GSimpleCache_CopyConstructor value_copy, size_t size_item){ GSimpleCache* ret = (GSimpleCache*) g_new(struct _GSimpleCache_Handle,1); ret->table = g_hash_table_new_full(&g_str_hash, &hash_strings_are_equals, &free, &gsimplecache_destroy_item_internal ); ret->do_copy = value_copy; ret->size_item = size_item; ret->max_number_item = max_number_item; pthread_mutex_init(&ret->mux,NULL); return ret; } /** * delete a cache object, all internals object are free * */ void gsimplecache_delete(GSimpleCache* cache){ if(cache != NULL){ pthread_mutex_lock(&cache->mux); g_hash_table_destroy (cache->table); pthread_mutex_unlock(&cache->mux); pthread_mutex_destroy(&cache->mux); g_free(cache); } } Internal_item* gsimplecache_find_kstr_internal(GSimpleCache* cache, const char* key){ Internal_item* ret = (Internal_item*) g_hash_table_lookup(cache->table, (gconstpointer) key); if(ret != NULL ){ return ret; } return NULL; } static gboolean gsimplecache_remove_internal_kstr(GSimpleCache* cache, const char* key){ return g_hash_table_remove(cache->table, (gconstpointer) key); } // simple lazy space maker, can be improved static void gsimplecache_manage_space(GSimpleCache* cache){ size_t len = (size_t) g_hash_table_size (cache->table); if(len >= cache->max_number_item){ g_hash_table_remove_all(cache->table); } } void gsimplecache_add_item_internal(GSimpleCache* cache, const char* key, void* item){ Internal_item* ret = gsimplecache_find_kstr_internal(cache, key); if(ret == NULL){ gsimplecache_manage_space(cache); ret = malloc(sizeof(struct _Internal_item) + cache->size_item); ret->ref_count = 2; cache->do_copy(item, ret->item); g_hash_table_insert(cache->table, strdup(key), ret); }else{ (ret->ref_count)++; } } /** * Add an item to the cache or increment the reference of this item of one if already exist * */ void gsimplecache_add_item_kstr(GSimpleCache* cache, const char* key, void* item){ pthread_mutex_lock(&cache->mux); gsimplecache_add_item_internal(cache, key, item); pthread_mutex_unlock(&cache->mux); } /** * remove the item in the cache, return TRUE if removed else FALSE * destroy the internal item automatically * */ gboolean gsimplecache_remove_kstr(GSimpleCache* cache, const char* key){ pthread_mutex_lock(&cache->mux); gboolean ret = gsimplecache_remove_internal_kstr(cache, key); pthread_mutex_unlock(&cache->mux); return ret; } /** * find the value in the cache, and decrease its internal reference count of 1. * If the item exist, set the item resu to the correct value and return 0 else return -1 * * */ int gsimplecache_take_one_kstr(GSimpleCache* cache, const char* key, void* res){ pthread_mutex_lock(&cache->mux); Internal_item* ret = gsimplecache_find_kstr_internal(cache, key); if(ret){ (ret->ref_count)--; cache->do_copy(ret->item, res); if(ret->ref_count <= 0) gsimplecache_remove_internal_kstr(cache, key); } pthread_mutex_unlock(&cache->mux); return (ret)?0:-1; } gfal2-v2.23.0/src/utils/gsimplecache/gcachemain.h000066400000000000000000000025561465240014500215730ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #pragma once #include #define MAX_LIST_LEN 20000 /** * copy the original object to a new one */ typedef void (*GSimpleCache_CopyConstructor)(gpointer original, gpointer copy); typedef struct _GSimpleCache_Handle GSimpleCache; GSimpleCache* gsimplecache_new(guint64 max_number_item, GSimpleCache_CopyConstructor value_copy, size_t size_item); void gsimplecache_delete(GSimpleCache* cache); void gsimplecache_add_item_kstr(GSimpleCache* cache, const char* key, void* item); int gsimplecache_take_one_kstr(GSimpleCache* cache, const char* key, void* res); gboolean gsimplecache_remove_kstr(GSimpleCache* cache, const char* key); gfal2-v2.23.0/src/utils/mds/000077500000000000000000000000001465240014500154725ustar00rootroot00000000000000gfal2-v2.23.0/src/utils/mds/gfal_mds.c000066400000000000000000000120361465240014500174140ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include #include #include #include "gfal_mds_internal.h" pthread_mutex_t m_mds =PTHREAD_MUTEX_INITIALIZER; const char* bdii_env_var = "LCG_GFAL_INFOSYS"; const char* bdii_config_var = "LCG_GFAL_INFOSYS"; const char* bdii_config_group = "BDII"; const char* bdii_config_enable = "ENABLED"; const char* bdii_config_timeout = "TIMEOUT"; gboolean gfal_get_nobdiiG(gfal2_context_t handle) { return (!gfal2_get_opt_boolean_with_default(handle, bdii_config_group, bdii_config_enable, TRUE)); } /* * define the bdii endpoint for the current handle * same purpose that the old LCG_GFAL_INFOSYS environment variable */ void gfal_mds_set_infosys(gfal2_context_t handle, const char * infosys, GError** err){ g_return_if_fail(handle && infosys); // no manner to define infosys in is interface currently, just setup the env var, // TODO : change this in is-interface and integrated module g_setenv(bdii_env_var, infosys, TRUE); } void gfal_mds_define_bdii_endpoint(gfal2_context_t handle, GError** err){ if(g_getenv(bdii_env_var) == NULL){ pthread_mutex_lock(&m_mds); gchar * bdii_host = gfal2_get_opt_string(handle,bdii_config_group, bdii_config_var,NULL); if(bdii_host ){ gfal2_log(G_LOG_LEVEL_DEBUG, " define LCG_GFAL_INFOSYS : %s", bdii_host); gfal_mds_set_infosys (handle, bdii_host, NULL); } g_free(bdii_host); pthread_mutex_unlock(&m_mds); } } /* * return the srm endpoints and their types, in the old way * */ int gfal_mds_get_se_types_and_endpoints (gfal2_context_t handle, const char *host, char ***se_types, char ***se_endpoints, GError** err){ GError* tmp_err = NULL; gfal_mds_endpoint tabend[GFAL_MDS_MAX_SRM_ENDPOINT]; int n = gfal_mds_resolve_srm_endpoint(handle, host, tabend, GFAL_MDS_MAX_SRM_ENDPOINT, &tmp_err); if (n > 0) { int i; *se_types = calloc(n + 1, sizeof(char*)); *se_endpoints = calloc(n + 1, sizeof(char*)); for (i = 0; i < n; ++i) { (*se_endpoints)[i] = strdup(tabend[i].url); (*se_types)[i] = strdup(((tabend[i].type == SRMv2) ? "srm_v2" : "srm_v1")); } } if (tmp_err) g_propagate_prefixed_error(err, tmp_err, "[%s]", __func__); return (n > 0) ? 0 : -1; } #if MDS_BDII_EXTERNAL /* * external call to the is interface for external bdii resolution * */ int gfal_mds_isifce_wrapper(const char* base_url, gfal_mds_endpoint* endpoints, size_t s_endpoint, GError** err){ char ** name_endpoints; char** types_endpoints; char errbuff[GFAL_ERRMSG_LEN]= {0}; GError* tmp_err=NULL; int res = -1; if(sd_get_se_types_and_endpoints(base_url, &types_endpoints, &name_endpoints, errbuff, GFAL_ERRMSG_LEN-1) != 0){ g_set_error(&tmp_err, gfal2_get_core_quark(), ENXIO, "IS INTERFACE ERROR : %s ", errbuff); }else{ int i; char ** p1 =name_endpoints; char **p2 =types_endpoints; for(i=0; i< s_endpoint && *p1 != NULL && *p2 != NULL; ++i,++p2,++p1){ g_strlcpy(endpoints[i].url, *p1, GFAL_URL_MAX_LEN); endpoints[i].type = (strcmp(*p2, "srm_v2")==0)?SRMv2:SRMv1; } res =i+1; } if(tmp_err) g_propagate_prefixed_error(err, tmp_err, "[%s]", __func__); return res; } #endif int gfal_mds_resolve_srm_endpoint(gfal2_context_t handle, const char* base_url, gfal_mds_endpoint* endpoints, size_t s_endpoint, GError** err) { #ifndef MDS_WITHOUT_CACHE int cached_result = gfal_mds_cache_resolve_endpoint(handle, base_url, endpoints, s_endpoint, err); if (cached_result < 0) { return cached_result; } else if (cached_result > 0) { int i; gfal2_log(G_LOG_LEVEL_DEBUG, "%s found in the cache!", base_url); for (i = 0; i < cached_result; ++i) { gfal2_log(G_LOG_LEVEL_DEBUG, "\tFound %s", endpoints[i].url); } return cached_result; } #endif #if MDS_BDII_EXTERNAL // call the is interface if configured for gfal_mds_define_bdii_endpoint(handle, err); if(err && *err==NULL) return gfal_mds_isifce_wrapper(base_url, endpoints, s_endpoint, err); return NULL; #else return gfal_mds_bdii_get_srm_endpoint(handle, base_url, endpoints, s_endpoint, err); #endif } gfal2-v2.23.0/src/utils/mds/gfal_mds.h000066400000000000000000000033661465240014500174270ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #pragma once #define GFAL_MDS_MAX_SRM_ENDPOINT 100 #include #include #ifdef __cplusplus extern "C" { #endif #ifndef MDS_BDII_EXTERNAL #define MDS_BDII_EXTERNAL 0 #endif typedef enum { SRMv2=0, SRMv1, WebDav, UnknownEndpointType } mds_type_endpoint; /* * @struct gfal_mds_endpoint * represente an endpoint URL and its type */ typedef struct _gfal_mds_endpoint{ char url[GFAL_URL_MAX_LEN]; mds_type_endpoint type; } gfal_mds_endpoint; extern const char* bdii_env_var; extern const char* bdii_config_var; extern const char* bdii_config_group; extern const char* bdii_config_timeout; // int gfal_mds_resolve_srm_endpoint(gfal2_context_t handle, const char* base_url, gfal_mds_endpoint* endpoints, size_t s_endpoint, GError** err); // deprecated, use gfal_mds_resolve_srm_endpoint instead int gfal_mds_get_se_types_and_endpoints(gfal2_context_t handle, const char *host, char ***se_types, char ***se_endpoints, GError** err); gboolean gfal_get_nobdiiG(gfal2_context_t handle); #ifdef __cplusplus } #endif gfal2-v2.23.0/src/utils/mds/gfal_mds_cache.cpp000066400000000000000000000077041465240014500211050ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2015 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include #include "gfal_mds_internal.h" const char* bdii_cache_file = "CACHE_FILE"; static mds_type_endpoint gfal_mds_cache_type(const std::string& type, const std::string &version) { if (strcasecmp(type.c_str(), "srm") == 0) { if (version.compare(0, 2, "1.") == 0) return SRMv1; else if (version.compare(0, 2, "2.") == 0) return SRMv2; else return UnknownEndpointType; } else if (strcasecmp(type.c_str(), "webdav") == 0) { return WebDav; } else { return UnknownEndpointType; } } static void gfal_mds_cache_insert(gfal_mds_endpoint* endpoints, size_t s_endpoints, size_t index, const pugi::xml_node& entry) { if (index > s_endpoints) return; std::string endpoint = entry.child("endpoint").last_child().value(); std::string type = entry.child("type").last_child().value(); std::string version = entry.child("version").last_child().value(); mds_type_endpoint typeEnum = gfal_mds_cache_type(type, version); if (!endpoint.empty() && typeEnum != UnknownEndpointType) { g_strlcpy(endpoints[index].url, endpoint.c_str(), sizeof(endpoints[index].url)); endpoints[index].type = typeEnum; } } int gfal_mds_cache_resolve_endpoint(gfal2_context_t handle, const char* host, gfal_mds_endpoint* endpoints, size_t s_endpoints, GError** err) { // Cache configuration gchar *cache_file = gfal2_get_opt_string(handle, bdii_config_group, bdii_cache_file, NULL); if (!cache_file) return 0; gfal2_log(G_LOG_LEVEL_DEBUG, "BDII CACHE_FILE set to %s", cache_file); // Open the file, but do not fail if it can not be open // (A cache may not be present!) pugi::xml_document cache; pugi::xml_parse_result loadResult = cache.load_file(cache_file); g_free(cache_file); if (loadResult.status != pugi::status_ok) { gfal2_log(G_LOG_LEVEL_DEBUG, "Could not load BDII CACHE_FILE: %s", loadResult.description()); return 0; } // Iterate. Get all endpoints and check if the contain the requested host size_t hostLen = strlen(host); pugi::xpath_node_set allEndpoints = cache.document_element().select_nodes("/entry/endpoint"); pugi::xpath_node_set::const_iterator i; size_t endpointIndex = 0; for (i = allEndpoints.begin(); i != allEndpoints.end() && endpointIndex < s_endpoints; ++i) { const char* endpoint = i->node().child_value(); const char* hostname = strstr(endpoint, "://"); if (hostname) hostname += 3; else hostname = endpoint; if (strncasecmp(hostname, host, hostLen) == 0) { std::string xpathQuery("/entry[endpoint='"); xpathQuery += endpoint; xpathQuery += "']"; pugi::xpath_node entry = cache.document_element().select_single_node(xpathQuery.c_str()); gfal_mds_cache_insert(endpoints, s_endpoints, endpointIndex++, entry.node()); } } // Done here return endpointIndex; } gfal2-v2.23.0/src/utils/mds/gfal_mds_internal.c000066400000000000000000000240011465240014500213030ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include #include #include #include #include "gfal_mds_internal.h" #include "gfal_mds_ldap_internal_layer.h" #ifndef ECOMM #define ECOMM EIO #endif static char* tabattr[] = {"GlueServiceVersion", "GlueServiceEndpoint", "GlueServiceType", NULL}; static const char *sbasedn = "o=grid"; static const char* srm_endpoint_filter = "(|(GlueSEUniqueID=*%s*)(&(GlueServiceType=srm*)(GlueServiceEndpoint=*://%s*)))"; static const char* SRM_PREFIX_NAME="SRM"; static pthread_mutex_t mux_init_lap = PTHREAD_MUTEX_INITIALIZER; LDAP *gfal_mds_ldap_connect(gfal2_context_t context, const char *uri, GError **err) { g_return_val_err_if_fail(uri != NULL, NULL, err, "invalid arg uri"); LDAP *ld = NULL; GError *tmp_err = NULL; int rc; pthread_mutex_lock(&mux_init_lap); // libldap suffers of a thread-safety bug inside initialize function if ((rc = gfal_mds_ldap.ldap_initialize(&ld, uri)) != LDAP_SUCCESS) { g_set_error(&tmp_err, gfal2_get_core_quark(), ECOMM, "Error with contacting ldap %s : %s", uri, ldap_err2string(rc)); } else { struct timeval timeout = {0}; timeout.tv_sec = gfal2_get_opt_integer_with_default(context, bdii_config_group, bdii_config_timeout, -1); gfal_mds_ldap.ldap_set_option(ld, LDAP_OPT_NETWORK_TIMEOUT, &timeout); gfal_mds_ldap.ldap_set_option(ld, LDAP_OPT_TIMEOUT, &timeout); gfal2_log(G_LOG_LEVEL_DEBUG, " use BDII TIMEOUT : %lld", (long long) timeout.tv_sec); gfal2_log(G_LOG_LEVEL_DEBUG, " Try to bind with the bdii %s", uri); struct berval cred = {.bv_val = NULL, .bv_len = 0}; if ((rc = gfal_mds_ldap.ldap_sasl_bind_s(ld, NULL, LDAP_SASL_SIMPLE, &cred, NULL, NULL, NULL)) != LDAP_SUCCESS) { g_set_error(&tmp_err, gfal2_get_core_quark(), ECOMM, "Error while bind to bdii with %s : %s", uri, ldap_err2string(rc)); ld = NULL; } } pthread_mutex_unlock(&mux_init_lap); G_RETURN_ERR(ld, tmp_err, err); } /* * Execute a ldap query on a connected bdii * */ int gfal_mds_ldap_search(LDAP *ld, const char *basedn, const char *filter, char **tabattr, LDAPMessage **res, GError **err) { GError *tmp_err = NULL; int ret = -1; int rc; if ((rc = gfal_mds_ldap.ldap_search_ext_s(ld, basedn, LDAP_SCOPE_SUBTREE, filter, tabattr, 0, NULL, NULL, LDAP_NO_LIMIT, LDAP_NO_LIMIT, res)) != LDAP_SUCCESS) { g_set_error(&tmp_err, gfal2_get_core_quark(), ECOMM, "Error while request %s to bdii : %s", filter, ldap_err2string(rc)); } else ret = 0; if (tmp_err) g_propagate_prefixed_error(err, tmp_err, "[%s]", __func__); return ret; } /* * convert bdii returned value to a srm_endpoint struct * @return 0 if success else -1 * */ static int gfal_mds_srm_endpoint_struct_builder(char *srm_name, char *srm_version, char *srm_endpoint, gfal_mds_endpoint *endpoints, GError **err) { int ret = 0; GError *tmp_err = NULL; if (strncasecmp(srm_name, SRM_PREFIX_NAME, strlen(SRM_PREFIX_NAME)) != 0) { g_set_error(&tmp_err, gfal2_get_core_quark(), EINVAL, "bad value of srm endpoint returned by bdii : %s, excepted : %s ", srm_name, SRM_PREFIX_NAME); ret = -1; } else { if (strncmp(srm_version, "1.", 2) == 0) endpoints->type = SRMv1; else if (strncmp(srm_version, "2.", 2) == 0) endpoints->type = SRMv2; else { g_set_error(&tmp_err, gfal2_get_core_quark(), EINVAL, "bad value of srm version returned by bdii : %s, excepted 1.x or 2.x ", srm_version); ret = -1; } if (strstr(srm_endpoint, ":/") != NULL) g_strlcpy(endpoints->url, srm_endpoint, GFAL_URL_MAX_LEN); else { g_set_error(&tmp_err, gfal2_get_core_quark(), EINVAL, "bad value of srm endpoint returned by bdii : %s, excepted a correct endpoint url ( httpg://, https://, ... ) ", srm_endpoint); ret = -1; } } if (tmp_err) g_propagate_prefixed_error(err, tmp_err, "[%s]", __func__); return ret; } /* * Analyse attr fields * return > 0 if success, 0 if not it is an empty entry or -1 if error * */ static int gfal_mds_convert_entry_to_srm_information(LDAP *ld, LDAPMessage *entry, gfal_mds_endpoint *endpoints, GError **err) { struct berval **vals; char *a; int ret = 0; GError *tmp_err = NULL; BerElement *ber; char srm_name[GFAL_URL_MAX_LEN + 1] = {0}; char srm_version[GFAL_URL_MAX_LEN + 1] = {0}; char srm_endpoint[GFAL_URL_MAX_LEN + 1] = {0}; for (a = gfal_mds_ldap.ldap_first_attribute(ld, entry, &ber); a != NULL; a = gfal_mds_ldap.ldap_next_attribute(ld, entry, ber)) { // for each attribute if ((vals = gfal_mds_ldap.ldap_get_values_len(ld, entry, a)) != NULL) { if (strncmp(a, "GlueServiceVersion", GFAL_URL_MAX_LEN) == 0) { g_strlcpy(srm_version, vals[0]->bv_val, sizeof(srm_version)); ret += 1; } else if (strncmp(a, "GlueServiceEndpoint", GFAL_URL_MAX_LEN) == 0) { g_strlcpy(srm_endpoint, vals[0]->bv_val, sizeof(srm_endpoint)); ret += 1; } else if (strncmp(a, "GlueServiceType", GFAL_URL_MAX_LEN) == 0) { g_strlcpy(srm_name, vals[0]->bv_val, sizeof(srm_name)); ret += 1; } else { g_set_error(&tmp_err, gfal2_get_core_quark(), EINVAL, " Bad attribute retrieved from bdii "); gfal_mds_ldap.ldap_value_free_len(vals); gfal_mds_ldap.ldap_memfree(a); ret = -1; break; } gfal_mds_ldap.ldap_value_free_len(vals); } gfal_mds_ldap.ldap_memfree(a); } if (ber) gfal_mds_ldap.ber_free(ber, 0); if (ret > 0) ret = (gfal_mds_srm_endpoint_struct_builder(srm_name, srm_version, srm_endpoint, endpoints, &tmp_err) == 0) ? ret : -1; if (tmp_err) g_propagate_prefixed_error(err, tmp_err, "[%s]", __func__); return ret; } /* * * parse the result of a query to get the srm endpoint */ int gfal_mds_get_srm_types_endpoint(LDAP *ld, LDAPMessage *result, gfal_mds_endpoint *endpoints, size_t s_endpoint, GError **err) { GError *tmp_err = NULL; int ret = 0; int tmp_ret; int i = 0; if ((tmp_ret = gfal_mds_ldap.ldap_count_entries(ld, result)) >= 1) { LDAPMessage *e = gfal_mds_ldap.ldap_first_entry(ld, result); while (e != NULL && i < s_endpoint) { /* Iterate through each attribute in the entry. */ if ((tmp_ret = gfal_mds_convert_entry_to_srm_information(ld, e, &endpoints[i], &tmp_err)) < 0) { ret = -1; break; } if (tmp_ret > 0) { // if 0 returned, empty field, try again without increment i++; ret++; } e = gfal_mds_ldap.ldap_next_entry(ld, e); } } else if (tmp_ret == -1) { int myld_errno = 0; ldap_get_option(ld, LDAP_OPT_RESULT_CODE, &myld_errno); g_set_error(&tmp_err, gfal2_get_core_quark(), EINVAL, " error returned in ldap results : %s", ldap_err2string(myld_errno)); ret = -1; } else { g_set_error(&tmp_err, gfal2_get_core_quark(), ENXIO, " no entries for the endpoint returned by the bdii : %d ", tmp_ret); ret = -1; } if (tmp_err) g_propagate_prefixed_error(err, tmp_err, "[%s]", __func__); return ret; } /* * * get the current ldap URI **/ int gfal_mds_get_ldapuri(gfal2_context_t context, char *buff, size_t s_buff, GError **err) { char *bdii_var = g_strdup(getenv(bdii_env_var)); if (bdii_var == NULL) { bdii_var = gfal2_get_opt_string(context, bdii_config_group, bdii_config_var, NULL); } if (bdii_var == NULL || bdii_var[0] == '\0') { g_set_error(err, gfal2_get_core_quark(), EINVAL, " no valid value for BDII found:" " please, configure the plugin properly, or try setting in the environment LCG_GFAL_INFOSYS"); g_free(bdii_var); return -1; } gfal2_log(G_LOG_LEVEL_DEBUG, " use LCG_GFAL_INFOSYS : %s", bdii_var); char *save_ptr = NULL; char *token = NULL; size_t i = 0; buff[i] = '\0'; token = strtok_r(bdii_var, ",", &save_ptr); while (token && i < s_buff) { i += g_strlcpy(buff + i, "ldap://", s_buff - i); i += g_strlcpy(buff + i, token, s_buff - i); i += g_strlcpy(buff + i, ",", s_buff - i); token = strtok_r(NULL, ",", &save_ptr); } // Remove trailing ',' buff[i - 1] = '\0'; g_free(bdii_var); return 0; } void gfal_mds_ldap_disconnect(LDAP *ld) { gfal_mds_ldap.ldap_unbind_ext_s(ld, NULL, NULL); } /* * resolve the SRM endpoint associated with a given base_url with the bdii * @param base_url : basic url to resolve * @param endpoints : table of gfal_mds_endpoint to set with a size of s_endpoint * @param s_endpoint : maximum number of endpoints to set * @param err: Gerror system for the report of the errors. * @return : number of endpoints set or -1 if error */ int gfal_mds_bdii_get_srm_endpoint(gfal2_context_t context, const char *base_url, gfal_mds_endpoint *endpoints, size_t s_endpoint, GError **err) { int ret = -1; GError *tmp_err = NULL; char uri[GFAL_URL_MAX_LEN]; LDAP *ld; gfal2_log(G_LOG_LEVEL_DEBUG, " gfal_mds_bdii_get_srm_endpoint ->"); if (gfal_mds_get_ldapuri(context, uri, GFAL_URL_MAX_LEN, &tmp_err) >= 0) { if ((ld = gfal_mds_ldap_connect(context, uri, &tmp_err)) != NULL) { LDAPMessage *res; char buff_filter[GFAL_URL_MAX_LEN]; snprintf(buff_filter, GFAL_URL_MAX_LEN, srm_endpoint_filter, base_url, base_url); // construct the request if (gfal_mds_ldap_search(ld, sbasedn, buff_filter, tabattr, &res, &tmp_err) >= 0) { ret = gfal_mds_get_srm_types_endpoint(ld, res, endpoints, s_endpoint, &tmp_err); gfal_mds_ldap.ldap_msgfree(res); } gfal_mds_ldap_disconnect(ld); } } gfal2_log(G_LOG_LEVEL_DEBUG, " gfal_mds_bdii_get_srm_endpoint <-"); if (tmp_err) g_propagate_prefixed_error(err, tmp_err, "[%s]", __func__); return ret; } gfal2-v2.23.0/src/utils/mds/gfal_mds_internal.h000066400000000000000000000033541465240014500213200ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #pragma once #include #include #include #include "gfal_mds.h" #ifdef __cplusplus extern "C" { #endif LDAP* gfal_mds_ldap_connect(gfal2_context_t context, const char* uri, GError** err); int gfal_mds_get_ldapuri(gfal2_context_t context, char* buff, size_t s_buff, GError** err); int gfal_mds_get_srm_types_endpoint(LDAP* ld, LDAPMessage *result, gfal_mds_endpoint* endpoints, size_t s_endpoint, GError** err); int gfal_mds_bdii_get_srm_endpoint(gfal2_context_t handle, const char* base_url, gfal_mds_endpoint* endpoints, size_t s_endpoint, GError** err); #ifndef MDS_WITHOUT_CACHE /** Tries to resolve the available endpoints from a cache file * compatible with FTS3 bdii cache format * @return The number of entries found, -1 on error. */ int gfal_mds_cache_resolve_endpoint(gfal2_context_t handle, const char* host, gfal_mds_endpoint* endpoints, size_t s_endpoints, GError** err); #endif #ifdef __cplusplus } #endif gfal2-v2.23.0/src/utils/mds/gfal_mds_ldap_internal_layer.c000066400000000000000000000027121465240014500235040ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include "gfal_mds_ldap_internal_layer.h" struct _gfal_mds_ldap gfal_mds_ldap={ .ldap_initialize= ldap_initialize, .ldap_sasl_bind_s = ldap_sasl_bind_s, .ldap_search_ext_s = ldap_search_ext_s, .ldap_unbind_ext_s = ldap_unbind_ext_s, .ldap_first_entry= ldap_first_entry, .ldap_next_entry= ldap_next_entry, .ldap_first_attribute = ldap_first_attribute, .ldap_next_attribute = ldap_next_attribute, .ldap_get_values_len= ldap_get_values_len, .ldap_value_free_len = ldap_value_free_len, .ldap_memfree = ldap_memfree, .ldap_msgfree = ldap_msgfree, .ber_free = ber_free, .ldap_count_entries = ldap_count_entries, .ldap_set_option = ldap_set_option }; gfal2-v2.23.0/src/utils/mds/gfal_mds_ldap_internal_layer.h000066400000000000000000000042701465240014500235120ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include struct _gfal_mds_ldap{ int (*ldap_initialize)(LDAP **ldp, const char *uri); int (*ldap_sasl_bind_s)(LDAP *ld, const char *dn, const char *mechanism, struct berval *cred, LDAPControl *sctrls[], LDAPControl *cctrls[], struct berval **servercredp); int (*ldap_search_ext_s)( LDAP *ld, LDAP_CONST char *base, int scope, LDAP_CONST char *filter, char **attrs, int attrsonly, LDAPControl **serverctrls, LDAPControl **clientctrls, struct timeval *timeout, int sizelimit, LDAPMessage **res ); int (*ldap_unbind_ext_s) ( LDAP *ld, LDAPControl **serverctrls, LDAPControl **clientctrls); LDAPMessage* (*ldap_first_entry)( LDAP *ld, LDAPMessage *result ); LDAPMessage* (*ldap_next_entry)( LDAP *ld, LDAPMessage *entry ); int (*ldap_count_entries)( LDAP *ld, LDAPMessage *result ); char* (*ldap_first_attribute)( LDAP *ld, LDAPMessage *entry, BerElement **berptr ); char* (*ldap_next_attribute)( LDAP *ld, LDAPMessage *entry, BerElement *ber ); struct berval **(*ldap_get_values_len)(LDAP *ld, LDAPMessage *entry, const char *attr); void (*ldap_value_free_len) ( struct berval **vals ); void (*ldap_memfree)(void * p); int (*ldap_msgfree)( LDAPMessage *msg ); void (*ber_free)(BerElement *ber, int freebuf); int (*ldap_set_option)(LDAP *ld, int option, const void *invalue); }; extern struct _gfal_mds_ldap gfal_mds_ldap; gfal2-v2.23.0/src/utils/network/000077500000000000000000000000001465240014500164005ustar00rootroot00000000000000gfal2-v2.23.0/src/utils/network/gfal2_network.c000066400000000000000000000100361465240014500213100ustar00rootroot00000000000000/* * Copyright (c) CERN 2022 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include "gfal2_network.h" #include "gfal_plugins_api.h" #include char* resolve_dns_helper(const char* host_uri, const char* msg) { char* resolved_str; GError *error = NULL; gfal2_uri *parsed = gfal2_parse_uri(host_uri, &error); if (error) { gfal2_log(G_LOG_LEVEL_WARNING, "Failed to parse host uri while resolving DNS alias: %s", host_uri); return NULL; } char *resolved = gfal2_resolve_dns_to_hostname(parsed->host); if (!resolved) { return NULL; } gfal2_log(G_LOG_LEVEL_INFO, "%s: %s => %s", msg, parsed->host, resolved); g_free(parsed->host); parsed->host = resolved; resolved_str = gfal2_join_uri(parsed); gfal2_free_uri(parsed); return resolved_str; } char* gfal2_resolve_dns_to_hostname(const char* dnshost) { struct addrinfo hints; struct addrinfo* addresses = NULL; struct addrinfo* addrP = NULL; GString* log_str = g_string_sized_new(512); char addrstr[INET6_ADDRSTRLEN * 2]; char hostname[256]; void* ptr = NULL; int count = 0; memset(&hints, 0, sizeof(hints)); hints.ai_family = AF_UNSPEC; hints.ai_socktype = SOCK_STREAM; hints.ai_flags |= AI_CANONNAME; int rc = getaddrinfo(dnshost, NULL, &hints, &addresses); if (rc || !addresses) { if (addresses) { freeaddrinfo(addresses); } gfal2_log(G_LOG_LEVEL_WARNING, "Could not resolve DNS alias: %s", dnshost); return NULL; } // Count and log all resolved addresses for (addrP = addresses; addrP != NULL; addrP = addrP->ai_next) { inet_ntop(addrP->ai_family, addrP->ai_addr->sa_data, addrstr, sizeof(addrstr)); switch (addrP->ai_family) { case AF_INET: ptr = &((struct sockaddr_in *) addrP->ai_addr)->sin_addr; if (ptr) { inet_ntop(addrP->ai_family, ptr, addrstr, sizeof(addrstr)); } break; case AF_INET6: ptr = &((struct sockaddr_in6 *) addrP->ai_addr)->sin6_addr; if (ptr) { inet_ntop(addrP->ai_family, ptr, addrstr, sizeof(addrstr)); } break; } // Reverse DNS. Try to translate the address to a hostname. If successful save hostname for logging if (getnameinfo(addrP->ai_addr, addrP->ai_addrlen, hostname, sizeof(hostname), NULL, 0, NI_NAMEREQD)) { gfal2_log(G_LOG_LEVEL_WARNING, "Failed reverse address %s into hostname", addrstr); } else { g_string_append_printf(log_str, "%s[%s] ", hostname, addrstr); } count++; } gfal2_log(G_LOG_LEVEL_DEBUG, "Resolved DNS alias %s into: %s", dnshost, log_str->str); g_string_free(log_str, TRUE); // Select at random an address between [0, count) srand(time(NULL)); int selected = rand() % count; for (addrP = addresses; addrP != NULL; addrP = addrP->ai_next) { if (selected-- == 0) { if (getnameinfo(addrP->ai_addr, addrP->ai_addrlen, hostname, sizeof(hostname), NULL, 0, NI_NAMEREQD)) { freeaddrinfo(addresses); gfal2_log(G_LOG_LEVEL_WARNING, "Failed reverse DNS resolution for %s ", dnshost); return NULL; } break; } } freeaddrinfo(addresses); return strdup(hostname); } gfal2-v2.23.0/src/utils/network/gfal2_network.h000066400000000000000000000020171465240014500213150ustar00rootroot00000000000000/* * Copyright (c) CERN 2022 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #pragma once #include #ifdef __cplusplus extern "C" { #endif /* * Helper function to resolve a DNS URI to a specific a host * Returns a newly allocated string with the resolved uri */ char* resolve_dns_helper(const char* host_uri, const char* msg); /* * Given a DNS alias, resolve the list of underlying addresses and select one at random. */ char* gfal2_resolve_dns_to_hostname(const char* dnshost); #ifdef __cplusplus } #endif gfal2-v2.23.0/src/utils/space/000077500000000000000000000000001465240014500160025ustar00rootroot00000000000000gfal2-v2.23.0/src/utils/space/gfal2_space.c000066400000000000000000000057741465240014500203310ustar00rootroot00000000000000/* * Copyright (c) CERN 2016 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include "gfal2_space.h" #include #include #include static const char *retention2str(enum space_retention_policy policy) { switch (policy) { case POLICY_REPLICA: return "REPLICA"; case POLICY_OUTPUT: return "OUTPUT"; case POLICY_CUSTODIAL: return "CUSTODIAL"; default: return "UNKNOWN"; } } static const char *accesslatency2str(enum space_latency latency) { switch (latency) { case LATENCY_ONLINE: return "ONLINE"; case LATENCY_NEARLINE: return "NEARLINE"; default: return "UNKNOWN"; } } size_t gfal2_space_generate_json(struct space_report *report, char *buffer, size_t s_buff) { struct json_object *root = json_object_new_object(); if (report->spacetoken) { json_object_object_add(root, "spacetoken", json_object_new_string(report->spacetoken)); } if (report->owner) { json_object_object_add(root, "owner", json_object_new_string(report->owner)); } json_object_object_add(root, "totalsize", json_object_new_int64(report->total)); json_object_object_add(root, "unusedsize", json_object_new_int64(report->free)); json_object_object_add(root, "usedsize", json_object_new_int64(report->used)); if (report->largest_chunk) { json_object_object_add(root, "guaranteedsize", json_object_new_int64(*report->largest_chunk)); } if (report->lifetime_assigned) { json_object_object_add(root, "lifetimeassigned", json_object_new_int(*report->lifetime_assigned)); } if (report->lifetime_left) { json_object_object_add(root, "lifetimeleft", json_object_new_int(*report->lifetime_left)); } if (report->retention != POLICY_UNKNOWN) { json_object_object_add(root, "retention", json_object_new_string(retention2str(report->retention))); } if (report->latency != LATENCY_UNKNOWN) { json_object_object_add(root, "accesslatency", json_object_new_string(accesslatency2str(report->latency))); } const char *serialized = json_object_to_json_string(root); g_strlcpy(buffer, serialized, s_buff); json_object_put(root); return strnlen(buffer, s_buff); } gfal2-v2.23.0/src/utils/space/gfal2_space.h000066400000000000000000000032221465240014500203200ustar00rootroot00000000000000/* * Copyright (c) CERN 2016 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #ifndef GFAL_SPACE_H_ #define GFAL_SPACE_H_ #include #ifdef __cplusplus extern "C" { #endif /* * Utilities for space reporting via extended attributes */ enum space_latency { LATENCY_UNKNOWN = 0, LATENCY_ONLINE, LATENCY_NEARLINE }; enum space_retention_policy { POLICY_UNKNOWN = 0, POLICY_REPLICA, POLICY_OUTPUT, POLICY_CUSTODIAL }; struct space_report { uint64_t used; uint64_t free; uint64_t total; // Optional values uint64_t *largest_chunk; int *lifetime_assigned; int *lifetime_left; enum space_latency latency; enum space_retention_policy retention; char *owner; char *spacetoken; }; /* * Generate a json containing the information in space_report. * Optional fields set to NULL are skipped. */ size_t gfal2_space_generate_json(struct space_report *report, char *buffer, size_t s_buff); #ifdef __cplusplus } #endif #endif // GFAL_SPACE_H_ gfal2-v2.23.0/src/utils/time_utils.h000066400000000000000000000062741465240014500172470ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #pragma once #ifndef _TIME_UTILS_H #define _TIME_UTILS_H /** similar to timeradd, timercmp, etc... function but for timerspec */ #define timespec_cmp(a, b, CMP) \ (((a)->tv_sec == (b)->tv_sec) ? \ ((a)->tv_nsec CMP (b)->tv_nsec) : \ ((a)->tv_sec CMP (b)->tv_sec)) #define timespec_add(a, b, result) \ do { \ (result)->tv_sec = (a)->tv_sec + (b)->tv_sec; \ (result)->tv_nsec = (a)->tv_nsec + (b)->tv_nsec; \ if ((result)->tv_nsec > 1000000000) { \ ++(result)->tv_sec; \ (result)->tv_nsec -= 1000000000; \ } \ } while (0) #define timespec_sub(a, b, result) \ do { \ (result)->tv_sec = (a)->tv_sec - (b)->tv_sec; \ (result)->tv_nsec = (a)->tv_nsec - (b)->tv_nsec; \ if ((result)->tv_nsec < 0) { \ --(result)->tv_sec; \ (result)->tv_nsec += 1000000000; \ } \ } while (0) #define timespec_clear(a) \ do { \ (a)->tv_sec = 0; \ (a)->tv_nsec = 0; \ } while (0) #define timespec_isset(a) \ ( !((a)->tv_sec == 0 && (a)->tv_nsec == 0) ) #define timespec_copy(a,b) \ do { \ (a)->tv_sec = (b)->tv_sec; \ (a)->tv_nsec = (b)->tv_nsec; \ } while (0) #endif gfal2-v2.23.0/src/utils/uri/000077500000000000000000000000001465240014500155065ustar00rootroot00000000000000gfal2-v2.23.0/src/utils/uri/gfal2_parsing.c000066400000000000000000000057741465240014500204050ustar00rootroot00000000000000/* * Copyright (c) CERN 2020 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "gfal2_parsing.h" gchar* gfal2_utf8escape_string(const char* str, size_t str_len, const char* ignore) { // Allocate initial string size // and relay on automatic GString reallocation further GString *escaped_str = g_string_sized_new(str_len); const char *p = str; while (p < str+str_len) { gunichar uchar = g_utf8_get_char_validated(p, str_len-(p-str)); if (uchar == (gunichar) -1 || uchar == (gunichar) -2) { g_string_append_printf(escaped_str, "\\x%02x", *(guint8 *) p); p++; continue; } if ((uchar < 32 || uchar == '\\') && (uchar == '\0' || !ignore || !strchr(ignore, uchar))) { switch (uchar) { case '\b': g_string_append(escaped_str, "\\b"); break; case '\f': g_string_append(escaped_str, "\\f"); break; case '\n': g_string_append(escaped_str, "\\n"); break; case '\r': g_string_append(escaped_str, "\\r"); break; case '\t': g_string_append(escaped_str, "\\t"); break; case '\\': g_string_append(escaped_str, "\\\\"); break; default: g_string_append_printf(escaped_str, "\\x%02x", uchar); break; } } else { g_string_append_unichar(escaped_str, uchar); } p = g_utf8_next_char(p); } gchar *res = escaped_str->str; g_string_free(escaped_str, FALSE); return res; } char* gfal2_path_collapse_slashes(const char* path) { if (path == NULL) { return NULL; } size_t len = strlen(path); char* collapsed = g_malloc0(len + 1); int pos = 0; int i; collapsed[pos] = '\0'; // Use a simple stack algorithm: // Insert elements from 'path' one-by-one into the stack // When inserting a new '/', make sure top of the stack is not also a '/' for (i = 0; i < len; i++) { if (path[i] != '/') { collapsed[pos++] = path[i]; } else if (pos == 0 || collapsed[pos - 1] != '/') { collapsed[pos++] = path[i]; } } collapsed[pos] = '\0'; collapsed = g_realloc(collapsed, pos + 1); return collapsed; } gfal2-v2.23.0/src/utils/uri/gfal2_parsing.h000066400000000000000000000023161465240014500203770ustar00rootroot00000000000000/* * Copyright (c) CERN 2020 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #pragma once #include #ifdef __cplusplus extern "C" { #endif /* * Given a string and it's length, returns a pointer to an UTF8-escaped Glib string. * The last parameter takes a list of characters for which to ignore escaping. * Note: the caller must handle memory deallocation of the new string */ gchar* gfal2_utf8escape_string(const char* str, size_t str_len, const char* ignore); /* * Given a path, returns a pointer to a new string with all slashes collapsed. * Note: the caller must handle memory deallocation of the new string */ char* gfal2_path_collapse_slashes(const char* path); #ifdef __cplusplus } #endif gfal2-v2.23.0/src/utils/uri/gfal2_uri.c000066400000000000000000000131301465240014500175220ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include #include #include "gfal2_uri.h" // From RFC3986, appendix B #define URI_REGEX "^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\\?([^#]*))?(#(.*))?" // 12 3 4 5 6 7 8 9 #define AUTHORITY_REGEX "^(([^@]*)@)?(([[:alnum:]][-_[:alnum:]]*(\\.[-_[:alnum:]]+)*)|(\\[[a-zA-Z0-9:]+\\]))?(:[[:digit:]]+)?" // 12 34 5 6 3 7 static GQuark scope_uri(){ return g_quark_from_static_string("Gfal::Uri_util"); } static char *_strdupmatch(const char *str, regmatch_t *match) { if (match->rm_so < 0) return NULL; size_t match_len = match->rm_eo - match->rm_so; return g_strndup(str + match->rm_so, match_len); } gfal2_uri *gfal2_parse_uri(const char *uri, GError **err) { char buffer[128]; regex_t preg; int ret = regcomp(&preg, URI_REGEX, REG_EXTENDED | REG_ICASE); assert(ret == 0); regmatch_t pmatch[10]; ret = regexec(&preg, uri, 10, pmatch, 0); if (ret != 0) { regerror(ret, &preg, buffer, sizeof(buffer)); regfree(&preg); gfal2_set_error(err, scope_uri(), EINVAL, __func__, "Could not match the uri: %s", buffer); return NULL; } // URI gfal2_uri *parsed = g_malloc0(sizeof(*parsed)); parsed->scheme = _strdupmatch(uri, &pmatch[2]); parsed->path = _strdupmatch(uri, &pmatch[5]); parsed->query = _strdupmatch(uri, &pmatch[7]); parsed->fragment = _strdupmatch(uri, &pmatch[9]); parsed->original = uri; // Authority not defined if (pmatch[4].rm_so < 0) { parsed->userinfo = parsed->host = NULL; parsed->port = 0; } // Authority defined but empty else if (pmatch[4].rm_so == pmatch[4].rm_eo) { parsed->host = g_strdup(""); } // Authority has content if (pmatch[4].rm_so != pmatch[4].rm_eo) { char *authority = _strdupmatch(uri, &pmatch[4]); regex_t authreg; ret = regcomp(&authreg, AUTHORITY_REGEX, REG_EXTENDED | REG_ICASE); assert(ret == 0); regmatch_t authmatch[8]; ret = regexec(&authreg, authority, 8, authmatch, 0); if (ret != 0) { regerror(ret, &preg, buffer, sizeof(buffer)); regfree(&preg); gfal2_free_uri(parsed); gfal2_set_error(err, scope_uri(), EINVAL, __func__, "Could not match the authority: %s", buffer); return NULL; } parsed->userinfo = _strdupmatch(authority, &authmatch[2]); parsed->host = _strdupmatch(authority, &authmatch[3]); if (authmatch[7].rm_so > -1) { parsed->port = atol(authority + authmatch[7].rm_so + 1); } regfree(&authreg); g_free(authority); } regfree(&preg); return parsed; } void gfal2_free_uri(gfal2_uri* uri) { if (uri) { g_free(uri->scheme); g_free(uri->userinfo); g_free(uri->host); g_free(uri->path); g_free(uri->query); g_free(uri->fragment); } g_free(uri); } int _push_component(gchar **str_array, int i, gchar *component) { if (component) { str_array[i++] = component; } return i; } char *gfal2_join_uri(gfal2_uri* uri) { gchar *str_array[11]; char port[12]; int i = 0; if (uri->scheme) { i = _push_component(str_array, i, uri->scheme); i = _push_component(str_array, i, "://"); } if (uri->userinfo) { i = _push_component(str_array, i, uri->userinfo); i = _push_component(str_array, i, "@"); } i = _push_component(str_array, i, uri->host); if (uri->port) { snprintf(port, sizeof(port), ":%d", uri->port); i = _push_component(str_array, i, port); } i = _push_component(str_array, i, uri->path); if (uri->query) { i = _push_component(str_array, i, "?"); i = _push_component(str_array, i, uri->query); } if (uri->fragment) { i = _push_component(str_array, i, "#"); i = _push_component(str_array, i, uri->fragment); } str_array[i] = NULL; return g_strjoinv("", str_array); } static int _hex(char digit) { if (digit >= '0' && digit <= '9') { return digit - '0'; } else if (digit >= 'A' && digit <= 'F') { return (digit - 'A') + 0xA; } else if (digit >= 'a' && digit <= 'f') { return (digit - 'a') + 0xA; } return 0; } char *gfal2_urldecode(char *str) { if (str == NULL) { return NULL; } char *r = str, *w = str; while (*r != '\0') { if (*r == '%' && isxdigit(*(r + 1)) && isxdigit(*(r + 2))) { *w = (_hex(*(r + 1)) << 4) + _hex(*(r + 2)); r += 3; ++w; } else { *w = *r; ++r; ++w; } } *w = '\0'; return str; } gfal2-v2.23.0/src/utils/uri/gfal2_uri.h000066400000000000000000000033571465240014500175410ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #pragma once #ifndef GFAL_URI_H #define GFAL_URI_H #include #include #ifdef __cplusplus extern "C" { #endif // URI struct // Please, note that there is a difference between a component being NULL (undefined) // and being the empty string. // i.e. empty = separator was present, but value missing // undefined = separator was not present typedef struct gfal2_uri { char *scheme; char *userinfo; char *host; unsigned port; char *path; char *query; char *fragment; const char *original; } gfal2_uri; /* * Parse an URI */ gfal2_uri* gfal2_parse_uri(const char *uri, GError **err); /* * Free an URI. It is safe to call if uri is NULL. */ void gfal2_free_uri(gfal2_uri* uri); /* * Returns a newly allocated string with the serialized uri */ char *gfal2_join_uri(gfal2_uri* uri); /* * url-decodes the string, and returns a pointer to the same string * Modifies the buffer pointed by str!! */ char *gfal2_urldecode(char *str); #ifdef __cplusplus } #endif #endif gfal2-v2.23.0/src/version/000077500000000000000000000000001465240014500152345ustar00rootroot00000000000000gfal2-v2.23.0/src/version/CMakeLists.txt000066400000000000000000000003221465240014500177710ustar00rootroot00000000000000cmake_minimum_required (VERSION 2.6) add_executable (gfal2_version gfal_version) target_link_libraries (gfal2_version "gfal2") install (TARGETS gfal2_version RUNTIME DESTINATION ${BIN_INSTALL_DIR}) gfal2-v2.23.0/src/version/gfal_version.c000066400000000000000000000016031465240014500200560ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include int main(int argc, char **argv) { printf("GFAL-client-%s\n", gfal2_version()); return (0); } gfal2-v2.23.0/test/000077500000000000000000000000001465240014500137375ustar00rootroot00000000000000gfal2-v2.23.0/test/CMakeLists.txt000066400000000000000000000016261465240014500165040ustar00rootroot00000000000000 set (UNIT_TESTS FALSE CACHE STRING "enable compilation of unit tests") set (FUNCTIONAL_TESTS FALSE CACHE STRING "functional tests for gfal ") set (STRESS_TESTS FALSE CACHE STRING "stress tests for gfal ") include_directories (${CMAKE_SOURCE_DIR}/src) if (ONLY_TESTS) find_package (GFAL2 REQUIRED) include_directories (${GFAL2_INCLUDE_DIRS}) else (ONLY_TESTS) set(GFAL2_LIBRARIES gfal2 gfal2_transfer) endif (ONLY_TESTS) find_package(GTEST) include_directories(${GTEST_INCLUDE_DIR}) execute_process(COMMAND cmake -E copy_directory ${CMAKE_CURRENT_SOURCE_DIR}/conf_test ${CMAKE_CURRENT_BINARY_DIR}/conf_test) # Common methods for the tests add_subdirectory(common) # Tests if (UNIT_TESTS) add_subdirectory(unit) endif (UNIT_TESTS) if (FUNCTIONAL_TESTS) add_subdirectory(functional) endif (FUNCTIONAL_TESTS) if (STRESS_TESTS) add_subdirectory(stress-test) endif (STRESS_TESTS) gfal2-v2.23.0/test/Dockerfile000066400000000000000000000007051465240014500157330ustar00rootroot00000000000000FROM gitlab-registry.cern.ch/dmc/gfal2:latest RUN yum install -y cmake ca-policy-egi-core voms-clients voms-config-vo-dteam RUN chmod a+rw /etc/gfal2.d/* # We need to mount /etc/passwd so the current user id can be resolved from # inside the container. And this is needed because of a bug in XRootD # (See RQF0573851) VOLUME [".ssh", ".globus", "/usr/share/gfal2/tests/Testing"] ADD "gfal2-tests.sh" "/gfal2-tests.sh" ENTRYPOINT ["/gfal2-tests.sh"] gfal2-v2.23.0/test/clean_directory.py000077500000000000000000000077371465240014500175000ustar00rootroot00000000000000#!/usr/bin/env python import gfal2 import logging import optparse import stat import sys log = logging.getLogger('gfal2.clean_directory') class Cleaner(object): def __init__(self, abort_on_error=False, recursive=False, only_files=False, chmod=False): self.abort_on_error = abort_on_error self.recursive = recursive self.only_files = only_files self.chmod = chmod self.context = gfal2.creat_context() def _get_list(self, surl): files = [] directories = [] if surl.startswith('srm://') and False: dh = self.context.opendir(surl + ";offset=0;count=100") else: dh = self.context.opendir(surl) d_entry, d_stat = dh.readpp() i = 0 while d_entry: full_path = surl + '/' + d_entry.d_name if stat.S_ISREG(d_stat.st_mode): files.append((full_path, d_stat)) elif stat.S_ISDIR(d_stat.st_mode): directories.append((full_path, d_stat)) d_entry, d_stat = dh.readpp() print i, '\r', i += 1 print return files, directories def __call__(self, surl): log.info("Cleaning %s" % surl) try: files, directories = self._get_list(surl) except gfal2.GError, e: if self.abort_on_error: raise logging.error("Could not list %s (%s)" % (surl, e.message)) return 0, 0 n_files, n_directories = 0, 0 for file, f_stat in files: try: self.context.unlink(file) log.info("Unlink %s" % file) n_files += 1 except gfal2.GError, e: if self.abort_on_error: raise log.error("Could not unlink %s (%s)" % (file, e.message)) for directory, d_stat in directories: if self.chmod and not (d_stat.st_mode & 0666): try: self.context.chmod(directory, 0775) log.info("Chmod for %s" % directory) except gfal2.GError, e: log.warn("Failed chmod for %s (%s)" % (directory, e.message)) sub_files, sub_directories = self(directory) n_files += sub_files n_directories += sub_directories if not self.only_files: try: log.info("Rmdir %s" % directory) self.context.rmdir(directory) n_directories += 1 except gfal2.GError, e: if self.abort_on_error: raise log.error("Failed to rmdir %s (%s)" % (directory, e.message)) return n_files, n_directories if __name__ == '__main__': parser = optparse.OptionParser(usage='usage: %prog [options] surl') parser.add_option('-x', '--abort', dest='abort_on_error', default=False, action='store_true', help='Abort cleaning on the first error') parser.add_option('-r', '--recursive', dest='recursive', default=False, action='store_true', help='Traverse directories recursively') parser.add_option('-f', '--files', dest='only_files', default=False, action='store_true', help='Unlink only files') parser.add_option('-c', '--chmod', dest='chmod', default=False, action='store_true', help='Attempt a chmod when a directory is not writeable') (options, args) = parser.parse_args() if len(args) != 1: parser.error('Wrong number of arguments') stdout_handler = logging.StreamHandler(sys.stdout) stdout_handler.setFormatter(logging.Formatter('[%(levelname)s] %(message)s')) log.addHandler(stdout_handler) log.setLevel(logging.INFO) cleaner = Cleaner(options.abort_on_error, options.recursive, options.only_files, options.chmod) n_files, n_directories = cleaner(args[0]) logging.info("Removed %d files and %d directories" % (n_files, n_directories)) gfal2-v2.23.0/test/common/000077500000000000000000000000001465240014500152275ustar00rootroot00000000000000gfal2-v2.23.0/test/common/CMakeLists.txt000066400000000000000000000004661465240014500177750ustar00rootroot00000000000000add_library(gfal2_test_shared SHARED gfal_lib_test.c gfal_gtest_asserts.cpp) target_link_libraries (gfal2_test_shared ${GFAL2_LIBRARIES} ${GTEST_LIBRARIES} ${JSONC_LIBRARIES}) if (FUNCTIONAL_TESTS OR UNIT_TESTS) install (TARGETS gfal2_test_shared LIBRARY DESTINATION ${LIB_INSTALL_DIR}) endif () gfal2-v2.23.0/test/common/gfal_gtest_asserts.cpp000066400000000000000000000065761465240014500216340ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include "gfal_gtest_asserts.h" testing::AssertionResult AssertGfalSuccess( const char* ret_expr, const char* error_expr, int ret, const GError* error) { if (ret >= 0 && error == NULL) return testing::AssertionSuccess(); if (ret >= 0 && error != NULL) return testing::AssertionFailure() << ret_expr << " >= 0, but " << error_expr << " is not NULL: (" << error->code << ") " << error->message; if (ret < 0 && error == NULL) return testing::AssertionFailure() << ret_expr << " < 0, but " << error_expr << " is NULL"; return testing::AssertionFailure() << error_expr << " is not NULL: (" << error->code << ") " << error->message; } testing::AssertionResult AssertGfalErrno( const char* ret_expr, const char* error_expr, const char* errno_expr, int ret, const GError* error, int err) { if (ret >= 0 && error == NULL) return testing::AssertionFailure() << "Operation succeeded, but errno " << err << " was expected"; if (ret >= 0 && error != NULL) return testing::AssertionFailure() << "Return status is >= 0, but error has been set: (" << error->code << ") " << error->message << " (Was expecting errno " << err << ")"; if (ret < 0 && error == NULL) return testing::AssertionFailure() << "Return status is < 0, but error has not been set " << "(was expecting errno " << err << ")"; if (error->code != err) return testing::AssertionFailure() << "Expecting errno " << err << " but got " << error->code << " (" << error->message << ")"; return testing::AssertionSuccess(); } // Same as AssertGfalError, but accepts a list of possible error codes // Accommodate plugins returning different error codes for the same function call (ideally, this is to be avoided) testing::AssertionResult AssertGfalOneOfErrno( const char* ret_expr, const char* error_expr, const char* errno_expr, int ret, const GError* error, std::list errcodes) { for (auto it = errcodes.begin(); it != errcodes.end(); it++) { auto res = AssertGfalErrno(ret_expr, error_expr, errno_expr, ret, error, *it); if (res == testing::AssertionSuccess()) { return res; } } std::ostringstream oss; std::copy(errcodes.begin(), errcodes.end(), std::ostream_iterator(oss, ", ")); return testing::AssertionFailure() << "Expecting one of following errno [" << oss.str().substr(0, oss.str().size() - 2) << "] but got " << error->code << " (" << error->message << ")"; } gfal2-v2.23.0/test/common/gfal_gtest_asserts.h000066400000000000000000000026111465240014500212630ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #pragma once #include #include #include #define SKIP_TEST(NAME) do { std::cout << "[ SKIPPED ] " << #NAME << std::endl; } while (0); testing::AssertionResult AssertGfalSuccess( const char* ret_expr, const char* error_expr, int ret, const GError* error ); testing::AssertionResult AssertGfalErrno( const char* ret_expr, const char* error_expr, const char* errno_expr, int ret, const GError* error, int err ); testing::AssertionResult AssertGfalOneOfErrno( const char* ret_expr, const char* error_expr, const char* errcodes_expr, int ret, const GError* error, std::list errcodes ); gfal2-v2.23.0/test/common/gfal_lib_test.c000066400000000000000000000122311465240014500201700ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include #include #include #include #include "gfal_lib_test.h" #include char* generate_random_uri(const char* uri_dir, const char* prefix, char* buff, size_t s_buff) { struct utsname info; memset(&info, 0, sizeof(struct utsname)); uname(&info); if (uri_dir[strlen(uri_dir)-1] != '/') { snprintf(buff, s_buff, "%s/%s_%s_%d%lld%ld", uri_dir, info.nodename, prefix, (int) getpid(), (long long) time(NULL), (long) rand()); } else { snprintf(buff, s_buff, "%s%s_%s_%d%lld%ld", uri_dir, info.nodename, prefix, (int) getpid(), (long long) time(NULL), (long) rand()); } return buff; } int generate_file_if_not_exists(gfal2_context_t handle, const char* surl, const char* src, GError** error) { struct stat st; if (gfal2_stat(handle, surl, &st, error) == 0) { return 0; } if ((*error)->code != ENOENT) { return -1; } g_clear_error(error); return gfalt_copy_file(handle, NULL, src, surl, error); } /* Everything that links against this will, automatically, * setup the segfault handler */ #define MAX_STACK_DEPTH 25 static void getFileAndLine(void* addr, const char* sname, char* buffer, size_t bufsize) { char fnameBuffer[512]; // Extract file object from the symbol name strncpy(fnameBuffer, sname, sizeof(fnameBuffer) - 1); char *p = strchr(fnameBuffer, '('); if (p) *p = '\0'; // Build the command char command[1024]; snprintf(command, sizeof(command), "addr2line -e '%s' 0x%lx", fnameBuffer, (long)addr); // Run it buffer[0] = '\0'; FILE *proc = popen(command, "r"); if (proc) { size_t nbytes = fread(buffer, 1, bufsize, proc); pclose(proc); if (nbytes <= 0) { strncpy(buffer, "Could not read from addr2line\n", bufsize); } } else { strncpy(buffer, "Could not execute addr2line\n", bufsize); } if (buffer[0] == '?') { buffer[0] = '\n'; buffer[1] = '\0'; } } int expect_third_party_copy(const char *url1, const char *url2) { int expect = 0; gfal2_uri *uri1 = gfal2_parse_uri(url1, NULL); gfal2_uri *uri2 = gfal2_parse_uri(url2, NULL); // If file is involved, surely not if (g_strcmp0(uri1->scheme, "file") == 0 || g_strcmp0(uri2->scheme, "file") == 0) { expect = 0; goto done_expect; } // If SRM involved, let's assume yes if (g_strcmp0(uri1->scheme, "srm") == 0 && g_strcmp0(uri2->scheme, "gsiftp") == 0) { expect = 1; goto done_expect; } if (g_strcmp0(uri1->scheme, "gsiftp") == 0 && g_strcmp0(uri2->scheme, "srm") == 0) { expect = 1; goto done_expect; } // If both are the same, depending on the protocol int is_same = (g_strcmp0(uri1->scheme, uri2->scheme) == 0); if (is_same) { expect = g_strcmp0(uri1->scheme, "srm") == 0 || g_strcmp0(uri1->scheme, "gsiftp") == 0 || g_strcmp0(uri1->scheme, "root") == 0 || g_strcmp0(uri1->scheme, "davs") == 0; goto done_expect; } // Otherwise, not expected // Mind that there are other combinations here that could happen, and even // some protocols can do copies both with third party or not, depending on the storages involved // (i.e. davs, s3, root), so this is an incomplete heuristic only suitable for the storages normally // involved on the tests done_expect: gfal2_free_uri(uri1); gfal2_free_uri(uri2); return expect; } static void dump_stack(int sig) { if (sig == SIGSEGV || sig == SIGBUS || sig == SIGABRT) { fprintf(stderr, "FATAL ERROR!\n"); void *array[MAX_STACK_DEPTH] = { 0 }; char fileBuffer[1024]; int nFrames = backtrace(array, MAX_STACK_DEPTH); char **symbols = backtrace_symbols(array, nFrames); int i; for (i = 0; symbols && i < nFrames; ++i) { if (symbols[i]) { getFileAndLine(array[i], symbols[i], fileBuffer, sizeof(fileBuffer)); fprintf(stderr, "%s\n", symbols[i]); fprintf(stderr, "\t%s", fileBuffer); } } if (symbols) { free(symbols); } exit(1); } } __attribute__((constructor)) void setup_segfault_handler(void) { signal(SIGSEGV, dump_stack); } gfal2-v2.23.0/test/common/gfal_lib_test.h000066400000000000000000000025261465240014500202030ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #pragma once #ifndef GFAL_LIB_TEST_H #define GFAL_LIB_TEST_H #include #ifdef __cplusplus extern "C" { #endif char* generate_random_uri(const char* uri_dir, const char* prefix, char* buff, size_t s_buff); /** * If surl does not exist, it creates it putting the content of src. */ int generate_file_if_not_exists(gfal2_context_t handle, const char* surl, const char* src, GError** error); /** * Return 1 if the copy from url1 to url2 should be third party copy */ int expect_third_party_copy(const char *url1, const char *url2); #ifdef __cplusplus } #endif #endif /* GFAL_LIB_TEST_H */ gfal2-v2.23.0/test/conf_test/000077500000000000000000000000001465240014500157235ustar00rootroot00000000000000gfal2-v2.23.0/test/conf_test/bdii.conf000066400000000000000000000003341465240014500175010ustar00rootroot00000000000000# # basic configuration for the gfal 2 BDII related options [BDII] # enable or disable bdii query system ( default : enabled ) ENABLED=true # BDII default server to contact LCG_GFAL_INFOSYS=lcg-bdii.cern.ch:2170 gfal2-v2.23.0/test/conf_test/dcap_plugin.conf000066400000000000000000000002571465240014500210630ustar00rootroot00000000000000# # basic configuration for the gfal 2 DCAP plugin [DCAP PLUGIN] # set the passive mode enabled or now # active mode can cause troubles with firewalls MODE_PASSIVE=TRUE gfal2-v2.23.0/test/conf_test/gfal2_core.conf000066400000000000000000000001141465240014500205710ustar00rootroot00000000000000# # Core configuration of GFAL 2.0 [CORE] # # nothing special for now gfal2-v2.23.0/test/conf_test/gsiftp_plugin.conf000066400000000000000000000016211465240014500214440ustar00rootroot00000000000000# # basic configuration for the gfal 2 griftp plugin [GRIDFTP PLUGIN] # enable or disable gridFTPv2 support GRIDFTP_V2=true # enable or disable session re-use support # warning : disable this feature can slow-down a lot the performances SESSION_REUSE=true # default number of streams used for file transfers # 0 means in-order-stream mode RD_NB_STREAM=0 # default checksum algorithm type used for transfer content verification COPY_CHECKSUM_TYPE=ADLER32 # enable or disable the encryption for the data channel DCAU=false # enable or disable Ipv6 support IPV6=true # define the maximum time in s # for a gsiftp checksum request CHECKSUM_CALC_TIMEOUT=1800 # maximum time between two performance marker # disable if equal to 0 PERF_MARKER_TIMEOUT=600 ## enable or disable the delay passive option of gridftpv2 ## this option need to be enabled to support the gridftp redirection features DELAY_PASSV=true gfal2-v2.23.0/test/conf_test/http_plugin.conf000066400000000000000000000001151465240014500211240ustar00rootroot00000000000000## configuration file for http plugin ## [HTTP PLUGIN] ENABLE_REMOTE_COPY=1 gfal2-v2.23.0/test/conf_test/lfc_plugin.conf000066400000000000000000000004051465240014500207130ustar00rootroot00000000000000# # basic configuration for the gfal 2 lfc plugin [LFC PLUGIN] # HOSTNAME of the lfc to contact LFC_HOST=lfc-puppet01.cern.ch # connexion timeout in second LFC_CONNTIMEOUT=15 # maximum number of try for opening a connexion LFC_CONRETRY=2 # LFC_CONRETRYINT=1 gfal2-v2.23.0/test/conf_test/rfio_plugin.conf000066400000000000000000000002671465240014500211140ustar00rootroot00000000000000# # basic configuration for the gfal 2 rfio plugin [RFIO PLUGIN] # type of RFIO library to use ( cf : rfio castor vs rfio dpm ) # value can be castor or dpm LCG_RFIO_TYPE="dpm" gfal2-v2.23.0/test/conf_test/srm_plugin.conf000066400000000000000000000013771465240014500207610ustar00rootroot00000000000000# # basic configuration for the gfal 2 srm plugin [SRM PLUGIN] # timeout for SRM operations in seconds # Overrides CORE:NAMESPACE_TIMEOUT OPERATION_TIMEOUT=180 # timeout for the HTTP connexion in seconds CONN_TIMEOUT=60 # desired request lifetime REQUEST_LIFETIME=3600 # default checksum type for transfer check COPY_CHECKSUM_TYPE=ADLER32 # ordered list of turls protocols for remote I/O # the top priority protocol is the first one TURL_PROTOCOLS=gsiftp;rfio;gsidcap;dcap;kdcap # ordered list of turls protocols for third party transfer # only protocol supporting third party copy should be here # the top priority protocol is the first one TURL_3RD_PARTY_PROTOCOLS=gsiftp # enable or disable the srm session re-use # no parameter : disabled KEEP_ALIVE=true gfal2-v2.23.0/test/functional/000077500000000000000000000000001465240014500161015ustar00rootroot00000000000000gfal2-v2.23.0/test/functional/CMakeLists.txt000066400000000000000000000272631465240014500206530ustar00rootroot00000000000000function (add_test_executable name source) add_executable(${name} ${source}) install(TARGETS ${name} DESTINATION ${SHARE_INSTALL_PREFIX}/gfal2/tests) endfunction(add_test_executable name) if(FUNCTIONAL_TESTS) find_package(GTEST) include_directories( "${CMAKE_SOURCE_DIR}/src" "${CMAKE_SOURCE_DIR}/test" "${CMAKE_SOURCE_DIR}/src/posix/" "${GLIB2_INCLUDE_DIRS}" "${GTEST_INCLUDE_DIR}" "${JSONC_INCLUDE_DIRS}" ) # Bring online tests add_test_executable(gfal_test_bringonline "gfal_test_bringonline.cpp") target_link_libraries(gfal_test_bringonline ${GFAL2_LIBRARIES} gfal2_test_shared) function(test_bringonline name dir_path) add_test(gfal_test_bringonline_${name} gfal_test_bringonline ${dir_path}) endfunction(test_bringonline name valid_file) # Archive tests add_test_executable(gfal_test_archive "gfal_test_archive.cpp") target_link_libraries(gfal_test_archive ${GFAL2_LIBRARIES} gfal2_test_shared) function(test_archive name dir_path) add_test(gfal_test_archive_${name} gfal_test_archive ${dir_path}) endfunction(test_archive name dir_path) # Checksum tests add_test_executable(gfal_test_checksum "gfal_test_checksum.cpp") target_link_libraries(gfal_test_checksum ${GFAL2_LIBRARIES} gfal2_test_shared) function(test_checksum_simple name dir_path algo) add_test(gfal_checksum_simple_${name} gfal_test_checksum ${dir_path} ${algo}) endfunction(test_checksum_simple name valid_file algo) # Stat tests add_test_executable(gfal_test_stat "gfal_test_stat.cpp") target_link_libraries(gfal_test_stat ${GFAL2_LIBRARIES} gfal2_test_shared) function(test_stat_all name valid) add_test(gfal_stat_${name} gfal_test_stat ${valid}) endfunction(test_stat_all valid) # Access test add_test_executable(gfal_test_access "gfal_test_access.cpp") target_link_libraries(gfal_test_access ${GFAL2_LIBRARIES} gfal2_test_shared) function(test_access name dir_path) add_test(gfal_test_access_${name} gfal_test_access ${dir_path}) endfunction(test_access name dir_path) # XAttr test add_test_executable(gfal_test_xattr "gfal_test_xattr.cpp") target_link_libraries(gfal_test_xattr ${GFAL2_LIBRARIES} gfal2_test_shared) function(test_xattr name dir_path) add_test(gfal_test_xattr_${name} gfal_test_xattr ${dir_path} ${ARGV2}) endfunction(test_xattr name dir_path) # Deletion tests add_test_executable(gfal_test_del "gfal_test_del.cpp") target_link_libraries(gfal_test_del ${GFAL2_LIBRARIES} gfal2_test_shared) function(test_del name dir_path) add_test(gfal_test_del_${name} gfal_test_del ${dir_path}) endfunction(test_del name path) # QoS tests add_test_executable(gfal_test_qos "gfal_test_qos.cpp") target_link_libraries(gfal_test_qos ${GFAL2_LIBRARIES} gfal2_test_shared) function(test_qos name) add_test(gfal_test_qos_${name} gfal_test_qos) endfunction(test_qos name) # Token tests add_test_executable(gfal_test_token "gfal_test_token.cpp") target_link_libraries(gfal_test_token ${GFAL2_LIBRARIES} gfal2_test_shared) function(test_token name) add_test(gfal_test_token_${name} gfal_test_token) endfunction(test_token name) # Rename tests add_test_executable(gfal_test_rename "gfal_test_rename.cpp") target_link_libraries(gfal_test_rename ${GFAL2_LIBRARIES} gfal2_test_shared) function(test_rename name dir_path) add_test(gfal_test_rename_${name} gfal_test_rename ${dir_path}) endfunction(test_rename name dir_path) # Chmod tests add_test_executable(gfal_test_chmod "gfal_test_chmod.cpp") target_link_libraries(gfal_test_chmod ${GFAL2_LIBRARIES} gfal2_test_shared) function(test_chmod_all name valid right1 right2 right3 right4) add_test(gfal_test_right_change_${name} gfal_test_chmod ${valid} ${right1} ${right2} ${right3} ${right4}) endfunction(test_chmod_all name valid right1 right2 right3 right4) # Rmdir tests add_test_executable(gfal_test_rmdir_full "gfal_test_rmdir_full.cpp") target_link_libraries(gfal_test_rmdir_full ${GFAL2_LIBRARIES} gfal2_test_shared) function(test_rmdir_all name valid_dir) add_test(gfal_test_rmdir_full_${name} gfal_test_rmdir_full ${valid_dir}) endfunction(test_rmdir_all name valid_dir) # Readdir tests add_test_executable(gfal_test_readdir_full "gfal_test_readdir_full.cpp") target_link_libraries(gfal_test_readdir_full ${GFAL2_LIBRARIES} gfal2_test_shared) function(test_readdir_full name valid) add_test(gfal_test_readdir_full_${name} gfal_test_readdir_full ${valid}) endfunction(test_readdir_full valid) # IO tests add_test_executable(gfal_test_rw_full "gfal_test_rw_full.cpp") target_link_libraries(gfal_test_rw_full ${GFAL2_LIBRARIES} gfal2_test_shared) add_test_executable(gfal_test_rw_seq "gfal_test_rw_seq.cpp") target_link_libraries(gfal_test_rw_seq ${GFAL2_LIBRARIES} gfal2_test_shared) add_test_executable(gfal_test_rw_seek "gfal_test_rw_seek.cpp") target_link_libraries(gfal_test_rw_seek ${GFAL2_LIBRARIES} gfal2_test_shared) function(test_rwt_all name path size) add_test(gfal_rw_size_${name} gfal_test_rw_full ${path} ${size}) endfunction(test_rwt_all name path size) function(test_rwt_seq name path size_read size) add_test(gfal_rw_seq_${name} gfal_test_rw_seq ${path} ${size_read} ${size}) endfunction(test_rwt_seq name path size_read size) function(test_rwt_seek name path size_read size) add_test(gfal_rw_seek_${name} gfal_test_rw_seek ${path} ${size_read} ${size}) endfunction(test_rwt_seek name path size) # Mkdir tests add_test_executable(gfal_test_mkdir_full "gfal_test_mkdir_full.cpp") target_link_libraries(gfal_test_mkdir_full ${GFAL2_LIBRARIES} gfal2_test_shared) function(test_mkdir_all name prefix) add_test(gfal_mkdir_full_${name} gfal_test_mkdir_full ${prefix}) endfunction(test_mkdir_all name prefix) # Register test add_test_executable(gfal_test_register "gfal_test_register.cpp") target_link_libraries(gfal_test_register ${GFAL2_LIBRARIES} gfal2_test_shared) function(test_register name origin lfc) add_test(gfal_test_register_${name} gfal_test_register ${origin} ${lfc}) endfunction(test_register name origin lfc) # Set credentials test add_test_executable(gfal_test_set_creds "gfal_test_set_creds.cpp") target_link_libraries(gfal_test_set_creds ${GFAL2_LIBRARIES} gfal2_test_shared) function(gfal_test_set_credentials name prefix) add_test(gfal_test_set_credentials_${name} gfal_test_set_creds ${prefix}) endfunction(gfal_test_set_credentials name prefix) # Posix test add_test_executable(gfal_test_posix "gfal_test_posix.cpp") target_link_libraries(gfal_test_posix ${GFAL2_LIBRARIES} gfal2_test_shared) function(gfal_test_posix name prefix) add_test(gfal_test_posix_${name} gfal_test_posix ${prefix}) endfunction(gfal_test_posix name prefix) # Space test add_test_executable(gfal_test_space "gfal_test_space.cpp") target_link_libraries(gfal_test_space ${GFAL2_LIBRARIES} gfal2_test_shared) function(test_space name prefix) add_test(gfal_test_space_${name} gfal_test_space ${prefix}) endfunction(test_space name prefix) # Tests for file transfer if(MAIN_TRANSFER) add_test_executable(gfalt_test_rd3_reorder_protocols "gfalt_test_rd3_reorder_protocols.cpp") target_link_libraries(gfalt_test_rd3_reorder_protocols ${GFAL2_TRANSFER_LINK} ${GFAL2_LIBRARIES} gfal2_test_shared) function(test_rd3_reorder_protocols name src_dir dst_dir) add_test(rd3_reorder_protocols_${name} gfalt_test_rd3_reorder_protocols ${src_dir} ${dst_dir}) endfunction(test_rd3_reorder_protocols name dst_dir) add_test_executable(gfalt_test_copy_file_cancel "gfalt_test_copy_file_cancel.cpp") target_link_libraries(gfalt_test_copy_file_cancel ${GFAL2_TRANSFER_LINK} ${GFAL2_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} pthread) add_test_executable(gfalt_test_copy_file "gfalt_test_copy_file.cpp") target_link_libraries(gfalt_test_copy_file ${GFAL2_TRANSFER_LINK} ${GFAL2_LIBRARIES} gfal2_test_shared) add_test_executable(gfalt_test_copy_file_timeout "gfalt_test_copy_file_timeout.cpp") target_link_libraries(gfalt_test_copy_file_timeout ${GFAL2_TRANSFER_LINK} ${GFAL2_LIBRARIES} gfal2_test_shared) add_test_executable(gfalt_test_copy_file_mkdir "gfalt_test_copy_file_mkdir.cpp") target_link_libraries(gfalt_test_copy_file_mkdir ${GFAL2_TRANSFER_LINK} ${GFAL2_LIBRARIES} gfal2_test_shared) add_test_executable(gfalt_test_copy_file_replace "gfalt_test_copy_file_replace.cpp") target_link_libraries(gfalt_test_copy_file_replace ${GFAL2_TRANSFER_LINK} ${GFAL2_LIBRARIES} gfal2_test_shared) add_test_executable(gfalt_test_copy_file_checksum "gfalt_test_copy_file_checksum.cpp") target_link_libraries(gfalt_test_copy_file_checksum ${GFAL2_TRANSFER_LINK} ${GFAL2_LIBRARIES} gfal2_test_shared) add_test_executable(gfalt_test_copy_file_checksum_user "gfalt_test_copy_file_checksum_user.cpp") target_link_libraries(gfalt_test_copy_file_checksum_user ${GFAL2_TRANSFER_LINK} ${GFAL2_LIBRARIES} gfal2_test_shared) add_test_executable(gfalt_test_copy_bulk "gfalt_test_copy_bulk.cpp") target_link_libraries(gfalt_test_copy_bulk ${GFAL2_TRANSFER_LINK} ${GFAL2_LIBRARIES} gfal2_test_shared) add_test_executable(gfalt_test_pasv "gfalt_test_pasv.cpp") target_link_libraries(gfalt_test_pasv ${GFAL2_TRANSFER_LINK} ${GFAL2_LIBRARIES} gfal2_test_shared) function(test_copy_file_full name src_dir dst_dir) add_test(copyfile_checksum_${name} gfalt_test_copy_file_checksum ${src_dir} ${dst_dir}) add_test(copyfile_checksum_user_${name} gfalt_test_copy_file_checksum_user ${src_dir} ${dst_dir}) add_test(copyfile_replace_${name} gfalt_test_copy_file_replace ${src_dir} ${dst_dir}) add_test(copyfile_test_${name} gfalt_test_copy_file ${src_dir} ${dst_dir}) add_test(copyfile_test_mkdir_${name} gfalt_test_copy_file_mkdir ${src_dir} ${dst_dir}) endfunction(test_copy_file_full name dst_dir) function (test_copy_bulk name src_dir dst_dir) add_test(copyfile_bulk_${name} gfalt_test_copy_bulk ${src_dir} ${dst_dir}) endfunction (test_copy_bulk name src_dir dst_dir) function(test_copy_file_no_checksum name src_dir dst_dir) add_test(copyfile_replace_${name} gfalt_test_copy_file_replace ${src_dir} ${dst_dir}) add_test(copyfile_test_${name} gfalt_test_copy_file ${src_dir} ${dst_dir}) add_test(copyfile_test_mkdir_${name} gfalt_test_copy_file_mkdir ${src_dir} ${dst_dir}) endfunction(test_copy_file_no_checksum name src_dir dst_dir) function(test_pasv name src_dir dst_dir) add_test(gfalt_test_pasv_${name} gfalt_test_pasv ${src_dir} ${dst_dir}) endfunction(test_pasv name src_dir dst_dir) add_test_executable(gfalt_test_double_cred "gfalt_test_double_cred.cpp") target_link_libraries(gfalt_test_double_cred ${GFAL2_TRANSFER_LINK} ${GFAL2_LIBRARIES} gfal2_test_shared) function(test_double_cred name src_dir dst_dir) add_test(gfalt_test_double_cred_${name} gfalt_test_double_cred ${src_dir} ${dst_dir}) endfunction(test_double_cred name src_dir dst_dir) endif(MAIN_TRANSFER) include(functional-test-parameters.cmake) include(functional-test-parameters.cmake) # Install ctest file install(FILES ${CMAKE_CURRENT_BINARY_DIR}/CTestTestfile.cmake DESTINATION ${SHARE_INSTALL_PREFIX}/gfal2/tests) endif(FUNCTIONAL_TESTS) gfal2-v2.23.0/test/functional/functional-test-parameters.cmake000066400000000000000000000376751465240014500244050ustar00rootroot00000000000000# # compilation lines for test parameters ## STAT Tests SET(MY_VO "dteam") SET(MY_VO_STORM "${MY_VO}") SET(TEST_ENVIRONMENT "PRODUCTION" CACHE STRING "Define the target for functional test") ## Global environment SET(ftp_prefix "ftp://mirror.switch.ch/mirror/centos/") IF(TEST_ENVIRONMENT STREQUAL "TESTBED_RC") ## Testbed environment SET(file_base_path "/tmp/") SET(srm_prefix_storm "srm://se01.esc.qmul.ac.uk:8444/srm/managerv2?SFN=/${MY_VO_STORM}/gfal2-tests/") SET(srm_prefix_dcache "srm://dcache-se-cms.desy.de:8443/srm/managerv2?SFN=/pnfs/desy.de/${MY_VO}/gfal2-tests/") SET(srm_prefix_castor "srm://srm-public.cern.ch:8443/srm/managerv2?SFN=/castor/cern.ch/grid/${MY_VO}/gfal2-tests/") SET(lfc_prefix "lfn:/grid/${MY_VO}/gfal2-tests/") SET(lfc_host_name "lfc-cc7.cern.ch") SET(gsiftp_prefix_dpm "gsiftp://dpmhead-rc.cern.ch/dpm/cern.ch/home/${MY_VO}/gfal2-tests") SET(srm_prefix_dpm "srm://dpmhead-rc.cern.ch:8446/dpm/cern.ch/home/${MY_VO}/gfal2-tests") SET(davs_prefix_dpm "davs://dpmhead-rc.cern.ch/dpm/cern.ch/home/${MY_VO}/gfal2-tests") SET(davs_prefix_dcache "davs://prometheus.desy.de/VOs/${MY_VO}/gfal2-tests") SET(root_prefix_dpm "root://dpmhead-rc.cern.ch/dpm/cern.ch/home/${MY_VO}/gfal2-tests") SET(root_prefix_eos "root://eospps.cern.ch//eos/opstest/gfal2-tests") SET(sftp_prefix "sftp://gfal2@arioch.cern.ch/home/gfal2/gfal2-tests") ELSEIF(TEST_ENVIRONMENT STREQUAL "TESTBED_TRUNK") ## Testbed trunk environment SET(file_base_path "/tmp/") SET(srm_prefix_storm "srm://se01.esc.qmul.ac.uk:8444/srm/managerv2?SFN=/${MY_VO_STORM}/gfal2-tests/") SET(srm_prefix_dcache "srm://vm-dcache-deploy6.desy.de:8443/data/${MY_VO}/gfal2-tests") SET(srm_prefix_castor "srm://srm-public.cern.ch:8443/srm/managerv2?SFN=/castor/cern.ch/grid/${MY_VO}/gfal2-tests/") SET(lfc_prefix "lfn:/grid/${MY_VO}/gfal2-tests/") SET(lfc_host_name "lfc-cc7.cern.ch") SET(gsiftp_prefix_dpm "gsiftp://dpmhead-trunk.cern.ch/dpm/cern.ch/home/${MY_VO}/gfal2-tests") SET(srm_prefix_dpm "srm://dpmhead-trunk.cern.ch:8446/dpm/cern.ch/home/${MY_VO}/gfal2-tests") SET(davs_prefix_dpm "davs://dpmhead-trunk.cern.ch/dpm/cern.ch/home/${MY_VO}/gfal2-tests") SET(davs_prefix_dcache "davs://prometheus.desy.de/VOs/${MY_VO}/gfal2-tests") SET(root_prefix_dpm "root://dpmhead-trunk.cern.ch/dpm/cern.ch/home/${MY_VO}/gfal2-tests") SET(root_prefix_eos "root://eospps.cern.ch//eos/opstest/tpc/gfal2-tests") SET(sftp_prefix "sftp://gfal2@arioch.cern.ch/home/gfal2/gfal2-tests") ELSE(TEST_ENVIRONMENT STREQUAL "TESTBED_RC") ## Production environment : default SET(file_base_path "/tmp/") SET(srm_prefix_storm "srm://se03.esc.qmul.ac.uk:8444/srm/managerv2?SFN=/${MY_VO_STORM}/gfal2-tests/") SET(srm_prefix_dcache "srm://dcache-se-cms.desy.de:8443/srm/managerv2?SFN=/pnfs/desy.de/${MY_VO}/gfal2-tests/") SET(srm_prefix_castor "srm://srm-public.cern.ch:8443/srm/managerv2?SFN=/castor/cern.ch/grid/${MY_VO}/gfal2-tests/") SET(lfc_prefix "lfn:/grid/${MY_VO}/gfal2-tests/") SET(lfc_host_name "lfc-cc7.cern.ch") SET(gsiftp_prefix_dpm "gsiftp://golias100.farm.particle.cz/dpm/farm.particle.cz/home/${MY_VO}/gfal2-tests") SET(srm_prefix_dpm "srm://node12.datagrid.cea.fr:8446/srm/managerv2?SFN=/dpm/datagrid.cea.fr/home/${MY_VO}/gfal2-tests/") SET(sftp_prefix "sftp://gfal2@arioch.cern.ch/home/gfal2/gfal2-tests") SET(root_prefix_dpm "root://dpmhead-rc.cern.ch/dpm/cern.ch/home/${MY_VO}/gfal2-tests") SET(root_prefix_eos "root://eospps.cern.ch//eos/opstest/tpc/gfal2-tests") # Need to find something better! SET(davs_prefix_dpm "davs://dpmhead-rc.cern.ch/dpm/cern.ch/home/${MY_VO}/gfal2-tests") SET(davs_prefix_dcache "davs://prometheus.desy.de/VOs/${MY_VO}/gfal2-tests") ENDIF(TEST_ENVIRONMENT STREQUAL "TESTBED_RC") ## lfc parameters SET(lfc_full_prefix "lfc://${lfc_host_name}/grid/${MY_VO}") ## local file parameters SET(file_prefix "file://${file_base_path}") IF(PLUGIN_FILE) test_del("FILE" "${file_prefix}") test_rename("FILE" "${file_prefix}") test_access("FILE" "${file_prefix}") test_stat_all("FILE" ${file_prefix}) test_chmod_all("FILE" ${file_prefix} 0565 060 360 767) test_mkdir_all("FILE" ${file_prefix}) test_rmdir_all("FILE" ${file_prefix}) test_readdir_full("FILE" ${file_prefix}) test_rwt_all("FILE" ${file_prefix} 4578) test_rwt_all("FILE" ${file_prefix} 1) test_rwt_all("FILE" ${file_prefix} 100000) test_rwt_seq("FILE" ${file_prefix} 100 4560) test_rwt_seek("FILE" ${file_prefix} 100 4560) test_checksum_simple("FILE_MD5" ${file_prefix} MD5) test_checksum_simple("FILE_ADLER32" ${file_prefix} ADLER32) test_checksum_simple("FILE_CRC32" ${file_prefix} CRC32) gfal_test_posix("FILE" ${file_prefix}) ENDIF(PLUGIN_FILE) IF(PLUGIN_SRM) # del dir test test_del("SRM_DPM" "${srm_prefix_dpm}") test_del("SRM_STORM" "${srm_prefix_storm}") test_del("SRM_DCACHE" "${srm_prefix_dcache}") test_del("SRM_CASTOR" "${srm_prefix_castor}") # stat tests test_stat_all("SRM_DPM" ${srm_prefix_dpm}) test_stat_all("SRM_DCACHE" ${srm_prefix_dcache}) test_stat_all("SRM_CASTOR" ${srm_prefix_castor}) test_access("SRM_DPM" ${srm_prefix_dpm}) # dCache says Operation not supported # test_access("SRM_DCACHE" ${srm_prefix_dcache}) # Rename test test_rename("SRM_DPM" ${srm_prefix_dpm}) test_rename("SRM_DCACHE" ${srm_prefix_dcache}) test_rename("SRM_CASTOR" ${srm_prefix_castor}) # checksum tests test_checksum_simple("SRM_DPM_ADLER32" ${srm_prefix_dpm} ADLER32) test_checksum_simple("SRM_DPM_MD5" ${srm_prefix_dpm} MD5) test_checksum_simple("SRM_DCACHE_ADLER32" ${srm_prefix_dcache} ADLER32) test_checksum_simple("SRM_CASTOR_ADLER32" ${srm_prefix_castor} ADLER32) # Mkdir test_mkdir_all("SRM_DPM" ${srm_prefix_dpm}) test_mkdir_all("SRM_DCACHE" ${srm_prefix_dcache}) test_mkdir_all("SRM_CASTOR" ${srm_prefix_castor}) test_chmod_all("SRM_DPM" ${srm_prefix_dpm} 0575 070 370 777) test_rmdir_all("SRM_DPM" ${srm_prefix_dpm}) test_readdir_full("SRM_DPM" ${srm_prefix_dpm} ) test_rwt_all("SRM_DPM" ${srm_prefix_dpm} 4578) test_rwt_all("SRM_DPM_little" ${srm_prefix_dpm} 10) test_rwt_all("SRM_DPM_single" ${srm_prefix_dpm} 1) test_rwt_all("SRM_DCAP" ${srm_prefix_dcache} 4578) test_rwt_all("SRM_DCAP_little" ${srm_prefix_dcache} 10) test_rwt_all("SRM_DCAP_single" ${srm_prefix_dcache} 1) test_rwt_all("SRM_CASTOR" ${srm_prefix_castor} 4578) test_rwt_all("SRM_CASTOR_little" ${srm_prefix_castor} 10) test_rwt_all("SRM_CASTOR_single" ${srm_prefix_castor} 1) test_rwt_seq("SRM_DPM" ${srm_prefix_dpm} 100 4560) test_rwt_seq("SRM_DPM_unit" ${srm_prefix_dpm} 1 10) test_rwt_seq("SRM_DCAP" ${srm_prefix_dpm} 100 4560) test_rwt_seq("SRM_STORM" ${srm_prefix_storm} 100 4560) # Bringonline test_bringonline("SRM_DPM" ${srm_prefix_dpm}) test_bringonline("SRM_DCACHE" ${srm_prefix_dcache}) test_bringonline("SRM_CASTOR" ${srm_prefix_castor}) # Note: STORM may not support tapes # test_chmod_all("SRM_DCACHE" ${srm_valid_dcache_chmod} 0565 060 360 767) -> disabled, since unavailable on dcache # test_stat_all( "SRM_EOS" ${srm_valid_EOS_stat}) test_xattr("DPM" ${srm_prefix_dpm}) test_xattr("DCACHE" ${srm_prefix_dcache}) test_xattr("CASTOR" ${srm_prefix_castor}) test_space("DPM" ${srm_prefix_dpm}) test_rd3_reorder_protocols("GRIDFTP_TO_SRM" ${gsiftp_prefix_dpm} ${srm_prefix_dcache}) test_rd3_reorder_protocols("SRM_TO_GRIDFTP" ${srm_prefix_dcache} ${gsiftp_prefix_dpm}) test_rd3_reorder_protocols("SRM_TO_SRM" ${srm_prefix_dcache} ${srm_prefix_dcache}) test_rd3_reorder_protocols("XROOTD_TO_SRM" ${root_prefix_dpm} ${srm_prefix_dcache}) ENDIF(PLUGIN_SRM) IF(PLUGIN_LFC) test_stat_all( "LFC" ${lfc_prefix}) test_mkdir_all("LFC" ${lfc_prefix}) test_chmod_all("LFC" ${lfc_prefix} 0565 060 0360 0767) test_rmdir_all("LFC" ${lfc_prefix}) test_readdir_full("LFC" ${lfc_prefix} ) # lfc full url style test test_stat_all( "LFC_FULL" ${lfc_full_prefix}) test_mkdir_all("LFC_FULL" ${lfc_full_prefix}) test_chmod_all("LFC_FULL" ${lfc_prefix} 0565 060 0360 0767) test_rmdir_all("LFC_FULL" ${lfc_full_prefix}) test_readdir_full("LFC_FULL" ${lfc_full_prefix} ) # Register test_register("LFC" ${gsiftp_prefix_dpm} ${lfc_full_prefix}) # Xattr test_xattr("LFC" ${lfc_full_prefix} ${gsiftp_prefix_dpm}) ENDIF(PLUGIN_LFC) IF(PLUGIN_GRIDFTP) test_del("GRIDFTP_DPM" "${gsiftp_prefix_dpm}") test_rename("GRIDFTP_DPM" ${gsiftp_prefix_dpm}) test_stat_all("GRIDFTP_DPM" ${gsiftp_prefix_dpm}) test_access("GRIDFTP_DPM" ${gsiftp_prefix_dpm}) test_checksum_simple("GRIDFTP_DPM__ADLER32" ${gsiftp_prefix_dpm} ADLER32) test_checksum_simple("GRIDFTP_DPM_MD5" ${gsiftp_prefix_dpm} MD5) test_checksum_simple("GRIDFTP_DPM_CRC32" ${gsiftp_prefix_dpm} CRC32) test_mkdir_all("GRIDFTP_DPM" ${gsiftp_prefix_dpm}) test_chmod_all("GRIDFTP_DPM" ${gsiftp_prefix_dpm} 0565 060 0360 0767) test_rmdir_all("GRIDFTP_DPM" ${gsiftp_prefix_dpm}) test_readdir_full("GRIDFTP_DPM" ${gsiftp_prefix_dpm}) test_rwt_all("GRIDFTP_DPM" ${gsiftp_prefix_dpm} 4578) test_rwt_all("GRIDFTP_DPM_single" ${gsiftp_prefix_dpm} 1) test_rwt_seq("GRIDFTP_DPM" ${gsiftp_prefix_dpm} 100 4560) test_rwt_seq("GRIDFTP_DPM_unit" ${gsiftp_prefix_dpm} 1 10) test_rwt_seek("GRIDFTP_DPM" ${gsiftp_prefix_dpm} 100 4560) test_space("GRIDFTP_DPM" ${gsiftp_prefix_dpm}) test_stat_all("FTP" ${ftp_prefix}) ENDIF(PLUGIN_GRIDFTP) IF(PLUGIN_HTTP) test_del("DAVS_DPM" ${davs_prefix_dpm}) test_del("DAVS_DCACHE" ${davs_prefix_dcache}) test_stat_all("DAVS_DPM" ${davs_prefix_dpm}) test_stat_all("DAVS_DCACHE" ${davs_prefix_dcache}) test_checksum_simple("DAVS_DPM_ADLER32" ${davs_prefix_dpm} ADLER32) test_checksum_simple("DAVS_DACHE_ADLER32" ${davs_prefix_dcache} ADLER32) test_checksum_simple("DAVS_DPM_MD5" ${davs_prefix_dpm} MD5) test_checksum_simple("DAVS_DCACHE_MD5" ${davs_prefix_dcache} MD5) test_checksum_simple("DAVS_DPM_CRC32" ${davs_prefix_dpm} CRC32) test_checksum_simple("DAVS_DCACHE_CRC32" ${davs_prefix_dcache} CRC32) test_mkdir_all("DAVS_DPM" ${davs_prefix_dpm}) test_mkdir_all("DAVS_DCACHE" ${davs_prefix_dcache}) test_rename("DAVS_DPM" ${davs_prefix_dpm}) test_rename("DAVS_DCACHE" ${davs_prefix_dcache}) # chmod not supported test_rmdir_all("DAVS_DPM" ${davs_prefix_dpm}) test_rmdir_all("DAVS_DCACHE" ${davs_prefix_dcache}) test_readdir_full("DAVS_DPM" ${davs_prefix_dpm}) test_readdir_full("DAVS_DCACHE" ${davs_prefix_dcache}) test_rwt_all("DAVS_DPM" ${davs_prefix_dpm} 4578) test_rwt_all("DAVS_DCACHE" ${davs_prefix_dcache} 4578) test_rwt_all("DAVS_DPM_single" ${davs_prefix_dpm} 1) test_rwt_all("DAVS_DCACHE_single" ${davs_prefix_dcache} 1) # sequencial writes not supported test_qos("FIRST") ENDIF(PLUGIN_HTTP) IF(PLUGIN_XROOTD) test_del("XROOTD_DPM" ${root_prefix_dpm}) test_del("XROOTD_EOS" ${root_prefix_eos}) test_stat_all("XROOTD_DPM" ${root_prefix_dpm}) test_stat_all("XROOTD_EOS" ${root_prefix_eos}) test_access("XROOTD_DPM" ${root_prefix_dpm}) #test_access fails on EOS, to check #test_access("XROOTD_EOS" ${root_prefix_eos}) test_rename("XROOTD_DPM" ${root_prefix_dpm}) test_rename("XROOTD_EOS" ${root_prefix_eos}) # Checksum not supported yet in the XrdCl library # Chmod does not work in posix-style, so the test can not be used test_mkdir_all("XROOTD_DPM" ${root_prefix_dpm}) test_mkdir_all("XROOTD_EOS" ${root_prefix_eos}) test_rmdir_all("XROOTD_DPM" ${root_prefix_dpm}) #test_rmdir_all fails on EOS, to check #test_rmdir_all("XROOTD_EOS" ${root_prefix_eos}) test_readdir_full("XROOTD_DPM" ${root_prefix_dpm}) test_readdir_full("XROOTD_EOS" ${root_prefix_eos}) test_rwt_all("XROOTD_DPM" ${root_prefix_dpm} 4578) test_rwt_all("XROOTD_EOS" ${root_prefix_eos} 4578) test_rwt_all("XROOTD_DPM_single" ${root_prefix_dpm} 1) test_rwt_all("XROOTD_EOS_single" ${root_prefix_eos} 1) test_rwt_seq("XROOTD_DPM" ${root_prefix_dpm} 100 4560) test_rwt_seq("XROOTD_EOS" ${root_prefix_eos} 100 4560) test_rwt_seq("XROOTD_DPM_single" ${root_prefix_dpm} 1 10) test_rwt_seq("XROOTD_EOS_single" ${root_prefix_eos} 1 10) test_rwt_seek("XROOTD_DPM" ${root_prefix_dpm} 100 4560) test_rwt_seek("XROOTD_EOS" ${root_prefix_eos} 100 4560) test_space("XROOTD_DPM" ${root_prefix_dpm}) #test_space fails on EOS, to check #test_space("XROOTD_EOS" ${root_prefix_eos}) # Copies IF (MAIN_TRANSFER) test_copy_file_no_checksum("XROOTD_DPM" ${root_prefix_dpm} ${root_prefix_dpm}) test_copy_file_no_checksum("XROOTD_EOS" ${root_prefix_eos} ${root_prefix_eos}) test_copy_bulk("XROOTD_DPM" ${root_prefix_dpm} ${root_prefix_dpm}) test_copy_bulk("XROOTD_EOS" ${root_prefix_eos} ${root_prefix_eos}) ENDIF (MAIN_TRANSFER) ENDIF() #IF (PLUGIN_SFTP) # test_del("SFTP" "${sftp_prefix}") # test_rename("SFTP" "${sftp_prefix}") # test_stat_all("SFTP" "${sftp_prefix}") # test_chmod_all("SFTP" "${sftp_prefix}" 0565 060 360 767) # test_mkdir_all("SFTP" "${sftp_prefix}") # test_rmdir_all("SFTP" "${sftp_prefix}") # test_readdir_full("SFTP" "${sftp_prefix}") # test_rwt_all("SFTP" "${sftp_prefix}" 4578) # test_rwt_all("SFTP" "${sftp_prefix}" 1) # test_rwt_all("SFTP" "${sftp_prefix}" 100000) # test_rwt_seq("SFTP" "${sftp_prefix}" 100 4560) # test_rwt_seek("SFTP" "${sftp_prefix}" 100 4560) #ENDIF () IF (MAIN_TRANSFER) test_copy_file_full("GRIDFTP_TO_GRIDFTP" ${gsiftp_prefix_dpm} ${gsiftp_prefix_dpm}) test_copy_file_full("SRM_DPM_TO_DCACHE" ${srm_prefix_dpm} ${srm_prefix_dcache}) test_copy_file_full("SRM_DCACHE_TO_DPM" ${srm_prefix_dcache} ${srm_prefix_dpm}) test_copy_file_full("GRIDFTP_TO_SRM" ${gsiftp_prefix_dpm} ${srm_prefix_dcache}) test_copy_file_full("SRM_TO_GRIDFTP" ${srm_prefix_dpm} ${gsiftp_prefix_dpm}) test_copy_file_full("DAVS_TO_DAVS" ${davs_prefix_dpm} ${davs_prefix_dpm}) test_copy_file_full("FILE_TO_SRM" ${file_prefix} ${srm_prefix_dpm}) test_copy_file_full("SRM_TO_FILE" ${srm_prefix_dcache} ${file_prefix}) test_copy_file_full("FILE_TO_FILE" ${file_prefix} ${file_prefix}) test_copy_file_full("GRIDFTP_TO_FILE" ${gsiftp_prefix_dpm} ${file_prefix}) test_copy_file_full("FILE_TO_GRIDFTP" ${file_prefix} ${gsiftp_prefix_dpm}) test_copy_file_full("STORM_TO_STORM" ${srm_prefix_storm} ${srm_prefix_storm}) test_copy_file_full("STORM_TO_SRM_DPM" ${srm_prefix_storm} ${srm_prefix_dpm}) test_copy_file_full("SRM_DPM_TO_CASTOR" ${srm_prefix_dpm} ${srm_prefix_castor}) test_copy_file_full("SRM_CASTOR_TO_DPM" ${srm_prefix_castor} ${srm_prefix_dpm}) # particular cases, storm to gridftp (i.e. can't reuse turl) test_copy_file_full("STORM_TO_GRIDFTP" ${srm_prefix_storm} ${gsiftp_prefix_dpm}) test_copy_file_full("GRIDFTP_TO_STORM" ${gsiftp_prefix_dpm} ${srm_prefix_storm}) # bulk, only a subset, otherwise this takes too long test_copy_bulk("GSIFTP" ${gsiftp_prefix_dpm} ${gsiftp_prefix_dpm}) test_copy_bulk("SRM" ${srm_prefix_dpm} ${srm_prefix_dcache}) # Passive plugin, which only makes sense for GridFTP test_pasv("PASV" ${gsiftp_prefix_dpm} ${gsiftp_prefix_dpm}) # Same for double credentials test_double_cred("GSIFTP" ${gsiftp_prefix_dpm} ${gsiftp_prefix_dpm}) ENDIF (MAIN_TRANSFER) gfal2-v2.23.0/test/functional/gfal_test_access.cpp000066400000000000000000000051561465240014500221050ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include #include #include #include class AccessTest: public testing::Test { public: static const char* root; char surl[2048]; gfal2_context_t context; AccessTest() { GError *error = NULL; context = gfal2_context_new(&error); Gfal::gerror_to_cpp(&error); } virtual ~AccessTest() { gfal2_context_free(context); } virtual void SetUp() { GError* error = NULL; generate_random_uri(root, "access_test", surl, sizeof(surl)); int ret = gfal2_mkdir(context, surl, 0775, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); } virtual void TearDown() { GError* error = NULL; gfal2_chmod(context, surl, 0775, &error); gfal2_rmdir(context, surl, &error); g_clear_error(&error); } }; const char* AccessTest::root; TEST_F(AccessTest, SimpleAccess) { GError *error = NULL; int ret = gfal2_access(context, surl, R_OK, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); // If running as root, and the surl is file://, it will always be possible to do anything if (strncmp(surl, "file://", 7) != 0 || geteuid() != 0) { ret = gfal2_chmod(context, surl, 0440, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); ret = gfal2_access(context, surl, W_OK, &error); EXPECT_PRED_FORMAT3(AssertGfalErrno, ret, error, EACCES); } } int main(int argc, char** argv) { testing::InitGoogleTest(&argc, argv); if (argc < 2) { printf("Missing base url and/or modes\n"); printf("\t%s [options] srm://host/base/path/\n", argv[0]); return 1; } AccessTest::root = argv[1]; return RUN_ALL_TESTS(); } gfal2-v2.23.0/test/functional/gfal_test_archive.cpp000066400000000000000000000142351465240014500222630ustar00rootroot00000000000000/* * Copyright (c) CERN 2020 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include #include #include #include #include class ArchiveTest: public testing::Test { public: static const char* root; static const char* test_file; char surl[2048]; gfal2_context_t handle; ArchiveTest() { GError* error = NULL; handle = gfal2_context_new(&error); Gfal::gerror_to_cpp(&error); } virtual ~ArchiveTest() { gfal2_context_free(handle); } virtual void SetUp() { setenv("XrdSecPROTOCOL", "gsi,unix", 1); uploadFile(surl, sizeof(surl)); } virtual void TearDown() { clearFile(surl); unsetenv("XrdSecPROTOCOL"); } protected: void uploadFile(char* surl, size_t s_surl) { generate_random_uri(root, "archive", surl, s_surl); GError* error = NULL; int ret = generate_file_if_not_exists(handle, surl, test_file, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); g_clear_error(&error); } void clearFile(char* surl) { GError* error = NULL; int ret = gfal2_unlink(handle, surl, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); g_clear_error(&error); } }; const char* ArchiveTest::root = NULL; const char* ArchiveTest::test_file = "file:///etc/hosts"; // Poll a non-existing file TEST_F(ArchiveTest, NoEntryPoll) { GError* error = NULL; char inexistent[2048]; generate_random_uri(root, "archive_enoent", inexistent, sizeof(inexistent)); int ret = gfal2_archive_poll(handle, inexistent, &error); ASSERT_EQ(-1, ret); ASSERT_PRED_FORMAT3(AssertGfalOneOfErrno, -1, error, (std::list{ENOENT, ENOMSG})); } // Poll invalid hostname TEST_F(ArchiveTest, InvalidHostPoll) { GError* error = NULL; char invalid_surl[2048]; const char* format = "%s://invalid.%sfile.test"; gfal2_uri* parsed = gfal2_parse_uri(root, &error); ASSERT_NE(parsed, (void *) NULL); if (root[strlen(root) - 1] != '/') { format = "%s://invalid.%s/file.test"; } snprintf(invalid_surl, sizeof(invalid_surl), format, parsed->scheme, (root + strlen(parsed->scheme) + 3)); g_clear_error(&error); gfal2_free_uri(parsed); int ret = gfal2_archive_poll(handle, invalid_surl, &error); ASSERT_EQ(-1, ret); ASSERT_PRED_FORMAT3(AssertGfalOneOfErrno, -1, error, (std::list{ECOMM, EHOSTUNREACH})); } // Poll a single file for archiving TEST_F(ArchiveTest, SingleFilePoll) { GError* error = NULL; int interval = 1; int ret; do { printf ("Polling for archive: %s\n", surl); g_clear_error(&error); ret = gfal2_archive_poll(handle, surl, &error); if (error != NULL) { ASSERT_EQ(EAGAIN, error->code); if (error->code == EAGAIN) { g_clear_error(&error); } } printf("Waiting %ds before new archive poll...\n", interval); fflush(stdout); sleep(interval); interval <<= 1; if (interval > 300) { interval = 300; } } while (ret == 0); ASSERT_EQ(1, ret); ASSERT_PRED_FORMAT2(AssertGfalSuccess, 1, error); } // Poll a file list where some entries exist in the namespace TEST_F(ArchiveTest, ListPoll) { char surl_second[2048]; char inexistent[2048]; int interval = 1; int ret; uploadFile(surl_second, sizeof(surl_second)); generate_random_uri(root, "archive_enoent", inexistent, sizeof(inexistent)); char* surls[] = { inexistent, surl, surl_second }; GError* errors[] = { NULL, NULL, NULL }; memset(errors, 0x00, sizeof(errors)); int nbfiles = sizeof(surls) / sizeof(char *); do { printf ("Polling for archive:\n"); for (int i = 0 ; i < nbfiles; i++) { printf("\t%s\n", surls[i]); g_clear_error(&errors[i]); } ret = gfal2_archive_poll_list(handle, nbfiles, surls, errors); // Check that the first file does not exist ASSERT_PRED_FORMAT3(AssertGfalOneOfErrno, -1, errors[0], (std::list{ENOENT, ENOMSG})); // Check that remaining files exist for (int i = 1; i < nbfiles; i++) { if (errors[1] != NULL) { ASSERT_EQ(EAGAIN, errors[i]->code); g_clear_error(&errors[i]); } } printf("Waiting %ds before new archive poll...\n", interval); fflush(stdout); sleep(interval); interval <<= 1; if (interval > 300) { interval = 300; } } while (ret == 0); ASSERT_EQ(2, ret); ASSERT_PRED_FORMAT3(AssertGfalOneOfErrno, -1, errors[0], (std::list{ENOENT, ENOMSG})); for (int i = 1; i < nbfiles; i++) { ASSERT_PRED_FORMAT2(AssertGfalSuccess, 1, errors[i]); } clearFile(surl_second); } int main(int argc, char** argv) { testing::InitGoogleTest(&argc, argv); if (argc < 2) { printf("Missing parameters:\n"); printf("\t%s [options] root://:1094// [test_file]\n", argv[0]); return 1; } ArchiveTest::root = argv[1]; if (argc == 3) { char* s_test_file = argv[2]; if (strncmp(argv[2], "file://", 7) != 0) { s_test_file = new char[sizeof(argv[2]) + 8]; strcpy(s_test_file, "file://"); strcat(s_test_file, argv[2]); } ArchiveTest::test_file = s_test_file; } return RUN_ALL_TESTS(); } gfal2-v2.23.0/test/functional/gfal_test_bringonline.cpp000066400000000000000000000250701465240014500231470ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include #include #include #include #include class BringonlineTest: public testing::Test { public: static const char* root; char surl[2048]; gfal2_context_t handle; BringonlineTest() { GError *error = NULL; handle = gfal2_context_new(&error); Gfal::gerror_to_cpp(&error); } virtual ~BringonlineTest() { gfal2_context_free(handle); } virtual void SetUp() { generate_random_uri(root, "bringonline", surl, 2048); RecordProperty("Surl", surl); GError* error = NULL; int ret = generate_file_if_not_exists(handle, surl, "file:///etc/hosts", &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); } virtual void TearDown() { GError *error = NULL; gfal2_unlink(handle, surl, &error); g_clear_error(&error); } }; const char* BringonlineTest::root; // Synchronous call, one single file TEST_F(BringonlineTest, SingleBringOnlineSync) { GError* error = NULL; char token[64]; int ret; ret = gfal2_bring_online(handle, surl, 10, 28800, token, sizeof(token), 0, &error); ASSERT_PRED_FORMAT2(AssertGfalSuccess, ret, error); ASSERT_GE(ret,0); } // Asynchronous call, one single file TEST_F(BringonlineTest, SingleBringOnlineAsync) { GError* error = NULL; char token[64] = {0}; int ret; ret = gfal2_bring_online(handle, surl, 10, 28800, token, sizeof(token), TRUE, &error); ASSERT_PRED_FORMAT2(AssertGfalSuccess, ret, error); if (ret == 0) { ASSERT_NE(0, token[0]); while (ret == 0) { sleep(1); printf("Poll\n"); ret = gfal2_bring_online_poll(handle, surl, token, &error); } } } // Synchronous call, two files (one does not exist) TEST_F(BringonlineTest, TwoBringOnlineSync) { GError* error[2] = {NULL, NULL}; char token[64] = {0}; int ret; char not_exist[2048]; generate_random_uri(root, "bringonline_enoent", not_exist, sizeof(not_exist)); char* surls[] = { not_exist, surl }; ret = gfal2_bring_online_list(handle, 2, surls, 10, 28800, token, sizeof(token), FALSE, error); ASSERT_GE(ret,0); // Tape REST API returns a success even if files in the request do not exist if (ret == 0) { ASSERT_PRED_FORMAT2(AssertGfalSuccess, ret, error[0]); } else { ASSERT_PRED_FORMAT3(AssertGfalErrno, -1, error[0], ENOENT); } ASSERT_PRED_FORMAT2(AssertGfalSuccess, 1, error[1]); } // Asynchronous call, two files (one does not exist) TEST_F(BringonlineTest, TwoBringOnlineAsync) { GError* error[2] = {NULL, NULL}; char token[64] = {0}; int ret; char not_exist[2048]; generate_random_uri(root, "bringonline_enoent", not_exist, sizeof(not_exist)); char* surls[] = { not_exist, surl }; ret = gfal2_bring_online_list(handle, 2, surls, 10, 28800, token, sizeof(token), TRUE, error); if (ret == 0) { ASSERT_NE(0, token[0]); while (ret == 0) { sleep(1); g_clear_error(&error[0]); g_clear_error(&error[1]); printf("Poll\n"); ret = gfal2_bring_online_poll_list(handle, 2, surls, token, error); if (error[0] != NULL) { ASSERT_TRUE(error[0]->code == EAGAIN || error[0]->code == ENOENT || error[0]->code == ENOMSG); if (error[0]->code == EAGAIN) { g_clear_error(&error[0]); } } if (error[1] != NULL) { ASSERT_EQ(error[1]->code, EAGAIN); g_clear_error(&error[1]); } } } ASSERT_GT(ret, 0); ASSERT_PRED_FORMAT3(AssertGfalOneOfErrno, -1, error[0], (std::list{ENOENT,ENOMSG})); ASSERT_PRED_FORMAT2(AssertGfalSuccess, 1, error[1]); } // Synchronous call, release TEST_F(BringonlineTest, SingleReleaseSync) { GError* error = NULL; char token[64]; int ret; ret = gfal2_bring_online(handle, surl, 10, 28800, token, sizeof(token), FALSE, &error); ASSERT_PRED_FORMAT2(AssertGfalSuccess, ret, error); ASSERT_GE(ret,0); if (token[0]) { printf("Release\n"); ret = gfal2_release_file(handle, surl, token, &error); ASSERT_PRED_FORMAT2(AssertGfalSuccess, ret, error); } } // Asynchronous call, two files, abort // Note that Castor returns an error (EIO) when a file has been processed before the abort TEST_F(BringonlineTest, TwoAbort) { GError* error[2] = {NULL, NULL}; char token[64] = {0}; int ret; char not_exist[2048]; generate_random_uri(root, "bringonline_enoent", not_exist, sizeof(not_exist)); char* surls[] = { not_exist, surl }; ret = gfal2_bring_online_list(handle, 2, surls, 10, 28800, token, sizeof(token), TRUE, error); if (ret == 0 && token[0]) { ret = gfal2_abort_files(handle, 2, surls, token, error); ASSERT_TRUE(error[0] == NULL || error[0]->code == ENOENT); ASSERT_TRUE(error[1] == NULL || error[1]->code == EIO || error[1]->code == ENOENT); while (ret == 0) { sleep(1); printf("Poll\n"); g_clear_error(&error[0]); g_clear_error(&error[1]); ret = gfal2_bring_online_poll_list(handle, 2, surls, token, error); } ASSERT_TRUE(error[0]->code == ECANCELED || error[0]->code == ENOENT || error[0]->code == ENOMSG); ASSERT_TRUE(error[1] == NULL ||error[1]->code == ECANCELED || error[1]->code == EIO || error[1]->code == ENOENT); } } // Poll with an invalid token TEST_F(BringonlineTest, InvalidPoll) { GError* error[2] = {NULL, NULL}; int ret; char* surls[] = { surl, surl }; ret = gfal2_bring_online_poll_list(handle, 2, surls, "1234-5678-badabad", error); // With xroot you can poll with an invalid token if(ret < 0) { ASSERT_TRUE(error[0]->code == EBADR || error[0]->code == EIO || error[0]->code == EINVAL || error[0]->code == ENOMSG); ASSERT_TRUE(error[1]->code == EBADR || error[1]->code == EIO || error[1]->code == EINVAL || error[1]->code == ENOMSG); } else{ ASSERT_PRED_FORMAT2(AssertGfalSuccess, ret, error[0]); ASSERT_PRED_FORMAT2(AssertGfalSuccess, ret, error[1]); } } // Release an invalid token TEST_F(BringonlineTest, InvalidRelease) { GError* error = NULL; int ret; ret = gfal2_release_file(handle, surl, "1234-5678-badabad", &error); // Some storages return a success even if the token does not exist if (ret) { ASSERT_PRED_FORMAT3(AssertGfalOneOfErrno, ret, error, (std::list{EBADR, EHOSTUNREACH})); } else { ASSERT_PRED_FORMAT2(AssertGfalSuccess, ret, error); } } // Poll invalid hostname TEST_F(BringonlineTest, InvalidHostPoll) { GError* error = NULL; char invalid_surl[2048]; const char* format = "%s://invalid.%sfile.test"; int ret; gfal2_uri* parsed = gfal2_parse_uri(root, &error); ASSERT_NE(parsed, (void *) NULL); if (root[strlen(root) - 1] != '/') { format = "%s://invalid.%s/file.test"; } snprintf(invalid_surl, sizeof(invalid_surl), format, parsed->scheme, (root + strlen(parsed->scheme) + 3)); g_clear_error(&error); gfal2_free_uri(parsed); ret = gfal2_bring_online_poll(handle, invalid_surl, "bringonline-token", &error); ASSERT_EQ(-1, ret); ASSERT_PRED_FORMAT3(AssertGfalOneOfErrno, -1, error, (std::list{ECOMM, EHOSTUNREACH})); } // Request with duplicated SURLs (see DMC-676) TEST_F(BringonlineTest, DuplicatedSURLs) { const int nbfiles = 100; GError* error[nbfiles]; char token[64] = {0}; int ret; memset(error, 0x00, sizeof(error)); char *surls[nbfiles]; // 0 exists surls[0] = surl; // 1 does not surls[1] = (char*)calloc(1, 2048); generate_random_uri(root, "bringonline_duplicated", surls[1], 2048); // all the rest are 0 or 1 duplicated for (int i = 1; i < nbfiles; ++i) { surls[i] = surls[i % 2]; } ret = gfal2_bring_online_list(handle, nbfiles, surls, 10, 28800, token, sizeof(token), TRUE, error); if (ret == 0) { ASSERT_NE(0, token[0]); while (ret == 0) { sleep(1); for (int i = 0; i < nbfiles; ++i) { g_clear_error(&error[i]); } printf("Poll\n"); ret = gfal2_bring_online_poll_list(handle, nbfiles, surls, token, error); for (int i = 0; i < nbfiles; ++i) { if (error[i] != NULL) { if (i % 2 == 0) { ASSERT_EQ(EAGAIN, error[i]->code); } else { ASSERT_TRUE(error[i]->code == EAGAIN || error[i]->code == ENOENT || error[i]->code == ENOMSG); } if (error[i] && error[i]->code == EAGAIN) { g_clear_error(&error[i]); } } } } } // Only the first one and duplicated should be successful for (int i = 0; i < nbfiles; ++i) { if (i % 2 == 0) ASSERT_PRED_FORMAT2(AssertGfalSuccess, 1, error[i]); else ASSERT_PRED_FORMAT3(AssertGfalOneOfErrno, -1, error[i], (std::list{ENOENT,ENOMSG})); } free(surls[1]); } int main(int argc, char** argv) { testing::InitGoogleTest(&argc, argv); if (argc < 2) { printf("Missing base urls\n"); printf("\t%s [options] srm://host/base/path/\n", argv[0]); return 1; } BringonlineTest::root = argv[1]; return RUN_ALL_TESTS(); } gfal2-v2.23.0/test/functional/gfal_test_checksum.cpp000066400000000000000000000076631465240014500224530ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include #include #include #include #include #include // Given an algorithm, returns a pre-calculated content with the hash in the requested // algorithm static int get_precomputed(const char* algorithm, const char** content, const char** checksum) { static const char *message = "THIS IS A STATIC MESSAGE USED FOR CHECKSUMING"; static struct { const char* algorithm; const char* hash; } precomputed[] = { { "MD5", "d0fb2562f39431dbe78c140581ca06b8" }, { "SHA1", "87c813e21babb87879588dc6009770f3c2b5d2f6" }, { "ADLER32", "111a0c0c" }, { "CRC32", "3971548410" }, { NULL, NULL } }; *content = message; size_t i; for (i = 0; precomputed[i].algorithm != NULL; ++i) { if (strcasecmp(algorithm, precomputed[i].algorithm) == 0) { *checksum = precomputed[i].hash; return 0; } } return -1; } class ChecksumTest: public testing::Test { public: static const char* root; static const char* algorithm; gfal2_context_t context; char surl[2048]; const char* precomputed_content; const char* precomputed_hash; ChecksumTest() { GError *error = NULL; context = gfal2_context_new(&error); Gfal::gerror_to_cpp(&error); } virtual ~ChecksumTest() { gfal2_context_free(context); } virtual void SetUp() { generate_random_uri(root, "test_checksum", surl, sizeof(surl)); GError* error = NULL; if (get_precomputed(algorithm, &precomputed_content, &precomputed_hash) != 0) { throw Gfal::CoreException(g_quark_from_static_string("TestChecksum"), EINVAL, "Could not get the pre-calculated checksum"); } int fd = gfal2_open(context, surl, O_CREAT | O_TRUNC | O_WRONLY, &error); Gfal::gerror_to_cpp(&error); size_t len = strlen(precomputed_content); gfal2_write(context, fd, precomputed_content, len, &error); Gfal::gerror_to_cpp(&error); gfal2_close(context, fd, &error); Gfal::gerror_to_cpp(&error); } virtual void TearDown() { GError* error = NULL; gfal2_unlink(context, surl, &error); g_clear_error(&error); } }; const char* ChecksumTest::root; const char* ChecksumTest::algorithm; TEST_F(ChecksumTest, ValidChecksum) { GError* error = NULL; char buffer[512]; int ret = gfal2_checksum(context, surl, algorithm, 0, 0, buffer, sizeof(buffer), &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); ret = gfal_compare_checksums(precomputed_hash, buffer, sizeof(buffer)); EXPECT_EQ(0, ret); } int main(int argc, char** argv) { testing::InitGoogleTest(&argc, argv); if (argc < 3) { printf("Missing base url\n"); printf("usage: %s root_dir [MD5|ADLER32|SHA1|CRC32]\n", argv[0]); return 1; } ChecksumTest::root = argv[1]; ChecksumTest::algorithm = argv[2]; return RUN_ALL_TESTS(); } gfal2-v2.23.0/test/functional/gfal_test_chmod.cpp000066400000000000000000000063451465240014500217370ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include #include #include #include class ChmodTest: public testing::Test { public: static const char* root; static int n_modes; static mode_t *modes; char surl[2048]; gfal2_context_t context; ChmodTest() { GError *error = NULL; context = gfal2_context_new(&error); Gfal::gerror_to_cpp(&error); } virtual ~ChmodTest() { gfal2_context_free(context); } virtual void SetUp() { GError* error = NULL; generate_random_uri(root, "chmod_test", surl, sizeof(surl)); int ret = gfal2_mkdir(context, surl, 0775, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); } virtual void TearDown() { GError* error = NULL; gfal2_chmod(context, surl, 0777, &error); g_clear_error(&error); gfal2_rmdir(context, surl, &error); g_clear_error(&error); } }; const char* ChmodTest::root; int ChmodTest::n_modes = 0; mode_t* ChmodTest::modes = NULL; TEST_F(ChmodTest, SimpleChmod) { GError* error = NULL; struct stat st; int ret = gfal2_stat(context, surl, &st, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); mode_t original_mode = st.st_mode; int i; for (i = 0; i < n_modes; ++i) { ret = gfal2_chmod(context, surl, modes[i], &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); ret = gfal2_stat(context, surl, &st, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); EXPECT_EQ(modes[i], st.st_mode & 0777); }; ret = gfal2_chmod(context, surl, original_mode, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); } int main(int argc, char** argv) { testing::InitGoogleTest(&argc, argv); if (argc < 3) { printf("Missing base url and/or modes\n"); printf("\t%s [options] srm://host/base/path/ 0777 0555 0000....\n", argv[0]); return 1; } ChmodTest::root = argv[1]; ChmodTest::n_modes = argc - 2; ChmodTest::modes = new mode_t[ChmodTest::n_modes]; int i; for (i = 2; i < argc; ++i) { errno = 0; ChmodTest::modes[i - 2] = (mode_t)strtol(argv[i], NULL, 8); if (errno) { perror ("strtol"); exit (1); } } int result = RUN_ALL_TESTS(); delete[] ChmodTest::modes; return result; } gfal2-v2.23.0/test/functional/gfal_test_del.cpp000066400000000000000000000140301465240014500213770ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include #include #include #include #define BLKLEN 65536 class DeleteTest: public testing::Test { public: static const char* root; char nested_file[2048]; const static int N_FILES = 5; char* files[N_FILES]; gfal2_context_t context; DeleteTest() { GError *error = NULL; context = gfal2_context_new(&error); Gfal::gerror_to_cpp(&error); for (int i = 0; i < N_FILES; ++i) { files[i] = new char[2048]; } } virtual ~DeleteTest() { gfal2_context_free(context); for (int i = 0; i < N_FILES; ++i) { delete [] files[i]; } } virtual void SetUp() { nested_file[0] = '\0'; int ret; GError* error = NULL; for (int i = 0; i < N_FILES; ++i) { generate_random_uri(root, "test_del", files[i], 2048); ret = generate_file_if_not_exists(context, files[i], "file:///etc/hosts", &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); } } virtual void TearDown() { GError *error = NULL; if (nested_file[0]) { gfal2_unlink(context, nested_file, &error); g_clear_error(&error); } for (int i = 0; i < N_FILES; ++i) { if (gfal2_unlink(context, files[i], &error) < 0) { g_clear_error(&error); gfal2_rmdir(context, files[i], &error); g_clear_error(&error); } } } }; TEST_F(DeleteTest, DeleteEnoent) { char buff[2048]; snprintf(buff, sizeof(buff), "%s/testfileunlink_enoent", root); GError *err = NULL; int ret = gfal2_unlink(context, buff, &err); EXPECT_PRED_FORMAT3(AssertGfalErrno, ret, err, ENOENT); } TEST_F(DeleteTest, DeleteIsDir) { GError *err = NULL; int ret = gfal2_unlink(context, files[0], &err); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, err); ret = gfal2_mkdir(context, files[0], 0775, &err); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, err); ret = gfal2_unlink(context, files[0], &err); EXPECT_PRED_FORMAT3(AssertGfalErrno, ret, err, EISDIR); } TEST_F(DeleteTest, DeleteSequential) { int ret; GError* error = NULL; for (int i = 0; i < N_FILES; ++i) { ret = gfal2_unlink(context, files[i], &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); } // Were they really removed? struct stat st; for (int i = 0; i < N_FILES; ++i) { GError *err = NULL; ret = gfal2_stat(context, files[i], &st, &err); EXPECT_PRED_FORMAT3(AssertGfalErrno, ret, err, ENOENT); } } TEST_F(DeleteTest, BulkDeletionOddFail) { int ret; // Remove odd files to force an error for (int i = 0; i < N_FILES; ++i) { GError* err = NULL; if (i % 2) { ret = gfal2_unlink(context, files[i], &err); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, err); } } GError *errors[N_FILES] = {0}; ret = gfal2_unlink_list(context, N_FILES, files, errors); EXPECT_LT(ret, 0); for (int i = 0; i < N_FILES; ++i) { if (i % 2) { EXPECT_PRED_FORMAT3(AssertGfalErrno, -1, errors[i], ENOENT); } else { EXPECT_EQ(NULL, errors[i]); } } // Were they really removed? struct stat st; for (int i = 0; i < N_FILES; ++i) { GError *err = NULL; if (i % 2 == 0) { ret = gfal2_stat(context, files[i], &st, &err); EXPECT_PRED_FORMAT3(AssertGfalErrno, ret, err, ENOENT); } } } TEST_F(DeleteTest, BulkDeletionIsDir) { int ret; GError* err = NULL; // Remove the first file, create dir instead ret = gfal2_unlink(context, files[0], &err); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, err); ret = gfal2_mkdir(context, files[0], 0775, &err); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, err); generate_random_uri(files[0], "test_del_nested_files", nested_file, 2048); ret = generate_file_if_not_exists(context, nested_file, "file:///etc/hosts", &err); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, err); // Call unlink all GError *errors[N_FILES] = {0}; ret = gfal2_unlink_list(context, N_FILES, files, errors); EXPECT_LT(ret, 0); // The first must fail for (int i = 0; i < N_FILES; ++i) { if (i == 0) { EXPECT_NE((void*)NULL, errors[i]); } else { EXPECT_EQ(NULL, errors[i]); } } // Were they really removed? 0 should have not! struct stat st; for (int i = 0; i < N_FILES; ++i) { GError *err = NULL; ret = gfal2_stat(context, files[i], &st, &err); if (i != 0) { EXPECT_PRED_FORMAT3(AssertGfalErrno, ret, err, ENOENT); } else { EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, err); } } } const char* DeleteTest::root; int main(int argc, char** argv) { testing::InitGoogleTest(&argc, argv); if (argc < 2) { printf("Missing base url\n"); printf("\t%s [options] srm://host/base/path/\n", argv[0]); return 1; } DeleteTest::root = argv[1]; return RUN_ALL_TESTS(); } gfal2-v2.23.0/test/functional/gfal_test_mkdir_full.cpp000066400000000000000000000047041465240014500227720ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include #include #include #include class MkDirTest: public testing::Test { public: static const char* root; char surl[2048]; gfal2_context_t context; MkDirTest() { GError *error = NULL; context = gfal2_context_new(&error); Gfal::gerror_to_cpp(&error); } virtual ~MkDirTest() { gfal2_context_free(context); } virtual void SetUp() { generate_random_uri(root, "mkdir_test", surl, sizeof(surl)); } virtual void TearDown() { GError* error = NULL; gfal2_rmdir(context, surl, &error); g_clear_error(&error); } }; const char* MkDirTest::root; TEST_F(MkDirTest, SimpleMkDir) { GError* error = NULL; int ret = gfal2_mkdir(context, surl, 0775, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); struct stat st; ret = gfal2_stat(context, surl, &st, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); EXPECT_TRUE(S_ISDIR(st.st_mode)); } TEST_F(MkDirTest, MkDirEEXIST) { GError* error = NULL; int ret = gfal2_mkdir(context, surl, 0775, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); ret = gfal2_mkdir(context, surl, 0775, &error); EXPECT_PRED_FORMAT3(AssertGfalErrno, ret, error, EEXIST); } int main(int argc, char** argv) { testing::InitGoogleTest(&argc, argv); if (argc < 2) { printf("Missing base url\n"); printf("\t%s [options] srm://host/base/path/\n", argv[0]); return 1; } MkDirTest::root = argv[1]; return RUN_ALL_TESTS(); } gfal2-v2.23.0/test/functional/gfal_test_posix.cpp000066400000000000000000000221311465240014500217760ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include #include class PosixTest: public testing::Test { public: static const char* root; std::list filesToClean; gfal2_context_t context; PosixTest() { GError *error = NULL; context = gfal2_context_new(&error); Gfal::gerror_to_cpp(&error); } ~PosixTest() { gfal2_context_free(context); } void SetUp() { } void TearDown() { typedef std::list::reverse_iterator iterator; for (iterator i = filesToClean.rbegin(); i != filesToClean.rend(); ++i) { GError *error = NULL; gfal2_unlink(context, i->c_str(), &error); g_clear_error(&error); gfal2_rmdir(context, i->c_str(), &error); g_clear_error(&error); } filesToClean.clear(); } std::string GenerateFile(std::string base = std::string()) { if (base.empty()) { base = root; } GError *error = NULL; char file[2048]; generate_random_uri(base.c_str(), "test_posix", file, sizeof(file)); int ret = generate_file_if_not_exists(context, file, "file:///etc/hosts", &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); filesToClean.push_back(file); return file; } std::string GenerateDir() { GError *error = NULL; char dir[2048]; generate_random_uri(root, "test_posix_dir", dir, sizeof(dir)); int ret = gfal2_mkdir(context, dir, 0775, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); filesToClean.push_back(dir); return dir; } }; const char *PosixTest::root; TEST_F(PosixTest, Access) { int ret = gfal_access(root, R_OK | W_OK | X_OK); ASSERT_EQ(0, ret); } TEST_F(PosixTest, ChMod) { std::string file = GenerateFile(); struct stat buf; int ret = gfal_chmod(file.c_str(), 0701); ASSERT_EQ(0, ret); ret = gfal_stat(file.c_str(), &buf); ASSERT_EQ(0, ret); ASSERT_EQ(buf.st_mode & ~S_IFMT, 0701); } TEST_F(PosixTest, Rename) { std::string file = GenerateFile(); int ret = gfal_access(file.c_str(), F_OK); ASSERT_EQ(0, ret); std::string newName = file + ".renamed"; filesToClean.push_back(newName); ret = gfal_rename(file.c_str(), newName.c_str()); ASSERT_EQ(0, ret); ret = gfal_access(newName.c_str(), F_OK); ASSERT_EQ(0, ret); ret = gfal_access(file.c_str(), F_OK); ASSERT_NE(0, ret); ASSERT_EQ(errno, ENOENT); } TEST_F(PosixTest, Stat) { std::string file = GenerateFile(); struct stat buf; int ret = gfal_stat(file.c_str(), &buf); ASSERT_EQ(0, ret); } TEST_F(PosixTest, LStat) { std::string file = GenerateFile(); struct stat buf; int ret = gfal_lstat(file.c_str(), &buf); ASSERT_EQ(0, ret); } TEST_F(PosixTest, OpenDir) { std::string dir = GenerateDir(); std::string file = GenerateFile(dir); int fnameIndex = file.rfind('/'); DIR* fd = gfal_opendir(dir.c_str()); ASSERT_NE((void*)NULL, fd); bool found = false; struct dirent *ent = gfal_readdir(fd); while (ent) { if (file.substr(fnameIndex + 1).compare(ent->d_name) == 0) { found = true; } ent = gfal_readdir(fd); } ASSERT_TRUE(found); gfal_closedir(fd); } TEST_F(PosixTest, OpenFile) { std::string file = GenerateFile(); int fd = gfal_open(file.c_str(), O_RDONLY); ASSERT_GT(fd, 0); char buffer[512] = {0}; ssize_t readSize = gfal_read(fd, buffer, sizeof(buffer)); ASSERT_GT(readSize, 0); off_t offset = gfal_lseek(fd, 2, SEEK_SET); ASSERT_EQ(offset, 2); readSize = gfal_read(fd, buffer + 50, sizeof(buffer) - 50); ASSERT_GT(readSize, 0); ASSERT_EQ(buffer[2], buffer[50]); ASSERT_EQ(buffer[3], buffer[51]); ASSERT_EQ(buffer[4], buffer[52]); gfal_close(fd); } TEST_F(PosixTest, Creat) { char file[2048]; generate_random_uri(root, "test_posix", file, sizeof(file)); filesToClean.push_back(file); int fd = gfal_creat(file, 0775); ASSERT_GT(fd, 0); char buffer[512] = {0}; ssize_t readSize = gfal_write(fd, buffer, sizeof(buffer)); ASSERT_EQ(readSize, sizeof(buffer)); gfal_close(fd); } TEST_F(PosixTest, SymLink) { if (strncmp(root, "file://", 7) != 0) { SKIP_TEST(SymLink); return; } std::string file = GenerateFile(); std::string link = file + ".link"; filesToClean.push_back(link); int ret = gfal_symlink(file.c_str(), link.c_str()); ASSERT_EQ(0, ret); struct stat buf; ret = gfal_lstat(link.c_str(), &buf); ASSERT_EQ(0, ret); ASSERT_TRUE(S_ISLNK(buf.st_mode)); ret = gfal_stat(link.c_str(), &buf); ASSERT_EQ(0, ret); ASSERT_FALSE(S_ISLNK(buf.st_mode)); char buffer[1024] = {0}; // Initialize to non-null value memset(buffer, 'x', sizeof(buffer)); ret = gfal_readlink(link.c_str(), buffer, sizeof(buffer)); ASSERT_GT(ret, 0); ASSERT_STREQ(file.substr(file.size() - ret).c_str(), buffer); } TEST_F(PosixTest, SymLinkTruncate) { if (strncmp(root, "file://", 7) != 0) { SKIP_TEST(SymLink); return; } std::string file = GenerateFile(); std::string link = file + ".link"; filesToClean.push_back(link); int ret = gfal_symlink(file.c_str(), link.c_str()); ASSERT_EQ(0, ret); size_t read_size = file.length() - 8; char buffer[1024]; GError* error; // Initialize to non-null value memset(buffer, 'x', sizeof(buffer)); // To capture the error object, we need to get the Posix thread handle gfal2_context_t handle = gfal_posix_get_handle(); ASSERT_NE(handle, (void *) NULL); ret = gfal2_readlink(handle, link.c_str(), buffer, read_size, &error); ASSERT_EQ(ret, read_size); ASSERT_STRNE(file.substr(file.size() - ret).c_str(), buffer); // Last read character should match the original ASSERT_EQ(buffer[read_size - 1], file[read_size + 6]); // Character at read_size should be the 'x' padding ASSERT_EQ(buffer[read_size], 'x'); // Readlink should return an error about possible truncation ASSERT_PRED_FORMAT3(AssertGfalErrno, -1, error, ENOMEM); } TEST_F(PosixTest, Xattr) { std::string file = GenerateFile(); char attrValue[] = "hello there"; int ret = gfal_setxattr(file.c_str(), "user.attr", attrValue, sizeof(attrValue), 0); ASSERT_EQ(0, ret); char buffer[64] = {0}; ret = gfal_getxattr(file.c_str(), "user.attr", buffer, sizeof(buffer)); ASSERT_EQ(sizeof(attrValue), ret); ASSERT_STREQ(buffer, attrValue); ret = gfal_listxattr(file.c_str(), buffer, sizeof(buffer)); ASSERT_GE(ret, 10); bool found = false; int i = 0; while (i < ret) { if (strncmp(buffer + i, "user.attr", 8) == 0) { found = true; break; } i += strlen(buffer + i) + 1; } EXPECT_TRUE(found); } TEST_F(PosixTest, PWrite) { char file[2048]; generate_random_uri(root, "test_posix", file, sizeof(file)); filesToClean.push_back(file); int fd = gfal_creat(file, 0775); ASSERT_GT(fd, 0); char buffer[512] = {0}; ssize_t write_size = gfal_pwrite(fd, buffer, sizeof(buffer), 0); ASSERT_EQ(write_size, sizeof(buffer)); gfal_close(fd); char read_buffer[512] = {0}; fd = gfal_open(file, O_RDONLY); ssize_t read_size = gfal_read(fd, read_buffer, sizeof(read_buffer)); ASSERT_EQ(read_size, write_size); ASSERT_TRUE(memcmp(buffer, read_buffer, read_size) == 0); gfal_close(fd); } TEST_F(PosixTest, PRead) { std::string file = GenerateFile(); int fd = gfal_open(file.c_str(), O_RDONLY); ASSERT_GT(fd, 0); char buffer[512] = {0}; ssize_t readSize = gfal_pread(fd, buffer, sizeof(buffer), 0); ASSERT_GT(readSize, 0); readSize = gfal_pread(fd, buffer + 50, sizeof(buffer) - 50, 2); ASSERT_GT(readSize, 0); ASSERT_EQ(buffer[2], buffer[50]); ASSERT_EQ(buffer[3], buffer[51]); ASSERT_EQ(buffer[4], buffer[52]); gfal_close(fd); } int main(int argc, char** argv) { testing::InitGoogleTest(&argc, argv); if (argc < 2) { printf("Missing base url\n"); printf("\t%s [options] srm://host/base/path/\n", argv[0]); return 1; } PosixTest::root = argv[1]; return RUN_ALL_TESTS(); } gfal2-v2.23.0/test/functional/gfal_test_qos.cpp000066400000000000000000000076421465240014500214500ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include class QosTest: public testing::Test { public: static const char* host; static const char* file; static const char* target_qos; static const char* token; gfal2_context_t context; gfal2_cred_t* cred; QosTest() { GError *error = NULL; context = gfal2_context_new(&error); Gfal::gerror_to_cpp(&error); cred = gfal2_cred_new(GFAL_CRED_BEARER, token); gfal2_cred_set(context, host, cred, &error); Gfal::gerror_to_cpp(&error); } virtual ~QosTest() { gfal2_cred_free(cred); gfal2_context_free(context); } virtual void SetUp() { unsetenv("X509_USER_PROXY"); unsetenv("X509_USER_CERT"); unsetenv("X509_USER_KEY"); // Clear automatic setup gfal2_remove_opt(context, "X509", "CERT", NULL); gfal2_remove_opt(context, "X509", "KEY", NULL); } }; const char* QosTest::host = NULL; const char* QosTest::file = NULL; const char* QosTest::target_qos = NULL; const char* QosTest::token = NULL; TEST_F(QosTest, TestQosClasses) { char buff[2048]; GError* err = NULL; ssize_t result = gfal2_qos_check_classes(context, host, "dataobject", buff, 2048, &err); if (result > 0) { std::cout << std::string(buff) << std::endl; } EXPECT_GT(result, 0); EXPECT_EQ(NULL, err); } TEST_F(QosTest, TestCheckFileQos) { char buff[2048]; GError* err = NULL; std::string url = std::string(host) + std::string(file); ssize_t result = gfal2_check_file_qos(context, url.c_str(), buff, 2048, &err); if (result > 0) { std::cout << std::string(buff) << std::endl; } EXPECT_GT(result, 0); EXPECT_EQ(NULL, err); } TEST_F(QosTest, TestCheckQoSTransitions) { char buff[2048]; GError* err = NULL; std::string url = std::string(host) + std::string("/cdmi_capabilities/dataobject/") + std::string(target_qos); ssize_t result = gfal2_check_available_qos_transitions(context, url.c_str(), buff, 2048, &err); if (result > 0) { std::cout << std::string(buff) << std::endl; } EXPECT_GT(result, 0); EXPECT_EQ(NULL, err); } TEST_F(QosTest, TestCheckTargetQoSOfFile) { char buff[2048]; GError* err = NULL; std::string url = std::string(host) + std::string(file); ssize_t result = gfal2_check_target_qos(context, url.c_str(), buff, 2048, &err); if (result > 0) { std::cout << std::string(buff) << std::endl; } EXPECT_GT(result, 0); EXPECT_EQ(NULL, err); } TEST_F(QosTest, TestChangeQosOfFile) { GError* err = NULL; std::string url = std::string(host) + std::string(file); std::string cdmi_target_qos = "/cdmi_capabilities/dataobject/" + std::string(target_qos); int result = gfal2_change_object_qos(context, url.c_str(), cdmi_target_qos.c_str(), &err); EXPECT_EQ(0, result); EXPECT_EQ(NULL, err); } int main(int argc, char** argv) { testing::InitGoogleTest(&argc, argv); if (argc < 5) { printf("Missing parameters: \n"); printf("\t%s [host] [file] [target_qos] [token]\n", argv[0]); return -1; } QosTest::host = argv[1]; QosTest::file = argv[2]; QosTest::target_qos = argv[3]; QosTest::token = argv[4]; return RUN_ALL_TESTS(); } gfal2-v2.23.0/test/functional/gfal_test_readdir_full.cpp000066400000000000000000000104321465240014500232710ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include #include #include #include #define NNESTED 10 class ReadDirTest: public testing::Test { public: static const char* root; char surl[2048]; char nested[NNESTED][2048]; gfal2_context_t context; ReadDirTest() { GError *error = NULL; context = gfal2_context_new(&error); Gfal::gerror_to_cpp(&error); } virtual ~ReadDirTest() { gfal2_context_free(context); } virtual void SetUp() { GError* error = NULL; generate_random_uri(root, "readdir_test", surl, sizeof(surl)); int ret = gfal2_mkdir(context, surl, 0777, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); // Populate for (int i = 0; i < NNESTED; ++i) { snprintf(nested[i], sizeof(nested[i]), "%s/nested_elem_%d", surl, i); ret = gfal2_mkdir(context, nested[i], 0777, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); g_clear_error(&error); } } virtual void TearDown() { GError* error = NULL; int i, ret; for (i = 0; i < NNESTED; ++i) { ret = gfal2_rmdir(context, nested[i], &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); g_clear_error(&error); } gfal2_rmdir(context, surl, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); g_clear_error(&error); } }; const char* ReadDirTest::root; TEST_F(ReadDirTest, ReadDirENOENT) { char enoent_dir[2048]; generate_random_uri(surl, "readdir_enoent", enoent_dir, sizeof(enoent_dir)); GError* error = NULL; DIR* dir = gfal2_opendir(context, enoent_dir, &error); EXPECT_PRED_FORMAT3(AssertGfalErrno, (dir == NULL)?-1:0, error, ENOENT); } TEST_F(ReadDirTest, ReadDir) { GError* error = NULL; DIR* dir = gfal2_opendir(context, surl, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, (dir == NULL)?-1:0, error); int count = 0; struct dirent* d = NULL; while ((d = gfal2_readdir(context, dir, &error)) != NULL) { // Just in case the plugin returns . and .. as entries if (strcmp(".", d->d_name) != 0 && strcmp("..", d->d_name) != 0) ++count; } EXPECT_PRED_FORMAT2(AssertGfalSuccess, 0, error); EXPECT_EQ(NNESTED, count); int ret = gfal2_closedir(context, dir, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); } TEST_F(ReadDirTest, ReadDirpp) { GError* error = NULL; DIR* dir = gfal2_opendir(context, surl, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, dir != NULL, error); int count = 0; struct dirent* d = NULL; struct stat st; while ((d = gfal2_readdirpp(context, dir, &st, &error)) != NULL) { // Just in case the plugin returns . and .. as entries if (strcmp(".", d->d_name) != 0 && strcmp("..", d->d_name) != 0) ++count; EXPECT_TRUE(S_ISDIR(st.st_mode)); } EXPECT_PRED_FORMAT2(AssertGfalSuccess, 0, error); EXPECT_EQ(NNESTED, count); int ret = gfal2_closedir(context, dir, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); } int main(int argc, char** argv) { testing::InitGoogleTest(&argc, argv); if (argc < 2) { printf("Missing base url\n"); printf("\t%s [options] srm://host/base/path/\n", argv[0]); return 1; } ReadDirTest::root = argv[1]; return RUN_ALL_TESTS(); } gfal2-v2.23.0/test/functional/gfal_test_register.cpp000066400000000000000000000116651465240014500224720ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ /* * NOTE: This test only makes sense for LFC as a destination! */ #include #include #include #include #include #include #include #include class RegisterTest: public testing::Test { public: static const char* lfc_root; static const char* origin_root; char origin[2048]; char origin2[2048]; char lfc[2048]; gfal2_context_t context; gfalt_params_t params; RegisterTest() { GError *error = NULL; context = gfal2_context_new(&error); Gfal::gerror_to_cpp(&error); params = gfalt_params_handle_new(NULL); } virtual ~RegisterTest() { GError *error = NULL; gfalt_params_handle_delete(params, &error); gfal2_context_free(context); } virtual void SetUp() { GError* error = NULL; generate_random_uri(lfc_root, "register_test", lfc, sizeof(origin)); generate_random_uri(origin_root, "register_test", origin, sizeof(origin)); generate_random_uri(origin_root, "register_test", origin2, sizeof(origin2)); int ret = generate_file_if_not_exists(context, origin, "file:///etc/hosts", &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); ret = generate_file_if_not_exists(context, origin2, "file:///etc/group", &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); } virtual void TearDown() { GError* error = NULL; gfal2_unlink(context, origin, &error); g_clear_error(&error); gfal2_unlink(context, origin2, &error); g_clear_error(&error); gfal2_unlink(context, lfc, &error); g_clear_error(&error); } }; const char* RegisterTest::lfc_root; const char* RegisterTest::origin_root; TEST_F(RegisterTest, RegisterNewEntry) { GError* error = NULL; int ret = gfalt_copy_file(context, params, origin, lfc, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); char replicas[2048]; ssize_t xattr_size = gfal2_getxattr(context, lfc, GFAL_XATTR_REPLICA, replicas, sizeof(replicas), &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, 0, error); EXPECT_GT(xattr_size, 0); EXPECT_STRCASEEQ(origin, replicas); } TEST_F(RegisterTest, RegisterExistingEntry) { GError* error = NULL; int ret = gfalt_copy_file(context, params, origin, lfc, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); ret = gfalt_copy_file(context, params, origin, lfc, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); } TEST_F(RegisterTest, RegisterMismatch) { GError* error = NULL; int ret = gfalt_copy_file(context, params, origin, lfc, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); ret = gfalt_copy_file(context, params, origin2, lfc, &error); EXPECT_PRED_FORMAT3(AssertGfalErrno, ret, error, EINVAL); } TEST_F(RegisterTest, RegisterAndOpen) { GError* error = NULL; int ret = gfalt_copy_file(context, params, origin, lfc, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); ret = gfal2_open(context, lfc, O_RDONLY, &error); EXPECT_GE(ret, 0); EXPECT_PRED_FORMAT2(AssertGfalSuccess, 0, error); char buffer[1024] = {0}; ssize_t read = gfal2_read(context, ret, buffer, sizeof(buffer), &error); EXPECT_GT(read, 0); EXPECT_PRED_FORMAT2(AssertGfalSuccess, 0, error); char buffer2[1024] = {0}; FILE* fd = fopen("/etc/hosts", "r"); EXPECT_NE((void*)NULL, fd); size_t read2 = fread(buffer2, 1, sizeof(buffer2), fd); fclose(fd); EXPECT_EQ(read, read2); EXPECT_STREQ(buffer, buffer2); ret = gfal2_close(context, ret, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); } int main(int argc, char** argv) { testing::InitGoogleTest(&argc, argv); if (argc < 3) { printf("Missing origin or lfc url\n"); printf("\t%s srm://host/base/path/ lfc://host/base/path\n", argv[0]); return 1; } RegisterTest::origin_root = argv[1]; RegisterTest::lfc_root = argv[2]; if (argc > 3 && strncmp(argv[3], "-v", 2) == 0) { gfal2_log_set_level(G_LOG_LEVEL_DEBUG); } return RUN_ALL_TESTS(); } gfal2-v2.23.0/test/functional/gfal_test_rename.cpp000066400000000000000000000067311465240014500221130ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include #include #include #include class RenameTest: public testing::Test { public: static const char* root; char origin[2048]; char destination[2048]; gfal2_context_t context; RenameTest() { GError *error = NULL; context = gfal2_context_new(&error); Gfal::gerror_to_cpp(&error); } virtual ~RenameTest() { gfal2_context_free(context); } virtual void SetUp() { generate_random_uri(root, "rename_test", origin, sizeof(origin)); generate_random_uri(root, "rename_test", destination, sizeof(destination)); } virtual void ForceRm(const char *surl) { GError* error = NULL; if (gfal2_rmdir(context, surl, &error) != 0) { g_clear_error(&error); gfal2_unlink(context, surl, &error); } g_clear_error(&error); } virtual void TearDown() { ForceRm(origin); ForceRm(destination); } }; const char* RenameTest::root; // Rename a file to a different name, destination does not exist TEST_F(RenameTest, SimpleRename) { GError* error = NULL; int ret = generate_file_if_not_exists(context, origin, "file:///etc/hosts", &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); ret = gfal2_rename(context, origin, destination, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); struct stat buf; ret = gfal2_stat(context, origin, &buf, &error); EXPECT_PRED_FORMAT3(AssertGfalErrno, ret, error, ENOENT); g_clear_error(&error); ret = gfal2_stat(context, destination, &buf, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); } // Rename a file, but the new name already exists and is a directory TEST_F(RenameTest, RenameToExisting) { if (strncmp(origin, "srm://", 6) == 0 || strncmp(origin, "davs", 4) == 0) { SKIP_TEST(RenameToExisting); return; } GError* error = NULL; int ret = generate_file_if_not_exists(context, origin, "file:///etc/hosts", &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); ret = gfal2_mkdir(context, destination, 0775, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); ret = gfal2_rename(context, origin, destination, &error); EXPECT_PRED_FORMAT3(AssertGfalErrno, ret, error, EISDIR); } int main(int argc, char** argv) { testing::InitGoogleTest(&argc, argv); if (argc < 2) { printf("Missing base url\n"); printf("\t%s [options] srm://host/base/path/\n", argv[0]); return 1; } RenameTest::root = argv[1]; return RUN_ALL_TESTS(); } gfal2-v2.23.0/test/functional/gfal_test_rmdir_full.cpp000066400000000000000000000131131465240014500227730ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include #include #include #include class RmDirTest: public testing::Test { public: static const char* root; char surl[2048]; char surl_nested[2048]; gfal2_context_t context; RmDirTest() { GError *error = NULL; context = gfal2_context_new(&error); Gfal::gerror_to_cpp(&error); } virtual ~RmDirTest() { gfal2_context_free(context); } virtual void SetUp() { generate_random_uri(root, "test_rmdir", surl, sizeof(surl)); generate_random_uri(surl, "test_rmdir_nested", surl_nested, sizeof(surl_nested)); } virtual void TearDown() { GError* error = NULL; gfal2_unlink(context, surl, &error); g_clear_error(&error); gfal2_chmod(context, surl, 0777, &error); g_clear_error(&error); gfal2_rmdir(context, surl, &error); g_clear_error(&error); gfal2_rmdir(context, surl_nested, &error); g_clear_error(&error); } }; const char* RmDirTest::root = NULL; static int is_dav(const char* surl) { return strncmp(surl, "dav:", 4) == 0 || strncmp(surl, "davs:", 5) == 0 || strncmp(surl, "http:", 5) == 0 || strncmp(surl, "https:", 6) == 0; } TEST_F(RmDirTest, RmDirENOENT) { GError* error = NULL; int ret = gfal2_rmdir(context, surl, &error); EXPECT_PRED_FORMAT3(AssertGfalErrno, ret, error, ENOENT); } TEST_F(RmDirTest, RmDirExists) { struct stat st; GError* error = NULL; int ret = gfal2_mkdir(context, surl, 0777, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); // Trigger a stat, this will activate the cache (i.e. srm) // Regression for DMC-584 ret = gfal2_stat(context, surl, &st, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); ret = gfal2_rmdir(context, surl, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); // Make sure it is not there! ret = gfal2_stat(context, surl, &st, &error); EXPECT_PRED_FORMAT3(AssertGfalErrno, ret, error, ENOENT); } TEST_F(RmDirTest, RmDirExists2) { struct stat st; GError* error = NULL; g_strlcat(surl, "/", sizeof(surl)); // Same thing but with trailing slash int ret = gfal2_mkdir(context, surl, 0777, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); // Trigger a stat, this will activate the cache (i.e. srm) // Regression for DMC-584 ret = gfal2_stat(context, surl, &st, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); ret = gfal2_rmdir(context, surl, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); // Make sure it is not there! ret = gfal2_stat(context, surl, &st, &error); EXPECT_PRED_FORMAT3(AssertGfalErrno, ret, error, ENOENT); } TEST_F(RmDirTest, RmDirNestedNotEmpty) { // dav removes recursively, so skip if (is_dav(surl)) { SKIP_TEST(RmDirNestedNotEmpty); return; } GError* error = NULL; int ret = gfal2_mkdir(context, surl, 0777, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); ret = gfal2_mkdir(context, surl_nested, 0777, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); ret = gfal2_rmdir(context, surl, &error); EXPECT_PRED_FORMAT3(AssertGfalErrno, ret, error, ENOTEMPTY); } TEST_F(RmDirTest, RmDirNestedEACCESS) { if (is_dav(surl) || strncmp("file:", surl, 5) == 0 || strncmp("root:", surl, 5) == 0) { SKIP_TEST(RmDirNestedEACCESS); return; } GError* error = NULL; int ret = gfal2_mkdir(context, surl, 0777, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); ret = gfal2_mkdir(context, surl_nested, 0777, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); ret = gfal2_chmod(context, surl, 0000, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); ret = gfal2_rmdir(context, surl_nested, &error); EXPECT_PRED_FORMAT3(AssertGfalErrno, ret, error, EACCES); } TEST_F(RmDirTest, RmDirNestedENOTDIR) { // Skip for LFC if (strncmp(surl, "lfc:/", 5) == 0 || strncmp(surl, "lfn:/", 5) == 0) { SKIP_TEST(RmDirNestedENOTDIR); return; } GError *error = NULL; int ret = generate_file_if_not_exists(context, surl, "file:///etc/hosts", &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); ret = gfal2_rmdir(context, surl, &error); EXPECT_PRED_FORMAT3(AssertGfalErrno, ret, error, ENOTDIR); } int main(int argc, char** argv) { testing::InitGoogleTest(&argc, argv); if (argc < 2) { printf("Missing base url\n"); printf("\t%s [options] srm://host/base/path/\n", argv[0]); return 1; } RmDirTest::root = argv[1]; return RUN_ALL_TESTS(); } gfal2-v2.23.0/test/functional/gfal_test_rw_full.cpp000066400000000000000000000061651465240014500223170ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include #include #include #include #define BLKLEN 65536 class RwFullTest: public testing::Test { public: static const char* root; static int size; char surl[2048]; gfal2_context_t context; char *original_data; RwFullTest() { GError *error = NULL; context = gfal2_context_new(&error); Gfal::gerror_to_cpp(&error); original_data = new char[size]; int i; for (i = 0; i < size; ++i) original_data[i] = i; } virtual ~RwFullTest() { gfal2_context_free(context); delete [] original_data; } virtual void SetUp() { generate_random_uri(root, "test_rw_full", surl, sizeof(surl)); } virtual void TearDown() { GError* error = NULL; gfal2_unlink(context, surl, &error); g_clear_error(&error); } }; const char* RwFullTest::root; int RwFullTest::size = 0; TEST_F(RwFullTest, OpenENOENT) { GError* error = NULL; int ret = gfal2_open(context, surl, O_RDONLY, &error); EXPECT_PRED_FORMAT3(AssertGfalErrno, ret, error, ENOENT); } TEST_F(RwFullTest, WriteAndRead) { GError* error = NULL; int fd = gfal2_open(context, surl, O_WRONLY | O_CREAT, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, fd, error); int ret = gfal2_write(context, fd, original_data, size, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); ret = gfal2_close(context, fd, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); fd = gfal2_open(context, surl, O_RDONLY, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, fd, error); char buffer[size]; ret = gfal2_read(context, fd, buffer, size, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, fd, error); ret = gfal2_close(context, fd, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); EXPECT_EQ(0, memcmp(original_data, buffer, size)); } int main(int argc, char** argv) { testing::InitGoogleTest(&argc, argv); if (argc != 3) { printf("Missing base url and/or size\n"); printf("\t%s [options] srm://host/base/path/ 1024\n", argv[0]); return 1; } RwFullTest::root = argv[1]; RwFullTest::size = atol(argv[2]); return RUN_ALL_TESTS(); } gfal2-v2.23.0/test/functional/gfal_test_rw_seek.cpp000066400000000000000000000102771465240014500223030ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ /* random read * regression for DMC-531 * random write is not tested because most storage elements will fail if this is attempted * Sequential writes calling seek should work, however */ #include #include #include #include #include #include #include #include class RwSeekTest: public testing::Test { public: static const char* root; static long block_size, file_size; char surl[2048]; gfal2_context_t context; RwSeekTest() { GError *error = NULL; context = gfal2_context_new(&error); Gfal::gerror_to_cpp(&error); } virtual ~RwSeekTest() { gfal2_context_free(context); } virtual void SetUp() { generate_random_uri(root, "rwseek_test", surl, sizeof(surl)); } virtual void TearDown() { GError* error = NULL; gfal2_unlink(context, surl, &error); g_clear_error(&error); } }; const char* RwSeekTest::root; long RwSeekTest::block_size = 0; long RwSeekTest::file_size = 0; // Only seeking sequentially TEST_F(RwSeekTest, SeqWriteSeek) { char buffer[file_size]; GError* error = NULL; int fd = gfal2_open(context, surl, O_WRONLY | O_CREAT, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, fd, error); int i = 0, ret, nb; while (i < file_size) { long j = ((i + block_size) < file_size) ? block_size : file_size - i; ret = gfal2_lseek(context, fd, i, SEEK_SET, &error); ASSERT_NE(-1, ret); EXPECT_PRED_FORMAT2(AssertGfalSuccess, fd, error); nb = gfal2_write(context, fd, buffer + i, j, &error); ASSERT_NE(-1, nb); EXPECT_PRED_FORMAT2(AssertGfalSuccess, fd, error); i += j; } ret = gfal2_close(context, fd, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, fd, error); } TEST_F(RwSeekTest, ReadSeek) { // Create char buffer[file_size]; GError* error = NULL; int fd = gfal2_open(context, surl, O_WRONLY | O_CREAT, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, fd, error); int ret = gfal2_write(context, fd, buffer, sizeof(buffer), &error); ASSERT_NE(-1, ret); EXPECT_PRED_FORMAT2(AssertGfalSuccess, fd, error); ret = gfal2_close(context, fd, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, fd, error); // Read fd = gfal2_open(context, surl, O_RDONLY, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, fd, error); off_t offset = file_size / 2; ret = gfal2_lseek(context, fd, offset, SEEK_SET, &error); ASSERT_NE(-1, ret); EXPECT_PRED_FORMAT2(AssertGfalSuccess, fd, error); ret = gfal2_read(context, fd, buffer, sizeof(buffer), &error); ASSERT_NE(-1, ret); EXPECT_PRED_FORMAT2(AssertGfalSuccess, fd, error); ret = gfal2_close(context, fd, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, fd, error); } int main(int argc, char** argv) { testing::InitGoogleTest(&argc, argv); if (argc != 4) { printf("Missing base url and/or sizes\n"); printf("\t%s [options] srm://host/base/path/ \n", argv[0]); return 1; } RwSeekTest::root = argv[1]; RwSeekTest::block_size = atol(argv[2]); RwSeekTest::file_size = atol(argv[3]); if (RwSeekTest::file_size / 2 < RwSeekTest::block_size) { fprintf(stderr, "block size should be less than half the file size\n"); exit(1); } return RUN_ALL_TESTS(); } gfal2-v2.23.0/test/functional/gfal_test_rw_seq.cpp000066400000000000000000000067711465240014500221500ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include #include #include #include class RwSeqTest: public testing::Test { public: static const char* root; static long block_size, file_size; char surl[2048]; gfal2_context_t context; RwSeqTest() { GError *error = NULL; context = gfal2_context_new(&error); Gfal::gerror_to_cpp(&error); } virtual ~RwSeqTest() { gfal2_context_free(context); } virtual void SetUp() { generate_random_uri(root, "rwseq_test", surl, sizeof(surl)); } virtual void TearDown() { GError* error = NULL; gfal2_unlink(context, surl, &error); g_clear_error(&error); } }; const char* RwSeqTest::root; long RwSeqTest::block_size = 0; long RwSeqTest::file_size = 0; TEST_F(RwSeqTest, WriteReadSeq) { // Create char buffer[file_size]; long i; for (i = 0; i < file_size; ++i) buffer[i] = i; GError* error = NULL; int fd = gfal2_open(context, surl, O_WRONLY | O_CREAT, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, fd, error); int ret; long j = 0, n = 0; i = 0; while (i < file_size) { j = ((i + block_size) < file_size) ? block_size : file_size - i; ret = gfal2_write(context, fd, buffer + i, j, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); i += j; } ret = gfal2_close(context, fd, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, fd, error); // Read char read_buffer[file_size]; fd = gfal2_open(context, surl, O_RDONLY, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, fd, error); i = j = n = 0; while (i < file_size && i >= 0) { j = ((i + block_size) < file_size) ? block_size : file_size - i; ret = gfal2_read(context, fd, read_buffer + i, j, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, fd, error); i += (long) ret; } ret = gfal2_close(context, fd, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, fd, error); // Compare EXPECT_EQ(0, memcmp(buffer, read_buffer, file_size)); } int main(int argc, char** argv) { testing::InitGoogleTest(&argc, argv); if (argc != 4) { printf("Missing base url and/or sizes\n"); printf("\t%s [options] srm://host/base/path/ \n", argv[0]); return 1; } RwSeqTest::root = argv[1]; RwSeqTest::block_size = atol(argv[2]); RwSeqTest::file_size = atol(argv[3]); if (RwSeqTest::file_size / 2 < RwSeqTest::block_size) { fprintf(stderr, "block size should be less than half the file size\n"); exit(1); } return RUN_ALL_TESTS(); } gfal2-v2.23.0/test/functional/gfal_test_set_creds.cpp000066400000000000000000000051571465240014500226200ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ /** * Regression test for DMC-529 * Set the credentials via configuration, and make sure that it does work! */ #include #include #include #include class CredsTest: public testing::Test { public: static const char* root; static const char* proxy; gfal2_context_t context; CredsTest() { GError *error = NULL; context = gfal2_context_new(&error); Gfal::gerror_to_cpp(&error); } virtual ~CredsTest() { gfal2_context_free(context); } }; const char* CredsTest::root; const char* CredsTest::proxy; TEST_F(CredsTest, SetCreds) { gfal2_set_opt_string(context, "X509", "CERT", proxy, NULL); gfal2_set_opt_string(context, "X509", "KEY", proxy, NULL); GError *error = NULL; struct stat statbuf; int ret; ret = gfal2_stat(context, root, &statbuf, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); if (ret == 0) { std::cout << "stat successful" << std::endl; std::cout << "mode = " << std::oct << statbuf.st_mode << std::dec << std::endl; std::cout << "nlink = " << statbuf.st_nlink << std::endl; std::cout << "uid = " << statbuf.st_uid << std::endl; std::cout << "gid = " << statbuf.st_gid << std::endl; std::cout << "size = " << statbuf.st_size << std::endl; } } int main(int argc, char **argv) { if (getenv("X509_USER_CERT") || getenv("X509_USER_KEY") || getenv("X509_USER_PROXY")) { std::cerr << "Unset X509_USER_* environment variables before calling" << std::endl; return 1; } testing::InitGoogleTest(&argc, argv); if (argc < 3) { std::cerr << "Usage: " << argv[0] << " proxy surl" << std::endl; return 1; } CredsTest::proxy = argv[1]; CredsTest::root = argv[2]; return RUN_ALL_TESTS(); } gfal2-v2.23.0/test/functional/gfal_test_space.cpp000066400000000000000000000121261465240014500217320ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include class SpaceTest: public testing::Test { public: static const char* root; gfal2_context_t context; static enum ResponseType { kUnsupported, kSpaceTokenArray, kSpaceInfo, } responseType; static char responseBuffer[1024]; SpaceTest() { GError *error = NULL; context = gfal2_context_new(&error); Gfal::gerror_to_cpp(&error); } virtual ~SpaceTest() { gfal2_context_free(context); } }; const char* SpaceTest::root; SpaceTest::ResponseType SpaceTest::responseType; char SpaceTest::responseBuffer[1024]; TEST_F(SpaceTest, SpaceInAttrList) { GError *error = NULL; char attrs[1024]; int ret = gfal2_listxattr(context, root, attrs, sizeof(attrs), &error); ASSERT_PRED_FORMAT2(AssertGfalSuccess, ret, error); for (int i = 0; i < ret;) { GTEST_LOG_(INFO) << (attrs + i); size_t attrlen = strlen(attrs + i); if (strcmp(attrs + i, GFAL_XATTR_SPACETOKEN) == 0) { SUCCEED(); return; } i += attrlen + 1; } // Raise an error if we reach here FAIL(); } TEST_F(SpaceTest, GetSpaceAttr) { GError *error = NULL; int ret = gfal2_getxattr(context, root, GFAL_XATTR_SPACETOKEN, responseBuffer, sizeof(responseBuffer), &error); if (ret < 0 && (error->code == ENOSYS || error->code == ECOMM)) { GTEST_LOG_(INFO) << "Not supported by the storage: " << error->message; g_error_free(error); responseType = kUnsupported; SKIP_TEST(GetSpaceAttr); return; } ASSERT_PRED_FORMAT2(AssertGfalSuccess, ret, error); // Now, there are two possible responses: either an array with spacetoken names (SRM), // or just space info (xrootd) if (responseBuffer[0] == '[') { responseType = kSpaceTokenArray; } else if (responseBuffer[0] == '{') { responseType = kSpaceInfo; } } static void _validate_space_info(const char *response) { json_object *info = json_tokener_parse(response); ASSERT_NE(info, (void*)NULL); ASSERT_TRUE(json_object_is_type(info, json_type_object)); json_object *totalsizeObj, *unusedObj, *usedObj; ASSERT_TRUE(json_object_object_get_ex(info, "totalsize", &totalsizeObj)); ASSERT_TRUE(json_object_object_get_ex(info, "unusedsize", &unusedObj)); ASSERT_TRUE(json_object_object_get_ex(info, "usedsize", &usedObj)); errno = 0; int64_t total = json_object_get_int64(totalsizeObj); ASSERT_EQ(errno, 0); errno = 0; int64_t unused = json_object_get_int64(unusedObj); ASSERT_EQ(errno, 0); errno = 0; int64_t used = json_object_get_int64(usedObj); ASSERT_EQ(errno, 0); GTEST_LOG_(INFO) << used << "/" << total << "(" << unused << ")"; json_object_put(info); } TEST_F(SpaceTest, SpaceTokenList) { GError *error = NULL; char buffer[1024]; if (responseType != kSpaceTokenArray) { SKIP_TEST(SpaceTokenList); return; } json_object *obj = json_tokener_parse(responseBuffer); ASSERT_NE(obj, (void*)NULL); ASSERT_TRUE(json_object_is_type(obj, json_type_array)); // Iterate and ask for the info of each token int arrayLen = json_object_array_length(obj); for (int i = 0; i < arrayLen; ++i) { json_object *tokenObj = json_object_array_get_idx(obj, i); ASSERT_TRUE(json_object_is_type(tokenObj, json_type_string)); const char *token = json_object_get_string(tokenObj); char *attrName = g_strconcat(GFAL_XATTR_SPACETOKEN, ".token?", token, NULL); GTEST_LOG_(INFO) << attrName; int ret = gfal2_getxattr(context, root, attrName, buffer, sizeof(buffer), &error); g_free(attrName); ASSERT_PRED_FORMAT2(AssertGfalSuccess, ret, error); _validate_space_info(buffer); } json_object_put(obj); } TEST_F(SpaceTest, SpaceInfo) { if (responseType != kSpaceInfo) { SKIP_TEST(SpaceTokenList); return; } _validate_space_info(responseBuffer); } int main(int argc, char** argv) { testing::InitGoogleTest(&argc, argv); if (argc < 2) { printf("Missing base url\n"); printf("\t%s srm://host/base/path/\n", argv[0]); return 1; } SpaceTest::root = argv[1]; return RUN_ALL_TESTS(); } gfal2-v2.23.0/test/functional/gfal_test_stat.cpp000066400000000000000000000040001465240014500216020ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include class StatTest: public testing::Test { public: static const char* root; gfal2_context_t context; StatTest() { GError *error = NULL; context = gfal2_context_new(&error); Gfal::gerror_to_cpp(&error); } virtual ~StatTest() { gfal2_context_free(context); } }; const char* StatTest::root; TEST_F(StatTest, SimpleStat) { GError *error = NULL; struct stat statbuf; int ret; ret = gfal2_stat(context, root, &statbuf, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); std::cout << "stat successful" << std::endl; std::cout << "mode = " << std::oct << statbuf.st_mode << std::dec << std::endl; std::cout << "nlink = " << statbuf.st_nlink << std::endl; std::cout << "uid = " << statbuf.st_uid << std::endl; std::cout << "gid = " << statbuf.st_gid << std::endl; std::cout << "size = " << statbuf.st_size << std::endl; } int main(int argc, char **argv) { testing::InitGoogleTest(&argc, argv); if (argc < 1) { printf("Missing base url\n"); printf("\t%s [options] srm://host/base/path/\n", argv[0]); return 1; } StatTest::root = argv[1]; return RUN_ALL_TESTS(); } gfal2-v2.23.0/test/functional/gfal_test_token.cpp000066400000000000000000000037751465240014500217710ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2021 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include class TokenTest: public testing::Test { public: static const char* url; static const char* issuer; gfal2_context_t context; TokenTest() { GError* error = NULL; context = gfal2_context_new(&error); Gfal::gerror_to_cpp(&error); } virtual ~TokenTest() { gfal2_context_free(context); } }; const char* TokenTest::url = NULL; const char* TokenTest::issuer = NULL; TEST_F(TokenTest, TestRetrieveNoIssuer) { char buff[2048]; GError* err = NULL; ssize_t result = gfal2_token_retrieve(context, url, "", true, 60, NULL, buff, 2048, &err); EXPECT_GT(result, 0); EXPECT_EQ(NULL, err); } TEST_F(TokenTest, TestRetrieveIssuer) { char buff[2048]; GError* err = NULL; ssize_t result = gfal2_token_retrieve(context, url, issuer, true, 60, NULL, buff, 2048, &err); EXPECT_GT(result, 0); EXPECT_EQ(NULL, err); } int main(int argc, char** argv) { testing::InitGoogleTest(&argc, argv); if (argc < 3) { printf("Missing parameters: \n"); printf("\t%s [url] [issuer]\n", argv[0]); return -1; } TokenTest::url = argv[1]; TokenTest::issuer = argv[2]; return RUN_ALL_TESTS(); } gfal2-v2.23.0/test/functional/gfal_test_xattr.cpp000066400000000000000000000205301465240014500217770ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include #include class XAttrTest: public testing::Test { public: static const char* root; static const char* origin_root; char surl[2048]; char origin[2048]; gfal2_context_t context; XAttrTest() { GError *error = NULL; context = gfal2_context_new(&error); Gfal::gerror_to_cpp(&error); origin[0] = '\0'; } ~XAttrTest() { gfal2_context_free(context); } void SetUp() { GError *error = NULL; generate_random_uri(root, "xattr_test", surl, sizeof(surl)); int ret; if (strncmp(root, "lfc://", 6) != 0) { ret = generate_file_if_not_exists(context, surl, "file:///etc/hosts", &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); } else { generate_random_uri(origin_root, "xattr_test", origin, sizeof(surl)); ret = generate_file_if_not_exists(context, origin, "file:///etc/hosts", &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); ret = gfalt_copy_file(context, NULL, origin, surl, &error); } } void TearDown() { GError *error = NULL; gfal2_unlink(context, surl, &error); g_clear_error(&error); gfal2_unlink(context, origin, &error); g_clear_error(&error); } }; const char *XAttrTest::root; const char *XAttrTest::origin_root; TEST_F(XAttrTest, SrmType) { if (strncmp(root, "srm://", 6) != 0) { SKIP_TEST(SrmType); return; } GError *error = NULL; char buffer[1024]; ssize_t ret = gfal2_getxattr(context, surl, "srm.type", buffer, sizeof(buffer), &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, 0, error); EXPECT_GT(ret, 0); EXPECT_GT(strlen(buffer), 0); ret = gfal2_listxattr(context, surl, buffer, sizeof(buffer), &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, 0, error); EXPECT_GT(ret, 0); bool found = false; int i = 0; while (i < ret) { if (strncmp(buffer + i, "srm.type", 8) == 0) { found = true; break; } i += strlen(buffer + i) + 1; } EXPECT_TRUE(found); } TEST_F(XAttrTest, Status) { GError *error = NULL; char buffer[1024]; ssize_t ret = gfal2_getxattr(context, surl, GFAL_XATTR_STATUS, buffer, sizeof(buffer), &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, 0, error); EXPECT_GT(ret, 0); EXPECT_GT(strlen(buffer), 0); bool found = false; int i = 0; while (i < ret) { if (strncmp(buffer + i, GFAL_XATTR_STATUS_ONLINE, sizeof(GFAL_XATTR_STATUS_ONLINE)) == 0) { found = true; break; } i += strlen(buffer + i) + 1; } EXPECT_TRUE(found); } TEST_F(XAttrTest, TapeAPIVersion) { if (strncmp("http:", root, 5) != 0 && strncmp("https:", root, 6) != 0 && strncmp("dav:", root, 4) != 0 && strncmp("davs:", root, 5) != 0) { SKIP_TEST(TapeAPIVersion); return; } GError *error = NULL; char buffer[1024]; ssize_t ret = gfal2_getxattr(context, surl, GFAL_XATTR_TAPE_API_VERSION, buffer, sizeof(buffer), &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, 0, error); EXPECT_GT(ret, 0); EXPECT_GT(strlen(buffer), 0); } TEST_F(XAttrTest, SrmReplicas) { if (strncmp(root, "srm://", 6) != 0) { SKIP_TEST(SrmReplicas); return; } GError *error = NULL; char buffer[1024]; ssize_t ret = gfal2_getxattr(context, surl, GFAL_XATTR_REPLICA, buffer, sizeof(buffer), &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, 0, error); EXPECT_GT(ret, 0); EXPECT_GT(strlen(buffer), 0); ret = gfal2_listxattr(context, surl, buffer, sizeof(buffer), &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, 0, error); EXPECT_GT(ret, 0); bool found = false; int i = 0; while (i < ret) { if (strncmp(buffer + i, GFAL_XATTR_REPLICA, sizeof(GFAL_XATTR_REPLICA)) == 0) { found = true; break; } i += strlen(buffer + i) + 1; } EXPECT_TRUE(found); } TEST_F(XAttrTest, LfcComment) { if (strncmp(root, "lfc://", 6) != 0) { SKIP_TEST(LfcComment); return; } GError *error = NULL; char comment[] = "this is a comment"; char buffer[1024]; int ret = gfal2_setxattr(context, surl, GFAL_XATTR_COMMENT, comment, sizeof(comment), 0, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); ret = gfal2_listxattr(context, surl, buffer, sizeof(buffer), &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, 0, error); EXPECT_GT(ret, 0); bool found = false; int i = 0; while (i < ret) { if (strncmp(buffer + i, GFAL_XATTR_COMMENT, sizeof(GFAL_XATTR_COMMENT)) == 0) { found = true; break; } i += strlen(buffer + i) + 1; } EXPECT_TRUE(found); ret = gfal2_getxattr(context, surl, GFAL_XATTR_COMMENT, buffer, sizeof(buffer), &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, 0, error); EXPECT_GT(ret, 0); EXPECT_STREQ(comment, buffer); } TEST_F(XAttrTest, LfcReplica) { if (strncmp(root, "lfc://", 6) != 0) { SKIP_TEST(LfcReplica); return; } GError *error = NULL; char buffer[1024]; int ret = gfal2_listxattr(context, surl, buffer, sizeof(buffer), &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, 0, error); EXPECT_GT(ret, 0); bool found = false; int i = 0; while (i < ret) { if (strncmp(buffer + i, GFAL_XATTR_REPLICA, sizeof(GFAL_XATTR_REPLICA)) == 0) { found = true; break; } i += strlen(buffer + i) + 1; } EXPECT_TRUE(found); ret = gfal2_getxattr(context, surl, GFAL_XATTR_REPLICA, buffer, sizeof(buffer), &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, 0, error); EXPECT_GT(ret, 0); EXPECT_STREQ(origin, buffer); } TEST_F(XAttrTest, LfcSetReplica) { if (strncmp(root, "lfc://", 6) != 0) { SKIP_TEST(LfcSetReplica); return; } struct stat buf; stat("/etc/hosts", &buf); GError *error = NULL; char buffer[1024]; char replica[1024]; snprintf(replica, sizeof(replica), "mock://fake/file?size=%lld", (long long)buf.st_size); // Add snprintf(buffer, sizeof(buffer), "+%s", replica); int ret = gfal2_setxattr(context, surl, GFAL_XATTR_REPLICA, buffer, sizeof(buffer), 0, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, 0, error); ret = gfal2_getxattr(context, surl, GFAL_XATTR_REPLICA, buffer, sizeof(buffer), &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, 0, error); EXPECT_EQ(ret, strlen(origin) + strlen(replica) + 2); // Remove snprintf(buffer, sizeof(buffer), "-%s", replica); ret = gfal2_setxattr(context, surl, GFAL_XATTR_REPLICA, buffer, sizeof(buffer), 0, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, 0, error); ret = gfal2_getxattr(context, surl, GFAL_XATTR_REPLICA, buffer, sizeof(buffer), &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, 0, error); EXPECT_EQ(ret, strlen(origin) + 1); } int main(int argc, char** argv) { testing::InitGoogleTest(&argc, argv); if (argc < 2) { printf("Missing base url\n"); printf("\t%s [options] srm://host/base/path/\n", argv[0]); return 1; } XAttrTest::root = argv[1]; if (strncmp(XAttrTest::root, "lfc://", 6) == 0) { if (argc < 3) { printf("Need a site URL for creating the replica!\n"); return 1; } XAttrTest::origin_root = argv[2]; } return RUN_ALL_TESTS(); } gfal2-v2.23.0/test/functional/gfalt_test_copy_bulk.cpp000066400000000000000000000250001465240014500230050ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include #include #include #include void transfer_callback(const gfalt_event_t e, gpointer user_data); #define NBPAIRS 3 class CopyBulk: public testing::Test { public: static const char* source_root; static const char* destination_root; char *sources[NBPAIRS]; char *destinations[NBPAIRS]; char *nested[NBPAIRS]; size_t done; gfal2_context_t handle; gfalt_params_t params; char original_checksum[32]; CopyBulk(): done(0) { GError *error = NULL; handle = gfal2_context_new(&error); Gfal::gerror_to_cpp(&error); params = gfalt_params_handle_new(NULL); gfalt_add_event_callback(params, transfer_callback, this, NULL, NULL); for (size_t i = 0; i < NBPAIRS; ++i) { sources[i] = new char[2048]; destinations[i] = new char[2048]; nested[i] = new char[2048]; } } virtual ~CopyBulk() { gfal2_context_free(handle); gfalt_params_handle_delete(params, NULL); for (size_t i = 0; i < NBPAIRS; ++i) { delete [] sources[i]; delete [] destinations[i]; delete [] nested[i]; } } virtual void SetUp() { char source_base[2048]; char dest_base[2048]; int ret; GError* error = NULL; generate_random_uri(source_root, "copyfile_bulk_source", source_base, 2048); generate_random_uri(destination_root, "copyfile_bulk_destination", dest_base, 2048); for (size_t i = 0; i < NBPAIRS; ++i) { snprintf(sources[i], 2048, "%s_%zu", source_base, i); snprintf(destinations[i], 2048, "%s_%zu", dest_base, i); ret = generate_file_if_not_exists(handle, sources[i], "file:///etc/hosts", &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); nested[i][0] = '\0'; } ret = gfal2_checksum(handle, "file:///etc/hosts", "ADLER32", 0, 0, original_checksum, sizeof original_checksum, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); done = 0; gfalt_set_checksum(params, GFALT_CHECKSUM_NONE, NULL, NULL, &error); gfalt_set_replace_existing_file(params, FALSE, NULL); gfalt_set_create_parent_dir(params, FALSE, NULL); } virtual void TearDown() { GError *error = NULL; for (size_t i = 0; i < NBPAIRS; ++i) { gfal2_unlink(handle, sources[i], &error); g_clear_error(&error); gfal2_unlink(handle, destinations[i], &error); g_clear_error(&error); gfal2_unlink(handle, nested[i], &error); g_clear_error(&error); gfal2_rmdir(handle, destinations[i], &error); g_clear_error(&error); } } }; const char* CopyBulk::source_root; const char* CopyBulk::destination_root; void transfer_callback(const gfalt_event_t e, gpointer user_data) { CopyBulk* copy = static_cast(user_data); const char* stage = g_quark_to_string(e->stage); const char* side; switch (e->side) { case GFAL_EVENT_SOURCE: side = "SOURCE"; break; case GFAL_EVENT_DESTINATION: side = "DESTINATION"; break; default: side = "BOTH"; } printf("%-15s %-15s %s\n", side, stage, e->description); if (e->stage == GFAL_EVENT_TRANSFER_EXIT) { copy->done++; } } TEST_F(CopyBulk, CopyBulk) { GError* op_error = NULL; GError** file_errors = NULL; int ret = 0; ret = gfalt_copy_bulk(handle, params, NBPAIRS, sources, destinations, NULL, &op_error, &file_errors); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, op_error); if (file_errors) { for (size_t i = 0; i < NBPAIRS; ++i) { if (file_errors[i]) { EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, file_errors[i]); g_error_free(file_errors[i]); } } g_free(file_errors); } if (op_error) g_error_free(op_error); ASSERT_EQ(NBPAIRS, done); // Do not trust! Make sure they do exist if (ret == 0) { struct stat st; GError* tmp_err = NULL; for (size_t i = 0; i < NBPAIRS; ++i) { ret = gfal2_stat(handle, destinations[i], &st, &tmp_err); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, tmp_err); if (tmp_err) g_error_free(tmp_err); tmp_err = NULL; } } } TEST_F(CopyBulk, CopyBulkSomeFail) { // Remove source for even transfers, they should fail int removed = 0; for (size_t i = 0; i < NBPAIRS; ++i) { if (i % 2 == 0) { GError* tmp_err = NULL; gfal2_unlink(handle, sources[i], &tmp_err); if (tmp_err) g_error_free(tmp_err); ++removed; } } GError* op_error = NULL; GError** file_errors = NULL; int ret = 0; ret = gfalt_copy_bulk(handle, params, NBPAIRS, sources, destinations, NULL, &op_error, &file_errors); ASSERT_LT(ret, 0); EXPECT_PRED_FORMAT2(AssertGfalSuccess, 0, op_error); ASSERT_NE((void*)NULL, file_errors); if (file_errors) { for (size_t i = 0; i < NBPAIRS; ++i) { if (i % 2 == 0) { EXPECT_PRED_FORMAT3(AssertGfalErrno, ret, file_errors[i], ENOENT); } else { EXPECT_PRED_FORMAT2(AssertGfalSuccess, 0, file_errors[i]); } if (file_errors[i]) g_error_free(file_errors[i]); } g_free(file_errors); } if (op_error) g_error_free(op_error); } TEST_F(CopyBulk, CopyBulkChecksuming) { // Xrootd is problematic, as several endpoints may not implement checksums, so skip this if (strncmp("root://", sources[0], 7) == 0) { SKIP_TEST(CopyBulkChecksuming); return; } const char *checksums[NBPAIRS] = {0}; // All checksums fine, except first for (size_t i = 0; i < NBPAIRS; ++i) checksums[i] = original_checksum; checksums[0] = "0D3902E7"; GError* op_error = NULL; GError** file_errors = NULL; int ret = 0; gfalt_set_checksum(params, GFALT_CHECKSUM_BOTH, "ADLER32", NULL, NULL); ret = gfalt_copy_bulk(handle, params, NBPAIRS, sources, destinations, checksums, &op_error, &file_errors); ASSERT_LT(ret, 0); EXPECT_PRED_FORMAT2(AssertGfalSuccess, 0, op_error); ASSERT_NE((void*)NULL, file_errors); if (file_errors) { EXPECT_PRED_FORMAT3(AssertGfalErrno, ret, file_errors[0], EIO); for (size_t i = 1; i < NBPAIRS; ++i) { EXPECT_PRED_FORMAT2(AssertGfalSuccess, 0, file_errors[i]); } g_free(file_errors); } if (op_error) g_error_free(op_error); } TEST_F(CopyBulk, CopyDestinationExists) { GError* error = NULL; int ret; for (size_t i = 0; i < NBPAIRS; ++i) { ret = generate_file_if_not_exists(handle, destinations[i], "file:///etc/hosts", &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); } GError* op_error = NULL; GError** file_errors = NULL; ret = gfalt_copy_bulk(handle, params, NBPAIRS, sources, destinations, NULL, &op_error, &file_errors); ASSERT_LT(ret, 0); EXPECT_PRED_FORMAT2(AssertGfalSuccess, 0, op_error); if (file_errors) { for (size_t i = 0; i < NBPAIRS; ++i) { EXPECT_PRED_FORMAT3(AssertGfalErrno, ret, file_errors[i], EEXIST); } g_free(file_errors); } if (op_error) g_error_free(op_error); } TEST_F(CopyBulk, CopyOverwrite) { GError* error = NULL; int ret; for (size_t i = 0; i < NBPAIRS; ++i) { ret = generate_file_if_not_exists(handle, destinations[i], "file:///etc/hosts", &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); } GError* op_error = NULL; GError** file_errors = NULL; gfalt_set_replace_existing_file(params, TRUE, NULL); ret = gfalt_copy_bulk(handle, params, NBPAIRS, sources, destinations, NULL, &op_error, &file_errors); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, op_error); if (file_errors) { for (size_t i = 0; i < NBPAIRS; ++i) { EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, file_errors[i]); } g_free(file_errors); } if (op_error) g_error_free(op_error); } TEST_F(CopyBulk, MkParentDir) { int ret; for (size_t i = 0; i < NBPAIRS; ++i) { char buffer[2048]; generate_random_uri(destinations[i], "mkparent", buffer, sizeof(buffer)); strncpy(nested[i], buffer, 2048); } GError* op_error = NULL; GError** file_errors = NULL; gfalt_set_replace_existing_file(params, TRUE, NULL); gfalt_set_create_parent_dir(params, TRUE, NULL); ret = gfalt_copy_bulk(handle, params, NBPAIRS, sources, nested, NULL, &op_error, &file_errors); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, op_error); if (file_errors) { for (size_t i = 0; i < NBPAIRS; ++i) { EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, file_errors[i]); } g_free(file_errors); } if (op_error) g_error_free(op_error); } int main(int argc, char** argv) { testing::InitGoogleTest(&argc, argv); if (argc < 3) { printf("Missing source and destination base urls\n"); printf("\t%s [options] gsiftp://host/base/path/ gsiftp://destination/base/path/\n", argv[0]); return 1; } CopyBulk::source_root = argv[1]; CopyBulk::destination_root = argv[2]; for (int i = 0; i < argc; ++i) { if (strcmp(argv[i], "-v") == 0) gfal2_log_set_level(G_LOG_LEVEL_DEBUG); } return RUN_ALL_TESTS(); } gfal2-v2.23.0/test/functional/gfalt_test_copy_file.cpp000066400000000000000000000100231465240014500227660ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include static void event_callback(const gfalt_event_t e, gpointer user_data); class CopyTest: public testing::Test { public: static const char* source_root; static const char* destination_root; char source[2048]; char destination[2048]; gfal2_context_t handle; gfalt_params_t params; bool isThirdPartyCopy, mustBeThirdPartyCopy; CopyTest() { GError *error = NULL; handle = gfal2_context_new(&error); Gfal::gerror_to_cpp(&error); params = gfalt_params_handle_new(&error); Gfal::gerror_to_cpp(&error); gfalt_add_event_callback(params, event_callback, this, NULL, &error); Gfal::gerror_to_cpp(&error); mustBeThirdPartyCopy = expect_third_party_copy(source_root, destination_root); } virtual ~CopyTest() { gfal2_context_free(handle); gfalt_params_handle_delete(params, NULL); } virtual void SetUp() { isThirdPartyCopy = false; generate_random_uri(source_root, "copyfile_source", source, 2048); generate_random_uri(destination_root, "copyfile", destination, 2048); RecordProperty("Source", source); RecordProperty("Destination", source); GError* error = NULL; int ret = generate_file_if_not_exists(handle, source, "file:///etc/hosts", &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); } virtual void TearDown() { GError *error = NULL; gfal2_unlink(handle, source, &error); g_clear_error(&error); gfal2_unlink(handle, destination, &error); g_clear_error(&error); } void VerifyThirdPartyCopy() { if (mustBeThirdPartyCopy) { EXPECT_TRUE(isThirdPartyCopy); } else { EXPECT_FALSE(isThirdPartyCopy); } } }; static void event_callback(const gfalt_event_t e, gpointer user_data) { CopyTest *copyTest = static_cast(user_data); if (e->stage == GFAL_EVENT_TRANSFER_TYPE) { copyTest->isThirdPartyCopy = (strncmp(e->description, "3rd", 3) == 0); gfal2_log(G_LOG_LEVEL_INFO, "Third party copy"); } } const char* CopyTest::source_root; const char* CopyTest::destination_root; TEST_F(CopyTest, SimpleFileCopy) { GError* error = NULL; int ret = gfalt_copy_file(handle, params, source, destination, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); VerifyThirdPartyCopy(); } TEST_F(CopyTest, SimpleFileCopyENOENT) { GError* error = NULL; int ret = gfal2_unlink(handle, source, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); ret = gfalt_copy_file(handle, params, source, destination, &error); EXPECT_PRED_FORMAT3(AssertGfalErrno, ret, error, ENOENT); } int main(int argc, char** argv) { testing::InitGoogleTest(&argc, argv); if (argc < 2) { printf("Missing source and destination base urls\n"); printf("\t%s [options] srm://host/base/path/ srm://destination/base/path/\n", argv[0]); return 1; } CopyTest::source_root = argv[1]; CopyTest::destination_root = argv[2]; // gfal2_log_set_level(G_LOG_LEVEL_DEBUG); return RUN_ALL_TESTS(); } gfal2-v2.23.0/test/functional/gfalt_test_copy_file_cancel.cpp000066400000000000000000000122771465240014500243100ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include #include typedef struct copier_params_t { gfal2_context_t handle; gfalt_params_t transfer_params; const char *source; const char *destination; } copier_params_t; GQuark test_domain() { return g_quark_from_static_string("gfalt_copyfile_cancel"); } void* copier_thread_method(void* data) { copier_params_t *params = (copier_params_t*)data; GError *error = NULL; struct stat stat_buffer; // Source must exist if (gfal2_stat(params->handle, params->source, &stat_buffer, &error) !=0) { g_prefix_error(&error, "Source file does not exist "); goto out; } // Do the copy if (gfalt_copy_file(params->handle, params->transfer_params, params->source, params->destination, &error) == 0) { if (gfal2_stat(params->handle, params->destination, &stat_buffer, &error) != 0) g_prefix_error(&error, "Destination file does not exist after the copy "); } // Exit sequence out: free(params); return error; } int spawn_copy(pthread_t* thread, gfal2_context_t handle, gfalt_params_t transfer_params, const char* source, const char* destination) { copier_params_t *copier_params = (copier_params_t*)malloc(sizeof(*copier_params)); copier_params->handle = handle; copier_params->transfer_params = transfer_params; copier_params->source = source; copier_params->destination = destination; return pthread_create(thread, NULL, copier_thread_method, copier_params); } void monitor_callback(gfalt_transfer_status_t h, const char* source, const char* destination, gpointer udata) { // We don't care about the performance data. We just want to trigger // all those internal locks. (void)h; (void)source; (void)destination; (void)udata; } void event_callback(const gfalt_event_t e, gpointer user_data) { fprintf(stderr, "[%s] %s %s\n", g_quark_to_string(e->domain), g_quark_to_string(e->stage), e->description); } int main(int argc, char** argv) { const char *source; const char *destination; gfal2_context_t handle = NULL; gfalt_params_t transfer_params = NULL; GError *error = NULL, *copier_error = NULL; pthread_t copier_thread; int i; // Parse params if (argc != 3) { fprintf(stderr, "Usage: %s SOURCE DESTINATION", argv[0]); return 1; } source = argv[1]; destination = argv[2]; // Create handle and transfer parameters handle = gfal2_context_new(&error); if (!handle) goto out; transfer_params = gfalt_params_handle_new(&error); if (!transfer_params) goto unwind_context; gfalt_set_replace_existing_file(transfer_params, TRUE, NULL); gfalt_add_monitor_callback(transfer_params, monitor_callback, NULL, NULL, NULL); gfalt_add_event_callback(transfer_params, event_callback, NULL, NULL, NULL); // Spawn the thread that will do the copy printf("Spawning the copy '%s' => '%s'\n", source, destination); if (spawn_copy(&copier_thread, handle, transfer_params, source, destination) != 0) { error = g_error_new(test_domain(), errno, "Could not spawn the copier thread."); goto unwind_params; } // Give it some time printf("Waiting some time before canceling\n"); for (i = 0; i < 30; ++i) { fputc('.', stdout); fflush(stdout); sleep(1); } fputc('\n', stdout); // Cancel printf("Calling gfal2_cancel.\n"); if (gfal2_cancel(handle) <= 0) { error = g_error_new(test_domain(), EINVAL, "No jobs canceled. Expected at least one."); goto unwind_thread; } printf("Transfer canceled normally\n"); // Unwinding unwind_thread: pthread_join(copier_thread, (void**)&copier_error); unwind_params: gfalt_params_handle_delete(transfer_params, NULL); unwind_context: gfal2_context_free(handle); out: if (error) fprintf(stderr, "FAILURE: %s (%d)\n", error->message, error->code); if (copier_error) fprintf(stderr, "COPIER FAILURE: %s (%d)\n", copier_error->message, copier_error->code); return error || (copier_error && copier_error->code != ECANCELED); } gfal2-v2.23.0/test/functional/gfalt_test_copy_file_checksum.cpp000066400000000000000000000132171465240014500246600ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include static void event_callback(const gfalt_event_t e, gpointer user_data); class CopyTestChecksum: public testing::Test { public: static const char* source_root; static const char* destination_root; char source[2048]; char destination[2048]; gfal2_context_t handle; gfalt_params_t params; bool isThirdPartyCopy, mustBeThirdPartyCopy; CopyTestChecksum() { GError *error = NULL; handle = gfal2_context_new(&error); Gfal::gerror_to_cpp(&error); params = gfalt_params_handle_new(NULL); gfalt_add_event_callback(params, &event_callback, this, NULL, NULL); mustBeThirdPartyCopy = expect_third_party_copy(source_root, destination_root); } virtual ~CopyTestChecksum() { gfal2_context_free(handle); gfalt_params_handle_delete(params, NULL); } virtual void SetUp() { isThirdPartyCopy = false; generate_random_uri(source_root, "copyfile_checksum_source", source, 2048); generate_random_uri(destination_root, "copyfile_checksum", destination, 2048); RecordProperty("Source", source); RecordProperty("Destination", source); GError* error = NULL; int ret = generate_file_if_not_exists(handle, source, "file:///etc/hosts", &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); } virtual void TearDown() { GError *error = NULL; gfal2_unlink(handle, source, &error); g_clear_error(&error); gfal2_unlink(handle, destination, &error); g_clear_error(&error); } void VerifyThirdPartyCopy() { if (mustBeThirdPartyCopy) { EXPECT_TRUE(isThirdPartyCopy); } else { EXPECT_FALSE(isThirdPartyCopy); } } }; void event_callback(const gfalt_event_t e, gpointer user_data) { CopyTestChecksum *copyTest = static_cast(user_data); if (e->stage == GFAL_EVENT_TRANSFER_TYPE) { copyTest->isThirdPartyCopy = (strncmp(e->description, "3rd", 3) == 0); gfal2_log(G_LOG_LEVEL_INFO, "Third party copy"); } } const char* CopyTestChecksum::source_root; const char* CopyTestChecksum::destination_root; TEST_F(CopyTestChecksum, CopyChecksumEnabled) { GError* error = NULL; int ret = 0; ret = gfalt_set_checksum(params, GFALT_CHECKSUM_BOTH, "ADLER32", NULL, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); ret = gfalt_copy_file(handle, params, source, destination, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); VerifyThirdPartyCopy(); } TEST_F(CopyTestChecksum, CopyChecksumAndReplaceEnabled) { GError* error = NULL; int ret = 0; ret = gfalt_set_checksum(params, GFALT_CHECKSUM_BOTH, "ADLER32", NULL, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); ret = gfalt_set_replace_existing_file(params, TRUE, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); ret = gfalt_copy_file(handle, params, source, destination, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); VerifyThirdPartyCopy(); } TEST_F(CopyTestChecksum, CopyChecksumAndReplaceEnabledENOENT) { GError* error = NULL; int ret = 0; ret = gfalt_set_checksum(params, GFALT_CHECKSUM_BOTH, "ADLER32", NULL, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); ret = gfalt_set_replace_existing_file(params, TRUE, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); gfal2_unlink(handle, source, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); ret = gfalt_copy_file(handle, params, source, destination, &error); EXPECT_PRED_FORMAT3(AssertGfalErrno, ret, error, ENOENT); } // Ask to compare only source checksum and do not give a checksum // Must fail TEST_F(CopyTestChecksum, CopyChecksumOnlySourceNoValue) { GError* error = NULL; int ret = 0; ret = gfalt_set_checksum(params, GFALT_CHECKSUM_SOURCE, "ADLER32", NULL, &error); EXPECT_PRED_FORMAT3(AssertGfalErrno, ret, error, EINVAL); } // Ask to compare only destination checksum and do not give a checksum // Must fail TEST_F(CopyTestChecksum, CopyChecksumOnlyDestinationNoValue) { GError* error = NULL; int ret = 0; ret = gfalt_set_checksum(params, GFALT_CHECKSUM_TARGET, "ADLER32", NULL, &error); EXPECT_PRED_FORMAT3(AssertGfalErrno, ret, error, EINVAL); } int main(int argc, char** argv) { testing::InitGoogleTest(&argc, argv); if (argc < 2) { printf("Missing source and destination base urls\n"); printf("\t%s [options] srm://host/base/path/ srm://destination/base/path/\n", argv[0]); return 1; } CopyTestChecksum::source_root = argv[1]; CopyTestChecksum::destination_root = argv[2]; // gfal2_log_set_level(G_LOG_LEVEL_DEBUG); return RUN_ALL_TESTS(); } gfal2-v2.23.0/test/functional/gfalt_test_copy_file_checksum_user.cpp000066400000000000000000000202401465240014500257100ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include static void event_callback(const gfalt_event_t e, gpointer user_data); class CopyTestUserChecksum: public testing::Test { public: static const char* source_root; static const char* destination_root; char source[2048]; char destination[2048]; gfal2_context_t handle; gfalt_params_t params; bool transfer_happened; bool isThirdPartyCopy, mustBeThirdPartyCopy; CopyTestUserChecksum(): transfer_happened(false) { GError *error = NULL; handle = gfal2_context_new(&error); Gfal::gerror_to_cpp(&error); params = gfalt_params_handle_new(NULL); gfalt_add_event_callback(params, &event_callback, this, NULL, NULL); mustBeThirdPartyCopy = expect_third_party_copy(source_root, destination_root); } virtual ~CopyTestUserChecksum() { gfal2_context_free(handle); gfalt_params_handle_delete(params, NULL); } virtual void SetUp() { isThirdPartyCopy = false; transfer_happened = false; gfalt_set_checksum(params, GFALT_CHECKSUM_BOTH, NULL, NULL, NULL); generate_random_uri(source_root, "copyfile_checksum_user_source", source, 2048); generate_random_uri(destination_root, "copyfile_checksum_user", destination, 2048); RecordProperty("Source", source); RecordProperty("Destination", source); GError* error = NULL; int ret = generate_file_if_not_exists(handle, source, "file:///etc/hosts", &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); } virtual void TearDown() { GError *error = NULL; gfal2_unlink(handle, source, &error); g_clear_error(&error); gfal2_unlink(handle, destination, &error); g_clear_error(&error); } void VerifyThirdPartyCopy() { if (mustBeThirdPartyCopy) { EXPECT_TRUE(isThirdPartyCopy); } else { EXPECT_FALSE(isThirdPartyCopy); } } }; void event_callback(const gfalt_event_t e, gpointer user_data) { CopyTestUserChecksum *copyTest = static_cast(user_data); if (e->stage == GFAL_EVENT_TRANSFER_ENTER) { copyTest->transfer_happened = true; } if (e->stage == GFAL_EVENT_TRANSFER_TYPE) { copyTest->isThirdPartyCopy = (strncmp(e->description, "3rd", 3) == 0); gfal2_log(G_LOG_LEVEL_INFO, "Third party copy"); } } const char* CopyTestUserChecksum::source_root; const char* CopyTestUserChecksum::destination_root; // Enabling checksum with the correct checksum, the copy must succeed TEST_F(CopyTestUserChecksum, CopyRightUserChecksum) { GError* error = NULL; int ret = 0; char checksum_source[2048]; ret = gfal2_checksum(handle, source, "ADLER32", 0, 0, checksum_source, sizeof(checksum_source), &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); gfalt_set_checksum(params, GFALT_CHECKSUM_BOTH, "ADLER32", checksum_source, NULL); ret = gfalt_copy_file(handle, params, source, destination, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); VerifyThirdPartyCopy(); } // Enabling checksum with a wrong checksum, the copy must fail // There must be no copy, since the source would not pass the validation TEST_F(CopyTestUserChecksum, CopyBadChecksum) { GError* error = NULL; int ret = 0; gfalt_set_checksum(params, GFALT_CHECKSUM_BOTH, "ADLER32", "aaaaaaa", NULL); ret = gfalt_copy_file(handle, params, source, destination, &error); EXPECT_PRED_FORMAT3(AssertGfalErrno, ret, error, EIO); EXPECT_FALSE(transfer_happened); } // Enabling checksum for the source only with the correct checksum, the copy must succeed TEST_F(CopyTestUserChecksum, CopyRightUserChecksumSource) { GError* error = NULL; int ret = 0; char checksum_source[2048]; ret = gfal2_checksum(handle, source, "ADLER32", 0, 0, checksum_source, sizeof(checksum_source), &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); gfalt_set_checksum(params, GFALT_CHECKSUM_SOURCE, "ADLER32", checksum_source, NULL); ret = gfalt_copy_file(handle, params, source, destination, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); VerifyThirdPartyCopy(); } // Enabling checksum for the destination only with the correct checksum, the copy must succeed TEST_F(CopyTestUserChecksum, CopyRightUserChecksumDestination) { GError* error = NULL; int ret = 0; char checksum_source[2048]; ret = gfal2_checksum(handle, source, "ADLER32", 0, 0, checksum_source, sizeof(checksum_source), &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); gfalt_set_checksum(params, GFALT_CHECKSUM_TARGET, "ADLER32", checksum_source, NULL); ret = gfalt_copy_file(handle, params, source, destination, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); VerifyThirdPartyCopy(); } // Enabling checksum for the source only with a wrong checksum, the copy must fail TEST_F(CopyTestUserChecksum, CopyRightUserChecksumSourceBad) { GError* error = NULL; int ret = 0; gfalt_set_checksum(params, GFALT_CHECKSUM_SOURCE, "ADLER32", "aaaaaaa", NULL); ret = gfalt_copy_file(handle, params, source, destination, &error); EXPECT_PRED_FORMAT3(AssertGfalErrno, ret, error, EIO); EXPECT_FALSE(transfer_happened); } // Enabling checksum for the destination only with a wrong checksum, the copy must fail // Note that the transfer must happen even if the source file does not match! TEST_F(CopyTestUserChecksum, CopyRightUserChecksumDestinationBad) { GError* error = NULL; int ret = 0; gfalt_set_checksum(params, GFALT_CHECKSUM_TARGET, "ADLER32", "aaaaaaa", NULL); ret = gfalt_copy_file(handle, params, source, destination, &error); EXPECT_PRED_FORMAT3(AssertGfalErrno, ret, error, EIO); EXPECT_TRUE(transfer_happened); VerifyThirdPartyCopy(); } // Disabling checksum with a wrong checksum, the copy must succeed TEST_F(CopyTestUserChecksum, CopyRightUserChecksumDisabledBad) { GError* error = NULL; int ret = 0; gfalt_set_checksum(params, GFALT_CHECKSUM_NONE, "ADLER32", "aaaaaaa", NULL); ret = gfalt_copy_file(handle, params, source, destination, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); VerifyThirdPartyCopy(); } // Set source or destination, with empty user defined TEST_F(CopyTestUserChecksum, CopyBadChecksumValue) { GError* error = NULL; int ret = gfalt_set_checksum(params, GFALT_CHECKSUM_SOURCE, "ADLER32", "", &error); EXPECT_PRED_FORMAT3(AssertGfalErrno, ret, error, EINVAL); g_clear_error(&error); ret = gfalt_set_checksum(params, GFALT_CHECKSUM_TARGET, "ADLER32", "", &error); EXPECT_PRED_FORMAT3(AssertGfalErrno, ret, error, EINVAL); g_clear_error(&error); ret = gfalt_set_checksum(params, GFALT_CHECKSUM_BOTH, "ADLER32", "", &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); g_clear_error(&error); } int main(int argc, char** argv) { testing::InitGoogleTest(&argc, argv); if (argc < 3) { printf("Missing source and destination base urls\n"); printf("\t%s [options] srm://host/base/path/ srm://destination/base/path/\n", argv[0]); return 1; } CopyTestUserChecksum::source_root = argv[1]; CopyTestUserChecksum::destination_root = argv[2]; // gfal2_log_set_level(G_LOG_LEVEL_DEBUG); return RUN_ALL_TESTS(); } gfal2-v2.23.0/test/functional/gfalt_test_copy_file_mkdir.cpp000066400000000000000000000114661465240014500241700ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include #include static void event_callback(const gfalt_event_t e, gpointer user_data); class CopyTestMkdir: public testing::Test { public: static const char* source_root; static const char* destination_root; char source[2048]; char destination[2048]; gfal2_context_t handle; gfalt_params_t params; bool isThirdPartyCopy, mustBeThirdPartyCopy; std::list directories; CopyTestMkdir() { GError *error = NULL; handle = gfal2_context_new(&error); Gfal::gerror_to_cpp(&error); params = gfalt_params_handle_new(NULL); gfalt_set_create_parent_dir(params, TRUE, NULL); gfalt_add_event_callback(params, &event_callback, this, NULL, NULL); mustBeThirdPartyCopy = expect_third_party_copy(source_root, destination_root); } virtual ~CopyTestMkdir() { gfal2_context_free(handle); gfalt_params_handle_delete(params, NULL); } virtual void SetUp() { isThirdPartyCopy = false; generate_random_uri(source_root, "copyfile_replace_source", source, 2048); RecordProperty("Source", source); RecordProperty("Destination", source); GError* error = NULL; int ret = generate_file_if_not_exists(handle, source, "file:///etc/hosts", &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); } virtual void TearDown() { GError *error = NULL; gfal2_unlink(handle, source, &error); g_clear_error(&error); gfal2_unlink(handle, destination, &error); g_clear_error(&error); std::list::const_iterator i; for (i = directories.begin(); i != directories.end(); ++i) { gfal2_rmdir(handle, i->c_str(), &error); g_clear_error(&error); } directories.clear(); } void VerifyThirdPartyCopy() { if (mustBeThirdPartyCopy) { EXPECT_TRUE(isThirdPartyCopy); } else { EXPECT_FALSE(isThirdPartyCopy); } } }; void event_callback(const gfalt_event_t e, gpointer user_data) { CopyTestMkdir *copyTest = static_cast(user_data); if (e->stage == GFAL_EVENT_TRANSFER_TYPE) { copyTest->isThirdPartyCopy = (strncmp(e->description, "3rd", 3) == 0); gfal2_log(G_LOG_LEVEL_INFO, "Third party copy"); } } const char* CopyTestMkdir::source_root; const char* CopyTestMkdir::destination_root; TEST_F(CopyTestMkdir, CopyNested) { char first_level[2048]; char second_level[2048]; char third_level[2048]; generate_random_uri(destination_root, "generate_folder", first_level, 2048); generate_random_uri(first_level, "generate_folder2", second_level, 2048); generate_random_uri(second_level, "generate_folder3", third_level, 2048); generate_random_uri(third_level, "generate_dest_file", destination, 2048); directories.push_front(first_level); directories.push_front(second_level); directories.push_front(third_level); GError *error = NULL; int ret = 0; ret = gfalt_copy_file(handle, params, source, destination, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); VerifyThirdPartyCopy(); } TEST_F(CopyTestMkdir, CopyAtRoot) { generate_random_uri(destination_root, "simple_file_at_root", destination, 2048); GError *error = NULL; int ret = 0; ret = gfalt_copy_file(handle, params, source, destination, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); VerifyThirdPartyCopy(); } int main(int argc, char** argv) { testing::InitGoogleTest(&argc, argv); if (argc < 2) { printf("Missing source and destination base urls\n"); printf("\t%s [options] srm://host/base/path/ srm://destination/base/path/\n", argv[0]); return 1; } CopyTestMkdir::source_root = argv[1]; CopyTestMkdir::destination_root = argv[2]; // gfal2_log_set_level(G_LOG_LEVEL_DEBUG); return RUN_ALL_TESTS(); } gfal2-v2.23.0/test/functional/gfalt_test_copy_file_replace.cpp000066400000000000000000000116421465240014500244710ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include static void event_callback(const gfalt_event_t e, gpointer user_data); class CopyTestReplace: public testing::Test { public: static const char* source_root; static const char* destination_root; char source[2048]; char destination[2048]; gfal2_context_t handle; gfalt_params_t params; bool isThirdPartyCopy, mustBeThirdPartyCopy; CopyTestReplace() { GError *error = NULL; handle = gfal2_context_new(&error); Gfal::gerror_to_cpp(&error); params = gfalt_params_handle_new(NULL); gfalt_add_event_callback(params, &event_callback, this, NULL, NULL); mustBeThirdPartyCopy = expect_third_party_copy(source_root, destination_root); } virtual ~CopyTestReplace() { gfal2_context_free(handle); gfalt_params_handle_delete(params, NULL); } virtual void SetUp() { isThirdPartyCopy = false; generate_random_uri(source_root, "copyfile_replace_source", source, 2048); generate_random_uri(destination_root, "copyfile_replace", destination, 2048); RecordProperty("Source", source); RecordProperty("Destination", source); GError* error = NULL; int ret = generate_file_if_not_exists(handle, source, "file:///etc/hosts", &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); } virtual void TearDown() { GError *error = NULL; gfal2_unlink(handle, source, &error); g_clear_error(&error); gfal2_unlink(handle, destination, &error); g_clear_error(&error); } void VerifyThirdPartyCopy() { if (mustBeThirdPartyCopy) { EXPECT_TRUE(isThirdPartyCopy); } else { EXPECT_FALSE(isThirdPartyCopy); } } }; void event_callback(const gfalt_event_t e, gpointer user_data) { CopyTestReplace *copyTest = static_cast(user_data); if (e->stage == GFAL_EVENT_TRANSFER_TYPE) { copyTest->isThirdPartyCopy = (strncmp(e->description, "3rd", 3) == 0); gfal2_log(G_LOG_LEVEL_INFO, "Third party copy"); } } const char* CopyTestReplace::source_root; const char* CopyTestReplace::destination_root; TEST_F(CopyTestReplace, CopyReplaceOverwriteNotSet) { GError* error = NULL; int ret = 0; ret = generate_file_if_not_exists(handle, destination, "file:///etc/hosts", &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); ret = gfalt_copy_file(handle, NULL, source, destination, &error); EXPECT_PRED_FORMAT3(AssertGfalErrno, ret, error, EEXIST); } TEST_F(CopyTestReplace, CopyReplaceOverwriteSet) { GError* error = NULL; int ret = 0; ret = generate_file_if_not_exists(handle, destination, "file:///etc/hosts", &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); ret = gfalt_set_replace_existing_file(params, TRUE, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); ret = gfalt_copy_file(handle, params, source, destination, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); VerifyThirdPartyCopy(); } TEST_F(CopyTestReplace, CopyReplaceOverwriteSet2) { GError* error = NULL; int ret = 0; struct stat st; ret = gfal2_stat(handle, destination, &st, &error); EXPECT_PRED_FORMAT3(AssertGfalErrno, ret, error, ENOENT); g_error_free(error); error = NULL; ret = gfalt_set_replace_existing_file(params, TRUE, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); ret = gfalt_copy_file(handle, params, source, destination, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); VerifyThirdPartyCopy(); } int main(int argc, char** argv) { testing::InitGoogleTest(&argc, argv); if (argc < 3) { printf("Missing source and destination base urls\n"); printf("\t%s [options] srm://host/base/path/ srm://destination/base/path/\n", argv[0]); return 1; } CopyTestReplace::source_root = argv[1]; CopyTestReplace::destination_root = argv[2]; // gfal2_log_set_level(G_LOG_LEVEL_DEBUG); return RUN_ALL_TESTS(); } gfal2-v2.23.0/test/functional/gfalt_test_copy_file_timeout.cpp000066400000000000000000000075231465240014500245470ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include static void event_callback(const gfalt_event_t e, gpointer user_data); class CopyTestTimeout: public testing::Test { public: static const char* source_root; static const char* destination_root; char source[2048]; char destination[2048]; gfal2_context_t handle; gfalt_params_t params; bool isThirdPartyCopy, mustBeThirdPartyCopy; CopyTestTimeout() { GError *error = NULL; handle = gfal2_context_new(&error); Gfal::gerror_to_cpp(&error); params = gfalt_params_handle_new(NULL); gfalt_add_event_callback(params, &event_callback, this, NULL, NULL); mustBeThirdPartyCopy = expect_third_party_copy(source_root, destination_root); } virtual ~CopyTestTimeout() { gfal2_context_free(handle); gfalt_params_handle_delete(params, NULL); } virtual void SetUp() { isThirdPartyCopy = false; generate_random_uri(source_root, "copyfile_timeout_source", source, 2048); generate_random_uri(destination_root, "copyfile_timeout", destination, 2048); RecordProperty("Source", source); RecordProperty("Destination", source); GError* error = NULL; int ret = generate_file_if_not_exists(handle, source, "file:///etc/hosts", &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); } virtual void TearDown() { GError *error = NULL; gfal2_unlink(handle, source, &error); g_clear_error(&error); gfal2_unlink(handle, destination, &error); g_clear_error(&error); } void VerifyThirdPartyCopy() { if (mustBeThirdPartyCopy) { EXPECT_TRUE(isThirdPartyCopy); } else { EXPECT_FALSE(isThirdPartyCopy); } } }; void event_callback(const gfalt_event_t e, gpointer user_data) { CopyTestTimeout *copyTest = static_cast(user_data); if (e->stage == GFAL_EVENT_TRANSFER_TYPE) { copyTest->isThirdPartyCopy = (strncmp(e->description, "3rd", 3) == 0); gfal2_log(G_LOG_LEVEL_INFO, "Third party copy"); } } const char* CopyTestTimeout::source_root; const char* CopyTestTimeout::destination_root; TEST_F(CopyTestTimeout, CopyTimeout) { GError* error = NULL; int ret = 0; ret = gfalt_set_timeout(params, 1, &error); ASSERT_PRED_FORMAT2(AssertGfalSuccess, ret, error); ret = gfalt_copy_file(handle, params, source, destination, &error); ASSERT_PRED_FORMAT3(AssertGfalErrno, ret, error, ETIMEDOUT); VerifyThirdPartyCopy(); } int main(int argc, char** argv) { testing::InitGoogleTest(&argc, argv); if (argc < 2) { printf("Missing source and destination base urls\n"); printf("\t%s [options] srm://host/base/path/ srm://destination/base/path/\n", argv[0]); return 1; } CopyTestTimeout::source_root = argv[1]; CopyTestTimeout::destination_root = argv[2]; // gfal2_log_set_level(G_LOG_LEVEL_DEBUG); return RUN_ALL_TESTS(); } gfal2-v2.23.0/test/functional/gfalt_test_double_cred.cpp000066400000000000000000000125431465240014500232750ustar00rootroot00000000000000/* * Copyright (c) CERN 2017 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include class CopyDoubleCredTest: public testing::Test { public: static const char *source_root, *destination_root; static const char *source_proxy, *destination_proxy; gfal2_context_t handle; gfalt_params_t params; gfal2_cred_t *source_cred, *dest_cred; char source[2048]; char destination[2048]; CopyDoubleCredTest() { source_cred = gfal2_cred_new(GFAL_CRED_X509_CERT, source_proxy); dest_cred = gfal2_cred_new(GFAL_CRED_X509_CERT, destination_proxy); } virtual ~CopyDoubleCredTest() { gfal2_cred_free(source_cred); gfal2_cred_free(dest_cred); } void SetUp() { unsetenv("X509_USER_PROXY"); unsetenv("X509_USER_CERT"); unsetenv("X509_USER_KEY"); int ret; GError *error = NULL; handle = gfal2_context_new(&error); Gfal::gerror_to_cpp(&error); params = gfalt_params_handle_new(&error); Gfal::gerror_to_cpp(&error); ret = gfal2_cred_clean(handle, &error); ASSERT_PRED_FORMAT2(AssertGfalSuccess, ret, error); // Clear automatic setup gfal2_remove_opt(handle, "X509", "CERT", NULL); gfal2_remove_opt(handle, "X509", "KEY", NULL); // Set configured values ret = gfal2_cred_set(handle, source_root, source_cred, &error); ASSERT_PRED_FORMAT2(AssertGfalSuccess, ret, error); ret = gfal2_cred_set(handle, destination_root, dest_cred, &error); ASSERT_PRED_FORMAT2(AssertGfalSuccess, ret, error); { setenv("X509_USER_PROXY", source_proxy, 1); generate_random_uri(source_root, "copyfile_doublecred_source", source, 2048); ret = generate_file_if_not_exists(handle, source, "file:///etc/hosts", &error); ASSERT_PRED_FORMAT2(AssertGfalSuccess, ret, error); unsetenv("X509_USER_PROXY"); } generate_random_uri(destination_root, "copyfile_doublecred_dest", destination, 2048); // Just make sure the environment is well prepared ASSERT_EQ((char*)NULL, getenv("X509_USER_CERT")); ASSERT_EQ((char*)NULL, getenv("X509_USER_KEY")); ASSERT_EQ((char*)NULL, getenv("X509_USER_PROXY")); ASSERT_EQ((gchar*)NULL, gfal2_get_opt_string_with_default(handle, "X509", "CERT", NULL)); ASSERT_EQ((gchar*)NULL, gfal2_get_opt_string_with_default(handle, "X509", "KEY", NULL)); } void TearDown() { GError *error = NULL; gfalt_params_handle_delete(params, NULL); { setenv("X509_USER_PROXY", source_proxy, 1); gfal2_unlink(handle, source, &error); g_clear_error(&error); unsetenv("X509_USER_PROXY"); } { setenv("X509_USER_PROXY", destination_proxy, 1); gfal2_unlink(handle, destination, &error); g_clear_error(&error); unsetenv("X509_USER_PROXY"); } gfal2_context_free(handle); } }; const char *CopyDoubleCredTest::source_root = NULL; const char *CopyDoubleCredTest::destination_root = NULL; const char *CopyDoubleCredTest::source_proxy = NULL; const char *CopyDoubleCredTest::destination_proxy = NULL; TEST_F(CopyDoubleCredTest, Stat) { GError *error = NULL; int ret; ret = gfal2_cred_set(handle, source_root, source_cred, &error); ASSERT_PRED_FORMAT2(AssertGfalSuccess, ret, error); struct stat buf; ret = gfal2_stat(handle, source, &buf, &error); ASSERT_PRED_FORMAT2(AssertGfalSuccess, ret, error); ASSERT_TRUE(S_ISREG(buf.st_mode)); } TEST_F(CopyDoubleCredTest, Simple) { GError *error = NULL; int ret; ret = gfal2_cred_set(handle, source_root, source_cred, &error); ASSERT_PRED_FORMAT2(AssertGfalSuccess, ret, error); ret = gfal2_cred_set(handle, destination_root, dest_cred, &error); ASSERT_PRED_FORMAT2(AssertGfalSuccess, ret, error); ret = gfalt_copy_file(handle, params, source, destination, &error); ASSERT_PRED_FORMAT2(AssertGfalSuccess, ret, error); } int main(int argc, char **argv) { testing::InitGoogleTest(&argc, argv); if (argc < 3 || (argc > 3 && argc != 5)) { printf("Usage: %s source_root destination [proxy-for-source_root proxy-for-destination]\n", argv[0]); return 1; } CopyDoubleCredTest::source_root = argv[1]; CopyDoubleCredTest::destination_root = argv[2]; if (argc > 3) { CopyDoubleCredTest::source_proxy = argv[3]; CopyDoubleCredTest::destination_proxy = argv[4]; } else { CopyDoubleCredTest::source_proxy = getenv("X509_USER_PROXY"); CopyDoubleCredTest::destination_proxy = getenv("X509_USER_PROXY"); } return RUN_ALL_TESTS(); } gfal2-v2.23.0/test/functional/gfalt_test_pasv.cpp000066400000000000000000000075451465240014500220050ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include #include #include #include static void eventCallback(const gfalt_event_t e, gpointer user_data); class PasvTest: public testing::Test { public: static const char* source_root; static const char* destination_root; char source[2048]; char destination[2048]; gfal2_context_t handle; gfalt_params_t params; bool pasvEventTriggered; PasvTest(): pasvEventTriggered(false) { GError *error = NULL; handle = gfal2_context_new(&error); Gfal::gerror_to_cpp(&error); params = gfalt_params_handle_new(&error); Gfal::gerror_to_cpp(&error); gfal2_set_opt_boolean(handle, "GRIDFTP PLUGIN", "ENABLE_PASV_PLUGIN", TRUE, &error); Gfal::gerror_to_cpp(&error); } virtual ~PasvTest() { gfal2_context_free(handle); GError *error = NULL; gfalt_params_handle_delete(params, &error); g_clear_error(&error); } virtual void SetUp() { pasvEventTriggered = false; generate_random_uri(source_root, "pasv_source", source, 2048); generate_random_uri(destination_root, "pasv", destination, 2048); RecordProperty("Source", source); RecordProperty("Destination", source); GError* error = NULL; int ret = generate_file_if_not_exists(handle, source, "file:///etc/hosts", &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); } virtual void TearDown() { GError *error = NULL; gfal2_unlink(handle, source, &error); g_clear_error(&error); gfal2_unlink(handle, destination, &error); g_clear_error(&error); gfalt_remove_event_callback(params, eventCallback, &error); g_clear_error(&error); } }; const char* PasvTest::source_root; const char* PasvTest::destination_root; static void eventCallback(const gfalt_event_t e, gpointer user_data) { const GQuark gridftpQuark = g_quark_from_string("GSIFTP"); const GQuark pasvQuark = g_quark_from_static_string("PASV"); if (e->domain == gridftpQuark && e->stage == pasvQuark) { PasvTest* pasv = (PasvTest*)user_data; pasv->pasvEventTriggered = true; printf("%s\n", e->description); } } TEST_F(PasvTest, EnablePasvPlugin) { GError* error = NULL; int ret = gfalt_add_event_callback(params, eventCallback, this, NULL, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); ret = gfalt_copy_file(handle, params, source, destination, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); EXPECT_TRUE(pasvEventTriggered); } int main(int argc, char** argv) { testing::InitGoogleTest(&argc, argv); if (argc < 3) { printf("Missing source and destination base urls\n"); printf("\t%s [options] srm://host/base/path/ srm://destination/base/path/\n", argv[0]); return 1; } PasvTest::source_root = argv[1]; PasvTest::destination_root = argv[2]; return RUN_ALL_TESTS(); } gfal2-v2.23.0/test/functional/gfalt_test_rd3_reorder_protocols.cpp000066400000000000000000000104741465240014500253450ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include static void event_callback(const gfalt_event_t e, gpointer user_data); class ReorderSRMProtocolsTest: public testing::Test { public: static const char* source_root; static const char* destination_root; char source[2048]; char destination[2048]; gfal2_context_t handle; gfalt_params_t params; bool isThirdPartyCopy, mustBeThirdPartyCopy; ReorderSRMProtocolsTest() { GError *error = NULL; handle = gfal2_context_new(&error); Gfal::gerror_to_cpp(&error); params = gfalt_params_handle_new(&error); Gfal::gerror_to_cpp(&error); gfalt_add_event_callback(params, event_callback, this, NULL, &error); Gfal::gerror_to_cpp(&error); gfal2_set_opt_string(handle, "SRM PLUGIN", "TURL_3RD_PARTY_PROTOCOLS", "rfio;gsiftp", &error); mustBeThirdPartyCopy = expect_third_party_copy(source_root, destination_root); } virtual ~ReorderSRMProtocolsTest() { gfal2_context_free(handle); gfalt_params_handle_delete(params, NULL); } virtual void SetUp() { isThirdPartyCopy = false; generate_random_uri(source_root, "copyfile_source", source, 2048); generate_random_uri(destination_root, "copyfile", destination, 2048); RecordProperty("Source", source); RecordProperty("Destination", source); GError* error = NULL; int ret = generate_file_if_not_exists(handle, source, "file:///etc/hosts", &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); } virtual void TearDown() { GError *error = NULL; gfal2_unlink(handle, source, &error); g_clear_error(&error); gfal2_unlink(handle, destination, &error); g_clear_error(&error); } void VerifyThirdPartyCopy() { if (mustBeThirdPartyCopy) { EXPECT_TRUE(isThirdPartyCopy); } else { EXPECT_FALSE(isThirdPartyCopy); } } }; static void event_callback(const gfalt_event_t e, gpointer user_data) { ReorderSRMProtocolsTest *reorderSRMProtocolsTest = static_cast(user_data); if (e->stage == GFAL_EVENT_TRANSFER_TYPE) { reorderSRMProtocolsTest->isThirdPartyCopy = (strncmp(e->description, "3rd", 3) == 0); gfal2_log(G_LOG_LEVEL_INFO, "Third party copy"); } } const char* ReorderSRMProtocolsTest::source_root; const char* ReorderSRMProtocolsTest::destination_root; TEST_F(ReorderSRMProtocolsTest, SimpleFileCopy) { GError* error = NULL; int ret = gfalt_copy_file(handle, params, source, destination, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); VerifyThirdPartyCopy(); } TEST_F(ReorderSRMProtocolsTest, SimpleFileCopyENOENT) { GError* error = NULL; int ret = gfal2_unlink(handle, source, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); ret = gfalt_copy_file(handle, params, source, destination, &error); EXPECT_PRED_FORMAT3(AssertGfalErrno, ret, error, ENOENT); } int main(int argc, char** argv) { testing::InitGoogleTest(&argc, argv); if (argc < 2) { printf("Missing source and destination base urls\n"); printf("\t%s [options] srm://host/base/path/ srm://destination/base/path/\n", argv[0]); return 1; } ReorderSRMProtocolsTest::source_root = argv[1]; ReorderSRMProtocolsTest::destination_root = argv[2]; // gfal2_log_set_level(G_LOG_LEVEL_DEBUG); return RUN_ALL_TESTS(); } gfal2-v2.23.0/test/gfal2-tests.sh000077500000000000000000000005711465240014500164340ustar00rootroot00000000000000#!/usr/bin/bash set -ex export HOME=/ export X509_USER_PROXY=/tmp/x509up_u`id -u` VOMS=${VOMS:=dteam} PASSWD=$1 if [[ -z "$PASSWD" ]]; then echo "Missing password" exit 1 fi cat > /etc/gfal2.d/sftp_plugin.conf < /etc/gfal2.d/sftp_plugin.conf" # Create proxy mock_cmd chroot "echo ${PASSWD} | voms-proxy-init --rfc --cert ${MOCK_HOME}/.globus/usercert.pem --key ${MOCK_HOME}/.globus/userkey.pem --voms ${VOMS} --pwstdin" # Run tests mock_cmd chroot "cd /usr/share/gfal2/tests/; ctest -T test" # Recover logs rm -rf "${OUTPUT_DIR}/Testing" mock_cmd --copyout "/usr/share/gfal2/tests/Testing" "${OUTPUT_DIR}/Testing" gfal2-v2.23.0/test/stress-test/000077500000000000000000000000001465240014500162375ustar00rootroot00000000000000gfal2-v2.23.0/test/stress-test/CMakeLists.txt000066400000000000000000000006651465240014500210060ustar00rootroot00000000000000FILE(GLOB src_loadtest "gfalt_copyfile_fts_style_load_test.c") IF (STRESS_TESTS) include_directories( "${CMAKE_SOURCE_DIR}/src " "${CMAKE_SOURCE_DIR}/src/posix/" ${GLIB2_PKG_INCLUDE_DIRS} ${NETTLE_PKG_INCLUDE_DIRS} ) add_executable(fts_seq_copy_files ${src_loadtest}) target_link_libraries(fts_seq_copy_files ${GFAL2_TRANSFER_LINK} ${GFAL2_LINK} gfal2_test_shared) ENDIF (STRESS_TESTS) gfal2-v2.23.0/test/stress-test/gfalt_copyfile_fts_style_load_test.c000066400000000000000000000107021465240014500255240ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include #include // // This test follows exactly the FTS 3.0 copy pattern and can simualte a long run of successive copy in different context // void call_perf(gfalt_transfer_status_t h, const char* src, const char* dst, gpointer user_data){ size_t avg = gfalt_copy_get_average_baudrate(h,NULL) / 1024; size_t inst = gfalt_copy_get_instant_baudrate(h,NULL) / 1024; size_t trans= gfalt_copy_get_bytes_transferred(h,NULL); time_t elapsed = gfalt_copy_get_elapsed_time(h,NULL); printf(" <%lld> perf marker avg : %zd, inst: %zd, elapsed: %lld, trans: %zd \n", (long long) time(NULL), avg, inst, (long long) elapsed, trans); } int internal_copy(gfal2_context_t* handle, gfalt_params_t* params, const char* src, const char* dst){ GError * tmp_err = NULL; // classical GError/glib error management struct stat buff; struct stat buff2; printf(" stat src files.... \n"); if( gfal2_stat(*handle, src, &buff, &tmp_err) !=0){ printf(" error while the file stat %d : %s.\n", tmp_err->code,tmp_err->message); return -1; }else{ g_assert(buff.st_size != 0); printf(" file size %lld", (long long) buff.st_size); } printf(" begin transfer .... \n"); if( gfalt_copy_file(*handle, *params, src, dst, &tmp_err) != 0){ printf(" error while the file transfer %d : %s.\n", tmp_err->code,tmp_err->message); return -1; }else printf(" transfer sucessfull ! \n"); printf(" stat dst file .... \n"); if( gfal2_stat(*handle, dst, &buff2, &tmp_err) !=0){ printf(" error while the file stat %d : %s.\n", tmp_err->code,tmp_err->message); return -1; }{ g_assert(buff2.st_size == buff.st_size); } return 0; } int main(int argc, char** argv){ int i, ret; if( argc <5 ){ printf(" Usage %s [src_url] [checksum] [dst_dir] [nbtime] \n",argv[0]); return 1; } GError * tmp_err = NULL; // classical GError/glib error management gfal2_context_t handle; // initialize gfal gfal_set_verbose(GFAL_VERBOSE_TRACE | GFAL_VERBOSE_VERBOSE | GFAL_VERBOSE_DEBUG); char buff[2048]; const int nbtime = atoi(argv[4]); const char * checksum_user = argv[2]; const char* src_file = argv[1]; printf(" sechedule %d transfets...", nbtime); for(i =0; i < nbtime;++i){ printf("execute %d....", i); if( (handle = gfal2_context_new(&tmp_err)) == NULL ) { printf(" bad initialization %d : %s.\n", tmp_err->code,tmp_err->message); return -1; } generate_random_uri(argv[3], "dest_fts_load_test_file", buff, 2048); // creat params gfalt_params_t my_params = gfalt_params_handle_new(NULL); gfalt_set_replace_existing_file(my_params, TRUE, NULL); gfalt_set_checksum_check(my_params, TRUE, NULL); gfalt_set_user_defined_checksum(my_params,"ADLER32",checksum_user, NULL); gfalt_set_monitor_callback(my_params, &call_perf,&tmp_err); gfalt_set_nbstreams(my_params, 4, &tmp_err); gfalt_set_timeout(my_params,3600, &tmp_err); // gfalt_set_src_spacetoken(my_params, "DTEAMLCGUTILSTEST", &tmp_err); // gfalt_set_dst_spacetoken(my_params, "DTEAMLCGUTILSTESTF", &tmp_err); // begin copy if(internal_copy(&handle, &my_params, src_file, buff) !=0){ return -1; } printf(" cleanup the copy ... %s",buff); if( (ret =gfal2_unlink(handle, buff, &tmp_err) ) < 0){ printf(" error message %s", tmp_err->message); return -1; } gfal2_context_free(handle); } return 0; } gfal2-v2.23.0/test/unit/000077500000000000000000000000001465240014500147165ustar00rootroot00000000000000gfal2-v2.23.0/test/unit/CMakeLists.txt000066400000000000000000000027211465240014500174600ustar00rootroot00000000000000find_package (PugiXML) include_directories( "${CMAKE_SOURCE_DIR}/test" "${CMAKE_SOURCE_DIR}/src/posix/" ) add_subdirectory(cancel) add_subdirectory(config) add_subdirectory(cred) add_subdirectory(global) add_subdirectory(http) add_subdirectory(mds) add_subdirectory(transfer) add_subdirectory(uri) if (PUGIXML_FOUND) set (TEST_MDS ./mds/test_mds.cpp) else (PUGIXML_FOUND) set (TEST_MDS "") endif (PUGIXML_FOUND) if (PLUGIN_HTTP) file(GLOB TEST_HTTP_PLUGIN "./http/test_*.cpp") set(HTTP_PLUGIN_LIBRARIES plugin_http_static) include (CheckLibraryExists) find_package(Davix REQUIRED) # Includes include_directories(${DAVIX_INCLUDE_DIR}) else(PLUGIN_HTTP) set(TEST_HTTP_PLUGIN "") set(HTTP_PLUGIN_LIBRARIES "") endif (PLUGIN_HTTP) add_executable(gfal2-unit-tests ./cancel/cancel_tests.cpp ./config/config_test.cpp ./cred/test_cred.cpp ./global/global_test.cpp ${TEST_HTTP_PLUGIN} ${TEST_MDS} ./transfer/tests_callbacks.cpp ./transfer/tests_params.cpp ./uri/test_uri.cpp ./uri/test_parsing.cpp ) target_include_directories(gfal2-unit-tests PRIVATE ${CMAKE_SOURCE_DIR}/test ${PROJECT_SOURCE_DIR}/src ${CMAKE_CURRENT_SOURCE_DIR} ) target_link_libraries(gfal2-unit-tests ${GFAL2_LIBRARIES} ${GTEST_LIBRARIES} ${GTEST_MAIN_LIBRARIES} gfal2_test_shared ${HTTP_PLUGIN_LIBRARIES} ) install(TARGETS gfal2-unit-tests DESTINATION ${BIN_INSTALL_DIR}/) add_test(unit-tests gfal2-unit-tests) gfal2-v2.23.0/test/unit/cancel/000077500000000000000000000000001465240014500161435ustar00rootroot00000000000000gfal2-v2.23.0/test/unit/cancel/CMakeLists.txt000066400000000000000000000004501465240014500207020ustar00rootroot00000000000000file (GLOB src_test_cancel "*.c*") add_executable(unit_test_transfer_cancel_exe ${src_test_cancel} ) target_link_libraries(unit_test_transfer_cancel_exe ${GFAL2_LIBRARIES} ${GTEST_LIBRARIES} ${GTEST_MAIN_LIBRARIES} m ) add_test(unit_test_transfer_cancel unit_test_transfer_cancel_exe) gfal2-v2.23.0/test/unit/cancel/cancel_tests.cpp000066400000000000000000000034451465240014500213240ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include TEST(gfalCancel, test_cancel_simple){ GError* tmp_err=NULL; gfal2_context_t c = gfal2_context_new(&tmp_err); ASSERT_TRUE(c != NULL); gfal2_cancel(c); gfal2_cancel(c); gfal2_cancel(c); gfal2_context_free(c); } void gfal_cancel_hook_cb_s(gfal2_context_t context, void* userdata){ int* p = (int*)userdata; *p += 1; (void) gfal2_version(); } TEST(gfalGlobal, testCancelCallback){ GError* tmp_err=NULL; gfal2_context_t c = gfal2_context_new(&tmp_err); if (tmp_err) { printf("%d %s\n", tmp_err->code, tmp_err->message); } ASSERT_TRUE(c != NULL); int i=0,res ; // add callback7 gfal_cancel_token_t tok = gfal2_register_cancel_callback(c, &gfal_cancel_hook_cb_s, &i); ASSERT_TRUE( tok != NULL); res= gfal2_cancel(c); ASSERT_EQ(res, 0); ASSERT_EQ(i, 1); gfal2_remove_cancel_callback(c, tok); res= gfal2_cancel(c); ASSERT_EQ(res, 0); ASSERT_EQ(i, 1); res= gfal2_cancel(c); ASSERT_EQ(res, 0); gfal2_context_free(c); } gfal2-v2.23.0/test/unit/config/000077500000000000000000000000001465240014500161635ustar00rootroot00000000000000gfal2-v2.23.0/test/unit/config/CMakeLists.txt000066400000000000000000000003271465240014500207250ustar00rootroot00000000000000 add_executable(config_test "config_test.cpp") target_link_libraries(config_test ${GFAL2_LIBRARIES} ${GTEST_LIBRARIES} ${GTEST_MAIN_LIBRARIES} gfal2_test_shared ) add_test(config_test config_test) gfal2-v2.23.0/test/unit/config/config_test.cpp000066400000000000000000000066571465240014500212110ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include class ConfigFixture: public testing::Test { protected: gfal2_context_t context; public: ConfigFixture() { GError* error = NULL; context = gfal2_context_new(&error); Gfal::gerror_to_cpp(&error); } ~ConfigFixture() { gfal2_context_free(context); } }; // Regression test for DMC-833 TEST_F(ConfigFixture, ClientData) { GError *error = NULL; int ret = 0; ret = gfal2_add_client_info(context, "TEST", "VALUE", &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); ret = gfal2_add_client_info(context, "TEST2", "VALUE2", &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); ret = gfal2_add_client_info(context, "TEST", "REPLACED", &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); ret = gfal2_get_client_info_count(context, &error); EXPECT_EQ(ret, 2); const char *value; ret = gfal2_get_client_info_value(context, "TEST", &value, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); EXPECT_STRCASEEQ(value, "REPLACED"); } TEST_F(ConfigFixture, String) { GError *error = NULL; int ret = 0; ret = gfal2_set_opt_string(context, "GROUP1", "KEY2", "MYVALUE3", &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); gchar *value = gfal2_get_opt_string(context, "GROUP1", "KEY2", &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); EXPECT_EQ(0, strncmp(value, "MYVALUE3", 8)); g_free(value); } TEST_F(ConfigFixture, Integer) { GError *error = NULL; int ret = 0; ret = gfal2_set_opt_integer(context, "GROUP1", "KEY2", 43215, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); int value = gfal2_get_opt_integer(context, "GROUP1", "KEY2", &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); EXPECT_EQ(43215, value); } TEST_F(ConfigFixture, KeyList) { GError *error = NULL; int ret = 0; ret = gfal2_set_opt_string(context, "GROUP1", "KEY1", "abcd", &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); ret = gfal2_set_opt_string(context, "GROUP1", "KEY2", "efgh", &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); ret = gfal2_set_opt_string(context, "GROUP2", "KEY3", "1234", &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, ret, error); gsize count = 0; gchar **keys = gfal2_get_opt_keys(context, "GROUP1", &count, &error); EXPECT_PRED_FORMAT2(AssertGfalSuccess, 0, error); EXPECT_EQ(2, count); EXPECT_STREQ("KEY1", keys[0]); EXPECT_STREQ("KEY2", keys[1]); EXPECT_EQ(NULL, keys[2]); g_strfreev(keys); }gfal2-v2.23.0/test/unit/cred/000077500000000000000000000000001465240014500156335ustar00rootroot00000000000000gfal2-v2.23.0/test/unit/cred/CMakeLists.txt000066400000000000000000000003451465240014500203750ustar00rootroot00000000000000 add_executable(gfal2_cred_test "test_cred.cpp") target_link_libraries(gfal2_cred_test ${GFAL2_LIBRARIES} ${GTEST_LIBRARIES} ${GTEST_MAIN_LIBRARIES} gfal2_test_shared ) add_test(gfal2_cred_test gfal2_cred_test) gfal2-v2.23.0/test/unit/cred/test_cred.cpp000066400000000000000000000210171465240014500203140ustar00rootroot00000000000000/* * Copyright (c) CERN 2017 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include "common/gfal_gtest_asserts.h" class CredTest: public testing::Test { protected: gfal2_context_t context; gfal2_cred_t *x509, *user, *x509_2; gfal2_cred_t *token, *token_2; public: CredTest() { x509 = gfal2_cred_new(GFAL_CRED_X509_CERT, "ABCDE"); x509_2 = gfal2_cred_new(GFAL_CRED_X509_CERT, "12345"); user = gfal2_cred_new(GFAL_CRED_USER, "user"); token = gfal2_cred_new(GFAL_CRED_BEARER, "mytoken"); token_2 = gfal2_cred_new(GFAL_CRED_BEARER, "mytoken_2"); } virtual ~CredTest() { gfal2_cred_free(x509); gfal2_cred_free(x509_2); gfal2_cred_free(user); gfal2_cred_free(token); gfal2_cred_free(token_2); } virtual void SetUp() { GError *error = NULL; context = gfal2_context_new(&error); ASSERT_PRED_FORMAT2(AssertGfalSuccess, 0, error); int ret = gfal2_cred_clean(context, &error); ASSERT_PRED_FORMAT2(AssertGfalSuccess, ret, error); } virtual void TearDown() { gfal2_context_free(context); } }; TEST_F(CredTest, dup) { gfal2_cred_t *copy = gfal2_cred_dup(x509); ASSERT_NE(copy, (void*)NULL); ASSERT_NE(copy->type, x509->type); ASSERT_NE(copy->value, x509->value); ASSERT_STREQ(copy->type, x509->type); ASSERT_STREQ(copy->value, x509->value); gfal2_cred_free(copy); } TEST_F(CredTest, set_get) { const char *original_baseurl = "gsiftp://host.com/path"; GError *error = NULL; int ret = gfal2_cred_set(context, original_baseurl, x509, &error); ASSERT_PRED_FORMAT2(AssertGfalSuccess, ret, error); const char *baseurl = NULL; char *resp = gfal2_cred_get(context, GFAL_CRED_X509_CERT, "gsiftp://host.com/path/subdir/file", &baseurl, &error); ASSERT_PRED_FORMAT2(AssertGfalSuccess, 0, error); ASSERT_NE(resp, (void*)NULL); ASSERT_STREQ(resp, x509->value); ASSERT_STREQ(original_baseurl, baseurl); g_free(resp); } TEST_F(CredTest, compat) { GError *error = NULL; int ret; ret = gfal2_set_opt_string(context, "X509", "CERT", "/path/to/my/cert", &error); ASSERT_PRED_FORMAT2(AssertGfalSuccess, ret, error); ret = gfal2_set_opt_string(context, "X509", "KEY", "/path/to/my/key", &error); ASSERT_PRED_FORMAT2(AssertGfalSuccess, ret, error); const char *baseurl = NULL; char *resp = gfal2_cred_get(context, GFAL_CRED_X509_CERT, "gsiftp://host.com/path/subdir/file", &baseurl, &error); ASSERT_PRED_FORMAT2(AssertGfalSuccess, 0, error); ASSERT_NE(resp, (void*)NULL); ASSERT_STREQ(resp, "/path/to/my/cert"); ASSERT_STREQ("", baseurl); g_free(resp); resp = gfal2_cred_get(context, GFAL_CRED_X509_KEY, "gsiftp://host.com/path/subdir/file", &baseurl, &error); ASSERT_PRED_FORMAT2(AssertGfalSuccess, 0, error); ASSERT_NE(resp, (void*)NULL); ASSERT_STREQ(resp, "/path/to/my/key"); ASSERT_STREQ("", baseurl); g_free(resp); } TEST_F(CredTest, set_get_longer_prefix) { const char *short_base = "gsiftp://host.com/path"; const char *long_base = "gsiftp://host.com/path/subdir"; GError *error = NULL; int ret = gfal2_cred_set(context, short_base, x509, &error); ASSERT_PRED_FORMAT2(AssertGfalSuccess, ret, error); ret = gfal2_cred_set(context, long_base, user, &error); ASSERT_PRED_FORMAT2(AssertGfalSuccess, ret, error); ret = gfal2_cred_set(context, long_base, x509_2, &error); ASSERT_PRED_FORMAT2(AssertGfalSuccess, ret, error); const char *baseurl = NULL; char *resp = gfal2_cred_get(context, GFAL_CRED_X509_CERT, "gsiftp://host.com/path/subdir/file", &baseurl, &error); ASSERT_PRED_FORMAT2(AssertGfalSuccess, 0, error); ASSERT_NE(resp, (void*)NULL); ASSERT_STREQ(resp, x509_2->value); ASSERT_STREQ(long_base, baseurl); g_free(resp); } static void callback(const char *url_prefix, const gfal2_cred_t *cred, void *user_data) { char **value = (char**)user_data; *value = cred->value; } TEST_F(CredTest, foreach) { GError *error = NULL; int ret = gfal2_cred_set(context, "gsiftp://host.com/path", x509, &error); ASSERT_PRED_FORMAT2(AssertGfalSuccess, ret, error); char *value; gfal2_cred_foreach(context, callback, &value); ASSERT_STREQ(value, x509->value); } TEST_F(CredTest, copy) { GError *error = NULL; int ret = gfal2_cred_set(context, "gsiftp://host.com/path", x509, &error); ASSERT_PRED_FORMAT2(AssertGfalSuccess, ret, error); gfal2_context_t new_context = gfal2_context_new(&error); ASSERT_PRED_FORMAT2(AssertGfalSuccess, 0, error); ret = gfal2_cred_copy(new_context, context, &error); ASSERT_PRED_FORMAT2(AssertGfalSuccess, ret, error); const char *baseurl = NULL; char *resp = gfal2_cred_get(new_context, GFAL_CRED_X509_CERT, "gsiftp://host.com/path/subdir/file", &baseurl, &error); ASSERT_NE(resp, (void*)NULL); ASSERT_STREQ(resp, x509->value); g_free(resp); gfal2_context_free(new_context); } TEST_F(CredTest, set_get_del) { const char* short_base = "https://host.com/path"; const char* long_base = "https://host.com/path/subpath"; const char* full_path = "https://host.com/path/subpath/file"; GError* error = NULL; int ret = gfal2_cred_set(context, short_base, token, &error); ASSERT_PRED_FORMAT2(AssertGfalSuccess, ret, error); ret = gfal2_cred_set(context, long_base, token_2, &error); ASSERT_PRED_FORMAT2(AssertGfalSuccess, ret, error); const char* baseurl = NULL; char* resp = gfal2_cred_get(context, GFAL_CRED_BEARER, full_path, &baseurl, &error); ASSERT_PRED_FORMAT2(AssertGfalSuccess, 0, error); ASSERT_NE(resp, (void*) NULL); ASSERT_STREQ(resp, token_2->value); ASSERT_STREQ(long_base, baseurl); g_free(resp); ret = gfal2_cred_del(context, GFAL_CRED_BEARER, long_base, &error); ASSERT_PRED_FORMAT2(AssertGfalSuccess, ret, error); resp = gfal2_cred_get(context, GFAL_CRED_BEARER, full_path, &baseurl, &error); ASSERT_PRED_FORMAT2(AssertGfalSuccess, 0, error); ASSERT_NE(resp, (void*) NULL); ASSERT_STREQ(resp, token->value); ASSERT_STREQ(short_base, baseurl); g_free(resp); ret = gfal2_cred_del(context, GFAL_CRED_BEARER, long_base, &error); ASSERT_EQ(ret, -1); ret = gfal2_cred_del(context, GFAL_CRED_BEARER, full_path, &error); ASSERT_EQ(ret, -1); } TEST_F(CredTest, set_get_prefix) { const char* dir_base = "https://host.com/path"; const char* dir_full_base = "https://host.com/path/"; const char* file_base = "https://host.com/path/file"; const char* full_path = "https://host.com/path/file.extension"; GError* error = NULL; int ret = gfal2_cred_set(context, dir_base, token, &error); ASSERT_PRED_FORMAT2(AssertGfalSuccess, ret, error); ret = gfal2_cred_set(context, dir_full_base, token_2, &error); ASSERT_PRED_FORMAT2(AssertGfalSuccess, ret, error); ret = gfal2_cred_set(context, file_base, token_2, &error); ASSERT_PRED_FORMAT2(AssertGfalSuccess, ret, error); const char* baseurl = NULL; char* resp = gfal2_cred_get(context, GFAL_CRED_BEARER, full_path, &baseurl, &error); ASSERT_PRED_FORMAT2(AssertGfalSuccess, 0, error); ASSERT_NE(resp, (void*) NULL); ASSERT_STREQ(resp, token_2->value); ASSERT_STREQ(dir_full_base, baseurl); g_free(resp); ret = gfal2_cred_del(context, GFAL_CRED_BEARER, dir_full_base, &error); ASSERT_PRED_FORMAT2(AssertGfalSuccess, ret, error); resp = gfal2_cred_get(context, GFAL_CRED_BEARER, full_path, &baseurl, &error); ASSERT_PRED_FORMAT2(AssertGfalSuccess, 0, error); ASSERT_NE(resp, (void*) NULL); ASSERT_STREQ(resp, token->value); ASSERT_STREQ(dir_base, baseurl); g_free(resp); ret = gfal2_cred_del(context, GFAL_CRED_BEARER, dir_base, &error); ASSERT_PRED_FORMAT2(AssertGfalSuccess, ret, error); resp = gfal2_cred_get(context, GFAL_CRED_BEARER, full_path, &baseurl, &error); ASSERT_PRED_FORMAT2(AssertGfalSuccess, 0, error); ASSERT_EQ(resp, (void*) NULL); ASSERT_STREQ("", baseurl); } gfal2-v2.23.0/test/unit/global/000077500000000000000000000000001465240014500161565ustar00rootroot00000000000000gfal2-v2.23.0/test/unit/global/CMakeLists.txt000066400000000000000000000003401465240014500207130ustar00rootroot00000000000000 add_executable(gfal2_test_exe "global_test.cpp" ${gfal2_utils_src}) target_link_libraries(gfal2_test_exe ${GFAL2_LIBRARIES} ${GTEST_LIBRARIES} ${GTEST_MAIN_LIBRARIES} ) add_test(gfal2_test_exe gfal2_test_exe) gfal2-v2.23.0/test/unit/global/global_test.cpp000066400000000000000000000052251465240014500211650ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include TEST(gfalGlobal, testLogLevel) { // Restrict log level to "Warning" to reduce verbosity gfal2_log_set_level(G_LOG_LEVEL_WARNING); GLogLevelFlags r = gfal2_log_get_level(); ASSERT_EQ(G_LOG_LEVEL_WARNING, r); } TEST(gfalGlobal, testLoad) { GError *tmp_err = NULL; gfal2_context_t c = gfal2_context_new(&tmp_err); if (tmp_err) printf("%s\n", tmp_err->message); ASSERT_EQ(NULL, tmp_err); ASSERT_NE((void *) NULL, c); gfal2_context_free(c); c = NULL; gfal2_context_free(c); } static const char *test_plugin_get_name(void) { return "TEST PLUGIN"; } static gboolean test_plugin_url(plugin_handle plugin_data, const char *url, plugin_mode operation, GError **err) { return strncmp(url, "test://", 7) == 0 && operation == GFAL_PLUGIN_STAT; } static int test_plugin_stat(plugin_handle plugin_data, const char *url, struct stat *buf, GError **err) { buf->st_mode = 12345; return 0; } TEST(gfalGlobal, registerPlugin) { GError *tmp_err = NULL; gfal2_context_t c = gfal2_context_new(&tmp_err); ASSERT_NE((void *) NULL, c); gfal_plugin_interface test_plugin; memset(&test_plugin, 0, sizeof(test_plugin)); test_plugin.getName = test_plugin_get_name; test_plugin.check_plugin_url = test_plugin_url; test_plugin.statG = test_plugin_stat; int ret = gfal2_register_plugin(c, &test_plugin, &tmp_err); if (tmp_err) printf("%s\n", tmp_err->message); ASSERT_EQ(0, ret); struct stat st; ret = gfal2_stat(c, "test://blah", &st, &tmp_err); ASSERT_EQ(0, ret); ASSERT_EQ(12345, st.st_mode); gchar **plugins = gfal2_get_plugin_names(c); int found = 0; for (int i = 0; plugins[i] != NULL; ++i) { found += (strncmp(plugins[i], "TEST PLUGIN", 11) == 0); } ASSERT_EQ(found, 1); g_strfreev(plugins); gfal2_context_free(c); } gfal2-v2.23.0/test/unit/http/000077500000000000000000000000001465240014500156755ustar00rootroot00000000000000gfal2-v2.23.0/test/unit/http/CMakeLists.txt000066400000000000000000000027341465240014500204430ustar00rootroot00000000000000add_executable(gfal2_token_map_test "test_token_map.cpp") add_executable(gfal2_custom_http_options_test "test_custom_http_options.cpp") add_executable(gfal2_http_copy_mode_test "test_http_copy_mode.cpp") find_package(Davix REQUIRED) find_package(JSONC REQUIRED) file(GLOB src_http "${CMAKE_SOURCE_DIR}/src/plugins/http/gfal_http_*.cpp") add_library(test_plugin_http STATIC ${src_http}) target_link_libraries(test_plugin_http gfal2 ${DAVIX_LIBRARIES} ${JSONC_LIBRARIES} ${CRYPTOPP_LIBRARIES}) set(test_plugin_http_link_libraries ${GFAL2_LIBRARIES} ${GTEST_LIBRARIES} ${GTEST_MAIN_LIBRARIES} gfal2_test_shared ${DAVIX_LIBRARIES} ${CRYPTOPP_LIBRARIES} test_plugin_http) target_include_directories(test_plugin_http PRIVATE ${DAVIX_INCLUDE_DIR} ${JSONC_INCLUDE_DIRS}) target_link_libraries(gfal2_token_map_test ${test_plugin_http_link_libraries}) target_include_directories(gfal2_token_map_test PRIVATE ${DAVIX_INCLUDE_DIR}) target_link_libraries(gfal2_custom_http_options_test ${test_plugin_http_link_libraries}) target_include_directories(gfal2_custom_http_options_test PRIVATE ${DAVIX_INCLUDE_DIR}) target_link_libraries(gfal2_http_copy_mode_test ${test_plugin_http_link_libraries}) target_include_directories(gfal2_http_copy_mode_test PRIVATE ${DAVIX_INCLUDE_DIR}) add_test(gfal2_token_map_test gfal2_token_map_test) add_test(gfal2_custom_http_options_test gfal2_custom_http_options_test) add_test(gfal2_http_copy_mode_test gfal2_http_copy_mode_test) gfal2-v2.23.0/test/unit/http/test_custom_http_options.cpp000066400000000000000000000134571465240014500235760ustar00rootroot00000000000000/* * Copyright (c) CERN 2022 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #define __GFAL2_H_INSIDE__ #include #undef __GFAL2_H_INSIDE__ #include #include "plugins/http/gfal_http_plugin.h" class CustomHttpOptionsTest: public testing::Test { public: CustomHttpOptionsTest() = default; ~CustomHttpOptionsTest() = default; virtual void SetUp() { GError* error = NULL; context = gfal2_context_new(&error); Gfal::gerror_to_cpp(&error); storeConfig("HTTP PLUGIN", "ENABLE_STREAM_COPY", true); storeConfig("HTTP PLUGIN", "ENABLE_FALLBACK_TPC_COPY", true); } virtual void TearDown() { gfal2_context_free(context); context = NULL; } protected: gfal2_context_t context = NULL; const char* src = "https://eospublic.cern.ch:443/path/file.src"; const char* dst = "https://eospps.cern.ch:443/path/file.dst"; void storeConfig(const std::string& group, const char* key, bool value) { GError* error = NULL; gfal2_set_opt_boolean(context, group.c_str(), key, value, &error); ASSERT_PRED_FORMAT2(AssertGfalSuccess, 0, error); } std::string surlToConfigGroup(const char* surl) { Davix::Uri uri(surl); std::string prot = uri.getProtocol(); if (prot.back() == 's') { prot.pop_back(); } std::string group = prot + ":" + uri.getHost(); std::transform(group.begin(), group.end(), group.begin(), ::toupper); return group; } }; TEST_F(CustomHttpOptionsTest, GenericEnabled) { ASSERT_TRUE(is_http_streaming_enabled(context, src, dst)); ASSERT_TRUE(is_http_3rdcopy_fallback_enabled(context, src, dst)); } TEST_F(CustomHttpOptionsTest, GenericDisabled) { storeConfig("HTTP PLUGIN", "ENABLE_STREAM_COPY", false); ASSERT_FALSE(is_http_streaming_enabled(context, src, dst)); storeConfig("HTTP PLUGIN", "ENABLE_FALLBACK_TPC_COPY", false); ASSERT_FALSE(is_http_3rdcopy_fallback_enabled(context, src, dst)); } TEST_F(CustomHttpOptionsTest, SrcDisabled) { storeConfig(surlToConfigGroup(src), "ENABLE_STREAM_COPY", false); ASSERT_FALSE(is_http_streaming_enabled(context, src, dst)); storeConfig(surlToConfigGroup(src), "ENABLE_FALLBACK_TPC_COPY", false); ASSERT_FALSE(is_http_3rdcopy_fallback_enabled(context, src, dst)); } TEST_F(CustomHttpOptionsTest, DstDisabled) { storeConfig(surlToConfigGroup(dst), "ENABLE_STREAM_COPY", false); ASSERT_FALSE(is_http_streaming_enabled(context, src, dst)); storeConfig(surlToConfigGroup(dst), "ENABLE_FALLBACK_TPC_COPY", false); ASSERT_FALSE(is_http_3rdcopy_fallback_enabled(context, src, dst)); } TEST_F(CustomHttpOptionsTest, SrcEnabled) { storeConfig(surlToConfigGroup(src), "ENABLE_STREAM_COPY", true); ASSERT_TRUE(is_http_streaming_enabled(context, src, dst)); storeConfig(surlToConfigGroup(src), "ENABLE_FALLBACK_TPC_COPY", true); ASSERT_TRUE(is_http_3rdcopy_fallback_enabled(context, src, dst)); } TEST_F(CustomHttpOptionsTest, DstEnabled) { storeConfig(surlToConfigGroup(dst), "ENABLE_STREAM_COPY", true); ASSERT_TRUE(is_http_streaming_enabled(context, src, dst)); storeConfig(surlToConfigGroup(dst), "ENABLE_FALLBACK_TPC_COPY", true); ASSERT_TRUE(is_http_3rdcopy_fallback_enabled(context, src, dst)); } TEST_F(CustomHttpOptionsTest, SrcAndDstDisabled) { storeConfig(surlToConfigGroup(src), "ENABLE_STREAM_COPY", false); storeConfig(surlToConfigGroup(dst), "ENABLE_STREAM_COPY", false); ASSERT_FALSE(is_http_streaming_enabled(context, src, dst)); storeConfig(surlToConfigGroup(src), "ENABLE_FALLBACK_TPC_COPY", false); storeConfig(surlToConfigGroup(dst), "ENABLE_FALLBACK_TPC_COPY", false); ASSERT_FALSE(is_http_3rdcopy_fallback_enabled(context, src, dst)); } TEST_F(CustomHttpOptionsTest, SrcEnabledAndDstDisabled) { storeConfig(surlToConfigGroup(src), "ENABLE_STREAM_COPY", true); storeConfig(surlToConfigGroup(dst), "ENABLE_STREAM_COPY", false); ASSERT_FALSE(is_http_streaming_enabled(context, src, dst)); storeConfig(surlToConfigGroup(src), "ENABLE_FALLBACK_TPC_COPY", true); storeConfig(surlToConfigGroup(dst), "ENABLE_FALLBACK_TPC_COPY", false); ASSERT_FALSE(is_http_3rdcopy_fallback_enabled(context, src, dst)); } TEST_F(CustomHttpOptionsTest, SrcDisabledAndDstEnabled) { storeConfig(surlToConfigGroup(src), "ENABLE_STREAM_COPY", false); storeConfig(surlToConfigGroup(dst), "ENABLE_STREAM_COPY", true); ASSERT_FALSE(is_http_streaming_enabled(context, src, dst)); storeConfig(surlToConfigGroup(src), "ENABLE_FALLBACK_TPC_COPY", false); storeConfig(surlToConfigGroup(dst), "ENABLE_FALLBACK_TPC_COPY", true); ASSERT_FALSE(is_http_3rdcopy_fallback_enabled(context, src, dst)); } TEST_F(CustomHttpOptionsTest, SrcAndDstEnabled) { storeConfig(surlToConfigGroup(src), "ENABLE_STREAM_COPY", true); storeConfig(surlToConfigGroup(dst), "ENABLE_STREAM_COPY", true); ASSERT_TRUE(is_http_streaming_enabled(context, src, dst)); storeConfig(surlToConfigGroup(src), "ENABLE_FALLBACK_TPC_COPY", true); storeConfig(surlToConfigGroup(dst), "ENABLE_FALLBACK_TPC_COPY", true); ASSERT_TRUE(is_http_3rdcopy_fallback_enabled(context, src, dst)); } gfal2-v2.23.0/test/unit/http/test_http_copy_mode.cpp000066400000000000000000000256601465240014500224660ustar00rootroot00000000000000/* * Copyright (c) CERN 2023 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #define __GFAL2_H_INSIDE__ #include #undef __GFAL2_H_INSIDE__ #include #include "plugins/http/gfal_http_plugin.h" using CopyMode = HttpCopyMode::CopyMode; class HttpCopyModeTest: public testing::Test { public: HttpCopyModeTest() = default; ~HttpCopyModeTest() = default; virtual void SetUp() { GError* error = NULL; context = gfal2_context_new(&error); Gfal::gerror_to_cpp(&error); storeConfig("HTTP PLUGIN", "ENABLE_REMOTE_COPY", TRUE); storeConfig("HTTP PLUGIN", "DEFAULT_COPY_MODE", GFAL_TRANSFER_TYPE_PULL); } virtual void TearDown() { gfal2_context_free(context); context = NULL; } protected: gfal2_context_t context = NULL; const char* src = "https://eospublic.cern.ch:443/path/file.src"; const char* dst = "https://eospps.cern.ch:443/path/file.dst"; void storeConfig(const std::string& group, const char* key, int value) { GError* error = NULL; gfal2_set_opt_boolean(context, group.c_str(), key, value, &error); ASSERT_PRED_FORMAT2(AssertGfalSuccess, 0, error); } void storeConfig(const std::string& group, const char* key, const char* value) { GError* error = NULL; gfal2_set_opt_string(context, group.c_str(), key, value, &error); ASSERT_PRED_FORMAT2(AssertGfalSuccess, 0, error); } std::string surlToConfigGroup(const char* surl) { Davix::Uri uri(surl); std::string prot = uri.getProtocol(); if (prot.back() == 's') { prot.pop_back(); } std::string group = prot + ":" + uri.getHost(); std::transform(group.begin(), group.end(), group.begin(), ::toupper); return group; } }; TEST_F(HttpCopyModeTest, Default) { auto copyMode = HttpCopyMode::ConstructCopyMode(context, src, dst); ASSERT_EQ(CopyMode::PULL, copyMode.value()); } TEST_F(HttpCopyModeTest, DefaultConfigOptions) { storeConfig("HTTP PLUGIN", "DEFAULT_COPY_MODE", GFAL_TRANSFER_TYPE_PULL); auto copyMode = HttpCopyMode::ConstructCopyMode(context, src, dst); ASSERT_EQ(CopyMode::PULL, copyMode.value()); storeConfig("HTTP PLUGIN", "DEFAULT_COPY_MODE", GFAL_TRANSFER_TYPE_PUSH); copyMode = HttpCopyMode::ConstructCopyMode(context, src, dst); ASSERT_EQ(CopyMode::PUSH, copyMode.value()); storeConfig("HTTP PLUGIN", "DEFAULT_COPY_MODE", GFAL_TRANSFER_TYPE_STREAMED); copyMode = HttpCopyMode::ConstructCopyMode(context, src, dst); ASSERT_EQ(CopyMode::STREAM, copyMode.value()); ASSERT_TRUE(copyMode.isStreamingOnly()); storeConfig("HTTP PLUGIN", "DEFAULT_COPY_MODE", "bad-value"); copyMode = HttpCopyMode::ConstructCopyMode(context, src, dst); ASSERT_EQ(CopyMode::PULL, copyMode.value()); } TEST_F(HttpCopyModeTest, TPCDisabled) { storeConfig("HTTP PLUGIN", "ENABLE_REMOTE_COPY", FALSE); auto copyMode = HttpCopyMode::ConstructCopyMode(context, src, dst); ASSERT_EQ(CopyMode::STREAM, copyMode.value()); ASSERT_TRUE(copyMode.isStreamingOnly()); } TEST_F(HttpCopyModeTest, StreamingDisabled) { storeConfig("HTTP PLUGIN", "ENABLE_STREAM_COPY", FALSE); auto copyMode = HttpCopyMode::ConstructCopyMode(context, src, dst); ASSERT_EQ(CopyMode::PULL, copyMode.value()); ASSERT_FALSE(copyMode.isStreamingOnly()); } TEST_F(HttpCopyModeTest, TPCIncompatible) { src = "root://eospublic.cern.ch:1094//path/file.src"; auto copyMode = HttpCopyMode::ConstructCopyMode(context, src, dst); ASSERT_EQ(CopyMode::STREAM, copyMode.value()); ASSERT_TRUE(copyMode.isStreamingOnly()); } TEST_F(HttpCopyModeTest, SESpecificSource) { storeConfig(surlToConfigGroup(src), "DEFAULT_COPY_MODE", GFAL_TRANSFER_TYPE_PUSH); auto copyMode = HttpCopyMode::ConstructCopyMode(context, src, dst); ASSERT_EQ(CopyMode::PUSH, copyMode.value()); storeConfig(surlToConfigGroup(src), "DEFAULT_COPY_MODE", GFAL_TRANSFER_TYPE_STREAMED); copyMode = HttpCopyMode::ConstructCopyMode(context, src, dst); ASSERT_EQ(CopyMode::STREAM, copyMode.value()); } TEST_F(HttpCopyModeTest, SESpecificDest) { storeConfig(surlToConfigGroup(dst), "DEFAULT_COPY_MODE", GFAL_TRANSFER_TYPE_PUSH); auto copyMode = HttpCopyMode::ConstructCopyMode(context, src, dst); ASSERT_EQ(CopyMode::PUSH, copyMode.value()); storeConfig(surlToConfigGroup(dst), "DEFAULT_COPY_MODE", GFAL_TRANSFER_TYPE_STREAMED); copyMode = HttpCopyMode::ConstructCopyMode(context, src, dst); ASSERT_EQ(CopyMode::STREAM, copyMode.value()); } TEST_F(HttpCopyModeTest, SESpecificBoth) { storeConfig(surlToConfigGroup(src), "DEFAULT_COPY_MODE", GFAL_TRANSFER_TYPE_PUSH); storeConfig(surlToConfigGroup(dst), "DEFAULT_COPY_MODE", GFAL_TRANSFER_TYPE_STREAMED); auto copyMode = HttpCopyMode::ConstructCopyMode(context, src, dst); ASSERT_EQ(CopyMode::PUSH, copyMode.value()); storeConfig(surlToConfigGroup(src), "DEFAULT_COPY_MODE", GFAL_TRANSFER_TYPE_PULL); storeConfig(surlToConfigGroup(dst), "DEFAULT_COPY_MODE", GFAL_TRANSFER_TYPE_PUSH); copyMode = HttpCopyMode::ConstructCopyMode(context, src, dst); ASSERT_EQ(CopyMode::PULL, copyMode.value()); storeConfig(surlToConfigGroup(src), "DEFAULT_COPY_MODE", GFAL_TRANSFER_TYPE_STREAMED); storeConfig(surlToConfigGroup(dst), "DEFAULT_COPY_MODE", GFAL_TRANSFER_TYPE_PULL); copyMode = HttpCopyMode::ConstructCopyMode(context, src, dst); ASSERT_EQ(CopyMode::STREAM, copyMode.value()); } TEST_F(HttpCopyModeTest, TPCIncompatible_and_SESpecific) { src = "root://eospublic.cern.ch:1094//path/file.src"; storeConfig(surlToConfigGroup(src), "DEFAULT_COPY_MODE", GFAL_TRANSFER_TYPE_PULL); storeConfig(surlToConfigGroup(dst), "DEFAULT_COPY_MODE", GFAL_TRANSFER_TYPE_PUSH); auto copyMode = HttpCopyMode::ConstructCopyMode(context, src, dst); ASSERT_EQ(CopyMode::STREAM, copyMode.value()); ASSERT_TRUE(copyMode.isStreamingOnly()); } TEST_F(HttpCopyModeTest, QueryStringSource) { src = "https://eospublic.cern.ch:443/path/file.src?key=value©_mode=push"; storeConfig(surlToConfigGroup(src), "DEFAULT_COPY_MODE", GFAL_TRANSFER_TYPE_PULL); auto copyMode = HttpCopyMode::ConstructCopyMode(context, src, dst); ASSERT_EQ(CopyMode::PUSH, copyMode.value()); } TEST_F(HttpCopyModeTest, QueryStringDest) { dst = "https://eospps.cern.ch:443/path/file.dst?key=value©_mode=push"; storeConfig(surlToConfigGroup(dst), "DEFAULT_COPY_MODE", GFAL_TRANSFER_TYPE_PULL); auto copyMode = HttpCopyMode::ConstructCopyMode(context, src, dst); ASSERT_EQ(CopyMode::PUSH, copyMode.value()); } TEST_F(HttpCopyModeTest, QueryStringBoth) { src = "https://eospublic.cern.ch:443/path/file.src?key=value©_mode=push"; dst = "https://eospps.cern.ch:443/path/file.dst?key=value©_mode=push"; storeConfig(surlToConfigGroup(src), "DEFAULT_COPY_MODE", GFAL_TRANSFER_TYPE_PULL); storeConfig(surlToConfigGroup(dst), "DEFAULT_COPY_MODE", GFAL_TRANSFER_TYPE_PULL); auto copyMode = HttpCopyMode::ConstructCopyMode(context, src, dst); ASSERT_EQ(CopyMode::PUSH, copyMode.value()); } TEST_F(HttpCopyModeTest, QueryStringInvalid) { src = "https://eospublic.cern.ch:443/path/file.src?key=value©_mode=stream"; dst = "https://eospps.cern.ch:443/path/file.dst?key=value©_mode=stream"; auto copyMode = HttpCopyMode::ConstructCopyMode(context, src, dst); ASSERT_EQ(CopyMode::PULL, copyMode.value()); src = "https://eospublic.cern.ch:443/path/file.src?copy_mode=invalid"; dst = "https://eospps.cern.ch:443/path/file.dst?copy_mode=invalid"; copyMode = HttpCopyMode::ConstructCopyMode(context, src, dst); ASSERT_EQ(CopyMode::PULL, copyMode.value()); } TEST_F(HttpCopyModeTest, QueryString_and_SESpecific) { src = "https://eospublic.cern.ch:443/path/file.src?key=value©_mode=push"; dst = "https://eospps.cern.ch:443/path/file.dst?key=value©_mode=pull"; storeConfig(surlToConfigGroup(src), "DEFAULT_COPY_MODE", GFAL_TRANSFER_TYPE_PULL); storeConfig(surlToConfigGroup(dst), "DEFAULT_COPY_MODE", GFAL_TRANSFER_TYPE_PULL); auto copyMode = HttpCopyMode::ConstructCopyMode(context, src, dst); ASSERT_EQ(CopyMode::PUSH, copyMode.value()); } TEST_F(HttpCopyModeTest, TPCDisabled_and_QueryString) { storeConfig("HTTP PLUGIN", "ENABLE_REMOTE_COPY", FALSE); src = "https://eospublic.cern.ch:443/path/file.src?copy_mode=push"; dst = "https://eospps.cern.ch:443/path/file.dst?copy_mode=push"; auto copyMode = HttpCopyMode::ConstructCopyMode(context, src, dst); ASSERT_EQ(CopyMode::STREAM, copyMode.value()); ASSERT_TRUE(copyMode.isStreamingOnly()); } TEST_F(HttpCopyModeTest, CopyModeToStr) { ASSERT_STREQ(HttpCopyMode::CopyModeToStr(CopyMode::PULL), GFAL_TRANSFER_TYPE_PULL); ASSERT_STREQ(HttpCopyMode::CopyModeToStr(CopyMode::PUSH), GFAL_TRANSFER_TYPE_PUSH); ASSERT_STREQ(HttpCopyMode::CopyModeToStr(CopyMode::STREAM), GFAL_TRANSFER_TYPE_STREAMED); ASSERT_STREQ(HttpCopyMode::CopyModeToStr(CopyMode::NONE), "None"); } TEST_F(HttpCopyModeTest, CopyModeIteration) { auto copyMode = HttpCopyMode::ConstructCopyMode(context, src, dst); ASSERT_EQ(CopyMode::PULL, copyMode.value()); ASSERT_FALSE(copyMode.end()); copyMode.next(); ASSERT_EQ(CopyMode::PUSH, copyMode.value()); ASSERT_FALSE(copyMode.end()); copyMode.next(); ASSERT_EQ(CopyMode::STREAM, copyMode.value()); ASSERT_FALSE(copyMode.end()); copyMode.next(); ASSERT_EQ(CopyMode::NONE, copyMode.value()); ASSERT_TRUE(copyMode.end()); copyMode.next(); ASSERT_EQ(CopyMode::NONE, copyMode.value()); ASSERT_TRUE(copyMode.end()); } TEST_F(HttpCopyModeTest, CopyModeIteration_StreamingDisabled) { storeConfig("HTTP PLUGIN", "ENABLE_STREAM_COPY", FALSE); dst = "https://eospps.cern.ch:443/path/file.dst?key=value©_mode=push"; auto copyMode = HttpCopyMode::ConstructCopyMode(context, src, dst); ASSERT_EQ(CopyMode::PUSH, copyMode.value()); ASSERT_FALSE(copyMode.isStreamingEnabled()); ASSERT_FALSE(copyMode.end()); copyMode.next(); ASSERT_EQ(CopyMode::NONE, copyMode.value()); ASSERT_TRUE(copyMode.end()); } TEST_F(HttpCopyModeTest, CopyModeLoopIteration) { auto copyMode = HttpCopyMode::ConstructCopyMode(context, src, dst); ASSERT_EQ(CopyMode::PULL, copyMode.value()); do { copyMode.next(); } while (!copyMode.end()); ASSERT_TRUE(copyMode.end()); } gfal2-v2.23.0/test/unit/http/test_token_map.cpp000066400000000000000000000237571465240014500214330ustar00rootroot00000000000000/* * Copyright (c) CERN 2021 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #define __GFAL2_H_INSIDE__ #include #undef __GFAL2_H_INSIDE__ #include #include "plugins/http/gfal_http_plugin.h" class TokenMapTest: public testing::Test { public: TokenMapTest() { GError *error = NULL; context = gfal2_context_new(&error); Gfal::gerror_to_cpp(&error); gfal_plugin_interface* p = gfal_find_plugin(context, "https://", GFAL_PLUGIN_TOKEN, &error); Gfal::gerror_to_cpp(&error); httpData = static_cast(gfal_get_plugin_handle(p)); } virtual ~TokenMapTest() { gfal2_context_free(context); } virtual void TearDown() { GError *error = NULL; int ret = gfal2_cred_clean(context, &error); ASSERT_PRED_FORMAT2(AssertGfalSuccess, ret, error); } protected: using OP = GfalHttpPluginData::OP; gfal2_context_t context; GfalHttpPluginData* httpData; void storeInTokenMap(const char* path, const char* token, const OP& operation, bool user_set = false) { GError* error = NULL; gfal2_cred_t* cred = gfal2_cred_new(GFAL_CRED_BEARER, token); gfal2_cred_set(context, path, cred, &error); ASSERT_PRED_FORMAT2(AssertGfalSuccess, 0, error); if (!user_set) { httpData->token_map[std::string(token)] = httpData->writeFlagFromOperation(operation); } } char* findInTokenMap(const char* path, const OP& operation) { return httpData->find_se_token(Davix::Uri(path), operation); } }; TEST_F(TokenMapTest, ReadOperation) { const char* path = "davs://example.cern.ch:443/path/subpath/file"; storeInTokenMap(path, "token_read", OP::READ); ASSERT_STREQ(findInTokenMap(path, OP::READ), "token_read"); } TEST_F(TokenMapTest, WriteOperation) { const char* path = "davs://example.cern.ch:443/path/subpath/file"; storeInTokenMap(path, "token_read", OP::READ); ASSERT_STREQ(findInTokenMap(path, OP::WRITE), nullptr); storeInTokenMap(path, "token_write", OP::WRITE); ASSERT_STREQ(findInTokenMap(path, OP::WRITE), "token_write"); // Write token fulfills a read operation storeInTokenMap(path, "token_write", OP::WRITE); ASSERT_STREQ(findInTokenMap(path, OP::READ), "token_write"); } TEST_F(TokenMapTest, MkColOperation) { const char* path = "davs://example.cern.ch:443/path/subpath/location"; const char* reservedpath = "davs://example.cern.ch:443/path/subpath/location/gfal2_mkdir.reserved"; storeInTokenMap(path, "token_path", OP::MKCOL); storeInTokenMap(reservedpath, "token_reservedpath", OP::MKCOL); ASSERT_STREQ(findInTokenMap(path, OP::MKCOL), "token_reservedpath"); } TEST_F(TokenMapTest, MkColSubPath) { const char* path = "davs://example.cern.ch:443/path/subpath/location"; const char* otherpath = "davs://example.cern.ch:443/path/subpath/file"; const char* subpath = "davs://example.cern.ch:443/path/subpath/subdir/file"; const char* enclosingpath = "davs://example.cern.ch:443/path/subpath/location/file"; const char* enclosingreadpath = "davs://example.cern.ch:443/path/subpath/location/readfile"; storeInTokenMap(path, "token_path", OP::READ); storeInTokenMap(otherpath, "token_otherpath", OP::WRITE); ASSERT_STREQ(findInTokenMap(path, OP::MKCOL), nullptr); storeInTokenMap(path, "token_path", OP::MKCOL); ASSERT_STREQ(findInTokenMap(path, OP::MKCOL), "token_path"); storeInTokenMap(enclosingreadpath, "token_enclosingreadpath", OP::READ); storeInTokenMap(enclosingpath, "token_enclosingpath", OP::WRITE); storeInTokenMap(subpath, "token_subpath", OP::WRITE); ASSERT_STREQ(findInTokenMap(path, OP::MKCOL), "token_enclosingpath"); } TEST_F(TokenMapTest, ParentPath) { const char* path = "davs://example.cern.ch:443/path/subpath/location"; const char* parentpath = "davs://example.cern.ch:443/path/subpath"; const char* rootpath = "davs://example.cern.ch:443/path"; storeInTokenMap(path, "token_path", OP::READ); storeInTokenMap(parentpath, "token_parentpath", OP::READ); storeInTokenMap(rootpath, "token_rootpath", OP::WRITE); ASSERT_STREQ(findInTokenMap(path, OP::READ), "token_path"); ASSERT_STREQ(findInTokenMap(path, OP::WRITE), "token_rootpath"); } TEST_F(TokenMapTest, ParentPathSlashMatch) { const char* path = "davs://example.cern.ch:443/path/subpath/location"; const char* siblingpath = "davs://example.cern.ch:443/path/subpath/location_sibling"; const char* parentpath = "davs://example.cern.ch:443/path/subpath"; const char* parentpath_sibling = "davs://example.cern.ch:443/path/subpath_sibling"; const char* other = "davs://example.cern.ch:443/path/other/location"; const char* parentother = "davs://example.cern.ch:443/path/other/"; const char* parentother_sibling = "davs://example.cern.ch:443/path/other_sibling/"; storeInTokenMap(path, "token_path", OP::READ); storeInTokenMap(siblingpath, "token_siblingpath", OP::READ); storeInTokenMap(parentpath, "token_parentpath", OP::WRITE); storeInTokenMap(parentpath_sibling, "token_parentpath_sibling", OP::WRITE); ASSERT_STREQ(findInTokenMap(path, OP::WRITE), "token_parentpath"); storeInTokenMap(parentother, "token_parentother", OP::READ); storeInTokenMap(parentother_sibling, "token_parentother_sibling", OP::READ); ASSERT_STREQ(findInTokenMap(other, OP::READ), "token_parentother"); } TEST_F(TokenMapTest, HostCred) { const char* path = "davs://example.cern.ch:443/path/subpath/file"; const char* host = "example.cern.ch"; storeInTokenMap(host, "token_host", OP::READ, true); ASSERT_STREQ(findInTokenMap(path, OP::WRITE), "token_host"); ASSERT_STREQ(findInTokenMap(path, OP::READ), "token_host"); } TEST_F(TokenMapTest, UserSetCred) { const char* path = "davs://example.cern.ch:443/path/subpath/file"; const char* pathuser = "davs://example.cern.ch:443/path/subpath"; storeInTokenMap(path, "token_path", OP::READ); storeInTokenMap(pathuser, "token_user", OP::READ, true); ASSERT_STREQ(findInTokenMap(path, OP::WRITE), "token_user"); ASSERT_STREQ(findInTokenMap(path, OP::READ), "token_path"); } TEST_F(TokenMapTest, CopyTest) { const char* source = "davs://source.cern.ch:443/path/subpath/file"; const char* dest = "davs://destination.cern.ch:443/path/subpath/location/file"; const char* destparent_reserved = "davs://destination.cern.ch:443/path/subpath/location/gfal2_mkdir.reserved"; const char* destparent_enoent = "davs://destination.cern.ch:443/path/subpath/location/"; const char* rootparent = "davs://destination.cern.ch:443/path/subpath"; // Stat files storeInTokenMap(source, "token_source", OP::READ); storeInTokenMap(dest, "token_dest_read", OP::READ); ASSERT_STREQ(findInTokenMap(source, OP::READ), "token_source"); ASSERT_STREQ(findInTokenMap(dest, OP::READ), "token_dest_read"); // Stat destination directory ASSERT_STREQ(findInTokenMap(destparent_enoent, OP::HEAD), "token_dest_read"); ASSERT_STREQ(findInTokenMap(rootparent, OP::HEAD), "token_dest_read"); // Mkdir destination directory ASSERT_STREQ(findInTokenMap(destparent_enoent, OP::MKCOL), nullptr); storeInTokenMap(destparent_reserved, "token_destparent_reserved", OP::MKCOL); ASSERT_STREQ(findInTokenMap(rootparent, OP::MKCOL), "token_destparent_reserved"); ASSERT_STREQ(findInTokenMap(destparent_enoent, OP::MKCOL), "token_destparent_reserved"); // Read source file ASSERT_STREQ(findInTokenMap(source, OP::READ), "token_source"); // Write destination file ASSERT_STREQ(findInTokenMap(dest, OP::WRITE), nullptr); storeInTokenMap(dest, "token_dest_write", OP::WRITE); ASSERT_STREQ(findInTokenMap(dest, OP::WRITE), "token_dest_write"); // Stat files ASSERT_STREQ(findInTokenMap(source, OP::HEAD), "token_source"); ASSERT_STREQ(findInTokenMap(dest, OP::HEAD), "token_dest_write"); } TEST_F(TokenMapTest, CopyTestFTSCompatibility) { const char* source = "davs://source.cern.ch:443/path/subpath/file"; const char* dest = "davs://destination.cern.ch:443/path/subpath/location/file"; const char* dest_host = "destination.cern.ch"; const char* destparent_enoent = "davs://destination.cern.ch:443/path/subpath/location/"; const char* rootparent = "davs://destination.cern.ch:443/path/subpath"; // Stat files storeInTokenMap(source, "token_source", OP::READ, true); storeInTokenMap(dest_host, "token_dest_host", OP::WRITE, true); ASSERT_STREQ(findInTokenMap(source, OP::READ), "token_source"); ASSERT_STREQ(findInTokenMap(dest, OP::READ), "token_dest_host"); // Stat destination directory ASSERT_STREQ(findInTokenMap(destparent_enoent, OP::HEAD), "token_dest_host"); ASSERT_STREQ(findInTokenMap(rootparent, OP::HEAD), "token_dest_host"); // Mkdir destination directory ASSERT_STREQ(findInTokenMap(destparent_enoent, OP::MKCOL), "token_dest_host"); ASSERT_STREQ(findInTokenMap(rootparent, OP::MKCOL), "token_dest_host"); ASSERT_STREQ(findInTokenMap(destparent_enoent, OP::MKCOL), "token_dest_host"); // Read source file ASSERT_STREQ(findInTokenMap(source, OP::READ), "token_source"); // Write destination file ASSERT_STREQ(findInTokenMap(dest, OP::WRITE), "token_dest_host"); // Stat files ASSERT_STREQ(findInTokenMap(source, OP::HEAD), "token_source"); ASSERT_STREQ(findInTokenMap(dest, OP::HEAD), "token_dest_host"); } gfal2-v2.23.0/test/unit/mds/000077500000000000000000000000001465240014500155015ustar00rootroot00000000000000gfal2-v2.23.0/test/unit/mds/CMakeLists.txt000066400000000000000000000003761465240014500202470ustar00rootroot00000000000000 find_package (PugiXML) if (PUGIXML_FOUND) add_executable(mds_test "test_mds.cpp") target_link_libraries(mds_test ${GFAL2_LIBRARIES} ${GTEST_LIBRARIES} ${GTEST_MAIN_LIBRARIES} ) add_test(mds_test mds_test) endif (PUGIXML_FOUND) gfal2-v2.23.0/test/unit/mds/test_mds.cpp000066400000000000000000000051261465240014500200330ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include class MdsTestFixture : public ::testing::Test { private: void generate_cache() { std::ofstream cache(MDS_CACHE_FILE, std::ios_base::out | std::ios_base::trunc); cache << "" << std::endl << "" << std::endl << " httpg://test.domain.com:8442/srm/managerv2" << std::endl << " TEST-PROD" << std::endl << " SRM" << std::endl << " 2.2.0" << std::endl << "" << std::endl; cache.flush(); } protected: gfal2_context_t context; public: static const char *MDS_CACHE_FILE; MdsTestFixture() { generate_cache(); GError *error = NULL; context = gfal2_context_new(&error); assert(context != NULL); int ret = gfal2_set_opt_string(context, "BDII", "CACHE_FILE", MDS_CACHE_FILE, NULL); assert(ret == 0); } ~MdsTestFixture() { unlink(MDS_CACHE_FILE); gfal2_context_free(context); } }; const char *MdsTestFixture::MDS_CACHE_FILE = "/tmp/mds_cache.xml"; TEST_F(MdsTestFixture, test_cache_not_found) { gfal_mds_endpoint endpoints[5]; GError* err = NULL; int ret = gfal_mds_cache_resolve_endpoint(context, "noutfound.example.com", endpoints, 5, &err); ASSERT_EQ(err, (void*)NULL); ASSERT_EQ(ret, 0); } TEST_F(MdsTestFixture, test_cache_found) { gfal_mds_endpoint endpoints[5]; GError* err = NULL; int ret = gfal_mds_cache_resolve_endpoint(context, "test.domain.com", endpoints, 5, &err); ASSERT_EQ(err, (void*)NULL); ASSERT_EQ(ret, 1); ASSERT_EQ(endpoints[0].type, SRMv2); ASSERT_STREQ(endpoints[0].url, "httpg://test.domain.com:8442/srm/managerv2"); } gfal2-v2.23.0/test/unit/transfer/000077500000000000000000000000001465240014500165425ustar00rootroot00000000000000gfal2-v2.23.0/test/unit/transfer/CMakeLists.txt000066400000000000000000000011701465240014500213010ustar00rootroot00000000000000if (MAIN_TRANSFER) add_executable (unit_test_transfer_params_exe tests_params.cpp ) target_link_libraries(unit_test_transfer_params_exe ${GFAL2_LIBRARIES} ${GTEST_LIBRARIES} ${GTEST_MAIN_LIBRARIES} m ) add_executable (unit_test_transfer_callbacks_exe tests_callbacks.cpp ) target_link_libraries(unit_test_transfer_callbacks_exe ${GFAL2_LIBRARIES} ${GTEST_LIBRARIES} ${GTEST_MAIN_LIBRARIES} m ) add_test(unit_test_transfer_params unit_test_transfer_params_exe) add_test(unit_test_transfer_callbacks unit_test_transfer_callbacks_exe) endif (MAIN_TRANSFER) gfal2-v2.23.0/test/unit/transfer/tests_callbacks.cpp000066400000000000000000000144731465240014500224200ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include static GQuark domain = g_quark_from_static_string("TEST"); static void monitor_callback_1(gfalt_transfer_status_t h, const char* src, const char* dst, gpointer user_data) { int *data = (int*)(user_data); (*data) += 1; } static void monitor_callback_2(gfalt_transfer_status_t h, const char* src, const char* dst, gpointer user_data) { int *data = (int*)(user_data); (*data) += 10; } static void event_callback_1(const gfalt_event_t e, gpointer user_data) { int *data = (int*)(user_data); (*data) += 1; } static void event_callback_2(const gfalt_event_t e, gpointer user_data) { int *data = (int*)(user_data); (*data) += 10; } static void event_reset_counter(gpointer user_data) { int *data = (int*)(user_data); *data = 0; } TEST(gfalTransfer, test_monitor_callbacks) { int counter1 = 0, counter2 = 0; gfalt_params_t params = gfalt_params_handle_new(NULL); gfalt_add_monitor_callback(params, monitor_callback_1, &counter1, event_reset_counter, NULL); gfalt_add_monitor_callback(params, monitor_callback_2, &counter1, event_reset_counter, NULL); plugin_trigger_monitor(params, NULL, "source", "destination"); g_assert(counter1 == 11); g_assert(counter2 == 0); gfalt_add_monitor_callback(params, monitor_callback_1, &counter2, event_reset_counter, NULL); gfalt_add_monitor_callback(params, monitor_callback_2, &counter2, event_reset_counter, NULL); plugin_trigger_monitor(params, NULL, "source", "destination"); g_assert(counter1 == 0); // Was reset on the second call to gfalt_add_monitor_callback g_assert(counter2 == 11); gfalt_params_handle_delete(params, NULL); g_assert(counter1 == 0); g_assert(counter2 == 0); } TEST(gfalTransfer, test_event_callbacks) { int counter1 = 0, counter2 = 0; gfalt_params_t params = gfalt_params_handle_new(NULL); gfalt_add_event_callback(params, event_callback_1, &counter1, event_reset_counter, NULL); gfalt_add_event_callback(params, event_callback_2, &counter1, event_reset_counter, NULL); plugin_trigger_event(params, domain, GFAL_EVENT_NONE, domain, "TEST"); g_assert(counter1 == 11); g_assert(counter2 == 0); gfalt_add_event_callback(params, event_callback_1, &counter2, event_reset_counter, NULL); gfalt_add_event_callback(params, event_callback_2, &counter2, event_reset_counter, NULL); plugin_trigger_event(params, domain, GFAL_EVENT_NONE, domain, "TEST"); g_assert(counter1 == 0); // Was reset on the second call to gfalt_add_event_callback g_assert(counter2 == 11); gfalt_params_handle_delete(params, NULL); g_assert(counter1 == 0); g_assert(counter2 == 0); } static const char* test_plugin_name() { return "TEST-PLUGIN"; } static int test_plugin_enter_hook(plugin_handle plugin_data, gfal2_context_t context, gfalt_params_t params, GError** error) { gfalt_add_event_callback(params, event_callback_1, plugin_data, NULL, NULL); return 0; } static int test_plugin_check_transfer(plugin_handle plugin_data, gfal2_context_t context, const char* src, const char* dst, gfal_url2_check check) { return 1; } static int test_plugin_copy(plugin_handle plugin_data, gfal2_context_t context, gfalt_params_t params, const char* src, const char* dst, GError**) { plugin_trigger_event(params, domain, GFAL_EVENT_NONE, domain, "TEST"); return 0; } static int test_plugin_bulk(plugin_handle plugin_data, gfal2_context_t context, gfalt_params_t params, size_t nbfiles, const char* const* srcs, const char* const* dsts, const char* const* checksums, GError** op_error, GError*** file_errors) { plugin_trigger_event(params, domain, GFAL_EVENT_NONE, domain, "TEST"); plugin_trigger_event(params, domain, GFAL_EVENT_NONE, domain, "TEST"); return 0; } TEST(gfalTransfer, test_inject_callback) { int counter = 0; gfal_plugin_interface test_plugin; memset(&test_plugin, 0, sizeof(test_plugin)); test_plugin.getName = test_plugin_name; test_plugin.copy_enter_hook = test_plugin_enter_hook; test_plugin.plugin_data = &counter; test_plugin.check_plugin_url_transfer = test_plugin_check_transfer; test_plugin.copy_file = test_plugin_copy; test_plugin.copy_bulk = test_plugin_bulk; gfal2_context_t context = gfal2_context_new(NULL); gfal2_register_plugin(context, &test_plugin, NULL); gfalt_params_t params = gfalt_params_handle_new(NULL); gfalt_copy_file(context, params, "test://", "test://", NULL); g_assert(counter == 4); gfalt_params_handle_delete(params, NULL); } TEST(gfalTransfer, test_inject_callback_bulk) { int counter = 0; gfal_plugin_interface test_plugin; memset(&test_plugin, 0, sizeof(test_plugin)); test_plugin.getName = test_plugin_name; test_plugin.copy_enter_hook = test_plugin_enter_hook; test_plugin.plugin_data = &counter; test_plugin.check_plugin_url_transfer = test_plugin_check_transfer; test_plugin.copy_file = test_plugin_copy; test_plugin.copy_bulk = test_plugin_bulk; gfal2_context_t context = gfal2_context_new(NULL); gfal2_register_plugin(context, &test_plugin, NULL); const char* sources[] = {"test://", "test://"}; const char* destinations[] = {"test://", "test://"}; GError** file_errors; gfalt_params_t params = gfalt_params_handle_new(NULL); gfalt_copy_bulk(context, params, 2, sources, destinations, NULL, NULL, &file_errors); g_assert(counter == 6); gfalt_params_handle_delete(params, NULL); } gfal2-v2.23.0/test/unit/transfer/tests_params.cpp000066400000000000000000000076341465240014500217650ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include TEST(gfalTransfer, testparam){ GError * tmp_err=NULL; gfalt_params_t p = gfalt_params_handle_new(&tmp_err); g_assert(p != NULL); g_assert(tmp_err == NULL); gfalt_params_t p2 = gfalt_params_handle_copy(p, &tmp_err); g_assert(p2 != NULL); g_assert(tmp_err == NULL); gfalt_params_handle_delete(NULL, &tmp_err); g_assert(tmp_err == NULL); gfalt_params_handle_delete(p,&tmp_err); g_assert(tmp_err == NULL); gfalt_params_handle_delete(p2,&tmp_err); g_assert(tmp_err == NULL); } TEST(gfalTransfer, testtimeout){ GError * tmp_err=NULL; gfalt_params_t p = gfalt_params_handle_new(&tmp_err); ASSERT_TRUE( p != NULL && tmp_err==NULL); gfalt_get_timeout(p, &tmp_err); ASSERT_TRUE(tmp_err==NULL); guint64 r = (guint64) rand(); gfalt_set_timeout(p, r, &tmp_err); ASSERT_TRUE( gfalt_get_timeout(p, &tmp_err) == r); gfalt_params_handle_delete(p,NULL); } TEST(gfalTransfer, testnbstream){ GError * tmp_err=NULL; gfalt_params_t p = gfalt_params_handle_new(&tmp_err); ASSERT_TRUE( p != NULL && tmp_err==NULL); gfalt_get_nbstreams(p, &tmp_err); ASSERT_TRUE(tmp_err==NULL); long r = rand(); gfalt_set_nbstreams(p, r, &tmp_err); ASSERT_TRUE( gfalt_get_nbstreams(p, &tmp_err) == r); gfalt_params_handle_delete(p,NULL); } TEST(gfalTransfer, testlocaltransfer){ GError * tmp_err=NULL; gfalt_params_t p = gfalt_params_handle_new(&tmp_err); ASSERT_TRUE( p != NULL && tmp_err==NULL); gboolean res = gfalt_get_local_transfer_perm(p, &tmp_err); ASSERT_TRUE( res == TRUE && tmp_err==NULL); int ret = gfalt_set_local_transfer_perm(p, FALSE, &tmp_err); res = gfalt_get_local_transfer_perm(p, &tmp_err); ASSERT_TRUE( res == FALSE && ret == FALSE && tmp_err==NULL); gfalt_params_handle_delete(p,NULL); } TEST(gfalTransfer, testSciTagMinValue){ GError* tmp_err = NULL; gfalt_params_t p = gfalt_params_handle_new(&tmp_err); ASSERT_TRUE(p != NULL && tmp_err == NULL); gfalt_get_scitag(p, &tmp_err); ASSERT_TRUE(tmp_err == NULL); guint r = (1 << 6); gint res = gfalt_set_scitag(p, r, &tmp_err); ASSERT_TRUE(res == -1); ASSERT_TRUE(tmp_err != NULL); gfalt_params_handle_delete(p, NULL); g_error_free(tmp_err); } TEST(gfalTransfer, testSciTagMaxValue){ GError* tmp_err = NULL; gfalt_params_t p = gfalt_params_handle_new(&tmp_err); ASSERT_TRUE(p != NULL && tmp_err == NULL); gfalt_get_scitag(p, &tmp_err); ASSERT_TRUE(tmp_err == NULL); guint r = (1 << 16); gint res = gfalt_set_scitag(p, r, &tmp_err); ASSERT_TRUE(res == -1); ASSERT_TRUE(tmp_err != NULL); gfalt_params_handle_delete(p, NULL); g_error_free(tmp_err); } TEST(gfalTransfer, testSciTag){ GError* tmp_err = NULL; gfalt_params_t p = gfalt_params_handle_new(&tmp_err); ASSERT_TRUE(p != NULL && tmp_err == NULL); gfalt_get_scitag(p, &tmp_err); ASSERT_TRUE(tmp_err == NULL); guint r = (rand() % (GFAL_SCITAG_MAX_VALUE - GFAL_SCITAG_MIN_VALUE + 1)) + GFAL_SCITAG_MIN_VALUE; gfalt_set_scitag(p, r, &tmp_err); ASSERT_TRUE(tmp_err == NULL); ASSERT_TRUE(gfalt_get_scitag(p, &tmp_err) == r); gfalt_params_handle_delete(p, NULL); } gfal2-v2.23.0/test/unit/uri/000077500000000000000000000000001465240014500155155ustar00rootroot00000000000000gfal2-v2.23.0/test/unit/uri/CMakeLists.txt000066400000000000000000000006461465240014500202630ustar00rootroot00000000000000add_executable(gfal2_test_uri "test_uri.cpp") target_link_libraries(gfal2_test_uri ${GFAL2_LIBRARIES} ${GTEST_LIBRARIES} ${GTEST_MAIN_LIBRARIES} ) add_executable(gfal2_test_parsing "test_parsing.cpp") target_link_libraries(gfal2_test_parsing ${GFAL2_LIBRARIES} ${GTEST_LIBRARIES} ${GTEST_MAIN_LIBRARIES} ) add_test(gfal2_test_uri gfal2_test_uri) add_test(gfal2_test_parsing gfal2_test_parsing) gfal2-v2.23.0/test/unit/uri/test_parsing.cpp000066400000000000000000000044131465240014500207250ustar00rootroot00000000000000/* * Copyright (c) CERN 2022 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include TEST(ParsingTest, CollapseSlashes) { const char* path = "//eos/opstest/dteam/test"; char* collapsed = gfal2_path_collapse_slashes(path); ASSERT_STREQ("//eos/opstest/dteam/test", path); ASSERT_STREQ("/eos/opstest/dteam/test", collapsed); g_free(collapsed); path = "//eos/opstest////dteam/test//"; collapsed = gfal2_path_collapse_slashes(path); ASSERT_STREQ("//eos/opstest////dteam/test//", path); ASSERT_STREQ("/eos/opstest/dteam/test/", collapsed); g_free(collapsed); collapsed = gfal2_path_collapse_slashes("path//not/starting/in/slash///"); ASSERT_STREQ("path/not/starting/in/slash/", collapsed); g_free(collapsed); } TEST(ParsingTest, CollapseSlashesNullInput) { char* collapsed = gfal2_path_collapse_slashes(NULL); ASSERT_EQ(NULL, collapsed); } TEST(ParsingTest, CollapseSlashesEmptyInput) { char* collapsed = gfal2_path_collapse_slashes(""); ASSERT_STREQ("", collapsed); g_free(collapsed); } TEST(ParsingTest, CollapseSlashesStringOutput) { std::string path = "//eos/opstest/dteam/test"; char *collapsed = gfal2_path_collapse_slashes(path.c_str()); path = collapsed; ASSERT_STREQ("/eos/opstest/dteam/test", path.c_str()); ASSERT_STREQ(collapsed, path.c_str()); g_free(collapsed); } TEST(ParsingTest, CollapseSlashesEmptyStringOutput) { char* collapsed = gfal2_path_collapse_slashes(""); std::string path = collapsed; ASSERT_TRUE(path.empty()); g_free(collapsed); std::string sempty; collapsed = gfal2_path_collapse_slashes(sempty.c_str()); sempty = collapsed; ASSERT_TRUE(sempty.empty()); g_free(collapsed); } gfal2-v2.23.0/test/unit/uri/test_uri.cpp000066400000000000000000000134161465240014500200640ustar00rootroot00000000000000/* * Copyright (c) CERN 2013-2017 * * Copyright (c) Members of the EMI Collaboration. 2010-2013 * See http://www.eu-emi.eu/partners for details on the copyright * holders. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include TEST(gfalURI, regular_parsing) { const char *URI = "gsiftp://dcache-door-desy09.desy.de:2811/pnfs/desy.de/dteam/gfal2-tests/testread0011"; GError* tmp_err=NULL; gfal2_uri *parsed = gfal2_parse_uri(URI, &tmp_err); ASSERT_NE(parsed, (void*)NULL); ASSERT_STREQ("gsiftp", parsed->scheme); ASSERT_STREQ("dcache-door-desy09.desy.de", parsed->host); ASSERT_EQ(2811, parsed->port); ASSERT_STREQ("/pnfs/desy.de/dteam/gfal2-tests/testread0011", parsed->path); ASSERT_EQ(NULL, parsed->query); char *rebuilt = gfal2_join_uri(parsed); ASSERT_STREQ(URI, rebuilt); g_free(rebuilt); gfal2_free_uri(parsed); } TEST(gfalURI, malformed) { GError* tmp_err=NULL; gfal2_uri *parsed = gfal2_parse_uri( "malformed", &tmp_err); ASSERT_EQ(parsed->scheme, (void*)NULL); gfal2_free_uri(parsed); } TEST(gfalURI, no_port) { const char *URI = "https://some.domain.com/path"; GError* tmp_err=NULL; gfal2_uri *parsed = gfal2_parse_uri(URI, &tmp_err); ASSERT_NE(parsed, (void*)NULL); ASSERT_STREQ("https", parsed->scheme); ASSERT_STREQ("some.domain.com", parsed->host); ASSERT_EQ(0, parsed->port); ASSERT_STREQ("/path", parsed->path); char *rebuilt = gfal2_join_uri(parsed); ASSERT_STREQ(URI, rebuilt); g_free(rebuilt); gfal2_free_uri(parsed); } TEST(gfalURI, ipv4) { const char *URI = "gsiftp://192.168.1.1:1234/path"; GError* tmp_err=NULL; gfal2_uri *parsed = gfal2_parse_uri(URI, &tmp_err); ASSERT_NE(parsed, (void*)NULL); ASSERT_STREQ("gsiftp", parsed->scheme); ASSERT_STREQ("192.168.1.1", parsed->host); ASSERT_EQ(1234, parsed->port); ASSERT_STREQ("/path", parsed->path); char *rebuilt = gfal2_join_uri(parsed); ASSERT_STREQ(URI, rebuilt); g_free(rebuilt); gfal2_free_uri(parsed); } TEST(gfalURI, ipv6) { const char *URI ="gsiftp://[2001:1458:301:a8ae::100:24]:1234/path"; GError* tmp_err=NULL; gfal2_uri *parsed = gfal2_parse_uri(URI, &tmp_err); ASSERT_NE(parsed, (void*)NULL); ASSERT_STREQ("gsiftp", parsed->scheme); ASSERT_STREQ("[2001:1458:301:a8ae::100:24]", parsed->host); ASSERT_EQ(1234, parsed->port); ASSERT_STREQ("/path", parsed->path); char *rebuilt = gfal2_join_uri(parsed); ASSERT_STREQ(URI, rebuilt); g_free(rebuilt); gfal2_free_uri(parsed); } TEST(gfalURI, userinfo) { const char *URI ="gsiftp://user:patata@[2001:1458:301:a8ae::100:24]:1234/path"; GError* tmp_err=NULL; gfal2_uri *parsed = gfal2_parse_uri(URI, &tmp_err); ASSERT_NE(parsed, (void*)NULL); ASSERT_STREQ("gsiftp", parsed->scheme); ASSERT_STREQ("[2001:1458:301:a8ae::100:24]", parsed->host); ASSERT_EQ(1234, parsed->port); ASSERT_STREQ("/path", parsed->path); ASSERT_STREQ("user:patata", parsed->userinfo); char *rebuilt = gfal2_join_uri(parsed); ASSERT_STREQ(URI, rebuilt); g_free(rebuilt); gfal2_free_uri(parsed); } TEST(gfalURI, fragmentAndQuery) { const char *URI = "gsiftp://host/path?a=b&c=d#fragment"; GError* tmp_err=NULL; gfal2_uri *parsed = gfal2_parse_uri(URI, &tmp_err); ASSERT_NE(parsed, (void*)NULL); ASSERT_STREQ("gsiftp", parsed->scheme); ASSERT_STREQ("host", parsed->host); ASSERT_STREQ("/path", parsed->path); ASSERT_STREQ("a=b&c=d", parsed->query); ASSERT_STREQ("fragment", parsed->fragment); char *rebuilt = gfal2_join_uri(parsed); ASSERT_STREQ(URI, rebuilt); g_free(rebuilt); gfal2_free_uri(parsed); } TEST(gfalURI, file) { const char *URI = "file:///path"; GError* tmp_err=NULL; // With authority gfal2_uri *parsed = gfal2_parse_uri(URI, &tmp_err); ASSERT_NE(parsed, (void*)NULL); ASSERT_STREQ("file", parsed->scheme); ASSERT_STREQ("", parsed->host); ASSERT_STREQ("/path", parsed->path); gfal2_free_uri(parsed); // Without authority parsed = gfal2_parse_uri("file:/path", &tmp_err); ASSERT_NE(parsed, (void*)NULL); ASSERT_STREQ("file", parsed->scheme); ASSERT_STREQ(NULL, parsed->host); ASSERT_STREQ("/path", parsed->path); char *rebuilt = gfal2_join_uri(parsed); ASSERT_STREQ(URI, rebuilt); g_free(rebuilt); gfal2_free_uri(parsed); } TEST(gfalURI, special) { const char *URI = "davs://arioch.cern.ch/dpm/cern.ch/home/dteam/xrd-f1460564827990430820aBc0!@%23f%25%5e_-+=:%20.dat"; GError* tmp_err = NULL; gfal2_uri *parsed = gfal2_parse_uri(URI, &tmp_err); ASSERT_NE(parsed, (void*)NULL); ASSERT_STREQ("davs", parsed->scheme); ASSERT_STREQ("arioch.cern.ch", parsed->host); ASSERT_STREQ("/dpm/cern.ch/home/dteam/xrd-f1460564827990430820aBc0!@%23f%25%5e_-+=:%20.dat", parsed->path); ASSERT_EQ(NULL, parsed->query); ASSERT_EQ(NULL, parsed->fragment); char *rebuilt = gfal2_join_uri(parsed); ASSERT_STREQ(URI, rebuilt); g_free(rebuilt); gfal2_urldecode(parsed->path); ASSERT_STREQ("/dpm/cern.ch/home/dteam/xrd-f1460564827990430820aBc0!@#f%^_-+=: .dat", parsed->path); gfal2_free_uri(parsed); }