ftp-cloudfs-0.35/0000775000175000017500000000000012670503005013251 5ustar jjmjjm00000000000000ftp-cloudfs-0.35/setup.py0000775000175000017500000000251212441557636015005 0ustar jjmjjm00000000000000#!/usr/bin/env python import os from setuptools import setup, find_packages from ftpcloudfs.constants import version def read(fname): full_path = os.path.join(os.path.dirname(__file__), fname) if os.path.exists(fname): return open(full_path).read() else: return "" setup(name='ftp-cloudfs', version=version, download_url="http://pypi.python.org/packages/source/f/ftp-cloudfs/ftp-cloudfs-%s.tar.gz" % (version), description='FTP interface to OpenStack Object Storage (Swift)', author='Chmouel Boudjnah', author_email='chmouel@chmouel.com', url='https://pypi.python.org/pypi/ftp-cloudfs/', long_description = read('README.rst'), license='MIT', include_package_data=True, zip_safe=False, install_requires=['pyftpdlib>=1.3.0', 'python-swiftclient>=2.1.0', 'python-daemon>=1.5.5', 'python-memcached'], scripts=['bin/ftpcloudfs'], packages = find_packages(exclude=['tests',]), tests_require = ["nose"], classifiers = [ 'Development Status :: 4 - Beta', 'Environment :: Console', 'Programming Language :: Python', 'Operating System :: OS Independent', 'Environment :: No Input/Output (Daemon)', 'License :: OSI Approved :: MIT License', ], test_suite = "nose.collector", ) ftp-cloudfs-0.35/ChangeLog0000664000175000017500000003215712670314732015042 0ustar jjmjjm000000000000002016-03-10 Juan J. Martinez * 0.35 - Fix in large object support for swiftclient > 2.6.0. - Added swiftclient version to the FTP banner. 2015-08-20 Juan J. Martinez * 0.34 - Fixed a bug introduced in 0.33 that was preventing 0 byte files to be created. Thanks to Vil for the patch. 2015-07-08 Juan J. Martinez * 0.33 - Fixed a bug in split file support failing to propagate the "insecure" option for self-signed certificates. Thanks to "triton7" for the patch. - Delayed opening connection to Swift when a file is uploaded to avoid connection timeouts in ceirtain conditions. Thanks to Vil for the patch. 2015-02-11 Juan J. Martinez * 0.32 - Fixes to hide-part-dir by John Leach to support any segment format. 2014-12-11 Juan J. Martinez * 0.31 - New permit-foreign-addresses configuration token to expose that functionality from pyftpdlib. This is useful when the control connection is proxified. Check the example configuration file for more information. Thanks to Koert van der Veer for the patch. - swiftclient requirement updated to be >= 2.1.0. - Tokens won't be reused in large files support to avoid broken uploads because of expired tokens. - Fixed a bug in large files support with UTF-8 encoded filenames. - Added error translation in two methods of the file emulation. Accessing an unexistent file was resulting in an unhandled exception instead of sending an FTP error to the client. Thanks to John Leach for the report. - A --config command line option has been added to specify an alternative configuration option that will be used instead of the default one in /etc. 2014-10-06 Juan J. Martinez * 0.30 - Migrated to swiftclient >= 2.0.0 and requests. This means older versions of the library are not supported, so please upgrade. - Added new configuration token "insecure" to allow connections to auth servers with invalid SSL certs (eg, self-signed certificates). This is a potentially disruptive release, please test before upgrading your production systems! 2014-09-19 Juan J. Martinez * 0.29 - Fixed large file support to support non-ascii encodings. Thanks to Édouard Puginier for the report! 2014-08-12 Juan J. Martinez * 0.28.1 - Fixed a bug in syslog logging. 2014-08-12 Juan J. Martinez * 0.28 - Better swift connection management avoiding CLOSE_WAIT problems with long time FTP sessions. - Added new configuration token "rackspace-service-network" to support Rackspace's service network as implemented in swiftclient. 2014-05-21 Juan J. Martinez * 0.27 - Added new configuration token "passive-ports" to expose pyftpdlib's functionality. Check ftpcloudfs.conf.example for details. - Explicitly close connections instead of relying on the garbage collector. Some compatibility code has been added for those using python-swiftclient < 1.9.0. 2014-05-14 Juan J. Martinez * 0.26.2 - Changed setup.py to force swiftclient 1.9.0 as version 2.x uses Requests and we're currently incompatible. 2014-04-10 Juan J. Martinez * 0.26.1 - Fixed a bug in token cache code with auth 2.0 (keystone) when used with memcache. The same username in different tenants could get the same auth token. This bug was introduced in 0.24. Thanks to Igor Belikov for the report and the patch! - Auth 2.0 support in tests has been improved. 2014-03-20 Juan J. Martinez * 0.26 This release improves large file support: - Large file rename (manifest only, not the parts). - Delete large files (manifest and parts are removed). - Hide ".part" directory in directory listings. - keystone auth support in tests Thanks to Sokolov Ilya for his contributions to this release! 2014-02-20 Juan J. Martinez * 0.25.3 This is a small bug-fix release: - Added "requests" library support with python-swiftclient >= 2.0.2. Thanks to Chmouel Boudjnah for the patch! - Fixed a small issue with directory listings and swift 1.9.1, thanks to Pedro Perez for the patch! - Copyright year bump 2013-12-01 Juan J. Martinez * 0.25.1, 0.25.2 - Fixed a bug in ObjectStorageFS when used in "delayed authentication" mode that resulted in a information leak vulnerability in sftp-cloudfs. Under certain conditions it was possible to serve a cached directory listing from a different user. 2013-11-15 Juan J. Martinez * 0.25 - Large file support added. The server will split big files in parts to go over the object size limit of Swift. See 'split-large-files' configuration token. For further info please read: http://docs.openstack.org/developer/swift/overview_large_objects.html 2013-11-08 Juan J. Martinez * 0.24.3 - Updated requirements to pyftpdlib >= 1.3.0. 2013-10-17 Juan J. Martinez * 0.24.2 - Fixed a bug in token cache code. The server could get stuck with a cached invalid token not requesting a new one. 2013-10-12 Juan J. Martinez * 0.24.1 - Fixed a small bug that prevented users with no privileges to list the root directory to access containers with the appropriate ACL. This was affecting sftp-cloudfs mainly as some SFTP clients will perform 'stat' over a directory before 'cd'. The problem was introduced in 0.23. 2013-10-11 Juan J. Martinez * 0.24 - Introduced an auth token cache when memcache is available. Thanks to Vil Surkin for the RFE. 2013-10-02 Juan J. Martinez * 0.23.4 - Fixed a bug listing a directory with more than 10k objects that included a virtual directory ('subdir') as last object of a 10k batch. 2013-09-29 Juan J. Martinez * 0.23.3 - Fixed a bug that was preventing non privileged users to log in. That was introduced in 0.23.2. 2013-09-29 Juan J. Martinez * 0.23.2 - Fixed a bug that was raising an exception at the first swift operation after authentication. 2013-09-08 Juan J. Martinez * 0.23.1 - Fixed a bug that prevented Auth 2.0 (keystone) to work. Thanks to Dmitry (cyberflow) for the report. 2013-08-29 Juan J. Martinez * 0.23 ftpcloudfs ported from python-cloudfiles to python-swiftclient - Using python-swiftfiles instead of the abandoned python-cloudfiles - The code has been tidied up for better maintainability and there are some performance improvements. The functionality should be the same, but please test this release before upgrading your production systems! From now on the project will focus on OpenStack Object Storage (swift); although Rackspace Cloud Files is still compatible. If you want to keep using python-cloudfiles, please use ftp-cloudfs 0.22. 2013-07-11 Juan J. Martinez * 0.22 Cache performance improvements - Cache management has been improved limiting the calls to memcache - Cache entries are now serialized using JSON before being stored in memcache (memcache uses pickle by default) - Enabled cache compression IMPORTANT: this is the last release supporting python-cloudfiles. 2013-06-19 Juan J. Martinez * 0.21.1 Minor fixes and cache regression - Improved logging for socket errors (timeouts mainly) - Fixed cache regression that abused cache flushes when using Memcache - Proper handling of no user/password logins 2013-06-12 Juan J. Martinez * 0.21 Port to pyftpdlib 1.2.0 - Using pyftpdlib's MultiprocessFTPServer instead of our pre-fork model. - 'workers' configuration token has been removed (it had no effect in new pyftpdlib). - fully converted to use unicode (pyftpdlib requirement). 2013-03-15 Juan J. Martinez * 0.20.1 - This will be the last release supporting pyftpdlib <= 0.7.0. pyftpdlib 1.0.1 will work with a warning. 2013-02-07 Juan J. Martinez * 0.20 - Auth 2.0 (OpenStack Identity Service 2.0, aka Keystone) support, thanks to schuermannkl for his help testing the feature 2013-02-05 Juan J. Martinez * 0.19.1 - Minor release to fix Debian packaging support - Copyright year bump 2012-10-30 Juan J. Martinez * 0.19 - Support for manifest file (big files), showing real size/hash - Seek support for read operations (FTP REST + RETR support, AKA resume download) 2012-09-12 Juan J. Martinez * 0.18 - Bug fixes in the code to account the max number of connections per IP, thanks to Konstantin vz'One Enchant 2012-09-11 Juan J. Martinez * 0.17.1 - Bug fix, code cleaning - when the retry code on SSLError fails, an exception was raised making that worker crash; thanks to Konstantin vz'One Enchant for the patch 2012-09-07 Juan J. Martinez * 0.17 - Bug fixes from Maxim Mitroshin - Fixed a race condition when checking the max connections per IP - More efficient shared cache by limiting the number of Memcache connections 2012-09-04 Juan J. Martinez * 0.16 - Improved logging, including formatting, log in, log out and most important commands 2012-08-31 Juan J. Martinez * 0.15 - Bug fixes - Catch SSLErrors on Connection.make_request and retry - Added X-Forwarded-For header to PUT requests 2012-06-22 Juan J. Martinez * 0.14 Support for directory listings with more than 10000 objects 2012-06-07 Juan J. Martinez * 0.13 Shared cache bug fixes and improvements - Bug fixes - the cache failed to invalidate after write operations on the root directory - improved cache invalidation - tests were modified to run properly with a shared cache enabled server 2012-03-28 Juan J. Martinez * 0.12.2 Minor fix (unrequired cache flush) 2012-03-28 Juan J. Martinez * 0.12.1 Minor fixes (README.rst and version number) 2012-03-27 Juan J. Martinez * 0.12 Small fixes and performance improvements - New features - Cache code revisited, including support for a shared cache with Memcache 2012-01-03 Juan J. Martinez * 0.11 Fixes and performance improvements - Bug fixes - use cloudfiles_api_timeout - performance improvements on downloads - improved logging - handle SSL errors 2011-11-24 Juan J. Martinez * 0.10 Fixes, better packaging support and some new features - Bug fixes - Fix pypi tarball - Fix debian packaging - New features - MD5 FTP extension - Max connections per IP limit - X-Forwarded-For header support in API calls 2011-08-17 Nick Craig-Wood * 0.9 Mostly bug fixes - Bug fixes - Fix licence inconsistency - Improve cache management to avoid inconsistencies between workers - Don't allow DELE on directories - Fix masquerade option - Fix user and group - Fix allowing access to a container even if the root is inaccessible 2011-06-17 Nick Craig-Wood * 0.8 Lots of fixes and features after extensive load testing - Bug fixes - Fix cat file bug - Fix exceptions on closing files crashing the daemon - Fix excessive open files by running garbage collector periodically - Fix time parsing (timezone error (times wrong) and floating point parsing) - Fix crash on logging unicode strings - Fix timeout problems under heavy load - New features - Daemon improvements: logging, pidfile, drop privileges, syslog, multiple workers - Verbose switch - Configuration file support - Internal reorganisation to make re-usable and testable ftpcloudfs.fs interface 2011-03-18 Chmouel Boudjnah * 0.7 A lot of improvement and fixes from Nick Craig-Wood nick@craig-wood.com highlights are (see commmit for details) : - Pseudo-hierachical folders/directories supported - Containers show size in listings - Logging in server module - Attempt to catch and translate all cloudfiles errors with correct error numbers - Rename support - Lots of tests 2011-02-19 Chmouel Boudjnah * 0.6 - Daemonize ftp-cloudfs with python-daemon. 2011-02-12 Chmouel Boudjnah * 0.5: A lot of improvement and fixes from Nick Craig-Wood nick@craig-wood.com highlights are : - Make compatible with pyftpd >= 0.6 - Implement file modification times in RackspaceCloudFilesFS.stat - Fix crash if user doesn't have permissions to list containers - Remove custom directory list routines and replace with a listdir cache to improve compatibility - Allow to specify a custom auth url (ie: for OpenStack or UK CloudFiles). 2009-11-02 Chmouel Boudjnah * server.py (RackspaceCloudAuthorizer.validate_authentication): Add servicenet support. ftp-cloudfs-0.35/README.rst0000664000175000017500000002143412670502674014757 0ustar jjmjjm00000000000000================================================= FTP Interface to OpenStack Object Storage (Swift) ================================================= :Homepage: https://pypi.python.org/pypi/ftp-cloudfs/ :Credits: Copyright 2009--2016 Chmouel Boudjnah :Licence: MIT DESCRIPTION =========== ftp-cloudfs is a ftp server acting as a proxy to `OpenStack Object Storage (swift)`_. It allow you to connect via any FTP client to do upload/download or create containers. By default the server will bind to port 2021 which allow to be run as a non root/administrator user. .. _OpenStack Object Storage (Swift): http://launchpad.net/swift It supports pseudo-hierarchical folders/directories as described in the `OpenStack Object Storage API`_. .. _OpenStack Object Storage API: http://docs.openstack.org/openstack-object-storage/developer/content/ REQUIREMENTS ============ - Python 2 >= 2.6 - python-swiftclient >= 2.1.0 - https://github.com/openstack/python-swiftclient/ - pyftpdlib >= 1.3.0 - http://code.google.com/p/pyftpdlib/ - python-daemon >= 1.5.5 - http://pypi.python.org/pypi/python-daemon/ - python-memcache >= 1.45 - http://www.tummy.com/Community/software/python-memcached/ IMPORTANT: pyftpdlib 1.2.0 has a couple of known issues (memory leak, file descriptor leak) and it shouldn't be used in production systems. python-swiftclient 2.x uses Requests and it is currently incompatible with ftp-cloudfs < 0.30. Operating Systems ================= ftp-cloudfs is developed and tested in Ubuntu and Debian Linux distributions but it should work on any Unix-like (including Mac OS X) as long as you install the requirements listed above. INSTALL ======= Use standard setup.py directives ie.:: python setup.py install Or if you have `pip`_ installed you can just run:: pip install ftp-cloudfs which will install ftp-cloudfs with all the required dependencies. We also provide a `requirements.txt` file in case you want to install all the dependencies using `pip` without installing ftp-cloudfs:: pip install -r requirements.txt ftp-cloudfs has been `included in Debian Jessie`_. .. _`pip`: https://pip.pypa.io/ .. _included in Debian Jessie: http://packages.debian.org/jessie/ftp-cloudfs USAGE ====== The install should have created a /usr/bin/ftpcloudfs (or whatever prefix defined in your python distribution or command line arguments) which can be used like this: Usage: ftpcloudfs [options] Options: --version show program's version number and exit -h, --help show this help message and exit -p PORT, --port=PORT Port to bind the server (default: 2021) -b BIND_ADDRESS, --bind-address=BIND_ADDRESS Address to bind (default: 127.0.0.1) -a AUTHURL, --auth-url=AUTHURL Authentication URL (required) --insecure Allow to access servers without checking SSL certs --memcache=MEMCACHE Memcache server(s) to be used for cache (ip:port) -v, --verbose Be verbose on logging -f, --foreground Do not attempt to daemonize but run in foreground -l LOG_FILE, --log-file=LOG_FILE Log File: Default stdout when in foreground --syslog Enable logging to the system logger (daemon facility) --pid-file=PID_FILE Pid file location when in daemon mode --uid=UID UID to drop the privilige to when in daemon mode --gid=GID GID to drop the privilige to when in daemon mode --keystone-auth Use auth 2.0 (Keystone, requires keystoneclient) --keystone-region-name=REGION_NAME Region name to be used in auth 2.0 --keystone-tenant-separator=TENANT_SEPARATOR Character used to separate tenant_name/username in auth 2.0 (default: TENANT.USERNAME) --keystone-service-type=SERVICE_TYPE Service type to be used in auth 2.0 (default: object- store) --keystone-endpoint-type=ENDPOINT_TYPE Endpoint type to be used in auth 2.0 (default: publicURL) The defaults can be changed using a configuration file (by default in /etc/ftpcloudfs.conf). Check the example file included in the package. CACHE MANAGEMENT ================ `OpenStack Object Storage (Swift)`_ is an object storage and not a real file system. This proxy simulates enough file system functionality to be used over FTP, but it has a performance impact. To improve the performance a cache is used. It can be local or external (with Memcache). By default a local cache is used, unless one or more Memcache servers are configured. If you're using just one client the local cache may be fine, but if you're using several connections, configuring an external cache is highly recommended. If an external cache is available it will be used to cache authentication tokens too so any Memcache server must be secured to prevent unauthorized access as it could be possible to associate a token with a specific user (not trivial) or even use the cache key (MD5 hash) to brute-force the user password. AUTH 2.0 ======== By default ftp-cloudfs will use Swift auth 1.0, that is compatible with `OpenStack Object Storage` using `swauth`_ auth middleware and Swift implementations such as `Rackspace Cloud Files` or `Memset's Memstore Cloud Storage`. Optionally `OpenStack Identity Service 2.0`_ can be used. Currently python-keystoneclient (0.3.2+ recommended) is required to use auth 2.0 and it can be enabled with ``keystone-auth`` option. You can provide a tenant name in the FTP login user with TENANT.USERNAME (using a dot as separator). Please check the example configuration file for further details. .. _swauth: https://github.com/gholt/swauth .. _OpenStack Identity Service 2.0: http://docs.openstack.org/api/openstack-identity-service/2.0/content/index.html .. _RackSpace Cloud Files: http://www.rackspace.com/cloud/cloud_hosting_products/files/ .. _Memset's Memstore Cloud Storage: https://www.memset.com/cloud/storage/ LARGE FILE SUPPORT ================== The object storage has a limit on the size of a single uploaded object (by default this is 5GB). Files larger than that can be split in parts and merged back on the fly using a manifest file. ftp-cloudfs supports this transparently with the *split-large-files* configuration token, setting it to the number of megabytes wanted to use for each part (disabled by default). When a *FILE* is larger than the specified amount of MB, a *FILE.part* directory will be created and *n* parts will be created splitting the file automatically. The original file name will be used to store the manifest. If the original file is downloaded, the parts will be served as it was a single file. The *FILE.part* directory can be removed from directory listings using the *hide-part-dir* configuration token. Please be aware that the directory will still be visible when accessing the storage using swift API. SUPPORT ======= The project website is at: https://github.com/cloudfs/ftp-cloudfs/issues There you can file bug reports, ask for help or contribute patches. There's additional information at: https://github.com/cloudfs/ftp-cloudfs/wiki LICENSE ======= Unless otherwise noted, all files are released under the `MIT`_ license, exceptions contain licensing information in them. .. _`MIT`: http://en.wikipedia.org/wiki/MIT_License Copyright (C) 2009-2016 Chmouel Boudjnah Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. Authors ======= - Chmouel Boudjnah - Nick Craig-Wood - Juan J. Martinez Contributors ============ - Christophe Le Guern - Konstantin vz'One Enchant - Maxim Mitroshin - Sokolov Ilya - John Leach - Vil Surkin ftp-cloudfs-0.35/PKG-INFO0000664000175000017500000002630412670503005014353 0ustar jjmjjm00000000000000Metadata-Version: 1.1 Name: ftp-cloudfs Version: 0.35 Summary: FTP interface to OpenStack Object Storage (Swift) Home-page: https://pypi.python.org/pypi/ftp-cloudfs/ Author: Chmouel Boudjnah Author-email: chmouel@chmouel.com License: MIT Download-URL: http://pypi.python.org/packages/source/f/ftp-cloudfs/ftp-cloudfs-0.35.tar.gz Description: ================================================= FTP Interface to OpenStack Object Storage (Swift) ================================================= :Homepage: https://pypi.python.org/pypi/ftp-cloudfs/ :Credits: Copyright 2009--2016 Chmouel Boudjnah :Licence: MIT DESCRIPTION =========== ftp-cloudfs is a ftp server acting as a proxy to `OpenStack Object Storage (swift)`_. It allow you to connect via any FTP client to do upload/download or create containers. By default the server will bind to port 2021 which allow to be run as a non root/administrator user. .. _OpenStack Object Storage (Swift): http://launchpad.net/swift It supports pseudo-hierarchical folders/directories as described in the `OpenStack Object Storage API`_. .. _OpenStack Object Storage API: http://docs.openstack.org/openstack-object-storage/developer/content/ REQUIREMENTS ============ - Python 2 >= 2.6 - python-swiftclient >= 2.1.0 - https://github.com/openstack/python-swiftclient/ - pyftpdlib >= 1.3.0 - http://code.google.com/p/pyftpdlib/ - python-daemon >= 1.5.5 - http://pypi.python.org/pypi/python-daemon/ - python-memcache >= 1.45 - http://www.tummy.com/Community/software/python-memcached/ IMPORTANT: pyftpdlib 1.2.0 has a couple of known issues (memory leak, file descriptor leak) and it shouldn't be used in production systems. python-swiftclient 2.x uses Requests and it is currently incompatible with ftp-cloudfs < 0.30. Operating Systems ================= ftp-cloudfs is developed and tested in Ubuntu and Debian Linux distributions but it should work on any Unix-like (including Mac OS X) as long as you install the requirements listed above. INSTALL ======= Use standard setup.py directives ie.:: python setup.py install Or if you have `pip`_ installed you can just run:: pip install ftp-cloudfs which will install ftp-cloudfs with all the required dependencies. We also provide a `requirements.txt` file in case you want to install all the dependencies using `pip` without installing ftp-cloudfs:: pip install -r requirements.txt ftp-cloudfs has been `included in Debian Jessie`_. .. _`pip`: https://pip.pypa.io/ .. _included in Debian Jessie: http://packages.debian.org/jessie/ftp-cloudfs USAGE ====== The install should have created a /usr/bin/ftpcloudfs (or whatever prefix defined in your python distribution or command line arguments) which can be used like this: Usage: ftpcloudfs [options] Options: --version show program's version number and exit -h, --help show this help message and exit -p PORT, --port=PORT Port to bind the server (default: 2021) -b BIND_ADDRESS, --bind-address=BIND_ADDRESS Address to bind (default: 127.0.0.1) -a AUTHURL, --auth-url=AUTHURL Authentication URL (required) --insecure Allow to access servers without checking SSL certs --memcache=MEMCACHE Memcache server(s) to be used for cache (ip:port) -v, --verbose Be verbose on logging -f, --foreground Do not attempt to daemonize but run in foreground -l LOG_FILE, --log-file=LOG_FILE Log File: Default stdout when in foreground --syslog Enable logging to the system logger (daemon facility) --pid-file=PID_FILE Pid file location when in daemon mode --uid=UID UID to drop the privilige to when in daemon mode --gid=GID GID to drop the privilige to when in daemon mode --keystone-auth Use auth 2.0 (Keystone, requires keystoneclient) --keystone-region-name=REGION_NAME Region name to be used in auth 2.0 --keystone-tenant-separator=TENANT_SEPARATOR Character used to separate tenant_name/username in auth 2.0 (default: TENANT.USERNAME) --keystone-service-type=SERVICE_TYPE Service type to be used in auth 2.0 (default: object- store) --keystone-endpoint-type=ENDPOINT_TYPE Endpoint type to be used in auth 2.0 (default: publicURL) The defaults can be changed using a configuration file (by default in /etc/ftpcloudfs.conf). Check the example file included in the package. CACHE MANAGEMENT ================ `OpenStack Object Storage (Swift)`_ is an object storage and not a real file system. This proxy simulates enough file system functionality to be used over FTP, but it has a performance impact. To improve the performance a cache is used. It can be local or external (with Memcache). By default a local cache is used, unless one or more Memcache servers are configured. If you're using just one client the local cache may be fine, but if you're using several connections, configuring an external cache is highly recommended. If an external cache is available it will be used to cache authentication tokens too so any Memcache server must be secured to prevent unauthorized access as it could be possible to associate a token with a specific user (not trivial) or even use the cache key (MD5 hash) to brute-force the user password. AUTH 2.0 ======== By default ftp-cloudfs will use Swift auth 1.0, that is compatible with `OpenStack Object Storage` using `swauth`_ auth middleware and Swift implementations such as `Rackspace Cloud Files` or `Memset's Memstore Cloud Storage`. Optionally `OpenStack Identity Service 2.0`_ can be used. Currently python-keystoneclient (0.3.2+ recommended) is required to use auth 2.0 and it can be enabled with ``keystone-auth`` option. You can provide a tenant name in the FTP login user with TENANT.USERNAME (using a dot as separator). Please check the example configuration file for further details. .. _swauth: https://github.com/gholt/swauth .. _OpenStack Identity Service 2.0: http://docs.openstack.org/api/openstack-identity-service/2.0/content/index.html .. _RackSpace Cloud Files: http://www.rackspace.com/cloud/cloud_hosting_products/files/ .. _Memset's Memstore Cloud Storage: https://www.memset.com/cloud/storage/ LARGE FILE SUPPORT ================== The object storage has a limit on the size of a single uploaded object (by default this is 5GB). Files larger than that can be split in parts and merged back on the fly using a manifest file. ftp-cloudfs supports this transparently with the *split-large-files* configuration token, setting it to the number of megabytes wanted to use for each part (disabled by default). When a *FILE* is larger than the specified amount of MB, a *FILE.part* directory will be created and *n* parts will be created splitting the file automatically. The original file name will be used to store the manifest. If the original file is downloaded, the parts will be served as it was a single file. The *FILE.part* directory can be removed from directory listings using the *hide-part-dir* configuration token. Please be aware that the directory will still be visible when accessing the storage using swift API. SUPPORT ======= The project website is at: https://github.com/cloudfs/ftp-cloudfs/issues There you can file bug reports, ask for help or contribute patches. There's additional information at: https://github.com/cloudfs/ftp-cloudfs/wiki LICENSE ======= Unless otherwise noted, all files are released under the `MIT`_ license, exceptions contain licensing information in them. .. _`MIT`: http://en.wikipedia.org/wiki/MIT_License Copyright (C) 2009-2016 Chmouel Boudjnah Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. Authors ======= - Chmouel Boudjnah - Nick Craig-Wood - Juan J. Martinez Contributors ============ - Christophe Le Guern - Konstantin vz'One Enchant - Maxim Mitroshin - Sokolov Ilya - John Leach - Vil Surkin Platform: UNKNOWN Classifier: Development Status :: 4 - Beta Classifier: Environment :: Console Classifier: Programming Language :: Python Classifier: Operating System :: OS Independent Classifier: Environment :: No Input/Output (Daemon) Classifier: License :: OSI Approved :: MIT License ftp-cloudfs-0.35/MANIFEST.in0000664000175000017500000000006612321526605015015 0ustar jjmjjm00000000000000include README.rst ftpcloudfs.conf.example ChangeLog ftp-cloudfs-0.35/ftp_cloudfs.egg-info/0000775000175000017500000000000012670503005017253 5ustar jjmjjm00000000000000ftp-cloudfs-0.35/ftp_cloudfs.egg-info/not-zip-safe0000664000175000017500000000000112222277405021506 0ustar jjmjjm00000000000000 ftp-cloudfs-0.35/ftp_cloudfs.egg-info/SOURCES.txt0000664000175000017500000000075012670503005021141 0ustar jjmjjm00000000000000ChangeLog MANIFEST.in README.rst ftpcloudfs.conf.example setup.py bin/ftpcloudfs ftp_cloudfs.egg-info/PKG-INFO ftp_cloudfs.egg-info/SOURCES.txt ftp_cloudfs.egg-info/dependency_links.txt ftp_cloudfs.egg-info/not-zip-safe ftp_cloudfs.egg-info/requires.txt ftp_cloudfs.egg-info/top_level.txt ftpcloudfs/__init__.py ftpcloudfs/chunkobject.py ftpcloudfs/constants.py ftpcloudfs/errors.py ftpcloudfs/fs.py ftpcloudfs/main.py ftpcloudfs/monkeypatching.py ftpcloudfs/server.py ftpcloudfs/utils.pyftp-cloudfs-0.35/ftp_cloudfs.egg-info/PKG-INFO0000664000175000017500000002630412670503004020354 0ustar jjmjjm00000000000000Metadata-Version: 1.1 Name: ftp-cloudfs Version: 0.35 Summary: FTP interface to OpenStack Object Storage (Swift) Home-page: https://pypi.python.org/pypi/ftp-cloudfs/ Author: Chmouel Boudjnah Author-email: chmouel@chmouel.com License: MIT Download-URL: http://pypi.python.org/packages/source/f/ftp-cloudfs/ftp-cloudfs-0.35.tar.gz Description: ================================================= FTP Interface to OpenStack Object Storage (Swift) ================================================= :Homepage: https://pypi.python.org/pypi/ftp-cloudfs/ :Credits: Copyright 2009--2016 Chmouel Boudjnah :Licence: MIT DESCRIPTION =========== ftp-cloudfs is a ftp server acting as a proxy to `OpenStack Object Storage (swift)`_. It allow you to connect via any FTP client to do upload/download or create containers. By default the server will bind to port 2021 which allow to be run as a non root/administrator user. .. _OpenStack Object Storage (Swift): http://launchpad.net/swift It supports pseudo-hierarchical folders/directories as described in the `OpenStack Object Storage API`_. .. _OpenStack Object Storage API: http://docs.openstack.org/openstack-object-storage/developer/content/ REQUIREMENTS ============ - Python 2 >= 2.6 - python-swiftclient >= 2.1.0 - https://github.com/openstack/python-swiftclient/ - pyftpdlib >= 1.3.0 - http://code.google.com/p/pyftpdlib/ - python-daemon >= 1.5.5 - http://pypi.python.org/pypi/python-daemon/ - python-memcache >= 1.45 - http://www.tummy.com/Community/software/python-memcached/ IMPORTANT: pyftpdlib 1.2.0 has a couple of known issues (memory leak, file descriptor leak) and it shouldn't be used in production systems. python-swiftclient 2.x uses Requests and it is currently incompatible with ftp-cloudfs < 0.30. Operating Systems ================= ftp-cloudfs is developed and tested in Ubuntu and Debian Linux distributions but it should work on any Unix-like (including Mac OS X) as long as you install the requirements listed above. INSTALL ======= Use standard setup.py directives ie.:: python setup.py install Or if you have `pip`_ installed you can just run:: pip install ftp-cloudfs which will install ftp-cloudfs with all the required dependencies. We also provide a `requirements.txt` file in case you want to install all the dependencies using `pip` without installing ftp-cloudfs:: pip install -r requirements.txt ftp-cloudfs has been `included in Debian Jessie`_. .. _`pip`: https://pip.pypa.io/ .. _included in Debian Jessie: http://packages.debian.org/jessie/ftp-cloudfs USAGE ====== The install should have created a /usr/bin/ftpcloudfs (or whatever prefix defined in your python distribution or command line arguments) which can be used like this: Usage: ftpcloudfs [options] Options: --version show program's version number and exit -h, --help show this help message and exit -p PORT, --port=PORT Port to bind the server (default: 2021) -b BIND_ADDRESS, --bind-address=BIND_ADDRESS Address to bind (default: 127.0.0.1) -a AUTHURL, --auth-url=AUTHURL Authentication URL (required) --insecure Allow to access servers without checking SSL certs --memcache=MEMCACHE Memcache server(s) to be used for cache (ip:port) -v, --verbose Be verbose on logging -f, --foreground Do not attempt to daemonize but run in foreground -l LOG_FILE, --log-file=LOG_FILE Log File: Default stdout when in foreground --syslog Enable logging to the system logger (daemon facility) --pid-file=PID_FILE Pid file location when in daemon mode --uid=UID UID to drop the privilige to when in daemon mode --gid=GID GID to drop the privilige to when in daemon mode --keystone-auth Use auth 2.0 (Keystone, requires keystoneclient) --keystone-region-name=REGION_NAME Region name to be used in auth 2.0 --keystone-tenant-separator=TENANT_SEPARATOR Character used to separate tenant_name/username in auth 2.0 (default: TENANT.USERNAME) --keystone-service-type=SERVICE_TYPE Service type to be used in auth 2.0 (default: object- store) --keystone-endpoint-type=ENDPOINT_TYPE Endpoint type to be used in auth 2.0 (default: publicURL) The defaults can be changed using a configuration file (by default in /etc/ftpcloudfs.conf). Check the example file included in the package. CACHE MANAGEMENT ================ `OpenStack Object Storage (Swift)`_ is an object storage and not a real file system. This proxy simulates enough file system functionality to be used over FTP, but it has a performance impact. To improve the performance a cache is used. It can be local or external (with Memcache). By default a local cache is used, unless one or more Memcache servers are configured. If you're using just one client the local cache may be fine, but if you're using several connections, configuring an external cache is highly recommended. If an external cache is available it will be used to cache authentication tokens too so any Memcache server must be secured to prevent unauthorized access as it could be possible to associate a token with a specific user (not trivial) or even use the cache key (MD5 hash) to brute-force the user password. AUTH 2.0 ======== By default ftp-cloudfs will use Swift auth 1.0, that is compatible with `OpenStack Object Storage` using `swauth`_ auth middleware and Swift implementations such as `Rackspace Cloud Files` or `Memset's Memstore Cloud Storage`. Optionally `OpenStack Identity Service 2.0`_ can be used. Currently python-keystoneclient (0.3.2+ recommended) is required to use auth 2.0 and it can be enabled with ``keystone-auth`` option. You can provide a tenant name in the FTP login user with TENANT.USERNAME (using a dot as separator). Please check the example configuration file for further details. .. _swauth: https://github.com/gholt/swauth .. _OpenStack Identity Service 2.0: http://docs.openstack.org/api/openstack-identity-service/2.0/content/index.html .. _RackSpace Cloud Files: http://www.rackspace.com/cloud/cloud_hosting_products/files/ .. _Memset's Memstore Cloud Storage: https://www.memset.com/cloud/storage/ LARGE FILE SUPPORT ================== The object storage has a limit on the size of a single uploaded object (by default this is 5GB). Files larger than that can be split in parts and merged back on the fly using a manifest file. ftp-cloudfs supports this transparently with the *split-large-files* configuration token, setting it to the number of megabytes wanted to use for each part (disabled by default). When a *FILE* is larger than the specified amount of MB, a *FILE.part* directory will be created and *n* parts will be created splitting the file automatically. The original file name will be used to store the manifest. If the original file is downloaded, the parts will be served as it was a single file. The *FILE.part* directory can be removed from directory listings using the *hide-part-dir* configuration token. Please be aware that the directory will still be visible when accessing the storage using swift API. SUPPORT ======= The project website is at: https://github.com/cloudfs/ftp-cloudfs/issues There you can file bug reports, ask for help or contribute patches. There's additional information at: https://github.com/cloudfs/ftp-cloudfs/wiki LICENSE ======= Unless otherwise noted, all files are released under the `MIT`_ license, exceptions contain licensing information in them. .. _`MIT`: http://en.wikipedia.org/wiki/MIT_License Copyright (C) 2009-2016 Chmouel Boudjnah Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. Authors ======= - Chmouel Boudjnah - Nick Craig-Wood - Juan J. Martinez Contributors ============ - Christophe Le Guern - Konstantin vz'One Enchant - Maxim Mitroshin - Sokolov Ilya - John Leach - Vil Surkin Platform: UNKNOWN Classifier: Development Status :: 4 - Beta Classifier: Environment :: Console Classifier: Programming Language :: Python Classifier: Operating System :: OS Independent Classifier: Environment :: No Input/Output (Daemon) Classifier: License :: OSI Approved :: MIT License ftp-cloudfs-0.35/ftp_cloudfs.egg-info/dependency_links.txt0000664000175000017500000000000112670503004023320 0ustar jjmjjm00000000000000 ftp-cloudfs-0.35/ftp_cloudfs.egg-info/requires.txt0000664000175000017500000000012012670503004021643 0ustar jjmjjm00000000000000pyftpdlib>=1.3.0 python-swiftclient>=2.1.0 python-daemon>=1.5.5 python-memcachedftp-cloudfs-0.35/ftp_cloudfs.egg-info/top_level.txt0000664000175000017500000000001312670503004021776 0ustar jjmjjm00000000000000ftpcloudfs ftp-cloudfs-0.35/bin/0000775000175000017500000000000012670503005014021 5ustar jjmjjm00000000000000ftp-cloudfs-0.35/bin/ftpcloudfs0000775000175000017500000000050112211103543016106 0ustar jjmjjm00000000000000#!/usr/bin/env python """ FTP Cloud Files - A FTP Proxy to Cloud FIles. """ import sys import os sys.path.append( os.path.abspath( os.path.join( os.path.dirname( sys.argv[0]), ".."))) from ftpcloudfs.main import Main if __name__ == '__main__': main = Main() main.main() ftp-cloudfs-0.35/ftpcloudfs/0000775000175000017500000000000012670503005015422 5ustar jjmjjm00000000000000ftp-cloudfs-0.35/ftpcloudfs/chunkobject.py0000664000175000017500000000660012565322440020302 0ustar jjmjjm00000000000000 import logging from urllib import quote from httplib import HTTPException from socket import timeout from ssl import SSLError from swiftclient.client import ClientException, http_connection from ftpcloudfs.utils import smart_str class ChunkObject(object): def __init__(self, conn, container, name, content_type=None, reuse_token = True): self.raw_conn = None if reuse_token: self.url = conn.url token = conn.token else: self.url, token = conn.get_auth() self.parsed, self.conn = http_connection(self.url) self.path = '%s/%s/%s' % (self.parsed.path.rstrip('/'), quote(smart_str(container)), quote(smart_str(name)), ) self.headers = { 'X-Auth-Token': token, 'Content-Type': content_type or 'application/octet-stream', 'Transfer-Encoding': 'chunked', 'Connection': 'close', # User-Agent ? } if conn.real_ip: self.headers['X-Forwarded-For'] = conn.real_ip logging.debug("ChunkedObject: path=%r, headers=%r" % (self.path, self.headers)) self.already_sent = 0 def _open_connection(self): logging.debug("ChunkObject: new connection open (%r, %r)" % (self.parsed, self.conn)) # we can't use the generator interface offered by requests to do a # chunked transfer encoded PUT, so we do this is to get control over the # "real" http connection and do the HTTP request ourselves self.raw_conn = self.conn.request_session.get_adapter(self.url).get_connection(self.url)._get_conn() self.raw_conn.putrequest('PUT', self.path, skip_accept_encoding=True) for key, value in self.headers.iteritems(): self.raw_conn.putheader(key, value) self.raw_conn.endheaders() def send_chunk(self, chunk): if self.raw_conn is None: self._open_connection() logging.debug("ChunkObject: sending %s bytes" % len(chunk)) try: self.raw_conn.send("%X\r\n" % len(chunk)) self.raw_conn.send(chunk) self.raw_conn.send("\r\n") except (timeout, SSLError, HTTPException), err: raise ClientException(err.message) else: self.already_sent += len(chunk) logging.debug("ChunkObject: already sent %s bytes" % self.already_sent) def finish_chunk(self): if self.raw_conn is None: self._open_connection() logging.debug("ChunkObject: finish_chunk") try: self.raw_conn.send("0\r\n\r\n") response = self.raw_conn.getresponse() except (timeout, SSLError, HTTPException), err: self.raw_conn.close() raise ClientException(err.message) try: response.read() except (timeout, SSLError): # this is not relevant, keep going pass # we always close the connection self.raw_conn.close() self.conn.request_session.close() if response.status // 100 != 2: raise ClientException(response.reason, http_status=response.status, http_reason=response.reason, ) ftp-cloudfs-0.35/ftpcloudfs/fs.py0000664000175000017500000012127412670255110016414 0ustar jjmjjm00000000000000""" A filesystem like interface to an object storage. Authors: Chmouel Boudjnah Nick Craig-Wood Juan J. Martinez """ import os import sys import time import mimetypes import stat import logging from urllib import unquote from errno import EPERM, ENOENT, EACCES, EIO, ENOTDIR, ENOTEMPTY from swiftclient.client import Connection, ClientException, quote from chunkobject import ChunkObject from errors import IOSError import posixpath from utils import smart_str, smart_unicode from functools import wraps import memcache import multiprocessing try: from hashlib import md5 except ImportError: from md5 import md5 try: import json except ImportError: import simplejson as json __all__ = ['ObjectStorageFS'] class ProxyConnection(Connection): """ Add X-Forwarded-For header to all requests. """ # max time to cache auth tokens (seconds), based on swift defaults TOKEN_TTL = 86400 def __init__(self, memcache, *args, **kwargs): self.memcache = memcache self.real_ip = None self.ignore_auth_cache = False self.tenant_name = None if kwargs.get('auth_version') == "2.0": self.tenant_name = kwargs['tenant_name'] super(ProxyConnection, self).__init__(*args, **kwargs) def http_connection(self): def request_wrapper(fn): @wraps(fn) def request_header_injection(method, url, data=None, headers=None): if headers is None: headers = {} if self.real_ip: headers['X-Forwarded-For'] = self.real_ip fn(method, url, data=data, headers=headers) return request_header_injection parsed, conn = super(ProxyConnection, self).http_connection() conn.request = request_wrapper(conn.request) return parsed, conn def close(self): """Our own close that actually closes the connection""" if self.http_conn and type(self.http_conn) is tuple and len(self.http_conn) > 1: conn = self.http_conn[1] if hasattr(conn, "request_session"): conn.request_session.close() self.http_conn = None else: super(ProxyConnection, self).close() def get_auth(self): """Perform the authentication using a token cache if memcache is available""" if self.memcache: tenant_name = self.tenant_name or "-" key = "tk%s" % md5("%s%s%s%s" % (self.authurl, self.user, tenant_name, self.key)).hexdigest() cache = self.memcache.get(key) if not cache or self.ignore_auth_cache: logging.debug("token cache miss, key=%s" % key) cache = super(ProxyConnection, self).get_auth() self.memcache.set(key, cache, self.TOKEN_TTL) self.ignore_auth_cache = False else: logging.debug("token cache hit, key=%s" % key) self.ignore_auth_cache = True return cache # no memcache return super(ProxyConnection, self).get_auth() def translate_objectstorage_error(fn): """ Decorator to catch Object Storage errors and translating them into IOSError. Other exceptions are not caught. """ @wraps(fn) def wrapper(*args,**kwargs): name = getattr(fn, "func_name", "unknown") log = lambda msg: logging.debug("At %s: %s" % (name, msg)) try: return fn(*args, **kwargs) except ClientException, e: # some errno mapping if e.http_status == 404: err = ENOENT elif e.http_status == 400: err = EPERM elif e.http_status == 403: err = EACCES else: err = EIO msg = "%s: %s" % (smart_str(e.msg), smart_str(e.http_reason)) log(msg) raise IOSError(err, msg) return wrapper def close_when_done(fn): """ Decorator to close swift connection when the ftp command is done. """ @wraps(fn) def wrapper(obj, *args, **kwargs): try: return fn(obj, *args, **kwargs) finally: if obj.conn: obj.close() return wrapper def parse_fspath(path): """ Returns a (container, path) tuple. For shorter paths replaces not provided values with empty strings. May raise IOSError for invalid paths. """ if not path.startswith('/'): logging.warning('parse_fspath: You have to provide an absolute path: %r' % path) raise IOSError(ENOENT, 'Absolute path needed') parts = path.split('/', 2)[1:] while len(parts) < 2: parts.append('') return tuple(parts) class ObjectStorageFD(object): """File alike object attached to the Object Storage.""" split_size = 0 def __init__(self, connection, container, obj, mode): self.conn = connection self.container = container self.name = obj self.mode = mode self.closed = False self.total_size = 0 self.part_size = 0 self.part = 0 self.headers = dict() self.content_type = mimetypes.guess_type(self.name)[0] self.pending_copy_task = None self.obj = None # this is only used by `seek`, so we delay the HEAD request until is required self.size = None if not all([container, obj]): self.closed = True raise IOSError(EPERM, 'Container and object required') logging.debug("ObjectStorageFD object: %r (mode: %r)" % (obj, mode)) if 'r' in self.mode: logging.debug("read fd %r" % self.name) else: # write logging.debug("write fd %r" % self.name) self.obj = ChunkObject(self.conn, self.container, self.name, content_type=self.content_type) @property def part_base_name(self): return "%s.part" % self.name @property def part_name(self): return "%s/%.6d" % (self.part_base_name, self.part) def _start_copy_task(self): """ Copy the first part of a multi-part file to its final location and create the manifest file. This happens in the background, pending_copy_task must be cleaned up at the end. """ def copy_task(conn, container, name, part_name, part_base_name): # open a new connection url, token = conn.get_auth() conn = ProxyConnection(None, preauthurl=url, preauthtoken=token, insecure=conn.insecure) headers = { 'x-copy-from': quote("/%s/%s" % (container, name)) } logging.debug("copying first part %r/%r, %r" % (container, part_name, headers)) try: conn.put_object(container, part_name, headers=headers, contents=None) except ClientException as ex: logging.error("Failed to copy %s: %s" % (name, ex.http_reason)) sys.exit(1) # setup the manifest headers = { 'x-object-manifest': quote("%s/%s" % (container, part_base_name)) } logging.debug("creating manifest %r/%r, %r" % (container, name, headers)) try: conn.put_object(container, name, headers=headers, contents=None) except ClientException as ex: logging.error("Failed to store the manifest %s: %s" % (name, ex.http_reason)) sys.exit(1) logging.debug("copy task done") conn.close() self.pending_copy_task = multiprocessing.Process(target=copy_task, args=(self.conn, self.container, self.name, self.part_name, self.part_base_name, ), ) self.pending_copy_task.start() @translate_objectstorage_error def write(self, data): """Write data to the object.""" if 'r' in self.mode: raise IOSError(EPERM, "File is opened for read") # large file support if self.split_size: # data can be of any size, so we need to split it in split_size chunks offs = 0 while offs < len(data): if self.part_size + len(data) - offs > self.split_size: current_size = self.split_size-self.part_size logging.debug("data is to large (%r), using %s" % (len(data), current_size)) else: current_size = len(data)-offs self.part_size += current_size if not self.obj: self.obj = ChunkObject(self.conn, self.container, self.part_name, content_type=self.content_type, reuse_token=False) self.obj.send_chunk(data[offs:offs+current_size]) offs += current_size if self.part_size == self.split_size: logging.debug("current size is %r, split_file is %r" % (self.part_size, self.split_size)) self.obj.finish_chunk() # this obj is not valid anymore, will create a new one if a new part is required self.obj = None # make it the first part if self.part == 0: self._start_copy_task() self.part_size = 0 self.part += 1 else: self.obj.send_chunk(data) @translate_objectstorage_error def close(self): """Close the object and finish the data transfer.""" if 'r' not in self.mode: if self.pending_copy_task: logging.debug("waiting for a pending copy task...") self.pending_copy_task.join() logging.debug("wait is over") if self.pending_copy_task.exitcode != 0: raise IOSError(EIO, 'Failed to store the file') if self.obj is not None: self.obj.finish_chunk() self.obj = None self.closed = True self.conn.close() @translate_objectstorage_error def read(self, size=65536): """ Read data from the object. We can use just one request because 'seek' is not fully supported. NB: It uses the size passed into the first call for all subsequent calls. """ if self.obj is None: headers = { } if self.total_size > 0: headers["Range"] = "bytes=%d-" % self.total_size _, self.obj = self.conn.get_object(self.container, self.name, resp_chunk_size=size, headers=headers) logging.debug("read size=%r, total_size=%r (range_from: %s)" % (size, self.total_size, self.total_size)) try: buff = self.obj.next() self.total_size += len(buff) except StopIteration: return "" else: return buff @translate_objectstorage_error def seek(self, offset, whence=None): """ Seek in the object. It's supported only for read operations because of object storage limitations. """ logging.debug("seek offset=%s, whence=%s" % (str(offset), str(whence))) if 'r' in self.mode: if self.size is None: meta = self.conn.head_object(self.container, self.name) try: self.size = int(meta["content-length"]) except ValueError: raise IOSError(EPERM, "Invalid file size") if not whence: offs = offset elif whence == 1: offs = self.total_size + offset elif whence == 2: offs = self.size - offset else: raise IOSError(EPERM, "Invalid file offset") if offs < 0 or offs > self.size: raise IOSError(EPERM, "Invalid file offset") # we need to start over after a seek call if self.obj is not None: del self.obj # GC the generator self.obj = None self.total_size = offs else: raise IOSError(EPERM, "Seek not available for write operations") class CacheEncoder(json.JSONEncoder): """JSONEncoder to encode the os.stat_result values into a list.""" def default(self, obj): if isinstance(obj, os.stat_result): return tuple(obj) return json.JSONEncoder.default(self, obj) def serialize(obj): """Serialize a cache dict into a JSON object.""" return json.dumps(obj, cls=CacheEncoder) def unserialize(js): """Unserialize a JSON object into a cache dict.""" return dict(((smart_str(key), os.stat_result(value)) for key, value in json.loads(js).iteritems())) class ListDirCache(object): """ Cache for listdir. This is to cache the very common case when we call listdir and then immediately call stat() on all the objects. In the OS this would be cached in the VFS but we have to make our own caching here to avoid the stat calls each making a connection. """ MAX_CACHE_TIME = 10 # seconds to cache the listdir for MIN_COMPRESS_LEN = 4096 # min length in bytes to compress cache entries memcache = None def __init__(self, cffs): self.cffs = cffs self.path = None self.cache = {} self.when = time.time() if self.cffs.memcache_hosts and ListDirCache.memcache is None: logging.debug("connecting to memcache %r" % self.cffs.memcache_hosts) ListDirCache.memcache = memcache.Client(self.cffs.memcache_hosts) @property def conn(self): """Connection to the storage.""" return self.cffs.conn def key(self, index): """Returns a key for a user distributed cache.""" tenant_name = self.cffs.tenant_name or "-" logging.debug("cache key for %r" % [self.cffs.authurl, self.cffs.username, tenant_name, index]) if not hasattr(self, "_key_base"): self._key_base = md5("%s%s%s" % (self.cffs.authurl, self.cffs.username, tenant_name)).hexdigest() return "%s-%s" % (self._key_base, md5(smart_str(index)).hexdigest()) def flush(self, path=None): """Flush the listdir cache.""" logging.debug("cache flush, current path: %s request: %s" % (self.path, path)) if self.memcache: if path is not None: logging.debug("flushing memcache for %r" % path) self.memcache.delete(self.key(path)) if self.path == path: self.cache = None elif self.path is not None: logging.debug("flushing memcache for %r" % self.path) self.memcache.delete(self.key(path)) self.cache = None else: self.cache = None def _make_stat(self, last_modified=None, content_type="application/directory", count=1, bytes=0, **kwargs): """Make a stat object from the parameters passed in from""" if last_modified: if "." in last_modified: last_modified, microseconds = last_modified.rsplit(".", 1) if microseconds.endswith("Z"): microseconds = microseconds[:-1] microseconds = float("0."+microseconds) else: microseconds = 0.0 mtime_tuple = list(time.strptime(last_modified, "%Y-%m-%dT%H:%M:%S")) mtime_tuple[8] = 0 # Use GMT mtime = time.mktime(mtime_tuple) + microseconds else: mtime = time.time() if content_type == "application/directory": mode = 0755|stat.S_IFDIR else: mode = 0644|stat.S_IFREG #(mode, ino, dev, nlink, uid, gid, size, atime, mtime, ctime) return os.stat_result((mode, 0L, 0L, count, 0, 0, bytes, mtime, mtime, mtime)) def listdir_container(self, cache, container, path=""): """Fills cache with the list dir of the container""" container = smart_str(container) path = smart_str(path) logging.debug("listdir container %r path %r" % (container, path)) if path: prefix = path.rstrip("/")+"/" else: prefix = None _, objects = self.conn.get_container(container, prefix=prefix, delimiter="/") # override 10000 objects limit with markers nbobjects = len(objects) while nbobjects >= 10000: # get last object as a marker lastobject = objects[-1] if 'subdir' in lastobject: # {u'subdir': 'dirname'} lastobjectname = lastobject['subdir'].rstrip("/") else: lastobjectname = lastobject['name'] # get a new list with the marker _, newobjects = self.conn.get_container(container, prefix=prefix, delimiter="/", marker=lastobjectname) # get the new list length nbobjects = len(newobjects) logging.debug("number of objects after marker %s: %s" % (lastobjectname, nbobjects)) # add the new list to current list objects.extend(newobjects) logging.debug("total number of objects %s:" % len(objects)) if self.cffs.hide_part_dir: manifests = {} for obj in objects: # {u'bytes': 4820, u'content_type': '...', u'hash': u'...', u'last_modified': u'2008-11-05T00:56:00.406565', u'name': u'new_object'}, if 'subdir' in obj: # {u'subdir': 'dirname'} obj['name'] = obj['subdir'].rstrip("/") # If a manifest and it's segment directory have the # same name then we have to choose which we want to # show, we can't show both. So we choose to keep the # manifest if hide_part_dir is enabled. # # We can do this here because swift returns objects in # alphabetical order so the manifest will come before # its segments. if self.cffs.hide_part_dir and obj['name'] in manifests: logging.debug("Not adding subdir %s which would overwrite manifest" % obj['name']) continue elif obj.get('bytes') == 0 and obj.get('hash') and obj.get('content_type') != 'application/directory': # if it's a 0 byte file, has a hash and is not a directory, we make an extra call # to check if it's a manifest file and retrieve the real size / hash manifest_obj = self.conn.head_object(container, obj['name']) logging.debug("possible manifest file: %r" % manifest_obj) if 'x-object-manifest' in manifest_obj: if self.cffs.hide_part_dir: manifests[obj['name']] = smart_unicode(unquote(manifest_obj['x-object-manifest']), "utf-8") logging.debug("manifest found: %s" % manifest_obj['x-object-manifest']) obj['hash'] = manifest_obj['etag'] obj['bytes'] = int(manifest_obj['content-length']) obj['count'] = 1 # Keep all names in utf-8, just like the filesystem name = posixpath.basename(obj['name']).encode("utf-8") cache[name] = self._make_stat(**obj) if self.cffs.hide_part_dir: for manifest in manifests: manifest_container, manifest_obj = parse_fspath('/' + manifests[manifest]) if manifest_container == container: for cache_obj in cache.copy(): # hide any manifest segments, but not the manifest itself, if it # happens to share a prefix with its segments. if unicode(unquote(cache_obj), "utf-8") != manifest and \ unicode(unquote(os.path.join(path, cache_obj)), "utf-8").startswith(manifest_obj): logging.debug("hiding manifest %r segment %r" % (manifest, cache_obj)) del cache[cache_obj] def listdir_root(self, cache): """Fills cache with the list of containers""" logging.debug("listdir root") try: _, objects = self.conn.get_account() except ClientException: # when implementing contaniners' ACL, getting the containers # list can raise a ResponseError, but still access to the # the containers we have permissions to access to return for obj in objects: # {u'count': 0, u'bytes': 0, u'name': u'container1'}, # Keep all names in utf-8, just like the filesystem name = obj['name'].encode("utf-8") cache[name] = self._make_stat(**obj) def listdir(self, path): """Return the directory list of the path, filling the cache in the process""" path = path.rstrip("/") or "/" logging.debug("listdir %r" % path) cache = None if self.memcache: cache = self.memcache.get(self.key(path)) if cache: cache = unserialize(cache) logging.debug("memcache hit %r" % self.key(path)) else: logging.debug("memcache miss %r" % self.key(path)) if not cache: cache = {} if path == "/": self.listdir_root(cache) else: container, obj = parse_fspath(path) self.listdir_container(cache, container, obj) if self.memcache: if self.memcache.set(self.key(path), serialize(cache), self.MAX_CACHE_TIME, min_compress_len=self.MIN_COMPRESS_LEN): logging.debug("memcache stored %r" % self.key(path)) else: logging.warning("Failed to store the cache") self.cache = cache self.path = path self.when = time.time() leaves = sorted(self.cache.keys()) logging.debug(".. %r" % leaves) return leaves def listdir_with_stat(self, path): """ Return the directory list of the path with stat objects. The cache will be filled in in the process, as a list of tuples (leafname, stat_result). """ self.listdir(path) return sorted(self.cache.iteritems()) def valid(self, path): """Check the cache is valid for the container and directory path""" if not self.cache or self.path != path: if self.memcache: cache = self.memcache.get(self.key(path)) if cache: cache = unserialize(cache) logging.debug("memcache hit %r" % self.key(path)) self.cache = cache self.path = path return True return False age = time.time() - self.when return age < self.MAX_CACHE_TIME def stat(self, path): """ Returns an os.stat_result for path or raises IOSError. Returns the information from the cache if possible. """ path = path.rstrip("/") or "/" logging.debug("stat path %r" % (path)) directory, leaf = posixpath.split(path) # Refresh the cache it if is old, or wrong if not self.valid(directory): logging.debug("invalid cache for %r (path: %r)" % (directory, self.path)) self.listdir(directory) if path != "/": try: stat_info = self.cache[smart_str(leaf)] except KeyError: logging.debug("Didn't find %r in directory listing" % leaf) # it can be a container and the user doesn't have # permissions to list the root if directory == '/' and leaf: try: container = self.conn.head_container(leaf) except ClientException: raise IOSError(ENOENT, 'No such file or directory %s' % leaf) logging.debug("Accessing %r container without root listing" % leaf) stat_info = self._make_stat(count=int(container["x-container-object-count"]), bytes=int(container["x-container-bytes-used"]), ) else: raise IOSError(ENOENT, 'No such file or directory %s' % leaf) else: # Root directory size is sum of containers, count is containers bytes = sum(stat_info.st_size for stat_info in self.cache.values()) count = len(self.cache) stat_info = self._make_stat(count=count, bytes=bytes) logging.debug("stat path: %r" % stat_info) return stat_info class ObjectStorageFS(object): """ Object Storage File System emulation. All the methods on this class emulate os.* or os.path.* functions of the same name. """ memcache_hosts = None @translate_objectstorage_error def __init__(self, username, api_key, authurl, keystone=None, hide_part_dir=False, snet=False, insecure=False): """ Create the Object Storage connection. username - if None then don't make the connection (delayed auth) api_key authurl keystone - optional for auth 2.0 (keystone) hider_part_dirt - optional, hide multipart .part files snet - optional, use Rackspace's service network insecure - optional, allow using servers without checking their SSL certs """ self.conn = None self.authurl = authurl self.keystone = keystone self.hide_part_dir = hide_part_dir self.snet = snet self.insecure = insecure # A cache to hold the information from the last listdir self._listdir_cache = ListDirCache(self) self._cwd = '/' if username is not None: self.authenticate(username, api_key) @translate_objectstorage_error def authenticate(self, username, api_key): """Authenticates and opens the connection""" if not username or not api_key: raise ClientException("username/password required", http_status=401) kwargs = dict(authurl=self.authurl, auth_version="1.0", snet=self.snet) tenant_name = None if self.keystone: if self.keystone['tenant_separator'] in username: tenant_name, username = username.split(self.keystone['tenant_separator'], 1) logging.debug("keystone authurl=%r username=%r tenant_name=%r conf=%r" % (self.authurl, username, tenant_name, self.keystone)) ks = self.keystone kwargs["auth_version"] = "2.0" kwargs["tenant_name"] = tenant_name kwargs["os_options"] = dict(service_type=ks['service_type'], endpoint_type=ks['endpoint_type'], region_name=ks['region_name'], ) self.conn = ProxyConnection(self._listdir_cache.memcache, user=username, key=api_key, insecure=self.insecure, **kwargs ) # force authentication self.conn.url, self.conn.token = self.conn.get_auth() self.conn.close() # now we are authenticated and we have an username self.username = username self.tenant_name = tenant_name def close(self): """Explicitly close the connection, although it may not be required""" logging.debug("called fs.close()") if self.conn: self.conn.close() def isabs(self, path): """Test whether a path is absolute""" return posixpath.isabs(path) def normpath(self, path): """Normalize path, eliminating double slashes, etc""" return posixpath.normpath(path) def abspath(self, path): """Return an absolute path""" if not self.isabs(path): path = posixpath.join(self.getcwd(), path) return self.normpath(path) def mkstemp(self, suffix='', prefix='', dir=None, mode='wb'): """ A wrapper around tempfile.mkstemp creating a file with a unique name. Unlike mkstemp it returns an object with a file-like interface. """ e = "mkstemp suffix=%r prefix=%r, dir=%r mode=%r - not implemented" % (suffix, prefix, dir, mode) logging.debug(e) raise IOSError(EPERM, 'Operation not permitted: %s' % e) @close_when_done @translate_objectstorage_error def open(self, path, mode): """Open path with mode, raise IOError on error""" path = self.abspath(path) logging.debug("open %r mode %r" % (path, mode)) self._listdir_cache.flush(posixpath.dirname(path)) container, obj = parse_fspath(path) return ObjectStorageFD(self.conn, container, obj, mode) def chdir(self, path): """Change current directory, raise OSError on error""" path = self.abspath(path) logging.debug("chdir %r" % path) if not path.startswith("/"): raise IOSError(ENOENT, 'Failed to change directory.') container, obj = parse_fspath(path) if not container: logging.debug("cd to /") else: logging.debug("cd to container %r directory %r" % (container, obj)) if not self.isdir(path): raise IOSError(ENOTDIR, "Can't cd to a file") self._cwd = path def getcwd(self): """Returns the current working directory""" return self._cwd def _container_exists(self, container): # verify the container exsists try: self.conn.head_container(container) except ClientException, e: if e.http_status == 404: raise IOSError(ENOTDIR, "Container not found") raise return True @close_when_done @translate_objectstorage_error def mkdir(self, path): """ Make a directory. Raises OSError on error. """ path = self.abspath(path) logging.debug("mkdir %r" % path) container, obj = parse_fspath(path) if obj: self._listdir_cache.flush(posixpath.dirname(path)) logging.debug("Making directory %r in %r" % (obj, container)) self._container_exists(container) self.conn.put_object(container, obj, contents=None, content_type="application/directory") else: self._listdir_cache.flush("/") logging.debug("Making container %r" % (container,)) self.conn.put_container(container) @close_when_done @translate_objectstorage_error def listdir(self, path): """ List a directory. Raises OSError on error. """ path = self.abspath(path) logging.debug("listdir %r" % path) list_dir = map(lambda x: unicode(x, 'utf-8'), self._listdir_cache.listdir(path)) return list_dir @close_when_done @translate_objectstorage_error def listdir_with_stat(self, path): """ Return the directory list of the path with stat objects. The the cache is filled in the process, as a list of tuples (leafname, stat_result). """ path = self.abspath(path) logging.debug("listdir_with_stat %r" % path) return [(unicode(name, 'utf-8)'), stat) for name, stat in self._listdir_cache.listdir_with_stat(path)] @close_when_done @translate_objectstorage_error def rmdir(self, path): """ Remove a directory. Eaise OSError on error. """ path = self.abspath(path) logging.debug("rmdir %r" % path) container, obj = parse_fspath(path) if not self.isdir(path): if self.isfile(path): raise IOSError(ENOTDIR, "Not a directory") raise IOSError(ENOENT, 'No such file or directory') if self.listdir(path): raise IOSError(ENOTEMPTY, "Directory not empty: %s" % path) if obj: self._listdir_cache.flush(posixpath.dirname(path)) logging.debug("Removing directory %r in %r" % (obj, container)) self.conn.delete_object(container, obj) else: self._listdir_cache.flush("/") logging.debug("Removing container %r" % (container,)) self.conn.delete_container(container) @close_when_done @translate_objectstorage_error def remove(self, path): """ Remove a file. Raises OSError on error. """ path = self.abspath(path) logging.debug("remove %r" % path) logging.info("remove %r" % path) container, name = parse_fspath(path) if not name: raise IOSError(EACCES, "Can't remove a container") if self.isdir(path): raise IOSError(EACCES, "Can't remove a directory (use rmdir instead)") meta = self.conn.head_object(container, name) if 'x-object-manifest' in meta: self._remove_path_folder_files(u'/' + smart_unicode(unquote(meta['x-object-manifest']), "utf-8")) self.conn.delete_object(container, name) self._listdir_cache.flush(posixpath.dirname(path)) return not name def _remove_path_folder_files(self, path): logging.info("Removing manifest file's parts from: %s" % path) files = self.listdir(path) for file in files: self.remove(path + '/' + file) @translate_objectstorage_error def _rename_container(self, src_container_name, dst_container_name): """Rename src_container_name into dst_container_name""" logging.debug("rename container %r -> %r" % (src_container_name, dst_container_name)) # Delete the old container first, raising error if not empty self.conn.delete_container(src_container_name) self.conn.put_container(dst_container_name) self._listdir_cache.flush("/") @close_when_done @translate_objectstorage_error def rename(self, src, dst): """ Rename a file/directory from src to dst. Raises OSError on error. """ src = self.abspath(src) dst = self.abspath(dst) logging.debug("rename %r -> %r" % (src, dst)) self._listdir_cache.flush() # Check not renaming to itself if src == dst: logging.debug("Renaming %r to itself - doing nothing" % src) return # If dst is an existing directory, copy src inside it if self.isdir(dst): if dst: dst += "/" dst += posixpath.basename(src) # Check constraints for renaming a directory if self.isdir(src): if self.listdir(src): raise IOSError(ENOTEMPTY, "Can't rename non-empty directory: %s" % src) if self.isfile(dst): raise IOSError(ENOTDIR, "Can't rename directory to file") # Check not renaming to itself if src == dst: logging.debug("Renaming %r to itself - doing nothing" % src) return # Parse the paths now src_container_name, src_path = parse_fspath(src) dst_container_name, dst_path = parse_fspath(dst) logging.debug("`.. %r/%r -> %r/%r" % (src_container_name, src_path, dst_container_name, dst_path)) # Check if we are renaming containers if not src_path and not dst_path and src_container_name and dst_container_name: return self._rename_container(src_container_name, dst_container_name) # ...otherwise can't deal with root stuff if not src_container_name or not src_path or not dst_container_name or not dst_path: raise IOSError(EACCES, "Can't rename to / from root") # Check destination directory exists if not self.isdir(posixpath.split(dst)[0]): raise IOSError(ENOENT, "Can't copy %r to %r, destination directory doesn't exist" % (src, dst)) # check dst container self._container_exists(dst_container_name) # Do the rename of the file/dir meta = self.conn.head_object(src_container_name, src_path) if 'x-object-manifest' in meta: # a manifest file headers = { 'x-object-manifest': quote(meta['x-object-manifest']) } else: # regular file headers = { 'x-copy-from': quote("/%s/%s" % (src_container_name, src_path)) } self.conn.put_object(dst_container_name, dst_path, headers=headers, contents=None) # Delete src self.conn.delete_object(src_container_name, src_path) self._listdir_cache.flush(posixpath.dirname(src)) self._listdir_cache.flush(posixpath.dirname(dst)) def chmod(self, path, mode): """Change file/directory mode""" e = "chmod %03o %r - not implemented" % (mode, path) logging.debug(e) raise IOSError(EPERM, 'Operation not permitted: %s' % e) def isfile(self, path): """ Is this path a file. Shouldn't raise an error if not found like os.path.isfile. """ logging.debug("isfile %r" % path) try: return stat.S_ISREG(self.stat(path).st_mode) except EnvironmentError: return False def islink(self, path): """ Is this path a link. Shouldn't raise an error if not found like os.path.islink. """ logging.debug("islink %r" % path) return False def isdir(self, path): """ Is this path a directory. Shouldn't raise an error if not found like os.path.isdir. """ logging.debug("isdir %r" % path) try: return stat.S_ISDIR(self.stat(path).st_mode) except EnvironmentError: return False def getsize(self, path): """ Return the size of path. Raises OSError on error. """ logging.debug("getsize %r" % path) return self.stat(path).st_size def getmtime(self, path): """ Return the modification time of path. Raises OSError on error. """ logging.debug("getmtime %r" % path) return self.stat(path).st_mtime def realpath(self, path): """Return the canonical path of the specified path""" return self.abspath(path) def lexists(self, path): """ Test whether a path exists. Returns True for broken symbolic links. """ logging.debug("lexists %r" % path) try: self.stat(path) return True except EnvironmentError: return False @close_when_done @translate_objectstorage_error def stat(self, path): """ Return os.stat_result object for path. Raises OSError on error. """ path = self.abspath(path) logging.debug("stat %r" % path) return self._listdir_cache.stat(path) exists = lexists lstat = stat def validpath(self, path): """Check whether the path belongs to user's home directory""" return True def flush(self): """Flush cache""" if self._listdir_cache: self._listdir_cache.flush() def get_user_by_uid(self, uid): """ Return the username associated with user id. If this can't be determined return raw uid instead. """ return self.username def get_group_by_gid(self, gid): """ Return the groupname associated with group id. If this can't be determined return raw gid instead. On Windows just return "group". """ return self.username def readlink(self, path): """ Return a string representing the path to which a symbolic link points. We never return that we have a symlink in stat, so this should never be called. """ e = "readlink %r - not implemented" % path logging.debug(e) raise IOSError(EPERM, 'Operation not permitted: %s' % e) @close_when_done @translate_objectstorage_error def md5(self, path): """ Return the object MD5 for path. Raise OSError on error. """ path = self.abspath(path) logging.debug("md5 %r" % path) container, name = parse_fspath(path) if not name: raise IOSError(EACCES, "Can't return the MD5 of a container") if self.isdir(path): # this is only 100% accurate for virtual directories raise IOSError(EACCES, "Can't return the MD5 of a directory") meta = self.conn.head_object(container, name) return meta["etag"] ftp-cloudfs-0.35/ftpcloudfs/monkeypatching.py0000664000175000017500000001017112372355335021026 0ustar jjmjjm00000000000000import sys import socket from pyftpdlib.handlers import DTPHandler, FTPHandler, _strerror from ftpcloudfs.utils import smart_str from server import ObjectStorageAuthorizer from multiprocessing.managers import RemoteError class MyDTPHandler(DTPHandler): def send(self, data): data = smart_str(data) return DTPHandler.send(self, data) def close(self): if self.file_obj is not None and not self.file_obj.closed: try: self.file_obj.close() except Exception, e: msg = u"Data connection error (%s)" % e self.cmd_channel.log(msg) self.cmd_channel.respond(u"421 " + msg) finally: self.file_obj = None DTPHandler.close(self) class MyFTPHandler(FTPHandler): # don't kick off client in long time transactions timeout = 0 dtp_handler = MyDTPHandler authorizer = ObjectStorageAuthorizer() max_cons_per_ip = 0 use_sendfile = False @staticmethod def abstracted_fs(root, cmd_channel): """Get an AbstractedFs for the user logged in on the cmd_channel.""" cffs = cmd_channel.authorizer.get_abstracted_fs(cmd_channel.username) cffs.init_abstracted_fs(root, cmd_channel) return cffs def process_command(self, cmd, *args, **kwargs): """ Flush the FS cache with every new FTP command (non-shared cache). Also track the remote ip to set the X-Forwarded-For header. """ if self.fs: if self.fs.memcache_hosts is None: self.fs.flush() self.fs.conn.real_ip = self.remote_ip FTPHandler.process_command(self, cmd, *args, **kwargs) def ftp_MD5(self, path): line = self.fs.fs2ftp(path) try: md5_checksum = self.run_as_current_user(self.fs.md5, path) except OSError, err: why = _strerror(err) self.respond('550 %s.' % why) else: msg = md5_checksum.upper() self.respond('251 "%s" %s' % (line.replace('"', '""'), msg)) def handle(self): """Track the ip and check max cons per ip (if needed).""" if self.max_cons_per_ip and self.remote_ip and self.shared_ip_map != None: count = 0 try: self.shared_lock.acquire() count = self.shared_ip_map.get(self.remote_ip, 0) + 1 self.shared_ip_map[self.remote_ip] = count self.logline("Connected, shared ip map: %s" % self.shared_ip_map) except RemoteError, e: self.logerror("Connection tracking failed: %s" % e) finally: self.shared_lock.release() self.logline("Connection track: %s -> %s" % (self.remote_ip, count)) if count > self.max_cons_per_ip: self.handle_max_cons_per_ip() return FTPHandler.handle(self) def handle_error(self): """Catch some 'expected' exceptions not processed by FTPHandler/AsyncChat.""" # this is aesthetic only t, v, _ = sys.exc_info() if t == socket.error: self.log("Connection error: %s" % v) self.handle_close() return FTPHandler.handle_error(self) def close(self): """Remove the ip from the shared map before calling close.""" if not self._closed and self.max_cons_per_ip and self.shared_ip_map != None: try: self.shared_lock.acquire() if self.remote_ip in self.shared_ip_map: self.shared_ip_map[self.remote_ip] -= 1 if self.shared_ip_map[self.remote_ip] <= 0: del self.shared_ip_map[self.remote_ip] self.logline("Disconnected, shared ip map: %s" % self.shared_ip_map) except RemoteError, e: self.logerror("Connection tracking cleanup failed: %s" % e) finally: self.shared_lock.release() FTPHandler.close(self) # We want to log more commands. log_cmds_list = ["ABOR", "APPE", "DELE", "RMD", "RNFR", "RNTO", "RETR", "STOR", "MKD",] ftp-cloudfs-0.35/ftpcloudfs/server.py0000664000175000017500000000525512426707320017316 0ustar jjmjjm00000000000000#/usr/bin/env python # # Authors: Chmouel Boudjnah # Juan J. Martinez # import os from pyftpdlib.filesystems import AbstractedFS from pyftpdlib.authorizers import DummyAuthorizer, AuthenticationFailed from fs import ObjectStorageFS class ObjectStorageFtpFS(ObjectStorageFS, AbstractedFS): """Object Storage File system emulation for a FTP server.""" servicenet = False authurl = None insecure = False keystone = None hide_part_dir = None snet = False def __init__(self, username, api_key, authurl=None, keystone=None, hide_part_dir=None): ObjectStorageFS.__init__(self, username, api_key, authurl=authurl or self.authurl, keystone=keystone or self.keystone, hide_part_dir=hide_part_dir or self.hide_part_dir, snet = self.snet, insecure = self.insecure, ) def init_abstracted_fs(self, root, cmd_channel): AbstractedFS.__init__(self, root, cmd_channel) class ObjectStorageAuthorizer(DummyAuthorizer): """ FTP server authorizer. Logs the users into the object storage and keeps track of them. """ users = {} abstracted_fs_for_user = {} def validate_authentication(self, username, password, handler): """ Validates the username and passwords. This creates the AbstractedFS at the same time and caches it under the username for retrieval with get_abstracted_fs. """ try: cffs = ObjectStorageFtpFS(username, password) except EnvironmentError, e: msg = "Failed to authenticate user %s: %s" % (username, e) handler.logerror(msg) raise AuthenticationFailed(msg) self.abstracted_fs_for_user[username] = cffs handler.log("Authentication validated for user %s" % username) def get_abstracted_fs(self, username): """ Gets an AbstractedFs object for the user. Raises KeyError if username isn't found. """ return self.abstracted_fs_for_user.pop(username) def has_user(self, username): return username != 'anonymous' def has_perm(self, username, perm, path=None): return True def get_perms(self, username): return u'lrdw' def get_home_dir(self, username): return unicode(os.sep) def get_msg_login(self, username): return u'Welcome %s' % username def get_msg_quit(self, username): return u'Goodbye %s' % username ftp-cloudfs-0.35/ftpcloudfs/main.py0000664000175000017500000004032612670251072016731 0ustar jjmjjm00000000000000# -*- encoding: utf-8 -*- __author__ = "Chmouel Boudjnah " import sys import os import signal import socket from ConfigParser import RawConfigParser import logging from logging.handlers import SysLogHandler from optparse import OptionParser import pyftpdlib.servers import swiftclient from server import ObjectStorageFtpFS from fs import ObjectStorageFD from constants import version, default_address, default_port, \ default_config_file, default_banner, \ default_ks_tenant_separator, default_ks_service_type, default_ks_endpoint_type from monkeypatching import MyFTPHandler from multiprocessing import Manager def modify_supported_ftp_commands(): """Remove the FTP commands we don't / can't support, and add the extensions.""" unsupported = ( 'SITE CHMOD', ) for cmd in unsupported: if cmd in pyftpdlib.handlers.proto_cmds: del pyftpdlib.handlers.proto_cmds[cmd] # add the MD5 command, FTP extension according to IETF Draft: # http://tools.ietf.org/html/draft-twine-ftpmd5-00 pyftpdlib.handlers.proto_cmds.update({ 'MD5': dict(perm=None, auth=True, arg=True, help=u'Syntax: MD5 file-name (get MD5 of file)') }) class Main(object): """ftp-cloudfs: A FTP Proxy Interface to OpenStack Object Storage (Swift).""" def __init__(self): self.options = None def setup_log(self): """Setup Logging.""" if self.options.log_level: self.options.log_level = logging.DEBUG else: self.options.log_level = logging.INFO if self.options.syslog: logger = logging.getLogger() try: handler = SysLogHandler(address='/dev/log', facility=SysLogHandler.LOG_DAEMON) except IOError: # fall back to UDP handler = SysLogHandler(facility=SysLogHandler.LOG_DAEMON) finally: prefix = "%s[%%(process)d]: " % __package__ formatter = logging.Formatter(prefix + "%(message)s") handler.setFormatter(formatter) logger.addHandler(handler) logger.setLevel(self.options.log_level) else: log_format = '[%(process)d] %(asctime)-15s - %(levelname)s - %(message)s' logging.basicConfig(filename=self.options.log_file, format=log_format, level=self.options.log_level) # warnings if self.config.get("ftpcloudfs", "workers") is not None: logging.warning("workers configuration token has been deprecated and has no effect") if self.config.get("ftpcloudfs", "service-net") is not None: logging.warning("service-net configuration token has been deprecated and has no effect (see ChangeLog)") def parse_configuration(self): """Parse the configuration file""" # look for an alternative configuration file alt_config_file = False parser = OptionParser() # only for error reporting config_file = default_config_file for arg in sys.argv: if arg == '--config': try: alt_config_file = sys.argv[sys.argv.index(arg) + 1] config_file = alt_config_file except IndexError: # the parser will report the error later on pass elif arg.startswith('--config='): _, alt_config_file = arg.split('=', 1) if not alt_config_file: parser.error("--config option requires an argument") config_file = alt_config_file config = RawConfigParser({'banner': default_banner, 'port': default_port, 'bind-address': default_address, 'workers': None, 'memcache': None, 'max-cons-per-ip': '0', 'permit-foreign-addresses': 'no', 'auth-url': None, 'insecure': False, 'service-net': None, 'verbose': 'no', 'syslog': 'no', 'log-file': None, 'pid-file': None, 'uid': None, 'gid': None, 'masquerade-firewall': None, 'passive-ports': None, 'split-large-files': '0', 'hide-part-dir': 'no', # keystone auth 2.0 support 'keystone-auth': False, 'keystone-region-name': None, 'keystone-tenant-separator': default_ks_tenant_separator, 'keystone-service-type': default_ks_service_type, 'keystone-endpoint-type': default_ks_endpoint_type, 'rackspace-service-net' : 'no', }) if not config.read(config_file) and alt_config_file: # the default conf file is optional parser.error("failed to read %s" % config_file) if not config.has_section('ftpcloudfs'): config.add_section('ftpcloudfs') self.config = config def parse_arguments(self): """Parse command line options""" parser = OptionParser(version="%prog " + version) parser.add_option('-p', '--port', type="int", dest="port", default=self.config.get('ftpcloudfs', 'port'), help="Port to bind the server (default: %d)" % \ (default_port)) parser.add_option('-b', '--bind-address', type="str", dest="bind_address", default=self.config.get('ftpcloudfs', 'bind-address'), help="Address to bind (default: %s)" % \ (default_address)) parser.add_option('-a', '--auth-url', type="str", dest="authurl", default=self.config.get('ftpcloudfs', 'auth-url'), help="Authentication URL (required)") parser.add_option('--insecure', action="store_true", dest="insecure", default=self.config.get('ftpcloudfs', 'insecure'), help="Allow to access servers without checking SSL certs") memcache = self.config.get('ftpcloudfs', 'memcache') if memcache: memcache = [x.strip() for x in memcache.split(',')] parser.add_option('--memcache', type="str", dest="memcache", action="append", default=memcache, help="Memcache server(s) to be used for cache (ip:port)") parser.add_option('-v', '--verbose', action="store_true", dest="log_level", default=self.config.getboolean('ftpcloudfs', 'verbose'), help="Be verbose on logging") parser.add_option('-f', '--foreground', action="store_true", dest="foreground", default=False, help="Do not attempt to daemonize but run in foreground") parser.add_option('-l', '--log-file', type="str", dest="log_file", default=self.config.get('ftpcloudfs', 'log-file'), help="Log File: Default stdout when in foreground") parser.add_option('--syslog', action="store_true", dest="syslog", default=self.config.getboolean('ftpcloudfs', 'syslog'), help="Enable logging to the system logger " + \ "(daemon facility)") parser.add_option('--pid-file', type="str", dest="pid_file", default=self.config.get('ftpcloudfs', 'pid-file'), help="Pid file location when in daemon mode") parser.add_option('--uid', type="int", dest="uid", default=self.config.get('ftpcloudfs', 'uid'), help="UID to drop the privilige to when in daemon mode") parser.add_option('--gid', type="int", dest="gid", default=self.config.get('ftpcloudfs', 'gid'), help="GID to drop the privilige to when in daemon mode") parser.add_option('--keystone-auth', action="store_true", dest="keystone", default=self.config.get('ftpcloudfs', 'keystone-auth'), help="Use auth 2.0 (Keystone, requires keystoneclient)") parser.add_option('--keystone-region-name', type="str", dest="region_name", default=self.config.get('ftpcloudfs', 'keystone-region-name'), help="Region name to be used in auth 2.0") parser.add_option('--keystone-tenant-separator', type="str", dest="tenant_separator", default=self.config.get('ftpcloudfs', 'keystone-tenant-separator'), help="Character used to separate tenant_name/username in auth 2.0" + \ " (default: TENANT%sUSERNAME)" % default_ks_tenant_separator) parser.add_option('--keystone-service-type', type="str", dest="service_type", default=self.config.get('ftpcloudfs', 'keystone-service-type'), help="Service type to be used in auth 2.0 (default: %s)" % default_ks_service_type) parser.add_option('--keystone-endpoint-type', type="str", dest="endpoint_type", default=self.config.get('ftpcloudfs', 'keystone-endpoint-type'), help="Endpoint type to be used in auth 2.0 (default: %s)" % default_ks_endpoint_type) parser.add_option('--config', type="str", dest="config", default=default_config_file, help="Use an alternative configuration file (default: %s)" % default_config_file) (options, _) = parser.parse_args() if options.keystone: try: from keystoneclient.v2_0 import client as _test_ksclient except ImportError: parser.error("Auth 2.0 (keystone) requires python-keystoneclient.") keystone_keys = ('region_name', 'tenant_separator', 'service_type', 'endpoint_type') options.keystone = dict((key, getattr(options, key)) for key in keystone_keys) if not options.authurl: parser.error("An authentication URL is required and it wasn't provided") self.options = options def setup_server(self): """Run the main ftp server loop.""" banner = self.config.get('ftpcloudfs', 'banner').replace('%v', version) banner = banner.replace('%f', pyftpdlib.__ver__) banner = banner.replace('%s', swiftclient.version.version_string) MyFTPHandler.banner = banner ObjectStorageFtpFS.authurl = self.options.authurl ObjectStorageFtpFS.insecure = self.options.insecure ObjectStorageFtpFS.keystone = self.options.keystone ObjectStorageFtpFS.memcache_hosts = self.options.memcache ObjectStorageFtpFS.hide_part_dir = self.config.getboolean('ftpcloudfs', 'hide-part-dir') ObjectStorageFtpFS.snet = self.config.getboolean('ftpcloudfs', 'rackspace-service-net') try: # store bytes ObjectStorageFD.split_size = int(self.config.get('ftpcloudfs', 'split-large-files'))*10**6 except ValueError, errmsg: sys.exit('Split large files error: %s' % errmsg) masquerade = self.config.get('ftpcloudfs', 'masquerade-firewall') if masquerade: try: MyFTPHandler.masquerade_address = socket.gethostbyname(masquerade) except socket.gaierror, (_, errmsg): sys.exit('Masquerade address error: %s' % errmsg) passive_ports = self.config.get('ftpcloudfs', 'passive-ports') if passive_ports: try: passive_ports = [p.strip() for p in passive_ports.split(":", 2)] if len(passive_ports) != 2 or passive_ports[0] >= passive_ports[1]: raise ValueError() passive_ports = map(int, passive_ports) MyFTPHandler.passive_ports = range(passive_ports[0], passive_ports[1]+1) except (ValueError, TypeError): sys.exit('Passive ports error: int:int expected') MyFTPHandler.permit_foreign_addresses = self.config.getboolean('ftpcloudfs', 'permit-foreign-addresses') try: max_cons_per_ip = int(self.config.get('ftpcloudfs', 'max-cons-per-ip')) except ValueError, errmsg: sys.exit('Max connections per IP error: %s' % errmsg) ftpd = pyftpdlib.servers.MultiprocessFTPServer((self.options.bind_address, self.options.port), MyFTPHandler, ) # set it to unlimited, we use our own checks with a shared dict ftpd.max_cons_per_ip = 0 ftpd.handler.max_cons_per_ip = max_cons_per_ip return ftpd def setup_daemon(self, preserve=None): """Setup the daemon context for the server.""" import daemon from utils import PidFile import tempfile daemonContext = daemon.DaemonContext() if not self.options.pid_file: self.options.pid_file = "%s/ftpcloudfs.pid" % \ (tempfile.gettempdir()) self.pidfile = PidFile(self.options.pid_file) daemonContext.pidfile = self.pidfile if self.options.uid: daemonContext.uid = self.options.uid if self.options.gid: daemonContext.gid = self.options.gid if preserve: daemonContext.files_preserve = preserve return daemonContext def signal_handler(self, signal, frame): """Catch signals and propagate them to child processes.""" if self.shm_manager: self.shm_manager.shutdown() self.shm_manager = None self.old_signal_handler(signal, frame) def main(self): """Main entry point.""" self.pid = os.getpid() self.parse_configuration() self.parse_arguments() modify_supported_ftp_commands() ftpd = self.setup_server() if self.options.foreground: MyFTPHandler.shared_ip_map = None self.setup_log() ftpd.serve_forever() return daemonContext = self.setup_daemon([ftpd.socket.fileno(), ftpd.ioloop.fileno(),]) with daemonContext: self.old_signal_handler = signal.signal(signal.SIGTERM, self.signal_handler) self.shm_manager = Manager() MyFTPHandler.shared_ip_map = self.shm_manager.dict() MyFTPHandler.shared_lock = self.shm_manager.Lock() self.setup_log() ftpd.serve_forever() ftp-cloudfs-0.35/ftpcloudfs/constants.py0000664000175000017500000000050512670314744020022 0ustar jjmjjm00000000000000version = '0.35' default_banner = "ftp-cloudfs %v using pyftpdlib %f (swiftclient %s) ready." default_config_file = '/etc/ftpcloudfs.conf' default_address = '127.0.0.1' default_port = 2021 # keystone defaults default_ks_tenant_separator = '.' default_ks_service_type = 'object-store' default_ks_endpoint_type = 'publicURL' ftp-cloudfs-0.35/ftpcloudfs/errors.py0000664000175000017500000000117612211103543017307 0ustar jjmjjm00000000000000""" Errors for ObjectStorageFS """ class IOSError(OSError, IOError): """ Subclass of OSError and IOError. This is needed because pyftpdlib catches either OSError, or IOError depending on which operation it is performing, which is perfectly correct, but makes our life more difficult. However our operations don't map to simple functions, and have common infrastructure. These common infrastructure functions can be called from either context and so don't know which error to raise. Using this combined type everywhere fixes the problem at very small cost (multiple inheritance!). """ ftp-cloudfs-0.35/ftpcloudfs/utils.py0000664000175000017500000000322012670255213017136 0ustar jjmjjm00000000000000import types import fcntl import os class PidFile(object): """Context manager that locks a pid file.""" def __init__(self, path): self.path = path self.pidfile = None def close(self): pidfile = self.pidfile self.pidfile = None pidfile.close() def __enter__(self): self.pidfile = open(self.path, "a+") fcntl.flock(self.pidfile.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) self.pidfile.seek(0) self.pidfile.truncate() self.pidfile.write(str(os.getpid())) self.pidfile.flush() self.pidfile.seek(0) return self.pidfile def __exit__(self, exc_type=None, exc_value=None, exc_tb=None): if self.pidfile: self.pidfile.close() os.remove(self.path) # compatibility later for swifclient < 2.7.0 def smart_unicode(s, encoding='utf-8'): if isinstance(s, unicode): return s else: return unicode(s, encoding) #from django.utils def smart_str(s, encoding='utf-8', strings_only=False, errors='strict'): if strings_only and isinstance(s, (types.NoneType, int)): return s elif not isinstance(s, basestring): try: return str(s) except UnicodeEncodeError: if isinstance(s, Exception): return ' '.join([smart_str(arg, encoding, strings_only, errors) for arg in s]) return unicode(s).encode(encoding, errors) elif isinstance(s, unicode): return s.encode(encoding, errors) elif s and encoding != 'utf-8': return s.decode('utf-8', errors).encode(encoding, errors) else: return s ftp-cloudfs-0.35/ftpcloudfs/__init__.py0000664000175000017500000000000012211103543017513 0ustar jjmjjm00000000000000ftp-cloudfs-0.35/ftpcloudfs.conf.example0000664000175000017500000000443112670250642017733 0ustar jjmjjm00000000000000# ftpcloudfs example configuration file # # Defaults are shown in the comments. # Configuration tokens don't require quotes. # [ftpcloudfs] # FTP banner (%v version, %f ftp handler version, %s swift client version) # banner = ftp-cloudfs %v using pyftpdlib %f (switfclient %s) ready. # Port to bind. # port = 2021 # Address to bind. # bind-address = 127.0.0.1 # Authentication URL (required) # auth-url = (empty) # Allow to access servers without checking SSL certs # insecure = no # DEPRECATED: Number of workers to use (no effect) # workers = (empty) # Memcache server(s) for external cache (eg 127.0.0.1:11211) # Can be a comma-separated list. # memcache = (empty) # Maximum number of client connections per IP # default is 0 (no limit) # max-cons-per-ip = 0 # Allow data connection from a different IP than the control connection. # Useful in situations where the control connection is proxied. Enables # site-to-site transfers, but also introduces a security risk. # permit-foreign-addresses = no # Large file support. # Specify a size in MB to split large files. # split-large-files = (empty) # Hide .part directory from large files # hide-part-dir = no # Be verbose on logging. # verbose = no # Enable logging to the system logger. # syslog = no # Log file location. # log-file = (empty) # Pid file location when in daemon mode. # pid-file = (empty) # UID to drop privileges when in daemon mode. # uid = (empty) # GID to drop priviliges when in daemon mode. # gid = (empty) # Masquerade IP address in case your server is behind a firewall or NATed. # masquerade-firewall = (empty) # Passive ports to be used for data transfers. Expected to be a port range # (endpoints included) in integer:integer format (eg. 60000:65535). # By default the operating system will assign a port. # passive-ports = (empty) # Auth 2.0 (Keystone), requires keystoneclient # keystone-auth = no # Region name to be used with Auth 2.0 (optional) # keystone-region-name = (empty) # Tenant separator to be used with Auth 2.0 (eg. TENANT.USERNAME) # keystone-tenant-separator = . # Service type to be used with Auth 2.0. # keystone-service-type = object-store # Endpoint type to be used with Auth 2.0. # keystone-endpoint-type = publicURL # Use Rackspace's ServiceNet internal network. # rackspace-service-net = no # EOF ftp-cloudfs-0.35/setup.cfg0000664000175000017500000000007312670503005015072 0ustar jjmjjm00000000000000[egg_info] tag_build = tag_date = 0 tag_svn_revision = 0