ftputil-3.4/0000755000175000017470000000000013200532636012303 5ustar debiandebianftputil-3.4/ftputil/0000755000175000017470000000000013200532636013772 5ustar debiandebianftputil-3.4/ftputil/path.py0000644000175000017470000002014213175171032015277 0ustar debiandebian# Copyright (C) 2003-2013, Stefan Schwarzer # and ftputil contributors (see `doc/contributors.txt`) # See the file LICENSE for licensing terms. """ ftputil.path - simulate `os.path` for FTP servers """ from __future__ import absolute_import from __future__ import unicode_literals import posixpath import stat import ftputil.compat import ftputil.error import ftputil.tool # The `_Path` class shouldn't be used directly by clients of the # ftputil library. __all__ = [] class _Path(object): """ Support class resembling `os.path`, accessible from the `FTPHost` object, e. g. as `FTPHost().path.abspath(path)`. Hint: substitute `os` with the `FTPHost` object. """ # `_Path` needs to provide all methods of `os.path`. # pylint: disable=too-many-instance-attributes def __init__(self, host): self._host = host # Delegate these to the `posixpath` module. # pylint: disable=invalid-name pp = posixpath self.dirname = pp.dirname self.basename = pp.basename self.isabs = pp.isabs self.commonprefix = pp.commonprefix self.split = pp.split self.splitdrive = pp.splitdrive self.splitext = pp.splitext self.normcase = pp.normcase self.normpath = pp.normpath def abspath(self, path): """Return an absolute path.""" original_path = path path = ftputil.tool.as_unicode(path) if not self.isabs(path): path = self.join(self._host.getcwd(), path) return ftputil.tool.same_string_type_as(original_path, self.normpath(path)) def exists(self, path): """Return true if the path exists.""" try: lstat_result = self._host.lstat( path, _exception_for_missing_path=False) return lstat_result is not None except ftputil.error.RootDirError: return True def getmtime(self, path): """ Return the timestamp for the last modification for `path` as a float. This will raise `PermanentError` if the path doesn't exist, but maybe other exceptions depending on the state of the server (e. g. timeout). """ return self._host.stat(path).st_mtime def getsize(self, path): """ Return the size of the `path` item as an integer. This will raise `PermanentError` if the path doesn't exist, but maybe raise other exceptions depending on the state of the server (e. g. timeout). """ return self._host.stat(path).st_size @staticmethod def join(*paths): """ Join the path component from `paths` and return the joined path. All of these paths must be either unicode strings or byte strings. If not, `join` raises a `TypeError`. """ # These checks are implicitly done by Python 3, but not by # Python 2. all_paths_are_unicode = all( (isinstance(path, ftputil.compat.unicode_type) for path in paths)) all_paths_are_bytes = all( (isinstance(path, ftputil.compat.bytes_type) for path in paths)) if all_paths_are_unicode or all_paths_are_bytes: return posixpath.join(*paths) else: # Python 3 raises this exception for mixed strings # in `os.path.join`, so also use this exception. raise TypeError( "can't mix unicode strings and bytes in path components") # Check whether a path is a regular file/dir/link. For the first # two cases follow links (like in `os.path`). # # Implementation note: The previous implementations simply called # `stat` or `lstat` and returned `False` if they ended with # raising a `PermanentError`. That exception usually used to # signal a missing path. This approach has the problem, however, # that exceptions caused by code earlier in `lstat` are obscured # by the exception handling in `isfile`, `isdir` and `islink`. def _is_file_system_entity(self, path, dir_or_file): """ Return `True` if `path` represents the file system entity described by `dir_or_file` ("dir" or "file"). Return `False` if `path` isn't a directory or file, respectively or if `path` leads to an infinite chain of links. """ assert dir_or_file in ["dir", "file"] # Consider differences between directories and files. if dir_or_file == "dir": should_look_for_dir = True stat_function = stat.S_ISDIR else: should_look_for_dir = False stat_function = stat.S_ISREG # path = ftputil.tool.as_unicode(path) # Workaround if we can't go up from the current directory. # The result from `getcwd` should already be normalized. if self.normpath(path) == self._host.getcwd(): return should_look_for_dir try: stat_result = self._host.stat( path, _exception_for_missing_path=False) except ftputil.error.RecursiveLinksError: return False except ftputil.error.RootDirError: return should_look_for_dir else: if stat_result is None: # Non-existent path return False else: return stat_function(stat_result.st_mode) def isdir(self, path): """ Return true if the `path` exists and corresponds to a directory (no link). A non-existing path does _not_ cause a `PermanentError`. """ return self._is_file_system_entity(path, "dir") def isfile(self, path): """ Return true if the `path` exists and corresponds to a regular file (no link). A non-existing path does _not_ cause a `PermanentError`. """ return self._is_file_system_entity(path, "file") def islink(self, path): """ Return true if the `path` exists and is a link. A non-existing path does _not_ cause a `PermanentError`. """ path = ftputil.tool.as_unicode(path) try: lstat_result = self._host.lstat( path, _exception_for_missing_path=False) except ftputil.error.RootDirError: return False else: if lstat_result is None: # Non-existent path return False else: return stat.S_ISLNK(lstat_result.st_mode) def walk(self, top, func, arg): """ Directory tree walk with callback function. For each directory in the directory tree rooted at top (including top itself, but excluding "." and ".."), call func(arg, dirname, fnames). dirname is the name of the directory, and fnames a list of the names of the files and subdirectories in dirname (excluding "." and ".."). func may modify the fnames list in-place (e.g. via del or slice assignment), and walk will only recurse into the subdirectories whose names remain in fnames; this can be used to implement a filter, or to impose a specific order of visiting. No semantics are defined for, or required of, arg, beyond that arg is always passed to func. It can be used, e.g., to pass a filename pattern, or a mutable object designed to accumulate statistics. Passing None for arg is common. """ top = ftputil.tool.as_unicode(top) # This code (and the above documentation) is taken from # `posixpath.py`, with slight modifications. try: names = self._host.listdir(top) except OSError: return func(arg, top, names) for name in names: name = self.join(top, name) try: stat_result = self._host.lstat(name) except OSError: continue if stat.S_ISDIR(stat_result[stat.ST_MODE]): self.walk(name, func, arg) ftputil-3.4/ftputil/session_adapter.py0000644000175000017470000000706113135113162017527 0ustar debiandebian# Copyright (C) 2015, Stefan Schwarzer # and ftputil contributors (see `doc/contributors.txt`) # See the file LICENSE for licensing terms. """ Session adapter for `ftplib.FTP`-compatible session factories to make them usable with ftputil under Python 2. `ftplib.FTP` under Python 2 doesn't work with unicode strings that contain non-ASCII characters (see ticket #100). Since ftputil converts string arguments to unicode strings as soon as possible, this even affects calls to `ftputil.FTPHost` methods when the string arguments are byte strings. ftputil client code V V `ftputil.FTPHost` methods V V convert byte strings to unicode (inside `FTPHost` methods) V V session adapter converts from unicode to byte strings V V `ftplib.FTP` (or other session) methods You may wonder why we convert byte strings to unicode strings in `ftputil.FTPHost` and unicode strings back to byte strings in the adapter. To make the string handling in ftputil consistent for Python 2 and 3, ftputil tries to work everywhere with the same string type. Because the focus of ftputil is on modern Python (i. e. Python 3), this universal string type is unicode. Indeed the string arguments for `ftplib.FTP` under Python 3 are all unicode strings. Having different code for Python 2 and Python 3 all over in ftputil would be bad for maintainability. (ftputil is complicated enough as it is.) Therefore, this adapter is the only place to deal with the preferred string type of `ftplib` under Python 2 vs. Python 3. """ from __future__ import unicode_literals import ftputil.compat import ftputil.tool # Shouldn't be used by ftputil client code __all__ = [] # Shortcut as_bytes = ftputil.tool.as_bytes # We only have to adapt the methods that are directly called by # ftputil and only those that take string arguments. # # Keep the call signature of each adaptor method the same as the # signature of the adapted method so that code that introspects # the signature of the method still works. class SessionAdapter(object): def __init__(self, session): self._session = session def voidcmd(self, cmd): cmd = as_bytes(cmd) return self._session.voidcmd(cmd) def transfercmd(self, cmd, rest=None): cmd = as_bytes(cmd) return self._session.transfercmd(cmd, rest) def dir(self, *args): # This is somewhat tricky, since some of the args may not # be strings. The last argument may be a callback. args = list(args) for index, arg in enumerate(args): # Replace only unicode strings with a corresponding # byte string. if isinstance(arg, ftputil.compat.unicode_type): args[index] = as_bytes(arg) return self._session.dir(*args) def rename(self, fromname, toname): fromname = as_bytes(fromname) toname = as_bytes(toname) return self._session.rename(fromname, toname) def delete(self, filename): filename = as_bytes(filename) return self._session.delete(filename) def cwd(self, dirname): dirname = as_bytes(dirname) return self._session.cwd(dirname) def mkd(self, dirname): dirname = as_bytes(dirname) return self._session.mkd(dirname) def rmd(self, dirname): dirname = as_bytes(dirname) return self._session.rmd(dirname) # Dispatch to session itself for methods that don't need string # conversions. def __getattr__(self, name): return getattr(self._session, name) ftputil-3.4/ftputil/file.py0000644000175000017470000001742113141663165015276 0ustar debiandebian# Copyright (C) 2003-2015, Stefan Schwarzer # and ftputil contributors (see `doc/contributors.txt`) # See the file LICENSE for licensing terms. """ ftputil.file - support for file-like objects on FTP servers """ from __future__ import print_function from __future__ import unicode_literals import io import ftputil.compat import ftputil.error import ftputil.socket_file_adapter # This module shouldn't be used by clients of the ftputil library. __all__ = [] class FTPFile(object): """ Represents a file-like object associated with an FTP host. File and socket are closed appropriately if the `close` method is called. """ # Set timeout in seconds when closing file connections (see ticket #51). _close_timeout = 5 def __init__(self, host): """Construct the file(-like) object.""" self._host = host # pylint: disable=protected-access self._session = host._session # The file is still closed. self.closed = True self._conn = None self._fobj = None def _open(self, path, mode, buffering=None, encoding=None, errors=None, newline=None, rest=None): """ Open the remote file with given path name and mode. Contrary to the `open` builtin, this method returns `None`, instead this file object is modified in-place. """ # We use the same arguments as in `io.open`. # pylint: disable=too-many-arguments # # `buffering` argument isn't used at this time. # pylint: disable=unused-argument # # Python 3's `socket.makefile` supports the same interface as # the new `open` builtin, but Python 2 supports only a mode, # but doesn't return an object with the proper interface to # wrap it in `io.TextIOWrapper`. # # Therefore, to make the code work on Python 2 _and_ 3, use # `socket.makefile` to always create a binary file and under # Python 2 wrap it in an adapter class. # # Check mode. if "a" in mode: raise ftputil.error.FTPIOError("append mode not supported") if mode not in ("r", "rb", "rt", "w", "wb", "wt"): raise ftputil.error.FTPIOError("invalid mode '{0}'".format(mode)) if "b" in mode and "t" in mode: # Raise a `ValueError` like Python would. raise ValueError("can't have text and binary mode at once") # Convenience variables is_binary_mode = "b" in mode is_read_mode = "r" in mode # `rest` is only allowed for binary mode. if (not is_binary_mode) and (rest is not None): raise ftputil.error.CommandNotImplementedError( "`rest` argument can't be used for text files") # Always use binary mode (see comments above). transfer_type = "I" command = "TYPE {0}".format(transfer_type) with ftputil.error.ftplib_error_to_ftp_io_error: self._session.voidcmd(command) # Make transfer command. command_type = "RETR" if is_read_mode else "STOR" command = "{0} {1}".format(command_type, path) # Force to binary regardless of transfer type (see above). makefile_mode = mode makefile_mode = makefile_mode.replace("t", "") if not "b" in makefile_mode: makefile_mode += "b" # Get connection and file object. with ftputil.error.ftplib_error_to_ftp_io_error: self._conn = self._session.transfercmd(command, rest) # The file object. Under Python 3, this will already be a # `BufferedReader` or `BufferedWriter` object. fobj = self._conn.makefile(makefile_mode) if ftputil.compat.python_version == 2: BufferedIOAdapter = ftputil.socket_file_adapter.BufferedIOAdapter if is_read_mode: fobj = BufferedIOAdapter(fobj, is_readable=True) else: fobj = BufferedIOAdapter(fobj, is_writable=True) if not is_binary_mode: fobj = io.TextIOWrapper(fobj, encoding=encoding, errors=errors, newline=newline) self._fobj = fobj # This comes last so that `close` won't try to close `FTPFile` # objects without `_conn` and `_fobj` attributes in case of an # error. self.closed = False def __iter__(self): """Return a file iterator.""" return self def __next__(self): """ Return the next line or raise `StopIteration`, if there are no more. """ # Apply implicit line ending conversion for text files. line = self.readline() if line: return line else: raise StopIteration # Although Python 2.6+ has the `next` builtin function already, it # still requires iterators to have a `next` method. next = __next__ # # Context manager methods # def __enter__(self): # Return `self`, so it can be accessed as the variable # component of the `with` statement. return self def __exit__(self, exc_type, exc_val, exc_tb): # We don't need the `exc_*` arguments here # pylint: disable=unused-argument self.close() # Be explicit return False # # Other attributes # def __getattr__(self, attr_name): """ Handle requests for attributes unknown to `FTPFile` objects: delegate the requests to the contained file object. """ if attr_name in ("encoding flush isatty fileno read readline " "readlines seek tell truncate name softspace " "write writelines".split()): return getattr(self._fobj, attr_name) raise AttributeError( "'FTPFile' object has no attribute '{0}'".format(attr_name)) # TODO: Implement `__dir__`? (See # http://docs.python.org/whatsnew/2.6.html#other-language-changes ) def close(self): """Close the `FTPFile`.""" if self.closed: return # Timeout value to restore, see below. # Statement works only before the try/finally statement, # otherwise Python raises an `UnboundLocalError`. old_timeout = self._session.sock.gettimeout() try: self._fobj.close() self._fobj = None with ftputil.error.ftplib_error_to_ftp_io_error: self._conn.close() # Set a timeout to prevent waiting until server timeout # if we have a server blocking here like in ticket #51. self._session.sock.settimeout(self._close_timeout) try: with ftputil.error.ftplib_error_to_ftp_io_error: self._session.voidresp() except ftputil.error.FTPIOError as exc: # Ignore some errors, see tickets #51 and #17 at # http://ftputil.sschwarzer.net/trac/ticket/51 and # http://ftputil.sschwarzer.net/trac/ticket/17, # respectively. exc = str(exc) error_code = exc[:3] if exc.splitlines()[0] != "timed out" and \ error_code not in ("150", "426", "450", "451"): raise finally: # Restore timeout for socket of `FTPFile`'s `ftplib.FTP` # object in case the connection is reused later. self._session.sock.settimeout(old_timeout) # If something went wrong before, the file is probably # defunct and subsequent calls to `close` won't help # either, so we consider the file closed for practical # purposes. self.closed = True def __getstate__(self): raise TypeError("cannot serialize FTPFile object") ftputil-3.4/ftputil/socket_file_adapter.py0000644000175000017470000000660513135113162020336 0ustar debiandebian# Copyright (C) 2014, Stefan Schwarzer # and ftputil contributors (see `doc/contributors.txt`) # See the file LICENSE for licensing terms. """ See docstring of class `BufferedIO`. """ import io __all__ = ["BufferedIOAdapter"] class BufferedIOAdapter(io.BufferedIOBase): """ Adapt a file object returned from `socket.makefile` to the interfaces of `io.BufferedReader` or `io.BufferedWriter`, so that the new object can be wrapped by `io.TextIOWrapper`. This is only needed with Python 2, since in Python 3 `socket.makefile` already returns a `BufferedReader` or `BufferedWriter` object (depending on mode). """ def __init__(self, fobj, is_readable=False, is_writable=False): # Don't call baseclass constructor for this adapter. # pylint: disable=super-init-not-called # # This is the return value of `socket.makefile` and is already # buffered. self.raw = fobj self._is_readable = is_readable self._is_writable = is_writable @property def closed(self): # pylint: disable=missing-docstring return self.raw.closed def close(self): self.raw.close() def fileno(self): return self.raw.fileno() def isatty(self): # It's highly unlikely that this file is interactive. return False def seekable(self): return False # # Interface for `BufferedReader` # def readable(self): return self._is_readable def read(self, *arg): return self.raw.read(*arg) read1 = read def readline(self, *arg): return self.raw.readline(*arg) def readlines(self, *arg): return self.raw.readlines(*arg) def readinto(self, bytearray_): data = self.raw.read(len(bytearray_)) bytearray_[:len(data)] = data return len(data) # # Interface for `BufferedWriter` # def writable(self): return self._is_writable def flush(self): self.raw.flush() # Derived from `socket.py` in Python 2.6 and 2.7. # There doesn't seem to be a public API for this. def _write_buffer_size(self): """Return current size of the write buffer in bytes.""" # pylint: disable=protected-access if hasattr(self.raw, "_wbuf_len"): # Python 2.6.3 - 2.7.5 return self.raw._wbuf_len elif hasattr(self.raw, "_get_wbuf_len"): # Python 2.6 - 2.6.2. (Strictly speaking, all other # Python 2.6 versions have a `_get_wbuf_len` method, but # for 2.6.3 and up it returns `_wbuf_len`). return self.raw._get_wbuf_len() else: # Fallback. In the context of `write` this means the file # appears to be unbuffered. return 0 def write(self, bytes_or_bytearray): # `BufferedWriter.write` has to return the number of written # bytes, but files returned from `socket.makefile` in Python 2 # return `None`. Hence provide a workaround. old_buffer_byte_count = self._write_buffer_size() added_byte_count = len(bytes_or_bytearray) self.raw.write(bytes_or_bytearray) new_buffer_byte_count = self._write_buffer_size() return (old_buffer_byte_count + added_byte_count - new_buffer_byte_count) def writelines(self, lines): self.raw.writelines(lines) ftputil-3.4/ftputil/tool.py0000644000175000017470000000501613135113162015317 0ustar debiandebian# Copyright (C) 2013, Stefan Schwarzer # and ftputil contributors (see `doc/contributors.txt`) # See the file LICENSE for licensing terms. """ tool.py - helper code """ from __future__ import unicode_literals import ftputil.compat as compat __all__ = ["same_string_type_as", "as_bytes", "as_unicode", "as_default_string"] # Encoding to convert between byte string and unicode string. This is # a "lossless" encoding: Strings can be encoded/decoded back and forth # without information loss or causing encoding-related errors. The # `ftplib` module under Python 3 also uses the "latin1" encoding # internally. It's important to use the same encoding here, so that users who # used `ftplib` to create FTP items with non-ASCII characters can access them # in the same way with ftputil. LOSSLESS_ENCODING = "latin1" def same_string_type_as(type_source, content_source): """ Return a string of the same type as `type_source` with the content from `content_source`. If the `type_source` and `content_source` don't have the same type, use `LOSSLESS_ENCODING` above to encode or decode, whatever operation is needed. """ if ( isinstance(type_source, compat.bytes_type) and isinstance(content_source, compat.unicode_type)): return content_source.encode(LOSSLESS_ENCODING) elif ( isinstance(type_source, compat.unicode_type) and isinstance(content_source, compat.bytes_type)): return content_source.decode(LOSSLESS_ENCODING) else: return content_source def as_bytes(string): """ Return the argument `string` converted to a byte string if it's a unicode string. Otherwise just return the string. """ return same_string_type_as(b"", string) def as_unicode(string): """ Return the argument `string` converted to a unicode string if it's a byte string. Otherwise just return the string. """ return same_string_type_as("", string) def as_default_string(string): """ Return the argument `string` converted to a the default string type for the Python version. For unicode strings, `LOSSLESS_ENCODING` is used for encoding or decoding. """ return same_string_type_as(compat.default_string_type(), string) def encode_if_unicode(string, encoding): """ Return the string `string`, encoded with `encoding` if `string` is a unicode string. Otherwise return `string` unchanged. """ if isinstance(string, compat.unicode_type): return string.encode(encoding) else: return string ftputil-3.4/ftputil/lrucache.py0000644000175000017470000002337213135113162016135 0ustar debiandebian# lrucache.py -- a simple LRU (Least-Recently-Used) cache class # Copyright 2004 Evan Prodromou # # Copyright 2009-2013 Stefan Schwarzer # (some changes to the original version) # Licensed under the Academic Free License 2.1 # Licensed for ftputil under the revised BSD license # with permission by the author, Evan Prodromou. Many # thanks, Evan! :-) # # The original file is available at # http://pypi.python.org/pypi/lrucache/0.2 . # arch-tag: LRU cache main module """a simple LRU (Least-Recently-Used) cache module This module provides very simple LRU (Least-Recently-Used) cache functionality. An *in-memory cache* is useful for storing the results of an 'expensive' process (one that takes a lot of time or resources) for later re-use. Typical examples are accessing data from the filesystem, a database, or a network location. If you know you'll need to re-read the data again, it can help to keep it in a cache. You *can* use a Python dictionary as a cache for some purposes. However, if the results you're caching are large, or you have a lot of possible results, this can be impractical memory-wise. An *LRU cache*, on the other hand, only keeps _some_ of the results in memory, which keeps you from overusing resources. The cache is bounded by a maximum size; if you try to add more values to the cache, it will automatically discard the values that you haven't read or written to in the longest time. In other words, the least-recently-used items are discarded. [1]_ .. [1]: 'Discarded' here means 'removed from the cache'. """ from __future__ import unicode_literals import time # The suffix after the hyphen denotes modifications by the # ftputil project with respect to the original version. __version__ = "0.2-12" __all__ = ['CacheKeyError', 'LRUCache', 'DEFAULT_SIZE'] __docformat__ = 'reStructuredText en' # Default size of a new LRUCache object, if no 'size' argument is given. DEFAULT_SIZE = 16 # For Python 2/3 compatibility try: long int_types = (int, long) except NameError: int_types = (int,) class CacheKeyError(KeyError): """Error raised when cache requests fail. When a cache record is accessed which no longer exists (or never did), this error is raised. To avoid it, you may want to check for the existence of a cache record before reading or deleting it. """ pass class LRUCache(object): """Least-Recently-Used (LRU) cache. Instances of this class provide a least-recently-used (LRU) cache. They emulate a Python mapping type. You can use an LRU cache more or less like a Python dictionary, with the exception that objects you put into the cache may be discarded before you take them out. Some example usage:: cache = LRUCache(32) # new cache cache['foo'] = get_file_contents('foo') # or whatever if 'foo' in cache: # if it's still in cache... # use cached version contents = cache['foo'] else: # recalculate contents = get_file_contents('foo') # store in cache for next time cache['foo'] = contents print cache.size # Maximum size print len(cache) # 0 <= len(cache) <= cache.size cache.size = 10 # Auto-shrink on size assignment for i in range(50): # note: larger than cache size cache[i] = i if 0 not in cache: print 'Zero was discarded.' if 42 in cache: del cache[42] # Manual deletion for j in cache: # iterate (in LRU order) print j, cache[j] # iterator produces keys, not values """ class _Node(object): """Record of a cached value. Not for public consumption.""" def __init__(self, key, obj, timestamp, sort_key): object.__init__(self) self.key = key self.obj = obj self.atime = timestamp self.mtime = self.atime self._sort_key = sort_key def __lt__(self, other): # Seems to be preferred over `__cmp__`, at least in newer # Python versions. Uses only around 60 % of the time # with respect to `__cmp__`. return self._sort_key < other._sort_key def __repr__(self): return "<%s %s => %s (%s)>" % \ (self.__class__, self.key, self.obj, \ time.asctime(time.localtime(self.atime))) def __init__(self, size=DEFAULT_SIZE): """Init the `LRUCache` object. `size` is the initial _maximum_ size of the cache. The size can be changed by setting the `size` attribute. """ self.clear() # Maximum size of the cache. If more than 'size' elements are # added to the cache, the least-recently-used ones will be # discarded. This assignment implicitly checks the size value. self.size = size def clear(self): """Clear the cache, removing all elements. The `size` attribute of the cache isn't modified. """ # pylint: disable=attribute-defined-outside-init self.__heap = [] self.__dict = {} self.__counter = 0 def _sort_key(self): """Return a new integer value upon every call. Cache nodes need a monotonically increasing time indicator. `time.time()` and `time.clock()` don't guarantee this in a platform-independent way. See http://ftputil.sschwarzer.net/trac/ticket/32 for details. """ self.__counter += 1 return self.__counter def __len__(self): """Return _current_ number of cache entries. This may be different from the value of the `size` attribute. """ return len(self.__heap) def __contains__(self, key): """Return `True` if the item denoted by `key` is in the cache.""" return key in self.__dict def __setitem__(self, key, obj): """Store item `obj` in the cache under the key `key`. If the number of elements after the addition of a new key would exceed the maximum cache size, the least recently used item in the cache is "forgotten". """ heap = self.__heap dict_ = self.__dict if key in dict_: node = dict_[key] # Update node object in-place. node.obj = obj node.atime = time.time() node.mtime = node.atime node._sort_key = self._sort_key() else: # The size of the heap can be at most the value of # `self.size` because `__setattr__` decreases the cache # size if the new size value is smaller; so we don't # need a loop _here_. if len(heap) == self.size: lru_node = min(heap) heap.remove(lru_node) del dict_[lru_node.key] node = self._Node(key, obj, time.time(), self._sort_key()) dict_[key] = node heap.append(node) def __getitem__(self, key): """Return the item stored under `key` key. If no such key is present in the cache, raise a `CacheKeyError`. """ if not key in self.__dict: raise CacheKeyError(key) else: node = self.__dict[key] # Update node object in-place. node.atime = time.time() node._sort_key = self._sort_key() return node.obj def __delitem__(self, key): """Delete the item stored under `key` key. If no such key is present in the cache, raise a `CacheKeyError`. """ if not key in self.__dict: raise CacheKeyError(key) else: node = self.__dict[key] self.__heap.remove(node) del self.__dict[key] return node.obj def __iter__(self): """Iterate over the cache, from the least to the most recently accessed item. """ self.__heap.sort() for node in self.__heap: yield node.key def __setattr__(self, name, value): """If the name of the attribute is "size", set the _maximum_ size of the cache to the supplied value. """ object.__setattr__(self, name, value) # Automagically shrink heap on resize. if name == 'size': size = value if not isinstance(size, int_types): raise TypeError("cache size (%r) must be an integer" % size) if size <= 0: raise ValueError("cache size (%d) must be positive" % size) heap = self.__heap dict_ = self.__dict # Do we need to remove anything at all? if len(heap) <= self.size: return # Remove enough nodes to reach the new size. heap.sort() node_count_to_remove = len(heap) - self.size for node in heap[:node_count_to_remove]: del dict_[node.key] del heap[:node_count_to_remove] def __repr__(self): return "<%s (%d elements)>" % (str(self.__class__), len(self.__heap)) def mtime(self, key): """Return the last modification time for the cache record with key. May be useful for cache instances where the stored values can get "stale", such as caching file or network resource contents. """ if not key in self.__dict: raise CacheKeyError(key) else: node = self.__dict[key] return node.mtime if __name__ == "__main__": cache = LRUCache(25) print(cache) for i in range(50): cache[i] = str(i) print(cache) if 46 in cache: del cache[46] print(cache) cache.size = 10 print(cache) cache[46] = '46' print(cache) print(len(cache)) for c in cache: print(c) print(cache) print(cache.mtime(46)) for c in cache: print(c) ftputil-3.4/ftputil/sync.py0000644000175000017470000001367313135113162015326 0ustar debiandebian# Copyright (C) 2007-2012, Stefan Schwarzer # and ftputil contributors (see `doc/contributors.txt`) # See the file LICENSE for licensing terms. """ Tools for syncing combinations of local and remote directories. *** WARNING: This is an unfinished in-development version! """ # Sync combinations: # - remote -> local (download) # - local -> remote (upload) # - remote -> remote # - local -> local (maybe implicitly possible due to design, but not targeted) from __future__ import unicode_literals import os import shutil from ftputil import FTPHost import ftputil.error __all__ = ["FTPHost", "LocalHost", "Syncer"] # Used for copying file objects; value is 64 KB. CHUNK_SIZE = 64 * 1024 class LocalHost(object): """ Provide an API for local directories and files so we can use the same code as for `FTPHost` instances. """ def open(self, path, mode): """ Return a Python file object for file name `path`, opened in mode `mode`. """ # This is the built-in `open` function, not `os.open`! return open(path, mode) def time_shift(self): """ Return the time shift value (see methods `set_time_shift` and `time_shift` in class `FTPHost` for a definition). By definition, the value is zero for local file systems. """ return 0.0 def __getattr__(self, attr): return getattr(os, attr) class Syncer(object): """ Control synchronization between combinations of local and remote directories and files. """ def __init__(self, source, target): """ Init the `FTPSyncer` instance. Each of `source` and `target` is either an `FTPHost` or a `LocalHost` object. The source and target directories, resp. have to be set with the `chdir` command before passing them in. The semantics is so that the items under the source directory will show up under the target directory after the synchronization (unless there's an error). """ self._source = source self._target = target def _mkdir(self, target_dir): """ Try to create the target directory `target_dir`. If it already exists, don't do anything. If the directory is present but it's actually a file, raise a `SyncError`. """ #TODO Handle setting of target mtime according to source mtime # (beware of rootdir anomalies; try to handle them as well). #print "Making", target_dir if self._target.path.isfile(target_dir): raise ftputil.error.SyncError( "target dir '{0}' is actually a file".format(target_dir)) # Deliberately use an `isdir` test instead of `try/except`. The # latter approach might mask other errors we want to see, e. g. # insufficient permissions. if not self._target.path.isdir(target_dir): self._target.mkdir(target_dir) def _sync_file(self, source_file, target_file): #XXX This duplicates code from `FTPHost._copyfileobj`. Maybe # implement the upload and download methods in terms of # `_sync_file`, or maybe not? #TODO Handle `IOError`s #TODO Handle conditional copy #TODO Handle setting of target mtime according to source mtime # (beware of rootdir anomalies; try to handle them as well). #print "Syncing", source_file, "->", target_file source = self._source.open(source_file, "rb") try: target = self._target.open(target_file, "wb") try: shutil.copyfileobj(source, target, length=CHUNK_SIZE) finally: target.close() finally: source.close() def _fix_sep_for_target(self, path): """ Return the string `path` with appropriate path separators for the target file system. """ return path.replace(self._source.sep, self._target.sep) def _sync_tree(self, source_dir, target_dir): """ Synchronize the source and the target directory tree by updating the target to match the source as far as possible. Current limitations: - _don't_ delete items which are on the target path but not on the source path - files are always copied, the modification timestamps are not compared - all files are copied in binary mode, never in ASCII/text mode - incomplete error handling """ self._mkdir(target_dir) for dirpath, dir_names, file_names in self._source.walk(source_dir): for dir_name in dir_names: inner_source_dir = self._source.path.join(dirpath, dir_name) inner_target_dir = inner_source_dir.replace(source_dir, target_dir, 1) inner_target_dir = self._fix_sep_for_target(inner_target_dir) self._mkdir(inner_target_dir) for file_name in file_names: source_file = self._source.path.join(dirpath, file_name) target_file = source_file.replace(source_dir, target_dir, 1) target_file = self._fix_sep_for_target(target_file) self._sync_file(source_file, target_file) def sync(self, source_path, target_path): """ Synchronize `source_path` and `target_path` (both are strings, each denoting a directory or file path), i. e. update the target path so that it's a copy of the source path. This method handles both directory trees and single files. """ #TODO Handle making of missing intermediate directories source_path = self._source.path.abspath(source_path) target_path = self._target.path.abspath(target_path) if self._source.path.isfile(source_path): self._sync_file(source_path, target_path) else: self._sync_tree(source_path, target_path) ftputil-3.4/ftputil/error.py0000644000175000017470000001207613175164562015515 0ustar debiandebian# Copyright (C) 2003-2014, Stefan Schwarzer # and ftputil contributors (see `doc/contributors.txt`) # See the file LICENSE for licensing terms. """ ftputil.error - exception classes and wrappers """ # pylint: disable=too-many-ancestors from __future__ import unicode_literals import ftplib import ftputil.tool import ftputil.version # You _can_ import these with `from ftputil.error import *`, - but # it's _not_ recommended. __all__ = [ "InternalError", "RootDirError", "InaccessibleLoginDirError", "TimeShiftError", "ParserError", "KeepAliveError", "FTPOSError", "TemporaryError", "PermanentError", "CommandNotImplementedError", "SyncError", "FTPIOError", ] class FTPError(Exception): """General ftputil error class.""" # In Python 2, we can't use a keyword argument after `*args`, so # `pop` from `**kwargs`. def __init__(self, *args, **kwargs): super(FTPError, self).__init__(*args) if "original_exception" in kwargs: # Byte string under Python 2. exception_string = str(kwargs.pop("original_exception")) self.strerror = ftputil.tool.as_unicode(exception_string) elif args: # If there was no `original_exception` argument, assume # the first argument is a string. It may be a byte string # though. self.strerror = ftputil.tool.as_unicode(args[0]) else: self.strerror = "" try: self.errno = int(self.strerror[:3]) except ValueError: # `int()` argument couldn't be converted to an integer. self.errno = None self.file_name = None def __str__(self): return "{0}\nDebugging info: {1}".format(self.strerror, ftputil.version.version_info) # Internal errors are those that have more to do with the inner # workings of ftputil than with errors on the server side. class InternalError(FTPError): """Internal error.""" pass class RootDirError(InternalError): """Raised for generic stat calls on the remote root directory.""" pass class InaccessibleLoginDirError(InternalError): """May be raised if the login directory isn't accessible.""" pass class TimeShiftError(InternalError): """Raised for invalid time shift values.""" pass class ParserError(InternalError): """Raised if a line of a remote directory can't be parsed.""" pass class CacheMissError(InternalError): """Raised if a path isn't found in the cache.""" pass # Currently not used class KeepAliveError(InternalError): """Raised if the keep-alive feature failed.""" pass class FTPOSError(FTPError, OSError): """Generic FTP error related to `OSError`.""" pass class TemporaryError(FTPOSError): """Raised for temporary FTP errors (4xx).""" pass class PermanentError(FTPOSError): """Raised for permanent FTP errors (5xx).""" pass class CommandNotImplementedError(PermanentError): """Raised if the server doesn't implement a certain feature (502).""" pass class RecursiveLinksError(PermanentError): """Raised if an infinite link structure is detected.""" pass # Currently not used class SyncError(PermanentError): """Raised for problems specific to syncing directories.""" pass class FtplibErrorToFTPOSError(object): """ Context manager to convert `ftplib` exceptions to exceptions derived from `FTPOSError`. """ def __enter__(self): pass def __exit__(self, exc_type, exc_value, traceback): if exc_type is None: # No exception return if isinstance(exc_value, ftplib.error_temp): raise TemporaryError(*exc_value.args, original_exception=exc_value) elif isinstance(exc_value, ftplib.error_perm): # If `exc_value.args[0]` is present, assume it's a byte or # unicode string. if ( exc_value.args and ftputil.tool.as_unicode(exc_value.args[0]).startswith("502") ): raise CommandNotImplementedError(*exc_value.args) else: raise PermanentError(*exc_value.args, original_exception=exc_value) elif isinstance(exc_value, ftplib.all_errors): raise FTPOSError(*exc_value.args, original_exception=exc_value) else: raise ftplib_error_to_ftp_os_error = FtplibErrorToFTPOSError() class FTPIOError(FTPError, IOError): """Generic FTP error related to `IOError`.""" pass class FtplibErrorToFTPIOError(object): """ Context manager to convert `ftplib` exceptions to `FTPIOError` exceptions. """ def __enter__(self): pass def __exit__(self, exc_type, exc_value, traceback): if exc_type is None: # No exception return if isinstance(exc_value, ftplib.all_errors): raise FTPIOError(*exc_value.args, original_exception=exc_value) else: raise ftplib_error_to_ftp_io_error = FtplibErrorToFTPIOError() ftputil-3.4/ftputil/stat.py0000644000175000017470000007450713175165326015345 0ustar debiandebian# Copyright (C) 2002-2015, Stefan Schwarzer # and ftputil contributors (see `doc/contributors.txt`) # See the file LICENSE for licensing terms. """ ftputil.stat - stat result, parsers, and FTP stat'ing for `ftputil` """ from __future__ import absolute_import from __future__ import unicode_literals import datetime import math import re import stat import time import ftputil.error import ftputil.stat_cache # These can be used to write custom parsers. __all__ = ["StatResult", "Parser", "UnixParser", "MSParser"] # Datetime precision values in seconds. MINUTE_PRECISION = 60 DAY_PRECISION = 24 * 60 * 60 UNKNOWN_PRECISION = None class StatResult(tuple): """ Support class resembling a tuple like that returned from `os.(l)stat`. """ _index_mapping = { "st_mode": 0, "st_ino": 1, "st_dev": 2, "st_nlink": 3, "st_uid": 4, "st_gid": 5, "st_size": 6, "st_atime": 7, "st_mtime": 8, "st_ctime": 9, "_st_name": 10, "_st_target": 11} def __init__(self, sequence): # Don't call `__init__` via `super`. Construction from a # sequence is implicitly handled by `tuple.__new__`, not # `tuple.__init__`. As a by-product, this avoids a # `DeprecationWarning` in Python 2.6+ . # pylint: disable=super-init-not-called # # Use `sequence` parameter to remain compatible to `__new__` # interface. # pylint: disable=unused-argument # # These may be overwritten in a `Parser.parse_line` method. self._st_name = "" self._st_target = None self._st_mtime_precision = UNKNOWN_PRECISION def __getattr__(self, attr_name): if attr_name in self._index_mapping: return self[self._index_mapping[attr_name]] else: raise AttributeError("'StatResult' object has no attribute '{0}'". format(attr_name)) def __repr__(self): # "Invert" `_index_mapping` so that we can look up the names # for the tuple indices. index_to_name = dict((v, k) for k, v in self._index_mapping.items()) argument_strings = [] for index, item in enumerate(self): argument_strings.append("{0}={1!r}".format(index_to_name[index], item)) return "{0}({1})".format(type(self).__name__, ", ".join(argument_strings)) # # FTP directory parsers # class Parser(object): """ Represent a parser for directory lines. Parsers for specific directory formats inherit from this class. """ # Map month abbreviations to month numbers. _month_numbers = { "jan": 1, "feb": 2, "mar": 3, "apr": 4, "may": 5, "jun": 6, "jul": 7, "aug": 8, "sep": 9, "oct": 10, "nov": 11, "dec": 12} _total_regex = re.compile(r"^total\s+\d+") def ignores_line(self, line): """ Return a true value if the line should be ignored, i. e. is assumed to _not_ contain actual directory/file/link data. A typical example are summary lines like "total 23" which are emitted by some FTP servers. If the line should be used to extract stat data from it, return a false value. """ # Ignore empty lines stemming from only a line break. if not line.strip(): # Yes, ignore the line if it's empty. return True # Either a `_SRE_Match` instance or `None` match = self._total_regex.search(line) return bool(match) def parse_line(self, line, time_shift=0.0): """ Return a `StatResult` object as derived from the string `line`. The parser code to use depends on the directory format the FTP server delivers (also see examples at end of file). If the given text line can't be parsed, raise a `ParserError`. For the definition of `time_shift` see the docstring of `FTPHost.set_time_shift` in `ftputil.py`. Not all parsers use the `time_shift` parameter. """ raise NotImplementedError("must be defined by subclass") # # Helper methods for parts of a directory listing line # def parse_unix_mode(self, mode_string): """ Return an integer from the `mode_string`, compatible with the `st_mode` value in stat results. Such a mode string may look like "drwxr-xr-x". If the mode string can't be parsed, raise an `ftputil.error.ParserError`. """ # Allow derived classes to make use of `self`. # pylint: disable=no-self-use if len(mode_string) != 10: raise ftputil.error.ParserError("invalid mode string '{0}'". format(mode_string)) st_mode = 0 #TODO Add support for "S" and sticky bit ("t", "T"). for bit in mode_string[1:10]: bit = (bit != "-") st_mode = (st_mode << 1) + bit if mode_string[3] == "s": st_mode = st_mode | stat.S_ISUID if mode_string[6] == "s": st_mode = st_mode | stat.S_ISGID file_type_to_mode = {"b": stat.S_IFBLK, "c": stat.S_IFCHR, "d": stat.S_IFDIR, "l": stat.S_IFLNK, "p": stat.S_IFIFO, "s": stat.S_IFSOCK, "-": stat.S_IFREG, # Ignore types which `ls` can't make sense of # (assuming the FTP server returns listings # like `ls` does). "?": 0, } file_type = mode_string[0] if file_type in file_type_to_mode: st_mode = st_mode | file_type_to_mode[file_type] else: raise ftputil.error.ParserError( "unknown file type character '{0}'".format(file_type)) return st_mode def _as_int(self, int_string, int_description): """ Return `int_string` converted to an integer. If it can't be converted, raise a `ParserError`, using `int_description` in the error message. For example, if the integer value is a day, pass "day" for `int_description`. """ try: return int(int_string) except ValueError: raise ftputil.error.ParserError("non-integer {0} value {1!r}". format(int_description, int_string)) def _mktime(self, mktime_tuple): """ Return a float value like `time.mktime` does, but ... - Raise a `ParserError` if parts of `mktime_tuple` are invalid (say, a day is 32). - If the resulting float value would be smaller than 0.0 (indicating a time before the "epoch") return a sentinel value of 0.0. Do this also if the native `mktime` implementation would raise an `OverflowError`. """ datetime_tuple = mktime_tuple[:6] try: # Only for sanity checks, we're not interested in the # return value. datetime.datetime(*datetime_tuple) # For example, day == 32. Not all implementations of `mktime` # catch this kind of error. except ValueError: invalid_datetime = ("%04d-%02d-%02d %02d:%02d:%02d" % datetime_tuple) raise ftputil.error.ParserError("invalid datetime {0!r}". format(invalid_datetime)) try: time_float = time.mktime(mktime_tuple) except (OverflowError, ValueError): # Sentinel for times before the epoch, see ticket #83. time_float = 0.0 # Don't allow float values smaller than 0.0 because, according # to https://docs.python.org/3/library/time.html#module-time , # these might be undefined for some platforms. return max(0.0, time_float) def parse_unix_time(self, month_abbreviation, day, year_or_time, time_shift, with_precision=False): """ Return a floating point number, like from `time.mktime`, by parsing the string arguments `month_abbreviation`, `day` and `year_or_time`. The parameter `time_shift` is the difference "time on server" - "time on client" and is available as the `time_shift` parameter in the `parse_line` interface. If `with_precision` is true (default: false), return a two-element tuple consisting of the floating point number as described in the previous paragraph and the precision of the time in seconds. The default is `False` for backward compatibility with custom parsers. The precision value takes into account that, for example, a time string like "May 26 2005" has only a precision of one day. This information is important for the `upload_if_newer` and `download_if_newer` methods in the `FTPHost` class. Times in Unix-style directory listings typically have one of these formats: - "Nov 23 02:33" (month name, day of month, time) - "May 26 2005" (month name, day of month, year) If this method can't make sense of the given arguments, it raises an `ftputil.error.ParserError`. """ try: month = self._month_numbers[month_abbreviation.lower()] except KeyError: raise ftputil.error.ParserError("invalid month abbreviation {0!r}". format(month_abbreviation)) day = self._as_int(day, "day") if ":" not in year_or_time: # `year_or_time` is really a year. year, hour, minute = self._as_int(year_or_time, "year"), 0, 0 st_mtime = self._mktime( (year, month, day, hour, minute, 0, 0, 0, -1) ) st_mtime_precision = DAY_PRECISION else: # `year_or_time` is a time hh:mm. hour, minute = year_or_time.split(":") year, hour, minute = ( None, self._as_int(hour, "hour"), self._as_int(minute, "minute")) # Try the current year year = time.localtime()[0] st_mtime = self._mktime( (year, month, day, hour, minute, 0, 0, 0, -1) ) st_mtime_precision = MINUTE_PRECISION # Rhs of comparison: Transform client time to server time # (as on the lhs), so both can be compared with respect # to the set time shift (see the definition of the time # shift in `FTPHost.set_time_shift`'s docstring). The # last addend allows for small deviations between the # supposed (rounded) and the actual time shift. # # XXX The downside of this "correction" is that there is # a one-minute time interval exactly one year ago that # may cause that datetime to be recognized as the current # datetime, but after all the datetime from the server # can only be exact up to a minute. if st_mtime > time.time() + time_shift + st_mtime_precision: # If it's in the future, use previous year. st_mtime = self._mktime( (year-1, month, day, hour, minute, 0, 0, 0, -1) ) # If we had a datetime before the epoch, the resulting value # 0.0 doesn't tell us anything about the precision. if st_mtime == 0.0: st_mtime_precision = UNKNOWN_PRECISION # if with_precision: return st_mtime, st_mtime_precision else: return st_mtime def parse_ms_time(self, date, time_, time_shift): """ Return a floating point number, like from `time.mktime`, by parsing the string arguments `date` and `time_`. The parameter `time_shift` is the difference "time on server" - "time on client" and can be set as the `time_shift` parameter in the `parse_line` interface. Times in MS-style directory listings typically have the format "10-23-01 03:25PM" (month-day_of_month-two_digit_year, hour:minute, am/pm). If this method can't make sense of the given arguments, it raises an `ftputil.error.ParserError`. """ # Derived classes might want to use `self`. # pylint: disable=no-self-use # # Derived classes may need access to `time_shift`. # pylint: disable=unused-argument # # For the time being, I don't add a `with_precision` # parameter as in the Unix parser because the precision for # the DOS format is always a minute and can be set in # `MSParser.parse_line`. Should you find yourself needing # support for `with_precision` for a derived class, please # send a mail (see ftputil.txt/html). month, day, year = [self._as_int(part, "year/month/day") for part in date.split("-")] if year >= 1000: # We have a four-digit year, so no need for heuristics. pass elif year >= 70: year = 1900 + year else: year = 2000 + year try: hour, minute, am_pm = time_[0:2], time_[3:5], time_[5] except IndexError: raise ftputil.error.ParserError("invalid time string '{0}'". format(time_)) hour, minute = ( self._as_int(hour, "hour"), self._as_int(minute, "minute")) if hour == 12 and am_pm == "A": hour = 0 if hour != 12 and am_pm == "P": hour += 12 st_mtime = self._mktime( (year, month, day, hour, minute, 0, 0, 0, -1) ) return st_mtime class UnixParser(Parser): """`Parser` class for Unix-specific directory format.""" @staticmethod def _split_line(line): """ Split a line in metadata, nlink, user, group, size, month, day, year_or_time and name and return the result as an nine-element list of these values. If the name is a link, it will be encoded as a string "link_name -> link_target". """ # This method encapsulates the recognition of an unusual # Unix format variant (see ticket # http://ftputil.sschwarzer.net/trac/ticket/12 ). line_parts = line.split() FIELD_COUNT_WITHOUT_USERID = 8 FIELD_COUNT_WITH_USERID = FIELD_COUNT_WITHOUT_USERID + 1 if len(line_parts) < FIELD_COUNT_WITHOUT_USERID: # No known Unix-style format raise ftputil.error.ParserError("line '{0}' can't be parsed". format(line)) # If we have a valid format (either with or without user id field), # the field with index 5 is either the month abbreviation or a day. try: int(line_parts[5]) except ValueError: # Month abbreviation, "invalid literal for int" line_parts = line.split(None, FIELD_COUNT_WITH_USERID-1) else: # Day line_parts = line.split(None, FIELD_COUNT_WITHOUT_USERID-1) USER_FIELD_INDEX = 2 line_parts.insert(USER_FIELD_INDEX, None) return line_parts def parse_line(self, line, time_shift=0.0): """ Return a `StatResult` instance corresponding to the given text line. The `time_shift` value is needed to determine to which year a datetime without an explicit year belongs. If the line can't be parsed, raise a `ParserError`. """ # The local variables are rather simple. # pylint: disable=too-many-locals try: mode_string, nlink, user, group, size, month, day, \ year_or_time, name = self._split_line(line) # We can get a `ValueError` here if the name is blank (see # ticket #69). This is a strange use case, but at least we # should raise the exception the docstring mentions. except ValueError as exc: raise ftputil.error.ParserError(str(exc)) # st_mode st_mode = self.parse_unix_mode(mode_string) # st_ino, st_dev, st_nlink, st_uid, st_gid, st_size, st_atime st_ino = None st_dev = None st_nlink = int(nlink) st_uid = user st_gid = group st_size = int(size) st_atime = None # st_mtime st_mtime, st_mtime_precision = \ self.parse_unix_time(month, day, year_or_time, time_shift, with_precision=True) # st_ctime st_ctime = None # st_name if name.count(" -> ") > 1: # If we have more than one arrow we can't tell where the link # name ends and the target name starts. raise ftputil.error.ParserError( '''name '{0}' contains more than one "->"'''.format(name)) elif name.count(" -> ") == 1: st_name, st_target = name.split(" -> ") else: st_name, st_target = name, None stat_result = StatResult( (st_mode, st_ino, st_dev, st_nlink, st_uid, st_gid, st_size, st_atime, st_mtime, st_ctime) ) # These attributes are kind of "half-official". I'm not # sure whether they should be used by ftputil client code. # pylint: disable=protected-access stat_result._st_mtime_precision = st_mtime_precision stat_result._st_name = st_name stat_result._st_target = st_target return stat_result class MSParser(Parser): """`Parser` class for MS-specific directory format.""" def parse_line(self, line, time_shift=0.0): """ Return a `StatResult` instance corresponding to the given text line from a FTP server which emits "Microsoft format" (see end of file). If the line can't be parsed, raise a `ParserError`. The parameter `time_shift` isn't used in this method but is listed for compatibility with the base class. """ # The local variables are rather simple. # pylint: disable=too-many-locals try: date, time_, dir_or_size, name = line.split(None, 3) except ValueError: # "unpack list of wrong size" raise ftputil.error.ParserError("line '{0}' can't be parsed". format(line)) # st_mode # Default to read access only; in fact, we can't tell. st_mode = 0o400 if dir_or_size == "": st_mode = st_mode | stat.S_IFDIR else: st_mode = st_mode | stat.S_IFREG # st_ino, st_dev, st_nlink, st_uid, st_gid st_ino = None st_dev = None st_nlink = None st_uid = None st_gid = None # st_size if dir_or_size != "": try: st_size = int(dir_or_size) except ValueError: raise ftputil.error.ParserError("invalid size {0}". format(dir_or_size)) else: st_size = None # st_atime st_atime = None # st_mtime st_mtime = self.parse_ms_time(date, time_, time_shift) # st_ctime st_ctime = None stat_result = StatResult( (st_mode, st_ino, st_dev, st_nlink, st_uid, st_gid, st_size, st_atime, st_mtime, st_ctime) ) # These attributes are kind of "half-official". I'm not # sure whether they should be used by ftputil client code. # pylint: disable=protected-access # _st_name and _st_target stat_result._st_name = name stat_result._st_target = None # mtime precision in seconds # If we had a datetime before the epoch, the resulting value # 0.0 doesn't tell us anything about the precision. if st_mtime == 0.0: stat_result._st_mtime_precision = UNKNOWN_PRECISION else: stat_result._st_mtime_precision = MINUTE_PRECISION return stat_result # # Stat'ing operations for files on an FTP server # class _Stat(object): """Methods for stat'ing directories, links and regular files.""" def __init__(self, host): self._host = host self._path = host.path # Use the Unix directory parser by default. self._parser = UnixParser() # Allow one chance to switch to another parser if the default # doesn't work. self._allow_parser_switching = True # Cache only lstat results. `stat` works locally on `lstat` results. self._lstat_cache = ftputil.stat_cache.StatCache() def _host_dir(self, path): """ Return a list of lines, as fetched by FTP's `LIST` command, when applied to `path`. """ return self._host._dir(path) def _stat_results_from_dir(self, path): """ Yield stat results extracted from the directory listing `path`. Omit the special entries for the directory itself and its parent directory. """ lines = self._host_dir(path) # `cache` is the "high-level" `StatCache` object whereas # `cache._cache` is the "low-level" `LRUCache` object. cache = self._lstat_cache # Auto-grow cache if the cache up to now can't hold as many # entries as there are in the directory `path`. if cache._enabled and len(lines) >= cache._cache.size: new_size = int(math.ceil(1.1 * len(lines))) cache.resize(new_size) # Yield stat results from lines. for line in lines: if self._parser.ignores_line(line): continue # For `listdir`, we are interested in just the names, # but we use the `time_shift` parameter to have the # correct timestamp values in the cache. stat_result = self._parser.parse_line(line, self._host.time_shift()) if stat_result._st_name in [self._host.curdir, self._host.pardir]: continue loop_path = self._path.join(path, stat_result._st_name) self._lstat_cache[loop_path] = stat_result yield stat_result def _real_listdir(self, path): """ Return a list of directories, files etc. in the directory named `path`. Like `os.listdir` the returned list elements have the type of the path argument. If the directory listing from the server can't be parsed, raise a `ParserError`. """ # We _can't_ put this check into `FTPHost._dir`; see its docstring. path = self._path.abspath(path) # `listdir` should only be allowed for directories and links to them. if not self._path.isdir(path): raise ftputil.error.PermanentError( "550 {0}: no such directory or wrong directory parser used". format(path)) # Set up for `for` loop. names = [] for stat_result in self._stat_results_from_dir(path): st_name = stat_result._st_name names.append(st_name) return names def _real_lstat(self, path, _exception_for_missing_path=True): """ Return an object similar to that returned by `os.lstat`. If the directory listing from the server can't be parsed, raise a `ParserError`. If the directory can be parsed and the `path` is not found, raise a `PermanentError`. That means that if the directory containing `path` can't be parsed we get a `ParserError`, independent on the presence of `path` on the server. (`_exception_for_missing_path` is an implementation aid and _not_ intended for use by ftputil clients.) """ path = self._path.abspath(path) # If the path is in the cache, return the lstat result. if path in self._lstat_cache: return self._lstat_cache[path] # Note: (l)stat works by going one directory up and parsing # the output of an FTP `LIST` command. Unfortunately, it is # not possible to do this for the root directory `/`. if path == "/": raise ftputil.error.RootDirError( "can't stat remote root directory") dirname, basename = self._path.split(path) # If even the directory doesn't exist and we don't want the # exception, treat it the same as if the path wasn't found in the # directory's contents (compare below). The use of `isdir` here # causes a recursion but that should be ok because that will at # the latest stop when we've gotten to the root directory. if not self._path.isdir(dirname) and not _exception_for_missing_path: return None # Loop through all lines of the directory listing. We # probably won't need all lines for the particular path but # we want to collect as many stat results in the cache as # possible. lstat_result_for_path = None for stat_result in self._stat_results_from_dir(dirname): # Needed to work without cache or with disabled cache. if stat_result._st_name == basename: lstat_result_for_path = stat_result if lstat_result_for_path is not None: return lstat_result_for_path # Path was not found during the loop. if _exception_for_missing_path: #TODO Use FTP `LIST` command on the file to implicitly use # the usual status code of the server for missing files # (450 vs. 550). raise ftputil.error.PermanentError( "550 {0}: no such file or directory".format(path)) else: # Be explicit. Returning `None` is a signal for # `_Path.exists/isfile/isdir/islink` that the path was # not found. If we would raise an exception, there would # be no distinction between a missing path or a more # severe error in the code above. return None def _real_stat(self, path, _exception_for_missing_path=True): """ Return info from a "stat" call on `path`. If the directory containing `path` can't be parsed, raise a `ParserError`. If the listing can be parsed but the `path` can't be found, raise a `PermanentError`. Also raise a `PermanentError` if there's an endless (cyclic) chain of symbolic links "behind" the `path`. (`_exception_for_missing_path` is an implementation aid and _not_ intended for use by ftputil clients.) """ # Save for error message. original_path = path # Most code in this method is used to detect recursive # link structures. visited_paths = set() while True: # Stat the link if it is one, else the file/directory. lstat_result = self._real_lstat(path, _exception_for_missing_path) if lstat_result is None: return None # If the file is not a link, the `stat` result is the # same as the `lstat` result. if not stat.S_ISLNK(lstat_result.st_mode): return lstat_result # If we stat'ed a link, calculate a normalized path for # the file the link points to. dirname, _ = self._path.split(path) path = self._path.join(dirname, lstat_result._st_target) path = self._path.abspath(self._path.normpath(path)) # Check for cyclic structure. if path in visited_paths: # We had seen this path already. raise ftputil.error.RecursiveLinksError( "recursive link structure detected for remote path '{0}'". format(original_path)) # Remember the path we have encountered. visited_paths.add(path) def __call_with_parser_retry(self, method, *args, **kwargs): """ Call `method` with the `args` and `kwargs` once. If that results in a `ParserError` and only one parser has been used yet, try the other parser. If that still fails, propagate the `ParserError`. """ # Do _not_ set `_allow_parser_switching` in a `finally` clause! # This would cause a `PermanentError` due to a not-found # file in an empty directory to finally establish the # parser - which is wrong. try: result = method(*args, **kwargs) # If a `listdir` call didn't find anything, we can't # say anything about the usefulness of the parser. if (method is not self._real_listdir) and result: self._allow_parser_switching = False return result except ftputil.error.ParserError: if self._allow_parser_switching: self._allow_parser_switching = False self._parser = MSParser() return method(*args, **kwargs) else: raise # Don't use these methods, but instead the corresponding methods # in the `FTPHost` class. def _listdir(self, path): """ Return a list of items in `path`. Raise a `PermanentError` if the path doesn't exist, but maybe raise other exceptions depending on the state of the server (e. g. timeout). """ return self.__call_with_parser_retry(self._real_listdir, path) def _lstat(self, path, _exception_for_missing_path=True): """ Return a `StatResult` without following links. Raise a `PermanentError` if the path doesn't exist, but maybe raise other exceptions depending on the state of the server (e. g. timeout). """ return self.__call_with_parser_retry(self._real_lstat, path, _exception_for_missing_path) def _stat(self, path, _exception_for_missing_path=True): """ Return a `StatResult` with following links. Raise a `PermanentError` if the path doesn't exist, but maybe raise other exceptions depending on the state of the server (e. g. timeout). """ return self.__call_with_parser_retry(self._real_stat, path, _exception_for_missing_path) ftputil-3.4/ftputil/session.py0000644000175000017470000001012613175150054016030 0ustar debiandebian# Copyright (C) 2014, Stefan Schwarzer # and ftputil contributors (see `doc/contributors.txt`) # See the file LICENSE for licensing terms. """ Session factory factory (the two "factory" are intential :-) ) for ftputil. """ from __future__ import unicode_literals import ftplib import ftputil.tool try: import M2Crypto import M2Crypto.ftpslib except ImportError: M2Crypto = None __all__ = ["session_factory"] # In a way, it would be appropriate to call this function # `session_factory_factory`, but that's cumbersome to use. Think of # the function returning a session factory and the shorter name should # be fine. def session_factory(base_class=ftplib.FTP, port=21, use_passive_mode=None, encrypt_data_channel=True, debug_level=None): """ Create and return a session factory according to the keyword arguments. base_class: Base class to use for the session class (e. g. `ftplib.FTP_TLS` or `M2Crypto.ftpslib.FTP_TLS`, default is `ftplib.FTP`). port: Port number (integer) for the command channel (default 21). If you don't know what "command channel" means, use the default or use what the provider gave you as "the FTP port". use_passive_mode: If `True`, explicitly use passive mode. If `False`, explicitly don't use passive mode. If `None` (default), let the `base_class` decide whether it wants to use active or passive mode. encrypt_data_channel: If `True` (the default), call the `prot_p` method of the base class if it has the method. If `False` or `None` (`None` is the default), don't call the method. debug_level: Debug level (integer) to be set on a session instance. The default is `None`, meaning no debugging output. This function should work for the base classes `ftplib.FTP`, `ftplib.FTP_TLS` and `M2Crypto.ftpslib.FTP_TLS` with TLS security. Other base classes should work if they use the same API as `ftplib.FTP`. Usage example: my_session_factory = session_factory( base_class=M2Crypto.ftpslib.FTP_TLS, use_passive_mode=True, encrypt_data_channel=True) with ftputil.FTPHost(host, user, password, session_factory=my_session_factory) as host: ... """ class Session(base_class): """Session factory class created by `session_factory`.""" def __init__(self, host, user, password): # Don't use `super` in case `base_class` isn't a new-style # class (e. g. `ftplib.FTP` in Python 2). base_class.__init__(self) self.connect(host, port) if self._use_m2crypto_ftpslib(): self.auth_tls() self._fix_socket() if debug_level is not None: self.set_debuglevel(debug_level) self.login(user, password) # `set_pasv` can be called with `True` (causing passive # mode) or `False` (causing active mode). if use_passive_mode is not None: self.set_pasv(use_passive_mode) if encrypt_data_channel and hasattr(base_class, "prot_p"): self.prot_p() def _use_m2crypto_ftpslib(self): """ Return `True` if the base class to use is `M2Crypto.ftpslib.FTP_TLS`, else return `False`. """ return (M2Crypto is not None and issubclass(base_class, M2Crypto.ftpslib.FTP_TLS)) def _fix_socket(self): """ Change the socket object so that arguments to `sendall` are converted to byte strings before being used. See the ftputil ticket #78 for details: http://ftputil.sschwarzer.net/trac/ticket/78 """ original_sendall = self.sock.sendall # Bound method, therefore no `self` argument. def sendall(data): data = ftputil.tool.as_bytes(data) return original_sendall(data) self.sock.sendall = sendall return Session ftputil-3.4/ftputil/file_transfer.py0000644000175000017470000001446413135113162017174 0ustar debiandebian# Copyright (C) 2013-2014, Stefan Schwarzer # and ftputil contributors (see `doc/contributors.txt`) # See the file LICENSE for licensing terms. """ file_transfer.py - upload, download and generic file copy """ from __future__ import unicode_literals import io import os import ftputil.stat #TODO Think a bit more about the API before making it public. # # Only `chunks` should be used by clients of the ftputil library. Any # # other functionality is supposed to be used via `FTPHost` objects. # __all__ = ["chunks"] __all__ = [] # Maximum size of chunk in `FTPHost.copyfileobj` in bytes. MAX_COPY_CHUNK_SIZE = 64 * 1024 class LocalFile(object): """ Represent a file on the local side which is to be transferred or is already transferred. """ def __init__(self, name, mode): self.name = os.path.abspath(name) self.mode = mode def exists(self): """ Return `True` if the path representing this file exists. Otherwise return `False`. """ return os.path.exists(self.name) def mtime(self): """Return the timestamp for the last modification in seconds.""" return os.path.getmtime(self.name) def mtime_precision(self): """Return the precision of the last modification time in seconds.""" # Derived classes might want to use `self`. # pylint: disable=no-self-use # # Assume modification timestamps for local file systems are # at least precise up to a second. return 1.0 def fobj(self): """Return a file object for the name/path in the constructor.""" return io.open(self.name, self.mode) class RemoteFile(object): """ Represent a file on the remote side which is to be transferred or is already transferred. """ def __init__(self, ftp_host, name, mode): self._host = ftp_host self._path = ftp_host.path self.name = self._path.abspath(name) self.mode = mode def exists(self): """ Return `True` if the path representing this file exists. Otherwise return `False`. """ return self._path.exists(self.name) def mtime(self): """Return the timestamp for the last modification in seconds.""" # Convert to client time zone (see definition of time # shift in docstring of `FTPHost.set_time_shift`). return self._path.getmtime(self.name) - self._host.time_shift() def mtime_precision(self): """Return the precision of the last modification time in seconds.""" # I think using `stat` instead of `lstat` makes more sense here. return self._host.stat(self.name)._st_mtime_precision def fobj(self): """Return a file object for the name/path in the constructor.""" return self._host.open(self.name, self.mode) def source_is_newer_than_target(source_file, target_file): """ Return `True` if the source is newer than the target, else `False`. Both arguments are `LocalFile` or `RemoteFile` objects. It's assumed that the actual modification time is reported_mtime <= actual_mtime <= reported_mtime + mtime_precision i. e. that the reported mtime is the actual mtime or rounded down (truncated). For the purpose of this test the source is newer than the target if any of the possible actual source modification times is greater than the reported target modification time. In other words: If in doubt, the file should be transferred. This is the only situation where the source is _not_ considered newer than the target: |/////////////////////| possible source mtime |////////| possible target mtime That is, the latest possible actual source modification time is before the first possible actual target modification time. """ if source_file.mtime_precision() is ftputil.stat.UNKNOWN_PRECISION: return True else: return (source_file.mtime() + source_file.mtime_precision() >= target_file.mtime()) def chunks(fobj, max_chunk_size=MAX_COPY_CHUNK_SIZE): """ Return an iterator which yields the contents of the file object. For each iteration, at most `max_chunk_size` bytes are read from `fobj` and yielded as a byte string. If the file object is exhausted, then don't yield any more data but stop the iteration, so the client does _not_ get an empty byte string. Any exceptions resulting from reading the file object are passed through to the client. """ while True: chunk = fobj.read(max_chunk_size) if not chunk: break yield chunk def copyfileobj(source_fobj, target_fobj, max_chunk_size=MAX_COPY_CHUNK_SIZE, callback=None): """Copy data from file-like object source to file-like object target.""" # Inspired by `shutil.copyfileobj` (I don't use the `shutil` # code directly because it might change) for chunk in chunks(source_fobj, max_chunk_size): target_fobj.write(chunk) if callback is not None: callback(chunk) def copy_file(source_file, target_file, conditional, callback): """ Copy a file from `source_file` to `target_file`. These are `LocalFile` or `RemoteFile` objects. Which of them is a local or a remote file, respectively, is determined by the arguments. If `conditional` is true, the file is only copied if the target doesn't exist or is older than the source. If `conditional` is false, the file is copied unconditionally. Return `True` if the file was copied, else `False`. """ if conditional: # Evaluate condition: The target file either doesn't exist or is # older than the source file. If in doubt (due to imprecise # timestamps), perform the transfer. transfer_condition = not target_file.exists() or \ source_is_newer_than_target(source_file, target_file) if not transfer_condition: # We didn't transfer. return False source_fobj = source_file.fobj() try: target_fobj = target_file.fobj() try: copyfileobj(source_fobj, target_fobj, callback=callback) finally: target_fobj.close() finally: source_fobj.close() # Transfer accomplished return True ftputil-3.4/ftputil/host.py0000644000175000017470000012036313175202272015327 0ustar debiandebian# Copyright (C) 2002-2015, Stefan Schwarzer # and ftputil contributors (see `doc/contributors.txt`) # See the file LICENSE for licensing terms. """ `FTPHost` is the central class of the `ftputil` library. See `__init__.py` for an example. """ from __future__ import absolute_import from __future__ import print_function from __future__ import unicode_literals import ftplib import stat import sys import time import warnings import ftputil.error import ftputil.file import ftputil.file_transfer import ftputil.path import ftputil.session_adapter import ftputil.stat import ftputil.tool __all__ = ["FTPHost"] # The "protected" attributes PyLint talks about aren't intended for # clients of the library. `FTPHost` objects need to use some of these # library-internal attributes though. # pylint: disable=protected-access ##################################################################### # `FTPHost` class with several methods similar to those of `os` class FTPHost(object): """FTP host class.""" # Implementation notes: # # Upon every request of a file (`FTPFile` object) a new FTP # session is created (or reused from a cache), leading to a child # session of the `FTPHost` object from which the file is requested. # # This is needed because opening an `FTPFile` will make the # local session object wait for the completion of the transfer. # In fact, code like this would block indefinitely, if the `RETR` # request would be made on the `_session` of the object host: # # host = FTPHost(ftp_server, user, password) # f = host.open("index.html") # host.getcwd() # would block! # # On the other hand, the initially constructed host object will # store references to already established `FTPFile` objects and # reuse an associated connection if its associated `FTPFile` # has been closed. def __init__(self, *args, **kwargs): """Abstract initialization of `FTPHost` object.""" # Store arguments for later operations. self._args = args self._kwargs = kwargs #XXX Maybe put the following in a `reset` method. # The time shift setting shouldn't be reset though. # Make a session according to these arguments. self._session = self._make_session() # Simulate `os.path`. self.path = ftputil.path._Path(self) # lstat, stat, listdir services. self._stat = ftputil.stat._Stat(self) self.stat_cache = self._stat._lstat_cache self.stat_cache.enable() with ftputil.error.ftplib_error_to_ftp_os_error: self._cached_current_dir = \ self.path.normpath(ftputil.tool.as_unicode(self._session.pwd())) # Associated `FTPHost` objects for data transfer. self._children = [] # This is only set to something else than `None` if this # instance represents an `FTPFile`. self._file = None # Now opened. self.closed = False # Set curdir, pardir etc. for the remote host. RFC 959 states # that this is, strictly speaking, dependent on the server OS # but it seems to work at least with Unix and Windows servers. self.curdir, self.pardir, self.sep = ".", "..", "/" # Set default time shift (used in `upload_if_newer` and # `download_if_newer`). self.set_time_shift(0.0) # Use `LIST -a` option by default. If this causes problems, # the user can set the attribute to `False`. warnings.warn( "`use_list_a_option` will default to `False` in ftputil 4.x.x", DeprecationWarning, stacklevel=2) self.use_list_a_option = True def keep_alive(self): """ Try to keep the connection alive in order to avoid server timeouts. Note that this won't help if the connection has already timed out! In this case, `keep_alive` will raise an `TemporaryError`. (Actually, if you get a server timeout, the error - for a specific connection - will be permanent.) """ # Warning: Don't call this method on `FTPHost` instances which # represent file transfers. This may fail in confusing ways. with ftputil.error.ftplib_error_to_ftp_os_error: # Ignore return value. self._session.pwd() # # Dealing with child sessions and file-like objects # (rather low-level) # def _make_session(self): """ Return a new session object according to the current state of this `FTPHost` instance. """ # Don't modify original attributes below. args = self._args[:] kwargs = self._kwargs.copy() # If a session factory has been given on the instantiation of # this `FTPHost` object, use the same factory for this # `FTPHost` object's child sessions. factory = kwargs.pop("session_factory", ftplib.FTP) with ftputil.error.ftplib_error_to_ftp_os_error: session = factory(*args, **kwargs) # Adapt session so that they accept unicode strings with # non-ASCII characters (as long as the string contains only # code points <= 255). See the docstring in module # `session_adapter` for details. if ftputil.compat.python_version == 2: session = ftputil.session_adapter.SessionAdapter(session) return session def _copy(self): """Return a copy of this `FTPHost` object.""" # The copy includes a new session factory return value (aka # session) but doesn't copy the state of `self.getcwd()`. return self.__class__(*self._args, **self._kwargs) def _available_child(self): """ Return an available (i. e. one whose `_file` object is closed and doesn't have a timed-out server connection) child (`FTPHost` object) from the pool of children or `None` if there aren't any. """ #TODO: Currently timed-out child sessions aren't removed and # may collect over time. In very busy or long running # processes, this might slow down an application because the # same stale child sessions have to be processed again and # again. for host in self._children: # Test for timeouts only after testing for a closed file: # - If a file isn't closed, save time; don't bother to access # the remote server. # - If a file transfer on the child is in progress, requesting # the directory is an invalid operation because of the way # the FTP state machine works (see RFC 959). if host._file.closed: try: host._session.pwd() # Under high load, a 226 status response from a # previous download may arrive too late, so that it's # "seen" in the `pwd` call. For now, skip the # potential child session; it will be considered again # when `_available_child` is called the next time. except ftplib.error_reply: continue # Timed-out sessions raise `error_temp`. except ftplib.error_temp: continue # The server may have closed the connection which may # cause `host._session.getline` to raise an `EOFError` # (see ticket #114). except EOFError: continue # Under high load, there may be a socket read timeout # during the last FTP file `close` (see ticket #112). # Note that a socket timeout is quite different from # an FTP session timeout. except OSError: continue else: # Everything's ok; use this `FTPHost` instance. return host # Be explicit. return None def open(self, path, mode="r", buffering=None, encoding=None, errors=None, newline=None, rest=None): """ Return an open file(-like) object which is associated with this `FTPHost` object. The arguments `path`, `mode`, `buffering`, `encoding`, `errors` and `newlines` have the same meaning as for `io.open`. If `rest` is given as an integer, - reading will start at the byte (zero-based) `rest` - writing will overwrite the remote file from byte `rest` This method tries to reuse a child but will generate a new one if none is available. """ # Support the same arguments as `io.open`. # pylint: disable=too-many-arguments path = ftputil.tool.as_unicode(path) host = self._available_child() if host is None: host = self._copy() self._children.append(host) host._file = ftputil.file.FTPFile(host) basedir = self.getcwd() # Prepare for changing the directory (see whitespace workaround # in method `_dir`). if host.path.isabs(path): effective_path = path else: effective_path = host.path.join(basedir, path) effective_dir, effective_file = host.path.split(effective_path) try: # This will fail if the directory isn't accessible at all. host.chdir(effective_dir) except ftputil.error.PermanentError: # Similarly to a failed `file` in a local file system, # raise an `IOError`, not an `OSError`. raise ftputil.error.FTPIOError("remote directory '{0}' doesn't " "exist or has insufficient access rights". format(effective_dir)) host._file._open(effective_file, mode=mode, buffering=buffering, encoding=encoding, errors=errors, newline=newline, rest=rest) if "w" in mode: # Invalidate cache entry because size and timestamps will change. self.stat_cache.invalidate(effective_path) return host._file def close(self): """Close host connection.""" if self.closed: return # Close associated children. for host in self._children: # Children have a `_file` attribute which is an `FTPFile` object. host._file.close() host.close() # Now deal with ourself. try: with ftputil.error.ftplib_error_to_ftp_os_error: self._session.close() finally: # If something went wrong before, the host/session is # probably defunct and subsequent calls to `close` won't # help either, so consider the host/session closed for # practical purposes. self.stat_cache.clear() self._children = [] self.closed = True # # Setting a custom directory parser # def set_parser(self, parser): """ Set the parser for extracting stat results from directory listings. The parser interface is described in the documentation, but here are the most important things: - A parser should derive from `ftputil.stat.Parser`. - The parser has to implement two methods, `parse_line` and `ignores_line`. For the latter, there's a probably useful default in the class `ftputil.stat.Parser`. - `parse_line` should try to parse a line of a directory listing and return a `ftputil.stat.StatResult` instance. If parsing isn't possible, raise `ftputil.error.ParserError` with a useful error message. - `ignores_line` should return a true value if the line isn't assumed to contain stat information. """ # The cache contents, if any, probably aren't useful. self.stat_cache.clear() # Set the parser explicitly, don't allow "smart" switching anymore. self._stat._parser = parser self._stat._allow_parser_switching = False # # Time shift adjustment between client (i. e. us) and server # def set_time_shift(self, time_shift): """ Set the time shift value (i. e. the time difference between client and server) for this `FTPHost` object. By (my) definition, the time shift value is positive if the local time of the server is greater than the local time of the client (for the same physical time), i. e. time_shift =def= t_server - t_client <=> t_server = t_client + time_shift <=> t_client = t_server - time_shift The time shift is measured in seconds. """ # Implicitly set via `set_time_shift` call in constructor # pylint: disable=attribute-defined-outside-init self._time_shift = time_shift def time_shift(self): """ Return the time shift between FTP server and client. See the docstring of `set_time_shift` for more on this value. """ return self._time_shift @staticmethod def __rounded_time_shift(time_shift): """ Return the given time shift in seconds, but rounded to full hours. The argument is also assumed to be given in seconds. """ minute = 60.0 hour = 60.0 * minute # Avoid division by zero below. if time_shift == 0: return 0.0 # Use a positive value for rounding. absolute_time_shift = abs(time_shift) signum = time_shift / absolute_time_shift # Round absolute time shift to 15-minute units. absolute_rounded_time_shift = ( int( (absolute_time_shift + (7.5*minute)) / (15.0*minute) ) * (15.0*minute)) # Return with correct sign. return signum * absolute_rounded_time_shift def __assert_valid_time_shift(self, time_shift): """ Perform sanity checks on the time shift value (given in seconds). If the value is invalid, raise a `TimeShiftError`, else simply return `None`. """ minute = 60.0 # seconds hour = 60.0 * minute absolute_rounded_time_shift = \ abs(self.__rounded_time_shift(time_shift)) # Test 1: Fail if the absolute time shift is greater than # a full day (24 hours). if absolute_rounded_time_shift > 24 * hour: raise ftputil.error.TimeShiftError( "time shift abs({0:.2f} s) > 1 day".format(time_shift)) # Test 2: Fail if the deviation between given time shift and # 15-minute units is greater than a certain limit. maximum_deviation = 5 * minute if abs(time_shift - self.__rounded_time_shift(time_shift)) > \ maximum_deviation: raise ftputil.error.TimeShiftError( "time shift ({0:.2f} s) deviates more than {1:d} s " "from 15-minute units".format( time_shift, int(maximum_deviation))) def synchronize_times(self): """ Synchronize the local times of FTP client and server. This is necessary to let `upload_if_newer` and `download_if_newer` work correctly. If `synchronize_times` isn't applicable (see below), the time shift can still be set explicitly with `set_time_shift`. This implementation of `synchronize_times` requires _all_ of the following: - The connection between server and client is established. - The client has write access to the directory that is current when `synchronize_times` is called. The common usage pattern of `synchronize_times` is to call it directly after the connection is established. (As can be concluded from the points above, this requires write access to the login directory.) If `synchronize_times` fails, it raises a `TimeShiftError`. """ helper_file_name = "_ftputil_sync_" # Open a dummy file for writing in the current directory # on the FTP host, then close it. try: # May raise `FTPIOError` if directory isn't writable. file_ = self.open(helper_file_name, "w") file_.close() except ftputil.error.FTPIOError: raise ftputil.error.TimeShiftError( """couldn't write helper file in directory '{0}'""". format(self.getcwd())) # If everything worked up to here it should be possible to stat # and then remove the just-written file. try: server_time = self.path.getmtime(helper_file_name) self.unlink(helper_file_name) except ftputil.error.FTPOSError: # If we got a `TimeShiftError` exception above, we should't # come here: if we did not get a `TimeShiftError` above, # deletion should be possible. The only reason for an exception # I can think of here is a race condition by removing write # permission from the directory or helper file after it has been # written to. raise ftputil.error.TimeShiftError( "could write helper file but not unlink it") # Calculate the difference between server and client. now = time.time() time_shift = server_time - now # As the time shift for this host instance isn't set yet, the # directory parser will calculate times one year in the past if # the time zone of the server is east from ours. Thus the time # shift will be off by a year as well (see ticket #55). if time_shift < -360 * 24 * 60 * 60: # Re-add one year and re-calculate the time shift. We don't # know how many days made up that year (it might have been # a leap year), so go the route via `time.localtime` and # `time.mktime`. server_time_struct = time.localtime(server_time) server_time_struct = (server_time_struct.tm_year+1,) + \ server_time_struct[1:] server_time = time.mktime(server_time_struct) time_shift = server_time - now # Do some sanity checks. self.__assert_valid_time_shift(time_shift) # If tests passed, store the time difference as time shift value. self.set_time_shift(self.__rounded_time_shift(time_shift)) # # Operations based on file-like objects (rather high-level), # like upload and download # # This code doesn't complain if the chunk size is passed as a # positional argument but emits a deprecation warning if `length` # is used as a keyword argument. @staticmethod def copyfileobj(source, target, max_chunk_size=ftputil.file_transfer.MAX_COPY_CHUNK_SIZE, callback=None): """ Copy data from file-like object `source` to file-like object `target`. """ ftputil.file_transfer.copyfileobj(source, target, max_chunk_size, callback) def _upload_files(self, source_path, target_path): """ Return a `LocalFile` and `RemoteFile` as source and target, respectively. The strings `source_path` and `target_path` are the (absolute or relative) paths of the local and the remote file, respectively. """ source_file = ftputil.file_transfer.LocalFile(source_path, "rb") # Passing `self` (the `FTPHost` instance) here is correct. target_file = ftputil.file_transfer.RemoteFile(self, target_path, "wb") return source_file, target_file def upload(self, source, target, callback=None): """ Upload a file from the local source (name) to the remote target (name). If a callable `callback` is given, it's called after every chunk of transferred data. The chunk size is a constant defined in `file_transfer`. The callback will be called with a single argument, the data chunk that was transferred before the callback was called. """ target = ftputil.tool.as_unicode(target) source_file, target_file = self._upload_files(source, target) ftputil.file_transfer.copy_file(source_file, target_file, conditional=False, callback=callback) def upload_if_newer(self, source, target, callback=None): """ Upload a file only if it's newer than the target on the remote host or if the target file does not exist. See the method `upload` for the meaning of the parameters. If an upload was necessary, return `True`, else return `False`. If a callable `callback` is given, it's called after every chunk of transferred data. The chunk size is a constant defined in `file_transfer`. The callback will be called with a single argument, the data chunk that was transferred before the callback was called. """ target = ftputil.tool.as_unicode(target) source_file, target_file = self._upload_files(source, target) return ftputil.file_transfer.copy_file(source_file, target_file, conditional=True, callback=callback) def _download_files(self, source_path, target_path): """ Return a `RemoteFile` and `LocalFile` as source and target, respectively. The strings `source_path` and `target_path` are the (absolute or relative) paths of the remote and the local file, respectively. """ source_file = ftputil.file_transfer.RemoteFile(self, source_path, "rb") target_file = ftputil.file_transfer.LocalFile(target_path, "wb") return source_file, target_file def download(self, source, target, callback=None): """ Download a file from the remote source (name) to the local target (name). If a callable `callback` is given, it's called after every chunk of transferred data. The chunk size is a constant defined in `file_transfer`. The callback will be called with a single argument, the data chunk that was transferred before the callback was called. """ source = ftputil.tool.as_unicode(source) source_file, target_file = self._download_files(source, target) ftputil.file_transfer.copy_file(source_file, target_file, conditional=False, callback=callback) def download_if_newer(self, source, target, callback=None): """ Download a file only if it's newer than the target on the local host or if the target file does not exist. See the method `download` for the meaning of the parameters. If a download was necessary, return `True`, else return `False`. If a callable `callback` is given, it's called after every chunk of transferred data. The chunk size is a constant defined in `file_transfer`. The callback will be called with a single argument, the data chunk that was transferred before the callback was called. """ source = ftputil.tool.as_unicode(source) source_file, target_file = self._download_files(source, target) return ftputil.file_transfer.copy_file(source_file, target_file, conditional=True, callback=callback) # # Helper methods to descend into a directory before executing a command # def _check_inaccessible_login_directory(self): """ Raise an `InaccessibleLoginDirError` exception if we can't change to the login directory. This test is only reliable if the current directory is the login directory. """ presumable_login_dir = self.getcwd() # Bail out with an internal error rather than modify the # current directory without hope of restoration. try: self.chdir(presumable_login_dir) except ftputil.error.PermanentError: raise ftputil.error.InaccessibleLoginDirError( "directory '{0}' is not accessible". format(presumable_login_dir)) def _robust_ftp_command(self, command, path, descend_deeply=False): """ Run an FTP command on a path. The return value of the method is the return value of the command. If `descend_deeply` is true (the default is false), descend deeply, i. e. change the directory to the end of the path. """ # If we can't change to the yet-current directory, the code # below won't work (see below), so in this case rather raise # an exception than giving wrong results. self._check_inaccessible_login_directory() # Some FTP servers don't behave as expected if the directory # portion of the path contains whitespace; some even yield # strange results if the command isn't executed in the # current directory. Therefore, change to the directory # which contains the item to run the command on and invoke # the command just there. # # Remember old working directory. old_dir = self.getcwd() try: if descend_deeply: # Invoke the command in (not: on) the deepest directory. self.chdir(path) # Workaround for some servers that give recursive # listings when called with a dot as path; see issue #33, # http://ftputil.sschwarzer.net/trac/ticket/33 return command(self, "") else: # Invoke the command in the "next to last" directory. head, tail = self.path.split(path) self.chdir(head) return command(self, tail) finally: self.chdir(old_dir) # # Miscellaneous utility methods resembling functions in `os` # def getcwd(self): """Return the current path name.""" return self._cached_current_dir def chdir(self, path): """Change the directory on the host.""" path = ftputil.tool.as_unicode(path) with ftputil.error.ftplib_error_to_ftp_os_error: self._session.cwd(path) # The path given as the argument is relative to the old current # directory, therefore join them. self._cached_current_dir = \ self.path.normpath(self.path.join(self._cached_current_dir, path)) # Ignore unused argument `mode` # pylint: disable=unused-argument def mkdir(self, path, mode=None): """ Make the directory path on the remote host. The argument `mode` is ignored and only "supported" for similarity with `os.mkdir`. """ path = ftputil.tool.as_unicode(path) def command(self, path): """Callback function.""" with ftputil.error.ftplib_error_to_ftp_os_error: self._session.mkd(path) self._robust_ftp_command(command, path) # TODO: The virtual directory support doesn't have unit tests yet # because the mocking most likely would be quite complicated. The # tests should be added when mainly the `mock` library is used # instead of the mock code in `test.mock_ftplib`. # # Ignore unused argument `mode` # pylint: disable=unused-argument def makedirs(self, path, mode=None): """ Make the directory `path`, but also make not yet existing intermediate directories, like `os.makedirs`. The value of `mode` is only accepted for compatibility with `os.makedirs` but otherwise ignored. """ path = ftputil.tool.as_unicode(path) path = self.path.abspath(path) directories = path.split(self.sep) old_dir = self.getcwd() try: # Try to build the directory chain from the "uppermost" to # the "lowermost" directory. for index in range(1, len(directories)): # Re-insert the separator which got lost by using # `path.split`. next_directory = (self.sep + self.path.join(*directories[:index+1])) # If we have "virtual directories" (see #86), just # listing the parent directory won't tell us if a # directory actually exists. So try to change into the # directory. try: self.chdir(next_directory) except ftputil.error.PermanentError: try: self.mkdir(next_directory) except ftputil.error.PermanentError: # Find out the cause of the error. Re-raise # the exception only if the directory didn't # exist already, else something went _really_ # wrong, e. g. there's a regular file with the # name of the directory. if not self.path.isdir(next_directory): raise finally: self.chdir(old_dir) def rmdir(self, path): """ Remove the _empty_ directory `path` on the remote host. Compatibility note: Previous versions of ftputil could possibly delete non- empty directories as well, - if the server allowed it. This is no longer supported. """ path = ftputil.tool.as_unicode(path) path = self.path.abspath(path) if self.listdir(path): raise ftputil.error.PermanentError("directory '{0}' not empty". format(path)) #XXX How does `rmd` work with links? def command(self, path): """Callback function.""" with ftputil.error.ftplib_error_to_ftp_os_error: self._session.rmd(path) self._robust_ftp_command(command, path) self.stat_cache.invalidate(path) def remove(self, path): """ Remove the file or link given by `path`. Raise a `PermanentError` if the path doesn't exist, but maybe raise other exceptions depending on the state of the server (e. g. timeout). """ path = ftputil.tool.as_unicode(path) path = self.path.abspath(path) # Though `isfile` includes also links to files, `islink` # is needed to include links to directories. if self.path.isfile(path) or self.path.islink(path) or \ not self.path.exists(path): # If the path doesn't exist, let the removal command trigger # an exception with a more appropriate error message. def command(self, path): """Callback function.""" with ftputil.error.ftplib_error_to_ftp_os_error: self._session.delete(path) self._robust_ftp_command(command, path) else: raise ftputil.error.PermanentError( "remove/unlink can only delete files and links, " "not directories") self.stat_cache.invalidate(path) unlink = remove def rmtree(self, path, ignore_errors=False, onerror=None): """ Remove the given remote, possibly non-empty, directory tree. The interface of this method is rather complex, in favor of compatibility with `shutil.rmtree`. If `ignore_errors` is set to a true value, errors are ignored. If `ignore_errors` is a false value _and_ `onerror` isn't set, all exceptions occurring during the tree iteration and processing are raised. These exceptions are all of type `PermanentError`. To distinguish between error situations, pass in a callable for `onerror`. This callable must accept three arguments: `func`, `path` and `exc_info`. `func` is a bound method object, _for example_ `your_host_object.listdir`. `path` is the path that was the recent argument of the respective method (`listdir`, `remove`, `rmdir`). `exc_info` is the exception info as it's got from `sys.exc_info`. Implementation note: The code is copied from `shutil.rmtree` in Python 2.4 and adapted to ftputil. """ path = ftputil.tool.as_unicode(path) # The following code is an adapted version of Python 2.4's # `shutil.rmtree` function. if ignore_errors: def new_onerror(*args): """Do nothing.""" # pylint: disable=unused-argument pass elif onerror is None: def new_onerror(*args): """Re-raise exception.""" # pylint: disable=unused-argument raise else: new_onerror = onerror names = [] try: names = self.listdir(path) except ftputil.error.PermanentError: new_onerror(self.listdir, path, sys.exc_info()) for name in names: full_name = self.path.join(path, name) try: mode = self.lstat(full_name).st_mode except ftputil.error.PermanentError: mode = 0 if stat.S_ISDIR(mode): self.rmtree(full_name, ignore_errors, new_onerror) else: try: self.remove(full_name) except ftputil.error.PermanentError: new_onerror(self.remove, full_name, sys.exc_info()) try: self.rmdir(path) except ftputil.error.FTPOSError: new_onerror(self.rmdir, path, sys.exc_info()) def rename(self, source, target): """Rename the source on the FTP host to target.""" source = ftputil.tool.as_unicode(source) target = ftputil.tool.as_unicode(target) # The following code is in spirit similar to the code in the # method `_robust_ftp_command`, though we do _not_ do # _everything_ imaginable. self._check_inaccessible_login_directory() source_head, source_tail = self.path.split(source) target_head, target_tail = self.path.split(target) paths_contain_whitespace = (" " in source_head) or (" " in target_head) if paths_contain_whitespace and source_head == target_head: # Both items are in the same directory. old_dir = self.getcwd() try: self.chdir(source_head) with ftputil.error.ftplib_error_to_ftp_os_error: self._session.rename(source_tail, target_tail) finally: self.chdir(old_dir) else: # Use straightforward command. with ftputil.error.ftplib_error_to_ftp_os_error: self._session.rename(source, target) #XXX One could argue to put this method into the `_Stat` class, but # I refrained from that because then `_Stat` would have to know # about `FTPHost`'s `_session` attribute and in turn about # `_session`'s `dir` method. def _dir(self, path): """Return a directory listing as made by FTP's `LIST` command.""" # Don't use `self.path.isdir` in this method because that # would cause a call of `(l)stat` and thus a call to `_dir`, # so we would end up with an infinite recursion. def _FTPHost_dir_command(self, path): """Callback function.""" lines = [] def callback(line): """Callback function.""" lines.append(ftputil.tool.as_unicode(line)) with ftputil.error.ftplib_error_to_ftp_os_error: if self.use_list_a_option: self._session.dir("-a", path, callback) else: self._session.dir(path, callback) return lines lines = self._robust_ftp_command(_FTPHost_dir_command, path, descend_deeply=True) return lines # The `listdir`, `lstat` and `stat` methods don't use # `_robust_ftp_command` because they implicitly already use # `_dir` which actually uses `_robust_ftp_command`. def listdir(self, path): """ Return a list of directories, files etc. in the directory named `path`. If the directory listing from the server can't be parsed with any of the available parsers raise a `ParserError`. """ original_path = path path = ftputil.tool.as_unicode(path) items = self._stat._listdir(path) return [ftputil.tool.same_string_type_as(original_path, item) for item in items] def lstat(self, path, _exception_for_missing_path=True): """ Return an object similar to that returned by `os.lstat`. If the directory listing from the server can't be parsed with any of the available parsers, raise a `ParserError`. If the directory _can_ be parsed and the `path` is _not_ found, raise a `PermanentError`. (`_exception_for_missing_path` is an implementation aid and _not_ intended for use by ftputil clients.) """ path = ftputil.tool.as_unicode(path) return self._stat._lstat(path, _exception_for_missing_path) def stat(self, path, _exception_for_missing_path=True): """ Return info from a "stat" call on `path`. If the directory containing `path` can't be parsed, raise a `ParserError`. If the directory containing `path` can be parsed but the `path` can't be found, raise a `PermanentError`. Also raise a `PermanentError` if there's an endless (cyclic) chain of symbolic links "behind" the `path`. (`_exception_for_missing_path` is an implementation aid and _not_ intended for use by ftputil clients.) """ path = ftputil.tool.as_unicode(path) return self._stat._stat(path, _exception_for_missing_path) def walk(self, top, topdown=True, onerror=None, followlinks=False): """ Iterate over directory tree and return a tuple (dirpath, dirnames, filenames) on each iteration, like the `os.walk` function (see https://docs.python.org/library/os.html#os.walk ). """ top = ftputil.tool.as_unicode(top) # The following code is copied from `os.walk` in Python 2.4 # and adapted to ftputil. try: names = self.listdir(top) except ftputil.error.FTPOSError as err: if onerror is not None: onerror(err) return dirs, nondirs = [], [] for name in names: if self.path.isdir(self.path.join(top, name)): dirs.append(name) else: nondirs.append(name) if topdown: yield top, dirs, nondirs for name in dirs: path = self.path.join(top, name) if followlinks or not self.path.islink(path): for item in self.walk(path, topdown, onerror, followlinks): yield item if not topdown: yield top, dirs, nondirs def chmod(self, path, mode): """ Change the mode of a remote `path` (a string) to the integer `mode`. This integer uses the same bits as the mode value returned by the `stat` and `lstat` commands. If something goes wrong, raise a `TemporaryError` or a `PermanentError`, according to the status code returned by the server. In particular, a non-existent path usually causes a `PermanentError`. """ path = ftputil.tool.as_unicode(path) path = self.path.abspath(path) def command(self, path): """Callback function.""" with ftputil.error.ftplib_error_to_ftp_os_error: self._session.voidcmd("SITE CHMOD 0{0:o} {1}". format(mode, path)) self._robust_ftp_command(command, path) self.stat_cache.invalidate(path) def __getstate__(self): raise TypeError("cannot serialize FTPHost object") # # Context manager methods # def __enter__(self): # Return `self`, so it can be accessed as the variable # component of the `with` statement. return self def __exit__(self, exc_type, exc_val, exc_tb): # We don't need the `exc_*` arguments here. # pylint: disable=unused-argument self.close() # Be explicit. return False ftputil-3.4/ftputil/stat_cache.py0000644000175000017470000001326513135113162016445 0ustar debiandebian# Copyright (C) 2006-2013, Stefan Schwarzer # and ftputil contributors (see `doc/contributors.txt`) # See the file LICENSE for licensing terms. """ ftp_stat_cache.py - cache for (l)stat data """ from __future__ import unicode_literals import time import ftputil.error import ftputil.lrucache # This module shouldn't be used by clients of the ftputil library. __all__ = [] class StatCache(object): """ Implement an LRU (least-recently-used) cache. `StatCache` objects have an attribute `max_age`. After this duration after _setting_ it a cache entry will expire. For example, if you code my_cache = StatCache() my_cache.max_age = 10 my_cache["/home"] = ... the value my_cache["/home"] can be retrieved for 10 seconds. After that, the entry will be treated as if it had never been in the cache and should be fetched again from the remote host. Note that the `__len__` method does no age tests and thus may include some or many already expired entries. """ # Disable "Badly implemented container" warning because of # "missing" `__delitem__`. # pylint: disable=incomplete-protocol # Default number of cache entries _DEFAULT_CACHE_SIZE = 5000 def __init__(self): # Can be reset with method `resize` self._cache = ftputil.lrucache.LRUCache(self._DEFAULT_CACHE_SIZE) # Never expire self.max_age = None self.enable() def enable(self): """Enable storage of stat results.""" self._enabled = True def disable(self): """ Disable the cache. Further storage attempts with `__setitem__` won't have any visible effect. Disabling the cache only effects new storage attempts. Values stored before calling `disable` can still be retrieved unless disturbed by a `resize` command or normal cache expiration. """ # `_enabled` is set via calling `enable` in the constructor. # pylint: disable=attribute-defined-outside-init self._enabled = False def resize(self, new_size): """ Set number of cache entries to the integer `new_size`. If the new size is smaller than the current cache size, relatively long-unused elements will be removed. """ self._cache.size = new_size def _age(self, path): """ Return the age of a cache entry for `path` in seconds. If the path isn't in the cache, raise a `CacheMissError`. """ try: return time.time() - self._cache.mtime(path) except ftputil.lrucache.CacheKeyError: raise ftputil.error.CacheMissError( "no entry for path {0} in cache".format(path)) def clear(self): """Clear (invalidate) all cache entries.""" self._cache.clear() def invalidate(self, path): """ Invalidate the cache entry for the absolute `path` if present. After that, the stat result data for `path` can no longer be retrieved, as if it had never been stored. If no stat result for `path` is in the cache, do _not_ raise an exception. """ #XXX To be 100 % sure, this should be `host.sep`, but I don't # want to introduce a reference to the `FTPHost` object for # only that purpose. assert path.startswith("/"), ("{0} must be an absolute path". format(path)) try: del self._cache[path] except ftputil.lrucache.CacheKeyError: # Ignore errors pass def __getitem__(self, path): """ Return the stat entry for the `path`. If there's no stored stat entry or the cache is disabled, raise `CacheMissError`. """ if not self._enabled: raise ftputil.error.CacheMissError("cache is disabled") # Possibly raise a `CacheMissError` in `_age` if (self.max_age is not None) and (self._age(path) > self.max_age): self.invalidate(path) raise ftputil.error.CacheMissError( "entry for path {0} has expired".format(path)) else: #XXX I don't know if this may raise a `CacheMissError` in # case of race conditions. I prefer robust code. try: return self._cache[path] except ftputil.lrucache.CacheKeyError: raise ftputil.error.CacheMissError( "entry for path {0} not found".format(path)) def __setitem__(self, path, stat_result): """ Put the stat data for the absolute `path` into the cache, unless it's disabled. """ assert path.startswith("/") if not self._enabled: return self._cache[path] = stat_result def __contains__(self, path): """ Support for the `in` operator. Return a true value, if data for `path` is in the cache, else return a false value. """ try: # Implicitly do an age test which may raise `CacheMissError`. self[path] except ftputil.error.CacheMissError: return False else: return True # # The following methods are only intended for debugging! # def __len__(self): """ Return the number of entries in the cache. Note that this may include some (or many) expired entries. """ return len(self._cache) def __str__(self): """Return a string representation of the cache contents.""" lines = [] for key in sorted(self._cache): lines.append("{0}: {1}".format(key, self[key])) return "\n".join(lines) ftputil-3.4/ftputil/version.py0000644000175000017470000000114513200531426016026 0ustar debiandebian# Copyright (C) 2006-2013, Stefan Schwarzer # and ftputil contributors (see `doc/contributors.txt`) # See the file LICENSE for licensing terms. """ Provide version information about ftputil and the runtime environment. """ from __future__ import unicode_literals import sys # ftputil version number; substituted by `make patch` __version__ = "3.4" _ftputil_version = __version__ _python_version = sys.version.split()[0] _python_platform = sys.platform version_info = "ftputil {0}, Python {1} ({2})".format( _ftputil_version, _python_version, _python_platform) ftputil-3.4/ftputil/__init__.py0000644000175000017470000000426113200423657016110 0ustar debiandebian# Copyright (C) 2002-2013, Stefan Schwarzer # and ftputil contributors (see `doc/contributors.txt`) # See the file LICENSE for licensing terms. """ ftputil - high-level FTP client library FTPHost objects This class resembles the `os` module's interface to ordinary file systems. In addition, it provides a method `file` which will return file-objects corresponding to remote files. # Example session with ftputil.FTPHost("ftp.domain.com", "me", "secret") as host: print host.getcwd() # e. g. "/home/me" host.mkdir("newdir") host.chdir("newdir") with host.open("sourcefile", "r") as source: with host.open("targetfile", "w") as target: host.copyfileobj(source, target) host.remove("targetfile") host.chdir(host.pardir) host.rmdir("newdir") There are also shortcuts for uploads and downloads: host.upload(local_file, remote_file) host.download(remote_file, local_file) Both accept an additional mode parameter. If it is "b", the transfer mode will be for binary files. For even more functionality refer to the documentation in `ftputil.txt` or `ftputil.html`. FTPFile objects `FTPFile` objects are constructed via the `file` method (`open` is an alias) of `FTPHost` objects. `FTPFile` objects support the usual file operations for non-seekable files (`read`, `readline`, `readlines`, `write`, `writelines`, `close`). Note: ftputil currently is not threadsafe. More specifically, you can use different `FTPHost` objects in different threads but not a single `FTPHost` object in different threads. """ from __future__ import absolute_import from __future__ import unicode_literals import sys import warnings from ftputil.host import FTPHost from ftputil.version import __version__ # `sys.version_info.major` isn't available in Python 2.6. if sys.version_info[0] == 2: warnings.warn("Python 2 suport will be removed in ftputil 4.0.0", DeprecationWarning, stacklevel=2) # Apart from `ftputil.error` and `ftputil.stat`, this is the whole # public API of `ftputil`. __all__ = ["FTPHost", "__version__"] ftputil-3.4/ftputil/compat.py0000644000175000017470000000176113135113162015630 0ustar debiandebian# encoding: utf-8 # Copyright (C) 2011-2013, Stefan Schwarzer # and ftputil contributors (see `doc/contributors.txt`) # See the file LICENSE for licensing terms. """ Help make the same code work in both Python 2 and 3. Comments given for the Python 2 versions of the helpers apply to the Python 3 helpers as well. """ from __future__ import unicode_literals import sys __all__ = ["int_types", "unicode_type", "bytes_type", "bytes_from_ints", "default_string_type"] python_version = sys.version_info[0] if python_version == 2: int_types = (int, long) unicode_type = unicode bytes_type = str def bytes_from_ints(int_list): """Return a `bytes` object from a list of integers.""" return b"".join((chr(i) for i in int_list)) else: int_types = (int,) unicode_type = str bytes_type = bytes bytes_from_ints = bytes # For Python 2 `str` means byte strings, for Python 3 unicode strings. default_string_type = str ftputil-3.4/doc/0000755000175000017470000000000013200532636013050 5ustar debiandebianftputil-3.4/doc/python3_support_outline.txt0000644000175000017470000001324712353637454020573 0ustar debiandebianWhat to do for Python 3 support =============================== Improving unit tests Review them before starting coding on them. Will probably remind me of what's tricky. Maybe this is the time (aka chance ;-) ) to clean up. After reading first tests ... I don't remember easily what all these tests were for, but I think I can recall when I look at the code. -> Add docstrings/comments to explain what the idea of each respective test is. Put custom session factory classes for certain tests before these tests the session factories are used in. Tests for ASCII/binary conversions certainly have to change. We're going to use Python's usual line ending normalization. We don't expect `\r` characters to be added during ASCII _reads_. Change certain methods to return the same string type that is passed in. The following list contains only user-visible methods or methods whose API is directly relevant for users. host.listdir(self, path) host.lstat(self, path, _exception_for_missing_path=True) host.stat(self, path, _exception_for_missing_path=True) host.walk(self, top, topdown=True, onerror=None) host.path.abspath(self, path) host.path.basename(path) host.path.commonprefix(path_list) host.path.dirname(path) host.path.exists(path) host.path.getmtime(path) host.path.getsize(path) host.path.isabs(path) host.path.isdir(path) host.path.isfile(path) host.path.islink(path) host.path.join(path1, path2, ...) host.path.normcase(path) host.path.normpath(path) host.path.split(path) host.path.splitdrive(path) host.path.splitext(path) host.path.walk(path, func, arg) parser.ignores_line(self, line) parser.parse_line(self, line, time_shift=0.0) mock_ftplib.cwd(self, path) mock_ftplib.dir(self, *args) mock_ftplib.transfercmd(self, cmd) mock_ftplib.voidcmd(self, cmd) mock_ftplib.voidresp(self) Can we somehow "generalize" unit tests? I. e. we don't want to code the "same string type" logic when testing each method. Which methods of the "normal" file system API accept byte strings, unicode strings or either? If the APIs are systematic, follow them for ftputil. If the APIs aren't systematic, don't support exceptional cases, e. g. only one method in a group of methods accepts byte strings beside unicode strings. Or support the exceptional cases only in a later ftputil version. Normalize cache string type? Does it make sense? What encoding to use to automatically convert? What to do if encoding to byte string isn't possible? (i. e. when the unicode strings contains characters that are invalid in the target encoding) Local files use the value of `sys.getfilesystemencoding()` whereas remote files - in ftplib - use latin1. Is it possible to be consistent? For what kind of "consistency"? Different caches for byte string paths and unicode string paths? File object API Files opened for reading When opened in binary mode, return byte strings. When opened in text mode, return unicode strings. Apply encoding given in `open` on the fly. Use helper classes in `io` module. Files opened for writing When opened in binary mode, accept only byte strings. When opened in text mode, accept only unicode strings. Allow byte strings during a "deprecation period"? No: Do the simplest thing that could possibly work. Might rather confuse people. Might cause surprises when the "deprecation period" is over! Apply encoding given in `open` on the fly. Use helper classes in `io` module. Deal consistently with local and remote files. What does "consistent" mean/involve here? How to deal with line ending conversion? We don't know for certain whether the remote server is a Posix or Windows machine. -> This would suggest to use the line endings of the local host when writing remote files. -> But I don't think this is logical. We would get different remote files depending on the operating system of the client which accesses these files! Always assume Posix or always assume Windows for the remote system? Let it depend on the returned listing format? Too subtle. What to do if we later move to using `MLSD`? Is there a way to specify a desired line ending type when using the `io` module? Yes, there's the `newline` argument in `io.open`. Probably it's better to let ftputil behave as for "regular" file systems. This also holds for remote file systems (NFS, CIFS). So it should be ok to use Python conventions here; mounted remote file systems are treated like local file systems. Provide user and password in `FTPHost` constructor as unicode strings? Probably very important since the server will only see the bytes, and an encoding problem will lead to a refused connection. How does ftplib handle the encoding? Different methods for returning byte strings or unicode strings (similar to `os.getcwd` vs. `os.getcwdu`)? Later! (do the simplest thing that could possibly work) ftputil-3.4/doc/ftputil.html0000644000175000017470000030550613200531432015427 0ustar debiandebian ftputil -- a high-level FTP client library

ftputil -- a high-level FTP client library

Version: 3.4
Date: 2017-11-08
Summary:high-level FTP client library for Python
Keywords:FTP, ftplib substitute, virtual filesystem, pure Python
Author: Stefan Schwarzer <sschwarzer@sschwarzer.net>

Introduction

The ftputil module is a high-level interface to the ftplib module. The FTPHost objects generated from it allow many operations similar to those of os, os.path and shutil.

Example:

import ftputil

# Download some files from the login directory.
with ftputil.FTPHost("ftp.domain.com", "user", "password") as ftp_host:
    names = ftp_host.listdir(ftp_host.curdir)
    for name in names:
        if ftp_host.path.isfile(name):
            ftp_host.download(name, name)  # remote, local
    # Make a new directory and copy a remote file into it.
    ftp_host.mkdir("newdir")
    with ftp_host.open("index.html", "rb") as source:
        with ftp_host.open("newdir/index.html", "wb") as target:
            ftp_host.copyfileobj(source, target)  # similar to shutil.copyfileobj

Also, there are FTPHost.lstat and FTPHost.stat to request size and modification time of a file. The latter can also follow links, similar to os.stat. FTPHost.walk and FTPHost.path.walk work, too.

ftputil features

  • Method names are familiar from Python's os, os.path and shutil modules. For example, use os.path.join to join paths for a local file system and ftp_host.path.join to join paths for a remote FTP file system.
  • Remote file system navigation (getcwd, chdir)
  • Upload and download files (upload, upload_if_newer, download, download_if_newer)
  • Time zone synchronization between client and server (needed for upload_if_newer and download_if_newer)
  • Create and remove directories (mkdir, makedirs, rmdir, rmtree) and remove files (remove)
  • Get information about directories, files and links (listdir, stat, lstat, exists, isdir, isfile, islink, abspath, split, join, dirname, basename etc.)
  • Iterate over remote file systems (walk)
  • Local caching of results from lstat and stat calls to reduce network access (also applies to exists, getmtime etc.).
  • Read files from and write files to remote hosts via file-like objects (FTPHost.open; the generated file-like objects have the familiar methods like read, readline, readlines, write, writelines and close. You can also iterate over these files line by line in a for loop.

Exception hierarchy

The exceptions are in the namespace of the ftputil.error module, e. g. ftputil.error.TemporaryError.

The exception classes are organized as follows:

FTPError
    FTPOSError(FTPError, OSError)
        PermanentError(FTPOSError)
            CommandNotImplementedError(PermanentError)
        TemporaryError(FTPOSError)
    FTPIOError(FTPError)
    InternalError(FTPError)
        InaccessibleLoginDirError(InternalError)
        ParserError(InternalError)
        RootDirError(InternalError)
        TimeShiftError(InternalError)

and are described here:

  • FTPError

    is the root of the exception hierarchy of the module.

  • FTPOSError

    is derived from OSError. This is for similarity between the os module and FTPHost objects. Compare

    try:
    os.chdir("nonexisting_directory")
except OSError:
    ...

    with

    host = ftputil.FTPHost("host", "user", "password")
try:
    host.chdir("nonexisting_directory")
except OSError:
    ...

    Imagine a function

    def func(path, file):
    ...

    which works on the local file system and catches OSErrors. If you change the parameter list to

    def func(path, file, os=os):
    ...

    where os denotes the os module, you can call the function also as

    host = ftputil.FTPHost("host", "user", "password")
func(path, file, os=host)

    to use the same code for both a local and remote file system. Another similarity between OSError and FTPOSError is that the latter holds the FTP server return code in the errno attribute of the exception object and the error text in strerror.

  • PermanentError

    is raised for 5xx return codes from the FTP server. This corresponds to ftplib.error_perm (though PermanentError and ftplib.error_perm are not identical).

  • CommandNotImplementedError

    indicates that an underlying command the code tries to use is not implemented. For an example, see the description of the FTPHost.chmod method.

  • TemporaryError

    is raised for FTP return codes from the 4xx category. This corresponds to ftplib.error_temp (though TemporaryError and ftplib.error_temp are not identical).

  • FTPIOError

    denotes an I/O error on the remote host. This appears mainly with file-like objects that are retrieved by calling FTPHost.open. Compare

    >>> try:
...     f = open("not_there")
... except IOError as obj:
...     print obj.errno
...     print obj.strerror
...
2
No such file or directory

    with

    >>> ftp_host = ftputil.FTPHost("host", "user", "password")
>>> try:
...     f = ftp_host.open("not_there")
... except IOError as obj:
...     print obj.errno
...     print obj.strerror
...
550
550 not_there: No such file or directory.

    As you can see, both code snippets are similar. However, the error codes aren't the same.

  • InternalError

    subsumes exception classes for signaling errors due to limitations of the FTP protocol or the concrete implementation of ftputil.

  • InaccessibleLoginDirError

    This exception is raised if the directory in which "you" are placed upon login is not accessible, i. e. a chdir call with the directory as argument would fail.

  • ParserError

    is used for errors during the parsing of directory listings from the server. This exception is used by the FTPHost methods stat, lstat, and listdir.

  • RootDirError

    Because of the implementation of the lstat method it is not possible to do a stat call on the root directory /. If you know any way to do it, please let me know. :-)

    This problem does not affect stat calls on items in the root directory.

  • TimeShiftError

    is used to denote errors which relate to setting the time shift.

Directory and file names

Note

Keep in mind that this section only applies to directory and file names, not file contents. Encoding and decoding for file contents is handled by the encoding argument for FTPHost.open.

First off: If your directory and file names (both as arguments and on the server) contain only ISO 8859-1 (latin-1) characters, you can use such names in the form of byte strings or unicode strings. However, you can't mix different string types (bytes and unicode) in one call (for example in FTPHost.path.join).

If you have directory or file names with characters that aren't in latin-1, it's recommended to use byte strings. In that case, returned paths will be byte strings, too.

Read on for details.

Note

The approach described below may look awkward and in a way it is. The intention of ftputil is to behave like the local file system APIs of Python 3 as far as it makes sense. Moreover, the taken approach makes sure that directory and file names that were used with Python 3's native ftplib module will be compatible with ftputil and vice versa. Otherwise you may be able to use a file name with ftputil, but get an exception when trying to read the same file with Python 3's ftplib module.

Methods that take names of directories and/or files can take either byte or unicode strings. If a method got a string argument and returns one or more strings, these strings will have the same string type as the argument(s). Mixing different string arguments in one call (for example in FTPHost.path.join) isn't allowed and will cause a TypeError. These rules are the same as for local file system operations in Python 3. Since ftputil uses the same API for Python 2, ftputil will do the same when run on Python 2.

Byte strings for directory and file names will be sent to the server as-is. On the other hand, unicode strings will be encoded to byte strings, assuming latin-1 encoding. This implies that such unicode strings must only contain code points 0-255 for the latin-1 character set. Using any other characters will result in a UnicodeEncodeError exception.

If you have directory or file names as unicode strings with non-latin-1 characters, encode the unicode strings to byte strings yourself, using the encoding you know the server uses. Decode received paths with the same encoding. Encapsulate these conversions as far as you can. Otherwise, you'd have to adapt potentially a lot of code if the server encoding changes.

If you don't know the encoding on the server side, it's probably the best to only use byte strings for directory and file names. That said, as soon as you show the names to a user, you -- or the library you use for displaying the names -- has to guess an encoding.

FTPHost objects

Construction

Introduction

FTPHost instances can be created with the following call:

ftp_host = ftputil.FTPHost(server, user, password, account,
                           session_factory=ftplib.FTP)

The first four parameters are strings with the same meaning as for the FTP class in the ftplib module. Usually the account and session_factory arguments aren't needed though.

FTPHost objects can also be used in a with statement:

import ftputil

with ftputil.FTPHost(server, user, password) as ftp_host:
    print ftp_host.listdir(ftp_host.curdir)

After the with block, the FTPHost instance and the associated FTP sessions will be closed automatically.

If something goes wrong during the FTPHost construction or in the body of the with statement, the instance is closed as well. Exceptions will be propagated (as with try ... finally).

Session factories

The keyword argument session_factory may be used to generate FTP connections with other factories than the default ftplib.FTP. For example, the standard library of Python 2.7 contains a class ftplib.FTP_TLS which extends ftplib.FTP to use an encrypted connection.

In fact, all positional and keyword arguments other than session_factory are passed to the factory to generate a new background session. This also happens for every remote file that is opened; see below.

This functionality of the constructor also allows to wrap ftplib.FTP objects to do something that wouldn't be possible with the ftplib.FTP constructor alone.

As an example, assume you want to connect to another than the default port, but ftplib.FTP only offers this by means of its connect method, not via its constructor. One solution is to use a custom class as a session factory:

import ftplib
import ftputil

EXAMPLE_PORT = 50001

class MySession(ftplib.FTP):

    def __init__(self, host, userid, password, port):
        """Act like ftplib.FTP's constructor but connect to another port."""
        ftplib.FTP.__init__(self)
        self.connect(host, port)
        self.login(userid, password)

# Try _not_ to use an _instance_ `MySession()` as factory, -
# use the class itself.
with ftputil.FTPHost(host, userid, password, port=EXAMPLE_PORT,
                     session_factory=MySession) as ftp_host:
    # Use `ftp_host` as usual.
    ...

On login, the format of the directory listings (needed for stat'ing files and directories) should be determined automatically. If not, please file a bug report.

For the most common uses you don't need to create your own session factory class though. The ftputil.session module has a function session_factory that can create session factories for a variety of parameters:

session_factory(base_class=ftplib.FTP,
                port=21,
                use_passive_mode=None,
                encrypt_data_channel=True,
                debug_level=None)

with

  • base_class is a base class to inherit a new session factory class from. By default, this is ftplib.FTP from the Python standard library.
  • port is the command channel port. The default is 21, used in most FTP server configurations.
  • use_passive_mode is either a boolean that determines whether passive mode should be used or None. None means to let the base class choose active or passive mode.
  • encrypt_data_channel defines whether to encrypt the data channel for secure connections. This is only supported for the base classes ftplib.FTP_TLS and M2Crypto.ftpslib.FTP_TLS, otherwise the the parameter is ignored.
  • debug_level sets the debug level for FTP session instances. The semantics is defined by the base class. For example, a debug level of 2 causes the most verbose output for Python's ftplib.FTP class.

All of these parameters can be combined. For example, you could use

import ftplib

import ftputil
import ftputil.session


my_session_factory = ftputil.session.session_factory(
                       base_class=ftpslib.FTP_TLS,
                       port=31,
                       encrypt_data_channel=True,
                       debug_level=2)

with ftputil.FTPHost(server, user, password,
                     session_factory=my_session_factory) as ftp_host:
    ...

to create and use a session factory derived from ftplib.FTP_TLS that connects on command channel 31, will encrypt the data channel and print output for debug level 2.

Note: Generally, you can achieve everything you can do with ftputil.session.session_factory with an explicit session factory as described at the start of this section. However, the class M2Crypto.ftpslib.FTP_TLS has a limitation so that you can't use it with ftputil out of the box. The function session_factory contains a workaround for this limitation. For details refer to this bug report.

Hidden files and directories

Whether ftputil sees "hidden" files and directories (usually files or directories whose names start with a dot) depends on the FTP server configuration. By default, ftputil uses the -a option in the FTP LIST command to find hidden files. However, the server may ignore this.

If using the -a option leads to problems, for example if an FTP server causes an exception, you may switch off the use of the option:

ftp_host = ftputil.FTPHost(server, user, password, account,
                           session_factory=ftplib.FTP)
ftp_host.use_list_a_option = False

FTPHost attributes and methods

Attributes

  • curdir, pardir, sep

    are strings which denote the current and the parent directory on the remote server. sep holds the path separator. Though RFC 959 (File Transfer Protocol) notes that these values may depend on the FTP server implementation, the Unix variants seem to work well in practice, even for non-Unix servers.

    Nevertheless, it's recommended that you don't hardcode these values for remote paths, but use FTPHost.path as you would use os.path to write platform-independent Python code for local filesystems. Keep in mind that most, but not all, arguments of FTPHost methods refer to remote directories or files. For example, in FTPHost.upload, the first argument is a local path and the second a remote path. Both of these should use their respective path separators.

Remote file system navigation

  • getcwd()

    returns the absolute current directory on the remote host. This method works like os.getcwd.

  • chdir(directory)

    sets the current directory on the FTP server. This resembles os.chdir, as you may have expected.

Uploading and downloading files

  • upload(source, target, callback=None)

    copies a local source file (given by a filename, i. e. a string) to the remote host under the name target. Both source and target may be absolute paths or relative to their corresponding current directory (on the local or the remote host, respectively).

    The file content is always transferred in binary mode.

    The callback, if given, will be invoked for each transferred chunk of data:

    callback(chunk)

    where chunk is a bytestring. An example usage of a callback method is to display a progress indicator.

  • download(source, target, callback=None)

    performs a download from the remote source file to a local target file. Both source and target are strings. See the description of upload for more details.

  • upload_if_newer(source, target, callback=None)

    is similar to the upload method. The only difference is that the upload is only invoked if the time of the last modification for the source file is more recent than that of the target file or the target doesn't exist at all. The check for the last modification time considers the precision of the timestamps and transfers a file "if in doubt". Consequently the code

    ftp_host.upload_if_newer("source_file", "target_file")
time.sleep(10)
ftp_host.upload_if_newer("source_file", "target_file")

    might upload the file again if the timestamp of the target file is precise up to a minute, which is typically the case because the remote datetime is determined by parsing a directory listing from the server. To avoid unnecessary transfers, wait at least a minute between calls of upload_if_newer for the same file. If it still seems that a file is uploaded unnecessarily (or not when it should), read the subsection on time shift settings.

    If an upload actually happened, the return value of upload_if_newer is a True, else False.

    Note that the method only checks the existence and/or the modification time of the source and target file; it doesn't compare any other file properties, say, the file size.

    This also means that if a transfer is interrupted, the remote file will have a newer modification time than the local file, and thus the transfer won't be repeated if upload_if_newer is used a second time. There are at least two possibilities after a failed upload:

    • use upload instead of upload_if_newer, or
    • remove the incomplete target file with FTPHost.remove, then use upload or upload_if_newer to transfer it again.

  • download_if_newer(source, target, callback=None)

    corresponds to upload_if_newer but performs a download from the server to the local host. Read the descriptions of download and upload_if_newer for more information. If a download actually happened, the return value is True, else False.

Time zone correction

If the client where ftputil runs and the server have a different understanding of their local times, this has to be taken into account for upload_if_newer and download_if_newer to work correctly.

Note that even if the client and the server are in the same time zone (or even on the same computer), the time shift value (see below) may be different from zero. For example, my computer is set to use local time whereas the server running on the very same host insists on using UTC time.

  • set_time_shift(time_shift)

    sets the so-called time shift value, measured in seconds. The time shift is the difference between the local time of the server and the local time of the client at a given moment, i. e. by definition

    time_shift = server_time - client_time

    Setting this value is important for upload_if_newer and download_if_newer to work correctly even if the time zone of the FTP server differs from that of the client. Note that the time shift value can be negative.

    If the time shift value is invalid, for example its absolute value is larger than 24 hours, a TimeShiftError is raised.

    See also synchronize_times for a way to set the time shift with a simple method call.

  • time_shift()

    returns the currently-set time shift value. See set_time_shift above for its definition.

  • synchronize_times()

    synchronizes the local times of the server and the client, so that upload_if_newer and download_if_newer work as expected, even if the client and the server use different time zones. For this to work, all of the following conditions must be true:

    • The connection between server and client is established.
    • The client has write access to the directory that is current when synchronize_times is called.

    If you can't fulfill these conditions, you can nevertheless set the time shift value explicitly with set_time_shift. Trying to call synchronize_times if the above conditions aren't met results in a TimeShiftError exception.

Creating and removing directories

  • mkdir(path, [mode])

    makes the given directory on the remote host. This does not construct "intermediate" directories that don't already exist. The mode parameter is ignored; this is for compatibility with os.mkdir if an FTPHost object is passed into a function instead of the os module. See the explanation in the subsection Exception hierarchy.

  • makedirs(path, [mode])

    works similar to mkdir (see above), but also makes intermediate directories like os.makedirs. The mode parameter is only there for compatibility with os.makedirs and is ignored.

  • rmdir(path)

    removes the given remote directory. If it's not empty, raise a PermanentError.

  • rmtree(path, ignore_errors=False, onerror=None)

    removes the given remote, possibly non-empty, directory tree. The interface of this method is rather complex, in favor of compatibility with shutil.rmtree.

    If ignore_errors is set to a true value, errors are ignored. If ignore_errors is a false value and onerror isn't set, all exceptions occurring during the tree iteration and processing are raised. These exceptions are all of type PermanentError.

    To distinguish between different kinds of errors, pass in a callable for onerror. This callable must accept three arguments: func, path and exc_info. func is a bound method object, for example your_host_object.listdir. path is the path that was the recent argument of the respective method (listdir, remove, rmdir). exc_info is the exception info as it is gotten from sys.exc_info.

    The code of rmtree is taken from Python's shutil module and adapted for ftputil.

Local caching of file system information

Many of the above methods need access to the remote file system to obtain data on directories and files. To get the most recent data, each call to lstat, stat, exists, getmtime etc. would require to fetch a directory listing from the server, which can make the program very slow. This effect is more pronounced for operations which mostly scan the file system rather than transferring file data.

For this reason, ftputil by default saves the results from directory listings locally and reuses those results. This reduces network accesses and so speeds up the software a lot. However, since data is more rarely fetched from the server, the risk of obsolete data also increases. This will be discussed below.

Caching can be controlled -- if necessary at all -- via the stat_cache object in an FTPHost's namespace. For example, after calling

ftp_host = ftputil.FTPHost(host, user, password)

the cache can be accessed as ftp_host.stat_cache.

While ftputil usually manages the cache quite well, there are two possible reasons for modifying cache parameters.

The first is when the number of possible entries is too low. You may notice that when you are processing very large directories and the program becomes much slower than before. It's common for code to read a directory with listdir and then process the found directories and files. This can also happen implicitly by a call to FTPHost.walk. Since version 2.6 ftputil automatically increases the cache size if directories with more entries than the current maximum cache size are to be scanned. Most of the time, this works fine.

However, if you need access to stat data for several directories at the same time, you may need to increase the cache explicitly. This is done by the resize method:

ftp_host.stat_cache.resize(20000)

where the argument is the maximum number of lstat results to store (the default is 5000, in versions before 2.6 it was 1000). Note that each path on the server, e. g. "/home/schwa/some_dir", corresponds to a single cache entry. Methods like exists or getmtime all derive their results from a previously fetched lstat result.

The value 5000 above means that the cache will hold at most 5000 entries (unless increased automatically by an explicit or implicit listdir call, see above). If more are about to be stored, the entries which haven't been used for the longest time will be deleted to make place for newer entries.

The second possible reason to change the cache parameters is to avoid stale cache data. Caching is so effective because it reduces network accesses. This can also be a disadvantage if the file system data on the remote server changes after a stat result has been retrieved; the client, when looking at the cached stat data, will use obsolete information.

There are two potential ways to get such out-of-date stat data. The first happens when an FTPHost instance modifies a file path for which it has a cache entry, e. g. by calling remove or rmdir. Such changes are handled transparently; the path will be deleted from the cache. A different matter are changes unknown to the FTPHost object which inspects its cache. Obviously, for example, these are changes by programs running on the remote host. On the other hand, cache inconsistencies can also occur if two FTPHost objects change a file system simultaneously:

with (
  ftputil.FTPHost(server, user1, password1) as ftp_host1,
  ftputil.FTPHost(server, user1, password1) as ftp_host2
):
    stat_result1 = ftp_host1.stat("some_file")
    stat_result2 = ftp_host2.stat("some_file")
    ftp_host2.remove("some_file")
    # `ftp_host1` will still see the obsolete cache entry!
    print ftp_host1.stat("some_file")
    # Will raise an exception since an `FTPHost` object
    # knows of its own changes.
    print ftp_host2.stat("some_file")

At first sight, it may appear to be a good idea to have a shared cache among several FTPHost objects. After some thinking, this turns out to be very error-prone. For example, it won't help with different processes using ftputil. So, if you have to deal with concurrent write/read accesses to a server, you have to handle them explicitly.

The most useful tool for this is the invalidate method. In the example above, it could be used like this:

with (
  ftputil.FTPHost(server, user1, password1) as ftp_host1,
  ftputil.FTPHost(server, user1, password1) as ftp_host2
):
    stat_result1 = ftp_host1.stat("some_file")
    stat_result2 = ftp_host2.stat("some_file")
    ftp_host2.remove("some_file")
    # Invalidate using an absolute path.
    absolute_path = ftp_host1.path.abspath(
                      ftp_host1.path.join(ftp_host1.getcwd(), "some_file"))
    ftp_host1.stat_cache.invalidate(absolute_path)
    # Will now raise an exception as it should.
    print ftp_host1.stat("some_file")
    # Would raise an exception since an `FTPHost` object
    # knows of its own changes, even without `invalidate`.
    print ftp_host2.stat("some_file")

The method invalidate can be used on any absolute path, be it a directory, a file or a link.

By default, the cache entries (if not replaced by newer ones) are stored for an infinite time. That is, if you start your Python process using ftputil and let it run for three days a stat call may still access cache data that old. To avoid this, you can set the max_age attribute:

with ftputil.FTPHost(server, user, password) as ftp_host:
    ftp_host.stat_cache.max_age = 60 * 60  # = 3600 seconds

This sets the maximum age of entries in the cache to an hour. This means any entry older won't be retrieved from the cache but its data instead fetched again from the remote host and then again stored for up to an hour. To reset max_age to the default of unlimited age, i. e. cache entries never expire, use None as value.

If you are certain that the cache will be in the way, you can disable and later re-enable it completely with disable and enable:

with ftputil.FTPHost(server, user, password) as ftp_host:
    ftp_host.stat_cache.disable()
    ...
    ftp_host.stat_cache.enable()

During that time, the cache won't be used; all data will be fetched from the network. After enabling the cache again, its entries will be the same as when the cache was disabled, that is, entries won't get updated with newer data during this period. Note that even when the cache is disabled, the file system data in the code can become inconsistent:

with ftputil.FTPHost(server, user, password) as ftp_host:
    ftp_host.stat_cache.disable()
    if ftp_host.path.exists("some_file"):
        mtime = ftp_host.path.getmtime("some_file")

In that case, the file some_file may have been removed by another process between the calls to exists and getmtime!

Iteration over directories

  • walk(top, topdown=True, onerror=None, followlinks=False)

    iterates over a directory tree, similar to os.walk. Actually, FTPHost.walk uses the code from Python with just the necessary modifications, so see the linked documentation.

  • path.walk(path, func, arg)

    Similar to os.path.walk, the walk method in FTPHost.path can be used, though FTPHost.walk is probably easier to use.

Other methods

  • close()

    closes the connection to the remote host. After this, no more interaction with the FTP server is possible with this FTPHost object. Usually you don't need to close an FTPHost instance with close if you set up the instance in a with statement.

  • rename(source, target)

    renames the source file (or directory) on the FTP server.

  • chmod(path, mode)

    sets the access mode (permission flags) for the given path. The mode is an integer as returned for the mode by the stat and lstat methods. Be careful: Usually, mode values are written as octal numbers, for example 0755 to make a directory readable and writable for the owner, but not writable for the group and others. If you want to use such octal values, rely on Python's support for them:

    ftp_host.chmod("some_directory", 0o755)

    Not all FTP servers support the chmod command. In case of an exception, how do you know if the path doesn't exist or if the command itself is invalid? If the FTP server complies with RFC 959, it should return a status code 502 if the SITE CHMOD command isn't allowed. ftputil maps this special error response to a CommandNotImplementedError which is derived from PermanentError.

    So you need to code like this:

    with ftputil.FTPHost(server, user, password) as ftp_host:
    try:
        ftp_host.chmod("some_file", 0o644)
    except ftputil.error.CommandNotImplementedError:
        # `chmod` not supported
        ...
    except ftputil.error.PermanentError:
        # Possibly a non-existent file
        ...

    Because the CommandNotImplementedError is more specific, you have to test for it first.

  • copyfileobj(source, target, length=64*1024)

    copies the contents from the file-like object source to the file-like object target. The only difference to shutil.copyfileobj is the default buffer size. Note that arbitrary file-like objects can be used as arguments (e. g. local files, remote FTP files).

    However, the interfaces of source and target have to match; the string type read from source must be an accepted string type when written to target. For example, if you open source in Python 3 as a local text file and target as a remote file object in binary mode, the transfer will fail since source.read gives unicode strings whereas target.write only accepts byte strings.

    See File-like objects for the construction and use of remote file-like objects.

  • set_parser(parser)

    sets a custom parser for FTP directories. Note that you have to pass in a parser instance, not the class.

    An extra section shows how to write own parsers if the default parsers in ftputil don't work for you.

  • keep_alive()

    attempts to keep the connection to the remote server active in order to prevent timeouts from happening. This method is primarily intended to keep the underlying FTP connection of an FTPHost object alive while a file is uploaded or downloaded. This will require either an extra thread while the upload or download is in progress or calling keep_alive from a callback function.

    The keep_alive method won't help if the connection has already timed out. In this case, a ftputil.error.TemporaryError is raised.

    If you want to use this method, keep in mind that FTP servers define a timeout for a reason. A timeout prevents running out of server connections because of clients that never disconnect on their own.

    Note that the keep_alive method does not affect the "hidden" FTP child connections established by FTPHost.open (see section FTPHost instances vs. FTP connections for details). You can't use keep_alive to avoid a timeout in a stalling transfer like this:

    with ftputil.FTPHost(server, userid, password) as ftp_host:
    with ftp_host.open("some_remote_file", "rb") as fobj:
        data = fobj.read(100)
        # _Futile_ attempt to avoid file connection timeout.
        for i in xrange(15):
            time.sleep(60)
            ftp_host.keep_alive()
        # Will raise an `ftputil.error.TemporaryError`.
        data += fobj.read()

File-like objects

Construction

Basics

FTPFile objects are returned by a call to FTPHost.open; never use the FTPFile constructor directly.

The API of remote file-like objects are is modeled after the API of the io module in Python 3, which has also been backported to Python 2.6 and 2.7.

  • FTPHost.open(path, mode="r", buffering=None, encoding=None, errors=None, newline=None, rest=None)

    returns a file-like object that refers to the path on the remote host. This path may be absolute or relative to the current directory on the remote host (this directory can be determined with the getcwd method). As with local file objects, the default mode is "r", i. e. reading text files. Valid modes are "r", "rb", "w", and "wb".

    If a file is opened in binary mode, you must not specify an encoding. On the other hand, if you open a file in text mode, an encoding is used. By default, this is the return value of locale.getpreferredencoding, but you can (and probably should) specify a distinct encoding.

    If you open a file in binary mode, the read and write operations use byte strings (str in Python 2, bytes in Python 3). That is, read operations return byte strings and write operations only accept byte strings.

    Similarly, text files always work with unicode strings (unicode in Python 2, str in Python 3). Here, read operations return unicode strings and write operations only accept unicode strings.

    Warning

    Note that the semantics of "text mode" has changed fundamentally from ftputil 2.8 and earlier. Previously, "text mode" implied converting newline characters to \r\n when writing remote files and converting newlines to \n when reading remote files. This is in line with the "text mode" notion of FTP command line clients. Now, "text mode" follows the semantics in Python's io module.

    The arguments errors and newline have the same semantics as in io.open. The argument buffering currently is ignored. It's only there for compatibility with the io.open interface.

    If the file is opened in binary mode, you may pass 0 or a positive integer for the rest argument. The argument is passed to the underlying FTP session instance (for example an instance of ftplib.FTP) to start reading or writing at the given byte offset. For example, if a remote file contains the letters "abcdef" in ASCII encoding, rest=3 will start reading at "d".

    Warning

    If you pass rest values which point after the file, the behavior is undefined and may even differ from one FTP server to another. Therefore, use the rest argument only for error recovery in case of interrupted transfers. You need to keep track of the transferred data so that you can provide a valid rest argument for a resumed transfer.

FTPHost.open can also be used in a with statement:

import ftputil

with ftputil.FTPHost(...) as ftp_host:
    ...
    with ftp_host.open("new_file", "w", encoding="utf8") as fobj:
        fobj.write("This is some text.")

At the end of the with block, the remote file will be closed automatically.

If something goes wrong during the construction of the file or in the body of the with statement, the file will be closed as well. Exceptions will be propagated as with try ... finally.

Attributes and methods

The methods

close()
read([count])
readline([count])
readlines()
write(data)
writelines(string_sequence)

and the attribute closed have the same semantics as for file objects of a local disk file system. The iterator protocol is supported as well, i. e. you can use a loop to read a file line by line:

with ftputil.FTPHost(server, user, password) as ftp_host:
    with ftp_host.open("some_file") as input_file:
        for line in input_file:
            # Do something with the line, e. g.
            print line.strip().replace("ftplib", "ftputil")

For more on file objects, see the section File objects in the Python Library Reference.

FTPHost instances vs. FTP connections

This section explains why keeping an FTPHost instance "alive" without timing out sometimes isn't trivial. If you always finish your FTP operations in time, you don't need to read this section.

The file transfer protocol is a stateful protocol. That means an FTP connection always is in a certain state. Each of these states can only change to certain other states under certain conditions triggered by the client or the server.

One of the consequences is that a single FTP connection can't be used at the same time, say, to transfer data on the FTP data channel and to create a directory on the remote host.

For example, consider this:

>>> import ftplib
>>> ftp = ftplib.FTP(server, user, password)
>>> ftp.pwd()
'/'
>>> # Start transfer. `CONTENTS` is a text file on the server.
>>> socket = ftp.transfercmd("RETR CONTENTS")
>>> socket
<socket._socketobject object at 0x7f801a6386e0>
>>> ftp.pwd()
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/usr/lib64/python2.7/ftplib.py", line 578, in pwd
    return parse257(resp)
  File "/usr/lib64/python2.7/ftplib.py", line 842, in parse257
    raise error_reply, resp
ftplib.error_reply: 226-File successfully transferred
226 0.000 seconds (measured here), 5.60 Mbytes per second
>>>

Note that ftp is a single FTP connection, represented by an ftplib.FTP instance, not an ftputil.FTPHost instance.

On the other hand, consider this:

>>> import ftputil
>>> ftp_host = ftputil.FTPHost(server, user, password)
>>> ftp_host.getcwd()
>>> fobj = ftp_host.open("CONTENTS")
>>> fobj
<ftputil.file.FTPFile object at 0x7f8019d3aa50>
>>> ftp_host.getcwd()
u'/'
>>> fobj.readline()
u'Contents of FTP test directory\n'
>>> fobj.close()
>>>

To be able to start a file transfer (i. e. open a remote file for reading or writing) and still be able to use other FTP commands, ftputil uses a trick. For every remote file, ftputil creates a new FTP connection, called a child connection in the ftputil source code. (Actually, FTP connections belonging to closed remote files are re-used if they haven't timed out yet.)

In most cases this approach isn't noticeable by code using ftputil. However, the nice abstraction of dealing with a single FTP connection falls apart if one of the child connections times out. For example, if you open a remote file and work only with the initial "main" connection to navigate the file system, the FTP connection for the remote file may eventually time out.

While it's often relatively easy to prevent the "main" connection from timing out it's unfortunately practically impossible to do this for a remote file connection (apart from transferring some data, of course). For this reason, FTPHost.keep_alive affects only the main connection. Child connections may still time out if they're idle for too long.

Some more details:

  • A kind of "straightforward" way of keeping the main connection alive would be to call ftp_host.getcwd(). However, this doesn't work because ftputil caches the current directory and returns it without actually contacting the server. That's the main reason why there's a keep_alive method since it calls pwd on the FTP connection (i. e. the session object), which isn't a public attribute.
  • Some servers define not only an idle timeout but also a transfer timeout. This means the connection times out unless there's some transfer on the data channel for this connection. So ftputil's keep_alive doesn't prevent this timeout, but an ftp_host.listdir(ftp_host.curdir) call should do it. However, this transfers the data for the whole directory listing which might take some time if the directory has many entries.

Bottom line: If you can, you should organize your FTP actions so that you finish everything before a timeout happens.

Writing directory parsers

ftputil recognizes the two most widely-used FTP directory formats, Unix and MS style, and adjusts itself automatically. Almost every FTP server uses one of these formats.

However, if your server uses a format which is different from the two provided by ftputil, you can plug in a custom parser with a single method call and have ftputil use this parser.

For this, you need to write a parser class by inheriting from the class Parser in the ftputil.stat module. Here's an example:

import ftputil.error
import ftputil.stat

class XyzParser(ftputil.stat.Parser):
    """
    Parse the default format of the FTP server of the XYZ
    corporation.
    """

    def parse_line(self, line, time_shift=0.0):
        """
        Parse a `line` from the directory listing and return a
        corresponding `StatResult` object. If the line can't
        be parsed, raise `ftputil.error.ParserError`.

        The `time_shift` argument can be used to fine-tune the
        parsing of dates and times. See the class
        `ftputil.stat.UnixParser` for an example.
        """
        # Split the `line` argument and examine it further; if
        # something goes wrong, raise an `ftputil.error.ParserError`.
        ...
        # Make a `StatResult` object from the parts above.
        stat_result = ftputil.stat.StatResult(...)
        # `_st_name`, `_st_target` and `_st_mtime_precision` are optional.
        stat_result._st_name = ...
        stat_result._st_target = ...
        stat_result._st_mtime_precision = ...
        return stat_result

    # Define `ignores_line` only if the default in the base class
    # doesn't do enough!
    def ignores_line(self, line):
        """
        Return a true value if the line should be ignored. For
        example, the implementation in the base class handles
        lines like "total 17". On the other hand, if the line
        should be used for stat'ing, return a false value.
        """
        is_total_line = super(XyzParser, self).ignores_line(line)
        my_test = ...
        return is_total_line or my_test

A StatResult object is similar to the value returned by os.stat and is usually built with statements like

stat_result = StatResult(
                (st_mode, st_ino, st_dev, st_nlink, st_uid,
                 st_gid, st_size, st_atime, st_mtime, st_ctime))
stat_result._st_name = ...
stat_result._st_target = ...
stat_result._st_mtime_precision = ...

with the arguments of the StatResult constructor described in the following table.

Index Attribute os.stat type StatResult type Notes
0 st_mode int int  
1 st_ino long long  
2 st_dev long long  
3 st_nlink int int  
4 st_uid int str usually only available as string
5 st_gid int str usually only available as string
6 st_size long long  
7 st_atime int/float float  
8 st_mtime int/float float  
9 st_ctime int/float float  
- _st_name - str file name without directory part
- _st_target - str link target (may be absolute or relative)
- _st_mtime_precision - int st_mtime precision in seconds

If you can't extract all the desirable data from a line (for example, the MS format doesn't contain any information about the owner of a file), set the corresponding values in the StatResult instance to None.

Parser classes can use several helper methods which are defined in the class Parser:

  • parse_unix_mode parses strings like "drwxr-xr-x" and returns an appropriate st_mode integer value.
  • parse_unix_time returns a float number usable for the st_...time values by parsing arguments like "Nov"/"23"/"02:33" or "May"/"26"/"2005". Note that the method expects the timestamp string already split at whitespace.
  • parse_ms_time parses arguments like "10-23-01"/"03:25PM" and returns a float number like from time.mktime. Note that the method expects the timestamp string already split at whitespace.

Additionally, there's an attribute _month_numbers which maps lowercase three-letter month abbreviations to integers.

For more details, see the two "standard" parsers UnixParser and MSParser in the module ftputil/stat.py.

To actually use the parser, call the method set_parser of the FTPHost instance.

If you can't write a parser or don't want to, please ask on the ftputil mailing list. Possibly someone has already written a parser for your server or can help with it.

FAQ / Tips and tricks

Where can I get the latest version?

See the download page. Announcements will be sent to the mailing list. Announcements on major updates will also be posted to the newsgroup comp.lang.python.announce .

Is there a mailing list on ftputil?

Yes, please visit http://ftputil.sschwarzer.net/mailinglist to subscribe or read the archives.

Though you can technically post without subscribing first I can't recommend it: The mails from non-subscribers have to be approved by me and because the arriving mails contain lots of spam, I rarely go through these mails.

I found a bug! What now?

Before reporting a bug, make sure that you already read this manual and tried the latest version of ftputil. There the bug might have already been fixed.

Please see http://ftputil.sschwarzer.net/issuetrackernotes for guidelines on entering a bug in ftputil's ticket system. If you are unsure if the behaviour you found is a bug or not, you should write to the ftputil mailing list. In either case you must not include confidential information (user id, password, file names, etc.) in the problem report! Be careful!

Does ftputil support SSL/TLS?

ftputil has no built-in SSL/TLS support.

On the other hand, there are two ways to get TLS support with ftputil:

  • In Python 2.7 and Python 3.2 and up, the ftplib library has a class FTP_TLS that you can use for the session_factory keyword argument in the FTPHost constructor. You can't use the class directly though if you need additional setup code in comparison to ftplib.FTP, for example calling prot_p, to secure the data connection. On the other hand, ftputil.session.session_factory can be used to create a custom session factory.

    If you have other requirements that session_factory can't fulfill, you may create your own session factory by inheriting from ftplib.FTP_TLS:

    import ftplib

import ftputil


class FTPTLSSession(ftplib.FTP_TLS):

    def __init__(self, host, user, password):
        ftplib.FTP_TLS.__init__(self)
        self.connect(host, port)
        self.login(user, password)
        # Set up encrypted data connection.
        self.prot_p()
        ...

# Note the `session_factory` parameter. Pass the class, not
# an instance.
with ftputil.FTPHost(server, user, password,
                     session_factory=FTPTLSSession) as ftp_host:
    # Use `ftp_host` as usual.
    ...

  • If you need to work with Python 2.6, you can use the ftpslib.FTP_TLS class from the M2Crypto project. Again, you can't use the class directly but need to use ftputil.session.session_factory or a recipe similar to that above.

    Unfortunately, M2Crypto.ftpslib.FTP_TLS (at least in version 0.22.3) doesn't work correctly if you pass unicode strings to its methods. Since ftputil does exactly that at some point (even if you used byte strings in ftputil calls) you need a workaround in the session factory class:

    import M2Crypto

import ftputil
import ftputil.tool


class M2CryptoSession(M2Crypto.ftpslib.FTP_TLS):

    def __init__(self, host, user, password):
        M2Crypto.ftpslib.FTP_TLS.__init__(self)
        # Change the port number if needed.
        self.connect(host, 21)
        self.auth_tls()
        self.login(user, password)
        self.prot_p()
        self._fix_socket()
        ...

    def _fix_socket(self):
        """
        Change the socket object so that arguments to `sendall`
        are converted to byte strings before being used.
        """
        original_sendall = self.sock.sendall
        # Bound method, therefore no `self` argument.
        def sendall(data):
            data = ftputil.tool.as_bytes(data)
            return original_sendall(data)
        self.sock.sendall = sendall

# Note the `session_factory` parameter. Pass the class, not
# an instance.
with ftputil.FTPHost(server, user, password,
                     session_factory=M2CryptoSession) as ftp_host:
    # Use `ftp_host` as usual.
    ...

    That said, session_factory has this workaround built in, so normally you don't need to define the session factory yourself!

How do I connect to a non-default port?

By default, an instantiated FTPHost object connects on the usual FTP port. If you have to use a different port, refer to the section Session factories.

How can I debug an FTP connection problem?

You can do this with a session factory. See Session factories.

If you want to change the debug level only temporarily after the connection is established, you can reach the session object as the _session attribute of the FTPHost instance and call _session.set_debuglevel. Note that the _session attribute should only be accessed for debugging. Calling arbitrary ftplib.FTP methods on the session object may cause bugs!

Conditional upload/download to/from a server in a different time zone

You may find that ftputil uploads or downloads files unnecessarily, or not when it should. This can happen when the FTP server is in a different time zone than the client on which ftputil runs. Please see the section on time zone correction. It may even be sufficient to call synchronize_times.

When I use ftputil, all I get is a ParserError exception

The FTP server you connect to may use a directory format that ftputil doesn't understand. You can either write and plug in an own parser or ask on the mailing list for help.

I don't find an answer to my problem in this document

Please send an email with your problem report or question to the ftputil mailing list, and we'll see what we can do for you. :-)

Bugs and limitations

  • ftputil needs at least Python 2.6 to work.
  • Whether ftputil "sees" "hidden" directory and file names (i. e. names starting with a dot) depends on the configuration of the FTP server. See Hidden files and directories for details.
  • Due to the implementation of lstat it can not return a sensible value for the root directory / though stat'ing entries in the root directory isn't a problem. If you know an implementation that can do this, please let me know. The root directory is handled appropriately in FTPHost.path.exists/isfile/isdir/islink, though.
  • In multithreaded programs, you can have each thread use one or more FTPHost instances as long as no instance is shared with other threads.
  • Currently, it is not possible to continue an interrupted upload or download. Contact me if this causes problems for you.
  • There's exactly one cache for lstat results for each FTPHost object, i. e. there's no sharing of cache results determined by several FTPHost objects. See Local caching of file system information for the reasons.

Files

If not overwritten via installation options, the ftputil files reside in the ftputil package. There's also documentation in reStructuredText and in HTML format. The locations of these files after installation is system-dependent.

The files test_*.py and mock_ftplib.py are for unit-testing. If you only use ftputil, i. e. don't modify it, you can delete these files.

References

Authors

ftputil is written by Stefan Schwarzer <sschwarzer@sschwarzer.net> and contributors (see doc/contributors.txt).

The original lrucache module was written by Evan Prodromou <evan@prodromou.name>.

Feedback is appreciated. :-)

ftputil-3.4/doc/contributors.txt0000644000175000017470000000010513135113162016336 0ustar debiandebianStefan Schwarzer Evan Prodromou Roger Demetrescu Philippe Ombredanne ftputil-3.4/doc/ftputil.txt0000644000175000017470000017567413200531426015320 0ustar debiandebian``ftputil`` -- a high-level FTP client library ============================================== :Version: 3.4 :Date: 2017-11-08 :Summary: high-level FTP client library for Python :Keywords: FTP, ``ftplib`` substitute, virtual filesystem, pure Python :Author: Stefan Schwarzer .. contents:: Introduction ------------ The ``ftputil`` module is a high-level interface to the ftplib_ module. The `FTPHost objects`_ generated from it allow many operations similar to those of os_, `os.path`_ and `shutil`_. .. _ftplib: https://docs.python.org/library/ftplib.html .. _os: https://docs.python.org/library/os.html .. _`os.stat`: https://docs.python.org/library/os.html#os.stat .. _`os.path`: https://docs.python.org/library/os.path.html .. _`shutil`: https://docs.python.org/library/shutil.html Example:: import ftputil # Download some files from the login directory. with ftputil.FTPHost("ftp.domain.com", "user", "password") as ftp_host: names = ftp_host.listdir(ftp_host.curdir) for name in names: if ftp_host.path.isfile(name): ftp_host.download(name, name) # remote, local # Make a new directory and copy a remote file into it. ftp_host.mkdir("newdir") with ftp_host.open("index.html", "rb") as source: with ftp_host.open("newdir/index.html", "wb") as target: ftp_host.copyfileobj(source, target) # similar to shutil.copyfileobj Also, there are `FTPHost.lstat`_ and `FTPHost.stat`_ to request size and modification time of a file. The latter can also follow links, similar to `os.stat`_. `FTPHost.walk`_ and `FTPHost.path.walk`_ work, too. ``ftputil`` features -------------------- * Method names are familiar from Python's ``os``, ``os.path`` and ``shutil`` modules. For example, use ``os.path.join`` to join paths for a local file system and ``ftp_host.path.join`` to join paths for a remote FTP file system. * Remote file system navigation (``getcwd``, ``chdir``) * Upload and download files (``upload``, ``upload_if_newer``, ``download``, ``download_if_newer``) * Time zone synchronization between client and server (needed for ``upload_if_newer`` and ``download_if_newer``) * Create and remove directories (``mkdir``, ``makedirs``, ``rmdir``, ``rmtree``) and remove files (``remove``) * Get information about directories, files and links (``listdir``, ``stat``, ``lstat``, ``exists``, ``isdir``, ``isfile``, ``islink``, ``abspath``, ``split``, ``join``, ``dirname``, ``basename`` etc.) * Iterate over remote file systems (``walk``) * Local caching of results from ``lstat`` and ``stat`` calls to reduce network access (also applies to ``exists``, ``getmtime`` etc.). * Read files from and write files to remote hosts via file-like objects (``FTPHost.open``; the generated file-like objects have the familiar methods like ``read``, ``readline``, ``readlines``, ``write``, ``writelines`` and ``close``. You can also iterate over these files line by line in a ``for`` loop. Exception hierarchy ------------------- The exceptions are in the namespace of the ``ftputil.error`` module, e. g. ``ftputil.error.TemporaryError``. The exception classes are organized as follows:: FTPError FTPOSError(FTPError, OSError) PermanentError(FTPOSError) CommandNotImplementedError(PermanentError) TemporaryError(FTPOSError) FTPIOError(FTPError) InternalError(FTPError) InaccessibleLoginDirError(InternalError) ParserError(InternalError) RootDirError(InternalError) TimeShiftError(InternalError) and are described here: - ``FTPError`` is the root of the exception hierarchy of the module. - ``FTPOSError`` is derived from ``OSError``. This is for similarity between the os module and ``FTPHost`` objects. Compare :: try: os.chdir("nonexisting_directory") except OSError: ... with :: host = ftputil.FTPHost("host", "user", "password") try: host.chdir("nonexisting_directory") except OSError: ... Imagine a function :: def func(path, file): ... which works on the local file system and catches ``OSErrors``. If you change the parameter list to :: def func(path, file, os=os): ... where ``os`` denotes the ``os`` module, you can call the function also as :: host = ftputil.FTPHost("host", "user", "password") func(path, file, os=host) to use the same code for both a local and remote file system. Another similarity between ``OSError`` and ``FTPOSError`` is that the latter holds the FTP server return code in the ``errno`` attribute of the exception object and the error text in ``strerror``. - ``PermanentError`` is raised for 5xx return codes from the FTP server. This corresponds to ``ftplib.error_perm`` (though ``PermanentError`` and ``ftplib.error_perm`` are *not* identical). - ``CommandNotImplementedError`` indicates that an underlying command the code tries to use is not implemented. For an example, see the description of the `FTPHost.chmod`_ method. - ``TemporaryError`` is raised for FTP return codes from the 4xx category. This corresponds to ``ftplib.error_temp`` (though ``TemporaryError`` and ``ftplib.error_temp`` are *not* identical). - ``FTPIOError`` denotes an I/O error on the remote host. This appears mainly with file-like objects that are retrieved by calling ``FTPHost.open``. Compare :: >>> try: ... f = open("not_there") ... except IOError as obj: ... print obj.errno ... print obj.strerror ... 2 No such file or directory with :: >>> ftp_host = ftputil.FTPHost("host", "user", "password") >>> try: ... f = ftp_host.open("not_there") ... except IOError as obj: ... print obj.errno ... print obj.strerror ... 550 550 not_there: No such file or directory. As you can see, both code snippets are similar. However, the error codes aren't the same. - ``InternalError`` subsumes exception classes for signaling errors due to limitations of the FTP protocol or the concrete implementation of ``ftputil``. - ``InaccessibleLoginDirError`` This exception is raised if the directory in which "you" are placed upon login is not accessible, i. e. a ``chdir`` call with the directory as argument would fail. - ``ParserError`` is used for errors during the parsing of directory listings from the server. This exception is used by the ``FTPHost`` methods ``stat``, ``lstat``, and ``listdir``. - ``RootDirError`` Because of the implementation of the ``lstat`` method it is not possible to do a ``stat`` call on the root directory ``/``. If you know *any* way to do it, please let me know. :-) This problem does *not* affect stat calls on items *in* the root directory. - ``TimeShiftError`` is used to denote errors which relate to setting the `time shift`_. Directory and file names ------------------------ .. note:: Keep in mind that this section only applies to directory and file *names*, not file *contents*. Encoding and decoding for file contents is handled by the ``encoding`` argument for `FTPHost.open`_. First off: If your directory and file names (both as arguments and on the server) contain only ISO 8859-1 (latin-1) characters, you can use such names in the form of byte strings or unicode strings. However, you can't mix different string types (bytes and unicode) in one call (for example in ``FTPHost.path.join``). If you have directory or file names with characters that aren't in latin-1, it's recommended to use byte strings. In that case, returned paths will be byte strings, too. Read on for details. .. note:: The approach described below may look awkward and in a way it is. The intention of ``ftputil`` is to behave like the local file system APIs of Python 3 as far as it makes sense. Moreover, the taken approach makes sure that directory and file names that were used with Python 3's native ``ftplib`` module will be compatible with ``ftputil`` and vice versa. Otherwise you may be able to use a file name with ``ftputil``, but get an exception when trying to read the same file with Python 3's ``ftplib`` module. Methods that take names of directories and/or files can take either byte or unicode strings. If a method got a string argument and returns one or more strings, these strings will have the same string type as the argument(s). Mixing different string arguments in one call (for example in ``FTPHost.path.join``) isn't allowed and will cause a ``TypeError``. These rules are the same as for local file system operations in Python 3. Since ``ftputil`` uses the same API for Python 2, ``ftputil`` will do the same when run on Python 2. Byte strings for directory and file names will be sent to the server as-is. On the other hand, unicode strings will be encoded to byte strings, assuming latin-1 encoding. This implies that such unicode strings must only contain code points 0-255 for the latin-1 character set. Using any other characters will result in a ``UnicodeEncodeError`` exception. If you have directory or file names as unicode strings with non-latin-1 characters, encode the unicode strings to byte strings yourself, using the encoding you know the server uses. Decode received paths with the same encoding. Encapsulate these conversions as far as you can. Otherwise, you'd have to adapt potentially a lot of code if the server encoding changes. If you *don't* know the encoding on the server side, it's probably the best to only use byte strings for directory and file names. That said, as soon as you *show* the names to a user, you -- or the library you use for displaying the names -- has to guess an encoding. ``FTPHost`` objects ------------------- .. _`FTPHost construction`: Construction ~~~~~~~~~~~~ Introduction ```````````` ``FTPHost`` instances can be created with the following call:: ftp_host = ftputil.FTPHost(server, user, password, account, session_factory=ftplib.FTP) The first four parameters are strings with the same meaning as for the FTP class in the ``ftplib`` module. Usually the ``account`` and ``session_factory`` arguments aren't needed though. ``FTPHost`` objects can also be used in a ``with`` statement:: import ftputil with ftputil.FTPHost(server, user, password) as ftp_host: print ftp_host.listdir(ftp_host.curdir) After the ``with`` block, the ``FTPHost`` instance and the associated FTP sessions will be closed automatically. If something goes wrong during the ``FTPHost`` construction or in the body of the ``with`` statement, the instance is closed as well. Exceptions will be propagated (as with ``try ... finally``). Session factories ````````````````` The keyword argument ``session_factory`` may be used to generate FTP connections with other factories than the default ``ftplib.FTP``. For example, the standard library of Python 2.7 contains a class ``ftplib.FTP_TLS`` which extends ``ftplib.FTP`` to use an encrypted connection. In fact, all positional and keyword arguments other than ``session_factory`` are passed to the factory to generate a new background session. This also happens for every remote file that is opened; see below. This functionality of the constructor also allows to wrap ``ftplib.FTP`` objects to do something that wouldn't be possible with the ``ftplib.FTP`` constructor alone. As an example, assume you want to connect to another than the default port, but ``ftplib.FTP`` only offers this by means of its ``connect`` method, not via its constructor. One solution is to use a custom class as a session factory:: import ftplib import ftputil EXAMPLE_PORT = 50001 class MySession(ftplib.FTP): def __init__(self, host, userid, password, port): """Act like ftplib.FTP's constructor but connect to another port.""" ftplib.FTP.__init__(self) self.connect(host, port) self.login(userid, password) # Try _not_ to use an _instance_ `MySession()` as factory, - # use the class itself. with ftputil.FTPHost(host, userid, password, port=EXAMPLE_PORT, session_factory=MySession) as ftp_host: # Use `ftp_host` as usual. ... On login, the format of the directory listings (needed for stat'ing files and directories) should be determined automatically. If not, please `file a bug report`_. .. _`file a bug report`: http://ftputil.sschwarzer.net/issuetrackernotes For the most common uses you don't need to create your own session factory class though. The ``ftputil.session`` module has a function ``session_factory`` that can create session factories for a variety of parameters:: session_factory(base_class=ftplib.FTP, port=21, use_passive_mode=None, encrypt_data_channel=True, debug_level=None) with - ``base_class`` is a base class to inherit a new session factory class from. By default, this is ``ftplib.FTP`` from the Python standard library. - ``port`` is the command channel port. The default is 21, used in most FTP server configurations. - ``use_passive_mode`` is either a boolean that determines whether passive mode should be used or ``None``. ``None`` means to let the base class choose active or passive mode. - ``encrypt_data_channel`` defines whether to encrypt the data channel for secure connections. This is only supported for the base classes ``ftplib.FTP_TLS`` and ``M2Crypto.ftpslib.FTP_TLS``, otherwise the the parameter is ignored. - ``debug_level`` sets the debug level for FTP session instances. The semantics is defined by the base class. For example, a debug level of 2 causes the most verbose output for Python's ``ftplib.FTP`` class. All of these parameters can be combined. For example, you could use :: import ftplib import ftputil import ftputil.session my_session_factory = ftputil.session.session_factory( base_class=ftpslib.FTP_TLS, port=31, encrypt_data_channel=True, debug_level=2) with ftputil.FTPHost(server, user, password, session_factory=my_session_factory) as ftp_host: ... to create and use a session factory derived from ``ftplib.FTP_TLS`` that connects on command channel 31, will encrypt the data channel and print output for debug level 2. Note: Generally, you can achieve everything you can do with ``ftputil.session.session_factory`` with an explicit session factory as described at the start of this section. However, the class ``M2Crypto.ftpslib.FTP_TLS`` has a limitation so that you can't use it with ftputil out of the box. The function ``session_factory`` contains a workaround for this limitation. For details refer to `this bug report`_. .. _`this bug report`: http://ftputil.sschwarzer.net/trac/ticket/78 Hidden files and directories ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Whether ftputil sees "hidden" files and directories (usually files or directories whose names start with a dot) depends on the FTP server configuration. By default, ftputil uses the ``-a`` option in the FTP ``LIST`` command to find hidden files. However, the server may ignore this. If using the ``-a`` option leads to problems, for example if an FTP server causes an exception, you may switch off the use of the option:: ftp_host = ftputil.FTPHost(server, user, password, account, session_factory=ftplib.FTP) ftp_host.use_list_a_option = False ``FTPHost`` attributes and methods ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Attributes `````````` - ``curdir``, ``pardir``, ``sep`` are strings which denote the current and the parent directory on the remote server. ``sep`` holds the path separator. Though `RFC 959`_ (File Transfer Protocol) notes that these values may depend on the FTP server implementation, the Unix variants seem to work well in practice, even for non-Unix servers. Nevertheless, it's recommended that you don't hardcode these values for remote paths, but use `FTPHost.path`_ as you would use ``os.path`` to write platform-independent Python code for local filesystems. Keep in mind that most, *but not all*, arguments of ``FTPHost`` methods refer to remote directories or files. For example, in `FTPHost.upload`_, the first argument is a local path and the second a remote path. Both of these should use their respective path separators. .. _`FTPHost.upload`: `Uploading and downloading files`_ Remote file system navigation ````````````````````````````` - ``getcwd()`` returns the absolute current directory on the remote host. This method works like ``os.getcwd``. - ``chdir(directory)`` sets the current directory on the FTP server. This resembles ``os.chdir``, as you may have expected. .. _`callback function`: Uploading and downloading files ``````````````````````````````` - ``upload(source, target, callback=None)`` copies a local source file (given by a filename, i. e. a string) to the remote host under the name target. Both ``source`` and ``target`` may be absolute paths or relative to their corresponding current directory (on the local or the remote host, respectively). The file content is always transferred in binary mode. The callback, if given, will be invoked for each transferred chunk of data:: callback(chunk) where ``chunk`` is a bytestring. An example usage of a callback method is to display a progress indicator. - ``download(source, target, callback=None)`` performs a download from the remote source file to a local target file. Both ``source`` and ``target`` are strings. See the description of ``upload`` for more details. .. _`upload_if_newer`: - ``upload_if_newer(source, target, callback=None)`` is similar to the ``upload`` method. The only difference is that the upload is only invoked if the time of the last modification for the source file is more recent than that of the target file or the target doesn't exist at all. The check for the last modification time considers the precision of the timestamps and transfers a file "if in doubt". Consequently the code :: ftp_host.upload_if_newer("source_file", "target_file") time.sleep(10) ftp_host.upload_if_newer("source_file", "target_file") might upload the file again if the timestamp of the target file is precise up to a minute, which is typically the case because the remote datetime is determined by parsing a directory listing from the server. To avoid unnecessary transfers, wait at least a minute between calls of ``upload_if_newer`` for the same file. If it still seems that a file is uploaded unnecessarily (or not when it should), read the subsection on `time shift`_ settings. If an upload actually happened, the return value of ``upload_if_newer`` is a ``True``, else ``False``. Note that the method only checks the existence and/or the modification time of the source and target file; it doesn't compare any other file properties, say, the file size. This also means that if a transfer is interrupted, the remote file will have a newer modification time than the local file, and thus the transfer won't be repeated if ``upload_if_newer`` is used a second time. There are at least two possibilities after a failed upload: - use ``upload`` instead of ``upload_if_newer``, or - remove the incomplete target file with ``FTPHost.remove``, then use ``upload`` or ``upload_if_newer`` to transfer it again. .. _`download_if_newer`: - ``download_if_newer(source, target, callback=None)`` corresponds to ``upload_if_newer`` but performs a download from the server to the local host. Read the descriptions of download and ``upload_if_newer`` for more information. If a download actually happened, the return value is ``True``, else ``False``. .. _`time shift`: .. _`time zone correction`: Time zone correction ```````````````````` If the client where ``ftputil`` runs and the server have a different understanding of their local times, this has to be taken into account for ``upload_if_newer`` and ``download_if_newer`` to work correctly. Note that even if the client and the server are in the same time zone (or even on the same computer), the time shift value (see below) may be different from zero. For example, my computer is set to use local time whereas the server running on the very same host insists on using UTC time. .. _`set_time_shift`: - ``set_time_shift(time_shift)`` sets the so-called time shift value, measured in seconds. The time shift is the difference between the local time of the server and the local time of the client at a given moment, i. e. by definition :: time_shift = server_time - client_time Setting this value is important for `upload_if_newer`_ and `download_if_newer`_ to work correctly even if the time zone of the FTP server differs from that of the client. Note that the time shift value *can be negative*. If the time shift value is invalid, for example its absolute value is larger than 24 hours, a ``TimeShiftError`` is raised. See also `synchronize_times`_ for a way to set the time shift with a simple method call. - ``time_shift()`` returns the currently-set time shift value. See ``set_time_shift`` above for its definition. .. _`synchronize_times`: - ``synchronize_times()`` synchronizes the local times of the server and the client, so that `upload_if_newer`_ and `download_if_newer`_ work as expected, even if the client and the server use different time zones. For this to work, *all* of the following conditions must be true: - The connection between server and client is established. - The client has write access to the directory that is current when ``synchronize_times`` is called. If you can't fulfill these conditions, you can nevertheless set the time shift value explicitly with `set_time_shift`_. Trying to call ``synchronize_times`` if the above conditions aren't met results in a ``TimeShiftError`` exception. Creating and removing directories ````````````````````````````````` - ``mkdir(path, [mode])`` makes the given directory on the remote host. This does *not* construct "intermediate" directories that don't already exist. The ``mode`` parameter is ignored; this is for compatibility with ``os.mkdir`` if an ``FTPHost`` object is passed into a function instead of the ``os`` module. See the explanation in the subsection `Exception hierarchy`_. - ``makedirs(path, [mode])`` works similar to ``mkdir`` (see above), but also makes intermediate directories like ``os.makedirs``. The ``mode`` parameter is only there for compatibility with ``os.makedirs`` and is ignored. - ``rmdir(path)`` removes the given remote directory. If it's not empty, raise a ``PermanentError``. - ``rmtree(path, ignore_errors=False, onerror=None)`` removes the given remote, possibly non-empty, directory tree. The interface of this method is rather complex, in favor of compatibility with ``shutil.rmtree``. If ``ignore_errors`` is set to a true value, errors are ignored. If ``ignore_errors`` is a false value *and* ``onerror`` isn't set, all exceptions occurring during the tree iteration and processing are raised. These exceptions are all of type ``PermanentError``. To distinguish between different kinds of errors, pass in a callable for ``onerror``. This callable must accept three arguments: ``func``, ``path`` and ``exc_info``. ``func`` is a bound method object, *for example* ``your_host_object.listdir``. ``path`` is the path that was the recent argument of the respective method (``listdir``, ``remove``, ``rmdir``). ``exc_info`` is the exception info as it is gotten from ``sys.exc_info``. The code of ``rmtree`` is taken from Python's ``shutil`` module and adapted for ``ftputil``. Removing files and links ```````````````````````` - ``remove(path)`` removes a file or link on the remote host, similar to ``os.remove``. - ``unlink(path)`` is an alias for ``remove``. Retrieving information about directories, files and links ````````````````````````````````````````````````````````` - ``listdir(path)`` returns a list containing the names of the files and directories in the given path, similar to `os.listdir`_. The special names ``.`` and ``..`` are not in the list. The methods ``lstat`` and ``stat`` (and some others) rely on the directory listing format used by the FTP server. When connecting to a host, ``FTPHost``'s constructor tries to guess the right format, which succeeds in most cases. However, if you get strange results or ``ParserError`` exceptions by a mere ``lstat`` call, please `file a bug report`_. If ``lstat`` or ``stat`` give wrong modification dates or times, look at the methods that deal with time zone differences (`time zone correction`_). .. _`FTPHost.lstat`: - ``lstat(path)`` returns an object similar to that from `os.lstat`_. This is a "tuple" with additional attributes; see the documentation of the ``os`` module for details. The result is derived by parsing the output of a ``LIST`` command on the server. Therefore, the result from ``FTPHost.lstat`` can not contain more information than the received text. In particular: - User and group ids can only be determined as strings, not as numbers, and that only if the server supplies them. This is usually the case with Unix servers but maybe not for other FTP server programs. - Values for the time of the last modification may be rough, depending on the information from the server. For timestamps older than a year, this usually means that the precision of the modification timestamp value is not better than days. For newer files, the information may be accurate to a minute. If the time of the last modification is before the epoch (usually 1970-01-01 UTC), set the time of the last modification to 0.0. - Links can only be recognized on servers that provide this information in the ``LIST`` output. - Stat attributes that can't be determined at all are set to ``None``. For example, a line of a directory listing may not contain the date/time of a directory's last modification. - There's a special problem with stat'ing the root directory. (Stat'ing things *in* the root directory is fine though.) In this case, a ``RootDirError`` is raised. This has to do with the algorithm used by ``(l)stat``, and I know of no approach which mends this problem. Currently, ``ftputil`` recognizes the common Unix-style and Microsoft/DOS-style directory formats. If you need to parse output from another server type, please write to the `ftputil mailing list`_. You may consider `writing your own parser`_. .. _`os.listdir`: https://docs.python.org/library/os.html#os.listdir .. _`os.lstat`: https://docs.python.org/library/os.html#os.lstat .. _`ftputil mailing list`: http://ftputil.sschwarzer.net/mailinglist .. _`writing your own parser`: `Writing directory parsers`_ .. _`FTPHost.stat`: - ``stat(path)`` returns ``stat`` information also for files which are pointed to by a link. This method follows multiple links until a regular file or directory is found. If an infinite link chain is encountered or the target of the last link in the chain doesn't exist, a ``PermanentError`` is raised. The limitations of the ``lstat`` method also apply to ``stat``. .. _`FTPHost.path`: ``FTPHost`` objects contain an attribute named ``path``, similar to `os.path`_. The following methods can be applied to the remote host with the same semantics as for ``os.path``: :: abspath(path) basename(path) commonprefix(path_list) dirname(path) exists(path) getmtime(path) getsize(path) isabs(path) isdir(path) isfile(path) islink(path) join(path1, path2, ...) normcase(path) normpath(path) split(path) splitdrive(path) splitext(path) walk(path, func, arg) Like Python's counterparts under `os.path`_, ``ftputil``'s ``is...`` methods return ``False`` if they can't find the path given by their argument. Local caching of file system information ```````````````````````````````````````` Many of the above methods need access to the remote file system to obtain data on directories and files. To get the most recent data, *each* call to ``lstat``, ``stat``, ``exists``, ``getmtime`` etc. would require to fetch a directory listing from the server, which can make the program *very* slow. This effect is more pronounced for operations which mostly scan the file system rather than transferring file data. For this reason, ``ftputil`` by default saves the results from directory listings locally and reuses those results. This reduces network accesses and so speeds up the software a lot. However, since data is more rarely fetched from the server, the risk of obsolete data also increases. This will be discussed below. Caching can be controlled -- if necessary at all -- via the ``stat_cache`` object in an ``FTPHost``'s namespace. For example, after calling :: ftp_host = ftputil.FTPHost(host, user, password) the cache can be accessed as ``ftp_host.stat_cache``. While ``ftputil`` usually manages the cache quite well, there are two possible reasons for modifying cache parameters. The first is when the number of possible entries is too low. You may notice that when you are processing very large directories and the program becomes much slower than before. It's common for code to read a directory with ``listdir`` and then process the found directories and files. This can also happen implicitly by a call to ``FTPHost.walk``. Since version 2.6 ``ftputil`` automatically increases the cache size if directories with more entries than the current maximum cache size are to be scanned. Most of the time, this works fine. However, if you need access to stat data for several directories at the same time, you may need to increase the cache explicitly. This is done by the ``resize`` method:: ftp_host.stat_cache.resize(20000) where the argument is the maximum number of ``lstat`` results to store (the default is 5000, in versions before 2.6 it was 1000). Note that each path on the server, e. g. "/home/schwa/some_dir", corresponds to a single cache entry. Methods like ``exists`` or ``getmtime`` all derive their results from a previously fetched ``lstat`` result. The value 5000 above means that the cache will hold *at most* 5000 entries (unless increased automatically by an explicit or implicit ``listdir`` call, see above). If more are about to be stored, the entries which haven't been used for the longest time will be deleted to make place for newer entries. The second possible reason to change the cache parameters is to avoid stale cache data. Caching is so effective because it reduces network accesses. This can also be a disadvantage if the file system data on the remote server changes after a stat result has been retrieved; the client, when looking at the cached stat data, will use obsolete information. There are two potential ways to get such out-of-date stat data. The first happens when an ``FTPHost`` instance modifies a file path for which it has a cache entry, e. g. by calling ``remove`` or ``rmdir``. Such changes are handled transparently; the path will be deleted from the cache. A different matter are changes unknown to the ``FTPHost`` object which inspects its cache. Obviously, for example, these are changes by programs running on the remote host. On the other hand, cache inconsistencies can also occur if two ``FTPHost`` objects change a file system simultaneously:: with ( ftputil.FTPHost(server, user1, password1) as ftp_host1, ftputil.FTPHost(server, user1, password1) as ftp_host2 ): stat_result1 = ftp_host1.stat("some_file") stat_result2 = ftp_host2.stat("some_file") ftp_host2.remove("some_file") # `ftp_host1` will still see the obsolete cache entry! print ftp_host1.stat("some_file") # Will raise an exception since an `FTPHost` object # knows of its own changes. print ftp_host2.stat("some_file") At first sight, it may appear to be a good idea to have a shared cache among several ``FTPHost`` objects. After some thinking, this turns out to be very error-prone. For example, it won't help with different processes using ``ftputil``. So, if you have to deal with concurrent write/read accesses to a server, you have to handle them explicitly. The most useful tool for this is the ``invalidate`` method. In the example above, it could be used like this:: with ( ftputil.FTPHost(server, user1, password1) as ftp_host1, ftputil.FTPHost(server, user1, password1) as ftp_host2 ): stat_result1 = ftp_host1.stat("some_file") stat_result2 = ftp_host2.stat("some_file") ftp_host2.remove("some_file") # Invalidate using an absolute path. absolute_path = ftp_host1.path.abspath( ftp_host1.path.join(ftp_host1.getcwd(), "some_file")) ftp_host1.stat_cache.invalidate(absolute_path) # Will now raise an exception as it should. print ftp_host1.stat("some_file") # Would raise an exception since an `FTPHost` object # knows of its own changes, even without `invalidate`. print ftp_host2.stat("some_file") The method ``invalidate`` can be used on any *absolute* path, be it a directory, a file or a link. By default, the cache entries (if not replaced by newer ones) are stored for an infinite time. That is, if you start your Python process using ``ftputil`` and let it run for three days a stat call may still access cache data that old. To avoid this, you can set the ``max_age`` attribute:: with ftputil.FTPHost(server, user, password) as ftp_host: ftp_host.stat_cache.max_age = 60 * 60 # = 3600 seconds This sets the maximum age of entries in the cache to an hour. This means any entry older won't be retrieved from the cache but its data instead fetched again from the remote host and then again stored for up to an hour. To reset `max_age` to the default of unlimited age, i. e. cache entries never expire, use ``None`` as value. If you are certain that the cache will be in the way, you can disable and later re-enable it completely with ``disable`` and ``enable``:: with ftputil.FTPHost(server, user, password) as ftp_host: ftp_host.stat_cache.disable() ... ftp_host.stat_cache.enable() During that time, the cache won't be used; all data will be fetched from the network. After enabling the cache again, its entries will be the same as when the cache was disabled, that is, entries won't get updated with newer data during this period. Note that even when the cache is disabled, the file system data in the code can become inconsistent:: with ftputil.FTPHost(server, user, password) as ftp_host: ftp_host.stat_cache.disable() if ftp_host.path.exists("some_file"): mtime = ftp_host.path.getmtime("some_file") In that case, the file ``some_file`` may have been removed by another process between the calls to ``exists`` and ``getmtime``! Iteration over directories `````````````````````````` .. _`FTPHost.walk`: - ``walk(top, topdown=True, onerror=None, followlinks=False)`` iterates over a directory tree, similar to `os.walk`_. Actually, ``FTPHost.walk`` uses the code from Python with just the necessary modifications, so see the linked documentation. .. _`os.walk`: https://docs.python.org/2/library/os.html#os.walk .. _`FTPHost.path.walk`: - ``path.walk(path, func, arg)`` Similar to ``os.path.walk``, the ``walk`` method in `FTPHost.path`_ can be used, though ``FTPHost.walk`` is probably easier to use. Other methods ````````````` - ``close()`` closes the connection to the remote host. After this, no more interaction with the FTP server is possible with this ``FTPHost`` object. Usually you don't need to close an ``FTPHost`` instance with ``close`` if you set up the instance in a ``with`` statement. - ``rename(source, target)`` renames the source file (or directory) on the FTP server. .. _`FTPHost.chmod`: - ``chmod(path, mode)`` sets the access mode (permission flags) for the given path. The mode is an integer as returned for the mode by the ``stat`` and ``lstat`` methods. Be careful: Usually, mode values are written as octal numbers, for example 0755 to make a directory readable and writable for the owner, but not writable for the group and others. If you want to use such octal values, rely on Python's support for them:: ftp_host.chmod("some_directory", 0o755) Not all FTP servers support the ``chmod`` command. In case of an exception, how do you know if the path doesn't exist or if the command itself is invalid? If the FTP server complies with `RFC 959`_, it should return a status code 502 if the ``SITE CHMOD`` command isn't allowed. ``ftputil`` maps this special error response to a ``CommandNotImplementedError`` which is derived from ``PermanentError``. So you need to code like this:: with ftputil.FTPHost(server, user, password) as ftp_host: try: ftp_host.chmod("some_file", 0o644) except ftputil.error.CommandNotImplementedError: # `chmod` not supported ... except ftputil.error.PermanentError: # Possibly a non-existent file ... Because the ``CommandNotImplementedError`` is more specific, you have to test for it first. .. _`RFC 959`: `RFC 959 - File Transfer Protocol (FTP)`_ - ``copyfileobj(source, target, length=64*1024)`` copies the contents from the file-like object ``source`` to the file-like object ``target``. The only difference to ``shutil.copyfileobj`` is the default buffer size. Note that arbitrary file-like objects can be used as arguments (e. g. local files, remote FTP files). However, the interfaces of ``source`` and ``target`` have to match; the string type read from ``source`` must be an accepted string type when written to ``target``. For example, if you open ``source`` in Python 3 as a local text file and ``target`` as a remote file object in binary mode, the transfer will fail since ``source.read`` gives unicode strings whereas ``target.write`` only accepts byte strings. See `File-like objects`_ for the construction and use of remote file-like objects. .. _`set_parser`: - ``set_parser(parser)`` sets a custom parser for FTP directories. Note that you have to pass in a parser *instance*, not the class. An `extra section`_ shows how to write own parsers if the default parsers in ``ftputil`` don't work for you. .. _`extra section`: `Writing directory parsers`_ .. _`keep_alive`: - ``keep_alive()`` attempts to keep the connection to the remote server active in order to prevent timeouts from happening. This method is primarily intended to keep the underlying FTP connection of an ``FTPHost`` object alive while a file is uploaded or downloaded. This will require either an extra thread while the upload or download is in progress or calling ``keep_alive`` from a `callback function`_. The ``keep_alive`` method won't help if the connection has already timed out. In this case, a ``ftputil.error.TemporaryError`` is raised. If you want to use this method, keep in mind that FTP servers define a timeout for a reason. A timeout prevents running out of server connections because of clients that never disconnect on their own. Note that the ``keep_alive`` method does *not* affect the "hidden" FTP child connections established by ``FTPHost.open`` (see section `FTPHost instances vs. FTP connections`_ for details). You *can't* use ``keep_alive`` to avoid a timeout in a stalling transfer like this:: with ftputil.FTPHost(server, userid, password) as ftp_host: with ftp_host.open("some_remote_file", "rb") as fobj: data = fobj.read(100) # _Futile_ attempt to avoid file connection timeout. for i in xrange(15): time.sleep(60) ftp_host.keep_alive() # Will raise an `ftputil.error.TemporaryError`. data += fobj.read() .. _`FTPHost.open`: File-like objects ----------------- Construction ~~~~~~~~~~~~ Basics `````` ``FTPFile`` objects are returned by a call to ``FTPHost.open``; never use the ``FTPFile`` constructor directly. The API of remote file-like objects are is modeled after the API of the io_ module in Python 3, which has also been backported to Python 2.6 and 2.7. .. _io: http://docs.python.org/library/io.html - ``FTPHost.open(path, mode="r", buffering=None, encoding=None, errors=None, newline=None, rest=None)`` returns a file-like object that refers to the path on the remote host. This path may be absolute or relative to the current directory on the remote host (this directory can be determined with the ``getcwd`` method). As with local file objects, the default mode is "r", i. e. reading text files. Valid modes are "r", "rb", "w", and "wb". If a file is opened in binary mode, you *must not* specify an encoding. On the other hand, if you open a file in text mode, an encoding is used. By default, this is the return value of ``locale.getpreferredencoding``, but you can (and probably should) specify a distinct encoding. If you open a file in binary mode, the read and write operations use byte strings (``str`` in Python 2, ``bytes`` in Python 3). That is, read operations return byte strings and write operations only accept byte strings. Similarly, text files always work with unicode strings (``unicode`` in Python 2, ``str`` in Python 3). Here, read operations return unicode strings and write operations only accept unicode strings. .. warning:: Note that the semantics of "text mode" has changed fundamentally from ftputil 2.8 and earlier. Previously, "text mode" implied converting newline characters to ``\r\n`` when writing remote files and converting newlines to ``\n`` when reading remote files. This is in line with the "text mode" notion of FTP command line clients. Now, "text mode" follows the semantics in Python's ``io`` module. The arguments ``errors`` and ``newline`` have the same semantics as in `io.open`_. The argument ``buffering`` currently is ignored. It's only there for compatibility with the ``io.open`` interface. If the file is opened in binary mode, you may pass 0 or a positive integer for the ``rest`` argument. The argument is passed to the underlying FTP session instance (for example an instance of ``ftplib.FTP``) to start reading or writing at the given byte offset. For example, if a remote file contains the letters "abcdef" in ASCII encoding, ``rest=3`` will start reading at "d". .. warning:: If you pass ``rest`` values which point *after* the file, the behavior is undefined and may even differ from one FTP server to another. Therefore, use the ``rest`` argument only for error recovery in case of interrupted transfers. You need to keep track of the transferred data so that you can provide a valid ``rest`` argument for a resumed transfer. .. _`io.open`: http://docs.python.org/library/io.html#io.open ``FTPHost.open`` can also be used in a ``with`` statement:: import ftputil with ftputil.FTPHost(...) as ftp_host: ... with ftp_host.open("new_file", "w", encoding="utf8") as fobj: fobj.write("This is some text.") At the end of the ``with`` block, the remote file will be closed automatically. If something goes wrong during the construction of the file or in the body of the ``with`` statement, the file will be closed as well. Exceptions will be propagated as with ``try ... finally``. Attributes and methods ~~~~~~~~~~~~~~~~~~~~~~ The methods :: close() read([count]) readline([count]) readlines() write(data) writelines(string_sequence) and the attribute ``closed`` have the same semantics as for file objects of a local disk file system. The iterator protocol is supported as well, i. e. you can use a loop to read a file line by line:: with ftputil.FTPHost(server, user, password) as ftp_host: with ftp_host.open("some_file") as input_file: for line in input_file: # Do something with the line, e. g. print line.strip().replace("ftplib", "ftputil") For more on file objects, see the section `File objects`_ in the Python Library Reference. .. _`file objects`: https://docs.python.org/2.7/library/stdtypes.html#file-objects .. _`child_connections`: ``FTPHost`` instances vs. FTP connections ----------------------------------------- This section explains why keeping an ``FTPHost`` instance "alive" without timing out sometimes isn't trivial. If you always finish your FTP operations in time, you don't need to read this section. The file transfer protocol is a stateful protocol. That means an FTP connection always is in a certain state. Each of these states can only change to certain other states under certain conditions triggered by the client or the server. One of the consequences is that a single FTP connection can't be used at the same time, say, to transfer data on the FTP data channel and to create a directory on the remote host. For example, consider this:: >>> import ftplib >>> ftp = ftplib.FTP(server, user, password) >>> ftp.pwd() '/' >>> # Start transfer. `CONTENTS` is a text file on the server. >>> socket = ftp.transfercmd("RETR CONTENTS") >>> socket >>> ftp.pwd() Traceback (most recent call last): File "", line 1, in File "/usr/lib64/python2.7/ftplib.py", line 578, in pwd return parse257(resp) File "/usr/lib64/python2.7/ftplib.py", line 842, in parse257 raise error_reply, resp ftplib.error_reply: 226-File successfully transferred 226 0.000 seconds (measured here), 5.60 Mbytes per second >>> Note that ``ftp`` is a single FTP connection, represented by an ``ftplib.FTP`` instance, not an ``ftputil.FTPHost`` instance. On the other hand, consider this:: >>> import ftputil >>> ftp_host = ftputil.FTPHost(server, user, password) >>> ftp_host.getcwd() >>> fobj = ftp_host.open("CONTENTS") >>> fobj >>> ftp_host.getcwd() u'/' >>> fobj.readline() u'Contents of FTP test directory\n' >>> fobj.close() >>> To be able to start a file transfer (i. e. open a remote file for reading or writing) and still be able to use other FTP commands, ftputil uses a trick. For every remote file, ftputil creates a new FTP connection, called a child connection in the ftputil source code. (Actually, FTP connections belonging to closed remote files are re-used if they haven't timed out yet.) In most cases this approach isn't noticeable by code using ftputil. However, the nice abstraction of dealing with a single FTP connection falls apart if one of the child connections times out. For example, if you open a remote file and work only with the initial "main" connection to navigate the file system, the FTP connection for the remote file may eventually time out. While it's often relatively easy to prevent the "main" connection from timing out it's unfortunately practically impossible to do this for a remote file connection (apart from transferring some data, of course). For this reason, `FTPHost.keep_alive`_ affects only the main connection. Child connections may still time out if they're idle for too long. .. _`FTPHost.keep_alive`: `keep_alive`_ Some more details: - A kind of "straightforward" way of keeping the main connection alive would be to call ``ftp_host.getcwd()``. However, this doesn't work because ftputil caches the current directory and returns it without actually contacting the server. That's the main reason why there's a ``keep_alive`` method since it calls ``pwd`` on the FTP connection (i. e. the session object), which isn't a public attribute. - Some servers define not only an idle timeout but also a transfer timeout. This means the connection times out unless there's some transfer on the data channel for this connection. So ftputil's ``keep_alive`` doesn't prevent this timeout, but an ``ftp_host.listdir(ftp_host.curdir)`` call should do it. However, this transfers the data for the whole directory listing which might take some time if the directory has many entries. Bottom line: If you can, you should organize your FTP actions so that you finish everything before a timeout happens. Writing directory parsers ------------------------- ``ftputil`` recognizes the two most widely-used FTP directory formats, Unix and MS style, and adjusts itself automatically. Almost every FTP server uses one of these formats. However, if your server uses a format which is different from the two provided by ``ftputil``, you can plug in a custom parser with a single method call and have ``ftputil`` use this parser. For this, you need to write a parser class by inheriting from the class ``Parser`` in the ``ftputil.stat`` module. Here's an example:: import ftputil.error import ftputil.stat class XyzParser(ftputil.stat.Parser): """ Parse the default format of the FTP server of the XYZ corporation. """ def parse_line(self, line, time_shift=0.0): """ Parse a `line` from the directory listing and return a corresponding `StatResult` object. If the line can't be parsed, raise `ftputil.error.ParserError`. The `time_shift` argument can be used to fine-tune the parsing of dates and times. See the class `ftputil.stat.UnixParser` for an example. """ # Split the `line` argument and examine it further; if # something goes wrong, raise an `ftputil.error.ParserError`. ... # Make a `StatResult` object from the parts above. stat_result = ftputil.stat.StatResult(...) # `_st_name`, `_st_target` and `_st_mtime_precision` are optional. stat_result._st_name = ... stat_result._st_target = ... stat_result._st_mtime_precision = ... return stat_result # Define `ignores_line` only if the default in the base class # doesn't do enough! def ignores_line(self, line): """ Return a true value if the line should be ignored. For example, the implementation in the base class handles lines like "total 17". On the other hand, if the line should be used for stat'ing, return a false value. """ is_total_line = super(XyzParser, self).ignores_line(line) my_test = ... return is_total_line or my_test A ``StatResult`` object is similar to the value returned by `os.stat`_ and is usually built with statements like :: stat_result = StatResult( (st_mode, st_ino, st_dev, st_nlink, st_uid, st_gid, st_size, st_atime, st_mtime, st_ctime)) stat_result._st_name = ... stat_result._st_target = ... stat_result._st_mtime_precision = ... with the arguments of the ``StatResult`` constructor described in the following table. ===== =================== ============ =================== ======================= Index Attribute os.stat type ``StatResult`` type Notes ===== =================== ============ =================== ======================= 0 st_mode int int 1 st_ino long long 2 st_dev long long 3 st_nlink int int 4 st_uid int str usually only available as string 5 st_gid int str usually only available as string 6 st_size long long 7 st_atime int/float float 8 st_mtime int/float float 9 st_ctime int/float float \- _st_name \- str file name without directory part \- _st_target \- str link target (may be absolute or relative) \- _st_mtime_precision \- int ``st_mtime`` precision in seconds ===== =================== ============ =================== ======================= If you can't extract all the desirable data from a line (for example, the MS format doesn't contain any information about the owner of a file), set the corresponding values in the ``StatResult`` instance to ``None``. Parser classes can use several helper methods which are defined in the class ``Parser``: - ``parse_unix_mode`` parses strings like "drwxr-xr-x" and returns an appropriate ``st_mode`` integer value. - ``parse_unix_time`` returns a float number usable for the ``st_...time`` values by parsing arguments like "Nov"/"23"/"02:33" or "May"/"26"/"2005". Note that the method expects the timestamp string already split at whitespace. - ``parse_ms_time`` parses arguments like "10-23-01"/"03:25PM" and returns a float number like from ``time.mktime``. Note that the method expects the timestamp string already split at whitespace. Additionally, there's an attribute ``_month_numbers`` which maps lowercase three-letter month abbreviations to integers. For more details, see the two "standard" parsers ``UnixParser`` and ``MSParser`` in the module ``ftputil/stat.py``. To actually *use* the parser, call the method `set_parser`_ of the ``FTPHost`` instance. If you can't write a parser or don't want to, please ask on the `ftputil mailing list`_. Possibly someone has already written a parser for your server or can help with it. FAQ / Tips and tricks --------------------- Where can I get the latest version? ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ See the `download page`_. Announcements will be sent to the `mailing list`_. Announcements on major updates will also be posted to the newsgroup `comp.lang.python.announce`_ . .. _`download page`: http://ftputil.sschwarzer.net/download .. _`mailing list`: http://ftputil.sschwarzer.net/mailinglist .. _`comp.lang.python.announce`: news:comp.lang.python.announce Is there a mailing list on ``ftputil``? ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Yes, please visit http://ftputil.sschwarzer.net/mailinglist to subscribe or read the archives. Though you can *technically* post without subscribing first I can't recommend it: The mails from non-subscribers have to be approved by me and because the arriving mails contain *lots* of spam, I rarely go through these mails. I found a bug! What now? ~~~~~~~~~~~~~~~~~~~~~~~~ Before reporting a bug, make sure that you already read this manual and tried the `latest version`_ of ``ftputil``. There the bug might have already been fixed. .. _`latest version`: http://ftputil.sschwarzer.net/download Please see http://ftputil.sschwarzer.net/issuetrackernotes for guidelines on entering a bug in ``ftputil``'s ticket system. If you are unsure if the behaviour you found is a bug or not, you should write to the `ftputil mailing list`_. In *either* case you *must not* include confidential information (user id, password, file names, etc.) in the problem report! Be careful! Does ``ftputil`` support SSL/TLS? ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ``ftputil`` has no *built-in* SSL/TLS support. On the other hand, there are two ways to get TLS support with ftputil: - In Python 2.7 and Python 3.2 and up, the ``ftplib`` library has a class ``FTP_TLS`` that you can use for the ``session_factory`` keyword argument in the ``FTPHost`` constructor. You can't use the class directly though if you need additional setup code in comparison to ``ftplib.FTP``, for example calling ``prot_p``, to secure the data connection. On the other hand, `ftputil.session.session_factory`_ can be used to create a custom session factory. If you have other requirements that ``session_factory`` can't fulfill, you may create your own session factory by inheriting from ``ftplib.FTP_TLS``:: import ftplib import ftputil class FTPTLSSession(ftplib.FTP_TLS): def __init__(self, host, user, password): ftplib.FTP_TLS.__init__(self) self.connect(host, port) self.login(user, password) # Set up encrypted data connection. self.prot_p() ... # Note the `session_factory` parameter. Pass the class, not # an instance. with ftputil.FTPHost(server, user, password, session_factory=FTPTLSSession) as ftp_host: # Use `ftp_host` as usual. ... .. _`ftputil.session.session_factory`: `Session factories`_ - If you need to work with Python 2.6, you can use the ``ftpslib.FTP_TLS`` class from the M2Crypto_ project. Again, you can't use the class directly but need to use ``ftputil.session.session_factory`` or a recipe similar to that above. Unfortunately, ``M2Crypto.ftpslib.FTP_TLS`` (at least in version 0.22.3) doesn't work correctly if you pass unicode strings to its methods. Since ``ftputil`` does exactly that at some point (even if you used byte strings in ``ftputil`` calls) you need a workaround in the session factory class:: import M2Crypto import ftputil import ftputil.tool class M2CryptoSession(M2Crypto.ftpslib.FTP_TLS): def __init__(self, host, user, password): M2Crypto.ftpslib.FTP_TLS.__init__(self) # Change the port number if needed. self.connect(host, 21) self.auth_tls() self.login(user, password) self.prot_p() self._fix_socket() ... def _fix_socket(self): """ Change the socket object so that arguments to `sendall` are converted to byte strings before being used. """ original_sendall = self.sock.sendall # Bound method, therefore no `self` argument. def sendall(data): data = ftputil.tool.as_bytes(data) return original_sendall(data) self.sock.sendall = sendall # Note the `session_factory` parameter. Pass the class, not # an instance. with ftputil.FTPHost(server, user, password, session_factory=M2CryptoSession) as ftp_host: # Use `ftp_host` as usual. ... That said, ``session_factory`` has this workaround built in, so normally you don't need to define the session factory yourself! .. _M2Crypto: https://github.com/martinpaljak/M2Crypto How do I connect to a non-default port? ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ By default, an instantiated ``FTPHost`` object connects on the usual FTP port. If you have to use a different port, refer to the section `Session factories`_. How do I set active or passive mode? ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Please see the section `Session factories`_. How can I debug an FTP connection problem? ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ You can do this with a session factory. See `Session factories`_. If you want to change the debug level only temporarily after the connection is established, you can reach the `session object`_ as the ``_session`` attribute of the ``FTPHost`` instance and call ``_session.set_debuglevel``. Note that the ``_session`` attribute should *only* be accessed for debugging. Calling arbitrary ``ftplib.FTP`` methods on the session object may *cause* bugs! .. _`session object`: `Session factories`_ Conditional upload/download to/from a server in a different time zone ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ You may find that ``ftputil`` uploads or downloads files unnecessarily, or not when it should. This can happen when the FTP server is in a different time zone than the client on which ``ftputil`` runs. Please see the section on `time zone correction`_. It may even be sufficient to call `synchronize_times`_. When I use ``ftputil``, all I get is a ``ParserError`` exception ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The FTP server you connect to may use a directory format that ``ftputil`` doesn't understand. You can either write and `plug in an own parser`_ or ask on the `mailing list`_ for help. .. _`plug in an own parser`: `Writing directory parsers`_ ``isdir``, ``isfile`` or ``islink`` incorrectly return ``False`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Like Python's counterparts under `os.path`_, ``ftputil``'s methods return ``False`` if they can't find the given path. Probably you used ``listdir`` on a directory and called ``is...()`` on the returned names. But if the argument for ``listdir`` wasn't the current directory, the paths won't be found and so all ``is...()`` variants will return ``False``. I don't find an answer to my problem in this document ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Please send an email with your problem report or question to the `ftputil mailing list`_, and we'll see what we can do for you. :-) Bugs and limitations -------------------- - ``ftputil`` needs at least Python 2.6 to work. - Whether ``ftputil`` "sees" "hidden" directory and file names (i. e. names starting with a dot) depends on the configuration of the FTP server. See `Hidden files and directories`_ for details. - Due to the implementation of ``lstat`` it can not return a sensible value for the root directory ``/`` though stat'ing entries *in* the root directory isn't a problem. If you know an implementation that can do this, please let me know. The root directory is handled appropriately in ``FTPHost.path.exists/isfile/isdir/islink``, though. - In multithreaded programs, you can have each thread use one or more ``FTPHost`` instances as long as no instance is shared with other threads. - Currently, it is not possible to continue an interrupted upload or download. Contact me if this causes problems for you. - There's exactly one cache for ``lstat`` results for each ``FTPHost`` object, i. e. there's no sharing of cache results determined by several ``FTPHost`` objects. See `Local caching of file system information`_ for the reasons. Files ----- If not overwritten via installation options, the ``ftputil`` files reside in the ``ftputil`` package. There's also documentation in `reStructuredText`_ and in HTML format. The locations of these files after installation is system-dependent. .. _`reStructuredText`: http://docutils.sourceforge.net/rst.html The files ``test_*.py`` and ``mock_ftplib.py`` are for unit-testing. If you only *use* ``ftputil``, i. e. *don't* modify it, you can delete these files. References ---------- - Mackinnon T, Freeman S, Craig P. 2000. `Endo-Testing: Unit Testing with Mock Objects`_. - Postel J, Reynolds J. 1985. `RFC 959 - File Transfer Protocol (FTP)`_. - Van Rossum G et al. 2013. `Python Library Reference`_. .. _`Endo-Testing: Unit Testing with Mock Objects`: http://www.connextra.com/aboutUs/mockobjects.pdf .. _`RFC 959 - File Transfer Protocol (FTP)`: http://www.ietf.org/rfc/rfc959.txt .. _`Python Library Reference`: https://docs.python.org/library/index.html Authors ------- ``ftputil`` is written by Stefan Schwarzer and contributors (see ``doc/contributors.txt``). The original ``lrucache`` module was written by Evan Prodromou . Feedback is appreciated. :-) ftputil-3.4/doc/announcements.txt0000644000175000017470000012240313200423657016472 0ustar debiandebianftputil 3.4 is now available from http://ftputil.sschwarzer.net/download . Changes since version 3.3.1 --------------------------- - Several bugs were fixed [1-5]. - Added deprecation warnings for backward incompatibilities in the upcoming ftputil 4.0.0. Important note -------------- The next version of ftputil will be 4.0.0 (apart from small fixes in possible 3.4.x versions). ftputil 4.0.0 will make some backward-incompatible changes: - Support for Python 2 will be removed. There are several reasons for this, which are explained in [6]. - The flag `use_list_a_option` will be set to `False` by default. This option was intended to make life easier for users, but turned out to be problematic (see [7]). What is ftputil? ---------------- ftputil is a high-level FTP client library for the Python programming language. ftputil implements a virtual file system for accessing FTP servers, that is, it can generate file-like objects for remote files. The library supports many functions similar to those in the os, os.path and shutil modules. ftputil has convenience functions for conditional uploads and downloads, and handles FTP clients and servers in different timezones. See the documentation for details: http://ftputil.sschwarzer.net/trac/wiki/Documentation License ------- ftputil is open source software, released under the revised BSD license (see http://opensource.org/licenses/BSD-3-Clause ). [1] http://ftputil.sschwarzer.net/trac/ticket/107 [2] http://ftputil.sschwarzer.net/trac/ticket/109 [3] http://ftputil.sschwarzer.net/trac/ticket/112 [4] http://ftputil.sschwarzer.net/trac/ticket/113 [5] http://ftputil.sschwarzer.net/trac/ticket/114 [6] http://lists.sschwarzer.net/pipermail/ftputil/2017q3/000465.html [7] http://ftputil.sschwarzer.net/trac/ticket/110 Stefan ---------------------------------------------------------------------- ftputil 3.3.1 is now available from http://ftputil.sschwarzer.net/download . Changes since version 3.3 ------------------------- - Fixed a bug where a 226 reply after a remote file close would only show up later when doing a `pwd` call on the session. [1] This resulted in an `ftplib.error_reply` exception when opening a remote file. Note that ftputil 3.0 broke backward compatibility with ftputil 2.8 and before. The differences are described here: http://ftputil.sschwarzer.net/trac/wiki/WhatsNewInFtputil3.0 What is ftputil? ---------------- ftputil is a high-level FTP client library for the Python programming language. ftputil implements a virtual file system for accessing FTP servers, that is, it can generate file-like objects for remote files. The library supports many functions similar to those in the os, os.path and shutil modules. ftputil has convenience functions for conditional uploads and downloads, and handles FTP clients and servers in different timezones. See the documentation for details: http://ftputil.sschwarzer.net/trac/wiki/Documentation License ------- ftputil is open source software, released under the revised BSD license (see http://opensource.org/licenses/BSD-3-Clause ). [1] http://ftputil.sschwarzer.net/trac/ticket/102 Stefan ---------------------------------------------------------------------- ftputil 3.3 is now available from http://ftputil.sschwarzer.net/download . Changes since version 3.2 ------------------------- - Added `rest` argument to `FTPHost.open` for recovery after interrupted transfers [1]. - Fixed handling of non-ASCII directory and file names under Python 2 [2]. Under Python 3, the directory and file names could already contain any characters from the ISO 5589-1 (latin-1) character set. Under Python 2, non-ASCII characters (even out of the latin-1 character set) resulted in a `UnicodeEncodeError`. Now Python 2 behaves like Python 3, supporting all latin-1 characters. Note that for interoperability between servers and clients it's still usually safest to use only ASCII characters for directory and file names. - Changed `FTPHost.makedirs` for better handling of "virtual directories" [3, 4]. Thanks to Roger Demetrescu for the implementation. - Small improvements [5, 6, 7] Upgrading is recommended. Note that ftputil 3.0 broke backward compatibility with ftputil 2.8 and before. The differences are described here: http://ftputil.sschwarzer.net/trac/wiki/WhatsNewInFtputil3.0 What is ftputil? ---------------- ftputil is a high-level FTP client library for the Python programming language. ftputil implements a virtual file system for accessing FTP servers, that is, it can generate file-like objects for remote files. The library supports many functions similar to those in the os, os.path and shutil modules. ftputil has convenience functions for conditional uploads and downloads, and handles FTP clients and servers in different timezones. See the documentation for details: http://ftputil.sschwarzer.net/trac/wiki/Documentation License ------- ftputil is open source software, released under the revised BSD license (see http://opensource.org/licenses/BSD-3-Clause ). [1] http://ftputil.sschwarzer.net/trac/ticket/61 [2] http://ftputil.sschwarzer.net/trac/ticket/100 [3] http://ftputil.sschwarzer.net/trac/ticket/86 [4] https://support.microsoft.com/en-us/kb/142853 [5] http://ftputil.sschwarzer.net/trac/ticket/89 [6] http://ftputil.sschwarzer.net/trac/ticket/91 [7] http://ftputil.sschwarzer.net/trac/ticket/92 Have fun! :-) Stefan ---------------------------------------------------------------------- ftputil 3.2 is now available from http://ftputil.sschwarzer.net/download . Changes since version 3.1 ------------------------- - For some platforms (notably Windows) modification datetimes before the epoch would cause an `OverflowError` [1]. Other platforms could return negative values. Since the Python documentation for the `time` module [2] points out that values before the epoch might cause problems, ftputil now sets the float value for such datetimes to 0.0. In theory, this might cause backward compatibility problems, but it's very unlikely since pre-epoch timestamps in directory listings should be very rare. - On some platforms, the `time.mktime` implementation could behave strange and accept invalid date/time values. For example, a day value of 32 would be accepted and implicitly cause a "wrap" to the next month. Such invalid values now result in a `ParserError`. - Make error handling more robust where the underlying FTP session factory (for example, `ftplib.FTP`) uses byte strings for exception messages. [3] - Improved error handling for directory listings. As just one example, previously a non-integer value for a day would unintentionally cause a `ValueError`. Now this causes a `ParserError`. - Extracted socket file adapter module [4] so that it can be used by other projects. Note that ftputil 3.0 broke backward compatibility with ftputil 2.8 and before. The differences are described here: http://ftputil.sschwarzer.net/trac/wiki/WhatsNewInFtputil3.0 What is ftputil? ---------------- ftputil is a high-level FTP client library for the Python programming language. ftputil implements a virtual file system for accessing FTP servers, that is, it can generate file-like objects for remote files. The library supports many functions similar to those in the os, os.path and shutil modules. ftputil has convenience functions for conditional uploads and downloads, and handles FTP clients and servers in different timezones. See the documentation for details: http://ftputil.sschwarzer.net/trac/wiki/Documentation License ------- ftputil is open source software, released under the revised BSD license (see http://opensource.org/licenses/BSD-3-Clause ). [1] http://ftputil.sschwarzer.net/trac/ticket/83 [2] https://docs.python.org/3/library/time.html [3] http://ftputil.sschwarzer.net/trac/ticket/85 [4] http://ftputil.sschwarzer.net/trac/wiki/SocketFileAdapter Have fun! :-) Stefan ---------------------------------------------------------------------- ftputil 3.1 is now available from http://ftputil.sschwarzer.net/download . Changes since version 3.0 ------------------------- - Added support for `followlinks` parameter in `FTPHost.walk`. [1] - Trying to pickle `FTPHost` and `FTPFile` objects now raises explicit `TypeError`s to make clear that not being able to pickle these objects is intentional. [2] - Improved exception messages for socket errors [3]. - Fixed handling of server error messages with non-ASCII characters when running under Python 2.x. [4] - Added a generic "session factory factory" to make creation of session factories easier for common use cases (encrypted connections, non-default port, active/passive mode, FTP session debug level and combination of these). [5] This includes a workaround for `M2Crypto.ftpslib.FTP_TLS`; this class won't be usable with ftputil 3.0 and up with just the session factory recipe described in the documentation. [6] - Don't assume time zone differences to always be full hours, but rather 15-minute units. [7] For example, according to [8], Nepal's time zone is UTC+05:45. - Improved documentation on timeout handling. This includes information on internal creation of additional FTP connections (for remote files, including uploads and downloads). This may help understand better why the `keep_alive` method is limited. Note that ftputil 3.0 broke backward compatibility with ftputil 2.8 and before. The differences are described here: http://ftputil.sschwarzer.net/trac/wiki/WhatsNewInFtputil3.0 What is ftputil? ---------------- ftputil is a high-level FTP client library for the Python programming language. ftputil implements a virtual file system for accessing FTP servers, that is, it can generate file-like objects for remote files. The library supports many functions similar to those in the os, os.path and shutil modules. ftputil has convenience functions for conditional uploads and downloads, and handles FTP clients and servers in different timezones. See the documentation for details: http://ftputil.sschwarzer.net/trac/wiki/Documentation License ------- ftputil is open source software, released under the revised BSD license (see http://opensource.org/licenses/BSD-3-Clause ). [1] http://ftputil.sschwarzer.net/trac/ticket/73 [2] http://ftputil.sschwarzer.net/trac/ticket/75 [3] http://ftputil.sschwarzer.net/trac/ticket/76 [4] http://ftputil.sschwarzer.net/trac/ticket/77 [5] http://ftputil.sschwarzer.net/trac/ticket/78 [6] http://ftputil.sschwarzer.net/trac/wiki/Documentation#session-factories [7] http://ftputil.sschwarzer.net/trac/ticket/81 [8] http://en.wikipedia.org/wiki/Timezone#List_of_UTC_offsets Have fun! :-) Stefan ---------------------------------------------------------------------- ftputil 3.0 is now available from http://ftputil.sschwarzer.net/download . Changes since version 2.8 ------------------------- Note: This version of ftputil is _not_ backward-compatible with earlier versions.See the links below for information on adapting existing client code. - This version adds Python 3 compatibility! :-) The same source is used for Python 2.x and Python 3.x. I had to change the API to find a good compromise for both Python versions. - ftputil now requires at least Python 2.6. - Remote file-like objects use the same semantics as Python's `io` module. (This is the same as for the built-in `open` function in Python 3.) - `ftputil.ftp_error` was renamed to `ftputil.error`. - For custom parsers, import `ftputil.parser` instead of `ftputil.stat`. For more information please read http://ftputil.sschwarzer.net/trac/wiki/Documentation http://ftputil.sschwarzer.net/trac/wiki/WhatsNewInFtputil3.0 What is ftputil? ---------------- ftputil is a high-level FTP client library for the Python programming language. ftputil implements a virtual file system for accessing FTP servers, that is, it can generate file-like objects for remote files. The library supports many functions similar to those in the os, os.path and shutil modules. ftputil has convenience functions for conditional uploads and downloads, and handles FTP clients and servers in different timezones. License ------- ftputil is Open Source software, released under the revised BSD license (see http://opensource.org/licenses/BSD-3-Clause ). Stefan ---------------------------------------------------------------------- ftputil 2.8 is now available from http://ftputil.sschwarzer.net/download . Changes since version 2.7.1 --------------------------- - After some discussion [1] I decided to remove the auto-probing before using the `-a` option for `LIST` [2] to find "hidden" files and directories. The option is used by default now, without probing for exceptions. If this new approach causes problems, you can use ftp_host = ftputil.FTPHost(...) ftp_host.use_list_a_option = False - Several bugs were fixed. [3] - The mailing lists have moved to ftputil@lists.sschwarzer.net ftputil-tickets@lists.sschwarzer.net The ftputil list [4] requires a subscription before you can post. The ftputil-tickets list [5] is read-only anyway. Thanks to Codespeak.net for having hosted the lists for almost ten years. :-) What is ftputil? ---------------- ftputil is a high-level FTP client library for the Python programming language. ftputil implements a virtual file system for accessing FTP servers, that is, it can generate file-like objects for remote files. The library supports many functions similar to those in the os, os.path and shutil modules. ftputil has convenience functions for conditional uploads and downloads, and handles FTP clients and servers in different timezones. Read the documentation at http://ftputil.sschwarzer.net/documentation . License ------- ftputil is Open Source software, released under the revised BSD license (see http://opensource.org/licenses/BSD-3-Clause ). [1] http://ftputil.sschwarzer.net/trac/ticket/65 [2] http://lists.sschwarzer.net/pipermail/ftputil/2012q3/000350.html [3] http://ftputil.sschwarzer.net/trac/ticket/39 http://ftputil.sschwarzer.net/trac/ticket/65 http://ftputil.sschwarzer.net/trac/ticket/66 http://ftputil.sschwarzer.net/trac/ticket/67 http://ftputil.sschwarzer.net/trac/ticket/69 [4] http://lists.sschwarzer.net/listinfo/ftputil http://lists.sschwarzer.net/listinfo/ftputil-tickets Stefan ---------------------------------------------------------------------- ftputil 2.7 is now available from http://ftputil.sschwarzer.net/download . Changes since version 2.6 ------------------------- - ftputil now explicitly tries to get hidden directory and file names (names starting with a dot) from the FTP server. [1] Before, ftputil used a `LIST` command to get directory listings, now it uses `LIST -a` if the server doesn't explicitly reject its usage upon login. Note that the server is free to ignore the `-a` option, so "hidden" directories and files may still not be visible. Please see [2] for details. If you have code that _relies_ on "hidden" directory or file names _not_ being visible, please update the code as necessary. If that's presumably not possible, please send feedback to the mailing list [3] or in private mail [4]. - A bug in the experimental synchronization code was fixed [5]. Thanks to Zhuo Qiang for his help. :-) What is ftputil? ---------------- ftputil is a high-level FTP client library for the Python programming language. ftputil implements a virtual file system for accessing FTP servers, that is, it can generate file-like objects for remote files. The library supports many functions similar to those in the os, os.path and shutil modules. ftputil has convenience functions for conditional uploads and downloads, and handles FTP clients and servers in different timezones. Read the documentation at http://ftputil.sschwarzer.net/documentation . License ------- ftputil is Open Source software, released under the revised BSD license (see http://www.opensource.org/licenses/bsd-license.php ). [1] http://ftputil.sschwarzer.net/trac/ticket/23 [2] http://ftputil.sschwarzer.net/trac/ticket/23#comment:4 [3] http://codespeak.net/mailman/listinfo/ftputil (note that you need to subscribe to the list to be able to post there) [4] sschwarzer@sschwarzer.net [5] http://ftputil.sschwarzer.net/trac/ticket/62 Stefan ---------------------------------------------------------------------- ftputil 2.6 is now available from http://ftputil.sschwarzer.net/download . Changes since version 2.5 ------------------------- - The stat caching has been improved. There's now an "auto-grow" feature for `FTPHost.listdir` which in turn applies to `FTPHost.walk`. Moreover, there were several performance optimizations. - A few bugs were fixed [1-3]. What is ftputil? ---------------- ftputil is a high-level FTP client library for the Python programming language. ftputil implements a virtual file system for accessing FTP servers, that is, it can generate file-like objects for remote files. The library supports many functions similar to those in the os, os.path and shutil modules. ftputil has convenience functions for conditional uploads and downloads, and handles FTP clients and servers in different timezones. Read the documentation at http://ftputil.sschwarzer.net/documentation . License ------- ftputil is Open Source software, released under the revised BSD license (see http://www.opensource.org/licenses/bsd-license.php ). [1] http://ftputil.sschwarzer.net/trac/ticket/53 [2] http://ftputil.sschwarzer.net/trac/ticket/55 [3] http://ftputil.sschwarzer.net/trac/ticket/56 Stefan ---------------------------------------------------------------------- ftputil 2.5 is now available from http://ftputil.sschwarzer.net/download . Changes since version 2.4.2 --------------------------- - As announced over a year ago [1], the `xreadlines` method for FTP file objects has been removed, and exceptions can no longer be accessed via the `ftputil` namespace. Only use `ftp_error` to access the exceptions. The distribution contains a small tool `find_deprecated_code.py` to scan a directory tree for the deprecated uses. Invoke the program with the `--help` option to see a description. - Upload and download methods now accept a `callback` argument to do things during a transfer. Modification time comparisons in `upload_if_newer` and `download_if_newer` now consider the timestamp precision of the remote file which may lead to some unnecessary transfers. These can be avoided by waiting at least a minute between calls of `upload_if_newer` (or `download_if_newer`) for the same file. See the documentation for details [2]. - The `FTPHost` class got a `keep_alive` method. It should be used carefully though, not routinely. Please read the description [3] in the documentation. - Several bugs were fixed [4-7]. - The source code was restructured. The tests are now in a `test` subdirectory and are no longer part of the release archive. You can still get them via the source repository. Licensing matters have been moved to a common `LICENSE` file. What is ftputil? ---------------- ftputil is a high-level FTP client library for the Python programming language. ftputil implements a virtual file system for accessing FTP servers, that is, it can generate file-like objects for remote files. The library supports many functions similar to those in the os, os.path and shutil modules. ftputil has convenience functions for conditional uploads and downloads, and handles FTP clients and servers in different timezones. Read the documentation at http://ftputil.sschwarzer.net/documentation . License ------- ftputil is Open Source software, released under the revised BSD license (see http://www.opensource.org/licenses/bsd-license.php ). [1] http://codespeak.net/pipermail/ftputil/2009q1/000256.html [2] http://ftputil.sschwarzer.net/trac/wiki/Documentation#uploading-and-downloading-files [3] http://ftputil.sschwarzer.net/trac/wiki/Documentation#keep-alive [4] http://ftputil.sschwarzer.net/trac/ticket/44 [5] http://ftputil.sschwarzer.net/trac/ticket/46 [6] http://ftputil.sschwarzer.net/trac/ticket/47 [7] http://ftputil.sschwarzer.net/trac/ticket/51 Stefan ---------------------------------------------------------------------- ftputil 2.4.2 is now available from http://ftputil.sschwarzer.net/download . Changes since version 2.4.1 --------------------------- - Some FTP servers seem to have problems using *any* directory argument which contains slashes. The new default for FTP commands now is to change into the directory before actually invoking the command on a relative path (report and fix suggestion by Nicola Murino). - Calling the method ``FTPHost.stat_cache.resize`` with an argument 0 caused an exception. This has been fixed; a zero cache size now of course doesn't cache anything but doesn't lead to a traceback either. - The installation script ``setup.py`` didn't work with the ``--home`` option because it still tried to install the documentation in a system directory (report by Albrecht Mühlenschulte). As a side effect, when using the *global* installation, the documentation is no longer installed in the ftputil package directory but in a subdirectory ``doc`` of a directory determined by Distutils. For example, on my system (Ubuntu 9.04) the documentation files are put into ``/usr/local/doc``. Upgrading is recommended. What is ftputil? ---------------- ftputil is a high-level FTP client library for the Python programming language. ftputil implements a virtual file system for accessing FTP servers, that is, it can generate file-like objects for remote files. The library supports many functions similar to those in the os, os.path and shutil modules. ftputil has convenience functions for conditional uploads and downloads, and handles FTP clients and servers in different timezones. Read the documentation at http://ftputil.sschwarzer.net/documentation . License ------- ftputil is Open Source software, released under the revised BSD license (see http://www.opensource.org/licenses/bsd-license.php ). Stefan ---------------------------------------------------------------------- ftputil 2.4.1 is now available from http://ftputil.sschwarzer.net/download . Changes since version 2.4 ------------------------- Several bugs were fixed: - On Windows, some accesses to the stat cache caused it to become inconsistent, which could also trigger exceptions (report and patch by Peter Stirling). - In ftputil 2.4, the use of ``super`` in the exception base class caused ftputil to fail on Python <2.5 (reported by Nicola Murino). ftputil is supposed to run with Python 2.3+. - The conversion of 12-hour clock times to 24-hour clock in the MS format parser was wrong for 12 AM and 12 PM. Upgrading is strongly recommended. What is ftputil? ---------------- ftputil is a high-level FTP client library for the Python programming language. ftputil implements a virtual file system for accessing FTP servers, that is, it can generate file-like objects for remote files. The library supports many functions similar to those in the os, os.path and shutil modules. ftputil has convenience functions for conditional uploads and downloads, and handles FTP clients and servers in different timezones. Read the documentation at http://ftputil.sschwarzer.net/documentation . License ------- ftputil is Open Source software, released under the revised BSD license (see http://www.opensource.org/licenses/bsd-license.php ). Stefan ---------------------------------------------------------------------- ftputil 2.4 is now available from http://ftputil.sschwarzer.net/download . Changes since version 2.3 ------------------------- The ``FTPHost`` class got a new method ``chmod``, similar to ``os.chmod``, to act on remote files. Thanks go to Tom Parker for the review. There's a new exception ``CommandNotImplementedError``, derived from ``PermanentError``, to denote commands not implemented by the FTP server or disabled by its administrator. Using the ``xreadlines`` method of FTP file objects causes a warning through Python's warnings framework. Upgrading is recommended. Incompatibility notice ---------------------- The ``xreadlines`` method will be removed in ftputil *2.5* as well as the direct access of exception classes via the ftputil module (e. g. ``ftputil.PermanentError``). However, the deprecated access causes no warning because that would be rather difficult to implement. The distribution contains a small tool find_deprecated_code.py to scan a directory tree for the deprecated uses. Invoke the program with the ``--help`` option to see a description. What is ftputil? ---------------- ftputil is a high-level FTP client library for the Python programming language. ftputil implements a virtual file system for accessing FTP servers, that is, it can generate file-like objects for remote files. The library supports many functions similar to those in the os, os.path and shutil modules. ftputil has convenience functions for conditional uploads and downloads, and handles FTP clients and servers in different timezones. Read the documentation at http://ftputil.sschwarzer.net/documentation . License ------- ftputil is Open Source software, released under the revised BSD license (see http://www.opensource.org/licenses/bsd-license.php ). Stefan ---------------------------------------------------------------------- ftputil 2.3 is now available from http://ftputil.sschwarzer.net/download . Changes since version 2.2.4 --------------------------- ftputil has got support for the ``with`` statement which was introduced by Python 2.5. You can now construct host and remote file objects in ``with`` statements and have them closed automatically (contributed by Roger Demetrescu). See the documentation for examples. What is ftputil? ---------------- ftputil is a high-level FTP client library for the Python programming language. ftputil implements a virtual file system for accessing FTP servers, that is, it can generate file-like objects for remote files. The library supports many functions similar to those in the os, os.path and shutil modules. ftputil has convenience functions for conditional uploads and downloads, and handles FTP clients and servers in different timezones. Read the documentation at http://ftputil.sschwarzer.net/documentation . License ------- ftputil is Open Source software, released under the revised BSD license (see http://www.opensource.org/licenses/bsd-license.php ). Stefan ---------------------------------------------------------------------- ftputil 2.2.4 is now available from http://ftputil.sschwarzer.net/download . Changes since version 2.2.3 --------------------------- This release fixes a bug in the ``makedirs`` call (report and fix by Richard Holden). Upgrading is recommended. What is ftputil? ---------------- ftputil is a high-level FTP client library for the Python programming language. ftputil implements a virtual file system for accessing FTP servers, that is, it can generate file-like objects for remote files. The library supports many functions similar to those in the os, os.path and shutil modules. ftputil has convenience functions for conditional uploads and downloads, and handles FTP clients and servers in different timezones. Read the documentation at http://ftputil.sschwarzer.net/documentation . License ------- ftputil is Open Source software, released under the revised BSD license (see http://www.opensource.org/licenses/bsd-license.php ). Stefan ---------------------------------------------------------------------- ftputil 2.2.3 is now available from http://ftputil.sschwarzer.net/download . Changes since version 2.2.2 --------------------------- This release fixes a bug in the ``makedirs`` call (report and fix by Julian, whose last name I don't know ;-) ). Upgrading is recommended. What is ftputil? ---------------- ftputil is a high-level FTP client library for the Python programming language. ftputil implements a virtual file system for accessing FTP servers, that is, it can generate file-like objects for remote files. The library supports many functions similar to those in the os, os.path and shutil modules. ftputil has convenience functions for conditional uploads and downloads, and handles FTP clients and servers in different timezones. Read the documentation at http://ftputil.sschwarzer.net/documentation . License ------- ftputil is Open Source software, released under the revised BSD license (see http://www.opensource.org/licenses/bsd-license.php ). Stefan ---------------------------------------------------------------------- ftputil 2.2.2 is now available from http://ftputil.sschwarzer.net/download . Changes since version 2.2.1 --------------------------- This bugfix release handles whitespace in path names more reliably (thanks to Johannes Strömberg). Upgrading is recommended. What is ftputil? ---------------- ftputil is a high-level FTP client library for the Python programming language. ftputil implements a virtual file system for accessing FTP servers, that is, it can generate file-like objects for remote files. The library supports many functions similar to those in the os, os.path and shutil modules. ftputil has convenience functions for conditional uploads and downloads, and handles FTP clients and servers in different timezones. Read the documentation at http://ftputil.sschwarzer.net/documentation . License ------- ftputil is Open Source software, released under the revised BSD license (see http://www.opensource.org/licenses/bsd-license.php ). Stefan ---------------------------------------------------------------------- ftputil 2.2.1 is now available from http://ftputil.sschwarzer.net/download . Changes since version 2.2 ------------------------- This bugfix release checks (and ignores) status code 451 when FTPFiles are closed (thanks go to Alexander Holyapin). Upgrading is recommended. What is ftputil? ---------------- ftputil is a high-level FTP client library for the Python programming language. ftputil implements a virtual file system for accessing FTP servers, that is, it can generate file-like objects for remote files. The library supports many functions similar to those in the os, os.path and shutil modules. ftputil has convenience functions for conditional uploads and downloads, and handles FTP clients and servers in different timezones. Read the documentation at http://ftputil.sschwarzer.net/trac/wiki/Documentation . License ------- ftputil is Open Source software, released under the revised BSD license (see http://www.opensource.org/licenses/bsd-license.php ). Stefan ---------------------------------------------------------------------- ftputil 2.2 is now available from http://ftputil.sschwarzer.net/download . Changes since version 2.1 ------------------------- - Results of stat calls (also indirect calls, i. e. listdir, isdir/isfile/islink, exists, getmtime etc.) are now cached and reused. This results in remarkable speedups for many use cases. Thanks to Evan Prodromou for his permission to add his lrucache module under ftputil's license. - The current directory is also locally cached, resulting in further speedups. - It's now possible to write and plug in custom parsers for directory formats which ftputil doesn't support natively. - File-like objects generated via ``FTPHost.file`` now support the iterator protocol (for line in some_file: ...). - The documentation has been updated accordingly. Read it under http://ftputil.sschwarzer.net/trac/wiki/Documentation . Possible incompatibilities: - This release requires at least Python 2.3. (Previous releases worked with Python versions from 2.1 up.) - The method ``FTPHost.set_directory_format`` has been removed, since the directory format (Unix or MS) is set automatically. (The new method ``set_parser`` is a different animal since it takes a parser object to parse "foreign" formats, not a string.) What is ftputil? ---------------- ftputil is a high-level FTP client library for the Python programming language. ftputil implements a virtual file system for accessing FTP servers, that is, it can generate file-like objects for remote files. The library supports many functions similar to those in the os, os.path and shutil modules. ftputil has convenience functions for conditional uploads and downloads, and handles FTP clients and servers in different timezones. License ------- ftputil 2.2 is Open Source software, released under the revised BSD license (see http://www.opensource.org/licenses/bsd-license.php ). Stefan ---------------------------------------------------------------------- The second beta version of ftputil 2.2 is available. You can download it from http://ftputil.sschwarzer.net/download . With respect to the first beta release, it's now possible to write and plug in custom parsers for FTP directory formats that ftputil doesn't know natively. The documentation has been updated accordingly. The documentation for this release is online at http://ftputil.sschwarzer.net/trac/wiki/Documentation#Documentationforftputil2.2b2 , so you can read about the extensions in more detail. Please download and test the release. Do you miss something which should be in this release? Are there any bugs? Stefan ---------------------------------------------------------------------- The first beta version of ftputil 2.2 is available. You can download it from http://ftputil.sschwarzer.net/download . With respect to the previous alpha release, the cache now uses the lrucache module by Evan Prodromou which is bundled with the ftputil distribution. (Evan also gave his permission to include the module under ftputil's modified BSD license instead of the Academic License.) The documentation for the cache and its control have been added to ftputil.txt / ftputil.html . File objects generated with FTPHost.file now support the iterator protocol (for line in some_file: ...). Please download and test the release. Do you miss something which should be in this release? Are there any bugs? Stefan ---------------------------------------------------------------------- Welcome to the first alpha release of ftputil 2.2, ftputil 2.2a1. Please download it from http://ftputil.sschwarzer.net/download . This version adds caching of stat results to ftputil. This also affects indirect calls via FTPHost.path, e. g. methods isfile, exists, getmtime, getsize. The test script at http://ftputil.sschwarzer.net/trac/browser/tags/release2_2a1/sandbox/list_dir_test.py runs about 20 times as fast as before adding caching! :-) As the "alpha" part implies, this release is not production-ready, it's even kind of experimental: The caching works but there's no cache entry expiration yet. (I plan to implement an LRU expiration strategy or something similar.) Apart from that, the release is tested as any production release. I suggest using the --prefix option for installing alpha releases. That said, it would be helpful if you tested this release and report your findings. When testing the code, please make sure that your code uses the ftputil version you intend (alpha vs. production version), e. g. by setting the PYTHONPATH environment variable. I'm very interested in the speedups - and any problems you encounter. Stefan ---------------------------------------------------------------------- ftputil 2.1.1 is now available from http://ftputil.sschwarzer.net/download . This release fixes a bug which happened when a client opened a large file on the server as a file-like object and read only a part of it. For details, see http://ftputil.sschwarzer.net/trac/ticket/17 . Stefan ---------------------------------------------------------------------- Changes since version 2.0 ------------------------- - Added new methods to the FTPHost class, namely makedirs, walk, rmtree. - The FTP server directory format ("Unix" vs. "Windows") is now set automatically (thanks to Andrew Ittner for testing it). - Border cases like inaccessible login directories and whitespace in directory names, are now handled more gracefully (based on input from Valeriy Pogrebitskiy, Tommy Sundström and H. Y. Chu). - The documentation was updated. - A Russian translation of the documentation (currently slightly behind) was contributed by Anton Stepanov. It's also on the website at http://ftputil.sschwarzer.net/trac/wiki/RussianDocumentation . - New website, http://ftputil.sschwarzer.net/ with wiki, issue tracker and Subversion repository (thanks to Trac!) Please enter not only bugs but also enhancement request into the issue tracker! Possible incompatibilities: - The exception hierarchy was changed slightly, which might break client code. See http://ftputil.sschwarzer.net/trac/changeset/489 for the change details and the possibly necessary code changes. - FTPHost.rmdir no longer removes non-empty directories. Use the new method FTPHost.rmtree for this. What is ftputil? ---------------- ftputil is a high-level FTP client library for the Python programming language. ftputil implements a virtual file system for accessing FTP servers, that is, it can generate file-like objects for remote files. The library supports many functions similar to those in the os, os.path and shutil modules. ftputil has convenience functions for conditional uploads and downloads, and handles FTP clients and servers in different timezones. License ------- ftputil 2.1 is Open Source software, released under the revised BSD license (see http://www.opensource.org/licenses/bsd-license.php ). Stefan ---------------------------------------------------------------------- ftputil 2.0.3 is now available at http://www.sschwarzer.net/python/python_software.html#ftputil . This release fixes (for most cases) some problems when logging into an FTP server with an inaccessible login directory, i. e. `getcwd()` returns "/some/login/dir" but `chdir("/some/login/dir")` fails. Thanks go to Valeriy Pogrebitskiy for investigating and reporting these problems. Stefan ---------------------------------------------------------------------- Here's ftputil 2.0 ! ftputil is a high-level alternative to Python's ftplib module. With ftputil, you can access directories and files on remote FTP servers almost as if they were in your local file system. This includes using file-like objects representing remote files. For future releases see http://www.sschwarzer.net/python/python_software.html or subscribe to the mailing list at http://codespeak.net/mailman/listinfo/ftputil What's new? ----------- From version 1.1 to 2.0, the following has changed: - ftputil has been re-organized and is now a Python package (the import statement is still the same) - installation via Python distutils - stat, upload_if_newer, download_if_newer etc. work correctly if the server is in another time zone than the client running ftputil (with help from Andrew Ittner); see section "Time zone correction" in the documentation - it's possible to set the directory listing format "manually" (though in most cases it's recognized automatically); see section "Stat'ing files and directories" - added a workaround regarding whitespace in directory names (thanks to Tommy Sundström and H. Y. Chu) - extended documentation and converted it to HTML format (now generated from reStructured Text) - several bugfixes - there's now a mailing list at http://codespeak.net/mailman/listinfo/ftputil (thanks to Holger Krekel) Documentation ------------- The documentation for ftputil can be found in the file ftputil.txt (reStructured Text format) or ftputil.html (recommended, generated from ftputil.txt). License ------- ftputil is Open Source Software. It is distributed under a BSD-style license (see the top of ftputil.py). Stefan ---------------------------------------------------------------------- ftputil 1.1 is released. You can find it at http://www.ndh.net/home/sschwarzer/python/python_software.html . ftputil provides a higher-level interface for FTP sessions than the ftplib module. FTP servers can be accessed via an interface similar to os and os.path. Remote files are accessible as file-like objects. New since version 1.0: - ftputil now runs under Python 2.1+ (not only 2.2+). - documentation - conditional upload/download (depending on local and remote file timestamps) - FTPHost.stat follows links - a session factory other than the default, ftplib.FTP, can be given in the FTPHost constructor; this allows to use classes derived from ftplib.FTP (like ftpslib.FTP_TLS from the M2Crypto package) - several bugfixes (mostly regarding byte count in text mode transfers) - unit test Stefan ---------------------------------------------------------------------- Hello Pythoneers :) I would like to announce ftputil.py, a module which provides a more friendly interface for FTP sessions than the ftplib module. The FTPHost objects generated from it allow many operations similar to those of os and os.path. Examples: # download some files from the login directory import ftputil host = ftputil.FTPHost('ftp.domain.com', 'user', 'secret') names = host.listdir(host.curdir) for name in names: if host.path.isreg(name): host.download(name, name, 'b') # remote, local, binary mode # make a new directory and copy a remote file into it host.mkdir('newdir') source = host.file('index.html', 'r') # file-like object target = host.file('newdir/index.html', 'w') # file-like object host.copyfileobj(source, target) # mimics shutil.copyfileobj source.close() target.close() Even host.path.walk works. :-) But slow. ;-) ftputil.py can be downloaded from http://www.ndh.net/home/sschwarzer/download/ftputil.py I would like to get your suggestions and comments. :-) Stefan P.S.: Thanks to Pedro Rodriguez for his helpful answer to my question in comp.lang.python :-) ---------------------------------------------------------------------- ftputil-3.4/doc/whats_new_in_ftputil_3.0.txt0000644000175000017470000002545313135113162020432 0ustar debiandebianWhat's new in ftputil 3.0? ========================== :Version: 3.0 :Date: 2013-09-29 :Author: Stefan Schwarzer .. contents:: Added support for Python 3 -------------------------- This ftputil release adds support for Python 3.0 and up. Python 2 and 3 are supported with the same source code. Also, the API including the semantics is the same. As for Python 3 code, in ftputil 3.0 unicode is somewhat preferred over byte strings. On the other hand, in line with the file system APIs of both Python 2 and 3, methods take either byte strings or unicode strings. Methods that take and return strings (for example, ``FTPHost.path.abspath`` or ``FTPHost.listdir``), return the same string type they get. .. Note:: Both Python 2 and 3 have two "string" types where one type represents a sequence of bytes and the other type character (text) data. ============== =========== =========== =========================== Python version Binary type Text type Default string literal type ============== =========== =========== =========================== 2 ``str`` ``unicode`` ``str`` (= binary type) 3 ``bytes`` ``str`` ``str`` (= text type) ============== =========== =========== =========================== So both lines of Python have an ``str`` type, but in Python 2 it's the byte type and in Python 3 the text type. The ``str`` type is also what you get when you write a literal string without any prefixes. For example ``"Python"`` is a binary string in Python 2 and a text (unicode) string in Python 3. If this seems confusing, please read `this description`_ in the Python documentation for more details. .. _`this description`: http://docs.python.org/3.0/whatsnew/3.0.html#text-vs-data-instead-of-unicode-vs-8-bit Dropped support for Python 2.4 and 2.5 -------------------------------------- To make it easier to use the same code for Python 2 and 3, I decided to use the Python 3 features backported to Python 2.6. As a consequence, ftputil 3.0 doesn't work with Python 2.4 and 2.5. Newlines and encoding of remote file content -------------------------------------------- Traditionally, "text mode" for FTP transfers meant translation to ``\r\n`` newlines, even between transfers of Unix clients and Unix servers. Since this presumably most of the time is neither the expected nor the desired behavior, the ``FTPHost.open`` method now has the API and semantics of the built-in ``open`` function in Python 3. If you want the same API for *local* files in Python 2.6 and 2.7, you can use the ``open`` function from the ``io`` module. Thus, when opening remote files in *binary* mode, the new API does *not* accept an encoding argument. On the other hand, opening a file in text mode always implies an encoding step when writing and decoding step when reading files. If the ``encoding`` argument isn't specified, it defaults to the value of ``locale.getpreferredencoding(False)``. Also as with Python 3's ``open`` builtin, opening a file in binary mode for reading will give you byte string data. If you write to a file opened in binary mode, you must write byte strings. Along the same lines, files opened in text mode will give you unicode strings when read, and require unicode strings to be passed to write operations. Module and method name changes ------------------------------ In earlier ftputil versions, most module names had a redundant ``ftp_`` prefix. In ftputil 3.0, these prefixes are removed. Of the module names that are part of the public ftputil API, this affects only ``ftputil.error`` and ``ftputil.stat``. In Python 2.2, ``file`` became an alias for ``open``, and previous ftputil versions also had an ``FTPHost.file`` besides the ``FTPHost.open`` method. In Python 3.0, the ``file`` builtin was removed and the return values from the built-in ``open`` methods are no longer ``file`` instances. Along the same lines, ftputil 3.0 also drops the ``FTPHost.file`` alias and requires ``FTPHost.open``. Upload and download modes ------------------------- The ``FTPHost`` methods for downloading and uploading files (``download``, ``download_if_newer``, ``upload`` and ``upload_if_newer``) now always use binary mode; a ``mode`` argument is no longer needed or even allowed. Although this behavior makes downloads and uploads slightly less flexible, it should cover almost all use cases. If you *really* want to do a transfer involving files opened in text mode, you can still do:: import ftputil.file_transfer ... with FTPHost.open("source.txt", "r", encoding="UTF-8") as source, \ FTPHost.open("target.txt", "w", encoding="latin1") as target: ftputil.file_transfer.copyfileobj(source, target) Note that it's not possible anymore to open one file in binary mode and the other file in text mode and transfer data between them with ``copyfileobj``. For example, opening the source in binary mode will read byte strings, but a target file opened in text mode will only allow writing of unicode strings. Then again, I assume that the cases where you want a mixed binary/text mode transfer should be *very* rare. Custom parsers receive lines as unicode strings ----------------------------------------------- Custom parsers, as described in the documentation_, receive a text line for each directory entry in the methods ``ignores_line`` and ``parse_line``. In previous ftputil versions, the ``line`` arguments were byte strings; now they're unicode strings. .. _documentation: http://ftputil.sschwarzer.net/documentation#writing-directory-parsers If you aren't sure what this is about, this may help: If you never used the ``FTPHost.set_parser`` method, you can ignore this section. :-) Porting to ftputil 3.0 ---------------------- - It's likely that you catch an ftputil exception here and there. In that case, you need to change ``import ftputil.ftp_error`` to ``import ftputil.error`` and modify the uses of the module accordingly. If you used ``from ftputil import ftp_error``, you can change this to ``from ftputil import error as ftp_error`` without changing the code using the module. - If you use the download or upload methods, you need to remove the ``mode`` argument from the call. If you used something else than ``"b"`` for binary mode (which I assume to be unlikely), you'll need to adapt the code that calls the download or upload methods. - If you use custom parsers, you'll need to change ``import ftputil.ftp_stat`` to ``import ftputil.stat`` and adapt your code in the module. Moreover, you might need to change your ``ignores_line`` or ``parse_line`` calls if they rely on their ``line`` argument being a byte string. - If you use remote files, especially ones opened in text mode, you may need to change your code to adapt to the changes in newline conversion, encoding and/or string type (see above sections). .. Note:: In the root directory of the installed ftputil package is a script ``find_invalid_code.py`` which, given a start directory as argument, will scan that directory tree for code that may need to be fixed. However, this script uses very simple heuristics, so it may miss some problematic code or list perfectly valid code. In particular, you may want to change the regular expression string ``HOST_REGEX`` for the names you usually use for ``FTPHost`` objects. Questions and answers --------------------- The advice to "adapt code to the new string types" is rather vague. Can't you be more specific? ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ It's difficult to be more specific without knowing your application. That said, best practices nowadays are: - If you're dealing with character data, use unicode strings whenever possible. In Python 2, this means the ``unicode`` type and in Python 3 the ``str`` type. - Whenever you deal with binary data which is actually character data, decode it as *soon* as possible when *reading* data. Encode the data as *late* as possible when *writing* data. Yes, I know that's not much more specific. Why don't you use a "Python 2 API" for Python 2 and a "Python 3 API" for Python 3? ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ (What's meant here is, for example, that if you opened a remote file as text, the read data could be of byte string type in Python 2 and of unicode type in Python 3. Similarly, under Python 2 a text file opened for writing could accept both byte strings and unicode strings in the ``write*`` methods.) Actually, I had at first thought of implementing this but dropped the idea because it has several problems: - Basically, I would have to support two APIs for the same set of methods. I can imagine that some things can be simplified by just using ``str`` to convert to the "right" string type automatically, but I assume these opportunities would be rather the exception than the rule. I'd certainly not look forward to maintaining such code. - Using two different APIs might require people to change their code if they move from using ftputil 3.x in Python 2 to using it in Python 3. - Developers who want to support both Python 2 and 3 with the same source code (as I do now in ftputil) would "inherit" the "dual API" and would have to use different wrapper code depending on the Python version their code is run under. For these reasons, I `ended up`_ choosing the same API semantics for Python 2 and 3. .. _`ended up`: https://groups.google.com/forum/?fromgroups=#!topic/comp.lang.python/XKof6DpNyH4 Why don't you use the six_ module to be able to support Python 2.4 and 2.5? ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. _six: https://pypi.python.org/pypi/six/ There are two reasons: - ftputil so far has no dependencies other than the Python standard library, and I think that's a nice feature. - Although ``six`` makes it easier to support Python 2.4/2.5 and Python 3 at the same time, the resulting code is somewhat awkward. I wanted a code base that feels more like "modern Python"; I wanted to use the Python 3 features backported to Python 2.6 and 2.7. Why don't you use 2to3_ to generate the Python 3 version of ftputil? ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. _2to3: http://docs.python.org/2/library/2to3.html I had considered this when I started adapting the ftputil source code for Python 3. On the other hand, although using 2to3 used to be the recommended approach for Python 3 support, even `rather large projects`_ have chosen the route of having one code base and using it unmodified for Python 2 and 3. .. _`rather large projects`: https://docs.djangoproject.com/en/dev/topics/python3/ When I looked into this approach for ftputil 3.0, it became quickly obvious that it would be easier and I found it worked out very well. ftputil-3.4/doc/README.txt0000644000175000017470000000715013200524625014550 0ustar debiandebianftputil ======= Purpose ------- ftputil is a high-level FTP client library for the Python programming language. ftputil implements a virtual file system for accessing FTP servers, that is, it can generate file-like objects for remote files. The library supports many functions similar to those in the os, os.path and shutil modules. ftputil has convenience functions for conditional uploads and downloads, and handles FTP clients and servers in different timezones. What's new? ----------- Since version 3.3.1 the following changed: - Several bugs were fixed [1-5]. - Added deprecation warnings for backward incompatibilities in the upcoming ftputil 4.0.0. Important note -------------- The next version of ftputil will be 4.0.0 (apart from small fixes in possible 3.4.x versions). ftputil 4.0.0 will make some backward-incompatible changes: - Support for Python 2 will be removed. There are several reasons for this, which are explained in [6]. - The flag `use_list_a_option` will be set to `False` by default. This option was intended to make life easier for users of the library, but turned out to be problematic (see [7]). Documentation ------------- The documentation for ftputil can be found in the file ftputil.txt (reStructuredText format) or ftputil.html (recommended, generated from ftputil.txt). Prerequisites ------------- To use ftputil, you need Python, at least version 2.6. Python 3.x versions work as well. Installation ------------ *If you have an older version of ftputil installed, delete it or move it somewhere else, so that it doesn't conflict with the new version!* If you have pip or easy_install available, you can install the current version of ftputil directly from the Python Package Index (PyPI) without downloading the package explicitly. You'll still need an internet connection, of course. - Just type pip install ftputil or easy_install ftputil on the command line, respectively. Unless you're installing ftputil in a virtual environment, you'll probably need root/administrator privileges to do that. Done. :-) If you don't have pip or easy_install, you need to download a tarball from the Python Package Index or from the ftputil website and install it: - Unpack the archive file containing the distribution files. If you had an ftputil version 2.8, you would type at the shell prompt: tar xzf ftputil-2.8.tar.gz - Make the directory to where the files were unpacked your current directory. Assume that after unpacking, you have a directory `ftputil-2.8`. Make it the current directory with cd ftputil-2.8 - Type python setup.py install at the shell prompt. On Unix/Linux, you have to be root to perform the installation. Likewise, you have to be logged in as administrator if you install on Windows. If you want to customize the installation paths, please read http://docs.python.org/inst/inst.html . License ------- ftputil is open source software. It is distributed under the new/modified/revised BSD license (see http://opensource.org/licenses/BSD-3-Clause ). Authors ------- Stefan Schwarzer Evan Prodromou (lrucache module) (See also the file `doc/contributors.txt`.) Please provide feedback! It's certainly appreciated. :-) [1] http://ftputil.sschwarzer.net/trac/ticket/107 [2] http://ftputil.sschwarzer.net/trac/ticket/109 [3] http://ftputil.sschwarzer.net/trac/ticket/112 [4] http://ftputil.sschwarzer.net/trac/ticket/113 [5] http://ftputil.sschwarzer.net/trac/ticket/114 [6] http://lists.sschwarzer.net/pipermail/ftputil/2017q3/000465.html [7] http://ftputil.sschwarzer.net/trac/ticket/110 ftputil-3.4/doc/whats_new_in_ftputil_3.0.html0000644000175000017470000005452313200531432020554 0ustar debiandebian What's new in ftputil 3.0?

What's new in ftputil 3.0?

Version: 3.0
Date: 2013-09-29
Author: Stefan Schwarzer <sschwarzer@sschwarzer.net>

Added support for Python 3

This ftputil release adds support for Python 3.0 and up.

Python 2 and 3 are supported with the same source code. Also, the API including the semantics is the same. As for Python 3 code, in ftputil 3.0 unicode is somewhat preferred over byte strings. On the other hand, in line with the file system APIs of both Python 2 and 3, methods take either byte strings or unicode strings. Methods that take and return strings (for example, FTPHost.path.abspath or FTPHost.listdir), return the same string type they get.

Note

Both Python 2 and 3 have two "string" types where one type represents a sequence of bytes and the other type character (text) data.

Python version Binary type Text type Default string literal type
2 str unicode str (= binary type)
3 bytes str str (= text type)

So both lines of Python have an str type, but in Python 2 it's the byte type and in Python 3 the text type. The str type is also what you get when you write a literal string without any prefixes. For example "Python" is a binary string in Python 2 and a text (unicode) string in Python 3.

If this seems confusing, please read this description in the Python documentation for more details.

Dropped support for Python 2.4 and 2.5

To make it easier to use the same code for Python 2 and 3, I decided to use the Python 3 features backported to Python 2.6. As a consequence, ftputil 3.0 doesn't work with Python 2.4 and 2.5.

Newlines and encoding of remote file content

Traditionally, "text mode" for FTP transfers meant translation to \r\n newlines, even between transfers of Unix clients and Unix servers. Since this presumably most of the time is neither the expected nor the desired behavior, the FTPHost.open method now has the API and semantics of the built-in open function in Python 3. If you want the same API for local files in Python 2.6 and 2.7, you can use the open function from the io module.

Thus, when opening remote files in binary mode, the new API does not accept an encoding argument. On the other hand, opening a file in text mode always implies an encoding step when writing and decoding step when reading files. If the encoding argument isn't specified, it defaults to the value of locale.getpreferredencoding(False).

Also as with Python 3's open builtin, opening a file in binary mode for reading will give you byte string data. If you write to a file opened in binary mode, you must write byte strings. Along the same lines, files opened in text mode will give you unicode strings when read, and require unicode strings to be passed to write operations.

Module and method name changes

In earlier ftputil versions, most module names had a redundant ftp_ prefix. In ftputil 3.0, these prefixes are removed. Of the module names that are part of the public ftputil API, this affects only ftputil.error and ftputil.stat.

In Python 2.2, file became an alias for open, and previous ftputil versions also had an FTPHost.file besides the FTPHost.open method. In Python 3.0, the file builtin was removed and the return values from the built-in open methods are no longer file instances. Along the same lines, ftputil 3.0 also drops the FTPHost.file alias and requires FTPHost.open.

Upload and download modes

The FTPHost methods for downloading and uploading files (download, download_if_newer, upload and upload_if_newer) now always use binary mode; a mode argument is no longer needed or even allowed. Although this behavior makes downloads and uploads slightly less flexible, it should cover almost all use cases.

If you really want to do a transfer involving files opened in text mode, you can still do:

import ftputil.file_transfer

...

with FTPHost.open("source.txt", "r", encoding="UTF-8") as source, \
     FTPHost.open("target.txt", "w", encoding="latin1") as target:
    ftputil.file_transfer.copyfileobj(source, target)

Note that it's not possible anymore to open one file in binary mode and the other file in text mode and transfer data between them with copyfileobj. For example, opening the source in binary mode will read byte strings, but a target file opened in text mode will only allow writing of unicode strings. Then again, I assume that the cases where you want a mixed binary/text mode transfer should be very rare.

Custom parsers receive lines as unicode strings

Custom parsers, as described in the documentation, receive a text line for each directory entry in the methods ignores_line and parse_line. In previous ftputil versions, the line arguments were byte strings; now they're unicode strings.

If you aren't sure what this is about, this may help: If you never used the FTPHost.set_parser method, you can ignore this section. :-)

Porting to ftputil 3.0

  • It's likely that you catch an ftputil exception here and there. In that case, you need to change import ftputil.ftp_error to import ftputil.error and modify the uses of the module accordingly. If you used from ftputil import ftp_error, you can change this to from ftputil import error as ftp_error without changing the code using the module.
  • If you use the download or upload methods, you need to remove the mode argument from the call. If you used something else than "b" for binary mode (which I assume to be unlikely), you'll need to adapt the code that calls the download or upload methods.
  • If you use custom parsers, you'll need to change import ftputil.ftp_stat to import ftputil.stat and adapt your code in the module. Moreover, you might need to change your ignores_line or parse_line calls if they rely on their line argument being a byte string.
  • If you use remote files, especially ones opened in text mode, you may need to change your code to adapt to the changes in newline conversion, encoding and/or string type (see above sections).

Note

In the root directory of the installed ftputil package is a script find_invalid_code.py which, given a start directory as argument, will scan that directory tree for code that may need to be fixed. However, this script uses very simple heuristics, so it may miss some problematic code or list perfectly valid code.

In particular, you may want to change the regular expression string HOST_REGEX for the names you usually use for FTPHost objects.

Questions and answers

The advice to "adapt code to the new string types" is rather vague. Can't you be more specific?

It's difficult to be more specific without knowing your application.

That said, best practices nowadays are:

  • If you're dealing with character data, use unicode strings whenever possible. In Python 2, this means the unicode type and in Python 3 the str type.
  • Whenever you deal with binary data which is actually character data, decode it as soon as possible when reading data. Encode the data as late as possible when writing data.

Yes, I know that's not much more specific.

Why don't you use a "Python 2 API" for Python 2 and a "Python 3 API" for Python 3?

(What's meant here is, for example, that if you opened a remote file as text, the read data could be of byte string type in Python 2 and of unicode type in Python 3. Similarly, under Python 2 a text file opened for writing could accept both byte strings and unicode strings in the write* methods.)

Actually, I had at first thought of implementing this but dropped the idea because it has several problems:

  • Basically, I would have to support two APIs for the same set of methods. I can imagine that some things can be simplified by just using str to convert to the "right" string type automatically, but I assume these opportunities would be rather the exception than the rule. I'd certainly not look forward to maintaining such code.
  • Using two different APIs might require people to change their code if they move from using ftputil 3.x in Python 2 to using it in Python 3.
  • Developers who want to support both Python 2 and 3 with the same source code (as I do now in ftputil) would "inherit" the "dual API" and would have to use different wrapper code depending on the Python version their code is run under.

For these reasons, I ended up choosing the same API semantics for Python 2 and 3.

Why don't you use the six module to be able to support Python 2.4 and 2.5?

There are two reasons:

  • ftputil so far has no dependencies other than the Python standard library, and I think that's a nice feature.
  • Although six makes it easier to support Python 2.4/2.5 and Python 3 at the same time, the resulting code is somewhat awkward. I wanted a code base that feels more like "modern Python"; I wanted to use the Python 3 features backported to Python 2.6 and 2.7.

Why don't you use 2to3 to generate the Python 3 version of ftputil?

I had considered this when I started adapting the ftputil source code for Python 3. On the other hand, although using 2to3 used to be the recommended approach for Python 3 support, even rather large projects have chosen the route of having one code base and using it unmodified for Python 2 and 3.

When I looked into this approach for ftputil 3.0, it became quickly obvious that it would be easier and I found it worked out very well.

ftputil-3.4/doc/python3_support.txt0000644000175000017470000001633012353637454017030 0ustar debiandebianMaking ftputil Ready for Python 3 ================================= Summary ------- First, here's a summary of my current plans: - The next ftputil version will be 3.0. It will work with all Python versions starting at 2.6. - This ftputil version will change some APIs in ways that will require changes in existing ftputil client code. This may include non-trivial changes where ftputil previously accepted either byte strings or unicode strings but will require unicode strings in the future. Please read on for details, including the reasons for these plans. Current Situation ----------------- Currently ftputil only supports Python 2. Several people have asked for Python 3 support; one entered a ticket on the ftputil project website. I also think that Python 3 becomes more and more widespread and so ftputil should be made compatible with Python 3. Future Development ------------------ Supported Python Versions ~~~~~~~~~~~~~~~~~~~~~~~~~ Although I want to adapt ftputil for use with Python 3, I have no plans to drop Python 2 support in ftputil anytime soon. Python 2 still is much more widely used. Originally the recommended approach for supporting Python 2 and 3 was to have a Python 2 version that could be translated with the `2to3` tool. However, nowadays another approach seems to be more common: supporting Python 2 and 3 with the very same code, without any conversions. From what I heard and read about the topic, I prefer the latter approach. In order to support Python 2 and 3 with the same code, I plan to drop support for Python versions before 2.6. Increasing the minimum Python version to 2.6 makes a common implementation easier because several significant Python 3 features have been backported to Python 2.6. Important examples are: - from __future__ import print_function - from __future__ import unicode_literals - `io` library, simplifying on-the-fly encoding/decoding Incompatibilities With Previous ftputil Versions (2.8 and Earlier) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ It seems that most of the semantics of ftputil can be kept. However, the handling of strings in Python 2 and 3 differs in non-trivial ways. Almost all APIs that used to expect byte strings in Python 2, expect unicode strings in Python 3. Actually, the native `str` type in Python 2 is a byte string type, whereas `str` in Python 3 is a unicode strings type. I can imagine two approaches for dealing with strings in the next version of ftputil: 1. Under Python 2, offer an API which is more "natural" for Python 2. Under Python 3, offer an API which is more "natural" for Python 3. Advantage: When run under Python 2, ftputil would behave the same as before. Only few code changes (e. g. import statements) would be necessary. Under Python 3, a more "modern" API could be used. Disadvantage: Implementing two different APIs in the same code base might be extremely difficult. Conditional testing for different Python versions would probably be a mess and error-prone. 2. Use the same API for Python 2 and 3. Advantages: Using the same string types (bytes/unicode) under Python 2 and 3 will make the migration of ftputil itself easier, in particular the unit tests. It will be easier for clients of ftputil to offer support for Python 2 and 3 because they don't have to access ftputil via different APIs that depend on the Python runtime environment. Disadvantage: Code using previous ftputil versions will have to be adapted, possibly in non-trivial ways. I prefer the second approach, using a unified API. Please read this thread on comp.lang.python for some discussion: https://groups.google.com/forum/?fromgroups=#!topic/comp.lang.python/XKof6DpNyH4 In particular, so far I plan the following changes: - Opening remote files will behave as for the `io.open` function, available in Python 2.6 and later (see http://docs.python.org/2/library/io.html#io.open ; this is the same as the built-in `open` function in Python 3). When opening a file for reading in text mode, methods like `read` and `readlines` will return unicode strings. When opening a file for writing in text mode, the `write` and `writelines` methods will only accept unicode strings. Also as documented for `io.open`, files opened in binary mode will use only byte strings in their interfaces. - Traditionally, using the "ascii" mode for file transfers has meant that newline characters on writing were translated to "\r\n" pairs. Versions of ftputil published so far have done the same when a file was opened in text mode. In the future, I'd like to handle the newline conversion as documented for `io.open`. That is, newlines in remote files will be written according to the operating system where the client runs. This is in line with accesses to remote file systems like NFS or CIFS. So ftputil here will introduce a backward incompatibility, but the API will correspond better to the behavior of the usual file system API, which actually is a key idea in the design of ftputil. - Methods which accept and return a string, say `FTPHost.path.abspath`, will return the same string type they've been called with. This is the same behavior as in Python 3. - Some module names are going to change. For example, `ftputil.ftp_error` will become `ftputil.error`. - Translation of file names from/to unicode will use ISO-8859-1, also known as latin-1. For local file system accesses, Python 3 uses the value of `sys.getfilesystemencoding()`, which often will be UTF-8. I think using ISO-8859-1 is the better choice since the `ftplib` in Python 3 also uses this encoding. Using the same encoding in ftputil will make sure that directory and file names created by ftplib can be read by ftputil and vice versa. That said, if you can you should avoid using anything but ASCII characters for directories and files on remote file systems. This advice is the same as for older ftputil versions. ;-) The mapping between actual bytes for directory and file names may be different from previous ftputil versions. On the other hand, this mapping hadn't even be defined previously. The above list contains the (in)compatibility changes that I've been able to think of so far. More issues may surface when I'm making the actual ftputil changes and after the first such version(s) of ftputil will have been published. I assume it was the same when the file system APIs in Python 3 were developed, so please bear with me. :-) If You Really Need Backward Compatibility: Creating an API Adapter ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ If there's a strong need for an API compatible to ftputil 2.8 and below, someone could write an adapter layer that could/would be used instead of the new ftputil API. I'm open to making simple API changes so that it becomes easier to use custom classes. For example, instead of hardcoding the use of the `ftputil._FTPFile` class in the `FTPHost` class, the `FTPHost` class can define the concrete FTP file class to use as a class attribute. What do you think about all this? Do you think it's reasonable? Have I overlooked something? Are there problems I may not have anticipated? Please give me feedback if there's something on your mind. :-) Best regards, Stefan ftputil-3.4/setup.py0000755000175000017470000000510512353637454014035 0ustar debiandebian#! /usr/bin/env python # Copyright (C) 2003-2013, Stefan Schwarzer # See the file LICENSE for licensing terms. """ setup.py - installation script for Python distutils """ from __future__ import print_function import os import sys from distutils import core _name = "ftputil" _package = "ftputil" _version = open("VERSION").read().strip() doc_files = [os.path.join("doc", name) for name in ["ftputil.txt", "ftputil.html", "whats_new_in_ftputil_3.0.txt", "whats_new_in_ftputil_3.0.html", "README.txt"]] doc_files_are_present = all((os.path.exists(doc_file) for doc_file in doc_files)) if "install" in sys.argv[1:] and not doc_files_are_present: print("One or more of the HTML documentation files are missing.") print("Please generate them with `make docs`.") sys.exit(1) core.setup( # Installation data name=_name, version=_version, packages=[_package], package_dir={_package: _package}, data_files=[("doc/ftputil", doc_files)], # Metadata author="Stefan Schwarzer", author_email="sschwarzer@sschwarzer.net", url="http://ftputil.sschwarzer.net/", description="High-level FTP client library (virtual file system and more)", keywords="FTP, client, library, virtual file system", license="Open source (revised BSD license)", platforms=["Pure Python"], long_description="""\ ftputil is a high-level FTP client library for the Python programming language. ftputil implements a virtual file system for accessing FTP servers, that is, it can generate file-like objects for remote files. The library supports many functions similar to those in the os, os.path and shutil modules. ftputil has convenience functions for conditional uploads and downloads, and handles FTP clients and servers in different timezones.""", download_url= "http://ftputil.sschwarzer.net/trac/attachment/wiki/Download/" "{0}-{1}.tar.gz?format=raw".format(_name, _version), classifiers=[ "Development Status :: 5 - Production/Stable", "Environment :: Other Environment", "Intended Audience :: Developers", "Intended Audience :: System Administrators", "License :: OSI Approved :: BSD License", "Operating System :: OS Independent", "Programming Language :: Python", "Programming Language :: Python :: 2", "Programming Language :: Python :: 3", "Topic :: Internet :: File Transfer Protocol (FTP)", "Topic :: Software Development :: Libraries :: Python Modules", "Topic :: System :: Filesystems", ] ) ftputil-3.4/LICENSE0000644000175000017470000002600313135113162013305 0ustar debiandebianLICENSE ======= All the software in the ftpuil distribution is covered by the modified BSD license. While ftputil has mainly been written by Stefan Schwarzer , others have also contributed suggestions and code. In particular, Evan Prodromou has contributed his lrucache module which is covered by both the modified BSD license and the Academic Free License. Modified BSD license: ---------------------------------------------------------------------- Copyright (C) 2002-2014, Stefan Schwarzer and contributors (see `doc/contributors.txt`) All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - Neither the name of the above author nor the names of the contributors to the software may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ---------------------------------------------------------------------- The Academic Free License: ---------------------------------------------------------------------- The Academic Free License v. 2.1 This Academic Free License (the "License") applies to any original work of authorship (the "Original Work") whose owner (the "Licensor") has placed the following notice immediately following the copyright notice for the Original Work: Licensed under the Academic Free License version 2.1 1) Grant of Copyright License. Licensor hereby grants You a world-wide, royalty-free, non-exclusive, perpetual, sublicenseable license to do the following: a) to reproduce the Original Work in copies; b) to prepare derivative works ("Derivative Works") based upon the Original Work; c) to distribute copies of the Original Work and Derivative Works to the public; d) to perform the Original Work publicly; and e) to display the Original Work publicly. 2) Grant of Patent License. Licensor hereby grants You a world-wide, royalty-free, non-exclusive, perpetual, sublicenseable license, under patent claims owned or controlled by the Licensor that are embodied in the Original Work as furnished by the Licensor, to make, use, sell and offer for sale the Original Work and Derivative Works. 3) Grant of Source Code License. The term "Source Code" means the preferred form of the Original Work for making modifications to it and all available documentation describing how to modify the Original Work. Licensor hereby agrees to provide a machine-readable copy of the Source Code of the Original Work along with each copy of the Original Work that Licensor distributes. Licensor reserves the right to satisfy this obligation by placing a machine-readable copy of the Source Code in an information repository reasonably calculated to permit inexpensive and convenient access by You for as long as Licensor continues to distribute the Original Work, and by publishing the address of that information repository in a notice immediately following the copyright notice that applies to the Original Work. 4) Exclusions From License Grant. Neither the names of Licensor, nor the names of any contributors to the Original Work, nor any of their trademarks or service marks, may be used to endorse or promote products derived from this Original Work without express prior written permission of the Licensor. Nothing in this License shall be deemed to grant any rights to trademarks, copyrights, patents, trade secrets or any other intellectual property of Licensor except as expressly stated herein. No patent license is granted to make, use, sell or offer to sell embodiments of any patent claims other than the licensed claims defined in Section 2. No right is granted to the trademarks of Licensor even if such marks are included in the Original Work. Nothing in this License shall be interpreted to prohibit Licensor from licensing under different terms from this License any Original Work that Licensor otherwise would have a right to license. 5) This section intentionally omitted. 6) Attribution Rights. You must retain, in the Source Code of any Derivative Works that You create, all copyright, patent or trademark notices from the Source Code of the Original Work, as well as any notices of licensing and any descriptive text identified therein as an "Attribution Notice." You must cause the Source Code for any Derivative Works that You create to carry a prominent Attribution Notice reasonably calculated to inform recipients that You have modified the Original Work. 7) Warranty of Provenance and Disclaimer of Warranty. Licensor warrants that the copyright in and to the Original Work and the patent rights granted herein by Licensor are owned by the Licensor or are sublicensed to You under the terms of this License with the permission of the contributor(s) of those copyrights and patent rights. Except as expressly stated in the immediately proceeding sentence, the Original Work is provided under this License on an "AS IS" BASIS and WITHOUT WARRANTY, either express or implied, including, without limitation, the warranties of NON-INFRINGEMENT, MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY OF THE ORIGINAL WORK IS WITH YOU. This DISCLAIMER OF WARRANTY constitutes an essential part of this License. No license to Original Work is granted hereunder except under this disclaimer. 8) Limitation of Liability. Under no circumstances and under no legal theory, whether in tort (including negligence), contract, or otherwise, shall the Licensor be liable to any person for any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or the use of the Original Work including, without limitation, damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses. This limitation of liability shall not apply to liability for death or personal injury resulting from Licensor's negligence to the extent applicable law prohibits such limitation. Some jurisdictions do not allow the exclusion or limitation of incidental or consequential damages, so this exclusion and limitation may not apply to You. 9) Acceptance and Termination. If You distribute copies of the Original Work or a Derivative Work, You must make a reasonable effort under the circumstances to obtain the express assent of recipients to the terms of this License. Nothing else but this License (or another written agreement between Licensor and You) grants You permission to create Derivative Works based upon the Original Work or to exercise any of the rights granted in Section 1 herein, and any attempt to do so except under the terms of this License (or another written agreement between Licensor and You) is expressly prohibited by U.S. copyright law, the equivalent laws of other countries, and by international treaty. Therefore, by exercising any of the rights granted to You in Section 1 herein, You indicate Your acceptance of this License and all of its terms and conditions. 10) Termination for Patent Action. This License shall terminate automatically and You may no longer exercise any of the rights granted to You by this License as of the date You commence an action, including a cross-claim or counterclaim, against Licensor or any licensee alleging that the Original Work infringes a patent. This termination provision shall not apply for an action alleging patent infringement by combinations of the Original Work with other software or hardware. 11) Jurisdiction, Venue and Governing Law. Any action or suit relating to this License may be brought only in the courts of a jurisdiction wherein the Licensor resides or in which Licensor conducts its primary business, and under the laws of that jurisdiction excluding its conflict-of-law provisions. The application of the United Nations Convention on Contracts for the International Sale of Goods is expressly excluded. Any use of the Original Work outside the scope of this License or after its termination shall be subject to the requirements and penalties of the U.S. Copyright Act, 17 U.S.C. § 101 et seq., the equivalent laws of other countries, and international treaty. This section shall survive the termination of this License. 12) Attorneys Fees. In any action to enforce the terms of this License or seeking damages relating thereto, the prevailing party shall be entitled to recover its costs and expenses, including, without limitation, reasonable attorneys' fees and costs incurred in connection with such action, including any appeal of such action. This section shall survive the termination of this License. 13) Miscellaneous. This License represents the complete agreement concerning the subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. 14) Definition of "You" in This License. "You" throughout this License, whether in upper or lower case, means an individual or a legal entity exercising rights under, and complying with all of the terms of, this License. For legal entities, "You" includes any entity that controls, is controlled by, or is under common control with you. For purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. 15) Right to Use. You may use the Original Work in all ways not otherwise restricted or conditioned by this License or by law, and Licensor promises not to interfere with or be responsible for such uses by You. This license is Copyright (C) 2003-2004 Lawrence E. Rosen. All rights reserved. Permission is hereby granted to copy and distribute this license without modification. This license may not be modified without the express written permission of its copyright owner. ---------------------------------------------------------------------- ftputil-3.4/VERSION0000644000175000017470000000000413200423657013347 0ustar debiandebian3.4 ftputil-3.4/test/0000755000175000017470000000000013200532636013262 5ustar debiandebianftputil-3.4/test/test_host.py0000644000175000017470000005343013175135274015665 0ustar debiandebian# encoding: utf-8 # Copyright (C) 2002-2016, Stefan Schwarzer # and ftputil contributors (see `doc/contributors.txt`) # See the file LICENSE for licensing terms. from __future__ import unicode_literals import ftplib import itertools import os import pickle import posixpath import random import time import warnings import pytest import ftputil import ftputil.compat import ftputil.error import ftputil.tool import ftputil.stat from test import mock_ftplib from test import test_base # # Helper functions to generate random data # def random_data(pool, size=10000): """ Return a byte string of characters consisting of those from the pool of integer numbers. """ ordinal_list = [random.choice(pool) for i in range(size)] return ftputil.compat.bytes_from_ints(ordinal_list) def ascii_data(): r""" Return a unicode string of "normal" ASCII characters, including `\r`. """ pool = list(range(32, 128)) # The idea is to have the "\r" converted to "\n" during the later # text write and check this conversion. pool.append(ord("\r")) return ftputil.tool.as_unicode(random_data(pool)) def binary_data(): """Return a binary character byte string.""" pool = list(range(0, 256)) return random_data(pool) # # Several customized `MockSession` classes # class FailOnLoginSession(mock_ftplib.MockSession): def __init__(self, host="", user="", password=""): raise ftplib.error_perm class FailOnKeepAliveSession(mock_ftplib.MockSession): def pwd(self): # Raise exception on second call to let the constructor work. if not hasattr(self, "pwd_called"): self.pwd_called = True return "/home" else: raise ftplib.error_temp class UnnormalizedCurrentWorkingDirectory(mock_ftplib.MockSession): def pwd(self): # Deliberately return the current working directory with a # trailing slash to test if it's removed when stored in the # `FTPHost` instance. return "/home/" class RecursiveListingForDotAsPathSession(mock_ftplib.MockSession): dir_contents = { ".": """\ lrwxrwxrwx 1 staff 7 Aug 13 2003 bin -> usr/bin dev: total 10 etc: total 10 pub: total 4 -rw-r--r-- 1 staff 74 Sep 25 2000 .message ---------- 1 staff 0 Aug 16 2003 .notar drwxr-xr-x 12 ftp 512 Nov 23 2008 freeware usr: total 4""", "": """\ total 10 lrwxrwxrwx 1 staff 7 Aug 13 2003 bin -> usr/bin d--x--x--x 2 staff 512 Sep 24 2000 dev d--x--x--x 3 staff 512 Sep 25 2000 etc dr-xr-xr-x 3 staff 512 Oct 3 2000 pub d--x--x--x 5 staff 512 Oct 3 2000 usr"""} def _transform_path(self, path): return path class BinaryDownloadMockSession(mock_ftplib.MockUnixFormatSession): mock_file_content = binary_data() class TimeShiftMockSession(mock_ftplib.MockSession): def delete(self, file_name): pass # # Customized `FTPHost` class for conditional upload/download tests # and time shift tests # class FailingUploadAndDownloadFTPHost(ftputil.FTPHost): def upload(self, source, target, mode=""): pytest.fail("`FTPHost.upload` should not have been called") def download(self, source, target, mode=""): pytest.fail("`FTPHost.download` should not have been called") class TimeShiftFTPHost(ftputil.FTPHost): class _Path: def split(self, path): return posixpath.split(path) def set_mtime(self, mtime): self._mtime = mtime def getmtime(self, file_name): return self._mtime def join(self, *args): return posixpath.join(*args) def normpath(self, path): return posixpath.normpath(path) def isabs(self, path): return posixpath.isabs(path) def abspath(self, path): return "/home/sschwarzer/_ftputil_sync_" # Needed for `isdir` in `FTPHost.remove` def isfile(self, path): return True def __init__(self, *args, **kwargs): ftputil.FTPHost.__init__(self, *args, **kwargs) self.path = self._Path() # # Test cases # class TestConstructor(object): """ Test initialization of `FTPHost` objects. """ def test_open_and_close(self): """ Test if opening and closing an `FTPHost` object works as expected. """ host = test_base.ftp_host_factory() host.close() assert host.closed is True assert host._children == [] def test_invalid_login(self): """Login to invalid host must fail.""" with pytest.raises(ftputil.error.FTPOSError): test_base.ftp_host_factory(FailOnLoginSession) def test_pwd_normalization(self): """ Test if the stored current directory is normalized. """ host = test_base.ftp_host_factory(UnnormalizedCurrentWorkingDirectory) assert host.getcwd() == "/home" class TestKeepAlive(object): def test_succeeding_keep_alive(self): """Assume the connection is still alive.""" host = test_base.ftp_host_factory() host.keep_alive() def test_failing_keep_alive(self): """Assume the connection has timed out, so `keep_alive` fails.""" host = test_base.ftp_host_factory( session_factory=FailOnKeepAliveSession) with pytest.raises(ftputil.error.TemporaryError): host.keep_alive() class TestSetParser(object): class TrivialParser(ftputil.stat.Parser): """ An instance of this parser always returns the same result from its `parse_line` method. This is all we need to check if ftputil uses the set parser. No actual parsing code is required here. """ def __init__(self): # We can't use `os.stat("/home")` directly because we # later need the object's `_st_name` attribute, which # we can't set on a `os.stat` stat value. default_stat_result = ftputil.stat.StatResult(os.stat("/home")) default_stat_result._st_name = "home" self.default_stat_result = default_stat_result def parse_line(self, line, time_shift=0.0): return self.default_stat_result def test_set_parser(self): """Test if the selected parser is used.""" host = test_base.ftp_host_factory() assert host._stat._allow_parser_switching is True trivial_parser = TestSetParser.TrivialParser() host.set_parser(trivial_parser) stat_result = host.stat("/home") assert stat_result == trivial_parser.default_stat_result assert host._stat._allow_parser_switching is False class TestCommandNotImplementedError(object): def test_command_not_implemented_error(self): """ Test if we get the anticipated exception if a command isn't implemented by the server. """ host = test_base.ftp_host_factory() with pytest.raises(ftputil.error.CommandNotImplementedError): host.chmod("nonexistent", 0o644) # `CommandNotImplementedError` is a subclass of `PermanentError`. with pytest.raises(ftputil.error.PermanentError): host.chmod("nonexistent", 0o644) class TestRecursiveListingForDotAsPath(object): """ Return a recursive directory listing when the path to list is a dot. This is used to test for issue #33, see http://ftputil.sschwarzer.net/trac/ticket/33 . """ def test_recursive_listing(self): host = test_base.ftp_host_factory( session_factory=RecursiveListingForDotAsPathSession) lines = host._dir(host.curdir) assert lines[0] == "total 10" assert lines[1].startswith("lrwxrwxrwx 1 staff") assert lines[2].startswith("d--x--x--x 2 staff") host.close() def test_plain_listing(self): host = test_base.ftp_host_factory( session_factory=RecursiveListingForDotAsPathSession) lines = host._dir("") assert lines[0] == "total 10" assert lines[1].startswith("lrwxrwxrwx 1 staff") assert lines[2].startswith("d--x--x--x 2 staff") host.close() def test_empty_string_instead_of_dot_workaround(self): host = test_base.ftp_host_factory( session_factory=RecursiveListingForDotAsPathSession) files = host.listdir(host.curdir) assert files == ["bin", "dev", "etc", "pub", "usr"] host.close() class TestUploadAndDownload(object): """Test ASCII upload and binary download as examples.""" def generate_file(self, data, file_name): """Generate a local data file.""" with open(file_name, "wb") as source_file: source_file.write(data) def test_download(self): """Test mode download.""" local_target = "_test_target_" host = test_base.ftp_host_factory( session_factory=BinaryDownloadMockSession) # Download host.download("dummy", local_target) # Read file and compare with open(local_target, "rb") as fobj: data = fobj.read() remote_file_content = mock_ftplib.content_of("dummy") assert data == remote_file_content # Clean up os.unlink(local_target) def test_conditional_upload(self): """Test conditional upload.""" local_source = "_test_source_" data = binary_data() self.generate_file(data, local_source) # Target is newer, so don't upload. host = test_base.ftp_host_factory( ftp_host_class=FailingUploadAndDownloadFTPHost) flag = host.upload_if_newer(local_source, "/home/newer") assert flag is False # Target is older, so upload. host = test_base.ftp_host_factory() flag = host.upload_if_newer(local_source, "/home/older") assert flag is True remote_file_content = mock_ftplib.content_of("older") assert data == remote_file_content # Target doesn't exist, so upload. host = test_base.ftp_host_factory() flag = host.upload_if_newer(local_source, "/home/notthere") assert flag is True remote_file_content = mock_ftplib.content_of("notthere") assert data == remote_file_content # Clean up. os.unlink(local_source) def compare_and_delete_downloaded_data(self, file_name): """ Compare content of downloaded file with its source, then delete the local target file. """ with open(file_name, "rb") as fobj: data = fobj.read() # The name `newer` is used by all callers, so use it here, too. remote_file_content = mock_ftplib.content_of("newer") assert data == remote_file_content # Clean up os.unlink(file_name) def test_conditional_download_without_target(self): """ Test conditional binary mode download when no target file exists. """ local_target = "_test_target_" # Target does not exist, so download. host = test_base.ftp_host_factory( session_factory=BinaryDownloadMockSession) flag = host.download_if_newer("/home/newer", local_target) assert flag is True self.compare_and_delete_downloaded_data(local_target) def test_conditional_download_with_older_target(self): """Test conditional binary mode download with newer source file.""" local_target = "_test_target_" # Make target file. open(local_target, "w").close() # Source is newer (date in 2020), so download. host = test_base.ftp_host_factory( session_factory=BinaryDownloadMockSession) flag = host.download_if_newer("/home/newer", local_target) assert flag is True self.compare_and_delete_downloaded_data(local_target) def test_conditional_download_with_newer_target(self): """Test conditional binary mode download with older source file.""" local_target = "_test_target_" # Make target file. open(local_target, "w").close() # Source is older (date in 1970), so don't download. host = test_base.ftp_host_factory( ftp_host_class=FailingUploadAndDownloadFTPHost, session_factory=BinaryDownloadMockSession) flag = host.download_if_newer("/home/older", local_target) assert flag is False # Remove target file os.unlink(local_target) class TestTimeShift(object): def test_rounded_time_shift(self): """Test if time shift is rounded correctly.""" host = test_base.ftp_host_factory(session_factory=TimeShiftMockSession) # Use private bound method. rounded_time_shift = host._FTPHost__rounded_time_shift # Pairs consisting of original value and expected result test_data = [ ( 0, 0), ( 0.1, 0), ( -0.1, 0), ( 1500, 1800), ( -1500, -1800), ( 1800, 1800), ( -1800, -1800), ( 2000, 1800), ( -2000, -1800), ( 5*3600-100, 5*3600), (-5*3600+100, -5*3600)] for time_shift, expected_time_shift in test_data: calculated_time_shift = rounded_time_shift(time_shift) assert calculated_time_shift == expected_time_shift def test_assert_valid_time_shift(self): """Test time shift sanity checks.""" host = test_base.ftp_host_factory(session_factory=TimeShiftMockSession) # Use private bound method. assert_time_shift = host._FTPHost__assert_valid_time_shift # Valid time shifts test_data = [23*3600, -23*3600, 3600+30, -3600+30] for time_shift in test_data: assert assert_time_shift(time_shift) is None # Invalid time shift (exceeds one day) with pytest.raises(ftputil.error.TimeShiftError): assert_time_shift(25*3600) with pytest.raises(ftputil.error.TimeShiftError): assert_time_shift(-25*3600) # Invalid time shift (too large deviation from 15-minute units # is unacceptable) with pytest.raises(ftputil.error.TimeShiftError): assert_time_shift(8*60) with pytest.raises(ftputil.error.TimeShiftError): assert_time_shift(-3600-8*60) def test_synchronize_times(self): """Test time synchronization with server.""" host = test_base.ftp_host_factory(ftp_host_class=TimeShiftFTPHost, session_factory=TimeShiftMockSession) # Valid time shifts test_data = [ (60*60+30, 60*60), (60*60-100, 60*60), (30*60+100, 30*60), (45*60-100, 45*60), ] for measured_time_shift, expected_time_shift in test_data: host.path.set_mtime(time.time() + measured_time_shift) host.synchronize_times() assert host.time_shift() == expected_time_shift # Invalid time shifts measured_time_shifts = [60*60+8*60, 45*60-6*60] for measured_time_shift in measured_time_shifts: host.path.set_mtime(time.time() + measured_time_shift) with pytest.raises(ftputil.error.TimeShiftError): host.synchronize_times() def test_synchronize_times_for_server_in_east(self): """Test for timestamp correction (see ticket #55).""" host = test_base.ftp_host_factory(ftp_host_class=TimeShiftFTPHost, session_factory=TimeShiftMockSession) # Set this explicitly to emphasize the problem. host.set_time_shift(0.0) hour = 60 * 60 # This could be any negative time shift. presumed_time_shift = -6 * hour # Set `mtime` to simulate a server east of us. # In case the `time_shift` value for this host instance is 0.0 # (as is to be expected before the time shift is determined), # the directory parser (more specifically # `ftputil.stat.Parser.parse_unix_time`) will return a time which # is a year too far in the past. The `synchronize_times` # method needs to deal with this and add the year "back". # I don't think it's a bug in `parse_unix_time` because the # method should work once the time shift is set correctly. local_time = time.localtime() local_time_with_wrong_year = (local_time.tm_year-1,) + local_time[1:] presumed_server_time = \ time.mktime(local_time_with_wrong_year) + presumed_time_shift host.path.set_mtime(presumed_server_time) host.synchronize_times() assert host.time_shift() == presumed_time_shift class TestAcceptEitherUnicodeOrBytes(object): """ Test whether certain `FTPHost` methods accept either unicode or byte strings for the path(s). """ def setup_method(self, method): self.host = test_base.ftp_host_factory() def test_upload(self): """Test whether `upload` accepts either unicode or bytes.""" host = self.host # The source file needs to be present in the current directory. host.upload("Makefile", "target") host.upload("Makefile", ftputil.tool.as_bytes("target")) def test_download(self): """Test whether `download` accepts either unicode or bytes.""" host = test_base.ftp_host_factory( session_factory=BinaryDownloadMockSession) local_file_name = "_local_target_" host.download("source", local_file_name) host.download(ftputil.tool.as_bytes("source"), local_file_name) os.remove(local_file_name) def test_rename(self): """Test whether `rename` accepts either unicode or bytes.""" # It's possible to mix argument types, as for `os.rename`. path_as_unicode = "/home/file_name_test/ä" path_as_bytes = ftputil.tool.as_bytes(path_as_unicode) paths = [path_as_unicode, path_as_bytes] for source_path, target_path in itertools.product(paths, paths): self.host.rename(source_path, target_path) def test_listdir(self): """Test whether `listdir` accepts either unicode or bytes.""" host = self.host as_bytes = ftputil.tool.as_bytes host.chdir("/home/file_name_test") # Unicode items = host.listdir("ä") assert items == ["ö", "o"] # Need explicit type check for Python 2 for item in items: assert isinstance(item, ftputil.compat.unicode_type) # Bytes items = host.listdir(as_bytes("ä")) assert items == [as_bytes("ö"), as_bytes("o")] # Need explicit type check for Python 2 for item in items: assert isinstance(item, ftputil.compat.bytes_type) def test_chmod(self): """Test whether `chmod` accepts either unicode or bytes.""" host = self.host # The `voidcmd` implementation in `MockSession` would raise an # exception for the `CHMOD` command. host._session.voidcmd = host._session._ignore_arguments path = "/home/file_name_test/ä" host.chmod(path, 0o755) host.chmod(ftputil.tool.as_bytes(path), 0o755) def _test_method_with_single_path_argument(self, method, path): method(path) method(ftputil.tool.as_bytes(path)) def test_chdir(self): """Test whether `chdir` accepts either unicode or bytes.""" self._test_method_with_single_path_argument( self.host.chdir, "/home/file_name_test/ö") def test_mkdir(self): """Test whether `mkdir` accepts either unicode or bytes.""" # This directory exists already in the mock session, but this # shouldn't matter for the test. self._test_method_with_single_path_argument( self.host.mkdir, "/home/file_name_test/ä") def test_makedirs(self): """Test whether `makedirs` accepts either unicode or bytes.""" self._test_method_with_single_path_argument( self.host.makedirs, "/home/file_name_test/ä") def test_rmdir(self): """Test whether `rmdir` accepts either unicode or bytes.""" empty_directory_as_required_by_rmdir = "/home/file_name_test/empty_ä" self._test_method_with_single_path_argument( self.host.rmdir, empty_directory_as_required_by_rmdir) def test_remove(self): """Test whether `remove` accepts either unicode or bytes.""" self._test_method_with_single_path_argument( self.host.remove, "/home/file_name_test/ö") def test_rmtree(self): """Test whether `rmtree` accepts either unicode or bytes.""" empty_directory_as_required_by_rmtree = "/home/file_name_test/empty_ä" self._test_method_with_single_path_argument( self.host.rmtree, empty_directory_as_required_by_rmtree) def test_lstat(self): """Test whether `lstat` accepts either unicode or bytes.""" self._test_method_with_single_path_argument( self.host.lstat, "/home/file_name_test/ä") def test_stat(self): """Test whether `stat` accepts either unicode or bytes.""" self._test_method_with_single_path_argument( self.host.stat, "/home/file_name_test/ä") def test_walk(self): """Test whether `walk` accepts either unicode or bytes.""" # We're not interested in the return value of `walk`. self._test_method_with_single_path_argument( self.host.walk, "/home/file_name_test/ä") class TestFailingPickling(object): def test_failing_pickling(self): """Test if pickling (intentionally) isn't supported.""" with test_base.ftp_host_factory() as host: with pytest.raises(TypeError): pickle.dumps(host) with host.open("/home/sschwarzer/index.html") as file_obj: with pytest.raises(TypeError): pickle.dumps(file_obj) ftputil-3.4/test/mock_ftplib.py0000644000175000017470000002210713175164317016137 0ustar debiandebian# encoding: utf-8 # Copyright (C) 2003-2013, Stefan Schwarzer # and ftputil contributors (see `doc/contributors.txt`) # See the file LICENSE for licensing terms. """ This module implements a mock version of the standard library's `ftplib.py` module. Some code is taken from there. Not all functionality is implemented, only what is needed to run the unit tests. """ from __future__ import unicode_literals import io import collections import ftplib import posixpath import ftputil.tool DEBUG = 0 # Use a global dictionary of the form `{path: `MockFile` object, ...}` # to make "remote" mock files that were generated during a test # accessible. For example, this is used for testing the contents of a # file after an `FTPHost.upload` call. mock_files = {} def content_of(path): """ Return the data stored in a mock remote file identified by `path`. """ return mock_files[path].getvalue() class MockFile(io.BytesIO, object): """ Mock class for the file objects _contained in_ `FTPFile` objects (not `FTPFile` objects themselves!). Contrary to `StringIO.StringIO` instances, `MockFile` objects can be queried for their contents after they have been closed. """ def __init__(self, path, content=b""): global mock_files mock_files[path] = self self._value_after_close = "" self._super = super(MockFile, self) self._super.__init__(content) def getvalue(self): if not self.closed: return self._super.getvalue() else: return self._value_after_close def close(self): if not self.closed: self._value_after_close = self._super.getvalue() self._super.close() class MockSocket(object): """ Mock class which is used to return something from `MockSession.transfercmd`. """ def __init__(self, path, mock_file_content=b""): if DEBUG: print("File content: *{0}*".format(mock_file_content)) self.file_path = path self.mock_file_content = mock_file_content self._timeout = 60 def makefile(self, mode): return MockFile(self.file_path, self.mock_file_content) def close(self): pass # Timeout-related methods are used in `FTPFile.close`. def gettimeout(self): return self._timeout def settimeout(self, timeout): self._timeout = timeout class MockSession(object): """ Mock class which works like `ftplib.FTP` for the purpose of the unit tests. """ # Used by `MockSession.cwd` and `MockSession.pwd` current_dir = "/home/sschwarzer" # Used by `MockSession.dir`. This is a mapping from absolute path # to the multi-line string that would show up in an FTP # command-line client for this directory. dir_contents = {} # File content to be used (indirectly) with `transfercmd`. mock_file_content = b"" def __init__(self, host="", user="", password=""): self.closed = 0 # Copy default from class. self.current_dir = self.__class__.current_dir # Count successful `transfercmd` invocations to ensure that # each has a corresponding `voidresp`. self._transfercmds = 0 # Dummy, only for getting/setting timeout in `FTPFile.close` self.sock = MockSocket("", b"") def voidcmd(self, cmd): if DEBUG: print(cmd) if cmd == "STAT": return "MockSession server awaiting your commands ;-)" elif cmd.startswith("TYPE "): return elif cmd.startswith("SITE CHMOD"): raise ftplib.error_perm("502 command not implemented") else: raise ftplib.error_perm def pwd(self): return self.current_dir def _remove_trailing_slash(self, path): if path != "/" and path.endswith("/"): path = path[:-1] return path def _transform_path(self, path): return posixpath.normpath(posixpath.join(self.pwd(), path)) def cwd(self, path): path = ftputil.tool.as_unicode(path) self.current_dir = self._transform_path(path) def _ignore_arguments(self, *args, **kwargs): pass delete = mkd = rename = rmd = _ignore_arguments def dir(self, *args): """ Provide a callback function for processing each line of a directory listing. Return nothing. """ # The callback comes last in `ftplib.FTP.dir`. if isinstance(args[-1], collections.Callable): # Get `args[-1]` _before_ removing it in the line after. callback = args[-1] args = args[:-1] else: callback = None # Everything before the path argument are options. path = args[-1] if DEBUG: print("dir: {0}".format(path)) path = self._transform_path(path) if path not in self.dir_contents: raise ftplib.error_perm dir_lines = self.dir_contents[path].split("\n") for line in dir_lines: if callback is None: print(line) else: callback(line) def voidresp(self): assert self._transfercmds == 1 self._transfercmds -= 1 return "2xx" def transfercmd(self, cmd, rest=None): """ Return a `MockSocket` object whose `makefile` method will return a mock file object. """ if DEBUG: print(cmd) # Fail if attempting to read from/write to a directory. cmd, path = cmd.split() # Normalize path for lookup. path = self._remove_trailing_slash(path) if path in self.dir_contents: raise ftplib.error_perm # Fail if path isn't available (this name is hard-coded here # and has to be used for the corresponding tests). if (cmd, path) == ("RETR", "notthere"): raise ftplib.error_perm assert self._transfercmds == 0 self._transfercmds += 1 return MockSocket(path, self.mock_file_content) def close(self): if not self.closed: self.closed = 1 assert self._transfercmds == 0 class MockUnixFormatSession(MockSession): dir_contents = { "/": """\ drwxr-xr-x 2 45854 200 512 May 4 2000 home""", "/home": """\ drwxr-sr-x 2 45854 200 512 May 4 2000 sschwarzer -rw-r--r-- 1 45854 200 4605 Jan 19 1970 older -rw-r--r-- 1 45854 200 4605 Jan 19 2020 newer lrwxrwxrwx 1 45854 200 21 Jan 19 2002 link -> sschwarzer/index.html lrwxrwxrwx 1 45854 200 15 Jan 19 2002 bad_link -> python/bad_link drwxr-sr-x 2 45854 200 512 May 4 2000 dir with spaces drwxr-sr-x 2 45854 200 512 May 4 2000 python drwxr-sr-x 2 45854 200 512 May 4 2000 file_name_test""", "/home/python": """\ lrwxrwxrwx 1 45854 200 7 Jan 19 2002 link_link -> ../link lrwxrwxrwx 1 45854 200 14 Jan 19 2002 bad_link -> /home/bad_link""", "/home/sschwarzer": """\ total 14 drwxr-sr-x 2 45854 200 512 May 4 2000 chemeng drwxr-sr-x 2 45854 200 512 Jan 3 17:17 download drwxr-sr-x 2 45854 200 512 Jul 30 17:14 image -rw-r--r-- 1 45854 200 4604 Jan 19 23:11 index.html drwxr-sr-x 2 45854 200 512 May 29 2000 os2 lrwxrwxrwx 2 45854 200 6 May 29 2000 osup -> ../os2 drwxr-sr-x 2 45854 200 512 May 25 2000 publications drwxr-sr-x 2 45854 200 512 Jan 20 16:12 python drwxr-sr-x 6 45854 200 512 Sep 20 1999 scios2""", "/home/dir with spaces": """\ total 1 -rw-r--r-- 1 45854 200 4604 Jan 19 23:11 file with spaces""", "/home/file_name_test": """\ drwxr-sr-x 2 45854 200 512 May 29 2000 ä drwxr-sr-x 2 45854 200 512 May 29 2000 empty_ä -rw-r--r-- 1 45854 200 4604 Jan 19 23:11 ö lrwxrwxrwx 2 45854 200 6 May 29 2000 ü -> ä""", "/home/file_name_test/ä": """\ -rw-r--r-- 1 45854 200 4604 Jan 19 23:11 ö -rw-r--r-- 1 45854 200 4604 Jan 19 23:11 o""", "/home/file_name_test/empty_ä": """\ """, # Fail when trying to write to this directory (the content isn't # relevant). "sschwarzer": "", } class MockMSFormatSession(MockSession): dir_contents = { "/": """\ 10-23-01 03:25PM home""", "/home": """\ 10-23-01 03:25PM msformat""", "/home/msformat": """\ 10-23-01 03:25PM WindowsXP 12-07-01 02:05PM XPLaunch 07-17-00 02:08PM 12266720 abcd.exe 07-17-00 02:08PM 89264 O2KKeys.exe""", "/home/msformat/XPLaunch": """\ 10-23-01 03:25PM WindowsXP 12-07-01 02:05PM XPLaunch 12-07-01 02:05PM empty 07-17-00 02:08PM 12266720 abcd.exe 07-17-00 02:08PM 89264 O2KKeys.exe""", "/home/msformat/XPLaunch/empty": "total 0", } ftputil-3.4/test/test_session.py0000644000175000017470000001255713135116467016377 0ustar debiandebian# Copyright (C) 2014-2016, Stefan Schwarzer # and ftputil contributors (see `doc/contributors.txt`) # See the file LICENSE for licensing terms. """ Unit tests for session factory helpers. """ from __future__ import unicode_literals import ftputil.session class MockSession(object): """ Mock session base class to determine if all expected calls have happened. """ def __init__(self): self.calls = [] def add_call(self, *args): self.calls.append(args) def connect(self, host, port): self.add_call("connect", host, port) def _fix_socket(self): self.add_call("_fix_socket") def set_debuglevel(self, value): self.add_call("set_debuglevel", value) def login(self, user, password): self.add_call("login", user, password) def set_pasv(self, flag): self.add_call("set_pasv", flag) class EncryptedMockSession(MockSession): def auth_tls(self): self.add_call("auth_tls") def prot_p(self): self.add_call("prot_p") class TestSessionFactory(object): """ Test if session factories created by `ftputil.session.session_factory` trigger the expected calls. """ def test_defaults(self): """Test defaults (apart from base class).""" factory = \ ftputil.session.session_factory(base_class=MockSession) session = factory("host", "user", "password") assert (session.calls == [("connect", "host", 21), ("login", "user", "password")]) def test_different_port(self): """Test setting the command channel port with `port`.""" factory = \ ftputil.session.session_factory(base_class=MockSession, port=2121) session = factory("host", "user", "password") assert (session.calls == [("connect", "host", 2121), ("login", "user", "password")]) def test_use_passive_mode(self): """ Test explicitly setting passive/active mode with `use_passive_mode`. """ # Passive mode factory = ftputil.session.session_factory(base_class=MockSession, use_passive_mode=True) session = factory("host", "user", "password") assert session.calls == [("connect", "host", 21), ("login", "user", "password"), ("set_pasv", True)] # Active mode factory = ftputil.session.session_factory(base_class=MockSession, use_passive_mode=False) session = factory("host", "user", "password") assert session.calls == [("connect", "host", 21), ("login", "user", "password"), ("set_pasv", False)] def test_encrypt_data_channel(self): """Test request to call `prot_p` with `encrypt_data_channel`.""" # With encrypted data channel (default for encrypted session). factory = ftputil.session.session_factory( base_class=EncryptedMockSession) session = factory("host", "user", "password") assert session.calls == [("connect", "host", 21), ("login", "user", "password"), ("prot_p",)] # factory = ftputil.session.session_factory( base_class=EncryptedMockSession, encrypt_data_channel=True) session = factory("host", "user", "password") assert session.calls == [("connect", "host", 21), ("login", "user", "password"), ("prot_p",)] # Without encrypted data channel. factory = ftputil.session.session_factory( base_class=EncryptedMockSession, encrypt_data_channel=False) session = factory("host", "user", "password") assert session.calls == [("connect", "host", 21), ("login", "user", "password")] def test_debug_level(self): """Test setting the debug level on the session.""" factory = ftputil.session.session_factory(base_class=MockSession, debug_level=1) session = factory("host", "user", "password") assert session.calls == [("connect", "host", 21), ("set_debuglevel", 1), ("login", "user", "password")] def test_m2crypto_session(self): """Test call sequence for M2Crypto session.""" factory = \ ftputil.session.session_factory(base_class=EncryptedMockSession) # Return `True` to fake that this is a session deriving from # `M2Crypto.ftpslib.FTP_TLS`. factory._use_m2crypto_ftpslib = lambda self: True # Override `_fix_socket` here, not in `MockSession`. Since # the created session class _inherits_ from `MockSession`, # it would override the `_fix_socket` there. factory._fix_socket = lambda self: self.add_call("_fix_socket") session = factory("host", "user", "password") assert session.calls == [("connect", "host", 21), ("auth_tls",), ("_fix_socket",), ("login", "user", "password"), ("prot_p",)] ftputil-3.4/test/test_stat.py0000644000175000017470000005641513135123207015656 0ustar debiandebian# Copyright (C) 2003-2016, Stefan Schwarzer # and ftputil contributors (see `doc/contributors.txt`) # See the file LICENSE for licensing terms. from __future__ import absolute_import from __future__ import division from __future__ import unicode_literals import stat import time import pytest import ftputil import ftputil.compat import ftputil.error import ftputil.stat from ftputil.stat import MINUTE_PRECISION, DAY_PRECISION, UNKNOWN_PRECISION from test import test_base from test import mock_ftplib def _test_stat(session_factory): host = test_base.ftp_host_factory(session_factory=session_factory) stat = ftputil.stat._Stat(host) # Use Unix format parser explicitly. This doesn't exclude switching # to the MS format parser later if the test allows this switching. stat._parser = ftputil.stat.UnixParser() return stat # Special value to handle special case of datetimes before the epoch. EPOCH = time.gmtime(0)[:6] def stat_tuple_to_seconds(t): """ Return a float number representing the local time associated with the six-element tuple `t`. """ assert len(t) == 6, \ "need a six-element tuple (year, month, day, hour, min, sec)" # Do _not_ apply `time.mktime` to the `EPOCH` value below. On some # platforms (e. g. Windows) this might cause an `OverflowError`. if t == EPOCH: return 0.0 else: return time.mktime(t + (0, 0, -1)) class TestParsers(object): # # Helper methods # def _test_valid_lines(self, parser_class, lines, expected_stat_results): parser = parser_class() for line, expected_stat_result in zip(lines, expected_stat_results): # Convert to list to compare with the list `expected_stat_results`. parse_result = parser.parse_line(line) stat_result = list(parse_result) + \ [parse_result._st_mtime_precision, parse_result._st_name, parse_result._st_target] # Convert time tuple to seconds. expected_stat_result[8] = \ stat_tuple_to_seconds(expected_stat_result[8]) # Compare lists. assert stat_result == expected_stat_result def _test_invalid_lines(self, parser_class, lines): parser = parser_class() for line in lines: with pytest.raises(ftputil.error.ParserError): parser.parse_line(line) def _expected_year(self): """ Return the expected year for the second line in the listing in `test_valid_unix_lines`. """ # If in this year it's after Dec 19, 23:11, use the current # year, else use the previous year. This datetime value # corresponds to the hard-coded value in the string lists # below. now = time.localtime() # We need only month, day, hour and minute. current_time_parts = now[1:5] time_parts_in_listing = (12, 19, 23, 11) if current_time_parts > time_parts_in_listing: return now[0] else: return now[0] - 1 # # Unix parser # def test_valid_unix_lines(self): lines = [ "drwxr-sr-x 2 45854 200 512 May 4 2000 " "chemeng link -> chemeng target", # The year value for this line will change with the actual time. "-rw-r--r-- 1 45854 200 4604 Dec 19 23:11 index.html", "drwxr-sr-x 2 45854 200 512 May 29 2000 os2", "---------- 2 45854 200 512 May 29 2000 some_file", "lrwxrwxrwx 2 45854 200 512 May 29 2000 osup -> " "../os2" ] expected_stat_results = [ [17901, None, None, 2, "45854", "200", 512, None, (2000, 5, 4, 0, 0, 0), None, DAY_PRECISION, "chemeng link", "chemeng target"], [33188, None, None, 1, "45854", "200", 4604, None, (self._expected_year(), 12, 19, 23, 11, 0), None, MINUTE_PRECISION, "index.html", None], [17901, None, None, 2, "45854", "200", 512, None, (2000, 5, 29, 0, 0, 0), None, DAY_PRECISION, "os2", None], [32768, None, None, 2, "45854", "200", 512, None, (2000, 5, 29, 0, 0, 0), None, DAY_PRECISION, "some_file", None], [41471, None, None, 2, "45854", "200", 512, None, (2000, 5, 29, 0, 0, 0), None, DAY_PRECISION, "osup", "../os2"] ] self._test_valid_lines(ftputil.stat.UnixParser, lines, expected_stat_results) def test_alternative_unix_format(self): # See http://ftputil.sschwarzer.net/trac/ticket/12 for a # description for the need for an alternative format. lines = [ "drwxr-sr-x 2 200 512 May 4 2000 " "chemeng link -> chemeng target", # The year value for this line will change with the actual time. "-rw-r--r-- 1 200 4604 Dec 19 23:11 index.html", "drwxr-sr-x 2 200 512 May 29 2000 os2", "lrwxrwxrwx 2 200 512 May 29 2000 osup -> ../os2" ] expected_stat_results = [ [17901, None, None, 2, None, "200", 512, None, (2000, 5, 4, 0, 0, 0), None, DAY_PRECISION, "chemeng link", "chemeng target"], [33188, None, None, 1, None, "200", 4604, None, (self._expected_year(), 12, 19, 23, 11, 0), None, MINUTE_PRECISION, "index.html", None], [17901, None, None, 2, None, "200", 512, None, (2000, 5, 29, 0, 0, 0), None, DAY_PRECISION, "os2", None], [41471, None, None, 2, None, "200", 512, None, (2000, 5, 29, 0, 0, 0), None, DAY_PRECISION, "osup", "../os2"] ] self._test_valid_lines(ftputil.stat.UnixParser, lines, expected_stat_results) def test_pre_epoch_times_for_unix(self): # See http://ftputil.sschwarzer.net/trac/ticket/83 . # `mirrors.ibiblio.org` returns dates before the "epoch" that # cause an `OverflowError` in `mktime` on some platforms, # e. g. Windows. lines = [ "-rw-r--r-- 1 45854 200 4604 May 4 1968 index.html", "-rw-r--r-- 1 45854 200 4604 Dec 31 1969 index.html", "-rw-r--r-- 1 45854 200 4604 May 4 1800 index.html", ] expected_stat_result = \ [33188, None, None, 1, "45854", "200", 4604, None, EPOCH, None, UNKNOWN_PRECISION, "index.html", None] # Make shallow copies to avoid converting the time tuple more # than once in _test_valid_lines`. expected_stat_results = [expected_stat_result[:], expected_stat_result[:], expected_stat_result[:]] self._test_valid_lines(ftputil.stat.UnixParser, lines, expected_stat_results) def test_invalid_unix_lines(self): lines = [ # Not intended to be parsed. Should have been filtered out by # `ignores_line`. "total 14", # Invalid month abbreviation "drwxr-sr-x 2 45854 200 512 Max 4 2000 chemeng", # Year value isn't an integer "drwxr-sr-x 2 45854 200 512 May 4 abcd chemeng", # Day value isn't an integer "drwxr-sr-x 2 45854 200 512 May ab 2000 chemeng", # Hour value isn't an integer "-rw-r--r-- 1 45854 200 4604 Dec 19 ab:11 index.html", # Minute value isn't an integer "-rw-r--r-- 1 45854 200 4604 Dec 19 23:ab index.html", # Day value too large "drwxr-sr-x 2 45854 200 512 May 32 2000 chemeng", # Incomplete mode "drwxr-sr- 2 45854 200 512 May 4 2000 chemeng", # Invalid first letter in mode "xrwxr-sr-x 2 45854 200 512 May 4 2000 chemeng", # Ditto, plus invalid size value "xrwxr-sr-x 2 45854 200 51x May 4 2000 chemeng", # Is this `os1 -> os2` pointing to `os3`, or `os1` pointing # to `os2 -> os3` or the plain name `os1 -> os2 -> os3`? We # don't know, so we consider the line invalid. "drwxr-sr-x 2 45854 200 512 May 29 2000 " "os1 -> os2 -> os3", # Missing name "-rwxr-sr-x 2 45854 200 51x May 4 2000 ", ] self._test_invalid_lines(ftputil.stat.UnixParser, lines) # # Microsoft parser # def test_valid_ms_lines_two_digit_year(self): lines = [ "07-27-01 11:16AM Test", "10-23-95 03:25PM WindowsXP", "07-17-00 02:08PM 12266720 test.exe", "07-17-09 12:08AM 12266720 test.exe", "07-17-09 12:08PM 12266720 test.exe" ] expected_stat_results = [ [16640, None, None, None, None, None, None, None, (2001, 7, 27, 11, 16, 0), None, MINUTE_PRECISION, "Test", None], [16640, None, None, None, None, None, None, None, (1995, 10, 23, 15, 25, 0), None, MINUTE_PRECISION, "WindowsXP", None], [33024, None, None, None, None, None, 12266720, None, (2000, 7, 17, 14, 8, 0), None, MINUTE_PRECISION, "test.exe", None], [33024, None, None, None, None, None, 12266720, None, (2009, 7, 17, 0, 8, 0), None, MINUTE_PRECISION, "test.exe", None], [33024, None, None, None, None, None, 12266720, None, (2009, 7, 17, 12, 8, 0), None, MINUTE_PRECISION, "test.exe", None] ] self._test_valid_lines(ftputil.stat.MSParser, lines, expected_stat_results) def test_valid_ms_lines_four_digit_year(self): # See http://ftputil.sschwarzer.net/trac/ticket/67 lines = [ "10-19-2012 03:13PM SYNCDEST", "10-19-2012 03:13PM SYNCSOURCE", "10-19-1968 03:13PM SYNC" ] expected_stat_results = [ [16640, None, None, None, None, None, None, None, (2012, 10, 19, 15, 13, 0), None, MINUTE_PRECISION, "SYNCDEST", None], [16640, None, None, None, None, None, None, None, (2012, 10, 19, 15, 13, 0), None, MINUTE_PRECISION, "SYNCSOURCE", None], [16640, None, None, None, None, None, None, None, EPOCH, None, UNKNOWN_PRECISION, "SYNC", None], ] self._test_valid_lines(ftputil.stat.MSParser, lines, expected_stat_results) def test_invalid_ms_lines(self): lines = [ # Neither "" nor a size present "07-27-01 11:16AM Test", # "AM"/"PM" missing "07-17-00 02:08 12266720 test.exe", # Year not an int "07-17-ab 02:08AM 12266720 test.exe", # Month not an int "ab-17-00 02:08AM 12266720 test.exe", # Day not an int "07-ab-00 02:08AM 12266720 test.exe", # Hour not an int "07-17-00 ab:08AM 12266720 test.exe", # Invalid size value "07-17-00 02:08AM 1226672x test.exe" ] self._test_invalid_lines(ftputil.stat.MSParser, lines) # # The following code checks if the decision logic in the Unix # line parser for determining the year works. # def datetime_string(self, time_float): """ Return a datetime string generated from the value in `time_float`. The parameter value is a floating point value as returned by `time.time()`. The returned string is built as if it were from a Unix FTP server (format: MMM dd hh:mm") """ time_tuple = time.localtime(time_float) return time.strftime("%b %d %H:%M", time_tuple) def dir_line(self, time_float): """ Return a directory line as from a Unix FTP server. Most of the contents are fixed, but the timestamp is made from `time_float` (seconds since the epoch, as from `time.time()`). """ line_template = \ "-rw-r--r-- 1 45854 200 4604 {0} index.html" return line_template.format(self.datetime_string(time_float)) def assert_equal_times(self, time1, time2): """ Check if both times (seconds since the epoch) are equal. For the purpose of this test, two times are "equal" if they differ no more than one minute from each other. """ abs_difference = abs(time1 - time2) assert abs_difference <= 60.0, \ "Difference is %s seconds" % abs_difference def _test_time_shift(self, supposed_time_shift, deviation=0.0): """ Check if the stat parser considers the time shift value correctly. `deviation` is the difference between the actual time shift and the supposed time shift, which is rounded to full hours. """ host = test_base.ftp_host_factory() # Explicitly use Unix format parser here. host._stat._parser = ftputil.stat.UnixParser() host.set_time_shift(supposed_time_shift) server_time = time.time() + supposed_time_shift + deviation stat_result = host._stat._parser.parse_line(self.dir_line(server_time), host.time_shift()) self.assert_equal_times(stat_result.st_mtime, server_time) def test_time_shifts(self): """Test correct year depending on time shift value.""" # 1. test: Client and server share the same local time self._test_time_shift(0.0) # 2. test: Server is three hours ahead of client self._test_time_shift(3 * 60 * 60) # 3. test: Client is three hours ahead of server self._test_time_shift(- 3 * 60 * 60) # 4. test: Server is supposed to be three hours ahead, but # is ahead three hours and one minute self._test_time_shift(3 * 60 * 60, 60) # 5. test: Server is supposed to be three hours ahead, but # is ahead three hours minus one minute self._test_time_shift(3 * 60 * 60, -60) # 6. test: Client is supposed to be three hours ahead, but # is ahead three hours and one minute self._test_time_shift(-3 * 60 * 60, -60) # 7. test: Client is supposed to be three hours ahead, but # is ahead three hours minus one minute self._test_time_shift(-3 * 60 * 60, 60) class TestLstatAndStat(object): """ Test `FTPHost.lstat` and `FTPHost.stat` (test currently only implemented for Unix server format). """ def setup_method(self, method): # Most tests in this class need the mock session class with # Unix format, so make this the default. Tests which need # the MS format can overwrite `self.stat` later. self.stat = \ _test_stat(session_factory=mock_ftplib.MockUnixFormatSession) def test_repr(self): """Test if the `repr` result looks like a named tuple.""" stat_result = self.stat._lstat("/home/sschwarzer/chemeng") # Only under Python 2, unicode strings have the `u` prefix. # TODO: Make the value for `st_mtime` robust against DST "time # zone" changes. if ftputil.compat.python_version == 2: expected_result = ( b"StatResult(st_mode=17901, st_ino=None, st_dev=None, " b"st_nlink=2, st_uid=u'45854', st_gid=u'200', st_size=512, " b"st_atime=None, st_mtime=957391200.0, st_ctime=None)") else: expected_result = ( "StatResult(st_mode=17901, st_ino=None, st_dev=None, " "st_nlink=2, st_uid='45854', st_gid='200', st_size=512, " "st_atime=None, st_mtime=957391200.0, st_ctime=None)") assert repr(stat_result) == expected_result def test_failing_lstat(self): """Test whether `lstat` fails for a nonexistent path.""" with pytest.raises(ftputil.error.PermanentError): self.stat._lstat("/home/sschw/notthere") with pytest.raises(ftputil.error.PermanentError): self.stat._lstat("/home/sschwarzer/notthere") def test_lstat_for_root(self): """ Test `lstat` for `/` . Note: `(l)stat` works by going one directory up and parsing the output of an FTP `LIST` command. Unfortunately, it's not possible to do this for the root directory `/`. """ with pytest.raises(ftputil.error.RootDirError) as exc_info: self.stat._lstat("/") # `RootDirError` is "outside" the `FTPOSError` hierarchy. assert not isinstance(exc_info.value, ftputil.error.FTPOSError) del exc_info def test_lstat_one_unix_file(self): """Test `lstat` for a file described in Unix-style format.""" stat_result = self.stat._lstat("/home/sschwarzer/index.html") # Second form is needed for Python 3 assert oct(stat_result.st_mode) in ("0100644", "0o100644") assert stat_result.st_size == 4604 assert stat_result._st_mtime_precision == 60 def test_lstat_one_ms_file(self): """Test `lstat` for a file described in DOS-style format.""" self.stat = _test_stat(session_factory=mock_ftplib.MockMSFormatSession) stat_result = self.stat._lstat("/home/msformat/abcd.exe") assert stat_result._st_mtime_precision == 60 def test_lstat_one_unix_dir(self): """Test `lstat` for a directory described in Unix-style format.""" stat_result = self.stat._lstat("/home/sschwarzer/scios2") # Second form is needed for Python 3 assert oct(stat_result.st_mode) in ("042755", "0o42755") assert stat_result.st_ino is None assert stat_result.st_dev is None assert stat_result.st_nlink == 6 assert stat_result.st_uid == "45854" assert stat_result.st_gid == "200" assert stat_result.st_size == 512 assert stat_result.st_atime is None assert (stat_result.st_mtime == stat_tuple_to_seconds((1999, 9, 20, 0, 0, 0))) assert stat_result.st_ctime is None assert stat_result._st_mtime_precision == 24*60*60 assert stat_result == (17901, None, None, 6, "45854", "200", 512, None, stat_tuple_to_seconds((1999, 9, 20, 0, 0, 0)), None) def test_lstat_one_ms_dir(self): """Test `lstat` for a directory described in DOS-style format.""" self.stat = _test_stat(session_factory=mock_ftplib.MockMSFormatSession) stat_result = self.stat._lstat("/home/msformat/WindowsXP") assert stat_result._st_mtime_precision == 60 def test_lstat_via_stat_module(self): """Test `lstat` indirectly via `stat` module.""" stat_result = self.stat._lstat("/home/sschwarzer/") assert stat.S_ISDIR(stat_result.st_mode) def test_stat_following_link(self): """Test `stat` when invoked on a link.""" # Simple link stat_result = self.stat._stat("/home/link") assert stat_result.st_size == 4604 # Link pointing to a link stat_result = self.stat._stat("/home/python/link_link") assert stat_result.st_size == 4604 stat_result = self.stat._stat("../python/link_link") assert stat_result.st_size == 4604 # Recursive link structures with pytest.raises(ftputil.error.PermanentError): self.stat._stat("../python/bad_link") with pytest.raises(ftputil.error.PermanentError): self.stat._stat("/home/bad_link") # # Test automatic switching of Unix/MS parsers # def test_parser_switching_with_permanent_error(self): """Test non-switching of parser format with `PermanentError`.""" self.stat = _test_stat(session_factory=mock_ftplib.MockMSFormatSession) assert self.stat._allow_parser_switching is True # With these directory contents, we get a `ParserError` for # the Unix parser first, so `_allow_parser_switching` can be # switched off no matter whether we got a `PermanentError` # afterward or not. with pytest.raises(ftputil.error.PermanentError): self.stat._lstat("/home/msformat/nonexistent") assert self.stat._allow_parser_switching is False def test_parser_switching_default_to_unix(self): """Test non-switching of parser format; stay with Unix.""" assert self.stat._allow_parser_switching is True assert isinstance(self.stat._parser, ftputil.stat.UnixParser) stat_result = self.stat._lstat("/home/sschwarzer/index.html") # The Unix parser worked, so keep it. assert isinstance(self.stat._parser, ftputil.stat.UnixParser) assert self.stat._allow_parser_switching is False def test_parser_switching_to_ms(self): """Test switching of parser from Unix to MS format.""" self.stat = _test_stat(session_factory=mock_ftplib.MockMSFormatSession) assert self.stat._allow_parser_switching is True assert isinstance(self.stat._parser, ftputil.stat.UnixParser) # Parsing the directory `/home/msformat` with the Unix parser # fails, so switch to the MS parser. stat_result = self.stat._lstat("/home/msformat/abcd.exe") assert isinstance(self.stat._parser, ftputil.stat.MSParser) assert self.stat._allow_parser_switching is False assert stat_result._st_name == "abcd.exe" assert stat_result.st_size == 12266720 def test_parser_switching_regarding_empty_dir(self): """Test switching of parser if a directory is empty.""" self.stat = _test_stat(session_factory=mock_ftplib.MockMSFormatSession) assert self.stat._allow_parser_switching is True # When the directory we're looking into doesn't give us any # lines we can't decide whether the first parser worked, # because it wasn't applied. So keep the parser for now. result = self.stat._listdir("/home/msformat/XPLaunch/empty") assert result == [] assert self.stat._allow_parser_switching is True assert isinstance(self.stat._parser, ftputil.stat.UnixParser) class TestListdir(object): """Test `FTPHost.listdir`.""" def setup_method(self, method): self.stat = \ _test_stat(session_factory=mock_ftplib.MockUnixFormatSession) def test_failing_listdir(self): """Test failing `FTPHost.listdir`.""" with pytest.raises(ftputil.error.PermanentError): self.stat._listdir("notthere") def test_succeeding_listdir(self): """Test succeeding `FTPHost.listdir`.""" # Do we have all expected "files"? assert len(self.stat._listdir(".")) == 9 # Have they the expected names? expected = ("chemeng download image index.html os2 " "osup publications python scios2").split() remote_file_list = self.stat._listdir(".") for file in expected: assert file in remote_file_list ftputil-3.4/test/test_with_statement.py0000644000175000017470000000567613135116467017757 0ustar debiandebian# Copyright (C) 2008-2016, Stefan Schwarzer # and ftputil contributors (see `doc/contributors.txt`) # See the file LICENSE for licensing terms. from __future__ import unicode_literals import pytest import ftputil.error from test import test_base from test.test_file import InaccessibleDirSession, ReadMockSession from test.test_host import FailOnLoginSession # Exception raised by client code, i. e. code using ftputil. Used to # test the behavior in case of client exceptions. class ClientCodeException(Exception): pass # # Test cases # class TestHostContextManager(object): def test_normal_operation(self): with test_base.ftp_host_factory() as host: assert host.closed is False assert host.closed is True def test_ftputil_exception(self): with pytest.raises(ftputil.error.FTPOSError): with test_base.ftp_host_factory(FailOnLoginSession) as host: pass # We arrived here, that's fine. Because the `FTPHost` object # wasn't successfully constructed, the assignment to `host` # shouldn't have happened. assert "host" not in locals() def test_client_code_exception(self): try: with test_base.ftp_host_factory() as host: assert host.closed is False raise ClientCodeException() except ClientCodeException: assert host.closed is True else: pytest.fail("`ClientCodeException` not raised") class TestFileContextManager(object): def test_normal_operation(self): with test_base.ftp_host_factory( session_factory=ReadMockSession) as host: with host.open("dummy", "r") as fobj: assert fobj.closed is False data = fobj.readline() assert data == "line 1\n" assert fobj.closed is False assert fobj.closed is True def test_ftputil_exception(self): with test_base.ftp_host_factory( session_factory=InaccessibleDirSession) as host: with pytest.raises(ftputil.error.FTPIOError): # This should fail since the directory isn't # accessible by definition. with host.open("/inaccessible/new_file", "w") as fobj: pass # The file construction shouldn't have succeeded, so `fobj` # should be absent from the local namespace. assert "fobj" not in locals() def test_client_code_exception(self): with test_base.ftp_host_factory( session_factory=ReadMockSession) as host: try: with host.open("dummy", "r") as fobj: assert fobj.closed is False raise ClientCodeException() except ClientCodeException: assert fobj.closed is True else: pytest.fail("`ClientCodeException` not raised") ftputil-3.4/test/test_stat_cache.py0000644000175000017470000000646313135116467017011 0ustar debiandebian# Copyright (C) 2006-2016, Stefan Schwarzer # and ftputil contributors (see `doc/contributors.txt`) # See the file LICENSE for licensing terms. from __future__ import unicode_literals import time import pytest import ftputil.error import ftputil.stat_cache from test import test_base class TestStatCache(object): def setup_method(self, method): self.cache = ftputil.stat_cache.StatCache() def test_get_set(self): with pytest.raises(ftputil.error.CacheMissError): self.cache.__getitem__("/path") self.cache["/path"] = "test" assert self.cache["/path"] == "test" def test_invalidate(self): # Don't raise a `CacheMissError` for missing paths self.cache.invalidate("/path") self.cache["/path"] = "test" self.cache.invalidate("/path") assert len(self.cache) == 0 def test_clear(self): self.cache["/path1"] = "test1" self.cache["/path2"] = "test2" self.cache.clear() assert len(self.cache) == 0 def test_contains(self): self.cache["/path1"] = "test1" assert "/path1" in self.cache assert "/path2" not in self.cache def test_len(self): assert len(self.cache) == 0 self.cache["/path1"] = "test1" self.cache["/path2"] = "test2" assert len(self.cache) == 2 def test_resize(self): self.cache.resize(100) # Don't grow the cache beyond it's set size. for i in range(150): self.cache["/{0:d}".format(i)] = i assert len(self.cache) == 100 def test_max_age1(self): """Set expiration after setting a cache item.""" self.cache["/path1"] = "test1" # Expire after one second self.cache.max_age = 1 time.sleep(0.5) # Should still be present assert self.cache["/path1"] == "test1" time.sleep(0.6) # Should have expired (_setting_ the cache counts) with pytest.raises(ftputil.error.CacheMissError): self.cache.__getitem__("/path1") def test_max_age2(self): """Set expiration before setting a cache item.""" # Expire after one second self.cache.max_age = 1 self.cache["/path1"] = "test1" time.sleep(0.5) # Should still be present assert self.cache["/path1"] == "test1" time.sleep(0.6) # Should have expired (_setting_ the cache counts) with pytest.raises(ftputil.error.CacheMissError): self.cache.__getitem__("/path1") def test_disabled(self): self.cache["/path1"] = "test1" self.cache.disable() self.cache["/path2"] = "test2" with pytest.raises(ftputil.error.CacheMissError): self.cache.__getitem__("/path1") with pytest.raises(ftputil.error.CacheMissError): self.cache.__getitem__("/path2") assert len(self.cache) == 1 # Don't raise a `CacheMissError` for missing paths. self.cache.invalidate("/path2") def test_cache_size_zero(self): host = test_base.ftp_host_factory() with pytest.raises(ValueError): host.stat_cache.resize(0) # If bug #38 was present, this raised an `IndexError`. items = host.listdir(host.curdir) assert items[:3] == ["chemeng", "download", "image"] ftputil-3.4/test/test_base.py0000644000175000017470000000117413135113162015604 0ustar debiandebian# Copyright (C) 2003-2013, Stefan Schwarzer # and ftputil contributors (see `doc/contributors.txt`) # See the file LICENSE for licensing terms. from __future__ import unicode_literals import ftputil from test import mock_ftplib # Factory to produce `FTPHost`-like classes from a given `FTPHost` # class and (usually) a given `MockSession` class. def ftp_host_factory(session_factory=mock_ftplib.MockUnixFormatSession, ftp_host_class=ftputil.FTPHost): return ftp_host_class("dummy_host", "dummy_user", "dummy_password", session_factory=session_factory) ftputil-3.4/test/test_file.py0000644000175000017470000002334013175127042015616 0ustar debiandebian# Copyright (C) 2002-2016, Stefan Schwarzer # and ftputil contributors (see `doc/contributors.txt`) # See the file LICENSE for licensing terms. from __future__ import unicode_literals import ftplib import pytest import ftputil.compat import ftputil.error from test import mock_ftplib from test import test_base # # Several customized `MockSession` classes # class ReadMockSession(mock_ftplib.MockSession): mock_file_content = b"line 1\r\nanother line\r\nyet another line" class ReadMockSessionWithMoreNewlines(mock_ftplib.MockSession): mock_file_content = b"\r\n".join(map(ftputil.compat.bytes_type, range(20))) class InaccessibleDirSession(mock_ftplib.MockSession): _login_dir = "/inaccessible" def pwd(self): return self._login_dir def cwd(self, dir): if dir in (self._login_dir, self._login_dir + "/"): raise ftplib.error_perm else: super(InaccessibleDirSession, self).cwd(dir) class TestFileOperations(object): """Test operations with file-like objects.""" def test_inaccessible_dir(self): """Test whether opening a file at an invalid location fails.""" host = test_base.ftp_host_factory( session_factory=InaccessibleDirSession) with pytest.raises(ftputil.error.FTPIOError): host.open("/inaccessible/new_file", "w") def test_caching(self): """Test whether `FTPFile` cache of `FTPHost` object works.""" host = test_base.ftp_host_factory() assert len(host._children) == 0 path1 = "path1" path2 = "path2" # Open one file and inspect cache. file1 = host.open(path1, "w") child1 = host._children[0] assert len(host._children) == 1 assert not child1._file.closed # Open another file. file2 = host.open(path2, "w") child2 = host._children[1] assert len(host._children) == 2 assert not child2._file.closed # Close first file. file1.close() assert len(host._children) == 2 assert child1._file.closed assert not child2._file.closed # Re-open first child's file. file1 = host.open(path1, "w") child1_1 = file1._host # Check if it's reused. assert child1 is child1_1 assert not child1._file.closed assert not child2._file.closed # Close second file. file2.close() assert child2._file.closed def test_write_to_directory(self): """Test whether attempting to write to a directory fails.""" host = test_base.ftp_host_factory() with pytest.raises(ftputil.error.FTPIOError): host.open("/home/sschwarzer", "w") def test_binary_read(self): """Read data from a binary file.""" host = test_base.ftp_host_factory(session_factory=ReadMockSession) with host.open("some_file", "rb") as fobj: data = fobj.read() assert data == ReadMockSession.mock_file_content def test_binary_write(self): """Write binary data with `write`.""" host = test_base.ftp_host_factory() data = b"\000a\001b\r\n\002c\003\n\004\r\005" with host.open("dummy", "wb") as output: output.write(data) child_data = mock_ftplib.content_of("dummy") expected_data = data assert child_data == expected_data def test_ascii_read(self): """Read ASCII text with plain `read`.""" host = test_base.ftp_host_factory(session_factory=ReadMockSession) with host.open("dummy", "r") as input_: data = input_.read(0) assert data == "" data = input_.read(3) assert data == "lin" data = input_.read(7) assert data == "e 1\nano" data = input_.read() assert data == "ther line\nyet another line" data = input_.read() assert data == "" def test_ascii_write(self): """Write ASCII text with `write`.""" host = test_base.ftp_host_factory() data = " \nline 2\nline 3" with host.open("dummy", "w", newline="\r\n") as output: output.write(data) child_data = mock_ftplib.content_of("dummy") # This corresponds to the byte stream, so expect a `bytes` object. expected_data = b" \r\nline 2\r\nline 3" assert child_data == expected_data # TODO: Add tests with given encoding and possibly buffering. def test_ascii_writelines(self): """Write ASCII text with `writelines`.""" host = test_base.ftp_host_factory() data = [" \n", "line 2\n", "line 3"] backup_data = data[:] output = host.open("dummy", "w", newline="\r\n") output.writelines(data) output.close() child_data = mock_ftplib.content_of("dummy") expected_data = b" \r\nline 2\r\nline 3" assert child_data == expected_data # Ensure that the original data was not modified. assert data == backup_data def test_binary_readline(self): """Read binary data with `readline`.""" host = test_base.ftp_host_factory(session_factory=ReadMockSession) input_ = host.open("dummy", "rb") data = input_.readline(3) assert data == b"lin" data = input_.readline(10) assert data == b"e 1\r\n" data = input_.readline(13) assert data == b"another line\r" data = input_.readline() assert data == b"\n" data = input_.readline() assert data == b"yet another line" data = input_.readline() assert data == b"" input_.close() def test_ascii_readline(self): """Read ASCII text with `readline`.""" host = test_base.ftp_host_factory(session_factory=ReadMockSession) input_ = host.open("dummy", "r") data = input_.readline(3) assert data == "lin" data = input_.readline(10) assert data == "e 1\n" data = input_.readline(13) assert data == "another line\n" data = input_.readline() assert data == "yet another line" data = input_.readline() assert data == "" input_.close() def test_ascii_readlines(self): """Read ASCII text with `readlines`.""" host = test_base.ftp_host_factory(session_factory=ReadMockSession) input_ = host.open("dummy", "r") data = input_.read(3) assert data == "lin" data = input_.readlines() assert data == ["e 1\n", "another line\n", "yet another line"] input_.close() def test_binary_iterator(self): """ Test the iterator interface of `FTPFile` objects (without newline conversion. """ host = test_base.ftp_host_factory(session_factory=ReadMockSession) input_ = host.open("dummy", "rb") input_iterator = iter(input_) assert next(input_iterator) == b"line 1\r\n" assert next(input_iterator) == b"another line\r\n" assert next(input_iterator) == b"yet another line" with pytest.raises(StopIteration): input_iterator.__next__() input_.close() def test_ascii_iterator(self): """ Test the iterator interface of `FTPFile` objects (with newline conversion). """ host = test_base.ftp_host_factory(session_factory=ReadMockSession) input_ = host.open("dummy") input_iterator = iter(input_) assert next(input_iterator) == "line 1\n" assert next(input_iterator) == "another line\n" assert next(input_iterator) == "yet another line" with pytest.raises(StopIteration): input_iterator.__next__() input_.close() def test_read_unknown_file(self): """Test whether reading a file which isn't there fails.""" host = test_base.ftp_host_factory() with pytest.raises(ftputil.error.FTPIOError): host.open("notthere", "r") class TestAvailableChild(object): def _failing_pwd(self, exception_class): """ Return a function that will be used instead of the `session.pwd` and will raise the exception `exception_to_raise`. """ def new_pwd(): raise exception_class("") return new_pwd def _test_with_pwd_error(self, exception_class): """ Test if reusing a child session fails because of `child_host._session.pwd` raising an exception of type `exception_class`. """ host = test_base.ftp_host_factory() # Implicitly create a child session. with host.open("/home/older") as _: pass assert len(host._children) == 1 # Make sure reusing the previous child session will fail. host._children[0]._session.pwd = self._failing_pwd(exception_class) # Try to create a new file. Since `pwd` now raises an # exception, a new child session should be created. with host.open("home/older") as _: pass assert len(host._children) == 2 def test_pwd_with_error_temp(self): """ Test if an `error_temp` in `_session.pwd` skips the child session. """ self._test_with_pwd_error(ftplib.error_temp) def test_pwd_with_error_reply(self): """ Test if an `error_reply` in `_session.pwd` skips the child session. """ self._test_with_pwd_error(ftplib.error_reply) def test_pwd_with_OSError(self): """ Test if an `OSError` in `_session.pwd` skips the child session. """ self._test_with_pwd_error(OSError) def test_pwd_with_EOFError(self): """ Test if an `EOFError` in `_session.pwd` skips the child session. """ self._test_with_pwd_error(EOFError) ftputil-3.4/test/test_file_transfer.py0000644000175000017470000001062113135116467017525 0ustar debiandebian# Copyright (C) 2010-2016, Stefan Schwarzer # and ftputil contributors (see `doc/contributors.txt`) # See the file LICENSE for licensing terms. from __future__ import unicode_literals import io import random import pytest import ftputil.compat import ftputil.file_transfer import ftputil.stat class MockFile(object): """Class compatible with `LocalFile` and `RemoteFile`.""" def __init__(self, mtime, mtime_precision): self._mtime = mtime self._mtime_precision = mtime_precision def mtime(self): return self._mtime def mtime_precision(self): return self._mtime_precision class TestTimestampComparison(object): def test_source_is_newer_than_target(self): """ Test whether the source is newer than the target, i. e. if the file should be transferred. """ # Define some time units/precisions. second = 1.0 minute = 60.0 * second hour = 60 * minute day = 24 * hour unknown = ftputil.stat.UNKNOWN_PRECISION # Define input arguments; modification datetimes are in seconds. # Fields are source datetime/precision, target datetime/precision, # expected comparison result. file_data = [ # Non-overlapping modification datetimes/precisions (1000.0, second, 900.0, second, True), (900.0, second, 1000.0, second, False), # Equal modification datetimes/precisions (if in doubt, transfer) (1000.0, second, 1000.0, second, True), # Just touching intervals (1000.0, second, 1000.0+second, minute, True), (1000.0+second, minute, 1000.0, second, True), # Other overlapping intervals (10000.0-0.5*hour, hour, 10000.0, day, True), (10000.0+0.5*hour, hour, 10000.0, day, True), (10000.0+0.2*hour, 0.2*hour, 10000.0, hour, True), (10000.0-0.2*hour, 2*hour, 10000.0, hour, True), # Unknown precision (1000.0, None, 1000.0, second, True), (1000.0, second, 1000.0, None, True), (1000.0, None, 1000.0, None, True), ] for (source_mtime, source_mtime_precision, target_mtime, target_mtime_precision, expected_result) in file_data: source_file = MockFile(source_mtime, source_mtime_precision) target_file = MockFile(target_mtime, target_mtime_precision) result = ftputil.file_transfer.source_is_newer_than_target( source_file, target_file) assert result == expected_result class FailingStringIO(io.BytesIO): """Mock class to test whether exceptions are passed on.""" # Kind of nonsense; we just want to see this exception raised. expected_exception = IndexError def read(self, count): raise self.expected_exception class TestChunkwiseTransfer(object): def _random_string(self, count): """Return a `BytesIO` object containing `count` "random" bytes.""" ints = (random.randint(0, 255) for i in range(count)) return ftputil.compat.bytes_from_ints(ints) def test_chunkwise_transfer_without_remainder(self): """Check if we get four chunks with 256 Bytes each.""" data = self._random_string(1024) fobj = io.BytesIO(data) chunks = list(ftputil.file_transfer.chunks(fobj, 256)) assert len(chunks) == 4 assert chunks[0] == data[:256] assert chunks[1] == data[256:512] assert chunks[2] == data[512:768] assert chunks[3] == data[768:1024] def test_chunkwise_transfer_with_remainder(self): """Check if we get three chunks with 256 Bytes and one with 253.""" data = self._random_string(1021) fobj = io.BytesIO(data) chunks = list(ftputil.file_transfer.chunks(fobj, 256)) assert len(chunks) == 4 assert chunks[0] == data[:256] assert chunks[1] == data[256:512] assert chunks[2] == data[512:768] assert chunks[3] == data[768:1021] def test_chunkwise_transfer_with_exception(self): """Check if we see the exception raised during reading.""" data = self._random_string(1024) fobj = FailingStringIO(data) iterator = ftputil.file_transfer.chunks(fobj, 256) with pytest.raises(FailingStringIO.expected_exception): next(iterator) ftputil-3.4/test/test_path.py0000644000175000017470000002121513175164400015631 0ustar debiandebian# encoding: utf-8 # Copyright (C) 2003-2016, Stefan Schwarzer # and ftputil contributors (see `doc/contributors.txt`) # See the file LICENSE for licensing terms. from __future__ import unicode_literals import ftplib import time import pytest import ftputil import ftputil.compat import ftputil.error import ftputil.tool from test import mock_ftplib from test import test_base class FailingFTPHost(ftputil.FTPHost): def _dir(self, path): raise ftputil.error.FTPOSError("simulate a failure, e. g. timeout") # Mock session, used for testing an inaccessible login directory class SessionWithInaccessibleLoginDirectory(mock_ftplib.MockSession): def cwd(self, dir): # Assume that `dir` is the inaccessible login directory. raise ftplib.error_perm("can't change into this directory") class TestPath(object): """Test operations in `FTPHost.path`.""" # TODO: Add unit tests for changes for ticket #113 # (commits [b4c9b089b6b8] and [4027740cdd2d]). def test_regular_isdir_isfile_islink(self): """Test regular `FTPHost._Path.isdir/isfile/islink`.""" host = test_base.ftp_host_factory() testdir = "/home/sschwarzer" host.chdir(testdir) # Test a path which isn't there. assert not host.path.isdir("notthere") assert not host.path.isfile("notthere") assert not host.path.islink("notthere") # This checks additional code (see ticket #66). assert not host.path.isdir("/notthere/notthere") assert not host.path.isfile("/notthere/notthere") assert not host.path.islink("/notthere/notthere") # Test a directory. assert host.path.isdir(testdir) assert not host.path.isfile(testdir) assert not host.path.islink(testdir) # Test a file. testfile = "/home/sschwarzer/index.html" assert not host.path.isdir(testfile) assert host.path.isfile(testfile) assert not host.path.islink(testfile) # Test a link. Since the link target of `osup` doesn't exist, # neither `isdir` nor `isfile` return `True`. testlink = "/home/sschwarzer/osup" assert not host.path.isdir(testlink) assert not host.path.isfile(testlink) assert host.path.islink(testlink) def test_workaround_for_spaces(self): """Test whether the workaround for space-containing paths is used.""" host = test_base.ftp_host_factory() testdir = "/home/sschwarzer" host.chdir(testdir) # Test a file name containing spaces. testfile = "/home/dir with spaces/file with spaces" assert not host.path.isdir(testfile) assert host.path.isfile(testfile) assert not host.path.islink(testfile) def test_inaccessible_home_directory_and_whitespace_workaround(self): "Test combination of inaccessible home directory + whitespace in path." host = test_base.ftp_host_factory( session_factory=SessionWithInaccessibleLoginDirectory) with pytest.raises(ftputil.error.InaccessibleLoginDirError): host._dir("/home dir") def test_isdir_isfile_islink_with_dir_failure(self): """ Test failing `FTPHost._Path.isdir/isfile/islink` because of failing `_dir` call. """ host = test_base.ftp_host_factory(ftp_host_class=FailingFTPHost) testdir = "/home/sschwarzer" host.chdir(testdir) # Test if exceptions are propagated. FTPOSError = ftputil.error.FTPOSError with pytest.raises(FTPOSError): host.path.isdir("index.html") with pytest.raises(FTPOSError): host.path.isfile("index.html") with pytest.raises(FTPOSError): host.path.islink("index.html") def test_isdir_isfile_with_infinite_link_chain(self): """ Test if `isdir` and `isfile` return `False` if they encounter an infinite link chain. """ host = test_base.ftp_host_factory() assert host.path.isdir("/home/bad_link") is False assert host.path.isfile("/home/bad_link") is False def test_exists(self): """Test `FTPHost.path.exists`.""" # Regular use of `exists` host = test_base.ftp_host_factory() testdir = "/home/sschwarzer" host.chdir(testdir) assert host.path.exists("index.html") assert not host.path.exists("notthere") # Test if exceptions are propagated. host = test_base.ftp_host_factory(ftp_host_class=FailingFTPHost) with pytest.raises(ftputil.error.FTPOSError): host.path.exists("index.html") class TestAcceptEitherBytesOrUnicode(object): def setup_method(self, method): self.host = test_base.ftp_host_factory() def _test_method_string_types(self, method, path): expected_type = type(path) assert isinstance(method(path), expected_type) def test_methods_that_take_and_return_one_string(self): """ Test whether the same string type as for the argument is returned. """ bytes_type = ftputil.compat.bytes_type unicode_type = ftputil.compat.unicode_type method_names = ("abspath basename dirname join normcase normpath". split()) for method_name in method_names: method = getattr(self.host.path, method_name) self._test_method_string_types(method, "/") self._test_method_string_types(method, ".") self._test_method_string_types(method, b"/") self._test_method_string_types(method, b".") def test_methods_that_take_a_string_and_return_a_bool(self): """Test whether the methods accept byte and unicode strings.""" host = self.host as_bytes = ftputil.tool.as_bytes host.chdir("/home/file_name_test") # `isabs` assert not host.path.isabs("ä") assert not host.path.isabs(as_bytes("ä")) # `exists` assert host.path.exists("ä") assert host.path.exists(as_bytes("ä")) # `isdir`, `isfile`, `islink` assert host.path.isdir("ä") assert host.path.isdir(as_bytes("ä")) assert host.path.isfile("ö") assert host.path.isfile(as_bytes("ö")) assert host.path.islink("ü") assert host.path.islink(as_bytes("ü")) def test_join(self): """ Test whether `FTPHost.path.join` accepts only arguments of the same string type and returns the same string type. """ join = self.host.path.join as_bytes = ftputil.tool.as_bytes # Only unicode parts = list("äöü") result = join(*parts) assert result == "ä/ö/ü" # Need explicit type check for Python 2 assert isinstance(result, ftputil.compat.unicode_type) # Only bytes parts = [as_bytes(s) for s in "äöü"] result = join(*parts) assert result == as_bytes("ä/ö/ü") # Need explicit type check for Python 2 assert isinstance(result, ftputil.compat.bytes_type) # Mixture of unicode and bytes parts = ["ä", as_bytes("ö")] with pytest.raises(TypeError): join(*parts) parts = [as_bytes("ä"), as_bytes("ö"), "ü"] with pytest.raises(TypeError): join(*parts) def test_getmtime(self): """ Test whether `FTPHost.path.getmtime` accepts byte and unicode paths. """ host = self.host as_bytes = ftputil.tool.as_bytes host.chdir("/home/file_name_test") # We don't care about the _exact_ time, so don't bother with # timezone differences. Instead, do a simple sanity check. day = 24 * 60 * 60 # seconds expected_mtime = time.mktime((2000, 5, 29, 0, 0, 0, 0, 0, 0)) mtime_makes_sense = (lambda mtime: expected_mtime - day <= mtime <= expected_mtime + day) assert mtime_makes_sense(host.path.getmtime("ä")) assert mtime_makes_sense(host.path.getmtime(as_bytes("ä"))) def test_getsize(self): """ Test whether `FTPHost.path.getsize` accepts byte and unicode paths. """ host = self.host as_bytes = ftputil.tool.as_bytes host.chdir("/home/file_name_test") assert host.path.getsize("ä") == 512 assert host.path.getsize(as_bytes("ä")) == 512 def test_walk(self): """Test whether `FTPHost.path.walk` accepts bytes and unicode paths.""" host = self.host as_bytes = ftputil.tool.as_bytes def noop(arg, top, names): del names[:] host.path.walk("ä", noop, None) host.path.walk(as_bytes("ä"), noop, None) ftputil-3.4/test/test_sync.py0000644000175000017470000000656213135116467015667 0ustar debiandebian# Copyright (C) 2007-2013, Stefan Schwarzer # and ftputil contributors (see `doc/contributors.txt`) # See the file LICENSE for licensing terms. from __future__ import absolute_import from __future__ import unicode_literals import io import ntpath import os import shutil import ftputil import ftputil.sync # Assume the test subdirectories are or will be in the current directory. TEST_ROOT = os.getcwd() class TestLocalToLocal(object): def setup_method(self, method): if not os.path.exists("test_empty"): os.mkdir("test_empty") if os.path.exists("test_target"): shutil.rmtree("test_target") os.mkdir("test_target") def test_sync_empty_dir(self): source = ftputil.sync.LocalHost() target = ftputil.sync.LocalHost() syncer = ftputil.sync.Syncer(source, target) source_dir = os.path.join(TEST_ROOT, "test_empty") target_dir = os.path.join(TEST_ROOT, "test_target") syncer.sync(source_dir, target_dir) def test_source_with_and_target_without_slash(self): source = ftputil.sync.LocalHost() target = ftputil.sync.LocalHost() syncer = ftputil.sync.Syncer(source, target) source_dir = os.path.join(TEST_ROOT, "test_source/") target_dir = os.path.join(TEST_ROOT, "test_target") syncer.sync(source_dir, target_dir) # Helper classes for `TestUploadFromWindows` class LocalWindowsHost(ftputil.sync.LocalHost): def __init__(self): self.path = ntpath self.sep = "\\" def open(self, path, mode): # Just return a dummy file object. return io.StringIO("") def walk(self, root): """ Return a list of tuples as `os.walk`, but use tuples as if the directory structure was dir1 dir11 file1 file2 where is the string passed in as `root`. """ join = ntpath.join return [(root, [join(root, "dir1")], []), (join(root, "dir1"), ["dir11"], ["file1", "file2"]) ] class DummyFTPSession(object): def pwd(self): return "/" class DummyFTPPath(object): def abspath(self, path): # Don't care here if the path is absolute or not. return path def isdir(self, path): return ("dir" in path) def isfile(self, path): return ("file" in path) class ArgumentCheckingFTPHost(ftputil.FTPHost): def __init__(self, *args, **kwargs): super(ArgumentCheckingFTPHost, self).__init__(*args, **kwargs) self.path = DummyFTPPath() def _make_session(self, *args, **kwargs): return DummyFTPSession() def mkdir(self, path): assert "\\" not in path def open(self, path, mode): assert "\\" not in path return io.StringIO("") class TestUploadFromWindows(object): def test_no_mixed_separators(self): source = LocalWindowsHost() target = ArgumentCheckingFTPHost() local_root = ntpath.join("some", "directory") syncer = ftputil.sync.Syncer(source, target) # If the following call raises any `AssertionError`s, the # test framework will catch them and show them. syncer.sync(local_root, "not_used_by_ArgumentCheckingFTPHost") ftputil-3.4/test/test_error.py0000644000175000017470000000567413135116467016047 0ustar debiandebian# encoding: utf-8 # Copyright (C) 2002-2016, Stefan Schwarzer # and ftputil contributors (see `doc/contributors.txt`) # See the file LICENSE for licensing terms. from __future__ import unicode_literals import ftplib import pytest import ftputil.error class TestFTPErrorArguments(object): """ The `*Error` constructors should accept either a byte string or a unicode string. """ def test_bytestring_argument(self): # An umlaut as latin-1 character io_error = ftputil.error.FTPIOError(b"\xe4") os_error = ftputil.error.FTPOSError(b"\xe4") def test_unicode_argument(self): # An umlaut as unicode character io_error = ftputil.error.FTPIOError("\xe4") os_error = ftputil.error.FTPOSError("\xe4") class TestErrorConversion(object): def callee(self): raise ftplib.error_perm() def test_ftplib_error_to_ftp_os_error(self): """ Ensure the `ftplib` exception isn't used as `FTPOSError` argument. """ with pytest.raises(ftputil.error.FTPOSError) as exc_info: with ftputil.error.ftplib_error_to_ftp_os_error: self.callee() exc = exc_info.value assert not (exc.args and isinstance(exc.args[0], ftplib.error_perm)) del exc_info def test_ftplib_error_to_ftp_os_error_non_ascii_server_message(self): """ Test that we don't get a `UnicodeDecodeError` if the server sends a message containing non-ASCII characters. """ # See ticket #77. message = \ ftputil.tool.as_bytes( "Não é possível criar um arquivo já existente.") with pytest.raises(ftputil.error.PermanentError): with ftputil.error.ftplib_error_to_ftp_os_error: raise ftplib.error_perm(message) def test_ftplib_error_to_ftp_io_error(self): """ Ensure the `ftplib` exception isn't used as `FTPIOError` argument. """ with pytest.raises(ftputil.error.FTPIOError) as exc_info: with ftputil.error.ftplib_error_to_ftp_io_error: self.callee() exc = exc_info.value assert not (exc.args and isinstance(exc.args[0], ftplib.error_perm)) del exc_info def test_error_message_reuse(self): """ Test if the error message string is retained if the caught exception has more than one element in `args`. """ # See ticket #76. with pytest.raises(ftputil.error.FTPOSError) as exc_info: # Format "host:port" doesn't work. host = ftputil.FTPHost("localhost:21", "", "") exc = exc_info.value # The error message may be different for different Python # versions. assert ( "No address associated with hostname" in str(exc) or "Name or service not known" in str(exc)) del exc_info ftputil-3.4/test/__init__.py0000644000175000017470000000000013135116467015370 0ustar debiandebianftputil-3.4/test/test_public_servers.py0000644000175000017470000001602213200526650017722 0ustar debiandebian# Copyright (C) 2009-2016, Stefan Schwarzer # and ftputil contributors (see `doc/contributors.txt`) # See the file LICENSE for licensing terms. from __future__ import print_function from __future__ import unicode_literals import os import socket import subprocess import pytest import ftputil import test def email_address(): """ Return the email address used to identify the client to an FTP server. If the hostname is "warpy", use my (Stefan's) email address, else try to use the content of the `$EMAIL` environment variable. If that doesn't exist, use a dummy address. """ hostname = socket.gethostname() if hostname == "warpy": email = "sschwarzer@sschwarzer.net" else: dummy_address = "anonymous@example.com" email = os.environ.get("EMAIL", dummy_address) if not email: # Environment variable exists but content is an empty string email = dummy_address return email EMAIL = email_address() def ftp_client_listing(server, directory): """ Log into the FTP server `server` using the command line client, then change to the `directory` and retrieve a listing with "dir". Return the list of items found as an `os.listdir` would return it. """ # The `-n` option prevents an auto-login. ftp_popen = subprocess.Popen(["ftp", "-n", server], stdin=subprocess.PIPE, stdout=subprocess.PIPE, universal_newlines=True) commands = ["user anonymous {0}".format(EMAIL), "dir", "bye"] if directory: # Change to this directory before calling "dir". commands.insert(1, "cd {0}".format(directory)) input_ = "\n".join(commands) stdout, unused_stderr = ftp_popen.communicate(input_) # Collect the directory/file names from the listing's text names = [] for line in stdout.strip().split("\n"): if line.startswith("total ") or line.startswith("Trying "): continue parts = line.split() if parts[-2] == "->": # Most likely a link name = parts[-3] else: name = parts[-1] names.append(name) # Remove entries for current and parent directory since they # aren't included in the result of `FTPHost.listdir` either. names = [name for name in names if name not in (".", "..")] return names class TestPublicServers(object): """ Get directory listings from various public FTP servers with a command line client and ftputil and compare both. An important aspect is to test different "spellings" of the same directory. For example, to list the root directory which is usually set after login, use "" (nothing), ".", "/", "/.", "./.", "././", "..", "../.", "../.." etc. The command line client `ftp` has to be in the path. """ # Implementation note: # # I (Stefan) implement the code so it works with Ubuntu's # client. Other clients may work or not. If you have problems # testing some other client, please send me a (small) patch. # Keep in mind that I don't plan supporting as many FTP # obscure commandline clients as servers. ;-) # List of pairs with server name and a directory "guaranteed # to exist" under the login directory which is assumed to be # the root directory. servers = [# Posix format ("ftp.de.debian.org", "debian"), ("ftp.gnome.org", "pub"), ("ftp.heanet.ie", "pub"), ("ftp.heise.de", "pub"), # DOS/Microsoft format # Do you know any FTP servers that use Microsoft # format? `ftp.microsoft.com` doesn't seem to be # reachable anymore. ] # This data structure contains the initial directories "." and # "DIR" (which will be replaced by a valid directory name for # each server). The list after the initial directory contains # paths that will be queried after changing into the initial # directory. All items in these lists are actually supposed to # yield the same directory contents. paths_table = [ (".", ["", ".", "/", "/.", "./.", "././", "..", "../.", "../..", "DIR/..", "/DIR/../.", "/DIR/../.."]), ("DIR", ["", ".", "/DIR", "/DIR/", "../DIR", "../../DIR"]) ] def inner_test_server(self, server, initial_directory, paths): """ Test one server for one initial directory. Connect to the server `server`; if the string argument `initial_directory` has a true value, change to this directory. Then iterate over all strings in the sequence `paths`, comparing the results of a listdir call with the listing from the command line client. """ canonical_names = ftp_client_listing(server, initial_directory) host = ftputil.FTPHost(server, "anonymous", EMAIL) try: host.chdir(initial_directory) for path in paths: path = path.replace("DIR", initial_directory) # Make sure that we don't recycle directory entries, i. e. # really repeatedly retrieve the directory contents # (shouldn't happen anyway with the current implementation). host.stat_cache.clear() names = host.listdir(path) # Filter out "hidden" names since the FTP command line # client won't include them in its listing either. names = [name for name in names if not ( name.startswith(".") or # The login directory of `ftp.microsoft.com` # contains this "hidden" entry that ftputil # finds but not the FTP command line client. name == "mscomtest" )] failure_message = ("For server {0}, directory {1}: {2} != {3}". format(server, initial_directory, names, canonical_names)) assert names == canonical_names, failure_message finally: host.close() @pytest.mark.slow_test def test_servers(self): """ Test all servers in `self.servers`. For each server, get the listings for the login directory and one other directory which is known to exist. Use different "spellings" to retrieve each list via ftputil and compare with the results gotten with the command line client. """ for server, actual_initial_directory in self.servers: for initial_directory, paths in self.paths_table: initial_directory = initial_directory.replace( "DIR", actual_initial_directory) print(server, initial_directory) self.inner_test_server(server, initial_directory, paths) ftputil-3.4/test/test_real_ftp.py0000644000175000017470000011012413166460744016501 0ustar debiandebian# encoding: UTF-8 # Copyright (C) 2003-2016, Stefan Schwarzer # and ftputil contributors (see `doc/contributors.txt`) # See the file LICENSE for licensing terms. # Execute tests on a real FTP server (other tests use a mock server). # # This test writes some files and directories on the local client and # the remote server. You'll need write access in the login directory. # This test can take a few minutes because it has to wait to test the # timezone calculation. from __future__ import absolute_import from __future__ import unicode_literals import ftplib import functools import gc import operator import os import time import stat import pytest import ftputil.compat import ftputil.error import ftputil.file_transfer import ftputil.session import ftputil.stat_cache import test def utc_local_time_shift(): """ Return the expected time shift in seconds assuming the server uses UTC in its listings and the client uses local time. This is needed because Pure-FTPd meanwhile seems to insist that the displayed time for files is in UTC. """ utc_tuple = time.gmtime() localtime_tuple = time.localtime() # To calculate the correct times shift, we need to ignore the # DST component in the localtime tuple, i. e. set it to 0. localtime_tuple = localtime_tuple[:-1] + (0,) time_shift_in_seconds = (time.mktime(utc_tuple) - time.mktime(localtime_tuple)) # To be safe, round the above value to units of 3600 s (1 hour). return round(time_shift_in_seconds / 3600.0) * 3600 # Difference between local times of server and client. If 0.0, server # and client use the same timezone. #EXPECTED_TIME_SHIFT = utc_local_time_shift() # Pure-FTPd seems to have changed its mind (see docstring of # `utc_local_time_shift`). EXPECTED_TIME_SHIFT = 0.0 class Cleaner(object): """ This class helps remove directories and files which might otherwise be left behind if a test fails in unexpected ways. """ def __init__(self, host): # The test class (probably `RealFTPTest`) and the helper # class share the same `FTPHost` object. self._host = host self._ftp_items = [] def add_dir(self, path): """Schedule a directory with path `path` for removal.""" self._ftp_items.append(("d", self._host.path.abspath(path))) def add_file(self, path): """Schedule a file with path `path` for removal.""" self._ftp_items.append(("f", self._host.path.abspath(path))) def clean(self): """ Remove the directories and files previously remembered. The removal works in reverse order of the scheduling with `add_dir` and `add_file`. Errors due to a removal are ignored. """ self._host.chdir("/") for type_, path in reversed(self._ftp_items): try: if type_ == "d": # If something goes wrong in `rmtree` we might # leave a mess behind. self._host.rmtree(path) elif type_ == "f": # Minor mess if `remove` fails self._host.remove(path) except ftputil.error.FTPError: pass class RealFTPTest(object): def setup_method(self, method): # Server, username, password. self.login_data = ("localhost", "ftptest", "d605581757de5eb56d568a4419f4126e") self.host = ftputil.FTPHost(*self.login_data) self.cleaner = Cleaner(self.host) def teardown_method(self, method): self.cleaner.clean() self.host.close() # # Helper methods # def make_remote_file(self, path): """Create a file on the FTP host.""" self.cleaner.add_file(path) with self.host.open(path, "wb") as file_: # Write something. Otherwise the FTP server might not update # the time of last modification if the file existed before. file_.write(b"\n") def make_local_file(self): """Create a file on the local host (= on the client side).""" with open("_local_file_", "wb") as fobj: fobj.write(b"abc\x12\x34def\t") class TestMkdir(RealFTPTest): def test_mkdir_rmdir(self): host = self.host dir_name = "_testdir_" file_name = host.path.join(dir_name, "_nonempty_") self.cleaner.add_dir(dir_name) # Make dir and check if the directory is there. host.mkdir(dir_name) files = host.listdir(host.curdir) assert dir_name in files # Try to remove a non-empty directory. self.cleaner.add_file(file_name) non_empty = host.open(file_name, "w") non_empty.close() with pytest.raises(ftputil.error.PermanentError): host.rmdir(dir_name) # Remove file. host.unlink(file_name) # `remove` on a directory should fail. try: try: host.remove(dir_name) except ftputil.error.PermanentError as exc: assert str(exc).startswith( "remove/unlink can only delete files") else: pytest.fail("we shouldn't have come here") finally: # Delete empty directory. host.rmdir(dir_name) files = host.listdir(host.curdir) assert dir_name not in files def test_makedirs_without_existing_dirs(self): host = self.host # No `_dir1_` yet assert "_dir1_" not in host.listdir(host.curdir) # Vanilla case, all should go well. host.makedirs("_dir1_/dir2/dir3/dir4") self.cleaner.add_dir("_dir1_") # Check host. assert host.path.isdir("_dir1_") assert host.path.isdir("_dir1_/dir2") assert host.path.isdir("_dir1_/dir2/dir3") assert host.path.isdir("_dir1_/dir2/dir3/dir4") def test_makedirs_from_non_root_directory(self): # This is a testcase for issue #22, see # http://ftputil.sschwarzer.net/trac/ticket/22 . host = self.host # No `_dir1_` and `_dir2_` yet assert "_dir1_" not in host.listdir(host.curdir) assert "_dir2_" not in host.listdir(host.curdir) # Part 1: Try to make directories starting from `_dir1_` and # change to non-root directory. self.cleaner.add_dir("_dir1_") host.mkdir("_dir1_") host.chdir("_dir1_") host.makedirs("_dir2_/_dir3_") # Test for expected directory hierarchy. assert host.path.isdir("/_dir1_") assert host.path.isdir("/_dir1_/_dir2_") assert host.path.isdir("/_dir1_/_dir2_/_dir3_") assert not host.path.isdir("/_dir1_/_dir1_") # Remove all but the directory we're in. host.rmdir("/_dir1_/_dir2_/_dir3_") host.rmdir("/_dir1_/_dir2_") # Part 2: Try to make directories starting from root. self.cleaner.add_dir("/_dir2_") host.makedirs("/_dir2_/_dir3_") # Test for expected directory hierarchy assert host.path.isdir("/_dir2_") assert host.path.isdir("/_dir2_/_dir3_") assert not host.path.isdir("/_dir1_/_dir2_") def test_makedirs_of_existing_directory(self): host = self.host # The (chrooted) login directory host.makedirs("/") def test_makedirs_with_file_in_the_way(self): host = self.host self.cleaner.add_dir("_dir1_") host.mkdir("_dir1_") self.make_remote_file("_dir1_/file1") # Try it. with pytest.raises(ftputil.error.PermanentError): host.makedirs("_dir1_/file1") with pytest.raises(ftputil.error.PermanentError): host.makedirs("_dir1_/file1/dir2") def test_makedirs_with_existing_directory(self): host = self.host self.cleaner.add_dir("_dir1_") host.mkdir("_dir1_") host.makedirs("_dir1_/dir2") # Check assert host.path.isdir("_dir1_") assert host.path.isdir("_dir1_/dir2") def test_makedirs_in_non_writable_directory(self): host = self.host # Preparation: `rootdir1` exists but is only writable by root. with pytest.raises(ftputil.error.PermanentError): host.makedirs("rootdir1/dir2") def test_makedirs_with_writable_directory_at_end(self): host = self.host self.cleaner.add_dir("rootdir2/dir2") # Preparation: `rootdir2` exists but is only writable by root. # `dir2` is writable by regular ftp users. Both directories # below should work. host.makedirs("rootdir2/dir2") host.makedirs("rootdir2/dir2/dir3") class TestRemoval(RealFTPTest): def test_rmtree_without_error_handler(self): host = self.host # Build a tree. self.cleaner.add_dir("_dir1_") host.makedirs("_dir1_/dir2") self.make_remote_file("_dir1_/file1") self.make_remote_file("_dir1_/file2") self.make_remote_file("_dir1_/dir2/file3") self.make_remote_file("_dir1_/dir2/file4") # Try to remove a _file_ with `rmtree`. with pytest.raises(ftputil.error.PermanentError): host.rmtree("_dir1_/file2") # Remove `dir2`. host.rmtree("_dir1_/dir2") assert not host.path.exists("_dir1_/dir2") assert host.path.exists("_dir1_/file2") # Re-create `dir2` and remove `_dir1_`. host.mkdir("_dir1_/dir2") self.make_remote_file("_dir1_/dir2/file3") self.make_remote_file("_dir1_/dir2/file4") host.rmtree("_dir1_") assert not host.path.exists("_dir1_") def test_rmtree_with_error_handler(self): host = self.host self.cleaner.add_dir("_dir1_") host.mkdir("_dir1_") self.make_remote_file("_dir1_/file1") # Prepare error "handler" log = [] def error_handler(*args): log.append(args) # Try to remove a file as root "directory". host.rmtree("_dir1_/file1", ignore_errors=True, onerror=error_handler) assert log == [] host.rmtree("_dir1_/file1", ignore_errors=False, onerror=error_handler) assert log[0][0] == host.listdir assert log[0][1] == "_dir1_/file1" assert log[1][0] == host.rmdir assert log[1][1] == "_dir1_/file1" host.rmtree("_dir1_") # Try to remove a non-existent directory. del log[:] host.rmtree("_dir1_", ignore_errors=False, onerror=error_handler) assert log[0][0] == host.listdir assert log[0][1] == "_dir1_" assert log[1][0] == host.rmdir assert log[1][1] == "_dir1_" def test_remove_non_existent_item(self): host = self.host with pytest.raises(ftputil.error.PermanentError): host.remove("nonexistent") def test_remove_existing_file(self): self.cleaner.add_file("_testfile_") self.make_remote_file("_testfile_") host = self.host assert host.path.isfile("_testfile_") host.remove("_testfile_") assert not host.path.exists("_testfile_") class TestWalk(RealFTPTest): """ Walk the directory tree walk_test ├── dir1 │   ├── dir11 │   └── dir12 │   ├── dir123 │   │   └── file1234 │   ├── file121 │   └── file122 ├── dir2 ├── dir3 │   ├── dir31 │   ├── dir32 -> ../dir1/dir12/dir123 │   ├── file31 │   └── file32 └── file4 and check if the results are the expected ones. """ def _walk_test(self, expected_result, **walk_kwargs): """Walk the directory and test results.""" # Collect data using `walk`. actual_result = [] for items in self.host.walk(**walk_kwargs): actual_result.append(items) # Compare with expected results. assert len(actual_result) == len(expected_result) for index, _ in enumerate(actual_result): assert actual_result[index] == expected_result[index] def test_walk_topdown(self): # Preparation: build tree in directory `walk_test`. expected_result = [ ("walk_test", ["dir1", "dir2", "dir3"], ["file4"]), # ("walk_test/dir1", ["dir11", "dir12"], []), # ("walk_test/dir1/dir11", [], []), # ("walk_test/dir1/dir12", ["dir123"], ["file121", "file122"]), # ("walk_test/dir1/dir12/dir123", [], ["file1234"]), # ("walk_test/dir2", [], []), # ("walk_test/dir3", ["dir31", "dir32"], ["file31", "file32"]), # ("walk_test/dir3/dir31", [], []), ] self._walk_test(expected_result, top="walk_test") def test_walk_depth_first(self): # Preparation: build tree in directory `walk_test` expected_result = [ ("walk_test/dir1/dir11", [], []), # ("walk_test/dir1/dir12/dir123", [], ["file1234"]), # ("walk_test/dir1/dir12", ["dir123"], ["file121", "file122"]), # ("walk_test/dir1", ["dir11", "dir12"], []), # ("walk_test/dir2", [], []), # ("walk_test/dir3/dir31", [], []), # ("walk_test/dir3", ["dir31", "dir32"], ["file31", "file32"]), # ("walk_test", ["dir1", "dir2", "dir3"], ["file4"]) ] self._walk_test(expected_result, top="walk_test", topdown=False) def test_walk_following_links(self): # Preparation: build tree in directory `walk_test`. expected_result = [ ("walk_test", ["dir1", "dir2", "dir3"], ["file4"]), # ("walk_test/dir1", ["dir11", "dir12"], []), # ("walk_test/dir1/dir11", [], []), # ("walk_test/dir1/dir12", ["dir123"], ["file121", "file122"]), # ("walk_test/dir1/dir12/dir123", [], ["file1234"]), # ("walk_test/dir2", [], []), # ("walk_test/dir3", ["dir31", "dir32"], ["file31", "file32"]), # ("walk_test/dir3/dir31", [], []), # ("walk_test/dir3/dir32", [], ["file1234"]), ] self._walk_test(expected_result, top="walk_test", followlinks=True) class TestRename(RealFTPTest): def test_rename(self): host = self.host # Make sure the target of the renaming operation is removed. self.cleaner.add_file("_testfile2_") self.make_remote_file("_testfile1_") host.rename("_testfile1_", "_testfile2_") assert not host.path.exists("_testfile1_") assert host.path.exists("_testfile2_") def test_rename_with_spaces_in_directory(self): host = self.host dir_name = "_dir with spaces_" self.cleaner.add_dir(dir_name) host.mkdir(dir_name) self.make_remote_file(dir_name + "/testfile1") host.rename(dir_name + "/testfile1", dir_name + "/testfile2") assert not host.path.exists(dir_name + "/testfile1") assert host.path.exists(dir_name + "/testfile2") class TestStat(RealFTPTest): def test_stat(self): host = self.host dir_name = "_testdir_" file_name = host.path.join(dir_name, "_nonempty_") # Make a directory and a file in it. self.cleaner.add_dir(dir_name) host.mkdir(dir_name) with host.open(file_name, "wb") as fobj: fobj.write(b"abc\x12\x34def\t") # Do some stats # - dir dir_stat = host.stat(dir_name) assert isinstance(dir_stat._st_name, ftputil.compat.unicode_type) assert host.listdir(dir_name) == ["_nonempty_"] assert host.path.isdir(dir_name) assert not host.path.isfile(dir_name) assert not host.path.islink(dir_name) # - file file_stat = host.stat(file_name) assert isinstance(file_stat._st_name, ftputil.compat.unicode_type) assert not host.path.isdir(file_name) assert host.path.isfile(file_name) assert not host.path.islink(file_name) assert host.path.getsize(file_name) == 9 # - file's modification time; allow up to two minutes difference host.synchronize_times() server_mtime = host.path.getmtime(file_name) client_mtime = time.mktime(time.localtime()) calculated_time_shift = server_mtime - client_mtime assert not abs(calculated_time_shift-host.time_shift()) > 120 def test_issomething_for_nonexistent_directory(self): host = self.host # Check if we get the right results if even the containing # directory doesn't exist (see ticket #66). nonexistent_path = "/nonexistent/nonexistent" assert not host.path.isdir(nonexistent_path) assert not host.path.isfile(nonexistent_path) assert not host.path.islink(nonexistent_path) def test_special_broken_link(self): # Test for ticket #39. host = self.host broken_link_name = os.path.join("dir_with_broken_link", "nonexistent") assert (host.lstat(broken_link_name)._st_target == "../nonexistent/nonexistent") assert not host.path.isdir(broken_link_name) assert not host.path.isfile(broken_link_name) assert host.path.islink(broken_link_name) def test_concurrent_access(self): self.make_remote_file("_testfile_") with ftputil.FTPHost(*self.login_data) as host1: with ftputil.FTPHost(*self.login_data) as host2: stat_result1 = host1.stat("_testfile_") stat_result2 = host2.stat("_testfile_") assert stat_result1 == stat_result2 host2.remove("_testfile_") # Can still get the result via `host1` stat_result1 = host1.stat("_testfile_") assert stat_result1 == stat_result2 # Stat'ing on `host2` gives an exception. with pytest.raises(ftputil.error.PermanentError): host2.stat("_testfile_") # Stat'ing on `host1` after invalidation absolute_path = host1.path.join(host1.getcwd(), "_testfile_") host1.stat_cache.invalidate(absolute_path) with pytest.raises(ftputil.error.PermanentError): host1.stat("_testfile_") def test_cache_auto_resizing(self): """Test if the cache is resized appropriately.""" host = self.host cache = host.stat_cache._cache # Make sure the cache size isn't adjusted towards smaller values. unused_entries = host.listdir("walk_test") assert cache.size == ftputil.stat_cache.StatCache._DEFAULT_CACHE_SIZE # Make the cache very small initially and see if it gets resized. cache.size = 2 entries = host.listdir("walk_test") # The adjusted cache size should be larger or equal to the # number of items in `walk_test` and its parent directory. The # latter is read implicitly upon `listdir`'s `isdir` call. expected_min_cache_size = max(len(host.listdir(host.curdir)), len(entries)) assert cache.size >= expected_min_cache_size class TestUploadAndDownload(RealFTPTest): """Test upload and download (including time shift test).""" def test_time_shift(self): self.host.synchronize_times() assert self.host.time_shift() == EXPECTED_TIME_SHIFT @pytest.mark.slow_test def test_upload(self): host = self.host host.synchronize_times() local_file = "_local_file_" remote_file = "_remote_file_" # Make local file to upload. self.make_local_file() # Wait, else small time differences between client and server # actually could trigger the update. time.sleep(65) try: self.cleaner.add_file(remote_file) host.upload(local_file, remote_file) # Retry; shouldn't be uploaded uploaded = host.upload_if_newer(local_file, remote_file) assert uploaded is False # Rewrite the local file. self.make_local_file() # Retry; should be uploaded now uploaded = host.upload_if_newer(local_file, remote_file) assert uploaded is True finally: # Clean up os.unlink(local_file) @pytest.mark.slow_test def test_download(self): host = self.host host.synchronize_times() local_file = "_local_file_" remote_file = "_remote_file_" # Make a remote file. self.make_remote_file(remote_file) # File should be downloaded as it's not present yet. downloaded = host.download_if_newer(remote_file, local_file) assert downloaded is True try: # If the remote file, taking the datetime precision into # account, _might_ be newer, the file will be downloaded # again. To prevent this, wait a bit over a minute (the # remote precision), then "touch" the local file. time.sleep(65) # Create empty file. with open(local_file, "w") as fobj: pass # Local file is present and newer, so shouldn't download. downloaded = host.download_if_newer(remote_file, local_file) assert downloaded is False # Re-make the remote file. self.make_remote_file(remote_file) # Local file is present but possibly older (taking the # possible deviation because of the precision into account), # so should download. downloaded = host.download_if_newer(remote_file, local_file) assert downloaded is True finally: # Clean up. os.unlink(local_file) def test_callback_with_transfer(self): host = self.host FILE_NAME = "debian-keyring.tar.gz" # Default chunk size as in `FTPHost.copyfileobj` MAX_COPY_CHUNK_SIZE = ftputil.file_transfer.MAX_COPY_CHUNK_SIZE file_size = host.path.getsize(FILE_NAME) chunk_count, _ = divmod(file_size, MAX_COPY_CHUNK_SIZE) # Add one chunk for remainder. chunk_count += 1 # Define a callback that just collects all data passed to it. transferred_chunks_list = [] def test_callback(chunk): transferred_chunks_list.append(chunk) try: host.download(FILE_NAME, FILE_NAME, callback=test_callback) # Construct a list of data chunks we expect. expected_chunks_list = [] with open(FILE_NAME, "rb") as downloaded_fobj: while True: chunk = downloaded_fobj.read(MAX_COPY_CHUNK_SIZE) if not chunk: break expected_chunks_list.append(chunk) # Examine data collected by callback function. assert len(transferred_chunks_list) == chunk_count assert transferred_chunks_list == expected_chunks_list finally: os.unlink(FILE_NAME) class TestFTPFiles(RealFTPTest): def test_only_closed_children(self): REMOTE_FILE_NAME = "CONTENTS" host = self.host with host.open(REMOTE_FILE_NAME, "rb") as file_obj1: # Create empty file and close it. with host.open(REMOTE_FILE_NAME, "rb") as file_obj2: pass # This should re-use the second child because the first isn't # closed but the second is. with host.open(REMOTE_FILE_NAME, "rb") as file_obj: assert len(host._children) == 2 assert file_obj._host is host._children[1] def test_no_timed_out_children(self): REMOTE_FILE_NAME = "CONTENTS" host = self.host # Implicitly create child host object. with host.open(REMOTE_FILE_NAME, "rb") as file_obj1: pass # Monkey-patch file to simulate an FTP server timeout below. def timed_out_pwd(): raise ftplib.error_temp("simulated timeout") file_obj1._host._session.pwd = timed_out_pwd # Try to get a file - which shouldn't be the timed-out file. with host.open(REMOTE_FILE_NAME, "rb") as file_obj2: assert file_obj1 is not file_obj2 # Re-use closed and not timed-out child session. with host.open(REMOTE_FILE_NAME, "rb") as file_obj3: pass assert file_obj2 is file_obj3 def test_no_delayed_226_children(self): REMOTE_FILE_NAME = "CONTENTS" host = self.host # Implicitly create child host object. with host.open(REMOTE_FILE_NAME, "rb") as file_obj1: pass # Monkey-patch file to simulate an FTP server timeout below. def timed_out_pwd(): raise ftplib.error_reply("delayed 226 reply") file_obj1._host._session.pwd = timed_out_pwd # Try to get a file - which shouldn't be the timed-out file. with host.open(REMOTE_FILE_NAME, "rb") as file_obj2: assert file_obj1 is not file_obj2 # Re-use closed and not timed-out child session. with host.open(REMOTE_FILE_NAME, "rb") as file_obj3: pass assert file_obj2 is file_obj3 class TestChmod(RealFTPTest): def assert_mode(self, path, expected_mode): """ Return an integer containing the allowed bits in the mode change command. The `FTPHost` object to test against is `self.host`. """ full_mode = self.host.stat(path).st_mode # Remove flags we can't set via `chmod`. # Allowed flags according to Python documentation # https://docs.python.org/library/stat.html allowed_flags = [stat.S_ISUID, stat.S_ISGID, stat.S_ENFMT, stat.S_ISVTX, stat.S_IREAD, stat.S_IWRITE, stat.S_IEXEC, stat.S_IRWXU, stat.S_IRUSR, stat.S_IWUSR, stat.S_IXUSR, stat.S_IRWXG, stat.S_IRGRP, stat.S_IWGRP, stat.S_IXGRP, stat.S_IRWXO, stat.S_IROTH, stat.S_IWOTH, stat.S_IXOTH] allowed_mask = functools.reduce(operator.or_, allowed_flags) mode = full_mode & allowed_mask assert mode == expected_mode, ( "mode {0:o} != {1:o}".format(mode, expected_mode)) def test_chmod_existing_directory(self): host = self.host host.mkdir("_test dir_") self.cleaner.add_dir("_test dir_") # Set/get mode of a directory. host.chmod("_test dir_", 0o757) self.assert_mode("_test dir_", 0o757) # Set/get mode in nested directory. host.mkdir("_test dir_/nested_dir") self.cleaner.add_dir("_test dir_/nested_dir") host.chmod("_test dir_/nested_dir", 0o757) self.assert_mode("_test dir_/nested_dir", 0o757) def test_chmod_existing_file(self): host = self.host host.mkdir("_test dir_") self.cleaner.add_dir("_test dir_") # Set/get mode on a file. file_name = host.path.join("_test dir_", "_testfile_") self.make_remote_file(file_name) host.chmod(file_name, 0o646) self.assert_mode(file_name, 0o646) def test_chmod_nonexistent_path(self): # Set/get mode of a non-existing item. with pytest.raises(ftputil.error.PermanentError): self.host.chmod("nonexistent", 0o757) def test_cache_invalidation(self): host = self.host host.mkdir("_test dir_") self.cleaner.add_dir("_test dir_") # Make sure the mode is in the cache. unused_stat_result = host.stat("_test dir_") # Set/get mode of the directory. host.chmod("_test dir_", 0o757) self.assert_mode("_test dir_", 0o757) # Set/get mode on a file. file_name = host.path.join("_test dir_", "_testfile_") self.make_remote_file(file_name) # Make sure the mode is in the cache. unused_stat_result = host.stat(file_name) host.chmod(file_name, 0o646) self.assert_mode(file_name, 0o646) class TestRestArgument(RealFTPTest): TEST_FILE_NAME = "rest_test" def setup_method(self, method): super(TestRestArgument, self).setup_method(method) # Write test file. with self.host.open(self.TEST_FILE_NAME, "wb") as fobj: fobj.write(b"abcdefghijkl") self.cleaner.add_file(self.TEST_FILE_NAME) def test_for_reading(self): """ If a `rest` argument is passed to `open`, the following read operation should start at the byte given by `rest`. """ with self.host.open(self.TEST_FILE_NAME, "rb", rest=3) as fobj: data = fobj.read() assert data == b"defghijkl" def test_for_writing(self): """ If a `rest` argument is passed to `open`, the following write operation should start writing at the byte given by `rest`. """ with self.host.open(self.TEST_FILE_NAME, "wb", rest=3) as fobj: fobj.write(b"123") with self.host.open(self.TEST_FILE_NAME, "rb") as fobj: data = fobj.read() assert data == b"abc123" def test_invalid_read_from_text_file(self): """ If the `rest` argument is used for reading from a text file, a `CommandNotImplementedError` should be raised. """ with pytest.raises(ftputil.error.CommandNotImplementedError): self.host.open(self.TEST_FILE_NAME, "r", rest=3) def test_invalid_write_to_text_file(self): """ If the `rest` argument is used for reading from a text file, a `CommandNotImplementedError` should be raised. """ with pytest.raises(ftputil.error.CommandNotImplementedError): self.host.open(self.TEST_FILE_NAME, "w", rest=3) # There are no tests for reading and writing beyond the end of a # file. For example, if the remote file is 10 bytes long and # `open(remote_file, "rb", rest=100)` is used, the server may # return an error status code or not. # # The server I use for testing returns a 554 status when # attempting to _read_ beyond the end of the file. On the other # hand, if attempting to _write_ beyond the end of the file, the # server accepts the request, but starts writing after the end of # the file, i. e. appends to the file. # # Instead of expecting certain responses that may differ between # server implementations, I leave the bahavior for too large # `rest` arguments undefined. In practice, this shouldn't be a # problem because the `rest` argument should only be used for # error recovery, and in this case a valid byte count for the # `rest` argument should be known. class TestOther(RealFTPTest): def test_open_for_reading(self): # Test for issues #17 and #51, # http://ftputil.sschwarzer.net/trac/ticket/17 and # http://ftputil.sschwarzer.net/trac/ticket/51 . file1 = self.host.open("debian-keyring.tar.gz", "rb") time.sleep(1) # Depending on the FTP server, this might return a status code # unexpected by `ftplib` or block the socket connection until # a server-side timeout. file1.close() def test_subsequent_reading(self): # Open a file for reading. with self.host.open("CONTENTS", "rb") as file1: pass # Make sure that there are no problems if the connection is reused. with self.host.open("CONTENTS", "rb") as file2: pass assert file1._session is file2._session def test_names_with_spaces(self): # Test if directories and files with spaces in their names # can be used. host = self.host assert host.path.isdir("dir with spaces") assert (host.listdir("dir with spaces") == ["second dir", "some file", "some_file"]) assert host.path.isdir("dir with spaces/second dir") assert host.path.isfile("dir with spaces/some_file") assert host.path.isfile("dir with spaces/some file") def test_synchronize_times_without_write_access(self): """Test failing synchronization because of non-writable directory.""" host = self.host # This isn't writable by the ftp account the tests are run under. host.chdir("rootdir1") with pytest.raises(ftputil.error.TimeShiftError): host.synchronize_times() def test_listdir_with_non_ascii_byte_string(self): """ `listdir` should accept byte strings with non-ASCII characters and return non-ASCII characters in directory or file names. """ host = self.host path = "äbc".encode("UTF-8") names = host.listdir(path) assert names[0] == b"file1" assert names[1] == "file1_ö".encode("UTF-8") def test_listdir_with_non_ascii_unicode_string(self): """ `listdir` should accept unicode strings with non-ASCII characters and return non-ASCII characters in directory or file names. """ host = self.host # `ftplib` under Python 3 only works correctly if the unicode # strings are decoded from latin1. Under Python 2, ftputil # is supposed to provide a compatible interface. path = "äbc".encode("UTF-8").decode("latin1") names = host.listdir(path) assert names[0] == "file1" assert names[1] == "file1_ö".encode("UTF-8").decode("latin1") def test_path_with_non_latin1_unicode_string(self): """ ftputil operations shouldn't accept file paths with non-latin1 characters. """ # Use some musical symbols. These are certainly not latin1. path = "𝄞𝄢" # `UnicodeEncodeError` is also the exception that `ftplib` # raises if it gets a non-latin1 path. with pytest.raises(UnicodeEncodeError): self.host.mkdir(path) def test_list_a_option(self): # For this test to pass, the server must _not_ list "hidden" # files by default but instead only when the `LIST` `-a` # option is used. host = self.host assert host.use_list_a_option directory_entries = host.listdir(host.curdir) assert ".hidden" in directory_entries host.use_list_a_option = False directory_entries = host.listdir(host.curdir) assert ".hidden" not in directory_entries def _make_objects_to_be_garbage_collected(self): for _ in range(10): with ftputil.FTPHost(*self.login_data) as host: for _ in range(10): unused_stat_result = host.stat("CONTENTS") with host.open("CONTENTS") as fobj: unused_data = fobj.read() def test_garbage_collection(self): """Test whether there are cycles which prevent garbage collection.""" gc.collect() objects_before_test = len(gc.garbage) self._make_objects_to_be_garbage_collected() gc.collect() objects_after_test = len(gc.garbage) assert not objects_after_test - objects_before_test @pytest.mark.skipif( ftputil.compat.python_version > 2, reason="test requires M2Crypto which only works on Python 2") def test_m2crypto_session(self): """ Test if a session with `M2Crypto.ftpslib.FTP_TLS` is set up correctly and works with unicode input. """ # See ticket #78. # # M2Crypto is only available for Python 2. import M2Crypto factory = ftputil.session.session_factory( base_class=M2Crypto.ftpslib.FTP_TLS, encrypt_data_channel=True) with ftputil.FTPHost(*self.login_data, session_factory=factory) as host: # Test if unicode argument works. files = host.listdir(".") assert "CONTENTS" in files ftputil-3.4/test/test_tool.py0000644000175000017470000000443313135116467015663 0ustar debiandebian# Copyright (C) 2013-2016, Stefan Schwarzer # and ftputil contributors (see `doc/contributors.txt`) # See the file LICENSE for licensing terms. from __future__ import unicode_literals import ftputil.compat as compat import ftputil.tool class TestSameStringTypeAs(object): # The first check for equality is enough for Python 3, where # comparing a byte string and unicode string would raise an # exception. However, we need the second test for Python 2. def test_to_bytes(self): result = ftputil.tool.same_string_type_as(b"abc", "def") assert result == b"def" assert isinstance(result, compat.bytes_type) def test_to_unicode(self): result = ftputil.tool.same_string_type_as("abc", b"def") assert result == "def" assert isinstance(result, compat.unicode_type) def test_both_bytes_type(self): result = ftputil.tool.same_string_type_as(b"abc", b"def") assert result == b"def" assert isinstance(result, compat.bytes_type) def test_both_unicode_type(self): result = ftputil.tool.same_string_type_as("abc", "def") assert result == "def" assert isinstance(result, compat.unicode_type) class TestSimpleConversions(object): def test_as_bytes(self): result = ftputil.tool.as_bytes(b"abc") assert result == b"abc" assert isinstance(result, compat.bytes_type) result = ftputil.tool.as_bytes("abc") assert result == b"abc" assert isinstance(result, compat.bytes_type) def test_as_unicode(self): result = ftputil.tool.as_unicode(b"abc") assert result == "abc" assert isinstance(result, compat.unicode_type) result = ftputil.tool.as_unicode("abc") assert result == "abc" assert isinstance(result, compat.unicode_type) class TestEncodeIfUnicode(object): def test_do_encode(self): string = "abc" converted_string = ftputil.tool.encode_if_unicode(string, "latin1") assert isinstance(converted_string, compat.bytes_type) def test_dont_encode(self): string = b"abc" not_converted_string = ftputil.tool.encode_if_unicode(string, "latin1") assert string == not_converted_string assert isinstance(not_converted_string, compat.bytes_type) ftputil-3.4/PKG-INFO0000644000175000017470000000301213200532636013374 0ustar debiandebianMetadata-Version: 1.1 Name: ftputil Version: 3.4 Summary: High-level FTP client library (virtual file system and more) Home-page: http://ftputil.sschwarzer.net/ Author: Stefan Schwarzer Author-email: sschwarzer@sschwarzer.net License: Open source (revised BSD license) Download-URL: http://ftputil.sschwarzer.net/trac/attachment/wiki/Download/ftputil-3.4.tar.gz?format=raw Description: ftputil is a high-level FTP client library for the Python programming language. ftputil implements a virtual file system for accessing FTP servers, that is, it can generate file-like objects for remote files. The library supports many functions similar to those in the os, os.path and shutil modules. ftputil has convenience functions for conditional uploads and downloads, and handles FTP clients and servers in different timezones. Keywords: FTP,client,library,virtual file system Platform: Pure Python Classifier: Development Status :: 5 - Production/Stable Classifier: Environment :: Other Environment Classifier: Intended Audience :: Developers Classifier: Intended Audience :: System Administrators Classifier: License :: OSI Approved :: BSD License Classifier: Operating System :: OS Independent Classifier: Programming Language :: Python Classifier: Programming Language :: Python :: 2 Classifier: Programming Language :: Python :: 3 Classifier: Topic :: Internet :: File Transfer Protocol (FTP) Classifier: Topic :: Software Development :: Libraries :: Python Modules Classifier: Topic :: System :: Filesystems