Pydap-3.2.2/ 0000775 0001750 0001750 00000000000 13111405714 013660 5 ustar hiebert hiebert 0000000 0000000 Pydap-3.2.2/src/ 0000775 0001750 0001750 00000000000 13111405714 014447 5 ustar hiebert hiebert 0000000 0000000 Pydap-3.2.2/src/pydap/ 0000775 0001750 0001750 00000000000 13111405714 015564 5 ustar hiebert hiebert 0000000 0000000 Pydap-3.2.2/src/pydap/lib.py 0000664 0001750 0001750 00000021651 13111210026 016677 0 ustar hiebert hiebert 0000000 0000000 """Basic functions related to the DAP spec."""
import operator
from pkg_resources import get_distribution
from six.moves.urllib.parse import quote as quote_
from six.moves import reduce, zip_longest
from six import binary_type, MAXSIZE
from .exceptions import ConstraintExpressionError
__dap__ = '2.15'
__version__ = get_distribution("Pydap").version
START_OF_SEQUENCE = b'\x5a\x00\x00\x00'
END_OF_SEQUENCE = b'\xa5\x00\x00\x00'
STRING = '|S128'
DEFAULT_TIMEOUT = 120 # 120 seconds = 2 minutes
NUMPY_TO_DAP2_TYPEMAP = {
'd': 'Float64',
'f': 'Float32',
'h': 'Int16',
'H': 'UInt16',
'i': 'Int32', 'l': 'Int32', 'q': 'Int32',
'I': 'UInt32', 'L': 'UInt32', 'Q': 'UInt32',
# DAP2 does not support signed bytes.
# Its Byte type is unsigned and thus corresponds
# to numpy's 'B'.
# The consequence is that there is no natural way
# in DAP2 to represent numpy's 'b' type.
# Ideally, DAP2 would have a signed Byte type
# and an unsigned UByte type and we would have the
# following mapping: {'b': 'Byte', 'B': 'UByte'}
# but this not how the protocol has been defined.
# This means that numpy's 'b' must be mapped to Int16
# and data must be upconverted in the DODS response.
'b': 'Int16',
'B': 'Byte',
# There are no boolean types in DAP2. Upconvert to
# Byte:
'?': 'Byte',
'S': 'String',
# Map numpy's 'U' to String b/c
# DAP2 does not explicitly support unicode.
'U': 'String'
}
# DAP2 demands big-endian 32 bytes signed integers
# www.opendap.org/pdf/dap_2_data_model.pdf
# Before pydap 3.2.2, length was
# big-endian 32 bytes UNSIGNED integers:
# DAP2_ARRAY_LENGTH_NUMPY_TYPE = '>I'
# Since pydap 3.2.2, the length type is accurate:
DAP2_ARRAY_LENGTH_NUMPY_TYPE = '>i'
DAP2_TO_NUMPY_RESPONSE_TYPEMAP = {
'Float64': '>d',
'Float32': '>f',
# This is a weird aspect of the DAP2 specification.
# For backward-compatibility, Int16 and UInt16 are
# encoded as 32 bits integers in the response,
# respectively:
'Int16': '>i',
'UInt16': '>I',
'Int32': '>i',
'UInt32': '>I',
# DAP2 does not support signed bytes.
# It's Byte type is unsigned and thus corresponds
# to numpy 'B'.
# The consequence is that there is no natural way
# in DAP2 to represent numpy's 'b' type.
# Ideally, DAP2 would have a signed Byte type
# and a usigned UByte type and we would have the
# following mapping: {'Byte': 'b', 'UByte': 'B'}
# but this not how the protocol has been defined.
# This means that DAP2 Byte is unsigned and must be
# mapped to numpy's 'B' type, usigned byte.
'Byte': 'B',
# Map String to numpy's string type 'S' b/c
# DAP2 does not explicitly support unicode.
'String': 'S',
'URL': 'S',
#
# These two types are not DAP2 but it is useful
# to include them for compatiblity with other
# data sources:
'Int': '>i',
'UInt': '>I',
}
# Typemap from lower case DAP2 types to
# numpy dtype string with specified endiannes.
# Here, the endianness is very important:
LOWER_DAP2_TO_NUMPY_PARSER_TYPEMAP = {
'float64': '>d',
'float32': '>f',
'int16': '>h',
'uint16': '>H',
'int32': '>i',
'uint32': '>I',
'byte': 'B',
'string': STRING,
'url': STRING,
'int': '>i',
'uint': '>I',
}
def quote(name):
"""Return quoted name according to the DAP specification.
>>> quote("White space")
'White%20space'
>>> quote("Period.")
'Period%2E'
"""
safe = '%_!~*\'-"'
return quote_(name.encode('utf-8'), safe=safe).replace('.', '%2E')
def encode(obj):
"""Return an object encoded to its DAP representation."""
try:
return '%.6g' % obj
except:
return '"{0}"'.format(obj)
def fix_slice(slice_, shape):
"""Return a normalized slice.
This function returns a slice so that it has the same length of `shape`,
and no negative indexes, if possible.
This is based on this document:
http://docs.scipy.org/doc/numpy/reference/arrays.indexing.html
"""
# convert `slice_` to a tuple
if not isinstance(slice_, tuple):
slice_ = (slice_,)
# expand Ellipsis and make `slice_` at least as long as `shape`
expand = len(shape) - len(slice_)
out = []
for s in slice_:
if s is Ellipsis:
out.extend((slice(None),) * (expand+1))
expand = 0
else:
out.append(s)
slice_ = tuple(out) + (slice(None),) * expand
out = []
for s, n in zip(slice_, shape):
if isinstance(s, int):
if s < 0:
s += n
out.append(s)
else:
k = s.step or 1
i = s.start
if i is None:
i = 0
elif i < 0:
i += n
j = s.stop
if (j is None or
j > n):
j = n
elif j < 0:
j += n
out.append(slice(i, j, k))
return tuple(out)
def combine_slices(slice1, slice2):
"""Return two tuples of slices combined sequentially.
These two should be equal:
x[ combine_slices(s1, s2) ] == x[s1][s2]
"""
out = []
for exp1, exp2 in zip_longest(
slice1, slice2, fillvalue=slice(None)):
if isinstance(exp1, int):
exp1 = slice(exp1, exp1+1)
if isinstance(exp2, int):
exp2 = slice(exp2, exp2+1)
start = (exp1.start or 0) + (exp2.start or 0)
step = (exp1.step or 1) * (exp2.step or 1)
if exp1.stop is None and exp2.stop is None:
stop = None
elif exp1.stop is None:
stop = (exp1.start or 0) + exp2.stop
elif exp2.stop is None:
stop = exp1.stop
else:
stop = min(exp1.stop, (exp1.start or 0) + exp2.stop)
out.append(slice(start, stop, step))
return tuple(out)
def hyperslab(slice_):
"""Return a DAP representation of a multidimensional slice."""
if not isinstance(slice_, tuple):
slice_ = [slice_]
else:
slice_ = list(slice_)
while slice_ and slice_[-1] == slice(None):
slice_.pop(-1)
return ''.join('[%s:%s:%s]' % (
s.start or 0, s.step or 1, (s.stop or MAXSIZE)-1) for s in slice_)
def walk(var, type=object):
"""Yield all variables of a given type from a dataset.
The iterator returns also the parent variable.
"""
if isinstance(var, type):
yield var
for child in var.children():
for var in walk(child, type):
yield var
def fix_shorthand(projection, dataset):
"""Fix shorthand notation in the projection.
Some clients request variables by their name, not by the id. This is called
the "shorthand notation", and it has to be fixed. This function will return
a new projection with no shorthand calls.
"""
out = []
for var in projection:
if len(var) == 1 and var[0][0] not in list(dataset.keys()):
token, slice_ = var.pop(0)
for child in walk(dataset):
if token == child.name:
if var:
raise ConstraintExpressionError(
'Ambiguous shorthand notation request: %s' % token)
var = [
(parent, ()) for parent in child.id.split('.')[:-1]
] + [(token, slice_)]
out.append(var)
return out
def get_var(dataset, id_):
"""Given an id, return the corresponding variable from the dataset."""
tokens = id_.split('.')
return reduce(operator.getitem, [dataset] + tokens)
def decode_np_strings(numpy_var):
"""Given a fixed-width numpy string, decode it to a unicode type"""
if isinstance(numpy_var, binary_type) and hasattr(numpy_var, 'tostring'):
return numpy_var.tostring().decode('utf-8')
else:
return numpy_var
def load_from_entry_point_relative(r, package):
try:
loaded = getattr(__import__(r.module_name
.replace(package + '.', '', 1),
globals(), None, [r.attrs[0]], 1),
r.attrs[0])
return r.name, loaded
except ImportError:
# This is only used in handlers testing:
return r.name, r.load()
class StreamReader(object):
"""Class to allow reading a `urllib3.HTTPResponse`."""
def __init__(self, stream):
self.stream = stream
self.buf = bytearray()
def read(self, n):
"""Read and return `n` bytes."""
while len(self.buf) < n:
bytes_read = next(self.stream)
self.buf.extend(bytes_read)
out = bytes(self.buf[:n])
self.buf = self.buf[n:]
return out
class BytesReader(object):
"""Class to allow reading a `bytes` object."""
def __init__(self, data):
self.data = data
def read(self, n):
"""Read and return `n` bytes."""
out = self.data[:n]
self.data = self.data[n:]
return out
Pydap-3.2.2/src/pydap/wsgi/ 0000775 0001750 0001750 00000000000 13111405714 016535 5 ustar hiebert hiebert 0000000 0000000 Pydap-3.2.2/src/pydap/wsgi/functions.py 0000664 0001750 0001750 00000014103 13110635621 021117 0 ustar hiebert hiebert 0000000 0000000 """Server-side functions for Pydap.
Pydap maps function calls in the request URL to functions defined through entry
points. For example, Pydap defines an entry point on the group "pydap.function"
with the name "density"; from ``setup.py``:
[pydap.function]
density = pydap.wsgi.functions.density
When a DAP client makes a request to the URL calling this function:
?cast.oxygen&density(cast.salt,cast.temp,cast.press)>1024
Pydap will call the ``density`` function passing the dataset itself as the
first argument, followed by the variables from the sequence "cast".
New functions can be defined in third-party modules. Simply create a package
and declare the entry point in its ``setup.py`` file. The server will
automatically discover the new functions from the system.
"""
from datetime import datetime, timedelta
import re
import numpy as np
import coards
import gsw
from ..model import SequenceType, GridType, BaseType
from ..lib import walk
from ..exceptions import ConstraintExpressionError, ServerError
def density(dataset, salinity, temperature, pressure):
"""Calculate in-situ density.
This function calculated in-situ density from absolute salinity and
conservative temperature, using the `gsw.rho` function. Returns a new
sequence with the data.
"""
# find sequence
for sequence in walk(dataset, SequenceType):
break
else:
raise ConstraintExpressionError(
'Function "bounds" should be used on a Sequence.')
selection = sequence[salinity.name, temperature.name, pressure.name]
rows = [tuple(row) for row in selection.iterdata()]
data = np.rec.fromrecords(
rows, names=['salinity', 'temperature', 'pressure'])
rho = gsw.rho(data['salinity'], data['temperature'], data['pressure'])
out = SequenceType("result")
out['rho'] = BaseType("rho", units="kg/m**3")
out.data = np.rec.fromrecords(rho.reshape(-1, 1), names=['rho'])
return out
density.__version__ = "0.1"
def bounds(dataset, xmin, xmax, ymin, ymax, zmin, zmax, tmin, tmax):
r"""Bound a sequence in space and time.
This function is used by GrADS to access Sequences, eg:
http://server.example.com/dataset.dods?sequence& \
bounds(0,360,-90,90,500,500,00Z01JAN1970,00Z01JAN1970)
We assume the dataset has only a single Sequence, which will be returned
modified in place.
"""
# find sequence
for sequence in walk(dataset, SequenceType):
break # get first sequence
else:
raise ConstraintExpressionError(
'Function "bounds" should be used on a Sequence.')
for child in sequence.children():
if child.attributes.get('axis', '').lower() == 'x':
if xmin == xmax:
sequence.data = sequence[child == xmin].data
else:
sequence.data = sequence[
(child >= xmin) & (child <= xmax)].data
elif child.attributes.get('axis', '').lower() == 'y':
if ymin == ymax:
sequence.data = sequence[child == ymin].data
else:
sequence.data = sequence[
(child >= ymin) & (child <= ymax)].data
elif child.attributes.get('axis', '').lower() == 'z':
if zmin == zmax:
sequence.data = sequence[child == zmin].data
else:
sequence.data = sequence[
(child >= zmin) & (child <= zmax)].data
elif child.attributes.get('axis', '').lower() == 't':
start = datetime.strptime(tmin, '%HZ%d%b%Y')
end = datetime.strptime(tmax, '%HZ%d%b%Y')
units = child.attributes.get('units', 'seconds since 1970-01-01')
# if start and end are equal, add the step
if start == end and 'grads_step' in child.attributes:
dt = parse_step(child.attributes['grads_step'])
end = start + dt
tmin = coards.format(start, units)
tmax = coards.format(end, units)
sequence.data = sequence[
(child >= tmin) & (child < tmax)].data
else:
tmin = coards.format(start, units)
tmax = coards.format(end, units)
sequence.data = sequence[
(child >= tmin) & (child <= tmax)].data
return sequence
bounds.__version__ = "1.0"
def parse_step(step):
"""Parse a GrADS time step returning a timedelta."""
value, units = re.search(r'(\d+)(.*)', step).groups()
value = int(value)
if units.lower() == 'mn':
return timedelta(minutes=value)
if units.lower() == 'hr':
return timedelta(hours=value)
if units.lower() == 'dy':
return timedelta(days=value)
if units.lower() == 'mo':
raise NotImplementedError('Need to implement month time step')
if units.lower() == 'yr':
raise NotImplementedError('Need to implement year time step')
raise ServerError('Unknown units: "%s".' % units)
def mean(dataset, var, axis=0):
"""Calculate the mean of an array along a given axis.
The input variable should be either a ``GridType`` or ``BaseType``. The
function will return an object of the same type with the mean applied.
"""
if not isinstance(var, (GridType, BaseType)):
raise ConstraintExpressionError(
'Function "mean" should be used on an array or grid.')
axis = int(axis)
dims = tuple(dim for i, dim in enumerate(var.dimensions) if i != axis)
# process basetype
if isinstance(var, BaseType):
return BaseType(
name=var.name, data=np.mean(var.data[:], axis=axis),
dimensions=dims, attributes=var.attributes)
# process grid
out = GridType(name=var.name, attributes=var.attributes)
out[var.array.name] = BaseType(
name=var.array.name, data=np.mean(var.array.data[:], axis=axis),
dimensions=dims, attributes=var.array.attributes)
for dim in dims:
out[dim] = BaseType(
name=dim, data=var[dim].data[:], dimensions=(dim,),
attributes=var[dim].attributes)
return out
mean.__version__ = "1.0"
Pydap-3.2.2/src/pydap/wsgi/__init__.py 0000664 0001750 0001750 00000000070 13105357414 020651 0 ustar hiebert hiebert 0000000 0000000 __import__('pkg_resources').declare_namespace(__name__)
Pydap-3.2.2/src/pydap/wsgi/app.py 0000664 0001750 0001750 00000020077 13111403046 017671 0 ustar hiebert hiebert 0000000 0000000 """A file-based Pydap server running on Gunicorn.
Usage:
pydap [options]
Options:
-h --help Show this help message and exit
--version Show version
-i --init DIR Create directory with templates
-b ADDRESS --bind ADDRESS The ip to listen to [default: 127.0.0.1]
-p PORT --port PORT The port to connect [default: 8001]
-d DIR --data DIR The directory with files [default: .]
-t DIR --templates DIR The directory with templates
--worker-class=CLASS Gunicorn worker class [default: sync]
"""
import os
import re
import mimetypes
from datetime import datetime
import shutil
from jinja2 import Environment, PackageLoader, FileSystemLoader, ChoiceLoader
from webob import Response
from webob.dec import wsgify
from webob.exc import HTTPNotFound, HTTPForbidden
from webob.static import FileApp, DirectoryApp
import pkg_resources
from six.moves.urllib.parse import unquote
from six import string_types
from ..lib import __version__
from ..handlers.lib import get_handler, load_handlers
from ..exceptions import ExtensionNotSupportedError
from .ssf import ServerSideFunctions
class DapServer(object):
"""A directory app that creates file listings and handle DAP requests."""
def __init__(self, path, templates=None):
self.path = os.path.abspath(path)
# the default loader reads templates from the package
loaders = [PackageLoader("pydap.wsgi", "templates")]
# optionally, the user can also specify a template directory that will
# override the default templates; this should have precedence over the
# default templates
if templates is not None:
loaders.insert(0, FileSystemLoader(templates))
# set the rendering environment; this is also used by pydap responses
# that need to render templates (like HTML, WMS, KML, etc.)
self.env = Environment(loader=ChoiceLoader(loaders))
self.env.filters["datetimeformat"] = datetimeformat
self.env.filters["unquote"] = unquote
# cache available handlers, so we don't need to load them every request
self.handlers = load_handlers()
@wsgify
def __call__(self, req):
"""WSGI application callable.
Returns either a file download, directory listing or DAP response.
"""
path = os.path.abspath(
os.path.join(self.path, *req.path_info.split("/")))
if not path.startswith(self.path):
return HTTPForbidden()
elif os.path.exists(path):
if os.path.isdir(path):
return self.index(path, req)
else:
return FileApp(path)
# strip DAP extension (``.das``, eg) and see if the file exists
base, ext = os.path.splitext(path)
if os.path.isfile(base):
req.environ["pydap.jinja2.environment"] = self.env
app = ServerSideFunctions(get_handler(base, self.handlers))
return req.get_response(app)
else:
return HTTPNotFound(comment=path)
def index(self, directory, req):
"""Return a directory listing."""
content = [
os.path.join(directory, name) for name in os.listdir(directory)]
files = [{
"name": os.path.split(path)[1],
"size": os.path.getsize(path),
"last_modified": datetime.fromtimestamp(os.path.getmtime(path)),
"supported": supported(path, self.handlers),
} for path in content if os.path.isfile(path)]
files.sort(key=lambda d: alphanum_key(d["name"]))
directories = [{
"name": os.path.split(path)[1],
"last_modified": datetime.fromtimestamp(os.path.getmtime(path)),
} for path in content if os.path.isdir(path)]
directories.sort(key=lambda d: alphanum_key(d["name"]))
tokens = req.path_info.split("/")[1:]
breadcrumbs = [{
"url": "/".join([req.application_url] + tokens[:i+1]),
"title": token,
} for i, token in enumerate(tokens) if token]
context = {
"root": req.application_url,
"location": req.path_url,
"breadcrumbs": breadcrumbs,
"directories": directories,
"files": files,
"version": __version__,
}
template = self.env.get_template("index.html")
return Response(
body=template.render(context),
content_type="text/html",
charset="utf-8")
def supported(filepath, handlers=None):
"""Test if a file has a corresponding handler.
Returns a boolean.
"""
try:
get_handler(filepath, handlers)
return True
except ExtensionNotSupportedError:
return False
def alphanum_key(s):
"""Parse a string, returning a list of string and number chunks.
>>> alphanum_key("z23a")
['z', 23, 'a']
Useful for sorting names in a natural way.
From http://nedbatchelder.com/blog/200712.html#e20071211T054956
"""
def tryint(s):
try:
return int(s)
except:
return s
return [tryint(c) for c in re.split('([0-9]+)', s)]
def datetimeformat(value, format='%Y-%m-%d %H:%M:%S'):
"""Return a formatted datetime object."""
return value.strftime(format)
class StaticMiddleware(object):
"""WSGI middleware for static assets.
The assets can be either specified as a directory, or retrieved from a
Python package. Inspired by ``werkezeug.wsgi.SharedDataMiddleware``.
"""
def __init__(self, app, static):
self.app = app
self.static = static
@wsgify
def __call__(self, req):
if req.path_info_peek() != "static":
return req.get_response(self.app)
# strip "/static"
req.path_info_pop()
# statically serve the directory
if isinstance(self.static, string_types):
return req.get_response(DirectoryApp(self.static))
# otherwise, load resource from package
package, resource_path = self.static
resource = os.path.join(resource_path, *req.path_info.split('/'))
if not pkg_resources.resource_exists(package, resource):
return HTTPNotFound(req.path_info)
content_type, content_encoding = mimetypes.guess_type(resource)
return Response(
body=pkg_resources.resource_string(package, resource),
content_type=content_type,
content_encoding=content_encoding)
def init(directory):
"""Create directory with default templates."""
# copy main templates
templates = pkg_resources.resource_filename("pydap.wsgi", "templates")
shutil.copytree(templates, directory)
# copy templates from HTML response
for resource in pkg_resources.resource_listdir(
"pydap.responses.html", "templates"):
path = pkg_resources.resource_filename(
"pydap.responses.html", "templates/{0}".format(resource))
shutil.copy(path, directory)
def main(): # pragma: no cover
"""Run server from the command line."""
import multiprocessing
from docopt import docopt
from gunicorn.app.pasterapp import PasterServerApplication
arguments = docopt(__doc__, version="Pydap %s" % __version__)
# init templates?
if arguments["--init"]:
init(arguments["--init"])
return
# create pydap app
data, templates = arguments["--data"], arguments["--templates"]
app = DapServer(data, templates)
# configure app so that is reads static assets from the template directory
# or from the package
if templates and os.path.exists(os.path.join(templates, "static")):
static = os.path.join(templates, "static")
else:
static = ("pydap.wsgi", "templates/static")
app = StaticMiddleware(app, static)
# configure WSGI server
workers = multiprocessing.cpu_count() * 2 + 1
PasterServerApplication(
app,
host=arguments["--bind"],
port=int(arguments["--port"]),
workers=workers,
worker_class=arguments["--worker-class"]).run()
if __name__ == "__main__":
main()
Pydap-3.2.2/src/pydap/wsgi/ssf.py 0000664 0001750 0001750 00000015204 13110635621 017705 0 ustar hiebert hiebert 0000000 0000000 """An implementation of server-side functions.
Pydap implements DAP server-side functions throught a custom WSGI middleware.
This simplifies writing custom handlers, since they don't need to parse and
apply the function calls themselves.
"""
import re
import operator
import ast
from webob import Request
from pkg_resources import iter_entry_points
import numpy as np
from six.moves import reduce, map
from six import string_types
from ..model import DatasetType, SequenceType
from ..parsers import parse_ce
from ..lib import walk, fix_shorthand, load_from_entry_point_relative
from ..handlers.lib import BaseHandler, apply_projection
from ..exceptions import ServerError
FUNCTION = re.compile(r'([^(]*)\((.*)\)')
RELOP = re.compile(r'(<=|<|>=|>|=~|=|!=)')
def load_functions():
"""Load all available functions from the system, returning a dictionary."""
# Relative import of functions:
package = 'pydap'
entry_points = 'pydap.function'
base_dict = dict(load_from_entry_point_relative(r, package)
for r in iter_entry_points(entry_points)
if r.module_name.startswith(package))
opts_dict = dict((r.name, r.load())
for r in iter_entry_points(entry_points)
if not r.module_name.startswith(package))
base_dict.update(opts_dict)
return base_dict
class ServerSideFunctions(object):
"""A WebOb based middleware for handling server-side function calls.
The middleware works by removing function calls from the request,
forwarding the request to Pydap, and then applying the functions calls to
the returned dataset.
"""
def __init__(self, app, **kwargs):
self.app = app
self.functions = load_functions()
self.functions.update(kwargs)
def __call__(self, environ, start_response):
# specify that we want the parsed dataset
environ['x-wsgiorg.want_parsed_response'] = True
req = Request(environ)
projection, selection = parse_ce(req.query_string)
# check if there are any functions calls in the request
called = (
any(s for s in selection if FUNCTION.match(s)) or
any(p for p in projection if isinstance(p, string_types)))
# ignore DAS requests and requests without functions
path, response = req.path.rsplit('.', 1)
if response == 'das' or not called:
return self.app(environ, start_response)
# apply selection without any function calls
req.query_string = '&'.join(
s for s in selection if not FUNCTION.match(s))
res = req.get_response(self.app)
# get the dataset
method = getattr(res.app_iter, 'x_wsgiorg_parsed_response', False)
if not method:
raise ServerError("Unable to call server-side function!")
dataset = method(DatasetType)
# apply selection containing server-side functions
selection = (s for s in selection if FUNCTION.match(s))
for expr in selection:
if RELOP.search(expr):
call, op, other = RELOP.split(expr)
op = {
'<': operator.lt,
'>': operator.gt,
'!=': operator.ne,
'=': operator.eq,
'>=': operator.ge,
'<=': operator.le,
'=~': lambda a, b: re.match(b, a),
}[op]
other = ast.literal_eval(other)
else:
call, op, other = expr, operator.eq, 1
# evaluate the function call
sequence = eval_function(dataset, call, self.functions)
# is this an inplace call?
for var in walk(dataset, SequenceType):
if sequence is var:
break
else:
# get the data from the resulting variable, and use it to
# constrain the original dataset
child = list(sequence.children())[0]
data = np.fromiter(child.data, child.dtype)
if data.dtype.char == "S":
valid = np.array(
list(map(lambda v: op(str(v), str(other)), data)),
bool)
else:
valid = op(data, other)
for sequence in walk(dataset, SequenceType):
sequence.data = np.rec.fromrecords(
[tuple(row) for row in sequence.iterdata()],
names=list(sequence.keys()))[valid]
# now apply projection
if projection:
projection = fix_shorthand(projection, dataset)
base = [p for p in projection if not isinstance(p, string_types)]
func = [p for p in projection if isinstance(p, string_types)]
# apply non-function projection
out = apply_projection(base, dataset)
# apply function projection
for call in func:
var = eval_function(dataset, call, self.functions)
for child in walk(var):
parent = reduce(
operator.getitem, [out] + child.id.split('.')[:-1])
if child.name not in parent.keys():
parent[child.name] = child
break
dataset = out
# Return the original response (DDS, DAS, etc.)
path, response = req.path.rsplit('.', 1)
res = BaseHandler.responses[response](dataset)
return res(environ, start_response)
def eval_function(dataset, function, functions):
"""Evaluate a given function on a dataset.
This function parses and evaluates a (possibly nested) function call,
returning its result.
"""
name, args = FUNCTION.match(function).groups()
def tokenize(input):
start = pos = count = 0
for char in input:
if char == '(':
count += 1
elif char == ')':
count -= 1
elif char == ',' and count == 0:
yield input[start:pos]
start = pos+1
pos += 1
yield input[start:]
def parse(token):
if FUNCTION.match(token):
return eval_function(dataset, token, functions)
else:
try:
names = re.sub(r'\[.*?\]', '', str(token)).split('.')
return reduce(operator.getitem, [dataset] + names)
except:
try:
return ast.literal_eval(token)
except:
return token
args = map(parse, tokenize(args))
func = functions[name]
return func(dataset, *args)
Pydap-3.2.2/src/pydap/wsgi/templates/ 0000775 0001750 0001750 00000000000 13111405714 020533 5 ustar hiebert hiebert 0000000 0000000 Pydap-3.2.2/src/pydap/wsgi/templates/static/ 0000775 0001750 0001750 00000000000 13111405714 022022 5 ustar hiebert hiebert 0000000 0000000 Pydap-3.2.2/src/pydap/wsgi/templates/static/style.css 0000664 0001750 0001750 00000003044 13105357414 023703 0 ustar hiebert hiebert 0000000 0000000 body {
font-size: 86%;
margin: 0 auto;
line-height: 1.7em;
}
a, .pure-menu a {
color: #15c;
}
.header {
margin: 0 0;
}
.header .pure-menu {
padding: 0.5em;
}
.header .pure-menu li a:hover,
.header .pure-menu li a:focus {
background: none;
border: none;
}
.content {
margin-top: 3.5em;
}
.footer {
background: #111;
color: #666;
text-align: center;
padding: 0.5em;
font-size: 80%;
}
.footer a {
color: #888;
}
.icon-fixed-width {
width: 1.28571429em;
display: inline-block;
text-align: center;
}
.pydap-listing {
margin: 0;
width: 100%;
}
.dap, .dap a {
color: #aaa;
font-size: 90%;
visibility: hidden;
}
tbody tr {
border-left: 4px solid #fff;
}
tbody tr:hover {
border-left: 4px solid #15c;
}
tbody tr:hover td {
background-color: #fafafa;
}
tbody tr:hover td .dap,
tbody tr:hover td .dap a {
visibility: visible;
}
div.attributes {
font-size: 90%;
}
dl {
padding-top: 1.65em;
}
dt {
width: 9em;
text-align: right;
color: #999;
overflow: hidden;
}
dd {
position: relative;
margin: -1.65em 0 0 10em;
}
dd.value {
white-space: pre;
}
div.form {
padding: 1.0em;
}
div.nested {
margin-left: -8em;
clear: left;
}
fieldset {
clear: left;
}
.pure-button-submit {
color: white;
border-radius: 4px;
text-shadow: 0 1px 1px rgba(0, 0, 0, 0.2);
}
.pure-button-submit {
background: rgb(28, 184, 65); /* this is a green */
}
Pydap-3.2.2/src/pydap/wsgi/templates/index.html 0000664 0001750 0001750 00000004364 13105357414 022545 0 ustar hiebert hiebert 0000000 0000000 {% extends "base.html" %}
{% block title %}Index of {{ location }}{% endblock %}
{% block breadcrumbs %}
Pydap-3.2.2/src/pydap/parsers/ 0000775 0001750 0001750 00000000000 13111405714 017243 5 ustar hiebert hiebert 0000000 0000000 Pydap-3.2.2/src/pydap/parsers/__init__.py 0000664 0001750 0001750 00000011756 13110635621 021367 0 ustar hiebert hiebert 0000000 0000000 """Parsing related functions.
This module defines functions to parse DAP objects, including a base parser for
DDS and DAS responses.
"""
import re
import operator
import ast
from six.moves.urllib.parse import unquote
from six.moves import map
from ..exceptions import ConstraintExpressionError
from ..lib import get_var
def parse_projection(input):
"""Split a projection into items.
The function takes into account server-side functions, and parse slices
into Python slice objects.
Returns a list of names and slices.
"""
def tokenize(input):
start = pos = count = 0
for char in input:
if char == '(':
count += 1
elif char == ')':
count -= 1
elif char == ',' and count == 0:
yield input[start:pos]
start = pos+1
pos += 1
yield input[start:]
def parse(token):
if '(' not in token:
token = token.split('.')
token = [
re.match(r'(.*?)(\[.*\])?$', part).groups()
for part in token]
token = [
(name, parse_hyperslab(slice_ or ''))
for (name, slice_) in token]
return token
return list(map(parse, tokenize(input)))
def parse_selection(expression, dataset):
"""Parse a selection expression into its elements.
This function will parse a selection expression into three tokens: two
variables or values and a comparison operator. Variables are returned as
Pydap objects from a given dataset, while values are parsed using
``ast.literal_eval``.
"""
id1, op, id2 = re.split('(<=|>=|!=|=~|>|<|=)', expression, 1)
op = {
'<=': operator.le,
'>=': operator.ge,
'!=': operator.ne,
'=': operator.eq,
'>': operator.gt,
'<': operator.lt,
}[op]
try:
id1 = get_var(dataset, id1)
except:
id1 = ast.literal_eval(id1)
try:
id2 = get_var(dataset, id2)
except:
id2 = ast.literal_eval(id2)
return id1, op, id2
def parse_ce(query_string):
"""Extract the projection and selection from the QUERY_STRING.
>>> parse_ce('a,b[0:2:9],c&a>1&b<2') # doctest: +NORMALIZE_WHITESPACE
([[('a', ())], [('b', (slice(0, 10, 2),))], [('c', ())]],
['a>1', 'b<2'])
>>> parse_ce('a>1&b<2')
([], ['a>1', 'b<2'])
This function can also handle function calls in the URL, according to the
DAP specification:
>>> ce = 'time&bounds(0,360,-90,90,0,500,00Z01JAN1970,00Z04JAN1970)'
>>> print(parse_ce(ce)) # doctest: +NORMALIZE_WHITESPACE
([[('time', ())]],
['bounds(0,360,-90,90,0,500,00Z01JAN1970,00Z04JAN1970)'])
>>> ce = 'time,bounds(0,360,-90,90,0,500,00Z01JAN1970,00Z04JAN1970)'
>>> print(parse_ce(ce)) # doctest: +NORMALIZE_WHITESPACE
([[('time', ())],
'bounds(0,360,-90,90,0,500,00Z01JAN1970,00Z04JAN1970)'], [])
>>> parse_ce('mean(g,0)')
(['mean(g,0)'], [])
>>> parse_ce('mean(mean(g.a,1),0)')
(['mean(mean(g.a,1),0)'], [])
Returns a tuple with the projection and the selection.
"""
tokens = [token for token in unquote(query_string).split('&') if token]
if not tokens:
projection = []
selection = []
elif re.search('<=|>=|!=|=~|>|<|=', tokens[0]):
projection = []
selection = tokens
else:
projection = parse_projection(tokens[0])
selection = tokens[1:]
return projection, selection
def parse_hyperslab(hyperslab):
"""Parse a hyperslab, returning a Python tuple of slices."""
exprs = [expr for expr in hyperslab[1:-1].split('][') if expr]
out = []
for expr in exprs:
tokens = list(map(int, expr.split(':')))
start = tokens[0]
step = 1
if len(tokens) == 1:
stop = start + 1
elif len(tokens) == 2:
stop = tokens[1] + 1
elif len(tokens) == 3:
step = tokens[1]
stop = tokens[2] + 1
else:
raise ConstraintExpressionError("Invalid hyperslab %s" % hyperslab)
out.append(slice(start, stop, step))
return tuple(out)
class SimpleParser(object):
"""A very simple parser."""
def __init__(self, input, flags=0):
self.buffer = input
self.flags = flags
def peek(self, regexp):
"""Check if a token is present and return it."""
p = re.compile(regexp, self.flags)
m = p.match(self.buffer)
if m:
token = m.group()
else:
token = ''
return token
def consume(self, regexp):
"""Consume a token from the buffer and return it."""
p = re.compile(regexp, self.flags)
m = p.match(self.buffer)
if m:
token = m.group()
self.buffer = self.buffer[len(token):]
else:
raise Exception("Unable to parse token: %s" % self.buffer[:10])
return token
Pydap-3.2.2/src/pydap/parsers/das.py 0000664 0001750 0001750 00000010102 13111210026 020344 0 ustar hiebert hiebert 0000000 0000000 """A parser for the Dataset Attribute Structure (DAS) response.
This module implements a DAS parser. The ``parse_das`` function will convert a
DAS response into a dictionary of attributes, which can be applied to an
existing dataset using the ``add_attributes`` function.
"""
import re
import ast
import operator
from six.moves import reduce
from . import SimpleParser
from ..lib import walk
class DASParser(SimpleParser):
"""A parser for the Dataset Attribute Structure response."""
def __init__(self, das):
super(DASParser, self).__init__(
das, re.IGNORECASE | re.VERBOSE | re.DOTALL)
def consume(self, regexp):
"""Return a token from the buffer.
Not that it will Ignore white space when consuming tokens.
"""
token = super(DASParser, self).consume(regexp)
self.buffer = self.buffer.lstrip()
return token
def parse(self):
"""Start the parsing, returning a nested dictionary of attributes."""
out = {}
self.consume('attributes')
self.container(out)
return out
def container(self, target):
"""Collect the attributes for a DAP variable."""
self.consume('{')
while not self.peek('}'):
if self.peek(r'[^\s]+\s+{'):
name = self.consume(r'[^\s]+')
target[name] = {}
self.container(target[name])
else:
name, values = self.attribute()
target[name] = values
self.consume('}')
def attribute(self):
"""Parse attributes.
The function will parse attributes from the DAS, converting them to the
corresponding Python object. Returns the name of the attribute and the
attribute(s).
"""
type = self.consume(r'[^\s]+')
name = self.consume(r'[^\s]+')
values = []
while not self.peek(';'):
value = self.consume(
r'''
"" # empty attribute
| # or
".*?[^\\]" # from quote up to an unquoted quote
| # or
[^;,]+ # up to semicolon or comma
'''
)
if type.lower() in ['string', 'url']:
value = str(value).strip('"')
elif value.lower() in ['nan', 'nan.', '-nan']:
value = float('nan')
else:
value = ast.literal_eval(value)
values.append(value)
if self.peek(','):
self.consume(',')
self.consume(';')
if len(values) == 1:
values = values[0]
return name, values
def parse_das(das):
"""Parse the DAS, returning nested dictionaries."""
return DASParser(das).parse()
def add_attributes(dataset, attributes):
"""Add attributes from a parsed DAS to a dataset.
Returns the dataset with added attributes.
"""
dataset.attributes['NC_GLOBAL'] = attributes.get('NC_GLOBAL', {})
dataset.attributes['DODS_EXTRA'] = attributes.get('DODS_EXTRA', {})
for var in list(walk(dataset))[::-1]:
# attributes can be flat, eg, "foo.bar" : {...}
if var.id in attributes:
var.attributes.update(attributes.pop(var.id))
# or nested, eg, "foo" : { "bar" : {...} }
try:
nested = reduce(
operator.getitem, [attributes] + var.id.split('.')[:-1])
k = var.id.split('.')[-1]
value = nested.pop(k)
except KeyError:
pass
else:
try:
var.attributes.update(value)
except (TypeError, ValueError):
# This attribute should be given to the parent.
# Keep around:
nested.update({k: value})
# add attributes that don't belong to any child
for k, v in attributes.items():
dataset.attributes[k] = v
return dataset
Pydap-3.2.2/src/pydap/parsers/dds.py 0000664 0001750 0001750 00000010736 13111210026 020364 0 ustar hiebert hiebert 0000000 0000000 """A DDS parser."""
import re
import numpy as np
from . import SimpleParser
from ..model import (DatasetType, BaseType,
SequenceType, StructureType,
GridType)
from ..lib import (quote, LOWER_DAP2_TO_NUMPY_PARSER_TYPEMAP)
constructors = ('grid', 'sequence', 'structure')
name_regexp = r'[\w%!~"\'\*-]+'
def DAP2_parser_typemap(type_string):
"""
This function takes a numpy dtype object
and returns a dtype object that is compatible with
the DAP2 specification.
"""
dtype_str = LOWER_DAP2_TO_NUMPY_PARSER_TYPEMAP[type_string.lower()]
return np.dtype(dtype_str)
class DDSParser(SimpleParser):
"""A parser for the DDS."""
def __init__(self, dds):
super(DDSParser, self).__init__(dds, re.IGNORECASE)
self.dds = dds
def consume(self, regexp):
"""Consume and return a token."""
token = super(DDSParser, self).consume(regexp)
self.buffer = self.buffer.lstrip()
return token
def parse(self):
"""Parse the DAS, returning a dataset."""
dataset = DatasetType('nameless')
self.consume('dataset')
self.consume('{')
while not self.peek('}'):
var = self.declaration()
dataset[var.name] = var
self.consume('}')
dataset.name = quote(self.consume('[^;]+'))
dataset._set_id(dataset.name)
self.consume(';')
return dataset
def declaration(self):
"""Parse and return a declaration."""
token = self.peek(r'\w+').lower()
map = {
'grid': self.grid,
'sequence': self.sequence,
'structure': self.structure,
}
method = map.get(token, self.base)
return method()
def base(self):
"""Parse a base variable, returning a ``BaseType``."""
data_type_string = self.consume('\w+')
parser_dtype = DAP2_parser_typemap(data_type_string)
name = quote(self.consume('[^;\[]+'))
shape, dimensions = self.dimensions()
self.consume(';')
data = DummyData(parser_dtype, shape)
var = BaseType(name, data, dimensions=dimensions)
return var
def dimensions(self):
"""Parse variable dimensions, returning tuples of dimensions/names."""
shape = []
names = []
while not self.peek(';'):
self.consume(r'\[')
token = self.consume(name_regexp)
if self.peek('='):
names.append(token)
self.consume('=')
token = self.consume(r'\d+')
shape.append(int(token))
self.consume(r'\]')
return tuple(shape), tuple(names)
def sequence(self):
"""Parse a DAS sequence, returning a ``SequenceType``."""
sequence = SequenceType('nameless')
self.consume('sequence')
self.consume('{')
while not self.peek('}'):
var = self.declaration()
sequence[var.name] = var
self.consume('}')
sequence.name = quote(self.consume('[^;]+'))
self.consume(';')
return sequence
def structure(self):
"""Parse a DAP structure, returning a ``StructureType``."""
structure = StructureType('nameless')
self.consume('structure')
self.consume('{')
while not self.peek('}'):
var = self.declaration()
structure[var.name] = var
self.consume('}')
structure.name = quote(self.consume('[^;]+'))
self.consume(';')
return structure
def grid(self):
"""Parse a DAP grid, returning a ``GridType``."""
grid = GridType('nameless')
self.consume('grid')
self.consume('{')
self.consume('array')
self.consume(':')
array = self.base()
grid[array.name] = array
self.consume('maps')
self.consume(':')
while not self.peek('}'):
var = self.base()
grid[var.name] = var
self.consume('}')
grid.name = quote(self.consume('[^;]+'))
self.consume(';')
return grid
def build_dataset(dds):
"""Return a dataset object from a DDS representation."""
return DDSParser(dds).parse()
class DummyData(object):
def __init__(self, dtype, shape):
self.dtype = dtype
self.shape = shape
Pydap-3.2.2/src/pydap/net.py 0000664 0001750 0001750 00000007730 13111210026 016721 0 ustar hiebert hiebert 0000000 0000000 from webob.request import Request
from webob.exc import HTTPError
from contextlib import closing
import requests
from requests.exceptions import (MissingSchema, InvalidSchema,
Timeout)
from six.moves.urllib.parse import urlsplit, urlunsplit
from .lib import DEFAULT_TIMEOUT
def GET(url, application=None, session=None, timeout=DEFAULT_TIMEOUT):
"""Open a remote URL returning a webob.response.Response object
Optional parameters:
session: a requests.Session() object (potentially) containing
authentication cookies.
Optionally open a URL to a local WSGI application
"""
if application:
_, _, path, query, fragment = urlsplit(url)
url = urlunsplit(('', '', path, query, fragment))
return follow_redirect(url, application=application, session=session,
timeout=timeout)
def raise_for_status(response):
# Raise error if status is above 300:
if response.status_code >= 300:
raise HTTPError(
detail=response.status+'\n'+response.text,
headers=response.headers,
comment=response.body
)
def follow_redirect(url, application=None, session=None,
timeout=DEFAULT_TIMEOUT):
"""
This function essentially performs the following command:
>>> Request.blank(url).get_response(application) # doctest: +SKIP
It however makes sure that the request possesses the same cookies and
headers as the passed session.
"""
req = create_request(url, session=session, timeout=timeout)
return req.get_response(application)
def create_request(url, session=None, timeout=DEFAULT_TIMEOUT):
if session is not None:
# If session is set and cookies were loaded using pydap.cas.get_cookies
# using the check_url option, then we can legitimately expect that
# the connection will go through seamlessly. However, there might be
# redirects that might want to modify the cookies. Webob is not
# really up to the task here. The approach used here is to
# piggy back on the requests library and use it to fetch the
# head of the requested url. Requests will follow redirects and
# adjust the cookies as needed. We can then use the final url and
# the final cookies to set up a webob Request object that will
# be guaranteed to have all the needed credentials:
return create_request_from_session(url, session, timeout=timeout)
else:
# If a session object was not passed, we simply pass a new
# requests.Session() object. The requests library allows the
# handling of redirects that are not naturally handled by Webob.
return create_request_from_session(url, requests.Session(),
timeout=timeout)
def create_request_from_session(url, session, timeout=DEFAULT_TIMEOUT):
try:
# Use session to follow redirects:
with closing(session.head(url, allow_redirects=True,
timeout=timeout)) as head:
req = Request.blank(head.url)
req.environ['webob.client.timeout'] = timeout
# Get cookies from head:
cookies_dict = head.cookies.get_dict()
# Set request cookies to the head cookies:
req.headers['Cookie'] = ','.join(name + '=' +
cookies_dict[name]
for name in cookies_dict)
# Set the headers to the session headers:
for item in head.request.headers:
req.headers[item] = head.request.headers[item]
return req
except (MissingSchema, InvalidSchema):
# Missing schema can occur in tests when the url
# is not pointing to any resource. Simply pass.
req = Request.blank(url)
req.environ['webob.client.timeout'] = timeout
return req
except Timeout:
raise HTTPError('Timeout')
Pydap-3.2.2/src/pydap/cas/ 0000775 0001750 0001750 00000000000 13111405714 016332 5 ustar hiebert hiebert 0000000 0000000 Pydap-3.2.2/src/pydap/cas/esgf.py 0000664 0001750 0001750 00000003157 13105357414 017644 0 ustar hiebert hiebert 0000000 0000000 from six.moves.urllib.parse import quote_plus
from . import get_cookies
def setup_session(openid, password, username=None,
check_url=None,
session=None, verify=False):
"""
A special call to get_cookies.setup_session that is tailored for
ESGF credentials.
username should only be necessary for a CEDA openid.
"""
session = get_cookies.setup_session(_uri(openid),
username=username,
password=password,
check_url=check_url,
session=session,
verify=verify)
# Connections can be kept alive on the ESGF:
session.headers.update([('Connection', 'keep-alive')])
return session
def _uri(openid):
'''
Create ESGF authentication url.
This function might be sensitive to a
future evolution of the ESGF security.
'''
def generate_url(dest_url):
dest_node = _get_node(dest_url)
try:
url = (dest_node +
'/esg-orp/j_spring_openid_security_check.htm?'
'openid_identifier=' +
quote_plus(openid))
except TypeError:
raise UserWarning('OPENID was not set. '
'ESGF connection cannot succeed.')
if _get_node(openid) == 'https://ceda.ac.uk':
return [url, None]
else:
return url
return generate_url
def _get_node(url):
return '/'.join(url.split('/')[:3]).replace('http:', 'https:')
Pydap-3.2.2/src/pydap/cas/__init__.py 0000664 0001750 0001750 00000000000 13105357414 020437 0 ustar hiebert hiebert 0000000 0000000 Pydap-3.2.2/src/pydap/cas/urs.py 0000664 0001750 0001750 00000001417 13105357414 017526 0 ustar hiebert hiebert 0000000 0000000 from . import get_cookies
def setup_session(username, password, check_url=None,
session=None, verify=True):
"""
A special call to get_cookies.setup_session that is tailored for
URS EARTHDATA at NASA credentials.
"""
if session is not None:
# URS connections cannot be kept alive at the moment.
session.headers.update({'Connection': 'close'})
session = get_cookies.setup_session('https://urs.earthdata.nasa.gov',
username=username,
password=password,
session=session,
check_url=check_url,
verify=verify)
return session
Pydap-3.2.2/src/pydap/cas/get_cookies.py 0000664 0001750 0001750 00000014063 13111210026 021171 0 ustar hiebert hiebert 0000000 0000000 from bs4 import BeautifulSoup
from six.moves.urllib.parse import urlsplit, urlunsplit
import warnings
import requests
from requests.packages.urllib3.exceptions import (InsecureRequestWarning,
InsecurePlatformWarning)
import copy
from .. import lib
ssl_verify_categories = [InsecureRequestWarning,
InsecurePlatformWarning]
def setup_session(uri,
username=None,
password=None,
check_url=None,
session=None,
verify=True,
username_field='username',
password_field='password'):
'''
A general function to set-up requests session with cookies
using beautifulsoup and by calling the right url.
'''
if session is None:
# Connections must be closed since some CAS
# will cough when connections are kept alive:
headers = [('User-agent', 'Pydap/{}'.format(lib.__version__)),
('Connection', 'close')]
session = requests.Session()
session.headers.update(headers)
if uri is None:
return session
if not verify:
verify_flag = session.verify
session.verify = False
if isinstance(uri, str):
url = uri
else:
url = uri(check_url)
if password is None or password == '':
warnings.warn('password was not set. '
'this was likely unintentional '
'but will result is much fewer datasets.')
if not verify:
session.verify = verify_flag
return session
# Allow for several subsequent security layers:
full_url = copy.copy(url)
if isinstance(full_url, list):
url = full_url[0]
with warnings.catch_warnings():
if not verify:
# Catch warnings. It is assumed that the
# user that explicitly uses verify=False
# is either fully aware of the risks
# or cannot avoid the risks because of
# an improperly configured server.
# This error will usually occur with
# ESGF authentication.
for category in ssl_verify_categories:
warnings.filterwarnings("ignore",
category=category)
response = soup_login(session, url, username, password,
username_field=username_field,
password_field=password_field)
# If there are further security levels.
# At the moment only used for CEDA OPENID:
if (isinstance(full_url, list) and
len(full_url) > 1):
for url in full_url[1:]:
response = soup_login(session, response.url,
username, password,
username_field=None,
password_field=None)
response.close()
if check_url:
if (username is not None and
password is not None):
res = session.get(check_url, auth=(username, password))
if res.status_code == 401:
res = session.get(res.url, auth=(username, password))
res.close()
raise_if_form_exists(check_url, session)
if not verify:
session.verify = verify_flag
return session
def raise_if_form_exists(url, session):
"""
This function raises a UserWarning if the link has forms
"""
user_warning = ('Navigate to {0}, '.format(url) +
'login and follow instructions. '
'It is likely that you have to perform some one-time '
'registration steps before acessing this data.')
resp = session.get(url)
soup = BeautifulSoup(resp.content, 'lxml')
if len(soup.select('form')) > 0:
raise UserWarning(user_warning)
def soup_login(session, url, username, password,
username_field='username',
password_field='password'):
resp = session.get(url)
soup = BeautifulSoup(resp.content, 'lxml')
login_form = soup.select('form')[0]
def get_to_url(current_url, to_url):
split_current = urlsplit(current_url)
split_to = urlsplit(to_url)
comb = [val2 if val1 == '' else val1
for val1, val2 in zip(split_to, split_current)]
return urlunsplit(comb)
to_url = get_to_url(resp.url, login_form.get('action'))
session.headers['Referer'] = resp.url
payload = {}
if username_field is not None:
if len(login_form.findAll('input', {'name': username_field})) > 0:
payload.update({username_field: username})
if password_field is not None:
if len(login_form.findAll('input', {'name': password_field})) > 0:
payload.update({password_field: password})
else:
# If there is no password_field, it might be because
# something should be handled in the browser
# for the first attempt. This is common when using
# pydap with the ESGF for the first time.
raise Exception('Navigate to {0}. '
'If you are unable to '
'login, you must either '
'wait or use authentication '
'from another service.'
.format(url))
# Replicate all other fields:
for input in login_form.findAll('input'):
if (input.get('name') not in payload and
input.get('name') is not None):
payload.update({input.get('name'): input.get('value')})
# Remove other submit fields:
submit_type = 'submit'
submit_names = [input.get('name') for input
in login_form.findAll('input', {'type': submit_type})]
for input in login_form.findAll('input', {'type': submit_type}):
if ('submit' in submit_names and
input.get('name').lower() != 'submit'):
payload.pop(input.get('name'), None)
return session.post(to_url, data=payload)
Pydap-3.2.2/src/pydap/__init__.py 0000664 0001750 0001750 00000000147 13105357414 017705 0 ustar hiebert hiebert 0000000 0000000 '''
Declare the namespace ``pydap`` here.
'''
__import__('pkg_resources').declare_namespace(__name__)
Pydap-3.2.2/src/pydap/exceptions.py 0000664 0001750 0001750 00000001761 13105357414 020332 0 ustar hiebert hiebert 0000000 0000000 """DAP exceptions.
These exceptions are mostly used by the server. When an exception is captured,
proper error message is displayed (according to the DAP 2.0 spec), with
information about the exception.
"""
class DapError(Exception):
"""Base DAP exception."""
def __init__(self, value):
self.value = value
def __str__(self):
return repr(self.value)
class ClientError(DapError):
"""Generic error with the client."""
pass
class ServerError(DapError):
"""Generic error with the server."""
pass
class ConstraintExpressionError(ServerError):
"""Exception raised when an invalid constraint expression is given."""
pass
class HandlerError(DapError):
"""Generic error with a handler."""
pass
class ExtensionNotSupportedError(HandlerError):
"""Exception raised when opening a file not supported by any handlers."""
pass
class OpenFileError(HandlerError):
"""Exception raised when unable to open a file."""
pass
Pydap-3.2.2/src/pydap/client.py 0000664 0001750 0001750 00000014324 13111210026 017406 0 ustar hiebert hiebert 0000000 0000000 """
Pydap client.
This module contains functions to access DAP servers. The most common use is to
open a dataset by its canonical URL, ie, without any DAP related extensions
like dds/das/dods/html. Here is an example:
>>> from pydap.client import open_url
>>> dataset = open_url("http://test.pydap.org/coads.nc")
This will return a `DatasetType` object, which is a container for lazy
evaluated objects. Data is downloaded automatically when arrays are sliced or
when sequences are iterated.
It is also possible to download data directly from a dods (binary) response.
This allows calling server-specific functions, like those supported by the
Ferret and the GrADS data servers:
>>> from pydap.client import open_dods
>>> dataset = open_dods(
... "http://test.pydap.org/coads.nc.dods",
... metadata=True)
Setting the `metadata` flag will also request the das response, populating the
dataset with the corresponding metadata.
If the dods response has already been downloaded, it is possible to open it as
if it were a remote dataset. Optionally, it is also possible to specify a das
response:
>>> from pydap.client import open_file
>>> dataset = open_file(
... "/path/to/file.dods", "/path/to/file.das") #doctest: +SKIP
Remote datasets opened with `open_url` can call server functions. Pydap has a
lazy mechanism for function call, supporting any function. Eg, to call the
`geogrid` function on the server:
>>> dataset = open_url(
... 'http://test.opendap.org/dap/data/nc/coads_climatology.nc')
>>> new_dataset = dataset.functions.geogrid(dataset.SST, 10, 20, -10, 60)
>>> print(new_dataset.SST.SST.shape) #doctest: +SKIP
(12, 12, 21)
"""
from io import open, BytesIO
from six.moves.urllib.parse import urlsplit, urlunsplit
from .model import DapType
from .lib import encode, DEFAULT_TIMEOUT
from .net import GET, raise_for_status
from .handlers.dap import DAPHandler, unpack_data, StreamReader
from .parsers.dds import build_dataset
from .parsers.das import parse_das, add_attributes
def open_url(url, application=None, session=None, output_grid=True,
timeout=DEFAULT_TIMEOUT):
"""
Open a remote URL, returning a dataset.
set output_grid to False to retrieve only main arrays and
never retrieve coordinate axes.
"""
dataset = DAPHandler(url, application, session, output_grid,
timeout).dataset
# attach server-side functions
dataset.functions = Functions(url, application, session)
return dataset
def open_file(dods, das=None):
"""Open a file downloaded from a `.dods` response, returning a dataset.
Optionally, read also the `.das` response to assign attributes to the
dataset.
"""
dds = ''
# This file contains both ascii _and_ binary data
# Let's handle them separately in sequence
# Without ignoring errors, the IO library will
# actually read past the ascii part of the
# file (despite our break from iteration) and
# will error out on the binary data
with open(dods, "rt", buffering=1, encoding='ascii',
newline='\n', errors='ignore') as f:
for line in f:
if line.strip() == 'Data:':
break
dds += line
dataset = build_dataset(dds)
pos = len(dds) + len('Data:\n')
with open(dods, "rb") as f:
f.seek(pos)
dataset.data = unpack_data(f, dataset)
if das is not None:
with open(das) as f:
add_attributes(dataset, parse_das(f.read()))
return dataset
def open_dods(url, metadata=False, application=None, session=None,
timeout=DEFAULT_TIMEOUT):
"""Open a `.dods` response directly, returning a dataset."""
r = GET(url, application, session, timeout=timeout)
raise_for_status(r)
dds, data = r.body.split(b'\nData:\n', 1)
dds = dds.decode(r.content_encoding or 'ascii')
dataset = build_dataset(dds)
stream = StreamReader(BytesIO(data))
dataset.data = unpack_data(stream, dataset)
if metadata:
scheme, netloc, path, query, fragment = urlsplit(url)
dasurl = urlunsplit(
(scheme, netloc, path[:-4] + 'das', query, fragment))
r = GET(dasurl, application, session, timeout=timeout)
raise_for_status(r)
das = r.text
add_attributes(dataset, parse_das(das))
return dataset
class Functions(object):
"""Proxy for server-side functions."""
def __init__(self, baseurl, application=None, session=None):
self.baseurl = baseurl
self.application = application
self.session = session
def __getattr__(self, attr):
return ServerFunction(self.baseurl, attr, self.application,
self.session)
class ServerFunction(object):
"""A proxy for a server-side function.
Instead of returning datasets, the function will return a proxy object,
allowing nested requests to be performed on the server.
"""
def __init__(self, baseurl, name, application=None, session=None):
self.baseurl = baseurl
self.name = name
self.application = application
self.session = None
def __call__(self, *args):
params = []
for arg in args:
if isinstance(arg, (DapType, ServerFunctionResult)):
params.append(arg.id)
else:
params.append(encode(arg))
id_ = self.name + '(' + ','.join(params) + ')'
return ServerFunctionResult(self.baseurl, id_, self.application,
self.session)
class ServerFunctionResult(object):
"""A proxy for the result from a server-side function call."""
def __init__(self, baseurl, id_, application=None, session=None):
self.id = id_
self.dataset = None
self.application = application
self.session = session
scheme, netloc, path, query, fragment = urlsplit(baseurl)
self.url = urlunsplit((scheme, netloc, path + '.dods', id_, None))
def __getitem__(self, key):
if self.dataset is None:
self.dataset = open_dods(self.url, True, self.application,
self.session)
return self.dataset[key]
def __getattr__(self, name):
return self[name]
Pydap-3.2.2/src/pydap/server/ 0000775 0001750 0001750 00000000000 13111405714 017072 5 ustar hiebert hiebert 0000000 0000000 Pydap-3.2.2/src/pydap/server/__init__.py 0000664 0001750 0001750 00000000000 13110635621 021172 0 ustar hiebert hiebert 0000000 0000000 Pydap-3.2.2/src/pydap/server/devel.py 0000664 0001750 0001750 00000010017 13110635621 020543 0 ustar hiebert hiebert 0000000 0000000 from webob.request import Request
from webob.exc import HTTPError
import threading
import time
import math
import numpy as np
import socket
from wsgiref.simple_server import make_server
from ..handlers.lib import BaseHandler
from ..model import BaseType, DatasetType
from ..wsgi.ssf import ServerSideFunctions
DefaultDataset = DatasetType("Default")
DefaultDataset["byte"] = BaseType("byte", np.arange(5, dtype="B"))
DefaultDataset["string"] = BaseType("string", np.array(["one", "two"]))
DefaultDataset["short"] = BaseType("short", np.array(1, dtype="h"))
def get_open_port():
# http://stackoverflow.com/questions/2838244/
# get-open-tcp-port-in-python/2838309#2838309
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind(("", 0))
s.listen(1)
port = s.getsockname()[1]
s.close()
return port
def run_simple_server(port, application):
application = ServerSideFunctions(application)
return make_server('0.0.0.0', port, application)
class LocalTestServer:
"""
Simple server instance that can be used to test pydap.
Relies on threading and is usually slow (it has to
start and shutdown which typically takes ~2 sec).
Usage:
>>> import numpy as np
>>> from pydap.handlers.lib import BaseHandler
>>> from pydap.model import DatasetType, BaseType
>>> DefaultDataset = DatasetType("Default")
>>> DefaultDataset["byte"] = BaseType("byte", np.arange(5, dtype="B"))
>>> DefaultDataset["string"] = BaseType("string", np.array(["one", "two"]))
>>> DefaultDataset["short"] = BaseType("short", np.array(1, dtype="h"))
>>> DefaultDataset
>>> application = BaseHandler(DefaultDataset)
>>> from pydap.client import open_url
As an instance:
>>> with LocalTestServer(application) as server:
... dataset = open_url("http://localhost:%s" % server.port)
... dataset
... print(dataset['byte'].data[:])
... print(dataset['string'].data[:])
... print(dataset['short'].data[:])
[0 1 2 3 4]
[b'one' b'two']
1
Or by managing connection and deconnection:
>>> server = LocalTestServer(application)
>>> server.start()
>>> dataset = open_url("http://localhost:%s" % server.port)
>>> dataset
>>> print(dataset['byte'].data[:])
[0 1 2 3 4]
>>> server.shutdown()
"""
def __init__(self, application=BaseHandler(DefaultDataset),
port=None, wait=0.5, polling=1e-2):
self._port = port or get_open_port()
self.application = application
self._wait = wait
self._polling = polling
def start(self):
# Start a simple WSGI server:
self.httpd = run_simple_server(self.port, self.application)
self.server_process = (threading
.Thread(target=self.httpd.serve_forever,
kwargs={'poll_interval': 1e-2}))
self.server_process.start()
# Poll the server
ok = False
for trial in range(int(math.ceil(self._wait/self._polling))):
try:
resp = (Request
.blank("http://0.0.0.0:%s/.dds" % self.port)
.get_response())
ok = (resp.status_code == 200)
except HTTPError:
pass
if ok:
break
time.sleep(self._polling)
if not ok:
raise Exception(('LocalTestServer did not start in {0}s. '
'Try using LocalTestServer(..., wait={1}')
.format(self._wait, 2*self._wait))
@property
def port(self):
return self._port
def __enter__(self):
self.start()
return self
def shutdown(self):
# Shutdown the server:
self.httpd.shutdown()
self.server_process.join()
def __exit__(self, *_):
self.shutdown()
Pydap-3.2.2/src/pydap/responses/ 0000775 0001750 0001750 00000000000 13111405714 017605 5 ustar hiebert hiebert 0000000 0000000 Pydap-3.2.2/src/pydap/responses/version.py 0000664 0001750 0001750 00000003030 13110635621 021641 0 ustar hiebert hiebert 0000000 0000000 """Response with information about Pydap version."""
import sys
from six import text_type
from webob import Response
from json import dumps
from pkg_resources import iter_entry_points
from ..lib import __version__, __dap__
class VersionResponse(object):
"""A specialized response for debugging.
This is a special response used to display information about the server.
"""
__version__ = __version__
def __init__(self, dataset):
output = {
"pydap": __version__,
"dap": __dap__,
"handlers": dict(
(ep.name, getattr(ep.load(), "__version__", "Unknown"))
for ep in iter_entry_points("pydap.handler")
),
"responses": dict(
(ep.name, getattr(ep.load(), "__version__", "Unknown"))
for ep in iter_entry_points("pydap.response")
),
"functions": dict(
(ep.name, getattr(ep.load(), "__version__", "Unknown"))
for ep in iter_entry_points("pydap.function")
),
"python": sys.version,
}
self.body = dumps(output, indent=4)
def __call__(self, environ, start_response):
res = Response()
res.text = text_type(self.body)
res.status = '200 OK'
res.content_type = 'application/json'
res.charset = 'utf-8'
res.headers.add('Content-description', 'dods_version')
res.headers.add('XDODS-Server', 'pydap/%s' % __version__)
return res(environ, start_response)
Pydap-3.2.2/src/pydap/responses/lib.py 0000664 0001750 0001750 00000006066 13110635621 020736 0 ustar hiebert hiebert 0000000 0000000 """Fundamental functions for Pydap responses.
Pydap responses are WSGI applications that convert a dataset into different
representations, like the DDS, DAS and DODS responses described in the DAP
specification.
In addition to the official responses, Pydap also has responses that generate
KML, WMS, JSON, etc., installed as third-party Python packages that declare the
"pydap.response" entry point.
"""
from pkg_resources import iter_entry_points
from ..model import DatasetType
from ..lib import __version__, load_from_entry_point_relative
def load_responses():
"""Load all available responses from the system, returning a dictionary."""
# Relative import of responses:
package = 'pydap'
entry_points = 'pydap.response'
base_dict = dict(load_from_entry_point_relative(r, package)
for r in iter_entry_points(entry_points)
if r.module_name.startswith(package))
opts_dict = dict((r.name, r.load())
for r in iter_entry_points(entry_points)
if not r.module_name.startswith(package))
base_dict.update(opts_dict)
return base_dict
class BaseResponse(object):
"""A base class for Pydap responses.
A Pydap response is a WSGI application that converts a dataset into any
other representation. The most know responses are the DDS, DAS and DODS
responses from the DAP spec, which describe the dataset structure,
attributes and data, respectively.
According to the WSGI specification, WSGI applications must returned an
iterable object when called. While this is traditionally a list of strings
representing an HTML response, this is not the case for Pydap. Pydap will
return an object (the response instance itself), which is an iterable that
yields the corresponding output (a DDS response, eg).
In practice, this means that the generation of the response is delayed
until the data is being sent to the client. But since the response object
also carries the original dataset, this means it's possible to write WSGI
middleware that modifies the dataset directly. A WSGI middleware can add
additional metadata to a dataset, eg, by adding attributes directly to the
dataset object, without having to generate a new response.
"""
def __init__(self, dataset):
self.dataset = dataset
self.headers = [
('XDODS-Server', 'pydap/%s' % __version__),
]
def __call__(self, environ, start_response):
start_response('200 OK', self.headers)
return self
def x_wsgiorg_parsed_response(self, type):
r"""Avoid serialization of datasets.
This function will return the contained dataset if ``type`` is a
``pydap.model.DatasetType`` object. Based on this proposal:
http://wsgi.readthedocs.org/en/latest/specifications/ \
avoiding_serialization.html
"""
if type is DatasetType:
return self.dataset
def __iter__(self):
raise NotImplementedError(
'Subclasses must implement __iter__')
Pydap-3.2.2/src/pydap/responses/__init__.py 0000664 0001750 0001750 00000000070 13105357414 021721 0 ustar hiebert hiebert 0000000 0000000 __import__('pkg_resources').declare_namespace(__name__)
Pydap-3.2.2/src/pydap/responses/ascii.py 0000664 0001750 0001750 00000004400 13110635621 021246 0 ustar hiebert hiebert 0000000 0000000 """The ASCII response.
The ASCII response is an unnoficial response used to return the data as ASCII.
Pydap's implementation is reverse engineered from the official server.
"""
try:
from functools import singledispatch
except ImportError:
from singledispatch import singledispatch
import copy
import numpy as np
from six.moves import zip
from ..model import (BaseType,
SequenceType, StructureType)
from ..lib import encode, __version__
from .lib import BaseResponse
from .dds import dds
class ASCIIResponse(BaseResponse):
"""The ASCII response."""
__version__ = __version__
def __init__(self, dataset):
BaseResponse.__init__(self, dataset)
self.headers.extend([
('Content-description', 'dods_ascii'),
('Content-type', 'text/plain; charset=ascii'),
])
def __iter__(self):
for line in dds(self.dataset):
yield line.encode('ascii')
yield (45 * '-' + '\n').encode('ascii')
for line in ascii(self.dataset):
yield line.encode('ascii')
@singledispatch
def ascii(var, printname=True):
"""A single dispatcher for the ASCII response."""
raise StopIteration
@ascii.register(SequenceType)
def _sequenctype(var, printname=True):
yield ', '.join([child.id for child in var.children()])
yield '\n'
for rec in var.iterdata():
out = copy.copy(var)
out.__class__ = StructureType
out.data = rec
for i, line in enumerate(ascii(out, printname=False)):
line = line.strip()
if line and i > 0:
yield ', '
yield line
yield '\n'
@ascii.register(StructureType)
def _structuretype(var, printname=True):
for child in var.children():
for line in ascii(child, printname):
yield line
yield '\n'
@ascii.register(BaseType)
def _basetype(var, printname=True):
if printname:
yield var.id
yield '\n'
if not getattr(var, "shape", ()):
yield encode(var.data)
else:
for indexes, value in zip(np.ndindex(var.shape), var.data.flat):
yield "{indexes} {value}\n".format(
indexes="[" + "][".join([str(idx) for idx in indexes]) + "]",
value=encode(value))
Pydap-3.2.2/src/pydap/responses/html/ 0000775 0001750 0001750 00000000000 13111405714 020551 5 ustar hiebert hiebert 0000000 0000000 Pydap-3.2.2/src/pydap/responses/html/__init__.py 0000664 0001750 0001750 00000006500 13110635621 022664 0 ustar hiebert hiebert 0000000 0000000 """HTML DAP response
This is a simple HTML response that allows that to be analysed on the browser.
The user can select a subset of the data and download in different formats.
"""
from jinja2 import Environment, PackageLoader, ChoiceLoader
from webob import Response
from webob.dec import wsgify
from webob.exc import HTTPSeeOther
from six.moves.urllib.parse import unquote
from ..lib import BaseResponse
from ...lib import __version__
class HTMLResponse(BaseResponse):
"""A simple HTML response for browsing and downloading data."""
__version__ = __version__
def __init__(self, dataset):
BaseResponse.__init__(self, dataset)
self.headers.extend([
("Content-description", "dods_form"),
("Content-type", "text/html; charset=utf-8"),
])
# our default environment; we need to include the base template from
# pydap as well since our template extends it
self.loaders = [
PackageLoader("pydap.responses.html", "templates"),
PackageLoader("pydap.wsgi", "templates"),
]
@wsgify
def __call__(self, req):
# if request is a post we should redict to ASCII response
if req.method == "POST":
return self.redirect(req)
# check if the server has specified a render environment; if it has,
# make a copy and add our loaders to it
if "pydap.jinja2.environment" in req.environ:
env = req.environ["pydap.jinja2.environment"].overlay()
env.loader = ChoiceLoader([
loader for loader in [env.loader] + self.loaders if loader])
else:
env = Environment(loader=ChoiceLoader(self.loaders))
env.filters["unquote"] = unquote
template = env.get_template("html.html")
tokens = req.path_info.split("/")[1:]
breadcrumbs = [{
"url": "/".join([req.application_url] + tokens[:i+1]),
"title": token,
} for i, token in enumerate(tokens) if token]
context = {
"root": req.application_url,
"location": req.path_url,
"breadcrumbs": breadcrumbs,
"dataset": self.dataset,
"version": __version__,
}
return Response(
body=template.render(context),
headers=self.headers)
def redirect(self, req):
"""Return a redirect to the ASCII response."""
projection, selection = [], []
for k in req.params:
# selection
if k.startswith("var1_") and req.params[k] != "--":
name = k[5:]
tokens = (
req.params[k],
req.params["op_%s" % name],
req.params["var2_%s" % name])
selection.append("".join(tokens))
# projection
if req.params[k] == "on":
tokens = [k]
i = 0
while "%s[%d]" % (k, i) in req.params:
tokens.append("[%s]" % req.params["%s[%d]" % (k, i)])
i += 1
projection.append("".join(tokens))
# send to ASCII response
location = "{0}.ascii?{1}&{2}".format(
req.path_url[:-5],
",".join(projection),
"&".join(selection)).rstrip("?&")
return HTTPSeeOther(location=location)
Pydap-3.2.2/src/pydap/responses/html/templates/ 0000775 0001750 0001750 00000000000 13111405714 022547 5 ustar hiebert hiebert 0000000 0000000 Pydap-3.2.2/src/pydap/responses/html/templates/macros.html 0000664 0001750 0001750 00000005477 13105357414 024744 0 ustar hiebert hiebert 0000000 0000000 {% macro dispatch(var) %}