pax_global_header00006660000000000000000000000064146122070010014503gustar00rootroot0000000000000052 comment=64a5df00eb086cd9389ae602d8d7f456cd9e4be2 cdsapi-0.7.0/000077500000000000000000000000001461220700100127525ustar00rootroot00000000000000cdsapi-0.7.0/.github/000077500000000000000000000000001461220700100143125ustar00rootroot00000000000000cdsapi-0.7.0/.github/workflows/000077500000000000000000000000001461220700100163475ustar00rootroot00000000000000cdsapi-0.7.0/.github/workflows/check-and-publish.yml000066400000000000000000000047531461220700100223640ustar00rootroot00000000000000name: Check and publish on: push: branches: [master] pull_request: branches: [master] # Trigger on public pull request approval pull_request_target: types: [labeled] release: types: [created] jobs: quality-checks: name: Code QA runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - run: pip install black flake8 isort - run: black --version - run: isort --version - run: flake8 --version - run: isort --check . - run: black --check . - run: flake8 . platform-checks: needs: quality-checks if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} strategy: fail-fast: false matrix: platform: [windows-latest, ubuntu-latest, macos-latest] python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] exclude: - platform: macos-latest python-version: "3.8" - platform: macos-latest python-version: "3.9" name: Python ${{ matrix.python-version }} on ${{ matrix.platform }} runs-on: ${{ matrix.platform }} timeout-minutes: 20 steps: - uses: actions/checkout@v3 with: ref: ${{ github.event.pull_request.head.sha || github.ref }} - uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - name: Tests env: CDSAPI_URL: https://cds.climate.copernicus.eu/api/v2 CDSAPI_KEY: ${{ secrets.CDSAPI_KEY }} run: | pip install setuptools python setup.py develop pip install pytest pytest deploy: needs: platform-checks if: ${{ github.event_name == 'release' }} name: Upload to Pypi runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 with: ref: ${{ github.event.pull_request.head.sha || github.ref }} - name: Build distributions run: | $CONDA/bin/python -m pip install build $CONDA/bin/python -m build - name: Publish a Python distribution to PyPI uses: pypa/gh-action-pypi-publish@release/v1 with: user: __token__ password: ${{ secrets.PYPI_API_TOKEN }} - name: Notify climetlab uses: mvasigh/dispatch-action@main with: token: ${{ secrets.NOTIFY_ECMWFLIBS }} repo: climetlab owner: ecmwf event_type: cdsapi-updated cdsapi-0.7.0/.github/workflows/label-public-pr.yml000066400000000000000000000003531461220700100220450ustar00rootroot00000000000000# Manage labels of pull requests that originate from forks name: label-public-pr on: pull_request_target: types: [opened, synchronize] jobs: label: uses: ecmwf-actions/reusable-workflows/.github/workflows/label-pr.yml@v2 cdsapi-0.7.0/.gitignore000066400000000000000000000001131461220700100147350ustar00rootroot00000000000000*.pyc *.data *.zip *.tgz *.tar *.tar.gz *.grib *.nc cdsapi.egg-info build/ cdsapi-0.7.0/CONTRIBUTING.rst000066400000000000000000000021561461220700100154170ustar00rootroot00000000000000 .. highlight:: console How to develop -------------- Install the package following README.rst and then install development dependencies:: $ pip install -U -r tests/requirements-dev.txt Unit tests can be run with `pytest `_ with:: $ pytest -v --flakes --cov=cdsapi --cov-report=html --cache-clear Coverage can be checked opening in a browser the file ``htmlcov/index.html`` for example with:: $ open htmlcov/index.html Code quality control checks can be run with:: $ pytest -v --pep8 --mccabe The complete python versions tests are run via `tox `_ with:: $ tox Please ensure the coverage at least stays the same before you submit a pull request. Dependency management --------------------- Update the `requirements-tests.txt` file with versions with:: pip-compile -U -o tests/requirements-tests.txt setup.py tests/requirements-tests.in # -U is optional Release procedure ----------------- Quality check release:: $ git status $ check-manifest $ tox Release with zest.releaser:: $ prerelease $ release $ postrelease cdsapi-0.7.0/LICENSE.txt000066400000000000000000000236761461220700100146130ustar00rootroot00000000000000 Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS cdsapi-0.7.0/MANIFEST.in000066400000000000000000000003141461220700100145060ustar00rootroot00000000000000include *.in include *.rst include *.txt include *.py include LICENSE include tox.ini recursive-include cdsapi *.py recursive-include tests *.in recursive-include tests *.py recursive-include tests *.txt cdsapi-0.7.0/README.rst000066400000000000000000000033041461220700100144410ustar00rootroot00000000000000 Install ------- Install via `pip` with:: $ pip install cdsapi Configure --------- Get your user ID (UID) and API key from the CDS portal at the address https://cds.climate.copernicus.eu/user and write it into the configuration file, so it looks like:: $ cat ~/.cdsapirc url: https://cds.climate.copernicus.eu/api/v2 key: : Remember to agree to the Terms and Conditions of every dataset that you intend to download. Test ---- Perform a small test retrieve of ERA5 data:: $ python >>> import cdsapi >>> cds = cdsapi.Client() >>> cds.retrieve('reanalysis-era5-pressure-levels', { "variable": "temperature", "pressure_level": "1000", "product_type": "reanalysis", "date": "2017-12-01/2017-12-31", "time": "12:00", "format": "grib" }, 'download.grib') >>> License ------- Copyright 2018 - 2019 European Centre for Medium-Range Weather Forecasts (ECMWF) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. In applying this licence, ECMWF does not waive the privileges and immunities granted to it by virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction. cdsapi-0.7.0/cdsapi/000077500000000000000000000000001461220700100142155ustar00rootroot00000000000000cdsapi-0.7.0/cdsapi/__init__.py000066400000000000000000000016621461220700100163330ustar00rootroot00000000000000# Copyright 2018 European Centre for Medium-Range Weather Forecasts (ECMWF) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # In applying this licence, ECMWF does not waive the privileges and immunities # granted to it by virtue of its status as an intergovernmental organisation nor # does it submit to any jurisdiction. from __future__ import absolute_import, division, print_function, unicode_literals from . import api Client = api.Client cdsapi-0.7.0/cdsapi/api.py000066400000000000000000000471171461220700100153520ustar00rootroot00000000000000# (C) Copyright 2018 ECMWF. # # This software is licensed under the terms of the Apache Licence Version 2.0 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. # In applying this licence, ECMWF does not waive the privileges and immunities # granted to it by virtue of its status as an intergovernmental organisation nor # does it submit to any jurisdiction. from __future__ import absolute_import, division, print_function, unicode_literals import json import logging import os import time import uuid import pkg_resources import requests try: from urllib.parse import urljoin except ImportError: from urlparse import urljoin from tqdm import tqdm def bytes_to_string(n): u = ["", "K", "M", "G", "T", "P"] i = 0 while n >= 1024: n /= 1024.0 i += 1 return "%g%s" % (int(n * 10 + 0.5) / 10.0, u[i]) def read_config(path): config = {} with open(path) as f: for line in f.readlines(): if ":" in line: k, v = line.strip().split(":", 1) if k in ("url", "key", "verify"): config[k] = v.strip() return config def get_url_key_verify(url, key, verify): if url is None: url = os.environ.get("CDSAPI_URL") if key is None: key = os.environ.get("CDSAPI_KEY") dotrc = os.environ.get("CDSAPI_RC", os.path.expanduser("~/.cdsapirc")) if url is None or key is None: if os.path.exists(dotrc): config = read_config(dotrc) if key is None: key = config.get("key") if url is None: url = config.get("url") if verify is None: verify = bool(int(config.get("verify", 1))) if url is None or key is None or key is None: raise Exception("Missing/incomplete configuration file: %s" % (dotrc)) # If verify is still None, then we set to default value of True if verify is None: verify = True return url, key, verify def toJSON(obj): to_json = getattr(obj, "toJSON", None) if callable(to_json): return to_json() if isinstance(obj, (list, tuple)): return [toJSON(x) for x in obj] if isinstance(obj, dict): r = {} for k, v in obj.items(): r[k] = toJSON(v) return r return obj class Result(object): def __init__(self, client, reply): self.reply = reply self._url = client.url self.session = client.session self.robust = client.robust self.verify = client.verify self.cleanup = client.delete self.debug = client.debug self.info = client.info self.warning = client.warning self.error = client.error self.sleep_max = client.sleep_max self.retry_max = client.retry_max self.timeout = client.timeout self.progress = client.progress self._deleted = False def toJSON(self): r = dict( resultType="url", contentType=self.content_type, contentLength=self.content_length, location=self.location, ) return r def _download(self, url, size, target): if target is None: target = url.split("/")[-1] self.info("Downloading %s to %s (%s)", url, target, bytes_to_string(size)) start = time.time() mode = "wb" total = 0 sleep = 10 tries = 0 headers = None while tries < self.retry_max: r = self.robust(self.session.get)( url, stream=True, verify=self.verify, headers=headers, timeout=self.timeout, ) try: r.raise_for_status() with tqdm( total=size, unit_scale=True, unit_divisor=1024, unit="B", disable=not self.progress, leave=False, ) as pbar: pbar.update(total) with open(target, mode) as f: for chunk in r.iter_content(chunk_size=1024): if chunk: f.write(chunk) total += len(chunk) pbar.update(len(chunk)) except requests.exceptions.ConnectionError as e: self.error("Download interupted: %s" % (e,)) finally: r.close() if total >= size: break self.error( "Download incomplete, downloaded %s byte(s) out of %s" % (total, size) ) self.warning("Sleeping %s seconds" % (sleep,)) time.sleep(sleep) mode = "ab" total = os.path.getsize(target) sleep *= 1.5 if sleep > self.sleep_max: sleep = self.sleep_max headers = {"Range": "bytes=%d-" % total} tries += 1 self.warning("Resuming download at byte %s" % (total,)) if total != size: raise Exception( "Download failed: downloaded %s byte(s) out of %s" % (total, size) ) elapsed = time.time() - start if elapsed: self.info("Download rate %s/s", bytes_to_string(size / elapsed)) return target def download(self, target=None): return self._download(self.location, self.content_length, target) @property def content_length(self): return int(self.reply["content_length"]) @property def location(self): return urljoin(self._url, self.reply["location"]) @property def content_type(self): return self.reply["content_type"] def __repr__(self): return "Result(content_length=%s,content_type=%s,location=%s)" % ( self.content_length, self.content_type, self.location, ) def check(self): self.debug("HEAD %s", self.location) metadata = self.robust(self.session.head)( self.location, verify=self.verify, timeout=self.timeout ) metadata.raise_for_status() self.debug(metadata.headers) return metadata def update(self, request_id=None): if request_id is None: request_id = self.reply["request_id"] task_url = "%s/tasks/%s" % (self._url, request_id) self.debug("GET %s", task_url) result = self.robust(self.session.get)( task_url, verify=self.verify, timeout=self.timeout ) result.raise_for_status() self.reply = result.json() def delete(self): if self._deleted: return if "request_id" in self.reply: rid = self.reply["request_id"] task_url = "%s/tasks/%s" % (self._url, rid) self.debug("DELETE %s", task_url) delete = self.session.delete( task_url, verify=self.verify, timeout=self.timeout ) self.debug("DELETE returns %s %s", delete.status_code, delete.reason) try: delete.raise_for_status() except Exception: self.warning( "DELETE %s returns %s %s", task_url, delete.status_code, delete.reason, ) self._deleted = True def __del__(self): try: if self.cleanup: self.delete() except Exception as e: print(e) class Client(object): logger = logging.getLogger("cdsapi") def __new__(cls, url=None, key=None, *args, **kwargs): _, token, _ = get_url_key_verify(url, key, None) if ":" in token: return super().__new__(cls) import cads_api_client.legacy_api_client return super().__new__(cads_api_client.legacy_api_client.LegacyApiClient) def __init__( self, url=None, key=None, quiet=False, debug=False, verify=None, timeout=60, progress=True, full_stack=False, delete=True, retry_max=500, sleep_max=120, wait_until_complete=True, info_callback=None, warning_callback=None, error_callback=None, debug_callback=None, metadata=None, forget=False, session=requests.Session(), ): if not quiet: if debug: level = logging.DEBUG else: level = logging.INFO self.logger.setLevel(level) # avoid duplicate handlers when creating more than one Client if not self.logger.handlers: formatter = logging.Formatter("%(asctime)s %(levelname)s %(message)s") handler = logging.StreamHandler() handler.setFormatter(formatter) self.logger.addHandler(handler) url, key, verify = get_url_key_verify(url, key, verify) self.url = url self.key = key self.quiet = quiet self.progress = progress and not quiet self.verify = True if verify else False self.timeout = timeout self.sleep_max = sleep_max self.retry_max = retry_max self.full_stack = full_stack self.delete = delete self.last_state = None self.wait_until_complete = wait_until_complete self.debug_callback = debug_callback self.warning_callback = warning_callback self.info_callback = info_callback self.error_callback = error_callback self.session = session self.session.auth = tuple(self.key.split(":", 2)) self.session.headers = { "User-Agent": "cdsapi/%s" % pkg_resources.get_distribution("cdsapi").version, } assert len(self.session.auth) == 2, ( "The cdsapi key provided is not the correct format, please ensure it conforms to:\n" ":" ) self.metadata = metadata self.forget = forget self.debug( "CDSAPI %s", dict( url=self.url, key=self.key, quiet=self.quiet, verify=self.verify, timeout=self.timeout, progress=self.progress, sleep_max=self.sleep_max, retry_max=self.retry_max, full_stack=self.full_stack, delete=self.delete, metadata=self.metadata, forget=self.forget, ), ) def retrieve(self, name, request, target=None): result = self._api("%s/resources/%s" % (self.url, name), request, "POST") if target is not None: result.download(target) return result def service(self, name, *args, **kwargs): self.delete = False # Don't delete results name = "/".join(name.split(".")) mimic_ui = kwargs.pop("mimic_ui", False) # To mimic the CDS ui the request should be populated directly with the kwargs if mimic_ui: request = kwargs else: request = dict(args=args, kwargs=kwargs) if self.metadata: request["_cds_metadata"] = self.metadata request = toJSON(request) result = self._api( "%s/tasks/services/%s/clientid-%s" % (self.url, name, uuid.uuid4().hex), request, "PUT", ) return result def workflow(self, code, *args, **kwargs): workflow_name = kwargs.pop("workflow_name", "application") params = dict(code=code, args=args, kwargs=kwargs, workflow_name=workflow_name) return self.service("tool.toolbox.orchestrator.run_workflow", params) def status(self, context=None): url = "%s/status.json" % (self.url,) r = self.session.get(url, verify=self.verify, timeout=self.timeout) r.raise_for_status() return r.json() def _status(self, url): try: status = self.status(url) info = status.get("info", []) if not isinstance(info, list): info = [info] for i in info: self.info("%s", i) warning = status.get("warning", []) if not isinstance(warning, list): warning = [warning] for w in warning: self.warning("%s", w) except Exception: pass def _api(self, url, request, method): self._status(url) session = self.session self.info("Sending request to %s", url) self.debug("%s %s %s", method, url, json.dumps(request)) if method == "PUT": action = session.put else: action = session.post result = self.robust(action)( url, json=request, verify=self.verify, timeout=self.timeout ) if self.forget: return result reply = None try: result.raise_for_status() reply = result.json() except Exception: if reply is None: try: reply = result.json() except Exception: reply = dict(message=result.text) self.debug(json.dumps(reply)) if "message" in reply: error = reply["message"] if "context" in reply and "required_terms" in reply["context"]: e = [error] for t in reply["context"]["required_terms"]: e.append( "To access this resource, you first need to accept the terms" "of '%s' at %s" % (t["title"], t["url"]) ) error = ". ".join(e) raise Exception(error) else: raise if not self.wait_until_complete: return Result(self, reply) sleep = 1 while True: self.debug("REPLY %s", reply) if reply["state"] != self.last_state: self.info("Request is %s" % (reply["state"],)) self.last_state = reply["state"] if reply["state"] == "completed": self.debug("Done") if "result" in reply: return reply["result"] return Result(self, reply) if reply["state"] in ("queued", "running"): rid = reply["request_id"] self.debug("Request ID is %s, sleep %s", rid, sleep) time.sleep(sleep) sleep *= 1.5 if sleep > self.sleep_max: sleep = self.sleep_max task_url = "%s/tasks/%s" % (self.url, rid) self.debug("GET %s", task_url) result = self.robust(session.get)( task_url, verify=self.verify, timeout=self.timeout ) result.raise_for_status() reply = result.json() continue if reply["state"] in ("failed",): self.error("Message: %s", reply["error"].get("message")) self.error("Reason: %s", reply["error"].get("reason")) for n in ( reply.get("error", {}) .get("context", {}) .get("traceback", "") .split("\n") ): if n.strip() == "" and not self.full_stack: break self.error(" %s", n) raise Exception( "%s. %s." % (reply["error"].get("message"), reply["error"].get("reason")) ) raise Exception("Unknown API state [%s]" % (reply["state"],)) def info(self, *args, **kwargs): if self.info_callback: self.info_callback(*args, **kwargs) else: self.logger.info(*args, **kwargs) def warning(self, *args, **kwargs): if self.warning_callback: self.warning_callback(*args, **kwargs) else: self.logger.warning(*args, **kwargs) def error(self, *args, **kwargs): if self.error_callback: self.error_callback(*args, **kwargs) else: self.logger.error(*args, **kwargs) def debug(self, *args, **kwargs): if self.debug_callback: self.debug_callback(*args, **kwargs) else: self.logger.debug(*args, **kwargs) def _download(self, results, targets=None): if isinstance(results, Result): if targets: path = targets.pop(0) else: path = None return results.download(path) if isinstance(results, (list, tuple)): return [self._download(x, targets) for x in results] if isinstance(results, dict): if "location" in results and "contentLength" in results: reply = dict( location=results["location"], content_length=results["contentLength"], content_type=results.get("contentType"), ) if targets: path = targets.pop(0) else: path = None return Result(self, reply).download(path) r = {} for k, v in results.items(): r[v] = self._download(v, targets) return r return results def download(self, results, targets=None): if targets: # Make a copy targets = [t for t in targets] return self._download(results, targets) def remote(self, url): r = requests.head(url) reply = dict( location=url, content_length=r.headers["Content-Length"], content_type=r.headers["Content-Type"], ) return Result(self, reply) def robust(self, call): def retriable(code, reason): if code in [ requests.codes.internal_server_error, requests.codes.bad_gateway, requests.codes.service_unavailable, requests.codes.gateway_timeout, requests.codes.too_many_requests, requests.codes.request_timeout, ]: return True return False def wrapped(*args, **kwargs): tries = 0 while True: txt = "Error" try: resp = call(*args, **kwargs) except ( requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout, ) as e: resp = None txt = f"Connection error: [{e}]" if resp is not None: if not retriable(resp.status_code, resp.reason): break try: self.warning(resp.json()["reason"]) except Exception: pass txt = f"HTTP error: [{resp.status_code} {resp.reason}]" tries += 1 self.warning(txt + f". Attempt {tries} of {self.retry_max}.") if tries < self.retry_max: self.warning(f"Retrying in {self.sleep_max} seconds") time.sleep(self.sleep_max) self.info("Retrying now...") else: raise Exception("Could not connect") return resp return wrapped cdsapi-0.7.0/docker/000077500000000000000000000000001461220700100142215ustar00rootroot00000000000000cdsapi-0.7.0/docker/.gitignore000066400000000000000000000000121461220700100162020ustar00rootroot00000000000000output/* cdsapi-0.7.0/docker/Dockerfile000066400000000000000000000002701461220700100162120ustar00rootroot00000000000000FROM python:3.7-alpine RUN pip3 install cdsapi WORKDIR /input COPY request.json request.json WORKDIR /output WORKDIR /app COPY retrieve.py retrieve.py CMD ["python", "retrieve.py"] cdsapi-0.7.0/docker/README.md000066400000000000000000000017031461220700100155010ustar00rootroot00000000000000## Simple wrapper around cdsapi cdsapi homepage : https://github.com/ecmwf/cdsapi ### How to use the dockerized version ? 1. Write a request in json file – don't forget the file format and name. Eg. ```js { "url": "https://cds.climate.copernicus.eu/api/v2", "uuid": "", "key": "", "variable": "reanalysis-era5-pressure-levels", "options": { "variable": "temperature", "pressure_level": "1000", "product_type": "reanalysis", "date": "2017-12-01/2017-12-31", "time": "12:00", "format": "grib" }, "filename":"test.grib" } ``` 2. Run the command ```sh docker run -it --rm \ -v $(pwd)/request.json:/input/request.json \ -v $(pwd)/.:/output \ /cdsretrieve ``` Note : the file will be downloaded in the current folder, if not specified otherwise in the docker command. Inside the container, `/input` folder include the request and `/output` is target folder for the downloaded file. cdsapi-0.7.0/docker/request.json000066400000000000000000000006001461220700100166000ustar00rootroot00000000000000{ "url": "https://cds.climate.copernicus.eu/api/v2", "uuid": "< YOUR USER ID >", "key": "< YOUR API KEY >", "variable": "reanalysis-era5-pressure-levels", "options": { "variable": "temperature", "pressure_level": "1000", "product_type": "reanalysis", "date": "2017-12-01/2017-12-31", "time": "12:00", "format": "grib" }, "filename":"test.grib" } cdsapi-0.7.0/docker/retrieve.py000066400000000000000000000004571461220700100164260ustar00rootroot00000000000000import json import cdsapi with open("/input/request.json") as req: request = json.load(req) cds = cdsapi.Client(request.get("url"), request.get("uuid") + ":" + request.get("key")) cds.retrieve( request.get("variable"), request.get("options"), "/output/" + request.get("filename"), ) cdsapi-0.7.0/example-era5.py000077500000000000000000000012431461220700100156140ustar00rootroot00000000000000#!/usr/bin/env python # (C) Copyright 2018 ECMWF. # # This software is licensed under the terms of the Apache Licence Version 2.0 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. # In applying this licence, ECMWF does not waive the privileges and immunities # granted to it by virtue of its status as an intergovernmental organisation nor # does it submit to any jurisdiction. import cdsapi c = cdsapi.Client() r = c.retrieve( "reanalysis-era5-single-levels", { "variable": "2t", "product_type": "reanalysis", "date": "2012-12-01", "time": "14:00", "format": "netcdf", }, ) r.download("test.nc") cdsapi-0.7.0/example-glaciers.py000077500000000000000000000010641461220700100165520ustar00rootroot00000000000000#!/usr/bin/env python # (C) Copyright 2018 ECMWF. # # This software is licensed under the terms of the Apache Licence Version 2.0 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. # In applying this licence, ECMWF does not waive the privileges and immunities # granted to it by virtue of its status as an intergovernmental organisation nor # does it submit to any jurisdiction. import cdsapi c = cdsapi.Client() c.retrieve( "insitu-glaciers-elevation-mass", {"variable": "elevation_change", "format": "tgz"}, "dowload.data", ) cdsapi-0.7.0/examples/000077500000000000000000000000001461220700100145705ustar00rootroot00000000000000cdsapi-0.7.0/examples/example-era5-update.py000077500000000000000000000030171461220700100207130ustar00rootroot00000000000000#!/usr/bin/env python # (C) Copyright 2018 ECMWF. # # This software is licensed under the terms of the Apache Licence Version 2.0 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. # In applying this licence, ECMWF does not waive the privileges and immunities # granted to it by virtue of its status as an intergovernmental organisation nor # does it submit to any jurisdiction. import time import cdsapi c = cdsapi.Client(debug=True, wait_until_complete=False) r = c.retrieve( "reanalysis-era5-single-levels", { "variable": "2t", "product_type": "reanalysis", "date": "2015-12-01", "time": "14:00", "format": "netcdf", }, ) sleep = 30 while True: r.update() reply = r.reply r.info("Request ID: %s, state: %s" % (reply["request_id"], reply["state"])) if reply["state"] == "completed": break elif reply["state"] in ("queued", "running"): r.info("Request ID: %s, sleep: %s", reply["request_id"], sleep) time.sleep(sleep) elif reply["state"] in ("failed",): r.error("Message: %s", reply["error"].get("message")) r.error("Reason: %s", reply["error"].get("reason")) for n in ( reply.get("error", {}).get("context", {}).get("traceback", "").split("\n") ): if n.strip() == "": break r.error(" %s", n) raise Exception( "%s. %s." % (reply["error"].get("message"), reply["error"].get("reason")) ) r.download("test.nc") cdsapi-0.7.0/setup.cfg000066400000000000000000000003071461220700100145730ustar00rootroot00000000000000[bdist_wheel] universal = 1 [aliases] test = pytest [tool:pytest] norecursedirs = build dist .tox .eggs pep8maxlinelength = 109 mccabe-complexity = 10 [coverage:run] branch = True cdsapi-0.7.0/setup.py000066400000000000000000000041601461220700100144650ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright 2018 European Centre for Medium-Range Weather Forecasts (ECMWF) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # In applying this licence, ECMWF does not waive the privileges and immunities # granted to it by virtue of its status as an intergovernmental organisation nor # does it submit to any jurisdiction. import io import os.path import setuptools def read(fname): file_path = os.path.join(os.path.dirname(__file__), fname) return io.open(file_path, encoding="utf-8").read() version = "0.7.0" setuptools.setup( name="cdsapi", version=version, author="ECMWF", author_email="software.support@ecmwf.int", license="Apache 2.0", url="https://github.com/ecmwf/cdsapi", description="Climate Data Store API", long_description=read("README.rst"), packages=setuptools.find_packages(), include_package_data=True, install_requires=[ "cads-api-client>=0.9.2", "requests>=2.5.0", "tqdm", ], zip_safe=True, classifiers=[ "Development Status :: 4 - Beta", "Intended Audience :: Developers", "Programming Language :: Python", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Operating System :: OS Independent", ], ) cdsapi-0.7.0/tests/000077500000000000000000000000001461220700100141145ustar00rootroot00000000000000cdsapi-0.7.0/tests/requirements.txt000066400000000000000000000001361461220700100174000ustar00rootroot00000000000000# pytest-cov pytest-env pytest-flakes pytest-mccabe pytest-pep8 pytest-runner pytest requests cdsapi-0.7.0/tests/test_api.py000066400000000000000000000020221461220700100162720ustar00rootroot00000000000000import os import cads_api_client.legacy_api_client import pytest import cdsapi def test_request(): c = cdsapi.Client() r = c.retrieve( "reanalysis-era5-single-levels", { "variable": "2t", "product_type": "reanalysis", "date": "2012-12-01", "time": "12:00", }, ) r.download("test.grib") assert os.path.getsize("test.grib") == 2076600 @pytest.mark.parametrize( "key,expected_client", [ ( ":", cdsapi.Client, ), ( "", cads_api_client.legacy_api_client.LegacyApiClient, ), ], ) @pytest.mark.parametrize("key_from_env", [True, False]) def test_instantiation(monkeypatch, key, expected_client, key_from_env): if key_from_env: monkeypatch.setenv("CDSAPI_KEY", key) c = cdsapi.Client() else: c = cdsapi.Client(key=key) assert isinstance(c, cdsapi.Client) assert isinstance(c, expected_client) assert c.key == key cdsapi-0.7.0/tox.ini000066400000000000000000000010151461220700100142620ustar00rootroot00000000000000[tox] envlist = qc, py312, py311, py310, py39, py38, pypy3, pypy, deps [testenv] setenv = PYTHONPATH = {toxinidir} deps = -r{toxinidir}/tests/requirements-tests.txt commands = pytest -v --flakes --cache-clear --basetemp={envtmpdir} {posargs} [testenv:qc] # needed for pytest-cov usedevelop = true commands = pytest -v --pep8 --mccabe --cov=cdsapi --cov-report=html --cache-clear {posargs} [testenv:deps] deps = commands = python setup.py test [black] line_length=120 [isort] profile=black [flake8] max-line-length = 120