pax_global_header00006660000000000000000000000064131520611110014501gustar00rootroot0000000000000052 comment=85a99ca3202c1c23461633cc99cb9f02a85d5b7f pyosmium-2.13.0/000077500000000000000000000000001315206111100134465ustar00rootroot00000000000000pyosmium-2.13.0/.gitignore000066400000000000000000000000411315206111100154310ustar00rootroot00000000000000build dist osmium.egg-info *.pyc pyosmium-2.13.0/.travis.yml000066400000000000000000000045361315206111100155670ustar00rootroot00000000000000#----------------------------------------------------------------------------- # # Configuration for continuous integration service at travis-ci.org # #----------------------------------------------------------------------------- language: cpp sudo: false dist: trusty matrix: include: - os: linux compiler: clang env: USE_PYTHON_VERSION=2 - os: linux compiler: clang env: USE_PYTHON_VERSION=3 - os: linux compiler: gcc env: USE_PYTHON_VERSION=2 - os: linux compiler: gcc env: USE_PYTHON_VERSION=3 - os: osx osx_image: xcode6.4 compiler: clang env: USE_PYTHON_VERSION=2 PYTHON_SUFFIX=2 BOOST_PYTHON_OPTION= - os: osx osx_image: xcode6.4 compiler: clang env: USE_PYTHON_VERSION=3 PYTHON_SUFFIX=3 BOOST_PYTHON_OPTION="--with-python3 --without-python" - os: osx osx_image: xcode7 compiler: clang env: USE_PYTHON_VERSION=2 PYTHON_SUFFIX= BOOST_PYTHON_OPTION= - os: osx osx_image: xcode7 compiler: clang env: USE_PYTHON_VERSION=3 PYTHON_SUFFIX=3 BOOST_PYTHON_OPTION="--with-python3 --without-python" # http://docs.travis-ci.com/user/apt/ addons: apt: sources: - boost-latest packages: - g++-4.8 - gcc-4.8 - libboost-python1.55-dev - libboost1.55-dev - libsparsehash-dev - python-dev - python-nose - python-mock - python3 - python3-dev - python3-nose - python3-setuptools install: - git clone --quiet --depth 1 https://github.com/osmcode/libosmium.git ../libosmium - if [ "$TRAVIS_OS_NAME" = 'osx' ]; then brew install python$PYTHON_SUFFIX google-sparsehash; brew install boost-python $BOOST_PYTHON_OPTION; pip$PYTHON_SUFFIX install -q nose mock; fi script: - if [ "${CXX}" = 'g++' ]; then CXX=g++-4.8; CC=gcc-4.8; fi - if [ "$TRAVIS_OS_NAME" = 'osx' ]; then PYTHON=python${USE_PYTHON_VERSION}; else PYTHON=/usr/bin/python${USE_PYTHON_VERSION}; fi - $PYTHON --version - $PYTHON setup.py build - cd test - $PYTHON run_tests.py pyosmium-2.13.0/CHANGELOG.md000066400000000000000000000070071315206111100152630ustar00rootroot00000000000000 # Change Log All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](http://semver.org/). ## [unreleased] - ### Added ### Changed ### Fixed ## [2.13.0] - 2017-08-31 ### Added - tests for WKB factories and replication server - str() and repr() implementations for all classes in osmium.osm - when applying diffs to a handler, a location cache may be used ### Changed - use new MultipolygonManager for building areas - allow to access nodes in a NodeRefList with negative index - use current libosmium ### Fixed - pyosmium-get-changes exits with an error when no start sequence can be found ## [2.12.4] - 2017-08-19 ### Added ### Changed ### Fixed - make apply_reader_simple a template again - minor fixes to documentation ## [2.12.3] - 2017-05-25 ### Added - links to appropriate mailing lists and issue trackers ### Changed ### Fixed - handler functions not called when using replication service (#38) - pyosmium-get-updates: bad variable name ## [2.12.2] - 2017-05-04 ### Added - build support for Windows - various tests ### Changed - python sources moved into src/ directory - use current libosmium - area.inner_rings() now takes an outer ring as parameter and returns an iterator ### Fixed - force use of C++ compiler - output type of index.map_types() function - write buffers growing unbound ## [2.12.1] - 2017-04-11 ### Added - geometry factories for WKT and GeoJSON - man pages for new tools - get() function for TagList - tests for TagList ### Changed - example code simplified - use current libosmium ### Fixed - area creator always called (#32) - various typos - TagList [] accessor properly throws KeyError on missing element ## [2.12.0] - 2017-03-19 ### Added - WriteHandler for writing data directly to a file - tools for downloading changes and updating a OSM files from these changes - get/set functions for io.Header ### Changed - use current libosmium ### Fixed - various typos in documentation ## [2.11.0] - 2017-01-15 ### Changed - Use current libosmium ## [2.10.2] - 2016-11-16 ### Added - support for sdist creation (now published via Pypi) - custom locations for libosmium and boost can be set via the environment variables `LIBOSMIUM_PREFIX` and `BOOST_PREFIX`. - export bounding box from osmium::io::Header ### Changed - Use libosmium 2.10.2 ### Fixed - various typos in documentation - crash in replication handler on incomplete state files ## [2.9.0] - 2016-09-15 ### Changed - Use current libosmium ## [2.8.0] - 2016-08-08 ### Changed - Use current libosmium ### Fixed - Works with different libosmium versions. ## [2.7.1] - 2016-06-01 ### Added - `apply_buffer()` for handling in-memory data - MergeInputReader for reading and sorting multiple input files - Functions using replication change files to update an OSM file or database. ### Changed - Use current libosmium ## [2.6.0] - 2016-02-04 ### Added - Experimental write support, see documentation - Multiple examples for writing data ### Changed - Use current libosmium - Improve timestamp to datetime conversion - Simplified package structure that uses the compiled libs directly ## [2.5.4] - 2015-12-03 ### Changed - Use current libosmium - README updates ## [2.5.3] - 2015-11-17 ### Changed - Use current libosmium ## [2.4.1] - 2015-08-31 ### Changed - Use current libosmium ## [2.3.0] - 2015-08-18 ### Changed - Use current libosmium ## [2.2.0] - 2015-07-04 ### Changed - Use current libosmium ### Fixed - Exception not caught in test. pyosmium-2.13.0/LICENSE.TXT000066400000000000000000000024171315206111100151350ustar00rootroot00000000000000 Copyright (c) 2014, Sarah Hoffmann All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. pyosmium-2.13.0/MANIFEST.in000066400000000000000000000001521315206111100152020ustar00rootroot00000000000000include *md include LICENSE.TXT include lib/*.hpp include doc/*.rst include examples/*py include test/*py pyosmium-2.13.0/README.md000066400000000000000000000062361315206111100147340ustar00rootroot00000000000000# pyosmium Provides Python bindings for the [Libosmium](https://github.com/osmcode/libosmium) C++ library, a library for working with OpenStreetMap data in a fast and flexible manner. [![Travis Build Status](https://api.travis-ci.org/osmcode/pyosmium.svg)](http://travis-ci.org/osmcode/pyosmium) [![Appveyor Build Status](https://ci.appveyor.com/api/projects/status/github/osmcode/pyosmium?svg=true)](https://ci.appveyor.com/project/Mapbox/pyosmium) ## Dependencies Python >= 2.7 is supported but a version >= 3.3 is strongly recommended. You'll need Python setuptools. On Debian/Ubuntu install `python-setuptools`. pyosmium uses [Boost.Python](http://www.boost.org/doc/libs/1_56_0/libs/python/doc/index.html) to create the bindings. On Debian/Ubuntu install `libboost-python-dev`. OS X run `brew install boost-python` or `brew install boost-python --with-python3` depending on which python version you want to use. You can also (re)install both. Other dependencies are expat, libz, libbz2 and Boost iterator. See [Libosmium dependencies](http://osmcode.org/libosmium/manual.html#dependencies) for required packages for various distributions. You have to compile with the same compiler version python is compiled with on your system, otherwise it might not work. ## Installation To compile the bindings, run python setup.py build To compile and install the bindings, run python setup.py install --user to install only for your user, or python setup.py install to install globally. The setup script uses per default either a globally installed libosmium or looks for the source in `../libosmium`. You can set a custom location with `LIBOSMIUM_PREFIX`. To use a custom boost installation, set `BOOST_PREFIX`. ## Examples The `example` directory contains small examples on how to use the library. They are mostly ports of the examples in Libosmium and osmium-contrib. ## Testing There is a small test suite in the test directory. This provides regression test for the python bindings, it is not meant to be a test suite for Libosmium. You'll need the Python `nose` module. On Debian/Ubuntu install the package `python-nose`. For Python2 `mock` is required as well (package `python-mock`). The suite can be run with: cd test python run_tests.py ## Documentation To build the documentation you need [Sphinx](http://sphinx-doc.org/) and the [autoprogram extension](https://pythonhosted.org/sphinxcontrib-autoprogram/) On Debian/Ubuntu install `python-sphinx sphinxcontrib-autoprogram` or `python3-sphinx python3-sphinxcontrib.autoprogram`. First compile the bindings as described above and then run: cd doc make html For building the man pages for the tools run: cd doc make man ## Bugs and Questions If you find bugs or have feature requests, please report those in the [github issue tracker](https://github.com/osmcode/pyosmium/issues/). For general questions about using pyosmium you can use the [OSM development mailing list](https://lists.openstreetmap.org/listinfo/dev) or ask on [OSM help](https://help.openstreetmap.org/). ## License Pyosmium is available under the BSD 2-Clause License. See LICENSE.TXT. ## Authors Sarah Hoffmann (lonvia@denofr.de) pyosmium-2.13.0/README.rst000066400000000000000000000037221315206111100151410ustar00rootroot00000000000000======== pyosmium ======== This library provides Python bindings for the `Libosmium`_ C++ library, a library for working with OpenStreetMap data in a fast and flexible manner. .. _Libosmium: https://github.com/osmcode/libosmium Installation ============ pyosmium can be installed with pip: pip install osmium The Pypi source package already comes bundled with a matching version of libosmium. The library uses `Boost.Python`_ to create the C++ bindings. On Debian/Ubuntu install `libboost-python-dev`. On OS X run `brew install boost-python` or `brew install boost-python --with-python3` depending on which python version you want to use. You can also (re)install both. Python >= 2.7 is supported but a version >= 3.3 is strongly recommended. Other dependencies are expat, libz, libbz2 and Boost iterator. See `Libosmium dependencies`_ for required packages for various distributions. .. _Boost.Python: http://www.boost.org/doc/libs/1_56_0/libs/python/doc/index.html .. _Libosmium dependencies: http://osmcode.org/libosmium/manual.html#dependencies Documentation ============= The documentation for the latest release is available at `osmcode.org`_. If you find bugs or have feature requests, please report those in the `Github issue tracker`_. For general questions about using pyosmium you can contanct the `OSM development mailing list`_ or ask on `OSM help`_. .. _osmcode.org: http://docs.osmcode.org/pyosmium/latest .. _Github issue tracker: https://github.com/osmcode/pyosmium/issues/ .. _OSM development mailing list: https://lists.openstreetmap.org/listinfo/dev .. _OSM help: https://help.openstreetmap.org/ Examples ======== The package contains an `example` directory with small examples on how to use the library. They are mostly ports of the examples in Libosmium and osmium-contrib. Fineprint ========= Pyosmium is available under the BSD 2-Clause License. See LICENSE.TXT. The source code can be found on `GitHub`_. .. _GitHub: https://github.com/osmcode/pyosmium pyosmium-2.13.0/appveyor.yml000066400000000000000000000031121315206111100160330ustar00rootroot00000000000000environment: matrix: - PYTHON: "C:\\Python27-x64" DEVPACK: pyosmium_libs27-b63.7z PIPINSTALLS: nose wheel mock - PYTHON: "C:\\Python36-x64" DEVPACK: pyosmium_libs36-b63.7z PIPINSTALLS: nose wheel os: Visual Studio 2015 install: - SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH% - SET VS90COMNTOOLS=%VS140COMNTOOLS% - cd "c:\Users\appveyor\AppData\Local\Programs\Common\Microsoft\" - ren "Visual C++ for Python" "Visual C++ for Python Do Not Use" - cd "C:\Program Files (x86)\" - ren "Microsoft Visual Studio 9.0" "Microsoft Visual Studio 9.0 Do Not Use" - python --version - cd c:\dev - git clone --depth 10 https://github.com/osmcode/libosmium.git - dir c:\dev - IF NOT EXIST pyosmium_libs_Release.7z ECHO downloading dependencies pack... && powershell Invoke-WebRequest https://github.com/alex85k/pyosmium_libs/releases/download/0.3/%DEVPACK% -OutFile %DEVPACK% - cd c:\ - 7z x c:\dev\%DEVPACK% - dir c:\libs\include - dir c:\libs\lib - pip install %PIPINSTALLS% # scripts that are called at very beginning, before repo cloning init: - git config --global core.autocrlf input # clone directory clone_folder: c:\dev\pyosmium build_script: - cd c:\dev\pyosmium - SET BOOST_PREFIX=c:/libs - SET BOOST_VERSION=1_63 - python setup.py build - python setup.py install --user test_script: - set PATH=%BOOST_PREFIX:/=\%\lib;%PATH% - cd test - python run_tests.py after_test: - cd c:\dev\pyosmium - copy c:\libs\lib\*.dll c:\dev\pyosmium\src\osmium - "%PYTHON%\\python.exe setup.py bdist_wheel" artifacts: - path: dist\* pyosmium-2.13.0/doc/000077500000000000000000000000001315206111100142135ustar00rootroot00000000000000pyosmium-2.13.0/doc/.gitignore000066400000000000000000000000071315206111100162000ustar00rootroot00000000000000_build pyosmium-2.13.0/doc/Makefile000066400000000000000000000151621315206111100156600ustar00rootroot00000000000000# Makefile for Sphinx documentation # # You can set these variables from the command line. SPHINXOPTS = SPHINXBUILD = sphinx-build PAPER = BUILDDIR = _build # User-friendly check for sphinx-build ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) endif # Internal variables. PAPEROPT_a4 = -D latex_paper_size=a4 PAPEROPT_letter = -D latex_paper_size=letter ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . # the i18n builder cannot share the environment and doctrees with the others I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext help: @echo "Please use \`make ' where is one of" @echo " html to make standalone HTML files" @echo " dirhtml to make HTML files named index.html in directories" @echo " singlehtml to make a single large HTML file" @echo " pickle to make pickle files" @echo " json to make JSON files" @echo " htmlhelp to make HTML files and a HTML help project" @echo " qthelp to make HTML files and a qthelp project" @echo " devhelp to make HTML files and a Devhelp project" @echo " epub to make an epub" @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" @echo " latexpdf to make LaTeX files and run them through pdflatex" @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" @echo " text to make text files" @echo " man to make manual pages" @echo " texinfo to make Texinfo files" @echo " info to make Texinfo files and run them through makeinfo" @echo " gettext to make PO message catalogs" @echo " changes to make an overview of all changed/added/deprecated items" @echo " xml to make Docutils-native XML files" @echo " pseudoxml to make pseudoxml-XML files for display purposes" @echo " linkcheck to check all external links for integrity" @echo " doctest to run all doctests embedded in the documentation (if enabled)" clean: rm -rf $(BUILDDIR)/* html: $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." dirhtml: $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." singlehtml: $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml @echo @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." pickle: $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle @echo @echo "Build finished; now you can process the pickle files." json: $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json @echo @echo "Build finished; now you can process the JSON files." htmlhelp: $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp @echo @echo "Build finished; now you can run HTML Help Workshop with the" \ ".hhp project file in $(BUILDDIR)/htmlhelp." qthelp: $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp @echo @echo "Build finished; now you can run "qcollectiongenerator" with the" \ ".qhcp project file in $(BUILDDIR)/qthelp, like this:" @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/Pyosmium.qhcp" @echo "To view the help file:" @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/Pyosmium.qhc" devhelp: $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp @echo @echo "Build finished." @echo "To view the help file:" @echo "# mkdir -p $$HOME/.local/share/devhelp/Pyosmium" @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/Pyosmium" @echo "# devhelp" epub: $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub @echo @echo "Build finished. The epub file is in $(BUILDDIR)/epub." latex: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." @echo "Run \`make' in that directory to run these through (pdf)latex" \ "(use \`make latexpdf' here to do that automatically)." latexpdf: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo "Running LaTeX files through pdflatex..." $(MAKE) -C $(BUILDDIR)/latex all-pdf @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." latexpdfja: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo "Running LaTeX files through platex and dvipdfmx..." $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." text: $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text @echo @echo "Build finished. The text files are in $(BUILDDIR)/text." man: $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man @echo @echo "Build finished. The manual pages are in $(BUILDDIR)/man." texinfo: $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo @echo @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." @echo "Run \`make' in that directory to run these through makeinfo" \ "(use \`make info' here to do that automatically)." info: $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo @echo "Running Texinfo files through makeinfo..." make -C $(BUILDDIR)/texinfo info @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." gettext: $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale @echo @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." changes: $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes @echo @echo "The overview file is in $(BUILDDIR)/changes." linkcheck: $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck @echo @echo "Link check complete; look for any errors in the above output " \ "or in $(BUILDDIR)/linkcheck/output.txt." doctest: $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest @echo "Testing of doctests in the sources finished, look at the " \ "results in $(BUILDDIR)/doctest/output.txt." xml: $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml @echo @echo "Build finished. The XML files are in $(BUILDDIR)/xml." pseudoxml: $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml @echo @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." pyosmium-2.13.0/doc/conf.py000066400000000000000000000217441315206111100155220ustar00rootroot00000000000000#!/usr/bin/env python3 # -*- coding: utf-8 -*- # # Pyosmium documentation build configuration file, created by # sphinx-quickstart on Tue Mar 10 18:09:49 2015. # # This file is execfile()d with the current directory set to its # containing dir. # # Note that not all possible configuration values are present in this # autogenerated file. # # All configuration values have a default; values that are commented out # serve to show the default. import sys import sysconfig import os # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. #sys.path.insert(0, os.path.abspath('.')) build_dir = "../build/lib.%s-%d.%d" % ( sysconfig.get_platform(), sys.version_info[0], sys.version_info[1]) # insert after the current directory sys.path.insert(0, os.path.normpath(os.path.join(os.path.abspath('.'), build_dir))) sys.path.insert(0, os.path.normpath(os.path.join(os.path.abspath('.'), '../tools'))) try: from osmium.version import pyosmium_major, pyosmium_release except ImportError: print(""" Compiled version of pyosmium not found, please build pyosmium for Python %d.%d. Expected build directory: %s """ % (sys.version_info[0], sys.version_info[1], build_dir)) raise # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. #needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ 'sphinx.ext.autodoc', 'sphinx.ext.todo', 'sphinxcontrib.autoprogram', ] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] # The suffix of source filenames. source_suffix = '.rst' # The encoding of source files. #source_encoding = 'utf-8-sig' # The master toctree document. master_doc = 'index' # General information about the project. project = 'Pyosmium' copyright = '2015-2017, Sarah Hoffmann' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. version = pyosmium_major # The full version, including alpha/beta/rc tags. release = pyosmium_release # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. #language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: #today = '' # Else, today_fmt is used as the format for a strftime call. #today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. exclude_patterns = ['_build'] # The reST default role (used for this markup: `text`) to use for all # documents. #default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. #add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). #add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. #show_authors = False # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' # A list of ignored prefixes for module index sorting. #modindex_common_prefix = [] # If true, keep warnings as "system message" paragraphs in the built documents. #keep_warnings = False # -- Options for HTML output ---------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. html_theme = 'default' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. #html_theme_options = {} # Add any paths that contain custom themes here, relative to this directory. #html_theme_path = [] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". #html_title = None # A shorter title for the navigation bar. Default is the same as html_title. #html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. #html_logo = None # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. #html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['_static'] # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied # directly to the root of the documentation. #html_extra_path = [] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. #html_last_updated_fmt = '%b %d, %Y' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. #html_use_smartypants = True # Custom sidebar templates, maps document names to template names. #html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. #html_additional_pages = {} # If false, no module index is generated. #html_domain_indices = True # If false, no index is generated. #html_use_index = True # If true, the index is split into individual pages for each letter. #html_split_index = False # If true, links to the reST sources are added to the pages. #html_show_sourcelink = True # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. #html_show_sphinx = True # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. #html_show_copyright = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. #html_use_opensearch = '' # This is the file name suffix for HTML files (e.g. ".xhtml"). #html_file_suffix = None # Output file base name for HTML help builder. htmlhelp_basename = 'Pyosmiumdoc' # -- Options for LaTeX output --------------------------------------------- latex_elements = { # The paper size ('letterpaper' or 'a4paper'). #'papersize': 'letterpaper', # The font size ('10pt', '11pt' or '12pt'). #'pointsize': '10pt', # Additional stuff for the LaTeX preamble. #'preamble': '', } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ ('index', 'Pyosmium.tex', 'Pyosmium Documentation', 'Sarah Hoffmann', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of # the title page. #latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. #latex_use_parts = False # If true, show page references after internal links. #latex_show_pagerefs = False # If true, show URL addresses after external links. #latex_show_urls = False # Documents to append as an appendix to all manuals. #latex_appendices = [] # If false, no module index is generated. #latex_domain_indices = True # -- Options for manual page output --------------------------------------- # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ ('tools_get_changes', 'pyosmium-get-changes', 'Download OSM change files', ['Sarah Hoffmann'], 1), ('tools_uptodate', 'pyosmium-up-to-date', 'Bring OSM files up-to-date', ['Sarah Hoffmann'], 1), ] # If true, show URL addresses after external links. #man_show_urls = False # -- Options for Texinfo output ------------------------------------------- # Grouping the document tree into Texinfo files. List of tuples # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ ('index', 'Pyosmium', 'Pyosmium Documentation', 'Sarah Hoffmann', 'Pyosmium', 'One line description of project.', 'Miscellaneous'), ] # Documents to append as an appendix to all manuals. #texinfo_appendices = [] # If false, no module index is generated. #texinfo_domain_indices = True # How to display URL addresses: 'footnote', 'no', or 'inline'. #texinfo_show_urls = 'footnote' # If true, do not generate a @detailmenu in the "Top" node's menu. #texinfo_no_detailmenu = False autodoc_member_order = 'bysource' autodoc_defaut_flags = ['members', 'undoc-members'] pyosmium-2.13.0/doc/index.rst000066400000000000000000000011051315206111100160510ustar00rootroot00000000000000.. Pyosmium documentation master file, created by sphinx-quickstart on Tue Mar 10 18:09:49 2015. You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. Welcome to Pyosmium's documentation! ==================================== Pyosmium is a library to process OSM files in different formats. It is a wrapper of the C++ library `osmium `_ and profits from its fast implementation. .. toctree:: :maxdepth: 2 intro reference tools * :ref:`genindex` * :ref:`search` pyosmium-2.13.0/doc/intro.rst000066400000000000000000000204641315206111100161060ustar00rootroot00000000000000Basic Usage =========== The following chapter gives a practical introduction on how to use Pyosmium. It is assumed that you already have a basic knowledge about the `OSM data model`_. For a more detailed introduction into the design of the osmium library, the reader is referred to the `osmium documentation`_. .. _OSM data model: http://wiki.openstreetmap.org/wiki/Elements .. _osmium documentation: http://osmcode.org/osmium-concepts/ Reading OSM Data ---------------- Using Handler Classes ^^^^^^^^^^^^^^^^^^^^^ OSM file parsing by osmium is built around the concept of handlers. A handler is a class with a set of callback functions. Each function processes exactly one type of object as it is read from the file. Let's start with a very simple handler that counts the nodes in the input file:: import osmium class CounterHandler(osmium.SimpleHandler): def __init__(self): osmium.SimpleHandler.__init__(self) self.num_nodes = 0 def node(self, n): self.num_nodes += 1 A handler first of all needs to inherit from one of the handler classes. At the moment this is always :py:class:`osmium.SimpleHandler`. Then it needs to implement functions for each object type it wants to process. In our case it is exactly one function `node()`. All other potential callbacks can be safely ignored. Now the handler needs to be applied to an OSM file. The easiest way to accomplish that is to call the :py:meth:`~osmium.SimpleHandler.apply_file` convenience function, which in its simplest form only requires the file name as a parameter. The main routine of the node counting application therefore looks like this:: if __name__ == '__main__': h = CounterHandler() h.apply_file("test.osm.pbf") print("Number of nodes: %d" % h.num_nodes) That already finishes our node counting program. Inspecting the OSM objects ^^^^^^^^^^^^^^^^^^^^^^^^^^ Counting nodes is actually boring because it completely ignores the content of the nodes. So let's change the handler to only count hotels (normally tagged with ``tourism=hotel``). They may be tagged as nodes, ways or relations, so handler functions for all three types need to be implemented:: import osmium class HotelCounterHandler(osmium.SimpleHandler): def __init__(self): osmium.SimpleHandler.__init__(self) self.num_nodes = 0 def count_hotel(self, tags): if tags['tourism'] == 'hotel': self.num_nodes += 1 def node(self, n): self.count_hotel(n.tags) def way(self, w): self.count_hotel(w.tags) def relation(self, r): self.count_hotel(r.tags) A reference to the object is always given as the only parameter to the handler functions. The objects have some common methods and attributes, listed in :py:class:`osmium.osm.OSMObject`, and some that are specific to each type. As all objects have tags, it is possible to reuse the same implementation for all types. The main function remains the same. It is important to remember that the object references that are handed to the handler are only temporary. They will become invalid as soon as the function returns. Handler functions *must* copy any data that should be kept for later use into their own data structures. This also includes attributes like tag lists. Handling Geometries ^^^^^^^^^^^^^^^^^^^ Because of the way that OSM data is structured, osmium needs to internally cache node geometries, when the handler wants to process the geometries of ways and areas. The :py:meth:`~!osmium.SimpleHandler.apply_file` method cannot deduce by itself if this cache is needed. Therefore locations need to be explicitly enabled by setting the locations parameter to True:: h.apply_file("test.osm.pbf", locations=True, idx='sparse_mem_array') The third parameter `idx` is optional and states what kind of cache osmium is supposed to use. The default `sparse_mem_array` is a good choice for small to medium size extracts of OSM data. If you plan to process the whole planet file, `dense_mmap_array` is better suited. If you want the cache to be persistent across invocations, you can use `dense_file_array` giving an additional file location for the cache like that:: h.apply_file("test.osm.pbf", locations=True, idx='dense_file_array,example.nodecache') where `example.nodecache` is the name of the cache file. For more details on the different kind of indexes, see the `Index section in the Osmium Manual`_. .. _Index section in the Osmium Manual: http://osmcode.org/osmium-concepts/#indexes Interfacing with Shapely ^^^^^^^^^^^^^^^^^^^^^^^^ Pyosmium is a library for processing OSM files and therefore offers almost no functionality for processing geometries further. There are other libraries for that purpose. To interface with these libraries you can simply convert the osmium geometries into WKB or WKT format and import the result. The following example uses the libgeos wrapper `Shapely`_ to compute the total way length:: import osmium import shapely.wkb as wkblib # A global factory that creates WKB from a osmium geometry wkbfab = osmium.geom.WKBFactory() class WayLenHandler(osmium.SimpleHandler): def __init__(self): osmium.SimpleHandler.__init__(self) self.total = 0 def way(self, w): wkb = wkbfab.create_linestring(w) line = wkblib.loads(wkb, hex=True) # Length is computed in WGS84 projection, which is practically meaningless. # Lets pretend we didn't notice, it is an example after all. self.total += line.length if __name__ == '__main__': h = WayLenHandler() h.apply_file("test.osm.pbf", locations=True) print("Total length: %f" % h.total) .. _Shapely: http://toblerity.org/shapely/index.html Writing OSM Data ---------------- :py:class:`osmium.SimpleWriter` is the main class that takes care of writing out OSM data to a file. The file name must be given when the writer is constructed. Its suffix determines the format of the data. For example:: writer = osmium.SimpleWriter('nodes.osm.bz2') opens a new writer for a packed OSM XML file. Objects can be written by using one of the writers ``add_*`` functions. A simple handler, that only writes out all the nodes from the input file into our new ``nodes.osm.bz2`` file would look like this:: import osmium class NodeWriter(osmium.SimpleHandler): def __init__(self, writer): osmium.SimpleHandler.__init__(self) self.writer = writer def node(self, n): self.writer.add_node(n) This example shows that an unmodified object can be written out directly to the writer. Normally, however, you want to modify some data. The native osmium OSM types are immutable and cannot be changed directly. Therefore you have to create a copy that can be changed. The ``node``, ``way`` and ``relation`` objects offer a convenient ``replace()`` function to achieve exactly that. The function makes a copy and at the same time replaces all attributes where new values are given as parameters to the function. Let's say you want to remove all the user names from your nodes before saving them to the new file (maybe to save some space), then the ``node()`` handler callback above needs to be changed like this:: class NodeWriter(osmium.SimpleHandler): ... def node(self, n): self.writer.add_node(n.replace(user="")) ``replace()`` creates a new instance of an ``osmium.osm.mutable`` object. These classes are python implementations of the native object types in ``osmium.osm``. They have exactly the same attributes but they are mutable. A writer is able to process the mutable datatypes just like the native osmium types. In fact, a writer is able to process any python object. It just expects suitably named attributes and will simply assume sensible default values for attributes that are missing. .. note:: It is important to understand that ``replace()`` only makes a shallow copy of the object. Tag, node and member lists are still native osmium objects. Normally this is what you want because the writer is much faster writing these native objects than pythonized copies. However, it means that you cannot use ``replace()`` to create a copy of the object that can be kept after the handler callback has finished. pyosmium-2.13.0/doc/ref_geom.rst000066400000000000000000000011301315206111100165230ustar00rootroot00000000000000``geom`` - Geometry Helper Functions ------------------------------------ This module provides various helper functions for geometry handling. Note: remember to apply a location handler before in order to use these geometry utilities on node locations. Geometry Factories ^^^^^^^^^^^^^^^^^^ .. autoclass:: osmium.geom.WKBFactory :members: :undoc-members: .. autoclass:: osmium.geom.WKTFactory :members: :undoc-members: .. autoclass:: osmium.geom.GeoJSONFactory :members: :undoc-members: Other Functions ^^^^^^^^^^^^^^^ .. autofunction:: osmium.geom.haversine_distance pyosmium-2.13.0/doc/ref_index.rst000066400000000000000000000013141315206111100167070ustar00rootroot00000000000000``index`` - Data Stores ----------------------- The ``index`` submodule provides efficient storage containers for preprocessed OSM data. Node Location Storage ^^^^^^^^^^^^^^^^^^^^^ Node location can be cached in a ``LocationTable``. There are different implementations available which should be choosen according to the size of data and whether or not the cache should be permanent. See the Osmium manual for a detailed explaination. The compiled in types can be listed with the ``map_types`` function, new stores can be created with ``create_map``. .. autofunction:: osmium.index.map_types .. autofunction:: osmium.index.create_map .. autoclass:: osmium.index.LocationTable :members: :undoc-members: pyosmium-2.13.0/doc/ref_io.rst000066400000000000000000000003561315206111100162140ustar00rootroot00000000000000``io`` - Data In- and Output ---------------------------- This module exposes the generic file reader. .. autoclass:: osmium.io.Reader :members: :undoc-members: .. autoclass:: osmium.io.Header :members: :undoc-members: pyosmium-2.13.0/doc/ref_osm.rst000066400000000000000000000051701315206111100164020ustar00rootroot00000000000000``osm`` - Basic Datatypes ------------------------- The ``osm`` submodule contains definitions of the basic data types used throughout the library. Native OSM Objects ^^^^^^^^^^^^^^^^^^ Native OSM object classes are lightwight wrappers around the osmium OSM data classes. They are immutable and generally bound to the lifetime of the buffer they are saved in. There are five classes representing the basic OSM entities. .. autoclass:: osmium.osm.OSMObject :members: :undoc-members: .. autoclass:: osmium.osm.Node :members: :undoc-members: .. autoclass:: osmium.osm.Way :members: :undoc-members: .. autoclass:: osmium.osm.Relation :members: :undoc-members: .. autoclass:: osmium.osm.Area :members: :undoc-members: .. autoclass:: osmium.osm.Changeset :members: :undoc-members: .. _mutable-objects: Mutable OSM Objects ^^^^^^^^^^^^^^^^^^^ The objects in ``osmium.osm.mutable`` are Python versions of the native OSM objects that can be modified. You can use these classes as a base class for your own objects or to modify objects read from a file. .. autoclass:: osmium.osm.mutable.OSMObject :members: :undoc-members: .. autoclass:: osmium.osm.mutable.Node :members: :undoc-members: .. autoclass:: osmium.osm.mutable.Way :members: :undoc-members: .. autoclass:: osmium.osm.mutable.Relation :members: :undoc-members: Node Reference Lists ^^^^^^^^^^^^^^^^^^^^ Line geometries in OSM are simply a sequence of nodes. To simplify processing osmium returns such node sequences using a special datatype that contains a reference to the node id and also the location geometry. The latter is only valid if the node locations have been cached by a location handler. .. autoclass:: osmium.osm.NodeRef :members: :undoc-members: .. autoclass:: osmium.osm.NodeRefList :members: :undoc-members: .. autoclass:: osmium.osm.WayNodeList :members: :undoc-members: .. autoclass:: osmium.osm.OuterRing :members: :undoc-members: .. autoclass:: osmium.osm.InnerRing :members: :undoc-members: Other OSM Entity Attributes ^^^^^^^^^^^^^^^^^^^^^^^^^^^ Some of the attributes of the OSM entities are represented with more complex classes. .. autoclass:: osmium.osm.Box :members: :undoc-members: .. autoclass:: osmium.osm.Location :members: :undoc-members: .. autoclass:: osmium.osm.RelationMember :members: :undoc-members: .. autoclass:: osmium.osm.RelationMemberList :members: :undoc-members: .. autoclass:: osmium.osm.Tag :members: :undoc-members: .. autoclass:: osmium.osm.TagList :members: :undoc-members: pyosmium-2.13.0/doc/ref_osmium.rst000066400000000000000000000031341315206111100171130ustar00rootroot00000000000000``osmium`` - Processing OSM files --------------------------------- Osmium processes files by reading data from a file and applying them to a processing chain. Pyosmium offers a simplified wrapper to this interface with the ``SimpleHandler`` class from which an OSM file processor can easily be derived. For more fine grained control of the processing chain, the more basic functions and processors are exported as well in this module. Input Handlers ^^^^^^^^^^^^^^ An input handler provides the base class for writing custom data processors. They take input data, usually from a file, and forward it to handler functions. .. autoclass:: osmium.SimpleHandler :members: :undoc-members: SimpleWriter ^^^^^^^^^^^^ The writer class can be used to create an OSM file. The writer is able to handle native ``osmium.osm`` objects as well as any Python object that exposes the same attributes. It is not necessary to implement the full list of attributes as any missing attributes will be replaced with a sensible default value when writing. See :ref:`mutable-objects` for a detailed discussion of the data formats understood for each attribute. .. warning:: Writers are considerably faster in handling native osmium data types than Python objects. You should therefore avoid converting objects whereever possible. This is not only true for the OSM data types like Node, Way and Relation but also for tag lists, node lists and member lists. .. autoclass:: osmium.SimpleWriter :members: :undoc-members: Low-level Functions and Classes ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. autofunction:: osmium.apply pyosmium-2.13.0/doc/ref_replication.rst000066400000000000000000000006061315206111100201140ustar00rootroot00000000000000``replication`` - Handling Updates of OSM Data ---------------------------------------------- Replication servers provide regular updates of OSM data. This module provides helper functions to access the servers and download and apply updates. Replication Server Class ^^^^^^^^^^^^^^^^^^^^^^^^ .. autoclass:: osmium.replication.server.ReplicationServer :members: :undoc-members: pyosmium-2.13.0/doc/reference.rst000066400000000000000000000006501315206111100167040ustar00rootroot00000000000000Pyosmium Reference ================== Pyosmium is a thin wrapper to the osmium library. Where possible it follows its structure and naming scheme. This reference provides a short description of the exported classes and interfaces. More details and background information can be found in the osmium manual. .. toctree:: :maxdepth: 2 ref_osmium ref_osm ref_io ref_index ref_geom ref_replication pyosmium-2.13.0/doc/tools.rst000066400000000000000000000002531315206111100161050ustar00rootroot00000000000000Pyosmium Tools ============== Pyosmium comes with a couple of scripts for handling change files: .. toctree:: :maxdepth: 1 tools_get_changes tools_uptodate pyosmium-2.13.0/doc/tools_get_changes.rst000066400000000000000000000002711315206111100204340ustar00rootroot00000000000000pyosmium-get-changes - Downloading OSM change files =================================================== Usage ----- .. autoprogram:: pyosmium-get-changes:get_arg_parser() :prog: pyosmium-2.13.0/doc/tools_uptodate.rst000066400000000000000000000002701315206111100200110ustar00rootroot00000000000000pyosmium-up-to-date - Bringing OSM files up-to-date =================================================== Usage ----- .. autoprogram:: pyosmium-up-to-date:get_arg_parser() :prog: pyosmium-2.13.0/examples/000077500000000000000000000000001315206111100152645ustar00rootroot00000000000000pyosmium-2.13.0/examples/amenity_list.py000066400000000000000000000021141315206111100203350ustar00rootroot00000000000000""" Extract all objects with an amenity tag from an osm file and list them with their name and position. This example shows how geometries from osmium objects can be imported into shapely using the WKBFactory. """ import osmium as o import sys import shapely.wkb as wkblib wkbfab = o.geom.WKBFactory() class AmenityListHandler(o.SimpleHandler): def print_amenity(amenity, tags, lon, lat): name = tags.get('name', '') print("%f %f %-15s %s" % (lon, lat, tags['amenity'], name)) def node(self, n): if 'amenity' in n.tags: self.print_amenity(n.tags, n.location.lon, n.location.lat) def area(self, a): if 'amenity' in a.tags: wkb = wkbfab.create_multipolygon(a) poly = wkblib.loads(wkb, hex=True) centroid = poly.representative_point() self.print_amenity(a.tags, centroid.x, centroid.y) if __name__ == '__main__': if len(sys.argv) != 2: print("Usage: python amenity_list.py ") sys.exit(-1) handler = AmenityListHandler() handler.apply_file(sys.argv[1]) pyosmium-2.13.0/examples/convert.py000066400000000000000000000013101315206111100173110ustar00rootroot00000000000000""" Converts a file from one format to another. This example shows how to write objects to a file. """ import osmium as o import sys class Convert(o.SimpleHandler): def __init__(self, writer): super(Convert, self).__init__() self.writer = writer def node(self, n): self.writer.add_node(n) def way(self, w): self.writer.add_way(w) def relation(self, r): self.writer.add_relation(r) if __name__ == '__main__': if len(sys.argv) != 3: print("Usage: python convert.py ") sys.exit(-1) writer = o.SimpleWriter(sys.argv[2]) handler = Convert(writer) handler.apply_file(sys.argv[1]) writer.close() pyosmium-2.13.0/examples/create_nodecache.py000066400000000000000000000005151315206111100210730ustar00rootroot00000000000000import osmium as o import sys if len(sys.argv) != 3: print("Usage: python create_nodecache.py ") exit(-1) reader = o.io.Reader(sys.argv[1], o.osm.osm_entity_bits.NODE) idx = o.index.create_map("sparse_file_array," + sys.argv[2]) lh = o.NodeLocationsForWays(idx) o.apply(reader, lh) reader.close() pyosmium-2.13.0/examples/filter_coastlines.py000066400000000000000000000026671315206111100213620ustar00rootroot00000000000000""" Filter all objects with a coastline tag. This example shows how to write objects to a file. We need to go twice over the file. First read the ways, filter the ones we are interested in and remember the nodes required. Then, in a second run all the relevant nodes and ways are written out. """ import osmium as o import sys class WayFilter(o.SimpleHandler): def __init__(self): super(WayFilter, self).__init__() self.nodes = set() def way(self, w): if 'natural' in w.tags and w.tags['natural'] == 'coastline': for n in w.nodes: self.nodes.add(n.ref) class CoastlineWriter(o.SimpleHandler): def __init__(self, writer, nodes): super(CoastlineWriter, self).__init__() self.writer = writer self.nodes = nodes def node(self, n): if n.id in self.nodes: self.writer.add_node(n) def way(self, w): if 'natural' in w.tags and w.tags['natural'] == 'coastline': self.writer.add_way(w) if __name__ == '__main__': if len(sys.argv) != 3: print("Usage: python filter_coastlines.py ") sys.exit(-1) # go through the ways to find all relevant nodes ways = WayFilter() ways.apply_file(sys.argv[1]) # go through the file again and write out the data writer = o.SimpleWriter(sys.argv[2]) CoastlineWriter(writer, ways.nodes).apply_file(sys.argv[1]) writer.close() pyosmium-2.13.0/examples/normalize_boolean.py000066400000000000000000000037011315206111100213360ustar00rootroot00000000000000""" This example shows how to filter and modify tags and write the results back. It changes all tag values 'yes/no' to '1/0'. """ import osmium as o import sys class BoolNormalizer(o.SimpleHandler): def __init__(self, writer): super(BoolNormalizer, self).__init__() self.writer = writer def normalize(self, o): # if there are no tags we are done if not o.tags: return o # new tags should be kept in a list so that the order is preserved newtags = [] # pyosmium is much faster writing an original osmium object than # a osmium.mutable.*. Therefore, keep track if the tags list was # actually changed. modified = False for t in o.tags: if t.v == 'yes': # custom tags should be added as a key/value tuple newtags.append((t.k, '1')) modified = True elif t.v == 'no': newtags.append((t.k, '0')) modified = True else: # if the tag is not modified, simply readd it to the list newtags.append(t) if modified: # We have changed tags. Create a new object as a copy of the # original one with the tag list replaced. return o.replace(tags=newtags) else: # Nothing changed, so simply return the original object # and discard the tag list we just created. return o def node(self, o): self.writer.add_node(self.normalize(o)) def way(self, o): self.writer.add_way(self.normalize(o)) def relation(self, o): self.writer.add_relation(self.normalize(o)) if __name__ == '__main__': if len(sys.argv) != 3: print("Usage: python normalize_boolean.py ") sys.exit(-1) writer = o.SimpleWriter(sys.argv[2]) BoolNormalizer(writer).apply_file(sys.argv[1]) writer.close() pyosmium-2.13.0/examples/osm_diff_stats.py000066400000000000000000000024311315206111100206420ustar00rootroot00000000000000""" Simple example that counts the number of changes in an osm diff file. Shows how to detect the different kind of modifications. """ import osmium as o import sys class Stats(object): def __init__(self): self.added = 0 self.modified = 0 self.deleted = 0 def add(self, o): if o.deleted: self.deleted += 1 elif o.version == 1: self.added += 1 else: self.modified += 1 def outstats(self, prefix): print("%s added: %d" % (prefix, self.added)) print("%s modified: %d" % (prefix, self.modified)) print("%s deleted: %d" % (prefix, self.deleted)) class FileStatsHandler(o.SimpleHandler): def __init__(self): super(FileStatsHandler, self).__init__() self.nodes = Stats() self.ways = Stats() self.rels = Stats() def node(self, n): self.nodes.add(n) def way(self, w): self.ways.add(w) def relation(self, r): self.rels.add(r) if __name__ == '__main__': if len(sys.argv) != 2: print("Usage: python osm_file_stats.py ") sys.exit(-1) h = FileStatsHandler() h.apply_file(sys.argv[1]) h.nodes.outstats("Nodes") h.ways.outstats("Ways") h.rels.outstats("Relations") pyosmium-2.13.0/examples/osm_file_stats.py000066400000000000000000000014311315206111100206500ustar00rootroot00000000000000""" Simple example that counts the number of objects in an osm file. Shows how to write a handler for the different types of objects. """ import osmium as o import sys class FileStatsHandler(o.SimpleHandler): def __init__(self): super(FileStatsHandler, self).__init__() self.nodes = 0 self.ways = 0 self.rels = 0 def node(self, n): self.nodes += 1 def way(self, w): self.ways += 1 def relation(self, r): self.rels += 1 if __name__ == '__main__': if len(sys.argv) != 2: print("Usage: python osm_file_stats.py ") sys.exit(-1) h = FileStatsHandler() h.apply_file(sys.argv[1]) print("Nodes: %d" % h.nodes) print("Ways: %d" % h.ways) print("Relations: %d" % h.rels) pyosmium-2.13.0/examples/osm_replication_stats.py000066400000000000000000000035441315206111100222510ustar00rootroot00000000000000""" Simple example that counts the number of changes on a replication server starting from a given timestamp for a maximum of n hours. Shows how to detect the different kind of modifications. """ import osmium as o import sys import datetime as dt import osmium.replication.server as rserv class Stats(object): def __init__(self): self.added = 0 self.modified = 0 self.deleted = 0 def add(self, o): if o.deleted: self.deleted += 1 elif o.version == 1: self.added += 1 else: self.modified += 1 def outstats(self, prefix): print("%s added: %d" % (prefix, self.added)) print("%s modified: %d" % (prefix, self.modified)) print("%s deleted: %d" % (prefix, self.deleted)) class FileStatsHandler(o.SimpleHandler): def __init__(self): super(FileStatsHandler, self).__init__() self.nodes = Stats() self.ways = Stats() self.rels = Stats() def node(self, n): self.nodes.add(n) def way(self, w): self.ways.add(w) def relation(self, r): self.rels.add(r) if __name__ == '__main__': if len(sys.argv) != 4: print("Usage: python osm_replication_stats.py ") sys.exit(-1) server_url = sys.argv[1] start = dt.datetime.strptime(sys.argv[2], "%Y-%m-%dT%H:%M:%SZ") if sys.version_info >= (3,0): start = start.replace(tzinfo=dt.timezone.utc) maxkb = min(int(sys.argv[3]), 10 * 1024) repserv = rserv.ReplicationServer(server_url) seqid = repserv.timestamp_to_sequence(start) print("Initial sequence id:", seqid) h = FileStatsHandler() seqid = repserv.apply_diffs(h, seqid, maxkb) print("Final sequence id:", seqid) h.nodes.outstats("Nodes") h.ways.outstats("Ways") h.rels.outstats("Relations") pyosmium-2.13.0/examples/osm_url_stats.py000066400000000000000000000015531315206111100205400ustar00rootroot00000000000000""" This also creates stats over an OSM file, only the file is first read into a string buffer. Shows how to use input from strings. """ import osmium as o import sys import urllib.request class FileStatsHandler(o.SimpleHandler): def __init__(self): super(FileStatsHandler, self).__init__() self.nodes = 0 self.ways = 0 self.rels = 0 def node(self, n): self.nodes += 1 def way(self, w): self.ways += 1 def relation(self, r): self.rels += 1 if __name__ == '__main__': if len(sys.argv) != 2: print("Usage: python osm_url_stats.py ") sys.exit(-1) data = urllib.request.urlopen(sys.argv[1]).read() h = FileStatsHandler() h.apply_buffer(data, sys.argv[1]) print("Nodes: %d" % h.nodes) print("Ways: %d" % h.ways) print("Relations: %d" % h.rels) pyosmium-2.13.0/examples/pub_names.py000066400000000000000000000010571315206111100176120ustar00rootroot00000000000000""" Search for pubs in an osm file and list their names. """ import osmium import sys class NamesHandler(osmium.SimpleHandler): def output_pubs(self, tags): if tags.get('amenity') == 'pub': if 'name' in tags: print(tags['name']) def node(self, n): self.output_pubs(n.tags) def way(self, w): self.output_pubs(w.tags) if __name__ == '__main__': if len(sys.argv) != 2: print("Usage: python pub_names.py ") sys.exit(-1) NamesHandler().apply_file(sys.argv[1]) pyosmium-2.13.0/examples/road_length.py000066400000000000000000000020301315206111100201170ustar00rootroot00000000000000""" Compute the total length of highways in an osm file. Shows how extract the geometry of a way. """ import osmium as o import sys class RoadLengthHandler(o.SimpleHandler): def __init__(self): super(RoadLengthHandler, self).__init__() self.length = 0.0 def way(self, w): if 'highway' in w.tags: try: self.length += o.geom.haversine_distance(w.nodes) except o.InvalidLocationError: # A location error might occur if the osm file is an extract # where nodes of ways near the boundary are missing. print("WARNING: way %d incomplete. Ignoring." % w.id) if __name__ == '__main__': if len(sys.argv) != 2: print("Usage: python road_length.py ") sys.exit(-1) h = RoadLengthHandler() # As we need the geometry, the node locations need to be cached. Therefore # set 'locations' to true. h.apply_file(sys.argv[1], locations=True) print('Total way length: %.2f km' % (h.length/1000)) pyosmium-2.13.0/examples/use_nodecache.py000066400000000000000000000012771315206111100204320ustar00rootroot00000000000000import osmium as o import sys class WayHandler(o.SimpleHandler): def __init__(self, idx): super(WayHandler, self).__init__() self.idx = idx def way(self, w): for n in w.nodes: n.lat, n.lon # throws an exception if the coordinates are missing loc = idx.get(n.ref) print("%d %s" %(w.id, len(w.nodes))) if len(sys.argv) != 3: print("Usage: python create_nodecache.py ") exit() reader = o.io.Reader(sys.argv[1], o.osm.osm_entity_bits.WAY) idx = o.index.create_map("sparse_file_array," + sys.argv[2]) lh = o.NodeLocationsForWays(idx) lh.ignore_errors() o.apply(reader, lh, WayHandler(idx)) reader.close() pyosmium-2.13.0/lib/000077500000000000000000000000001315206111100142145ustar00rootroot00000000000000pyosmium-2.13.0/lib/generic_handler.hpp000066400000000000000000000162001315206111100200350ustar00rootroot00000000000000#ifndef PYOSMIUM_GENERIC_HANDLER_HPP #define PYOSMIUM_GENERIC_HANDLER_HPP #include #include #include #include #include #include #include #include #include typedef osmium::index::map::Map index_type; class BaseHandler : public osmium::handler::Handler { protected: enum pre_handler { no_handler, location_handler, area_handler }; public: virtual void apply_start() {}; // handler functions virtual void node(const osmium::Node&) = 0; virtual void way(const osmium::Way&) = 0; virtual void relation(const osmium::Relation&) = 0; virtual void changeset(const osmium::Changeset&) = 0; virtual void area(const osmium::Area&) = 0; private: void apply_with_location(osmium::io::Reader &r, const std::string &idx) { const auto& map_factory = osmium::index::MapFactory::instance(); std::unique_ptr index = map_factory.create_map(idx); osmium::handler::NodeLocationsForWays location_handler(*index); location_handler.ignore_errors(); osmium::apply(r, location_handler, *this); } void apply_with_area(osmium::io::Reader &r, osmium::area::MultipolygonManager &mp_manager, const std::string &idx) { const auto& map_factory = osmium::index::MapFactory::instance(); std::unique_ptr index = map_factory.create_map(idx); osmium::handler::NodeLocationsForWays location_handler(*index); location_handler.ignore_errors(); osmium::apply(r, location_handler, *this, mp_manager.handler([this](const osmium::memory::Buffer& area_buffer) { osmium::apply(area_buffer, *this); }) ); } protected: void apply(const osmium::io::File &file, osmium::osm_entity_bits::type types, pre_handler pre = no_handler, const std::string &idx = "sparse_mem_array") { switch (pre) { case no_handler: { osmium::io::Reader reader(file, types); osmium::apply(reader, *this); reader.close(); break; } case location_handler: { osmium::io::Reader reader(file, types); apply_with_location(reader, idx); reader.close(); break; } case area_handler: { osmium::area::Assembler::config_type assembler_config; osmium::area::MultipolygonManager mp_manager{assembler_config}; osmium::relations::read_relations(file, mp_manager); osmium::io::Reader reader2(file); apply_with_area(reader2, mp_manager, idx); reader2.close(); break; } } } }; using namespace boost::python; struct SimpleHandlerWrap: BaseHandler, wrapper { void node(const osmium::Node& node) override { if (!(m_callbacks & osmium::osm_entity_bits::node)) return; if (override f = this->get_override("node")) { f(boost::ref(node)); } } void way(const osmium::Way& way) { if (!(m_callbacks & osmium::osm_entity_bits::way)) return; if (override f = this->get_override("way")) f(boost::ref(way)); } void relation(const osmium::Relation& rel) { if (!(m_callbacks & osmium::osm_entity_bits::relation)) return; if (override f = this->get_override("relation")) f(boost::ref(rel)); } void changeset(const osmium::Changeset& cs) { if (!(m_callbacks & osmium::osm_entity_bits::changeset)) return; if (override f = this->get_override("changeset")) f(boost::ref(cs)); } void area(const osmium::Area& area) { if (!(m_callbacks & osmium::osm_entity_bits::area)) return; if (override f = this->get_override("area")) f(boost::ref(area)); } void apply_file(const std::string &filename, bool locations = false, const std::string &idx = "sparse_mem_array") { apply_object(osmium::io::File(filename), locations, idx); } void apply_buffer(const boost::python::object &buf, const boost::python::str &format, bool locations = false, const std::string &idx = "sparse_mem_array") { Py_buffer pybuf; PyObject_GetBuffer(buf.ptr(), &pybuf, PyBUF_C_CONTIGUOUS); size_t len = (size_t) pybuf.len; const char *cbuf = reinterpret_cast(pybuf.buf); const char *cfmt = boost::python::extract(format); apply_object(osmium::io::File(cbuf, len, cfmt), locations, idx); } void apply_start() override { m_callbacks = osmium::osm_entity_bits::nothing; if (hasfunc("node")) m_callbacks |= osmium::osm_entity_bits::node; if (hasfunc("way")) m_callbacks |= osmium::osm_entity_bits::way; if (hasfunc("relation")) m_callbacks |= osmium::osm_entity_bits::relation; if (hasfunc("area")) m_callbacks |= osmium::osm_entity_bits::area; if (hasfunc("changeset")) m_callbacks |= osmium::osm_entity_bits::changeset; } private: void apply_object(osmium::io::File file, bool locations, const std::string &idx) { osmium::osm_entity_bits::type entities = osmium::osm_entity_bits::nothing; BaseHandler::pre_handler handler = locations? BaseHandler::location_handler :BaseHandler::no_handler; apply_start(); if (m_callbacks & osmium::osm_entity_bits::area) { entities = osmium::osm_entity_bits::object; handler = BaseHandler::area_handler; } else { if (locations || m_callbacks & osmium::osm_entity_bits::node) entities |= osmium::osm_entity_bits::node; if (m_callbacks & osmium::osm_entity_bits::way) entities |= osmium::osm_entity_bits::way; if (m_callbacks & osmium::osm_entity_bits::relation) entities |= osmium::osm_entity_bits::relation; } if (m_callbacks & osmium::osm_entity_bits::changeset) entities |= osmium::osm_entity_bits::changeset; apply(file, entities, handler, idx); } bool hasfunc(char const *name) { reference_existing_object::apply::type converter; PyObject* obj = converter( this ); if (PyObject_HasAttrString(obj, name)) { auto o = boost::python::object(handle<>(obj)); return o.attr(name) != boost::python::object(); } return false; } osmium::osm_entity_bits::type m_callbacks; }; #endif pyosmium-2.13.0/lib/generic_writer.hpp000066400000000000000000000216111315206111100177360ustar00rootroot00000000000000#ifndef PYOSMIUM_GENERIC_WRITER_HPP #define PYOSMIUM_GENERIC_WRITER_HPP #include #include #include #include #include #include class SimpleWriterWrap { enum { BUFFER_WRAP = 4096 }; public: SimpleWriterWrap(const char* filename, size_t bufsz=4096*1024) : writer(filename), buffer(bufsz < 2 * BUFFER_WRAP ? 2 * BUFFER_WRAP : bufsz, osmium::memory::Buffer::auto_grow::yes), buffer_size(buffer.capacity()) // same rounding to BUFFER_WRAP {} virtual ~SimpleWriterWrap() { close(); } void add_osmium_object(const osmium::OSMObject& o) { buffer.add_item(o); flush_buffer(); } void add_node(boost::python::object o) { boost::python::extract node(o); if (node.check()) { buffer.add_item(node()); } else { osmium::builder::NodeBuilder builder(buffer); if (hasattr(o, "location")) { osmium::Node& n = builder.object(); n.set_location(get_location(o.attr("location"))); } set_common_attributes(o, builder); if (hasattr(o, "tags")) set_taglist(o.attr("tags"), builder); } flush_buffer(); } void add_way(const boost::python::object& o) { boost::python::extract way(o); if (way.check()) { buffer.add_item(way()); } else { osmium::builder::WayBuilder builder(buffer); set_common_attributes(o, builder); if (hasattr(o, "nodes")) set_nodelist(o.attr("nodes"), &builder); if (hasattr(o, "tags")) set_taglist(o.attr("tags"), builder); } flush_buffer(); } void add_relation(boost::python::object o) { boost::python::extract rel(o); if (rel.check()) { buffer.add_item(rel()); } else { osmium::builder::RelationBuilder builder(buffer); set_common_attributes(o, builder); if (hasattr(o, "members")) set_memberlist(o.attr("members"), &builder); if (hasattr(o, "tags")) set_taglist(o.attr("tags"), builder); } flush_buffer(); } void close() { if (buffer) { writer(std::move(buffer)); writer.close(); buffer = osmium::memory::Buffer(); } } private: void set_object_attributes(const boost::python::object& o, osmium::OSMObject& t) { if (hasattr(o, "id")) t.set_id(boost::python::extract(o.attr("id"))); if (hasattr(o, "visible")) t.set_visible(boost::python::extract(o.attr("visible"))); if (hasattr(o, "version")) t.set_version(boost::python::extract(o.attr("version"))); if (hasattr(o, "changeset")) t.set_changeset(boost::python::extract(o.attr("changeset"))); if (hasattr(o, "uid")) t.set_uid_from_signed(boost::python::extract(o.attr("uid"))); if (hasattr(o, "timestamp")) { boost::python::object ts = o.attr("timestamp"); boost::python::extract ots(ts); if (ots.check()) { t.set_timestamp(ots()); } else { if (hasattr(ts, "timestamp")) { double epoch = boost::python::extract(ts.attr("timestamp")()); t.set_timestamp(osmium::Timestamp(uint32_t(epoch))); } else { // XXX terribly inefficient because of the double string conversion // but the only painless method for converting a datetime // in python < 3.3. if (hasattr(ts, "strftime")) ts = ts.attr("strftime")("%Y-%m-%dT%H:%M:%SZ"); t.set_timestamp(osmium::Timestamp(boost::python::extract(ts))); } } } } template void set_common_attributes(const boost::python::object& o, T& builder) { set_object_attributes(o, builder.object()); if (hasattr(o, "user")) { auto s = boost::python::extract(o.attr("user")); builder.set_user(s); } } template void set_taglist(const boost::python::object& o, T& obuilder) { // original taglist boost::python::extract otl(o); if (otl.check()) { if (otl().size() > 0) obuilder.add_item(otl()); return; } // dict boost::python::extract tagdict(o); if (tagdict.check()) { auto items = tagdict().items(); auto len = boost::python::len(items); if (len == 0) return; osmium::builder::TagListBuilder builder(buffer, &obuilder); auto iter = items.attr("__iter__")(); for (int i = 0; i < len; ++i) { #if PY_VERSION_HEX < 0x03000000 auto tag = iter.attr("next")(); #else auto tag = iter.attr("__next__")(); #endif builder.add_tag(boost::python::extract(tag[0]), boost::python::extract(tag[1])); } return; } // any other iterable auto l = boost::python::len(o); if (l == 0) return; osmium::builder::TagListBuilder builder(buffer, &obuilder); for (int i = 0; i < l; ++i) { auto tag = o[i]; boost::python::extract ot(tag); if (ot.check()) { builder.add_tag(ot); } else { builder.add_tag(boost::python::extract(tag[0]), boost::python::extract(tag[1])); } } } void set_nodelist(const boost::python::object& o, osmium::builder::WayBuilder *builder) { // original nodelist boost::python::extract onl(o); if (onl.check()) { if (onl().size() > 0) builder->add_item(onl()); return; } auto len = boost::python::len(o); if (len == 0) return; osmium::builder::WayNodeListBuilder wnl_builder(buffer, builder); for (int i = 0; i < len; ++i) { boost::python::extract ref(o[i]); if (ref.check()) wnl_builder.add_node_ref(ref()); else wnl_builder.add_node_ref(boost::python::extract(o[i])); } } void set_memberlist(const boost::python::object& o, osmium::builder::RelationBuilder *builder) { // original memberlist boost::python::extract oml(o); if (oml.check()) { if (oml().size() > 0) builder->add_item(oml()); return; } auto len = boost::python::len(o); if (len == 0) return; osmium::builder::RelationMemberListBuilder rml_builder(buffer, builder); for (int i = 0; i < len; ++i) { auto member = o[i]; auto type = osmium::char_to_item_type(boost::python::extract(member[0])()[0]); auto id = boost::python::extract(member[1])(); auto role = boost::python::extract(member[2])(); rml_builder.add_member(type, id, role); } } osmium::Location get_location(const boost::python::object& o) { boost::python::extract ol(o); if (ol.check()) return ol; // default is a tuple with two floats return osmium::Location(boost::python::extract(o[0]), boost::python::extract(o[1])); } bool hasattr(const boost::python::object& obj, char const *attr) { return PyObject_HasAttrString(obj.ptr(), attr) && (obj.attr(attr) != boost::python::object()); } void flush_buffer() { buffer.commit(); if (buffer.committed() > buffer_size - BUFFER_WRAP) { osmium::memory::Buffer new_buffer(buffer_size, osmium::memory::Buffer::auto_grow::yes); using std::swap; swap(buffer, new_buffer); writer(std::move(new_buffer)); } } osmium::io::Writer writer; osmium::memory::Buffer buffer; size_t buffer_size; }; #endif // PYOSMIUM_GENERIC_WRITER_HPP pyosmium-2.13.0/lib/geom.cc000066400000000000000000000173251315206111100154620ustar00rootroot00000000000000#include #include #include #include #include #include class WKBFactory : public osmium::geom::WKBFactory<> { public: WKBFactory() : osmium::geom::WKBFactory<>(osmium::geom::wkb_type::wkb, osmium::geom::out_type::hex) {} }; using WKTFactory = osmium::geom::WKTFactory<>; using GeoJSONFactory = osmium::geom::GeoJSONFactory<>; BOOST_PYTHON_MODULE(geom) { using namespace boost::python; docstring_options doc_options(true, true, false); enum_("use_nodes") .value("UNIQUE", osmium::geom::use_nodes::unique) .value("ALL", osmium::geom::use_nodes::all) ; enum_("direction") .value("BACKWARD", osmium::geom::direction::backward) .value("FORWARD", osmium::geom::direction::forward) ; def("haversine_distance", static_cast(&osmium::geom::haversine::distance), arg("list"), "Compute the distance using the Haversine algorithm which takes the " "curvature of earth into account. If a :py:class:`WayNodeList` is given " "as a parameter the total length of the way in meters is computed."); class_("WKBFactory", "Factory that creates WKB from osmium geometries.") .add_property("epsg", &WKBFactory::epsg, "(read-only) EPSG number of the output geometry.") .add_property("proj_string", &WKBFactory::proj_string, "(read-only) projection string of the output geometry.") .def("create_point", static_cast(&WKBFactory::create_point), (arg("self"), arg("location")), "Create a point geometry from a :py:class:`osmium.osm.Location`.") .def("create_point", static_cast(&WKBFactory::create_point), (arg("self"), arg("node")), "Create a point geometry from a :py:class:`osmium.osm.Node`.") .def("create_point", static_cast(&WKBFactory::create_point), (arg("self"), arg("ref")), "Create a point geometry from a :py:class:`osmium.osm.NodeRef`.") .def("create_linestring", static_cast(&WKBFactory::create_linestring), (arg("self"), arg("list"), arg("use_nodes")=osmium::geom::use_nodes::unique, arg("direction")=osmium::geom::direction::forward), "Create a LineString geometry from a :py:class:`osmium.osm.WayNodeList`.") .def("create_linestring", static_cast(&WKBFactory::create_linestring), (arg("self"), arg("way"), arg("use_nodes")=osmium::geom::use_nodes::unique, arg("direction")=osmium::geom::direction::forward), "Create a LineString geometry from a :py:class:`osmium.osm.Way`.") .def("create_multipolygon", &WKBFactory::create_multipolygon, (arg("self"), arg("area")), "Create a MultiPolygon geometry from a :py:class:`osmium.osm.Area`.") ; class_("WKTFactory", "Factory that creates WKT from osmium geometries.") .add_property("epsg", &WKTFactory::epsg, "(read-only) EPSG number of the output geometry.") .add_property("proj_string", &WKTFactory::proj_string, "(read-only) projection string of the output geometry.") .def("create_point", static_cast(&WKTFactory::create_point), (arg("self"), arg("location")), "Create a point geometry from a :py:class:`osmium.osm.Location`.") .def("create_point", static_cast(&WKTFactory::create_point), (arg("self"), arg("node")), "Create a point geometry from a :py:class:`osmium.osm.Node`.") .def("create_point", static_cast(&WKTFactory::create_point), (arg("self"), arg("ref")), "Create a point geometry from a :py:class:`osmium.osm.NodeRef`.") .def("create_linestring", static_cast(&WKTFactory::create_linestring), (arg("self"), arg("list"), arg("use_nodes")=osmium::geom::use_nodes::unique, arg("direction")=osmium::geom::direction::forward), "Create a LineString geometry from a :py:class:`osmium.osm.WayNodeList`.") .def("create_linestring", static_cast(&WKTFactory::create_linestring), (arg("self"), arg("way"), arg("use_nodes")=osmium::geom::use_nodes::unique, arg("direction")=osmium::geom::direction::forward), "Create a LineString geometry from a :py:class:`osmium.osm.Way`.") .def("create_multipolygon", &WKTFactory::create_multipolygon, (arg("self"), arg("area")), "Create a MultiPolygon geometry from a :py:class:`osmium.osm.Area`.") ; class_("GeoJSONFactory", "Factory that creates GeoJSON geometries from osmium geometries.") .add_property("epsg", &GeoJSONFactory::epsg, "(read-only) EPSG number of the output geometry.") .add_property("proj_string", &GeoJSONFactory::proj_string, "(read-only) projection string of the output geometry.") .def("create_point", static_cast(&GeoJSONFactory::create_point), (arg("self"), arg("location")), "Create a point geometry from a :py:class:`osmium.osm.Location`.") .def("create_point", static_cast(&GeoJSONFactory::create_point), (arg("self"), arg("node")), "Create a point geometry from a :py:class:`osmium.osm.Node`.") .def("create_point", static_cast(&GeoJSONFactory::create_point), (arg("self"), arg("ref")), "Create a point geometry from a :py:class:`osmium.osm.NodeRef`.") .def("create_linestring", static_cast(&GeoJSONFactory::create_linestring), (arg("self"), arg("list"), arg("use_nodes")=osmium::geom::use_nodes::unique, arg("direction")=osmium::geom::direction::forward), "Create a LineString geometry from a :py:class:`osmium.osm.WayNodeList`.") .def("create_linestring", static_cast(&GeoJSONFactory::create_linestring), (arg("self"), arg("way"), arg("use_nodes")=osmium::geom::use_nodes::unique, arg("direction")=osmium::geom::direction::forward), "Create a LineString geometry from a :py:class:`osmium.osm.Way`.") .def("create_multipolygon", &GeoJSONFactory::create_multipolygon, (arg("self"), arg("area")), "Create a MultiPolygon geometry from a :py:class:`osmium.osm.Area`.") ; } pyosmium-2.13.0/lib/index.cc000066400000000000000000000042661315206111100156420ustar00rootroot00000000000000#include #include #include #include #include "win_boost_fix.hpp" using namespace boost::python; typedef osmium::index::map::Map LocationTable; LocationTable *create_map(const std::string& config_string) { const auto& map_factory = osmium::index::MapFactory::instance(); return map_factory.create_map(config_string).release(); } PyObject *map_types() { const auto& map_factory = osmium::index::MapFactory::instance(); boost::python::list* l = new boost::python::list(); for (auto const &e : map_factory.map_types()) (*l).append(e); return l->ptr(); } BOOST_PYTHON_MODULE(index) { docstring_options doc_options(true, true, false); class_("LocationTable", "A map from a node ID to a location object. This implementation works " "only with positive node IDs.", no_init) .def("set", &LocationTable::set, (arg("self"), arg("id"), arg("loc")), "Set the location for a given node id.") .def("get", &LocationTable::get, (arg("self"), arg("id")), "Return the location for a given id.") .def("used_memory", &LocationTable::used_memory, arg("self"), "Return the size (in bytes) currently allocated by this location table.") .def("clear", &LocationTable::clear, arg("self"), "Remove all entries from the location table.") ; def("create_map", &create_map, return_value_policy(), (arg("map_type")), "Create a new location store. The string parameter takes the type " "and, where required, additional arguments separated by comma. For " "example, to create an array cache backed by a file ``foo.store``, " "the map_type should be ``dense_file_array,foo.store``."); def("map_types", &map_types, "Return a list of strings with valid types for the location table."); } pyosmium-2.13.0/lib/io.cc000066400000000000000000000051261315206111100151360ustar00rootroot00000000000000#include #include #include #include "osm.cc" BOOST_PYTHON_MODULE(io) { using namespace boost::python; docstring_options doc_options(true, true, false); class_("Header", "File header with global information about the file.") .add_property("has_multiple_object_versions", &osmium::io::Header::has_multiple_object_versions, make_function(&osmium::io::Header::set_has_multiple_object_versions, return_value_policy()), "True if there may be more than one version of the same " "object in the file. This happens normally only in history " "files.") .def("box", &osmium::io::Header::box, arg("self"), "Return the bounding box of the data in the file or an invalid " "box if the information is not available.") .def("get", &osmium::io::Header::get, (arg("self"), arg("key"), arg("default")=""), "Get the value of header option 'key' or default value if " "there is no header option with that name. The default cannot be " "None.") .def("set", static_cast(&osmium::io::Header::set), (arg("self"), arg("key"), arg("value")), "Set the value of header option 'key'.") ; class_("Reader", "A class that reads OSM data from a file.", init()) .def(init()) .def("eof", &osmium::io::Reader::eof, arg("self"), "Check if the end of file has been reached.") .def("close", &osmium::io::Reader::close, arg("self"), "Close any open file handles. The reader is unusable afterwards.") .def("header", &osmium::io::Reader::header, arg("self"), "Return the header with file information, see :py:class:`osmium.io.Header`.") ; class_("Writer", "Class for writing OSM data to a file. This class just encapsulates an " "OSM file,. Have a look `osmium.SimpleWriter` for a high-level interface " "for writing out data.", init()) .def(init()) .def("close", &osmium::io::Writer::close, arg("self"), "Close any open file handles. The writer is unusable afterwards.") ; } pyosmium-2.13.0/lib/merged_input.hpp000066400000000000000000000141261315206111100174130ustar00rootroot00000000000000#ifndef PYOSMIUM_MERGED_INPUT_HPP #define PYOSMIUM_MERGED_INPUT_HPP #include #include #include #include #include #include #include #include #include #include #include #include namespace { /** * Copy the first OSM object with a given Id to the output. Keep * track of the Id of each object to do this. * * We are using this functor class instead of a simple lambda, because the * lambda doesn't build on MSVC. */ class copy_first_with_id { osmium::io::Writer* writer; osmium::object_id_type id = 0; public: explicit copy_first_with_id(osmium::io::Writer& w) : writer(&w) { } void operator()(const osmium::OSMObject& obj) { if (obj.id() != id) { if (obj.visible()) { (*writer)(obj); } id = obj.id(); } } }; } // anonymous namespace namespace pyosmium { class MergeInputReader { void apply_without_location(BaseHandler& handler, bool simplify = true) { handler.apply_start(); if (simplify) { objects.sort(osmium::object_order_type_id_reverse_version()); osmium::item_type prev_type = osmium::item_type::undefined; osmium::object_id_type prev_id = 0; for (const auto &item: objects) { if (item.type() != prev_type || item.id() != prev_id) { prev_type = item.type(); prev_id = item.id(); osmium::apply_item(item, handler); } } } else { objects.sort(osmium::object_order_type_id_version()); osmium::apply(objects.cbegin(), objects.cend(), handler); } objects = osmium::ObjectPointerCollection(); changes.clear(); } void apply_with_location(BaseHandler& handler, const std::string &idx, bool simplify = true) { const auto& map_factory = osmium::index::MapFactory::instance(); std::unique_ptr index = map_factory.create_map(idx); osmium::handler::NodeLocationsForWays location_handler(*index); location_handler.ignore_errors(); handler.apply_start(); if (simplify) { objects.sort(osmium::object_order_type_id_reverse_version()); osmium::item_type prev_type = osmium::item_type::undefined; osmium::object_id_type prev_id = 0; for (auto &item: objects) { if (item.type() != prev_type || item.id() != prev_id) { prev_type = item.type(); prev_id = item.id(); osmium::apply_item(item, location_handler, handler); } } } else { objects.sort(osmium::object_order_type_id_version()); osmium::apply(objects.begin(), objects.end(), location_handler, handler); } objects = osmium::ObjectPointerCollection(); changes.clear(); } public: void apply(BaseHandler& handler, const std::string &idx = "", bool simplify = true) { if (idx.empty()) apply_without_location(handler, simplify); else apply_with_location(handler, idx, simplify); } void apply_to_reader(osmium::io::Reader &reader, osmium::io::Writer &writer, bool with_history = true) { auto input = osmium::io::make_input_iterator_range(reader); if (with_history) { // For history files this is a straightforward sort of the change // files followed by a merge with the input file. objects.sort(osmium::object_order_type_id_version()); auto out = osmium::io::make_output_iterator(writer); std::set_union(objects.begin(), objects.end(), input.begin(), input.end(), out); } else { // For normal data files we sort with the largest version of each // object first and then only copy this last version of any object // to the output. objects.sort(osmium::object_order_type_id_reverse_version()); auto output_it = boost::make_function_output_iterator( copy_first_with_id(writer) ); std::set_union(objects.begin(), objects.end(), input.begin(), input.end(), output_it, osmium::object_order_type_id_reverse_version()); } } size_t add_file(const std::string &filename) { return internal_add(osmium::io::File(filename)); } size_t add_buffer(const boost::python::object &buf, const boost::python::str &format) { Py_buffer pybuf; PyObject_GetBuffer(buf.ptr(), &pybuf, PyBUF_C_CONTIGUOUS); size_t len = (size_t) pybuf.len; const char *cbuf = reinterpret_cast(pybuf.buf); const char *cfmt = boost::python::extract(format); return internal_add(osmium::io::File(cbuf, len, cfmt)); } private: size_t internal_add(osmium::io::File change_file) { size_t sz = 0; osmium::io::Reader reader(change_file, osmium::osm_entity_bits::object); while (osmium::memory::Buffer buffer = reader.read()) { osmium::apply(buffer, objects); sz += buffer.committed(); changes.push_back(std::move(buffer)); } return sz; } std::vector changes; osmium::ObjectPointerCollection objects; }; } #endif /* PYOSMIUM_MERGED_INPUT_HPP */ pyosmium-2.13.0/lib/osm.cc000066400000000000000000000477521315206111100153400ustar00rootroot00000000000000 #include #include #include #include #include #include "std_pair.hpp" inline const char *get_tag_by_key(osmium::TagList const& obj, const char *key) { if (!key) { PyErr_SetString(PyExc_KeyError, "Key 'None' not allowed."); boost::python::throw_error_already_set(); } const char* v = obj.get_value_by_key(key); if (!v) { PyErr_SetString(PyExc_KeyError, "No tag with that key."); boost::python::throw_error_already_set(); } return v; } inline const char *get_tag_by_key_with_none(osmium::TagList const& obj, const char *key) { return key ? obj.get_value_by_key(key) : nullptr; } inline const osmium::NodeRef &get_item_nodereflist(osmium::NodeRefList const& list, ssize_t idx) { auto sz = list.size(); osmium::NodeRefList::size_type iout = (idx >= 0 ? idx : (ssize_t) sz + idx); if (iout >= sz) { PyErr_SetString(PyExc_IndexError, "Bad index."); boost::python::throw_error_already_set(); } return list[iout]; } inline bool taglist_contains_tag(osmium::TagList const& obj, const char *key) { return key && obj.has_key(key); } inline const char member_item_type(osmium::RelationMember& obj) { return item_type_to_char(obj.type()); } // Converter for osmium::Timestamp -> datetime.datetime struct Timestamp_to_python { static PyObject* convert(osmium::Timestamp const& s) { #if PY_VERSION_HEX >= 0x03000000 static auto fconv = boost::python::import("datetime").attr("datetime").attr("fromtimestamp"); static boost::python::object utc = boost::python::import("datetime").attr("timezone").attr("utc"); return boost::python::incref(fconv(s.seconds_since_epoch(), utc).ptr()); #else static auto fconv = boost::python::import("datetime").attr("datetime").attr("utcfromtimestamp"); return boost::python::incref(fconv(s.seconds_since_epoch()).ptr()); #endif } }; BOOST_PYTHON_MODULE(_osm) { using namespace boost::python; docstring_options doc_options(true, true, false); to_python_converter(); std_pair_to_python_converter(); enum_("osm_entity_bits") .value("NOTHING", osmium::osm_entity_bits::nothing) .value("NODE", osmium::osm_entity_bits::node) .value("WAY", osmium::osm_entity_bits::way) .value("RELATION", osmium::osm_entity_bits::relation) .value("AREA", osmium::osm_entity_bits::area) .value("OBJECT", osmium::osm_entity_bits::object) .value("CHANGESET", osmium::osm_entity_bits::changeset) .value("ALL", osmium::osm_entity_bits::all) ; class_("Location", "A geographic coordinate in WGS84 projection. A location doesn't " "necessarily have to be valid.") .def(init()) .add_property("x", &osmium::Location::x, "(read-only) X coordinate (longitude) as a fixed-point integer.") .add_property("y", &osmium::Location::y, "(read-only) Y coordinate (latitude) as a fixed-point integer.") .add_property("lon", &osmium::Location::lon, "(read-only) Longitude (x coordinate) as floating point number.") .add_property("lat", &osmium::Location::lat, "(read-only) Latitude (y coordinate) as floating point number.") .def("valid", &osmium::Location::valid, args("self"), "Check that the location is a valid WGS84 coordinate, i.e. " "that it is within the usual bounds.") .def("lat_without_check", &osmium::Location::lat_without_check, args("self"), "Return latitude (y coordinate) without checking if the location " "is valid.") .def("lon_without_check", &osmium::Location::lon_without_check, args("self"), "Return longitude (x coordinate) without checking if the location " "is valid.") ; class_("Box", "A bounding box around a geographic area. It is defined by an " ":py:class:`osmium.osm.Location` for the bottem-left corner and an " "``osmium.osm.Location`` for the top-right corner. Those locations may " " be invalid in which case the box is considered invalid, too.") .def(init()) .def(init()) .add_property("bottom_left", make_function(static_cast(&osmium::Box::bottom_left), return_value_policy()), "(read-only) Bottom-left corner of the bounding box.") .add_property("top_right", make_function(static_cast(&osmium::Box::top_right), return_value_policy()), "(read-only) Top-right corner of the bounding box.") .def("extend", make_function(static_cast(&osmium::Box::extend), return_value_policy()), //(arg("self"), arg("location")), "Extend the box to include the given location. If the location " "is invalid the box remains unchanged. If the box is invalid, it " "will contain only the location after the operation.") .def("extend", make_function(static_cast(&osmium::Box::extend), return_value_policy()), //(arg("self"), arg("box")), "Extend the box to include the given box. If the box to be added " "is invalid the input box remains unchanged. If the input box is invalid, it " "will become equal to the box that was added.") .def("valid", &osmium::Box::valid, args("self"), "Check if the box coordinates are defined and with the usual bounds.") .def("size", &osmium::Box::size, args("self"), "Return the size in square degrees.") .def("contains", &osmium::Box::contains, (arg("self"), arg("location")), "Check if the given location is inside the box.") ; class_("Tag", "A single OSM tag.", no_init) .add_property("k", &osmium::Tag::key, "(read-only) Tag key.") .add_property("v", &osmium::Tag::value, "(read-only) Tag value.") ; class_("TagList", "A fixed list of tags. The list is exported as an unmutable, " "dictionary-like object where the keys are tag strings and the " "items are :py:class:`osmium.osm.Tag`.", no_init) .def("__len__", &osmium::TagList::size) .def("__getitem__", &get_tag_by_key) .def("__contains__", &taglist_contains_tag) .def("__iter__", iterator>()) .def("get", &osmium::TagList::get_value_by_key, (arg("self"), arg("key"), arg("default"))) .def("get", &get_tag_by_key_with_none) ; class_("NodeRef", "A reference to a OSM node that also caches the nodes location.") .add_property("x", &osmium::NodeRef::x, "(read-only) X coordinate (longitude) as a fixed-point integer.") .add_property("y", &osmium::NodeRef::y, "(read-only) Y coordinate (latitude) as a fixed-point integer.") .add_property("lon", &osmium::NodeRef::lon, "(read-only) Longitude (x coordinate) as floating point number.") .add_property("lat", &osmium::NodeRef::lat, "(read-only) Latitude (y coordinate) as floating point number.") .add_property("ref", &osmium::NodeRef::ref, "(read-only) Id of the referenced node.") .add_property("location", static_cast(&osmium::NodeRef::location), "(read-only) Node coordinates as a :py:class:`osmium.osm.Location` object.") ; class_("RelationMember", "Member of a relation.", no_init) .add_property("ref", static_cast(&osmium::RelationMember::ref), "OSM ID of the object. Only unique within the type.") .add_property("type", &member_item_type, "Type of object referenced, a node, way or relation.") .add_property("role", &osmium::RelationMember::role, "The role of the member within the relation, a free-text string. " "If no role is set then the string is empty.") ; class_("RelationMemberList", "An immutable sequence of relation members :py:class:`osmium.osm.RelationMember`.", no_init) .def("__len__", &osmium::RelationMemberList::size) .def("__iter__", iterator>()) ; class_("NodeRefList", "A list of node references, implemented as " "an immutable sequence of :py:class:`osmium.osm.NodeRef`. This class " "is normally not used directly, use one of its subclasses instead.", no_init) .def("__len__", &osmium::NodeRefList::size) .def("__getitem__", make_function(&get_item_nodereflist, return_value_policy())) .def("__iter__", iterator>()) .def("is_closed", &osmium::NodeRefList::is_closed, args("self"), "True if the start and end node are the same (synonym for " "``ends_have_same_id``).") .def("ends_have_same_id", &osmium::NodeRefList::ends_have_same_id, args("self"), "True if the start and end node are exactly the same.") .def("ends_have_same_location", &osmium::NodeRefList::ends_have_same_location, args("self"), "True if the start and end node of the way are at the same location. " "Throws an exception if the location of one of the nodes is missing.") ; class_, boost::noncopyable>("WayNodeList", "List of nodes in a way. For its members see :py:class:`osmium.osm.NodeRefList`.", no_init) ; class_, boost::noncopyable>("OuterRing", "List of nodes in an outer ring. For its members see :py:class:`osmium.osm.NodeRefList`.", no_init) ; class_, boost::noncopyable>("InnerRing", "List of nodes in an inner ring. For its members see :py:class:`osmium.osm.NodeRefList`.", no_init) ; class_("OSMObject", "This is the base class for all OSM entity classes below and contains " "all common attributes.", no_init) .add_property("id", &osmium::OSMObject::id, "(read-only) OSM id of the object.") .add_property("deleted", &osmium::OSMObject::deleted, "(read-only) True if the object is no longer visible.") .add_property("visible", &osmium::OSMObject::visible, "(read-only) True if the object is visible.") .add_property("version", &osmium::OSMObject::version, "(read-only) Version number of the object.") .add_property("changeset", &osmium::OSMObject::changeset, "(read-only) Id of changeset where this version of the " "object was created.") .add_property("uid", &osmium::OSMObject::uid, "(read-only) Id of the user that created this version " "of the object. Only this ID uniquely identifies users.") .add_property("timestamp", &osmium::OSMObject::timestamp, "(read-only) Date when this version has been created, " "returned as a ``datetime.datetime``.") .add_property("user", &osmium::OSMObject::user, "(read-only) Name of the user that created this version. " "Be aware that user names can change, so that the same " "user ID may appear with different names and vice versa. ") .add_property("tags", make_function(&osmium::OSMObject::tags, return_value_policy()), "(read-only) List of tags describing the object. " "See :py:class:`osmium.osm.TagList`.") .def("positive_id", &osmium::OSMObject::positive_id, arg("self"), "Get the absolute value of the id of this object.") .def("user_is_anonymous", &osmium::OSMObject::user_is_anonymous, arg("self"), "Check if the user is anonymous. If true, the uid does not uniquely " "identify a single user but only the group of all anonymous users " "in general.") ; class_, boost::noncopyable>("Node", "Represents a single OSM node. It inherits from OSMObjects and " "adds a single attribute, the location.", no_init) .add_property("location", static_cast(&osmium::Node::location), "The geographic coordinates of the node. " "See :py:class:`osmium.osm.Location`.") ; class_, boost::noncopyable>("Way", "Represents a OSM way. It inherits the attributes from OSMObjects and " "adds an ordered list of nodes that describes the way.", no_init) .add_property("nodes", make_function(static_cast(&osmium::Way::nodes), return_value_policy()), "(read-only) Ordered list of nodes. See :py:class:`osmium.osm.WayNodeList`.") .def("is_closed", &osmium::Way::is_closed, args("self"), "True if the start and end node are the same (synonym for " "``ends_have_same_id``).") .def("ends_have_same_id", &osmium::Way::ends_have_same_id, args("self"), "True if the start and end node are exactly the same.") .def("ends_have_same_location", &osmium::Way::ends_have_same_location, args("self"), "True if the start and end node of the way are at the same location." "Throws an exception if the location of one of the nodes is missing.") ; class_, boost::noncopyable>("Relation", "Represents a OSM relation. It inherits the attributes from OSMObjects " "and adds an ordered list of members.", no_init) .add_property("members", make_function(static_cast(&osmium::Relation::members), return_value_policy()), "(read-only) Ordered list of relation members. " "See :py:class:`osmium.osm.RelationMemberList`") ; class_ >("InnerRingIterator", "Iterator over inner rings.", no_init) .def("__iter__", iterator, return_internal_reference<> >()) ; class_, boost::noncopyable>("Area", "Areas are a special kind of meta-object representing a polygon. " "They can either be derived from closed ways or from relations " "that represent multipolygons. They also inherit the attributes " "of OSMObjects and in addition contain polygon geometries. Areas have " "their own unique id space. This is computed as the OSM id times 2 " "and for relations 1 is added,", no_init) .def("from_way", &osmium::Area::from_way, args("self"), "Return true if the area was created from a way, false if it was " "created from a relation of multipolygon type.") .def("orig_id", &osmium::Area::orig_id, args("self"), "Compute the original OSM id of this object. Note that this is not " "necessarily unique because the object might be a way or relation " "which have an overlapping id space.") .def("is_multipolygon", &osmium::Area::is_multipolygon, args("self"), "Return true if this area is a true multipolygon, i.e. it consists " "of multiple outer rings.") .def("num_rings", &osmium::Area::num_rings, args("self"), "Return a tuple with the number of outer rings and inner rings.") .def("outer_rings", range >( &osmium::Area::cbegin, &osmium::Area::cend), "Return an iterator over all outer rings of the multipolygon.") .def("inner_rings", &osmium::Area::inner_rings, (arg("self"), arg("outer_ring")), "Return an iterator over all inner rings of the multipolygon.") ; class_("Changeset", "A changeset description.", no_init) .add_property("id", &osmium::Changeset::id, "(read-only) Unique ID of the changeset.") .add_property("uid", &osmium::Changeset::uid, "(read-only) User ID of the changeset creator.") .add_property("created_at", &osmium::Changeset::created_at, "(read-only) Timestamp when the changeset was first opened.") .add_property("closed_at", &osmium::Changeset::closed_at, "(read-only) Timestamp when the changeset was finalized. May be " "None when the changeset is still open.") .add_property("open", &osmium::Changeset::open, "(read-only) True when the changeset is still open.") .add_property("num_changes", &osmium::Changeset::num_changes, "(read-only) The total number of objects changed in this " "Changeset.") .add_property("bounds", make_function(static_cast(&osmium::Changeset::bounds), return_value_policy()), "(read-only) The bounding box of the area that was edited.") .add_property("user", &osmium::Changeset::user, "(read-only) Name of the user that created the changeset. " "Be aware that user names can change, so that the same " "user ID may appear with different names and vice versa. ") .add_property("tags", make_function(&osmium::Changeset::tags, return_value_policy()), "(read-only) List of tags describing the changeset. " "See :py:class:`osmium.osm.TagList`.") .def("user_is_anonymous", &osmium::Changeset::user_is_anonymous, arg("self"), "Check if the user anonymous. If true, the uid does not uniquely " "identify a single user but only the group of all anonymous users " "in general.") ; } pyosmium-2.13.0/lib/osmium.cc000066400000000000000000000244021315206111100160360ustar00rootroot00000000000000#include #include #include #include #include #include "generic_writer.hpp" #include "generic_handler.hpp" #include "merged_input.hpp" #include "write_handler.hpp" #include "win_boost_fix.hpp" // workaround for Visual Studio 2015 Update 3 // https://connect.microsoft.com/VisualStudio/Feedback/Details/2852624 #if (_MSC_VER > 1800 && _MSC_FULL_VER > 190023918) namespace boost { template<> const volatile SimpleHandlerWrap* get_pointer(const volatile SimpleHandlerWrap* p) { return p; } } #endif template void apply_reader_simple(osmium::io::Reader &rd, T &h) { osmium::apply(rd, h); } template <> void apply_reader_simple(osmium::io::Reader &rd, BaseHandler &h) { h.apply_start(); osmium::apply(rd, h); } template void apply_reader_simple_with_location(osmium::io::Reader &rd, osmium::handler::NodeLocationsForWays &l, BaseHandler &h) { h.apply_start(); osmium::apply(rd, l, h); } PyObject *invalidLocationExceptionType = NULL; PyObject *notFoundExceptionType = NULL; void translator1(osmium::invalid_location const&) { PyErr_SetString(invalidLocationExceptionType, "Invalid location"); } void translator2(osmium::not_found const&) { PyErr_SetString(notFoundExceptionType, "Element not found in index"); } PyObject* createExceptionClass(const char* name, PyObject* baseTypeObj = PyExc_Exception) { using std::string; namespace bp = boost::python; string scopeName = bp::extract(bp::scope().attr("__name__")); string qualifiedName0 = scopeName + "." + name; char* qualifiedName1 = const_cast(qualifiedName0.c_str()); PyObject* typeObj = PyErr_NewException(qualifiedName1, baseTypeObj, 0); if(!typeObj) bp::throw_error_already_set(); bp::scope().attr(name) = bp::handle<>(bp::borrowed(typeObj)); return typeObj; } #include "index.cc" BOOST_PYTHON_MODULE(_osmium) { using namespace boost::python; docstring_options doc_options(true, true, false); invalidLocationExceptionType = createExceptionClass("InvalidLocationError", PyExc_RuntimeError); register_exception_translator(&translator1); notFoundExceptionType = createExceptionClass("NotFoundError", PyExc_KeyError); register_exception_translator(&translator2); class_, boost::noncopyable>("NodeLocationsForWays", init()) .def("ignore_errors", &osmium::handler::NodeLocationsForWays::ignore_errors) ; class_("SimpleHandler", "The most generic of OSM data handlers. Derive your data processor " "from this class and implement callbacks for each object type you are " "interested in. The following data types are recognised: \n" " `node`, `way`, `relation`, `area` and `changeset`.\n " "A callback takes exactly one parameter which is the object. Note that " "all objects that are handed into the handler are only readable and are " "only valid until the end of the callback is reached. Any data that " "should be retained must be copied into other data structures.") .def("apply_file", &SimpleHandlerWrap::apply_file, (arg("self"), arg("filename"), arg("locations")=false, arg("idx")="sparse_mem_array"), "Apply the handler to the given file. If locations is true, then\n" "a location handler will be applied before, which saves the node\n" "positions. In that case, the type of this position index can be\n" "further selected in idx. If an area callback is implemented, then\n" "the file will be scanned twice and a location handler and a\n" "handler for assembling multipolygons and areas from ways will\n" "be executed.") .def("apply_buffer", &SimpleHandlerWrap::apply_buffer, (arg("self"), arg("buffer"), arg("format"), arg("locations")=false, arg("idx")="sparse_mem_array"), "Apply the handler to a string buffer. The buffer must be a\n" "byte string.") ; def("apply", &apply_reader_simple, "Apply a chain of handlers."); def("apply", &apply_reader_simple>); def("apply", &apply_reader_simple_with_location); class_("SimpleWriter", "The most generic class to write osmium objects into a file. The writer " "takes a file name as its mandatory parameter. The file must not yet " "exist. The file type to output is determined from the file extension. " "The second (optional) parameter is the buffer size. osmium caches the " "output data in an internal memory buffer before writing it on disk. This " "parameter allows changing the default buffer size of 4MB. Larger buffers " "are normally better but you should be aware that there are normally multiple " "buffers in use during the write process.", init()) .def(init()) .def("add_node", &SimpleWriterWrap::add_node, (arg("self"), arg("node")), "Add a new node to the file. The node may be an ``osmium.osm.Node`` object, " "an ``osmium.osm.mutable.Node`` object or any other Python object that " "implements the same attributes.") .def("add_way", &SimpleWriterWrap::add_way, (arg("self"), arg("way")), "Add a new way to the file. The way may be an ``osmium.osm.Way`` object, " "an ``osmium.osm.mutable.Way`` object or any other Python object that " "implements the same attributes.") .def("add_relation", &SimpleWriterWrap::add_relation, (arg("self"), arg("relation")), "Add a new relation to the file. The relation may be an " "``osmium.osm.Relation`` object, an ``osmium.osm.mutable.Way`` " "object or any other Python object that implements the same attributes.") .def("close", &SimpleWriterWrap::close, args("self"), "Flush the remaining buffers and close the writer. While it is not " "strictly necessary to call this function explicitly, it is still " "strongly recommended to close the writer as soon as possible, so " "that the buffer memory can be freed.") ; class_("WriteHandler", "Handler function that writes all data directly to a file." "The handler takes a file name as its mandatory parameter. The file " "must not yet exist. The file type to output is determined from the " "file extension. " "The second (optional) parameter is the buffer size. osmium caches the " "output data in an internal memory buffer before writing it on disk. This " "parameter allows changing the default buffer size of 4MB. Larger buffers " "are normally better but you should be aware that there are normally multiple " "buffers in use during the write process.", init()) .def(init()) .def("close", &WriteHandler::close, args("self"), "Flush the remaining buffers and close the writer. While it is not " "strictly necessary to call this function explicitly, it is still " "strongly recommended to close the writer as soon as possible, so " "that the buffer memory can be freed.") ; class_("MergeInputReader", "Collects data from multiple input files and sorts and optionally " "deduplicates the data before applying it to a handler.") .def("apply", &pyosmium::MergeInputReader::apply, (arg("self"), arg("handler"), arg("idx")="", arg("simplify")=true), "Apply collected data to a handler. The data will be sorted first. " "If `simplify` is true (default) then duplicates will be eliminated " "and only the newest version of each object kept. If `idx` is given " "a node location cache with the given type will be created and " "applied when creating the ways. Note that a diff file normally does " "not contain all node locations to reconstruct changed ways. If the " "full way geometries are needed, create a persistent node location " "cache during initial import of the area and reuse it when processing " "diffs. After the data " "has been applied the buffer of the MergeInputReader is empty and " "new data can be added for the next round of application.") .def("apply_to_reader", &pyosmium::MergeInputReader::apply_to_reader, (arg("self"), arg("reader"), arg("writer"), arg("with_history")=false), "Apply the collected data to data from the given `reader` and write " "the result to `writer`. This function can be used to merge the diff " "data together with other OSM data (for example when updating a " "planet file. If `with_history` is true, then the collected data will " "be applied verbatim without removing duplicates. This is important " "when using OSM history files as input.") .def("add_file", &pyosmium::MergeInputReader::add_file, (arg("self"), arg("file")), "Add data from a file to the internal cache. The file type will be " "determined from the file extension.") .def("add_buffer", &pyosmium::MergeInputReader::add_buffer, (arg("self"), arg("buffer"), arg("format")), "Add data from a byte buffer. The format of the input data must " "be given in the `format` argument as a string. The data will be " "copied into internal buffers, so that the input buffer can be " "safely discarded after the function has been called.") ; } pyosmium-2.13.0/lib/replication.cc000066400000000000000000000021041315206111100170310ustar00rootroot00000000000000#include #include #include #include #include namespace pyosmium { struct LastChangeHandler : public osmium::handler::Handler { osmium::Timestamp last_change; void osm_object(const osmium::OSMObject& obj) { set(obj.timestamp()); } private: void set(const osmium::Timestamp& ts) { if (ts > last_change) last_change = ts; } }; osmium::Timestamp compute_latest_change(const char* filename) { osmium::io::Reader reader(filename, osmium::osm_entity_bits::node | osmium::osm_entity_bits::way | osmium::osm_entity_bits::relation); LastChangeHandler handler; osmium::apply(reader, handler); reader.close(); return handler.last_change; } } BOOST_PYTHON_MODULE(_replication) { using namespace boost::python; def("newest_change_from_file", &pyosmium::compute_latest_change, "Find the date of the newest change in a file"); } pyosmium-2.13.0/lib/std_pair.hpp000066400000000000000000000021671315206111100165400ustar00rootroot00000000000000#ifndef PYOSMIUM_STD_PAIR_HPP #define PYOSMIUM_STD_PAIR_HPP // Borrowed from Boost Python examples. // Copyright Ralf W. Grosse-Kunstleve 2002-2004. Distributed under the Boost // Software License, Version 1.0. #include #include #include #include namespace { // Avoid cluttering the global namespace. // Converts a std::pair instance to a Python tuple. template struct std_pair_to_tuple { static PyObject* convert(std::pair const& p) { return boost::python::incref( boost::python::make_tuple(p.first, p.second).ptr()); } static PyTypeObject const *get_pytype () {return &PyTuple_Type; } }; // Helper for convenience. template struct std_pair_to_python_converter { std_pair_to_python_converter() { boost::python::to_python_converter< std::pair, std_pair_to_tuple, true //std_pair_to_tuple has get_pytype >(); } }; } // namespace anonymous #endif pyosmium-2.13.0/lib/win_boost_fix.hpp000066400000000000000000000010141315206111100175720ustar00rootroot00000000000000#ifndef PYOSMIUM_WIN_BOOST_FIX_HPP #define PYOSMIUM_WIN_BOOST_FIX_HPP // workarodund for Visual Studio 2015 Update 3 // https://connect.microsoft.com/VisualStudio/Feedback/Details/2852624 #if (_MSC_VER > 1800 && _MSC_FULL_VER > 190023918) namespace boost { template <> const volatile osmium::index::map::Map* get_pointer(const volatile osmium::index::map::Map *c) { return c; } } #endif #endif pyosmium-2.13.0/lib/write_handler.hpp000066400000000000000000000031531315206111100175560ustar00rootroot00000000000000#ifndef PYOSMIUM_WRITE_HANDLER_HPP #define PYOSMIUM_WRITE_HANDLER_HPP #include #include #include #include #include "generic_handler.hpp" class WriteHandler : public BaseHandler, public boost::python::wrapper { enum { BUFFER_WRAP = 4096 }; public: WriteHandler(const char* filename, size_t bufsz=4096*1024) : writer(filename), buffer(bufsz < 2*BUFFER_WRAP ? 2*BUFFER_WRAP : bufsz, osmium::memory::Buffer::auto_grow::yes) {} virtual ~WriteHandler() { close(); } void node(const osmium::Node& o) { buffer.add_item(o); flush_buffer(); } void way(const osmium::Way& o) { buffer.add_item(o); flush_buffer(); } void relation(const osmium::Relation& o) { buffer.add_item(o); flush_buffer(); } void changeset(const osmium::Changeset&) {} void area(const osmium::Area&) {} void close() { if (buffer) { writer(std::move(buffer)); writer.close(); buffer = osmium::memory::Buffer(); } } private: void flush_buffer() { buffer.commit(); if (buffer.committed() > buffer.capacity() - BUFFER_WRAP) { osmium::memory::Buffer new_buffer(buffer.capacity(), osmium::memory::Buffer::auto_grow::yes); using std::swap; swap(buffer, new_buffer); writer(std::move(new_buffer)); } } osmium::io::Writer writer; osmium::memory::Buffer buffer; }; #endif // PYOSMIUM_WRITE_HANDLER_HPP pyosmium-2.13.0/setup.py000066400000000000000000000156711315206111100151720ustar00rootroot00000000000000from setuptools import setup, Extension from setuptools.command.sdist import sdist as orig_sdist from distutils.command import build_ext as setuptools_build_ext from subprocess import call from sys import version_info as pyversion, platform as osplatform from ctypes.util import find_library import os includes = [] libs = [] libdirs = [] class My_sdist(orig_sdist): def make_release_tree(self, base_dir, files): orig_sdist.make_release_tree(self, base_dir, files) # checkout libosmium in the required version tarball = 'https://github.com/osmcode/libosmium/archive/v%s.tar.gz' % libosmium_version print("Downloading and adding libosmium sources from", tarball) call('cd %s && wget -O - -q %s | tar xz' % (base_dir, tarball), shell=True) def get_versions(): """ Read the version file. The file cannot be directly imported because it is not installed yet. """ version_py = os.path.join(os.path.split(__file__)[0], "src/osmium/version.py") v = {} with open(version_py) as version_file: # Execute the code in version.py. exec(compile(version_file.read(), version_py, 'exec'), v) return v['pyosmium_release'], v['libosmium_version'] pyosmium_release, libosmium_version = get_versions() ## boost dependencies boost_prefix = os.environ.get('BOOST_PREFIX', 'c:/libs' if osplatform == "win32" else '/usr') includes.append(os.path.join(boost_prefix, 'include')) if 'BOOST_VERSION' in os.environ: includes.append(os.path.join(boost_prefix, 'include', "boost-%s" %os.environ['BOOST_VERSION'])) if 'BOOST_VERSION' in os.environ: libdirs.append(os.path.join(boost_prefix, 'lib')) elif osplatform in ["linux", "linux2"]: libdirs.append('/usr/lib/x86_64-linux-gnu/') else: libdirs.append(os.path.join(boost_prefix, 'lib')) # try to find the boost library matching the python version suffixes = [ # Debian naming convention for version installed in parallel "-py%d%d" % (pyversion.major, pyversion.minor), # Gentoo naming convention for version installed in parallel "-%d.%d" % (pyversion.major, pyversion.minor), # standard suffix for Python3 "%d" % (pyversion.major), # standard naming "", # former naming schema? "-mt" ] for suf in suffixes: lib = find_library("boost_python%s" % suf) if lib is not None: libs.append("boost_python%s" % suf) break else: # Visual C++ supports auto-linking, no library needed if osplatform != "win32": raise Exception("Cannot find boost_python library") if osplatform != "win32": orig_compiler = setuptools_build_ext.customize_compiler def cpp_compiler(compiler): retval = orig_compiler(compiler) # force C++ compiler # Note that we only exchange the compiler as we want to keep the # original Python cflags. if len(compiler.compiler_cxx) > 0: compiler.compiler_so[0] = compiler.compiler_cxx[0] # remove warning that does not make sense for C++ try: compiler.compiler_so.remove('-Wstrict-prototypes') except (ValueError, AttributeError): pass return retval setuptools_build_ext.customize_compiler = cpp_compiler ### osmium dependencies osmium_prefixes = [ 'libosmium-' + libosmium_version, '../libosmium' ] if 'LIBOSMIUM_PREFIX' in os.environ: lo_version_h = os.path.join(os.environ['LIBOSMIUM_PREFIX'], 'include/osmium/version.hpp') if not os.path.isfile(lo_version_h): raise RuntimeError("LIBOSMIUM_PREFIX is set but no libosmium was found in '%s'" % os.environ['LIBOSMIUM_PREFIX']) includes.insert(0, os.path.join(os.environ['LIBOSMIUM_PREFIX'], 'include')) else: # default search paths for libosmium for prefix in [ 'libosmium-' + libosmium_version, '../libosmium' ]: if os.path.isfile(os.path.join(prefix, 'include/osmium/version.hpp')): print("libosmium found in '%s'" % prefix) includes.insert(0, os.path.join(prefix, 'include')) break else: print("Using global libosmium.") if osplatform == "win32" : osmium_libs = ('expat', 'zlib', 'bzip2', 'ws2_32') extra_compile_args = [ '-DWIN32_LEAN_AND_MEAN', '-D_CRT_SECURE_NO_WARNINGS', '-DNOMINMAX', '/wd4996', '/EHsc' ] else: osmium_libs = ('expat', 'pthread', 'z', 'bz2') extra_compile_args = [ '-std=c++11', '-D_LARGEFILE_SOURCE', '-D_FILE_OFFSET_BITS=64', '-D__STDC_FORMAT_MACROS' ] libs.extend(osmium_libs) extensions = [] extensions.append(Extension('osmium._osmium', sources = ['lib/osmium.cc'], include_dirs = includes, libraries = libs, library_dirs = libdirs, language = 'c++', extra_compile_args = extra_compile_args )) packages = ['osmium'] for ext in ('io', 'index', 'geom'): extensions.append(Extension('osmium.%s' % ext, sources = ['lib/%s.cc' % ext], include_dirs = includes, libraries = libs, library_dirs = libdirs, language = 'c++', extra_compile_args = extra_compile_args )) for ext in ('osm', 'replication'): extensions.append(Extension('osmium.%s._%s' % (ext, ext), sources = ['lib/%s.cc' % ext], include_dirs = includes, libraries = libs, library_dirs = libdirs, language = 'c++', extra_compile_args = extra_compile_args )) packages.append('osmium.%s' % ext) classifiers = [ "Development Status :: 4 - Beta", "Intended Audience :: Developers", "License :: OSI Approved :: BSD License", "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3.3", "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: C++", ] descfile=open('README.rst') long_description = descfile.read() descfile.close() setup (name = 'osmium', version = pyosmium_release, description = 'Python bindings for libosmium, the data processing library for OSM data', long_description=long_description, author='Sarah Hoffmann', author_email='lonvia@denofr.de', maintainer='Sarah Hoffmann', maintainer_email='lonvia@denofr.de', download_url='https://github.com/osmcode/pyosmium', url='http://osmcode.org/pyosmium', keywords=["OSM", "OpenStreetMap", "Osmium"], license='BSD', scripts=['tools/pyosmium-get-changes', 'tools/pyosmium-up-to-date'], classifiers=classifiers, packages = packages, package_dir = {'' : 'src'}, package_data = { 'osmium' : [ '*.dll' ] }, cmdclass={'sdist' : My_sdist}, ext_modules = extensions) pyosmium-2.13.0/src/000077500000000000000000000000001315206111100142355ustar00rootroot00000000000000pyosmium-2.13.0/src/osmium/000077500000000000000000000000001315206111100155465ustar00rootroot00000000000000pyosmium-2.13.0/src/osmium/__init__.py000066400000000000000000000001471315206111100176610ustar00rootroot00000000000000from osmium._osmium import * import osmium.io import osmium.osm import osmium.index import osmium.geom pyosmium-2.13.0/src/osmium/osm/000077500000000000000000000000001315206111100163445ustar00rootroot00000000000000pyosmium-2.13.0/src/osmium/osm/__init__.py000066400000000000000000000114521315206111100204600ustar00rootroot00000000000000from ._osm import * import osmium.osm.mutable def create_mutable_node(node, **args): """ Create a mutable node replacing the properties given in the named parameters. Note that this function only creates a shallow copy which is still bound to the scope of the original object. """ return osmium.osm.mutable.Node(base=node, **args) def create_mutable_way(way, **args): """ Create a mutable way replacing the properties given in the named parameters. Note that this function only creates a shallow copy which is still bound to the scope of the original object. """ return osmium.osm.mutable.Way(base=way, **args) def create_mutable_relation(rel, **args): """ Create a mutable relation replacing the properties given in the named parameters. Note that this function only creates a shallow copy which is still bound to the scope of the original object. """ return osmium.osm.mutable.Relation(base=rel, **args) Node.replace = create_mutable_node Way.replace = create_mutable_way Relation.replace = create_mutable_relation Location.__repr__ = lambda l : 'osmium.osm.Location(x=%r, y=%r)' \ % (l.x, l.y) \ if l.valid() else 'osmium.osm.Location()' Location.__str__ = lambda l : '%f/%f' % (l.lon_without_check(), l.lat_without_check()) \ if l.valid() else 'invalid' Box.__repr__ = lambda b : 'osmium.osm.Box(bottom_left=%r, top_right=%r)' \ % (b.bottom_left, b.top_right) Box.__str__ = lambda b : '(%s %s)' % (b.bottom_left, b.top_right) Tag.__repr__ = lambda t : 'osmium.osm.Tag(k=%s, v=%s)' % (t.k, t.v) Tag.__str__ = lambda t : '%s=%s' % (t.k, t.v) TagList.__repr__ = lambda t : "osmium.osm.TagList({%s})" \ % ",".join(["%r=%r" % (i.k, i.v) for i in t]) TagList.__str__ = lambda t : "{%s}" % ",".join([str(i) for i in t]) NodeRef.__repr__ = lambda n : 'osmium.osm.NodeRef(ref=%r, location=%r)' % (n.ref, n.location) NodeRef.__str__ = lambda n : '%s@%s' % (n.ref, n.location) if n.location.valid() \ else str(n.ref) NodeRefList.__repr__ = lambda t : "%s([%s])" % (t.__class__.__name__, ",".join([repr(i) for i in t])) NodeRefList.__str__ = lambda t : "[%s]" % ",".join([str(i) for i in t]) RelationMember.__repr__ = lambda r : 'osmium.osm.RelationMember(ref=%r, type=%r, role=%r)' \ % (r.ref, r.type, r.role) RelationMember.__str__ = lambda r : '%s%d@%s' % (r.type, r.ref, r.role) \ if r.role else '%s%d' % (r.type, r.ref) RelationMemberList.__repr__ = lambda t : "osmium.osm.RelationMemberList([%s])" \ % ",".join([repr(i) for i in t]) RelationMemberList.__str__ = lambda t : "[%s]" % ",".join([str(i) for i in t]) OSMObject.__repr__ = lambda o : '%s(id=%r, deleted=%r, visible=%r, version=%r, changeset=%r, uid=%r, timestamp=%r, user=%r, tags=%r)'% (o.__class__.__name__, o.id, o.deleted, o.visible, o.version, o.changeset, o.uid, o.timestamp, o.user, o.tags) def _str_ellipse(s, length=50): s = str(s) return s if len(s) <= length else (s[:length - 4] + '...' + s[-1]) Node.__repr__ = lambda o : '%s(id=%r, deleted=%r, visible=%r, version=%r, changeset=%r, uid=%r, timestamp=%r, user=%r, tags=%r, location=%r)'% (o.__class__.__name__, o.id, o.deleted, o.visible, o.version, o.changeset, o.uid, o.timestamp, o.user, o.tags, o.location) Node.__str__ = lambda n : 'n%d: location=%s tags=%s' \ % (n.id, n.location, _str_ellipse(n.tags)) Way.__repr__ = lambda o : '%s(id=%r, deleted=%r, visible=%r, version=%r, changeset=%r, uid=%r, timestamp=%r, user=%r, tags=%r, nodes=%r)'% (o.__class__.__name__, o.id, o.deleted, o.visible, o.version, o.changeset, o.uid, o.timestamp, o.user, o.tags, o.nodes) Way.__str__ = lambda o : 'w%d: nodes=%s tags=%s' \ % (o.id, _str_ellipse(o.nodes), _str_ellipse(o.tags)) Relation.__repr__ = lambda o : '%s(id=%r, deleted=%r, visible=%r, version=%r, changeset=%r, uid=%r, timestamp=%r, user=%r, tags=%r, members=%r)'% (o.__class__.__name__, o.id, o.deleted, o.visible, o.version, o.changeset, o.uid, o.timestamp, o.user, o.tags, o.members) Relation.__str__ = lambda o : 'r%d: members=%s, tags=%s' \ % (o.id, _str_ellipse(o.members), _str_ellipse(o.tags)) Changeset.__repr__ = lambda o : '%s(id=%r, uid=%r, created_at=%r, closed_at=%r, open=%r, num_changes=%r, bounds=%r, user=%r, tags=%s)' %(o.__class__.__name__, o.id, o.uid, o.created_at, o.closed_at, o.open, o.num_changes, o.bounds, o.user, o.tags) Changeset.__str__ = lambda o : 'c%d: closed_at=%s, bounds=%s, tags=%s' \ % (o.id, o.closed_at, o.bounds, _str_ellipse(o.tags)) pyosmium-2.13.0/src/osmium/osm/mutable.py000066400000000000000000000062201315206111100203470ustar00rootroot00000000000000class OSMObject(object): """Mutable version of ``osmium.osm.OSMObject``. It exposes the following attributes ``id``, ``version``, ``visible``, ``changeset``, ``timestamp``, ``uid`` and ``tags``. Timestamps may be strings or datetime objects. Tags can be an osmium.osm.TagList, a dict-like object or a list of tuples, where each tuple contains a (key value) string pair. If the ``base`` parameter is given in the constructor, then the object will be initialised first from the attributes of this base object. """ def __init__(self, base=None, id=None, version=None, visible=None, changeset=None, timestamp=None, uid=None, tags=None): if base is None: self.id = id self.version = version self.visible = visible self.changeset = changeset self.timestamp = timestamp self.uid = uid self.tags = tags else: self.id = base.id if id is None else id self.version = base.version if version is None else version self.visible = base.visible if visible is None else visible self.changeset = base.changeset if changeset is None else changeset self.timestamp = base.timestamp if timestamp is None else timestamp self.uid = base.uid if uid is None else uid self.tags = base.tags if tags is None else tags class Node(OSMObject): """The mutable version of ``osmium.osm.Node``. It inherits all attributes from osmium.osm.mutable.OSMObject and adds a `location` attribute. This may either be an `osmium.osm.Location` or a tuple of lon/lat coordinates. """ def __init__(self, base=None, location=None, **attrs): OSMObject.__init__(self, base=base, **attrs) if base is None: self.location = location else: self.location = location if location is not None else base.location class Way(OSMObject): """The mutable version of ``osmium.osm.Way``. It inherits all attributes from osmium.osm.mutable.OSMObject and adds a `nodes` attribute. This may either be and ``osmium.osm.NodeList`` or a list consisting of ``osmium.osm.NodeRef`` or simple node ids. """ def __init__(self, base=None, nodes=None, **attrs): OSMObject.__init__(self, base=base, **attrs) if base is None: self.nodes = nodes else: self.nodes = nodes if nodes is not None else base.nodes class Relation(OSMObject): """The mutable version of ``osmium.osm.Relation``. It inherits all attributes from osmium.osm.mutable.OSMObject and adds a `members` attribute. This may either be an ``osmium.osm.RelationMemberList`` or a list consisting of ``osmium.osm.RelationMember`` or tuples of (type, id, role). The member type should be a single character 'n', 'w' or 'r'. """ def __init__(self, base=None, members=None, **attrs): OSMObject.__init__(self, base=base, **attrs) if base is None: self.members = members else: self.members = members if members is not None else base.members pyosmium-2.13.0/src/osmium/replication/000077500000000000000000000000001315206111100200575ustar00rootroot00000000000000pyosmium-2.13.0/src/osmium/replication/__init__.py000066400000000000000000000000341315206111100221650ustar00rootroot00000000000000from ._replication import * pyosmium-2.13.0/src/osmium/replication/server.py000066400000000000000000000300071315206111100217370ustar00rootroot00000000000000""" Helper functions to communicate with replication servers. """ import sys try: import urllib.request as urlrequest except ImportError: import urllib2 as urlrequest try: import urllib.error as urlerror except ImportError: import urllib2 as urlerror import datetime as dt from collections import namedtuple from math import ceil from osmium import MergeInputReader from osmium import io as oio import logging log = logging.getLogger('pyosmium') log.addHandler(logging.NullHandler()) OsmosisState = namedtuple('OsmosisState', ['sequence', 'timestamp']) DownloadResult = namedtuple('DownloadResult', ['id', 'reader', 'newest']) class ReplicationServer(object): """ Represents a server that publishes replication data. Replication change files allow to keep local OSM data up-to-date without downloading the full dataset again. """ def __init__(self, url, diff_type='osc.gz'): self.baseurl = url self.diff_type = diff_type def collect_diffs(self, start_id, max_size=1024): """ Create a MergeInputReader and download diffs starting with sequence id `start_id` into it. `max_size` restricts the number of diffs that are downloaded. The download stops as soon as either a diff cannot be downloaded or the unpacked data in memory exceeds `max_size` kB. If some data was downloaded, returns a namedtuple with three fields: `id` contains the sequence id of the last downloaded diff, `reader` contains the MergeInputReader with the data and `newest` is a sequence id of the most recent diff available. Returns None if there was an error during download or no new data was available. """ left_size = max_size * 1024 current_id = start_id # must not read data newer than the published sequence id # or we might end up reading partial data newest = self.get_state_info() if newest is None or current_id > newest.sequence: return None rd = MergeInputReader() while left_size > 0 and current_id <= newest.sequence: try: diffdata = self.get_diff_block(current_id) except: diffdata = '' if len(diffdata) == 0: if start_id == current_id: return None break left_size -= rd.add_buffer(diffdata, self.diff_type) log.debug("Downloaded change %d. (%d kB available in download buffer)" % (current_id, left_size / 1024)) current_id += 1 return DownloadResult(current_id - 1, rd, newest.sequence) def apply_diffs(self, handler, start_id, max_size=1024, idx="", simplify=True): """ Download diffs starting with sequence id `start_id`, merge them together and then apply them to handler `handler`. `max_size` restricts the number of diffs that are downloaded. The download stops as soon as either a diff cannot be downloaded or the unpacked data in memory exceeds `max_size` kB. If `idx` is set, a location cache will be created and applied to the way nodes. You should be aware that diff files usually do not contain the complete set of nodes when a way is modified. That means that you cannot just create a new location cache, apply it to a diff and expect to get complete way geometries back. Instead you need to do an initial data import using a persistent location cache to obtain a full set of node locations and then reuse this location cache here when applying diffs. Diffs may contain multiple versions of the same object when it was changed multiple times during the period covered by the diff. If `simplify` is set to False then all versions are returned. If it is True (the default) then only the most recent version will be sent to the handler. The function returns the sequence id of the last diff that was downloaded or None if the download failed completely. """ diffs = self.collect_diffs(start_id, max_size) if diffs is None: return None diffs.reader.apply(handler, idx=idx, simplify=simplify) return diffs.id def apply_diffs_to_file(self, infile, outfile, start_id, max_size=1024, set_replication_header=True): """ Download diffs starting with sequence id `start_id`, merge them with the data from the OSM file named `infile` and write the result into a file with the name `outfile`. The output file must not yet exist. `max_size` restricts the number of diffs that are downloaded. The download stops as soon as either a diff cannot be downloaded or the unpacked data in memory exceeds `max_size` kB. If `set_replication_header` is true then the URL of the replication server and the sequence id and timestamp of the last diff applied will be written into the `writer`. Note that this currently works only for the PBF format. The function returns a tuple of last downloaded sequence id and newest available sequence id if new data has been written or None if no data was available or the download failed completely. """ diffs = self.collect_diffs(start_id, max_size) if diffs is None: return None reader = oio.Reader(infile) has_history = reader.header().has_multiple_object_versions h = oio.Header() h.has_multiple_object_versions = has_history if set_replication_header: h.set("osmosis_replication_base_url", self.baseurl) h.set("osmosis_replication_sequence_number", str(diffs.id)) info = self.get_state_info(diffs.id) h.set("osmosis_replication_timestamp", info.timestamp.strftime("%Y-%m-%dT%H:%M:%SZ")) writer = oio.Writer(outfile, h) log.debug("Merging changes into OSM file.") diffs.reader.apply_to_reader(reader, writer, has_history) reader.close() writer.close() return (diffs.id, diffs.newest) def timestamp_to_sequence(self, timestamp, balanced_search=False): """ Get the sequence number of the replication file that contains the given timestamp. The search algorithm is optimised for replication servers that publish updates in regular intervals. For servers with irregular change file publication dates 'balanced_search` should be set to true so that a standard binary search for the sequence will be used. The default is good for all known OSM replication services. """ # get the current timestamp from the server upper = self.get_state_info() if upper is None: return None if timestamp >= upper.timestamp or upper.sequence <= 0: return upper.sequence # find a state file that is before the required timestamp lower = None lowerid = 0 while lower is None: log.debug("Trying with Id %s" % lowerid) lower = self.get_state_info(lowerid) if lower is not None and lower.timestamp >= timestamp: if lower.sequence == 0 or lower.sequence + 1 >= upper.sequence: return lower.sequence upper = lower lower = None lowerid = 0 if lower is None: # no lower yet, so try a higher id (binary search wise) newid = int((lowerid + upper.sequence) / 2) if newid <= lowerid: # nothing suitable found, so upper is probably the best we can do return upper.sequence lowerid = newid # Now do a binary search between upper and lower. # We could be clever here and compute the most likely state file # by interpolating over the timestamps but that creates a whole ton of # special cases that need to be handled correctly. while True: if balanced_search: base_splitid = int((lower.sequence + upper.sequence) / 2) else: ts_int = (upper.timestamp - lower.timestamp).total_seconds() seq_int = upper.sequence - lower.sequence goal = (timestamp - lower.timestamp).total_seconds() base_splitid = lower.sequence + ceil(goal * seq_int / ts_int) if base_splitid >= upper.sequence: base_splitid = upper.sequence - 1 split = self.get_state_info(base_splitid) if split is None: # file missing, search the next towards lower splitid = base_splitid - 1 while split is None and splitid > lower.sequence: split = self.get_state_info(splitid) splitid -= 1 if split is None: # still nothing? search towards upper splitid = base_splitid + 1 while split is None and splitid < upper.sequence: split = self.get_state_info(splitid) splitid += 1 if split is None: # still nothing? Then lower has to do return lower.sequence # set new boundary if split.timestamp < timestamp: lower = split else: upper = split if lower.sequence + 1 >= upper.sequence: return lower.sequence def get_state_info(self, seq=None): """ Downloads and returns the state information for the given sequence. If the download is successful, a namedtuple with `sequence` and `timestamp` is returned, otherwise the function returns `None`. """ try: response = urlrequest.urlopen(self.get_state_url(seq)) except: return None ts = None seq = None line = response.readline() while line: line = line.decode('utf-8') if '#' in line: line = line[0:line.index('#')] else: line = line.strip() if line: kv = line.split('=', 2) if len(kv) != 2: return None if kv[0] == 'sequenceNumber': seq = int(kv[1]) elif kv[0] == 'timestamp': ts = dt.datetime.strptime(kv[1], "%Y-%m-%dT%H\\:%M\\:%SZ") if sys.version_info >= (3,0): ts = ts.replace(tzinfo=dt.timezone.utc) line = response.readline() return OsmosisState(sequence=seq, timestamp=ts) def get_diff_block(self, seq): """ Downloads the diff with the given sequence number and returns it as a byte sequence. Throws a `urllib.error.HTTPError` (or `urllib2.HTTPError` in python2) if the file cannot be downloaded. """ return urlrequest.urlopen(self.get_diff_url(seq)).read() def get_state_url(self, seq): """ Returns the URL of the state.txt files for a given sequence id. If seq is `None` the URL for the latest state info is returned, i.e. the state file in the root directory of the replication service. """ if seq is None: return self.baseurl + '/state.txt' return '%s/%03i/%03i/%03i.state.txt' % (self.baseurl, seq / 1000000, (seq % 1000000) / 1000, seq % 1000) def get_diff_url(self, seq): """ Returns the URL to the diff file for the given sequence id. """ return '%s/%03i/%03i/%03i.%s' % (self.baseurl, seq / 1000000, (seq % 1000000) / 1000, seq % 1000, self.diff_type) pyosmium-2.13.0/src/osmium/replication/utils.py000066400000000000000000000042561315206111100216000ustar00rootroot00000000000000""" Helper functions for change file handling. """ import logging import datetime as dt from collections import namedtuple from osmium.io import Reader as oreader from sys import version_info as python_version log = logging.getLogger('pyosmium') ReplicationHeader = namedtuple('ReplicationHeader', ['url', 'sequence', 'timestamp']) def get_replication_header(fname): """ Scans the given file for an Osmosis replication header. It returns a namedtuple with `url`, `sequence` and `timestamp`. Each or all fields may be None, if the piece of information is not avilable. If any of the fields has an invalid format, it is simply ignored. The given file must exist and be readable for osmium, otherwise a `RuntimeError` is raised. """ r = oreader(fname) h = r.header() ts = h.get("osmosis_replication_timestamp") url = h.get("osmosis_replication_base_url") if url or ts: log.debug("Replication information found in OSM file header.") if url: log.debug("Replication URL: %s" % url) # the sequence ID is only considered valid, if an URL is given seq = h.get("osmosis_replication_sequence_number") if seq: log.debug("Replication sequence: %s" % seq) try: seq = int(seq) if seq < 0: log.warning("Sequence id '%d' in OSM file header is negative. Ignored." % seq) seq = None except ValueError: log.warning("Sequence id '%s' in OSM file header is not a number.Ignored" % seq) seq = None else: seq = None else: url = None seq = None if ts: log.debug("Replication timestamp: %s" % ts) try: ts = dt.datetime.strptime(ts, "%Y-%m-%dT%H:%M:%SZ") if python_version >= (3,0): ts = ts.replace(tzinfo=dt.timezone.utc) except ValueError: log.warning("Date in OSM file header is not in ISO8601 format (e.g. 2015-12-24T08:08Z). Ignored") ts = None else: ts = None return ReplicationHeader(url, seq, ts) pyosmium-2.13.0/src/osmium/version.py000066400000000000000000000003251315206111100176050ustar00rootroot00000000000000""" Version information. """ # the major version pyosmium_major = '2.13' # current release (Pip version) pyosmium_release = '2.13.0' # libosmium version shipped with the Pip release libosmium_version = '2.13.1' pyosmium-2.13.0/test/000077500000000000000000000000001315206111100144255ustar00rootroot00000000000000pyosmium-2.13.0/test/helpers.py000066400000000000000000000132131315206111100164410ustar00rootroot00000000000000""" Provides some helper functions for test. """ import random import tempfile import os import sys from textwrap import dedent import osmium from datetime import datetime if sys.version_info[0] == 3: from datetime import timezone def mkdate(*args): return datetime(*args, tzinfo=timezone.utc) else: def mkdate(*args): return datetime(*args) def _complete_object(o): """Takes a hash with an incomplete OSM object description and returns a complete one. """ if o['type'] == 'C': ret = { 'created_at' : "2005-04-09T19:54:13Z", 'num_changes' : 2, 'closed_at' : "2005-04-09T20:54:39Z", 'open' : "false", 'min_lon' : -0.1465242, 'min_lat' : 51.5288506, 'max_lon' : -0.1464925, 'max_lat' : 51.5288620, 'user' : "Steve", 'uid' : "1", 'tags' : None } else: ret = { 'version' : '1', 'timestamp': "2012-05-01T15:06:20Z", 'changeset' : "11470653", 'uid' : "122294", 'user' : "foo", 'tags' : {} } ret.update(o) if ret['type'] == 'N': if 'lat' not in ret: ret['lat'] = random.random()*180 - 90 if 'lon' not in ret: ret['lon'] = random.random()*360 - 180 return ret def _write_osm_obj(fd, obj): if obj['type'] == 'N': fd.write(('\n'.encode('utf-8')) else: fd.write('>\n'.encode('utf-8')) for k,v in iter(obj['tags'].items()): fd.write((' \n' % (k, v)).encode('utf-8')) fd.write('\n'.encode('utf-8')) elif obj['type'] == 'W': fd.write(('\n' % obj).encode('utf-8')) for nd in obj['nodes']: fd.write(('\n' % (nd,)).encode('utf-8')) for k,v in iter(obj['tags'].items()): fd.write((' \n' % (k, v)).encode('utf-8')) fd.write('\n'.encode('utf-8')) elif obj['type'] == 'R': fd.write(('\n' % obj).encode('utf-8')) for mem in obj['members']: fd.write((' \n' % mem).encode('utf-8')) for k,v in iter(obj['tags'].items()): fd.write((' \n' % (k, v)).encode('utf-8')) fd.write('\n'.encode('utf-8')) elif obj['type'] == 'C': fd.write(('\n'.encode('utf-8')) else: fd.write('>\n'.encode('utf-8')) for k,v in iter(obj['tags'].items()): fd.write((' \n' % (k, v)).encode('utf-8')) fd.write('\n'.encode('utf-8')) def create_osm_file(data): """Creates a temporary osm XML file. The data is a list of OSM objects, each described by a hash of attributes. Most attributes are optional and will be filled with sensitive values, if missing. Mandatory are only `type` and `id`. For ways, nodes are obligatory and for relations the memberlist. """ data.sort(key=lambda x:('NWR'.find(x['type']), x['id'])) with tempfile.NamedTemporaryFile(dir=tempfile.gettempdir(), suffix='.osm', delete=False) as fd: fname = fd.name fd.write("\n".encode('utf-8')) fd.write('\n'.encode('utf-8')) fd.write('\t\n'.encode('utf-8')) for obj in data: _write_osm_obj(fd, _complete_object(obj)) fd.write('\n'.encode('utf-8')) return fname def create_opl_file(data): with tempfile.NamedTemporaryFile(dir=tempfile.gettempdir(), suffix='.opl', delete=False) as fd: fname = fd.name fd.write(dedent(data).encode('utf-8')) fd.write(b'\n') return fname def osmobj(kind, **args): ret = dict(args) ret['type'] = kind return ret def check_repr(o): return not str(o).startswith('<') and not repr(o).startswith('<') class HandlerTestBase: apply_locations = False apply_idx = 'sparse_mem_array' def test_func(self): if isinstance(self.data, (list, tuple)): fn = create_osm_file(self.data) else: fn = create_opl_file(self.data) try: self.handler = self.Handler() self.handler.apply_file(fn, self.apply_locations, self.apply_idx) finally: os.remove(fn) if hasattr(self, "check_result"): self.check_result() class CountingHandler(osmium.SimpleHandler): def __init__(self): super(CountingHandler, self).__init__() self.counts = [0, 0, 0, 0] def node(self, _): self.counts[0] += 1 def way(self, _): self.counts[1] += 1 def relation(self, _): self.counts[2] += 1 def area(self, _): self.counts[3] += 1 pyosmium-2.13.0/test/run_tests.py000066400000000000000000000010021315206111100170160ustar00rootroot00000000000000import sys import sysconfig import os # use some black magic to find the libraries in the build directory # borrowed from http://stackoverflow.com/questions/14320220/testing-python-c-libraries-get-build-path build_dir = "../../build/lib.%s-%d.%d" % ( sysconfig.get_platform(), sys.version_info[0], sys.version_info[1]) # insert after the current directory sys.path.insert(1, os.path.normpath(os.path.join(os.path.realpath(__file__), build_dir))) import nose nose.main() pyosmium-2.13.0/test/test_geom.py000066400000000000000000000032261315206111100167700ustar00rootroot00000000000000from nose.tools import * import unittest from helpers import create_osm_file, osmobj, HandlerTestBase import osmium as o wkbfab = o.geom.WKBFactory() class TestWkbCreateNode(HandlerTestBase, unittest.TestCase): data = [osmobj('N', id=1)] class Handler(o.SimpleHandler): wkbs = [] def node(self, n): self.wkbs.append(wkbfab.create_point(n)) def check_result(self): assert_equals(1, len(self.handler.wkbs)) class TestWkbCreateWay(HandlerTestBase, unittest.TestCase): data = [osmobj('N', id=1, lat=0, lon=0), osmobj('N', id=2, lat=0, lon=1), osmobj('N', id=3, lat=1, lon=0), osmobj('W', id=1, nodes = [1,2,3])] apply_locations = True class Handler(o.SimpleHandler): wkbs = [] def way(self, w): self.wkbs.append(wkbfab.create_linestring(w)) self.wkbs.append(wkbfab.create_linestring(w, direction=o.geom.direction.BACKWARD)) self.wkbs.append(wkbfab.create_linestring(w, use_nodes=o.geom.use_nodes.ALL)) def check_result(self): assert_equals(3, len(self.handler.wkbs)) class TestWkbCreatePoly(HandlerTestBase, unittest.TestCase): data = [osmobj('N', id=1, lat=0, lon=0), osmobj('N', id=2, lat=0, lon=1), osmobj('N', id=3, lat=1, lon=0), osmobj('W', id=23, nodes = [1,2,3,1], tags = { "area" : "yes" }), ] apply_locations = True class Handler(o.SimpleHandler): wkbs = [] def area(self, a): self.wkbs.append(wkbfab.create_multipolygon(a)) def check_result(self): assert_equals(1, len(self.handler.wkbs)) pyosmium-2.13.0/test/test_index.py000066400000000000000000000003601315206111100171440ustar00rootroot00000000000000from nose.tools import * import unittest import osmium as o class TestIndexMapTypes(unittest.TestCase): def runTest(self): ml = o.index.map_types() assert_true(isinstance(ml, list)) assert_greater(len(ml), 0) pyosmium-2.13.0/test/test_io.py000066400000000000000000000037401315206111100164510ustar00rootroot00000000000000from nose.tools import * import unittest import os from helpers import create_osm_file, osmobj import osmium as o class TestReaderFromFile(unittest.TestCase): def _run_file(self, fn): try: rd = o.io.Reader(fn) o.apply(rd, o.SimpleHandler()) rd.close() finally: os.remove(fn) def test_node_only(self): self._run_file(create_osm_file([osmobj('N', id=1)])) def test_way_only(self): self._run_file(create_osm_file([osmobj('W', id=1, nodes=[1,2,3])])) def test_relation_only(self): self._run_file(create_osm_file([osmobj('R', id=1, members=[('way', 1, '')])])) def test_node_with_tags(self): self._run_file(create_osm_file([osmobj('N', id=1, tags=dict(foo='bar', name='xx'))])) def test_way_with_tags(self): self._run_file(create_osm_file([osmobj('W', id=1, nodes=[1,2,3], tags=dict(foo='bar', name='xx'))])) def test_relation_with_tags(self): self._run_file(create_osm_file([osmobj('R', id=1, members=[('way', 1, '')], tags=dict(foo='bar', name='xx'))])) def test_broken_timestamp(self): fn = create_osm_file([osmobj('N', id=1, timestamp='x')]) try: rd = o.io.Reader(fn) with assert_raises(ValueError): o.apply(rd, o.SimpleHandler()) rd.close() finally: os.remove(fn) class TestFileHeader(unittest.TestCase): def test_file_header(self): fn = create_osm_file([osmobj('N', id=1)]) try: rd = o.io.Reader(fn) h = rd.header() assert_false(h.has_multiple_object_versions) assert_true(h.box().valid()) assert_equals(h.box().size(), 64800.0) rd.close() finally: os.remove(fn) if __name__ == '__main__': unittest.main() pyosmium-2.13.0/test/test_memberlist.py000066400000000000000000000021701315206111100202010ustar00rootroot00000000000000 from nose.tools import * import unittest import os import sys from datetime import datetime from helpers import create_osm_file, osmobj, HandlerTestBase import osmium as o class TestLength(HandlerTestBase, unittest.TestCase): data = """\ r2 Mn3@ r4 r45 Mw1@fo,r45@4,r45@5 """ class Handler(o.SimpleHandler): expected_length = { 2 : 1, 4 : 0, 45 : 3 } def relation(self, r): assert_equals(self.expected_length[r.id], len(r.members)) class TestMembers(HandlerTestBase, unittest.TestCase): data = """r34 Mn23@,n12@foo,w5@.,r34359737784@()""" class Handler(o.SimpleHandler): def relation(self, r): m = list(r.members) eq_(4, len(m)) eq_(23, m[0].ref) eq_('n', m[0].type) eq_('', m[0].role) eq_(12, m[1].ref) eq_('n', m[1].type) eq_('foo', m[1].role) eq_(5, m[2].ref) eq_('w', m[2].type) eq_('.', m[2].role) eq_(34359737784, m[3].ref) eq_('r', m[3].type) eq_('()', m[3].role) pyosmium-2.13.0/test/test_nodelist.py000066400000000000000000000034351315206111100176640ustar00rootroot00000000000000from nose.tools import * import unittest import os import sys from datetime import datetime from helpers import create_osm_file, osmobj, HandlerTestBase import osmium as o class TestLength(HandlerTestBase, unittest.TestCase): data = """\ w593 w4 Nn1,n2,n-34 w8 Nn12,n12,n12,n0 """ class Handler(o.SimpleHandler): expected_length = { 593 : 0, 4 : 3, 8 : 4 } def way(self, w): assert_equals(self.expected_length[w.id], len(w.nodes)) class TestNodeIds(HandlerTestBase, unittest.TestCase): data = """w4 Nn1,n1,n34359737784,n-34,n0""" class Handler(o.SimpleHandler): def way(self, w): eq_(1, w.nodes[0].ref) eq_(1, w.nodes[1].ref) eq_(34359737784, w.nodes[2].ref) eq_(-34, w.nodes[3].ref) eq_(0, w.nodes[4].ref) eq_(0, w.nodes[-1].ref) class TestMissingRef(HandlerTestBase, unittest.TestCase): data = """\ n1 x0.5 y10.0 w4 Nn1 """ class Handler(o.SimpleHandler): def way(self, w): eq_(1, w.nodes[0].ref) assert_false(w.nodes[0].location.valid()) with assert_raises(o.InvalidLocationError): w.nodes[0].location.lat with assert_raises(o.InvalidLocationError): w.nodes[0].location.lon class TestValidRefs(HandlerTestBase, unittest.TestCase): data = """\ n1 x0.5 y10.0 w4 Nn1 """ apply_locations = True class Handler(o.SimpleHandler): def way(self, w): eq_(1, w.nodes[0].ref) assert_true(w.nodes[0].location.valid()) assert_almost_equal(w.nodes[0].location.lat, 10.0) assert_almost_equal(w.nodes[0].location.lon, 0.5) pyosmium-2.13.0/test/test_osm.py000066400000000000000000000150761315206111100166450ustar00rootroot00000000000000from nose.tools import * import unittest import os import sys from helpers import create_osm_file, osmobj, check_repr, HandlerTestBase, mkdate import osmium as o class TestLocation(unittest.TestCase): def test_invalid_location(self): loc = o.osm.Location() assert_false(loc.valid()) assert_true(check_repr(loc)) def test_valid_location(self): loc = o.osm.Location(1,10) assert_equals(loc.lon, 1, 0.0001) assert_equals(loc.lat, 10, 0.00001) assert_equals(loc.x, 10000000) assert_equals(loc.y, 100000000) assert_true(check_repr(loc)) class TestNodeAttributes(HandlerTestBase, unittest.TestCase): data = [osmobj('N', id=1, version=5, changeset=58674, uid=42, timestamp='2014-01-31T06:23:35Z', user='anonymous')] class Handler(o.SimpleHandler): def node(self, n): assert_equals(n.id, 1) assert_equals(n.deleted, False) assert_equals(n.visible, True) assert_equals(n.version, 5) assert_equals(n.changeset, 58674) assert_equals(n.uid, 42) assert_equals(n.user_is_anonymous(), False) assert_equals(n.timestamp, mkdate(2014, 1, 31, 6, 23, 35)) assert_equals(n.user, 'anonymous') assert_equals(n.positive_id(), 1) assert_true(check_repr(n)) class TestNodePositiveId(HandlerTestBase, unittest.TestCase): data = [osmobj('N', id=-34, version=5, changeset=58674, uid=42, timestamp='2014-01-31T06:23:35Z', user='anonymous')] class Handler(o.SimpleHandler): def node(self, n): assert_equals(n.positive_id(), 34) class TestNodeLargeId(HandlerTestBase, unittest.TestCase): data = [osmobj('N', id=17179869418, version=5, changeset=58674, uid=42, timestamp='2014-01-31T06:23:35Z', user='anonymous')] class Handler(o.SimpleHandler): def node(self, n): assert_equals(n.id, 17179869418) class TestWayAttributes(HandlerTestBase, unittest.TestCase): apply_locations = True data = [osmobj('N', id=1, lat=0, lon=0), osmobj('N', id=3, lat=1, lon=1), osmobj('W', id=1, version=5, changeset=58674, uid=42, timestamp='2014-01-31T06:23:35Z', user='anonymous', nodes = [1,2,3])] class Handler(o.SimpleHandler): def way(self, n): assert_equals(n.id, 1) assert_equals(n.deleted, False) assert_equals(n.visible, True) assert_equals(n.version, 5) assert_equals(n.changeset, 58674) assert_equals(n.uid, 42) assert_equals(n.user_is_anonymous(), False) assert_equals(n.timestamp, mkdate(2014, 1, 31, 6, 23, 35)) assert_equals(n.user, 'anonymous') assert_equals(n.positive_id(), 1) assert_false(n.is_closed()) assert_false(n.ends_have_same_id()) assert_false(n.ends_have_same_location()) assert_true(check_repr(n)) assert_true(check_repr(n.nodes)) class TestRelationAttributes(HandlerTestBase, unittest.TestCase): data = [osmobj('R', id=1, version=5, changeset=58674, uid=42, timestamp='2014-01-31T06:23:35Z', user='anonymous', members=[('way',1,'')])] class Handler(o.SimpleHandler): def relation(self, n): assert_equals(n.id, 1) assert_equals(n.deleted, False) assert_equals(n.visible, True) assert_equals(n.version, 5) assert_equals(n.changeset, 58674) assert_equals(n.uid, 42) assert_equals(n.user_is_anonymous(), False) assert_equals(n.timestamp, mkdate(2014, 1, 31, 6, 23, 35)) assert_equals(n.user, 'anonymous') assert_equals(n.positive_id(), 1) assert_true(check_repr(n)) assert_true(check_repr(n.members)) class TestAreaFromWayAttributes(HandlerTestBase, unittest.TestCase): data = [osmobj('N', id=1, lat=0, lon=0), osmobj('N', id=2, lat=0, lon=1), osmobj('N', id=3, lat=1, lon=0), osmobj('W', id=23, version=5, changeset=58674, uid=42, timestamp='2014-01-31T06:23:35Z', user='anonymous', nodes = [1,2,3,1], tags = { "area" : "yes" }), ] class Handler(o.SimpleHandler): def area(self, n): assert_equals(n.id, 46) assert_equals(n.deleted, False) assert_equals(n.visible, True) assert_equals(n.version, 5) assert_equals(n.changeset, 58674) assert_equals(n.uid, 42) assert_equals(n.user_is_anonymous(), False) assert_equals(n.timestamp, mkdate(2014, 1, 31, 6, 23, 35)) assert_equals(n.user, 'anonymous') assert_equals(n.positive_id(), 46) assert_equals(n.orig_id(), 23) assert_equals(n.from_way(), True) assert_equals(n.is_multipolygon(), False) assert_equals(n.num_rings(), (1, 0)) assert_equals(len(list(n.outer_rings())), 1) oring = list(n.outer_rings())[0] assert_equals(len(list(oring)), 4) assert_equals(set((1,2,3)), set([x.ref for x in oring])) assert_true(oring.is_closed()) assert_true(oring.ends_have_same_id()) assert_true(oring.ends_have_same_location()) assert_equals(len(list(n.inner_rings(oring))), 0) class TestChangesetAttributes(HandlerTestBase, unittest.TestCase): data = [osmobj('C', id=34, created_at="2005-04-09T19:54:13Z", num_changes=2, closed_at="2005-04-09T20:54:39Z", open="false", min_lon=-0.1465242, min_lat=51.5288506, max_lon=-0.1464925, max_lat=51.5288620, user="Steve", uid="1") ] class Handler(o.SimpleHandler): def changeset(self,c): assert_equals(34, c.id) assert_equals(1, c.uid) assert_false(c.user_is_anonymous()) assert_equals("Steve", c.user) assert_equals(mkdate(2005, 4, 9, 19, 54, 13), c.created_at) assert_equals(mkdate(2005, 4, 9, 20, 54, 39), c.closed_at) assert_false(c.open) assert_equals(2, c.num_changes) assert_equals(0, len(c.tags)) assert_equals(-1464925, c.bounds.top_right.x) assert_equals(515288620, c.bounds.top_right.y) assert_equals(-1465242, c.bounds.bottom_left.x) assert_equals(515288506, c.bounds.bottom_left.y) assert_true(check_repr(c)) pyosmium-2.13.0/test/test_replication.py000066400000000000000000000151071315206111100203530ustar00rootroot00000000000000from nose.tools import * import unittest from io import BytesIO from textwrap import dedent from helpers import mkdate, CountingHandler try: from urllib.error import URLError except ImportError: from urllib2 import URLError try: from unittest.mock import MagicMock, patch except ImportError: from mock import MagicMock, patch import osmium as o import osmium.replication.server as rserv class UrllibMock(MagicMock): def set_result(self, s): self.return_value = BytesIO(dedent(s).encode()) def set_script(self, files): self.side_effect = [BytesIO(dedent(s).encode()) for s in files] def test_get_state_url(): svr = rserv.ReplicationServer("http://text.org") data = [ (None, 'http://text.org/state.txt'), (1, 'http://text.org/000/000/001.state.txt'), (999, 'http://text.org/000/000/999.state.txt'), (1000, 'http://text.org/000/001/000.state.txt'), (573923, 'http://text.org/000/573/923.state.txt'), (3290012, 'http://text.org/003/290/012.state.txt'), ] for i, o in data: assert_equals(o, svr.get_state_url(i)) def test_get_diff_url(): svr = rserv.ReplicationServer("https://who.is/me/") data = [ (1, 'https://who.is/me//000/000/001.osc.gz'), (500, 'https://who.is/me//000/000/500.osc.gz'), (83750, 'https://who.is/me//000/083/750.osc.gz'), (999999999, 'https://who.is/me//999/999/999.osc.gz'), ] for i, o in data: assert_equals(o, svr.get_diff_url(i)) @patch('osmium.replication.server.urlrequest.urlopen', new_callable=UrllibMock) def test_get_state_valid(mock): mock.set_result("""\ #Sat Aug 26 11:04:04 UTC 2017 txnMaxQueried=1219304113 sequenceNumber=2594669 timestamp=2017-08-26T11\:04\:02Z txnReadyList= txnMax=1219304113 txnActiveList=1219303583,1219304054,1219304104""") res = rserv.ReplicationServer("http://test.io").get_state_info() assert_is_not_none(res) assert_equals(res.timestamp, mkdate(2017, 8, 26, 11, 4, 2)) assert_equals(res.sequence, 2594669) assert_equal(mock.call_count, 1) @patch('osmium.replication.server.urlrequest.urlopen') def test_get_state_server_timeout(mock): mock.side_effect = URLError(reason='Mock') svr = rserv.ReplicationServer("http://test.io") assert_is_none(svr.get_state_info()) @patch('osmium.replication.server.urlrequest.urlopen', new_callable=UrllibMock) def test_apply_diffs_count(mock): mock.set_script(("""\ sequenceNumber=100 timestamp=2017-08-26T11\:04\:02Z """, """ n1 w1 r1 """)) svr = rserv.ReplicationServer("http://test.io", "opl") h = CountingHandler() assert_equals(100, svr.apply_diffs(h, 100, 10000)) assert_equals(h.counts, [1, 1, 1, 0]) @patch('osmium.replication.server.urlrequest.urlopen', new_callable=UrllibMock) def test_apply_diffs_without_simplify(mock): mock.set_script(("""\ sequenceNumber=100 timestamp=2017-08-26T11\:04\:02Z """, """ n1 v23 n1 v24 w1 r1 """)) svr = rserv.ReplicationServer("http://test.io", "opl") h = CountingHandler() assert_equals(100, svr.apply_diffs(h, 100, 10000, simplify=False)) assert_equals([2, 1, 1, 0], h.counts) @patch('osmium.replication.server.urlrequest.urlopen', new_callable=UrllibMock) def test_apply_diffs_with_simplify(mock): mock.set_script(("""\ sequenceNumber=100 timestamp=2017-08-26T11\:04\:02Z """, """ n1 v23 n1 v24 w1 r1 """)) svr = rserv.ReplicationServer("http://test.io", "opl") h = CountingHandler() assert_equals(100, svr.apply_diffs(h, 100, 10000, simplify=True)) assert_equals([1, 1, 1, 0], h.counts) @patch('osmium.replication.server.urlrequest.urlopen', new_callable=UrllibMock) def test_apply_with_location(mock): mock.set_script(("""\ sequenceNumber=100 timestamp=2017-08-26T11\:04\:02Z """, """ n1 x10.0 y23.0 w1 Nn1,n2 """)) svr = rserv.ReplicationServer("http://test.io", "opl") class Handler(CountingHandler): def way(self, w): self.counts[1] += 1 assert_equals(2, len(w.nodes)) assert_equals(1, w.nodes[0].ref) assert_equals(10, w.nodes[0].location.lon) assert_equals(23, w.nodes[0].location.lat) assert_equals(2, w.nodes[1].ref) assert_false(w.nodes[1].location.valid()) h = Handler() assert_equals(100, svr.apply_diffs(h, 100, 10000, idx="flex_mem")) assert_equals(h.counts, [1, 1, 0, 0]) @patch('osmium.replication.server.urlrequest.urlopen', new_callable=UrllibMock) def test_apply_reader_without_simplify(mock): mock.set_script(("""\ sequenceNumber=100 timestamp=2017-08-26T11\:04\:02Z """, """ n1 v23 n1 v24 w1 r1 """)) svr = rserv.ReplicationServer("http://test.io", "opl") h = CountingHandler() diffs = svr.collect_diffs(100, 100000) assert_is_not_none(diffs) diffs.reader.apply(h, simplify=False) assert_equals([2, 1, 1, 0], h.counts) @patch('osmium.replication.server.urlrequest.urlopen', new_callable=UrllibMock) def test_apply_reader_with_simplify(mock): mock.set_script(("""\ sequenceNumber=100 timestamp=2017-08-26T11\:04\:02Z """, """ n1 v23 n1 v24 w1 r1 """)) svr = rserv.ReplicationServer("http://test.io", "opl") h = CountingHandler() diffs = svr.collect_diffs(100, 100000) assert_is_not_none(diffs) diffs.reader.apply(h, simplify=True) assert_equals([1, 1, 1, 0], h.counts) @patch('osmium.replication.server.urlrequest.urlopen', new_callable=UrllibMock) def test_apply_reader_with_location(mock): mock.set_script(("""\ sequenceNumber=100 timestamp=2017-08-26T11\:04\:02Z """, """ n1 x10.0 y23.0 w1 Nn1,n2 """)) svr = rserv.ReplicationServer("http://test.io", "opl") class Handler(CountingHandler): def way(self, w): self.counts[1] += 1 assert_equals(2, len(w.nodes)) assert_equals(1, w.nodes[0].ref) assert_equals(10, w.nodes[0].location.lon) assert_equals(23, w.nodes[0].location.lat) assert_equals(2, w.nodes[1].ref) assert_false(w.nodes[1].location.valid()) h = Handler() diffs = svr.collect_diffs(100, 100000) assert_is_not_none(diffs) diffs.reader.apply(h, idx="flex_mem") assert_equals(h.counts, [1, 1, 0, 0]) pyosmium-2.13.0/test/test_taglist.py000066400000000000000000000055411315206111100175120ustar00rootroot00000000000000from nose.tools import * import unittest import os import sys from datetime import datetime from helpers import create_osm_file, osmobj, check_repr, HandlerTestBase import osmium as o class TestTagEmptyTagListLength(HandlerTestBase, unittest.TestCase): data = "n234 x1 y2" class Handler(o.SimpleHandler): def node(self, n): assert_equals(0, len(n.tags)) assert_false(n.tags) class TestTagEmptyTagListContains(HandlerTestBase, unittest.TestCase): data = "n234 x1 y2" class Handler(o.SimpleHandler): def node(self, n): assert_not_in("a", n.tags) class TestTagEmptyTagListGet(HandlerTestBase, unittest.TestCase): data = "n234 x1 y2" class Handler(o.SimpleHandler): def node(self, n): assert_equals(None, n.tags.get("foo")) assert_equals(None, n.tags.get("foo", None)) assert_equals("fs", n.tags.get("foo", "fs")) class TestTagEmptyTagListIndexOp(HandlerTestBase, unittest.TestCase): data = "n234 x1 y2" class Handler(o.SimpleHandler): def node(self, n): with assert_raises(KeyError): n.tags["foo"] with assert_raises(KeyError): n.tags[None] class TestTagListLen(HandlerTestBase, unittest.TestCase): data = """\ n1 x0 y0 Ta=a n2 Tkey=value n3 Tfoo=1,bar=2,foobar=33 """ class Handler(o.SimpleHandler): expected_len = { 1 : 1, 2 : 1, 3 : 3} def node(self, n): assert_true(n.tags) assert_equals(self.expected_len[n.id], len(n.tags)) class TestTagContains(HandlerTestBase, unittest.TestCase): data = "n234 Tabba=x,2=vvv,xx=abba" class Handler(o.SimpleHandler): def node(self, n): assert_in("abba", n.tags) assert_in("2", n.tags) assert_in("xx", n.tags) assert_not_in("x", n.tags) assert_not_in(None, n.tags) assert_not_in("", n.tags) assert_true(check_repr(n.tags)) class TestTagIndexOp(HandlerTestBase, unittest.TestCase): data = "n234 Tabba=x,2=vvv,xx=abba" class Handler(o.SimpleHandler): def node(self, n): eq_("x", n.tags["abba"]) eq_("vvv", n.tags["2"]) eq_("abba", n.tags["xx"]) for k in ("x", "addad", "..", None): with assert_raises(KeyError): n.tags[k] class TestTagGet(HandlerTestBase, unittest.TestCase): data = "n234 Tabba=x,2=vvv,xx=abba" class Handler(o.SimpleHandler): def node(self, n): eq_("x", n.tags.get("abba")) eq_("vvv", n.tags.get("2", None)) eq_("abba", n.tags.get("xx", "ff")) eq_("43 fg", n.tags.get("_", "43 fg")) assert_is_none(n.tags.get("gerger4")) assert_is_none(n.tags.get("ffleo", None)) pyosmium-2.13.0/test/test_writer.py000066400000000000000000000107501315206111100173550ustar00rootroot00000000000000from nose.tools import * import unittest import tempfile import os from contextlib import contextmanager from datetime import datetime from collections import OrderedDict import logging import sys import osmium as o log = logging.getLogger(__name__) if sys.version_info[0] == 3: from datetime import timezone def mkdate(*args): return datetime(*args, tzinfo=timezone.utc) else: def mkdate(*args): return datetime(*args) @contextmanager def WriteExpect(expected): fname = tempfile.mktemp(dir=tempfile.gettempdir(), suffix='.opl') writer = o.SimpleWriter(fname, 1024*1024) try: yield writer finally: writer.close() with open(fname, 'r') as fd: line = fd.readline().strip() assert_equals(line, expected) os.remove(fname) class O(object): def __init__(self, **params): for k,v in params.items(): setattr(self, k, v) class TestWriteSimpleAttributes(unittest.TestCase): test_data_simple_attr = ( (O(id=None), '0 v0 dV c0 t i0 u T'), (O(visible=None), '0 v0 dV c0 t i0 u T'), (O(version=None), '0 v0 dV c0 t i0 u T'), (O(uid=None), '0 v0 dV c0 t i0 u T'), (O(user=None), '0 v0 dV c0 t i0 u T'), (O(timestamp=None), '0 v0 dV c0 t i0 u T'), (O(id=1), '1 v0 dV c0 t i0 u T'), (O(id=-99), '-99 v0 dV c0 t i0 u T'), (O(visible=True), '0 v0 dV c0 t i0 u T'), (O(visible=False), '0 v0 dD c0 t i0 u T'), (O(version=23), '0 v23 dV c0 t i0 u T'), (O(user="Schmidt"), '0 v0 dV c0 t i0 uSchmidt T'), (O(user=""), '0 v0 dV c0 t i0 u T'), (O(uid=987), '0 v0 dV c0 t i987 u T'), (O(timestamp='2012-04-14T20:58:35Z'), '0 v0 dV c0 t2012-04-14T20:58:35Z i0 u T'), (O(timestamp=mkdate(2009, 4, 14, 20, 58, 35)), '0 v0 dV c0 t2009-04-14T20:58:35Z i0 u T'), (O(timestamp='1970-01-01T00:00:01Z'), '0 v0 dV c0 t1970-01-01T00:00:01Z i0 u T'), ) def test_node_simple_attr(self): for node, out in self.test_data_simple_attr: with WriteExpect('n' + out + ' x y') as w: w.add_node(node) def test_way_simple_attr(self): for way, out in self.test_data_simple_attr: with WriteExpect('w' + out + ' N') as w: w.add_way(way) def test_relation_simple_attr(self): for rel, out in self.test_data_simple_attr: with WriteExpect('r' + out + ' M') as w: w.add_relation(rel) class TestWriteTags(unittest.TestCase): test_data_tags = ( (None, 'T'), ([], 'T'), ({}, 'T'), ((("foo", "bar"), ), 'Tfoo=bar'), ((("foo", "bar"), ("2", "1")), 'Tfoo=bar,2=1'), ({'test' : 'drive'}, 'Ttest=drive'), (OrderedDict((('a', 'b'), ('c', '3'))), 'Ta=b,c=3'), ) def test_node_tags(self): for tags, out in self.test_data_tags: with WriteExpect('n0 v0 dV c0 t i0 u ' + out + ' x y') as w: w.add_node(O(tags=tags)) def test_way_tags(self): for tags, out in self.test_data_tags: with WriteExpect('w0 v0 dV c0 t i0 u ' + out + ' N') as w: w.add_way(O(tags=tags)) def test_relation_tags(self): for tags, out in self.test_data_tags: with WriteExpect('r0 v0 dV c0 t i0 u ' + out + ' M') as w: w.add_relation(O(tags=tags)) class TestWriteNode(unittest.TestCase): def test_location_tuple(self): with WriteExpect('n0 v0 dV c0 t i0 u T x1.1234561 y0.1234561') as w: w.add_node(O(location=(1.1234561, 0.1234561))) def test_location_none(self): with WriteExpect('n0 v0 dV c0 t i0 u T x y') as w: w.add_node(O(location=None)) class TestWriteWay(unittest.TestCase): def test_node_list(self): with WriteExpect('w0 v0 dV c0 t i0 u T Nn1,n2,n3,n-4') as w: w.add_way(O(nodes=(1, 2, 3, -4))) def test_node_list_none(self): with WriteExpect('w0 v0 dV c0 t i0 u T N') as w: w.add_way(O(nodes=None)) class TestWriteRelation(unittest.TestCase): def test_relation_members(self): with WriteExpect('r0 v0 dV c0 t i0 u T Mn34@foo,r200@,w1111@x') as w: w.add_relation(O(members=(('n', 34, 'foo'), ('r', 200, ''), ('w', 1111, 'x') ))) def test_relation_members_None(self): with WriteExpect('r0 v0 dV c0 t i0 u T M') as w: w.add_relation(O(members=None)) if __name__ == '__main__': unittest.main() pyosmium-2.13.0/tools/000077500000000000000000000000001315206111100146065ustar00rootroot00000000000000pyosmium-2.13.0/tools/pyosmium-get-changes000077500000000000000000000201041315206111100205760ustar00rootroot00000000000000#!/usr/bin/python """ Fetch diffs from an OSM planet server. The starting point of the diff must be given either as a sequence ID or a date or can be computed from an OSM file. If no output file is given, the program will just print the initial sequence ID it would use (or save it in a file, if requested) and exit. This can be used to bootstrap the update process. The program tries to download until the latest change on the server is found or the maximum requested diff size is reached. Note that diffs are kept in memory during download. On success, the program will print a single number to stdout, the sequence number where to continue updates in the next run. This output can also be written to (and later read from) a file. """ from argparse import ArgumentParser, RawDescriptionHelpFormatter, ArgumentTypeError import datetime as dt from sys import version_info as python_version from osmium.replication import server as rserv from osmium.replication import newest_change_from_file from osmium.replication.utils import get_replication_header from osmium import SimpleHandler, WriteHandler import re import sys import logging from textwrap import dedent as msgfmt log = logging.getLogger() class ReplicationStart(object): """ Represents the point where changeset download should begin. """ def __init__(self, date=None, seq_id=None, src=None): self.date = date self.seq_id = seq_id self.source = src def get_sequence(self, svr): if self.seq_id is not None: log.debug("Using given sequence ID %d" % self.seq_id) return self.seq_id + 1 log.debug("Looking up sequence ID for timestamp %s" % self.date) seq = svr.timestamp_to_sequence(self.date) return seq + 1 if seq is not None else None @staticmethod def from_id(idstr): try: seq_id=int(idstr) except ValueError: raise ArgumentTypeError("Sequence id '%s' is not a number" % idstr) if seq_id < 0: raise ArgumentTypeError("Sequence id '%s' is negative" % idstr) return ReplicationStart(seq_id=seq_id) @staticmethod def from_date(datestr): try: date = dt.datetime.strptime(datestr, "%Y-%m-%dT%H:%M:%SZ") if python_version >= (3,0): date = date.replace(tzinfo=dt.timezone.utc) except ValueError: raise ArgumentTypeError("Date needs to be in ISO8601 format (e.g. 2015-12-24T08:08:08Z).") return ReplicationStart(date=date) @staticmethod def from_osm_file(fname, ignore_headers): if ignore_headers: ts = None seq = None url = None else: try: (url, seq, ts) = get_replication_header(fname) except RuntimeError as e: raise ArgumentTypeError(e) if ts is None and seq is None: log.debug("OSM file has no replication headers. Looking for newest OSM object.") try: ts = newest_change_from_file(fname) except RuntimeError as e: raise ArgumentTypeError(e) if ts is None: raise ArgumentTypeError("OSM file does not seem to contain valid data.") return ReplicationStart(seq_id=seq, date=ts, src=url) def write_end_sequence(fname, seqid): """Either writes out the sequence file or prints the sequence id to stdout. """ if fname is None: print(seqid) else: with open(fname, 'w') as fd: fd.write(str(seqid)) def get_arg_parser(from_main=False): def h(s): return re.sub("\s\s+" , " ", s) parser = ArgumentParser(description=__doc__, usage=None if from_main else 'pyosmium-get-changes [options]', formatter_class=RawDescriptionHelpFormatter) parser.add_argument('-v', dest='loglevel', action='count', default=0, help='Increase verbosity') parser.add_argument('-o', '--outfile', dest='outfile', help=h("""Name of diff output file. If omitted, only the sequence ID will be printed where updates would start.""")) parser.add_argument('--server', action='store', dest='server_url', help='Base URL of the replication server') parser.add_argument('-s', '--size', dest='outsize', type=int, default=100, help='Maximum data to load in MB (default: 100MB).') group = parser.add_mutually_exclusive_group() group.add_argument('-I', '--start-id', dest='start', type=ReplicationStart.from_id, metavar='ID', help='Sequence ID to start with') group.add_argument('-D', '--start-date', dest='start', metavar='DATE', type=ReplicationStart.from_date, help='Date when to start updates') group.add_argument('-O', '--start-osm-data', dest='start_file', metavar='OSMFILE', help='start at the date of the newest OSM object in the file') parser.add_argument('-f', '--sequence-file', dest='seq_file', help=h("""Sequence file. If the file exists, then updates will start after the id given in the file. At the end of the process, the last sequence ID contained in the diff is written.""")) parser.add_argument('--ignore-osmosis-headers', dest='ignore_headers', action='store_true', help=h("""When determining the start from an OSM file, ignore potential replication information in the header and search for the newest OSM object.""")) parser.add_argument('-d', '--no-deduplicate', action='store_false', dest='simplify', help='Do not deduplicate and sort diffs.') return parser if __name__ == '__main__': logging.basicConfig(stream=sys.stderr, format='%(levelname)s: %(message)s') options = get_arg_parser(from_main=True).parse_args() log.setLevel(max(3 - options.loglevel, 0) * 10) if options.start_file is not None: options.start = ReplicationStart.from_osm_file(options.start_file, options.ignore_headers) if options.start is None: if options.seq_file is None: log.error(msgfmt(""" Don't know with which change to start. One of the parameters -I / -D / -O / -f needs to begiven.""")) exit(1) with open(opt.start_file, 'r') as f: seq = f.readline() options.start = ReplicationStart_from_id(seq) if options.server_url is not None and options.start.source is not None: if options.server_url != options.start.source: log.error(msgfmt(""" You asked to use server URL: %s but the referenced OSM file points to replication server: %s If you really mean to overwrite the URL, use --ignore-osmosis-headers.""" % (options.server_url, options.start.source))) exit(2) url = options.server_url \ or options.start.source \ or 'https://planet.osm.org/replication/minute/' logging.info("Using replication server at %s" % url) svr = rserv.ReplicationServer(url) startseq = options.start.get_sequence(svr) if startseq is None: log.error("Cannot read state file from server. Is the URL correct?") exit(1) if options.outfile is None: write_end_sequence(options.seq_file, startseq) exit(0) log.debug("Starting download at ID %d (max %d MB)" % (startseq, options.outsize)) outhandler = WriteHandler(options.outfile) endseq = svr.apply_diffs(outhandler, startseq, max_size=options.outsize*1024, simplify=options.simplify) outhandler.close() if endseq is None: exit(3) write_end_sequence(options.seq_file, endseq) pyosmium-2.13.0/tools/pyosmium-up-to-date000077500000000000000000000175441315206111100204060ustar00rootroot00000000000000#!/usr/bin/python """ Update an OSM file with changes from a OSM replication server. Diffs are downloaded and kept in memory. To avoid running out of memory, the maximum size of diffs that can be downloaded at once is limited to 1 GB per default. This corresponds to approximately 3 days of update. The limit can be changed with the --size parameter. However, you should take into account that processing the files requires additional memory (about 1GB more). The starting time is automatically determined from the data in the file. For PBF files, it is also possible to read and write the replication information from the osmosis headers. That means that after the first update, subsequent calls to pyosmium-up-to-date will continue the updates from the same server exactly where they have left of. The program returns 0, if updates have been successfully applied up to the newest data. It returns 1, if some updates have been applied but there is still data available on the server (either because the size limit has been reached or there was a network error which could not be resolved). Any other error results in a return code larger than 1. The output file is guaranteed to be unmodified in that case. """ import re import sys import logging from argparse import ArgumentParser, RawDescriptionHelpFormatter import datetime as dt from sys import version_info as python_version from osmium.replication import server as rserv from osmium.replication.utils import get_replication_header from osmium.replication import newest_change_from_file from textwrap import dedent as msgfmt from tempfile import mktemp import os.path log = logging.getLogger() def update_from_osm_server(ts, options): """Update the OSM file using the official OSM servers at https://planet.osm.org/replication. This strategy will attempt to start with daily updates before going down to minutelies. TODO: only updates from hourlies currently implemented. """ return update_from_custom_server("https://planet.osm.org/replication/hour/", None, ts, options) def update_from_custom_server(url, seq, ts, options): """Update from a custom URL, simply using the diff sequence as is.""" svr = rserv.ReplicationServer(url) log.info("Using replication service at %s" % url) if seq is None: log.info("Using timestamp %s as starting point." % ts) startseq = svr.timestamp_to_sequence(ts) if startseq is None: log.error("No starting point found for time %s on server %s" % (str(ts), url)) return 3 else: log.debug("Using given sequence ID %d" % seq) startseq = seq + 1 ts = svr.get_state_info(seq=startseq) if ts is None: log.error("Cannot download state information for ID %d. Is the URL correct?" % seq) return 3 ts = ts.timestamp if not options.force_update: cmpdate = dt.datetime.utcnow() - dt.timedelta(days=90) if python_version >= (3,0): cmpdate = cmpdate.replace(tzinfo=dt.timezone.utc) if ts < cmpdate: log.error( """The OSM file is more than 3 months old. You should download a more recent file instead of updating. If you really want to update the file, use --force-update-of-old-planet.""") return 3 log.info("Starting download at ID %d (max %d MB)" % (startseq, options.outsize)) outfile = options.outfile infile = options.infile if outfile is None: fdir, fname = os.path.split(infile) if options.tmpdir is not None: fdir = options.tmpdir ofname = mktemp(suffix='-' + fname, dir=fdir) else: ofname = outfile outseqs = svr.apply_diffs_to_file(infile, ofname, startseq, max_size=options.outsize*1024) if outseqs is None: log.info("No new updates found.") return 3 if outfile is None: os.rename(ofname, infile) log.info("Downloaded until %d. Server has data available until %d." % outseqs) return 0 if outseqs[1] == outseqs[0] else 1 def compute_start_point(options): if options.ignore_headers: url, seq, ts = None, None, None else: url, seq, ts = get_replication_header(options.infile) if options.server_url is not None: if url is not None and url != options.server_url: log.error(msgfmt(""" You asked to use server URL: %s but the referenced OSM file points to replication server: %s If you really mean to overwrite the URL, use --ignore-osmosis-headers.""" % (options.server_url, url))) exit(2) url = options.server_url if seq is None and ts is None: log.info("No replication information found, scanning for newest OSM object.") ts = newest_change_from_file(options.infile) if ts is None: log.error("OSM file does not seem to contain valid data.") exit(2) if ts is not None: ts -= dt.timedelta(minutes=options.wind_back) return url, seq, ts def get_arg_parser(from_main=False): def h(s): return re.sub("\s\s+" , " ", s) parser = ArgumentParser(description=__doc__, usage=None if from_main else 'pyosmium-up-to-date [options] ', formatter_class=RawDescriptionHelpFormatter) parser.add_argument('-v', dest='loglevel', action='count', default=0, help='Increase verbosity') parser.add_argument('infile', metavar='', help="OSM file to update") parser.add_argument('-o', '--outfile', dest='outfile', help=h("""Name output of file. If missing, the input file will be overwritten.""")) parser.add_argument('--server', action='store', dest='server_url', help=h("""Base URL of the replication server. Default: 'https://planet.osm.org/replication/hour/' (hourly diffs from osm.org).""")) parser.add_argument('-s', '--size', dest='outsize', metavar='SIZE', type=int, default=1024, help='Maximum size of change to apply at once in MB. Default: 1GB.') parser.add_argument('--tmpdir', dest='tmpdir', help='Directory to use for temporary files. Default: directory of input file') parser.add_argument('--ignore-osmosis-headers', dest='ignore_headers', action='store_true', help=h("""Ignore potential replication information in the header of the input file and search for the newest OSM object in the file instead.""")) parser.add_argument('-b', '--wind-back', dest='wind_back', type=int, default=60, help=h("""Number of minutes to start downloading before the newest addition to input data. (Ignored when the file contains a sequence ID.) Default: 60.""")) parser.add_argument('--force-update-of-old-planet', action='store_true', dest='force_update', help="Apply update even if the input data is really old.") return parser if __name__ == '__main__': logging.basicConfig(stream=sys.stderr, format='%(asctime)s %(levelname)s: %(message)s') options = get_arg_parser(from_main=True).parse_args() log.setLevel(max(3 - options.loglevel, 0) * 10) try: url, seq, ts = compute_start_point(options) except RuntimeError as e: log.error(str(e)) exit(2) if url is None: ret = update_from_osm_server(ts, options) else: ret = update_from_custom_server(url, seq, ts, options) exit(ret)