imposm.parser-1.0.7/0000755000076500000240000000000012454223055014241 5ustar oltstaff00000000000000imposm.parser-1.0.7/CHANGES0000644000076500000240000000136712454206010015233 0ustar oltstaff00000000000000Changelog --------- 1.0.7 2015-01-10 ~~~~~~~~~~~~~~~~ - updated protobuf parser 1.0.6 2014-10-31 ~~~~~~~~~~~~~~~~ - fixed bug in protobuf parser that could cause segfaults 1.0.5 2013-09-13 ~~~~~~~~~~~~~~~~ - support PBF without granularity value - improved support for non-pretty-printed XML 1.0.4 2012-12-10 ~~~~~~~~~~~~~~~~ - improved support for non-pretty-printed XML - fixed dependency check for multiprocessing 1.0.3 2011-07-21 ~~~~~~~~~~~~~~~~ - support for uncompressed PBF - bug fix for PBF without dense nodes 1.0.2 2011-03-10 ~~~~~~~~~~~~~~~~ - improved regexp based XML coord parser - prevent mmap overflow in XMLChunker without coord_callback - successfully parsed whole planet.osm 1.0.0 2011-02-22 ~~~~~~~~~~~~~~~~ - first release imposm.parser-1.0.7/imposm/0000755000076500000240000000000012454223055015545 5ustar oltstaff00000000000000imposm.parser-1.0.7/imposm/__init__.py0000644000076500000240000000006711766117447017675 0ustar oltstaff00000000000000__import__('pkg_resources').declare_namespace(__name__)imposm.parser-1.0.7/imposm/parser/0000755000076500000240000000000012454223055017041 5ustar oltstaff00000000000000imposm.parser-1.0.7/imposm/parser/__init__.py0000644000076500000240000000010311766117447021160 0ustar oltstaff00000000000000from imposm.parser.simple import OSMParser __all__ = ['OSMParser']imposm.parser-1.0.7/imposm/parser/pbf/0000755000076500000240000000000012454223055017610 5ustar oltstaff00000000000000imposm.parser-1.0.7/imposm/parser/pbf/__init__.py0000644000076500000240000000000011766117447021723 0ustar oltstaff00000000000000imposm.parser-1.0.7/imposm/parser/pbf/multiproc.py0000644000076500000240000000765312012456241022206 0ustar oltstaff00000000000000# Copyright 2011 Omniscale GmbH & Co. KG # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import multiprocessing from imposm.parser.pbf.parser import PBFFile, PBFParser from imposm.parser.util import setproctitle class PBFParserProcess(PBFParser, multiprocessing.Process): def __init__(self, pos_queue, *args, **kw): multiprocessing.Process.__init__(self) PBFParser.__init__(self, *args, **kw) self.daemon = True self.pos_queue = pos_queue def run(self): setproctitle('imposm pbf parser') while True: pos = self.pos_queue.get() if pos is None: self.pos_queue.task_done() break self.parse(pos['filename'], offset=pos['blob_pos'], size=pos['blob_size']) self.pos_queue.task_done() class PBFMultiProcParser(object): nodes_tag_filter = None ways_tag_filter = None relations_tag_filter = None def __init__(self, pool_size, nodes_queue=None, ways_queue=None, relations_queue=None, coords_queue=None, marshal_elem_data=False): self.pool_size = pool_size self.nodes_callback = nodes_queue.put if nodes_queue else None self.ways_callback = ways_queue.put if ways_queue else None self.relations_callback = relations_queue.put if relations_queue else None self.coords_callback = coords_queue.put if coords_queue else None self.marshal = marshal_elem_data def parse(self, filename): pos_queue = multiprocessing.JoinableQueue(32) pool = [] for _ in xrange(self.pool_size): proc = PBFParserProcess(pos_queue, nodes_callback=self.nodes_callback, coords_callback=self.coords_callback, ways_callback=self.ways_callback, relations_callback=self.relations_callback, nodes_tag_filter=self.nodes_tag_filter, ways_tag_filter=self.ways_tag_filter, relations_tag_filter=self.relations_tag_filter, marshal=self.marshal ) pool.append(proc) proc.start() reader = PBFFile(filename) for pos in reader.blob_offsets(): pos_queue.put(pos) pos_queue.join() for proc in pool: pos_queue.put(None) for proc in pool: proc.join() if __name__ == '__main__': import sys def count_proc(type, queue): def count(): count = 0 while True: nodes = queue.get() if nodes is None: queue.task_done() break count += len(nodes) queue.task_done() print type, count return count nodes_queue = multiprocessing.JoinableQueue(128) ways_queue = multiprocessing.JoinableQueue(128) relations_queue = multiprocessing.JoinableQueue(128) procs = [ multiprocessing.Process(target=count_proc('nodes', nodes_queue)), multiprocessing.Process(target=count_proc('ways', ways_queue)), multiprocessing.Process(target=count_proc('relations', relations_queue)) ] for proc in procs: proc.start() parser = PBFMultiProcParser(2, nodes_queue=nodes_queue, ways_queue=ways_queue, relations_queue=relations_queue) parser.parse(sys.argv[1]) nodes_queue.put(None) ways_queue.put(None) relations_queue.put(None) for proc in procs: proc.join()imposm.parser-1.0.7/imposm/parser/pbf/osm.cc0000644000076500000240000104367712454175040020737 0ustar oltstaff00000000000000// -*- C++ -*- #include #include #include #include "structmember.h" #include "osm.pb.h" #include #include static PyObject * fastpb_convert5(::google::protobuf::int32 value) { return PyLong_FromLong(value); } static PyObject * fastpb_convert3(::google::protobuf::int64 value) { return PyLong_FromLongLong(value); } static PyObject * fastpb_convert18(::google::protobuf::int64 value) { return PyLong_FromLongLong(value); } static PyObject * fastpb_convert17(::google::protobuf::int32 value) { return PyLong_FromLong(value); } static PyObject * fastpb_convert13(::google::protobuf::uint32 value) { return PyLong_FromUnsignedLong(value); } static PyObject * fastpb_convert7(::google::protobuf::int32 value) { return PyLong_FromLong(value); } static PyObject * fastpb_convert4(::google::protobuf::uint64 value) { return PyLong_FromUnsignedLong(value); } static PyObject * fastpb_convert1(double value) { return PyFloat_FromDouble(value); } static PyObject * fastpb_convert2(float value) { return PyFloat_FromDouble(value); } static PyObject * fastpb_convert9(const ::std::string &value) { return PyUnicode_Decode(value.data(), value.length(), "utf-8", NULL); } static PyObject * fastpb_convert12(const ::std::string &value) { return PyString_FromStringAndSize(value.data(), value.length()); } static PyObject * fastpb_convert8(bool value) { return PyBool_FromLong(value ? 1 : 0); } static PyObject * fastpb_convert14(int value) { // TODO(robbyw): Check EnumName_IsValid(value) return PyInt_FromLong(value); } // Lets try not to pollute the global namespace namespace { // Forward-declaration for recursive structures extern PyTypeObject BlobType; typedef struct { PyObject_HEAD OSMPBF::Blob *protobuf; } Blob; void Blob_dealloc(Blob* self) { delete self->protobuf; self->ob_type->tp_free((PyObject*)self); } PyObject * Blob_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { Blob *self; self = (Blob *)type->tp_alloc(type, 0); self->protobuf = new OSMPBF::Blob(); return (PyObject *)self; } PyObject * Blob_DebugString(Blob* self) { std::string result; Py_BEGIN_ALLOW_THREADS result = self->protobuf->Utf8DebugString(); Py_END_ALLOW_THREADS return PyUnicode_FromStringAndSize(result.data(), result.length()); } PyObject * Blob_SerializeToString(Blob* self) { std::string result; Py_BEGIN_ALLOW_THREADS self->protobuf->SerializeToString(&result); Py_END_ALLOW_THREADS return PyString_FromStringAndSize(result.data(), result.length()); } PyObject * Blob_SerializeMany(void *nothing, PyObject *values) { std::string result; google::protobuf::io::ZeroCopyOutputStream* output = new google::protobuf::io::StringOutputStream(&result); google::protobuf::io::CodedOutputStream* outputStream = new google::protobuf::io::CodedOutputStream(output); PyObject *sequence = PySequence_Fast(values, "The values to serialize must be a sequence."); for (Py_ssize_t i = 0, len = PySequence_Length(sequence); i < len; ++i) { Blob *value = (Blob *)PySequence_Fast_GET_ITEM(sequence, i); Py_BEGIN_ALLOW_THREADS outputStream->WriteVarint32(value->protobuf->ByteSize()); value->protobuf->SerializeToCodedStream(outputStream); Py_END_ALLOW_THREADS } Py_XDECREF(sequence); delete outputStream; delete output; return PyString_FromStringAndSize(result.data(), result.length()); } PyObject * Blob_ParseFromString(Blob* self, PyObject *value) { std::string serialized(PyString_AsString(value), PyString_Size(value)); Py_BEGIN_ALLOW_THREADS self->protobuf->ParseFromString(serialized); Py_END_ALLOW_THREADS Py_RETURN_NONE; } PyObject * Blob_ParseFromLongString(Blob* self, PyObject *value) { google::protobuf::io::ZeroCopyInputStream* input = new google::protobuf::io::ArrayInputStream(PyString_AsString(value), PyString_Size(value)); google::protobuf::io::CodedInputStream* inputStream = new google::protobuf::io::CodedInputStream(input); inputStream->SetTotalBytesLimit(512 * 1024 * 1024, 512 * 1024 * 1024); Py_BEGIN_ALLOW_THREADS self->protobuf->ParseFromCodedStream(inputStream); Py_END_ALLOW_THREADS delete inputStream; delete input; Py_RETURN_NONE; } PyObject * Blob_ParseMany(void* nothing, PyObject *args) { PyObject *value; PyObject *callback; int fail = 0; if (!PyArg_ParseTuple(args, "OO", &value, &callback)) { return NULL; } google::protobuf::io::ZeroCopyInputStream* input = new google::protobuf::io::ArrayInputStream(PyString_AsString(value), PyString_Size(value)); google::protobuf::io::CodedInputStream* inputStream = new google::protobuf::io::CodedInputStream(input); inputStream->SetTotalBytesLimit(512 * 1024 * 1024, 512 * 1024 * 1024); google::protobuf::uint32 bytes; PyObject *single = NULL; while (inputStream->ReadVarint32(&bytes)) { google::protobuf::io::CodedInputStream::Limit messageLimit = inputStream->PushLimit(bytes); if (single == NULL) { single = Blob_new(&BlobType, NULL, NULL); } Py_BEGIN_ALLOW_THREADS ((Blob *)single)->protobuf->ParseFromCodedStream(inputStream); Py_END_ALLOW_THREADS inputStream->PopLimit(messageLimit); PyObject *result = PyObject_CallFunctionObjArgs(callback, single, NULL); if (result == NULL) { fail = 1; break; }; if (single->ob_refcnt != 1) { // If the callback saved a reference to the item, don't re-use it. Py_XDECREF(single); single = NULL; } } if (single != NULL) { Py_XDECREF(single); } delete inputStream; delete input; if (fail) { return NULL; } else { Py_RETURN_NONE; } } PyObject * Blob_getraw(Blob *self, void *closure) { if (! self->protobuf->has_raw()) { Py_RETURN_NONE; } return fastpb_convert12( self->protobuf->raw()); } int Blob_setraw(Blob *self, PyObject *input, void *closure) { if (input == NULL || input == Py_None) { self->protobuf->clear_raw(); return 0; } PyObject *value = input; // string if (! PyString_Check(value)) { PyErr_SetString(PyExc_TypeError, "The raw attribute value must be a string"); return -1; } std::string protoValue(PyString_AsString(value), PyString_Size(value)); self->protobuf->set_raw(protoValue); return 0; } PyObject * Blob_getraw_size(Blob *self, void *closure) { if (! self->protobuf->has_raw_size()) { Py_RETURN_NONE; } return fastpb_convert5( self->protobuf->raw_size()); } int Blob_setraw_size(Blob *self, PyObject *input, void *closure) { if (input == NULL || input == Py_None) { self->protobuf->clear_raw_size(); return 0; } PyObject *value = input; ::google::protobuf::int32 protoValue; // int32 if (PyInt_Check(value)) { protoValue = PyInt_AsLong(value); } else { PyErr_SetString(PyExc_TypeError, "The raw_size attribute value must be an integer"); return -1; } self->protobuf->set_raw_size(protoValue); return 0; } PyObject * Blob_getzlib_data(Blob *self, void *closure) { if (! self->protobuf->has_zlib_data()) { Py_RETURN_NONE; } return fastpb_convert12( self->protobuf->zlib_data()); } int Blob_setzlib_data(Blob *self, PyObject *input, void *closure) { if (input == NULL || input == Py_None) { self->protobuf->clear_zlib_data(); return 0; } PyObject *value = input; // string if (! PyString_Check(value)) { PyErr_SetString(PyExc_TypeError, "The zlib_data attribute value must be a string"); return -1; } std::string protoValue(PyString_AsString(value), PyString_Size(value)); self->protobuf->set_zlib_data(protoValue); return 0; } PyObject * Blob_getlzma_data(Blob *self, void *closure) { if (! self->protobuf->has_lzma_data()) { Py_RETURN_NONE; } return fastpb_convert12( self->protobuf->lzma_data()); } int Blob_setlzma_data(Blob *self, PyObject *input, void *closure) { if (input == NULL || input == Py_None) { self->protobuf->clear_lzma_data(); return 0; } PyObject *value = input; // string if (! PyString_Check(value)) { PyErr_SetString(PyExc_TypeError, "The lzma_data attribute value must be a string"); return -1; } std::string protoValue(PyString_AsString(value), PyString_Size(value)); self->protobuf->set_lzma_data(protoValue); return 0; } PyObject * Blob_getOBSOLETE_bzip2_data(Blob *self, void *closure) { if (! self->protobuf->has_obsolete_bzip2_data()) { Py_RETURN_NONE; } return fastpb_convert12( self->protobuf->obsolete_bzip2_data()); } int Blob_setOBSOLETE_bzip2_data(Blob *self, PyObject *input, void *closure) { if (input == NULL || input == Py_None) { self->protobuf->clear_obsolete_bzip2_data(); return 0; } PyObject *value = input; // string if (! PyString_Check(value)) { PyErr_SetString(PyExc_TypeError, "The OBSOLETE_bzip2_data attribute value must be a string"); return -1; } std::string protoValue(PyString_AsString(value), PyString_Size(value)); self->protobuf->set_obsolete_bzip2_data(protoValue); return 0; } int Blob_init(Blob *self, PyObject *args, PyObject *kwds) { PyObject *raw = NULL; PyObject *raw_size = NULL; PyObject *zlib_data = NULL; PyObject *lzma_data = NULL; PyObject *OBSOLETE_bzip2_data = NULL; static char *kwlist[] = { (char *) "raw", (char *) "raw_size", (char *) "zlib_data", (char *) "lzma_data", (char *) "OBSOLETE_bzip2_data", NULL }; if (! PyArg_ParseTupleAndKeywords( args, kwds, "|OOOOO", kwlist, &raw,&raw_size,&zlib_data,&lzma_data,&OBSOLETE_bzip2_data)) return -1; if (raw) { if (Blob_setraw(self, raw, NULL) < 0) { return -1; } } if (raw_size) { if (Blob_setraw_size(self, raw_size, NULL) < 0) { return -1; } } if (zlib_data) { if (Blob_setzlib_data(self, zlib_data, NULL) < 0) { return -1; } } if (lzma_data) { if (Blob_setlzma_data(self, lzma_data, NULL) < 0) { return -1; } } if (OBSOLETE_bzip2_data) { if (Blob_setOBSOLETE_bzip2_data(self, OBSOLETE_bzip2_data, NULL) < 0) { return -1; } } return 0; } PyObject * Blob_richcompare(PyObject *self, PyObject *other, int op) { PyObject *result = NULL; if (!PyType_IsSubtype(other->ob_type, &BlobType)) { result = Py_NotImplemented; } else { // This is not a particularly efficient implementation since it never short circuits, but it's better // than nothing. It should probably only be used for tests. Blob *selfValue = (Blob *)self; Blob *otherValue = (Blob *)other; std::string selfSerialized; std::string otherSerialized; Py_BEGIN_ALLOW_THREADS selfValue->protobuf->SerializeToString(&selfSerialized); otherValue->protobuf->SerializeToString(&otherSerialized); Py_END_ALLOW_THREADS int cmp = selfSerialized.compare(otherSerialized); bool value = false; switch (op) { case Py_LT: value = cmp < 0; break; case Py_LE: value = cmp <= 0; break; case Py_EQ: value = cmp == 0; break; case Py_NE: value = cmp != 0; break; case Py_GT: value = cmp > 0; break; case Py_GE: value = cmp >= 0; break; } result = value ? Py_True : Py_False; } Py_XINCREF(result); return result; } static PyObject * Blob_repr(PyObject *selfObject) { Blob *self = (Blob *)selfObject; PyObject *member; PyObject *memberRepr; std::stringstream result; result << "Blob("; result << "raw="; member = Blob_getraw(self, NULL); memberRepr = PyObject_Repr(member); result << PyString_AsString(memberRepr); Py_XDECREF(memberRepr); Py_XDECREF(member); result << ", "; result << "raw_size="; member = Blob_getraw_size(self, NULL); memberRepr = PyObject_Repr(member); result << PyString_AsString(memberRepr); Py_XDECREF(memberRepr); Py_XDECREF(member); result << ", "; result << "zlib_data="; member = Blob_getzlib_data(self, NULL); memberRepr = PyObject_Repr(member); result << PyString_AsString(memberRepr); Py_XDECREF(memberRepr); Py_XDECREF(member); result << ", "; result << "lzma_data="; member = Blob_getlzma_data(self, NULL); memberRepr = PyObject_Repr(member); result << PyString_AsString(memberRepr); Py_XDECREF(memberRepr); Py_XDECREF(member); result << ", "; result << "OBSOLETE_bzip2_data="; member = Blob_getOBSOLETE_bzip2_data(self, NULL); memberRepr = PyObject_Repr(member); result << PyString_AsString(memberRepr); Py_XDECREF(memberRepr); Py_XDECREF(member); result << ")"; std::string resultString = result.str(); return PyUnicode_Decode(resultString.data(), resultString.length(), "utf-8", NULL); } PyMemberDef Blob_members[] = { {NULL} // Sentinel }; PyGetSetDef Blob_getsetters[] = { {(char *)"raw", (getter)Blob_getraw, (setter)Blob_setraw, (char *)"", NULL}, {(char *)"raw_size", (getter)Blob_getraw_size, (setter)Blob_setraw_size, (char *)"", NULL}, {(char *)"zlib_data", (getter)Blob_getzlib_data, (setter)Blob_setzlib_data, (char *)"", NULL}, {(char *)"lzma_data", (getter)Blob_getlzma_data, (setter)Blob_setlzma_data, (char *)"", NULL}, {(char *)"OBSOLETE_bzip2_data", (getter)Blob_getOBSOLETE_bzip2_data, (setter)Blob_setOBSOLETE_bzip2_data, (char *)"", NULL}, {NULL} // Sentinel }; PyMethodDef Blob_methods[] = { {"DebugString", (PyCFunction)Blob_DebugString, METH_NOARGS, "Generates a human readable form of this message, useful for debugging and other purposes." }, {"SerializeToString", (PyCFunction)Blob_SerializeToString, METH_NOARGS, "Serializes the protocol buffer to a string." }, {"SerializeMany", (PyCFunction)Blob_SerializeMany, METH_O | METH_CLASS, "Serializes a sequence of protocol buffers to a string." }, {"ParseFromString", (PyCFunction)Blob_ParseFromString, METH_O, "Parses the protocol buffer from a string." }, {"ParseFromLongString", (PyCFunction)Blob_ParseFromLongString, METH_O, "Parses the protocol buffer from a string as large as 512MB." }, {"ParseMany", (PyCFunction)Blob_ParseMany, METH_VARARGS | METH_CLASS, "Parses many protocol buffers of this type from a string." }, {NULL} // Sentinel }; PyTypeObject BlobType = { PyObject_HEAD_INIT(NULL) 0, /*ob_size*/ "OSMPBF.Blob", /*tp_name*/ sizeof(Blob), /*tp_basicsize*/ 0, /*tp_itemsize*/ (destructor)Blob_dealloc, /*tp_dealloc*/ 0, /*tp_print*/ 0, /*tp_getattr*/ 0, /*tp_setattr*/ 0, /*tp_compare*/ Blob_repr, /*tp_repr*/ 0, /*tp_as_number*/ 0, /*tp_as_sequence*/ 0, /*tp_as_mapping*/ 0, /*tp_hash */ 0, /*tp_call*/ 0, /*tp_str*/ 0, /*tp_getattro*/ 0, /*tp_setattro*/ 0, /*tp_as_buffer*/ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_RICHCOMPARE, /*tp_flags*/ "Blob objects", /* tp_doc */ 0, /* tp_traverse */ 0, /* tp_clear */ Blob_richcompare, /* tp_richcompare */ 0, /* tp_weaklistoffset */ 0, /* tp_iter */ 0, /* tp_iternext */ Blob_methods, /* tp_methods */ Blob_members, /* tp_members */ Blob_getsetters, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ 0, /* tp_descr_set */ 0, /* tp_dictoffset */ (initproc)Blob_init, /* tp_init */ 0, /* tp_alloc */ Blob_new, /* tp_new */ }; } // Lets try not to pollute the global namespace namespace { // Forward-declaration for recursive structures extern PyTypeObject BlobHeaderType; typedef struct { PyObject_HEAD OSMPBF::BlobHeader *protobuf; } BlobHeader; void BlobHeader_dealloc(BlobHeader* self) { delete self->protobuf; self->ob_type->tp_free((PyObject*)self); } PyObject * BlobHeader_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { BlobHeader *self; self = (BlobHeader *)type->tp_alloc(type, 0); self->protobuf = new OSMPBF::BlobHeader(); return (PyObject *)self; } PyObject * BlobHeader_DebugString(BlobHeader* self) { std::string result; Py_BEGIN_ALLOW_THREADS result = self->protobuf->Utf8DebugString(); Py_END_ALLOW_THREADS return PyUnicode_FromStringAndSize(result.data(), result.length()); } PyObject * BlobHeader_SerializeToString(BlobHeader* self) { std::string result; Py_BEGIN_ALLOW_THREADS self->protobuf->SerializeToString(&result); Py_END_ALLOW_THREADS return PyString_FromStringAndSize(result.data(), result.length()); } PyObject * BlobHeader_SerializeMany(void *nothing, PyObject *values) { std::string result; google::protobuf::io::ZeroCopyOutputStream* output = new google::protobuf::io::StringOutputStream(&result); google::protobuf::io::CodedOutputStream* outputStream = new google::protobuf::io::CodedOutputStream(output); PyObject *sequence = PySequence_Fast(values, "The values to serialize must be a sequence."); for (Py_ssize_t i = 0, len = PySequence_Length(sequence); i < len; ++i) { BlobHeader *value = (BlobHeader *)PySequence_Fast_GET_ITEM(sequence, i); Py_BEGIN_ALLOW_THREADS outputStream->WriteVarint32(value->protobuf->ByteSize()); value->protobuf->SerializeToCodedStream(outputStream); Py_END_ALLOW_THREADS } Py_XDECREF(sequence); delete outputStream; delete output; return PyString_FromStringAndSize(result.data(), result.length()); } PyObject * BlobHeader_ParseFromString(BlobHeader* self, PyObject *value) { std::string serialized(PyString_AsString(value), PyString_Size(value)); Py_BEGIN_ALLOW_THREADS self->protobuf->ParseFromString(serialized); Py_END_ALLOW_THREADS Py_RETURN_NONE; } PyObject * BlobHeader_ParseFromLongString(BlobHeader* self, PyObject *value) { google::protobuf::io::ZeroCopyInputStream* input = new google::protobuf::io::ArrayInputStream(PyString_AsString(value), PyString_Size(value)); google::protobuf::io::CodedInputStream* inputStream = new google::protobuf::io::CodedInputStream(input); inputStream->SetTotalBytesLimit(512 * 1024 * 1024, 512 * 1024 * 1024); Py_BEGIN_ALLOW_THREADS self->protobuf->ParseFromCodedStream(inputStream); Py_END_ALLOW_THREADS delete inputStream; delete input; Py_RETURN_NONE; } PyObject * BlobHeader_ParseMany(void* nothing, PyObject *args) { PyObject *value; PyObject *callback; int fail = 0; if (!PyArg_ParseTuple(args, "OO", &value, &callback)) { return NULL; } google::protobuf::io::ZeroCopyInputStream* input = new google::protobuf::io::ArrayInputStream(PyString_AsString(value), PyString_Size(value)); google::protobuf::io::CodedInputStream* inputStream = new google::protobuf::io::CodedInputStream(input); inputStream->SetTotalBytesLimit(512 * 1024 * 1024, 512 * 1024 * 1024); google::protobuf::uint32 bytes; PyObject *single = NULL; while (inputStream->ReadVarint32(&bytes)) { google::protobuf::io::CodedInputStream::Limit messageLimit = inputStream->PushLimit(bytes); if (single == NULL) { single = BlobHeader_new(&BlobHeaderType, NULL, NULL); } Py_BEGIN_ALLOW_THREADS ((BlobHeader *)single)->protobuf->ParseFromCodedStream(inputStream); Py_END_ALLOW_THREADS inputStream->PopLimit(messageLimit); PyObject *result = PyObject_CallFunctionObjArgs(callback, single, NULL); if (result == NULL) { fail = 1; break; }; if (single->ob_refcnt != 1) { // If the callback saved a reference to the item, don't re-use it. Py_XDECREF(single); single = NULL; } } if (single != NULL) { Py_XDECREF(single); } delete inputStream; delete input; if (fail) { return NULL; } else { Py_RETURN_NONE; } } PyObject * BlobHeader_gettype(BlobHeader *self, void *closure) { if (! self->protobuf->has_type()) { Py_RETURN_NONE; } return fastpb_convert9( self->protobuf->type()); } int BlobHeader_settype(BlobHeader *self, PyObject *input, void *closure) { if (input == NULL || input == Py_None) { self->protobuf->clear_type(); return 0; } PyObject *value = input; // string bool reallocated = false; if (PyUnicode_Check(value)) { value = PyUnicode_AsEncodedString(value, "utf-8", NULL); reallocated = true; } if (! PyString_Check(value)) { PyErr_SetString(PyExc_TypeError, "The type attribute value must be a string"); return -1; } std::string protoValue(PyString_AsString(value), PyString_Size(value)); if (reallocated) { Py_XDECREF(value); } self->protobuf->set_type(protoValue); return 0; } PyObject * BlobHeader_getindexdata(BlobHeader *self, void *closure) { if (! self->protobuf->has_indexdata()) { Py_RETURN_NONE; } return fastpb_convert12( self->protobuf->indexdata()); } int BlobHeader_setindexdata(BlobHeader *self, PyObject *input, void *closure) { if (input == NULL || input == Py_None) { self->protobuf->clear_indexdata(); return 0; } PyObject *value = input; // string if (! PyString_Check(value)) { PyErr_SetString(PyExc_TypeError, "The indexdata attribute value must be a string"); return -1; } std::string protoValue(PyString_AsString(value), PyString_Size(value)); self->protobuf->set_indexdata(protoValue); return 0; } PyObject * BlobHeader_getdatasize(BlobHeader *self, void *closure) { if (! self->protobuf->has_datasize()) { Py_RETURN_NONE; } return fastpb_convert5( self->protobuf->datasize()); } int BlobHeader_setdatasize(BlobHeader *self, PyObject *input, void *closure) { if (input == NULL || input == Py_None) { self->protobuf->clear_datasize(); return 0; } PyObject *value = input; ::google::protobuf::int32 protoValue; // int32 if (PyInt_Check(value)) { protoValue = PyInt_AsLong(value); } else { PyErr_SetString(PyExc_TypeError, "The datasize attribute value must be an integer"); return -1; } self->protobuf->set_datasize(protoValue); return 0; } int BlobHeader_init(BlobHeader *self, PyObject *args, PyObject *kwds) { PyObject *type = NULL; PyObject *indexdata = NULL; PyObject *datasize = NULL; static char *kwlist[] = { (char *) "type", (char *) "indexdata", (char *) "datasize", NULL }; if (! PyArg_ParseTupleAndKeywords( args, kwds, "|OOO", kwlist, &type,&indexdata,&datasize)) return -1; if (type) { if (BlobHeader_settype(self, type, NULL) < 0) { return -1; } } if (indexdata) { if (BlobHeader_setindexdata(self, indexdata, NULL) < 0) { return -1; } } if (datasize) { if (BlobHeader_setdatasize(self, datasize, NULL) < 0) { return -1; } } return 0; } PyObject * BlobHeader_richcompare(PyObject *self, PyObject *other, int op) { PyObject *result = NULL; if (!PyType_IsSubtype(other->ob_type, &BlobHeaderType)) { result = Py_NotImplemented; } else { // This is not a particularly efficient implementation since it never short circuits, but it's better // than nothing. It should probably only be used for tests. BlobHeader *selfValue = (BlobHeader *)self; BlobHeader *otherValue = (BlobHeader *)other; std::string selfSerialized; std::string otherSerialized; Py_BEGIN_ALLOW_THREADS selfValue->protobuf->SerializeToString(&selfSerialized); otherValue->protobuf->SerializeToString(&otherSerialized); Py_END_ALLOW_THREADS int cmp = selfSerialized.compare(otherSerialized); bool value = false; switch (op) { case Py_LT: value = cmp < 0; break; case Py_LE: value = cmp <= 0; break; case Py_EQ: value = cmp == 0; break; case Py_NE: value = cmp != 0; break; case Py_GT: value = cmp > 0; break; case Py_GE: value = cmp >= 0; break; } result = value ? Py_True : Py_False; } Py_XINCREF(result); return result; } static PyObject * BlobHeader_repr(PyObject *selfObject) { BlobHeader *self = (BlobHeader *)selfObject; PyObject *member; PyObject *memberRepr; std::stringstream result; result << "BlobHeader("; result << "type="; member = BlobHeader_gettype(self, NULL); memberRepr = PyObject_Repr(member); result << PyString_AsString(memberRepr); Py_XDECREF(memberRepr); Py_XDECREF(member); result << ", "; result << "indexdata="; member = BlobHeader_getindexdata(self, NULL); memberRepr = PyObject_Repr(member); result << PyString_AsString(memberRepr); Py_XDECREF(memberRepr); Py_XDECREF(member); result << ", "; result << "datasize="; member = BlobHeader_getdatasize(self, NULL); memberRepr = PyObject_Repr(member); result << PyString_AsString(memberRepr); Py_XDECREF(memberRepr); Py_XDECREF(member); result << ")"; std::string resultString = result.str(); return PyUnicode_Decode(resultString.data(), resultString.length(), "utf-8", NULL); } PyMemberDef BlobHeader_members[] = { {NULL} // Sentinel }; PyGetSetDef BlobHeader_getsetters[] = { {(char *)"type", (getter)BlobHeader_gettype, (setter)BlobHeader_settype, (char *)"", NULL}, {(char *)"indexdata", (getter)BlobHeader_getindexdata, (setter)BlobHeader_setindexdata, (char *)"", NULL}, {(char *)"datasize", (getter)BlobHeader_getdatasize, (setter)BlobHeader_setdatasize, (char *)"", NULL}, {NULL} // Sentinel }; PyMethodDef BlobHeader_methods[] = { {"DebugString", (PyCFunction)BlobHeader_DebugString, METH_NOARGS, "Generates a human readable form of this message, useful for debugging and other purposes." }, {"SerializeToString", (PyCFunction)BlobHeader_SerializeToString, METH_NOARGS, "Serializes the protocol buffer to a string." }, {"SerializeMany", (PyCFunction)BlobHeader_SerializeMany, METH_O | METH_CLASS, "Serializes a sequence of protocol buffers to a string." }, {"ParseFromString", (PyCFunction)BlobHeader_ParseFromString, METH_O, "Parses the protocol buffer from a string." }, {"ParseFromLongString", (PyCFunction)BlobHeader_ParseFromLongString, METH_O, "Parses the protocol buffer from a string as large as 512MB." }, {"ParseMany", (PyCFunction)BlobHeader_ParseMany, METH_VARARGS | METH_CLASS, "Parses many protocol buffers of this type from a string." }, {NULL} // Sentinel }; PyTypeObject BlobHeaderType = { PyObject_HEAD_INIT(NULL) 0, /*ob_size*/ "OSMPBF.BlobHeader", /*tp_name*/ sizeof(BlobHeader), /*tp_basicsize*/ 0, /*tp_itemsize*/ (destructor)BlobHeader_dealloc, /*tp_dealloc*/ 0, /*tp_print*/ 0, /*tp_getattr*/ 0, /*tp_setattr*/ 0, /*tp_compare*/ BlobHeader_repr, /*tp_repr*/ 0, /*tp_as_number*/ 0, /*tp_as_sequence*/ 0, /*tp_as_mapping*/ 0, /*tp_hash */ 0, /*tp_call*/ 0, /*tp_str*/ 0, /*tp_getattro*/ 0, /*tp_setattro*/ 0, /*tp_as_buffer*/ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_RICHCOMPARE, /*tp_flags*/ "BlobHeader objects", /* tp_doc */ 0, /* tp_traverse */ 0, /* tp_clear */ BlobHeader_richcompare, /* tp_richcompare */ 0, /* tp_weaklistoffset */ 0, /* tp_iter */ 0, /* tp_iternext */ BlobHeader_methods, /* tp_methods */ BlobHeader_members, /* tp_members */ BlobHeader_getsetters, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ 0, /* tp_descr_set */ 0, /* tp_dictoffset */ (initproc)BlobHeader_init, /* tp_init */ 0, /* tp_alloc */ BlobHeader_new, /* tp_new */ }; } // Lets try not to pollute the global namespace namespace { // Forward-declaration for recursive structures extern PyTypeObject ChangeSetType; typedef struct { PyObject_HEAD OSMPBF::ChangeSet *protobuf; } ChangeSet; void ChangeSet_dealloc(ChangeSet* self) { delete self->protobuf; self->ob_type->tp_free((PyObject*)self); } PyObject * ChangeSet_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { ChangeSet *self; self = (ChangeSet *)type->tp_alloc(type, 0); self->protobuf = new OSMPBF::ChangeSet(); return (PyObject *)self; } PyObject * ChangeSet_DebugString(ChangeSet* self) { std::string result; Py_BEGIN_ALLOW_THREADS result = self->protobuf->Utf8DebugString(); Py_END_ALLOW_THREADS return PyUnicode_FromStringAndSize(result.data(), result.length()); } PyObject * ChangeSet_SerializeToString(ChangeSet* self) { std::string result; Py_BEGIN_ALLOW_THREADS self->protobuf->SerializeToString(&result); Py_END_ALLOW_THREADS return PyString_FromStringAndSize(result.data(), result.length()); } PyObject * ChangeSet_SerializeMany(void *nothing, PyObject *values) { std::string result; google::protobuf::io::ZeroCopyOutputStream* output = new google::protobuf::io::StringOutputStream(&result); google::protobuf::io::CodedOutputStream* outputStream = new google::protobuf::io::CodedOutputStream(output); PyObject *sequence = PySequence_Fast(values, "The values to serialize must be a sequence."); for (Py_ssize_t i = 0, len = PySequence_Length(sequence); i < len; ++i) { ChangeSet *value = (ChangeSet *)PySequence_Fast_GET_ITEM(sequence, i); Py_BEGIN_ALLOW_THREADS outputStream->WriteVarint32(value->protobuf->ByteSize()); value->protobuf->SerializeToCodedStream(outputStream); Py_END_ALLOW_THREADS } Py_XDECREF(sequence); delete outputStream; delete output; return PyString_FromStringAndSize(result.data(), result.length()); } PyObject * ChangeSet_ParseFromString(ChangeSet* self, PyObject *value) { std::string serialized(PyString_AsString(value), PyString_Size(value)); Py_BEGIN_ALLOW_THREADS self->protobuf->ParseFromString(serialized); Py_END_ALLOW_THREADS Py_RETURN_NONE; } PyObject * ChangeSet_ParseFromLongString(ChangeSet* self, PyObject *value) { google::protobuf::io::ZeroCopyInputStream* input = new google::protobuf::io::ArrayInputStream(PyString_AsString(value), PyString_Size(value)); google::protobuf::io::CodedInputStream* inputStream = new google::protobuf::io::CodedInputStream(input); inputStream->SetTotalBytesLimit(512 * 1024 * 1024, 512 * 1024 * 1024); Py_BEGIN_ALLOW_THREADS self->protobuf->ParseFromCodedStream(inputStream); Py_END_ALLOW_THREADS delete inputStream; delete input; Py_RETURN_NONE; } PyObject * ChangeSet_ParseMany(void* nothing, PyObject *args) { PyObject *value; PyObject *callback; int fail = 0; if (!PyArg_ParseTuple(args, "OO", &value, &callback)) { return NULL; } google::protobuf::io::ZeroCopyInputStream* input = new google::protobuf::io::ArrayInputStream(PyString_AsString(value), PyString_Size(value)); google::protobuf::io::CodedInputStream* inputStream = new google::protobuf::io::CodedInputStream(input); inputStream->SetTotalBytesLimit(512 * 1024 * 1024, 512 * 1024 * 1024); google::protobuf::uint32 bytes; PyObject *single = NULL; while (inputStream->ReadVarint32(&bytes)) { google::protobuf::io::CodedInputStream::Limit messageLimit = inputStream->PushLimit(bytes); if (single == NULL) { single = ChangeSet_new(&ChangeSetType, NULL, NULL); } Py_BEGIN_ALLOW_THREADS ((ChangeSet *)single)->protobuf->ParseFromCodedStream(inputStream); Py_END_ALLOW_THREADS inputStream->PopLimit(messageLimit); PyObject *result = PyObject_CallFunctionObjArgs(callback, single, NULL); if (result == NULL) { fail = 1; break; }; if (single->ob_refcnt != 1) { // If the callback saved a reference to the item, don't re-use it. Py_XDECREF(single); single = NULL; } } if (single != NULL) { Py_XDECREF(single); } delete inputStream; delete input; if (fail) { return NULL; } else { Py_RETURN_NONE; } } PyObject * ChangeSet_getid(ChangeSet *self, void *closure) { if (! self->protobuf->has_id()) { Py_RETURN_NONE; } return fastpb_convert3( self->protobuf->id()); } int ChangeSet_setid(ChangeSet *self, PyObject *input, void *closure) { if (input == NULL || input == Py_None) { self->protobuf->clear_id(); return 0; } PyObject *value = input; ::google::protobuf::int64 protoValue; // int64 if (PyInt_Check(value)) { protoValue = PyInt_AsLong(value); } else if (PyLong_Check(value)) { protoValue = PyLong_AsLongLong(value); } else { PyErr_SetString(PyExc_TypeError, "The id attribute value must be an integer"); return -1; } self->protobuf->set_id(protoValue); return 0; } int ChangeSet_init(ChangeSet *self, PyObject *args, PyObject *kwds) { PyObject *id = NULL; static char *kwlist[] = { (char *) "id", NULL }; if (! PyArg_ParseTupleAndKeywords( args, kwds, "|O", kwlist, &id)) return -1; if (id) { if (ChangeSet_setid(self, id, NULL) < 0) { return -1; } } return 0; } PyObject * ChangeSet_richcompare(PyObject *self, PyObject *other, int op) { PyObject *result = NULL; if (!PyType_IsSubtype(other->ob_type, &ChangeSetType)) { result = Py_NotImplemented; } else { // This is not a particularly efficient implementation since it never short circuits, but it's better // than nothing. It should probably only be used for tests. ChangeSet *selfValue = (ChangeSet *)self; ChangeSet *otherValue = (ChangeSet *)other; std::string selfSerialized; std::string otherSerialized; Py_BEGIN_ALLOW_THREADS selfValue->protobuf->SerializeToString(&selfSerialized); otherValue->protobuf->SerializeToString(&otherSerialized); Py_END_ALLOW_THREADS int cmp = selfSerialized.compare(otherSerialized); bool value = false; switch (op) { case Py_LT: value = cmp < 0; break; case Py_LE: value = cmp <= 0; break; case Py_EQ: value = cmp == 0; break; case Py_NE: value = cmp != 0; break; case Py_GT: value = cmp > 0; break; case Py_GE: value = cmp >= 0; break; } result = value ? Py_True : Py_False; } Py_XINCREF(result); return result; } static PyObject * ChangeSet_repr(PyObject *selfObject) { ChangeSet *self = (ChangeSet *)selfObject; PyObject *member; PyObject *memberRepr; std::stringstream result; result << "ChangeSet("; result << "id="; member = ChangeSet_getid(self, NULL); memberRepr = PyObject_Repr(member); result << PyString_AsString(memberRepr); Py_XDECREF(memberRepr); Py_XDECREF(member); result << ")"; std::string resultString = result.str(); return PyUnicode_Decode(resultString.data(), resultString.length(), "utf-8", NULL); } PyMemberDef ChangeSet_members[] = { {NULL} // Sentinel }; PyGetSetDef ChangeSet_getsetters[] = { {(char *)"id", (getter)ChangeSet_getid, (setter)ChangeSet_setid, (char *)"", NULL}, {NULL} // Sentinel }; PyMethodDef ChangeSet_methods[] = { {"DebugString", (PyCFunction)ChangeSet_DebugString, METH_NOARGS, "Generates a human readable form of this message, useful for debugging and other purposes." }, {"SerializeToString", (PyCFunction)ChangeSet_SerializeToString, METH_NOARGS, "Serializes the protocol buffer to a string." }, {"SerializeMany", (PyCFunction)ChangeSet_SerializeMany, METH_O | METH_CLASS, "Serializes a sequence of protocol buffers to a string." }, {"ParseFromString", (PyCFunction)ChangeSet_ParseFromString, METH_O, "Parses the protocol buffer from a string." }, {"ParseFromLongString", (PyCFunction)ChangeSet_ParseFromLongString, METH_O, "Parses the protocol buffer from a string as large as 512MB." }, {"ParseMany", (PyCFunction)ChangeSet_ParseMany, METH_VARARGS | METH_CLASS, "Parses many protocol buffers of this type from a string." }, {NULL} // Sentinel }; PyTypeObject ChangeSetType = { PyObject_HEAD_INIT(NULL) 0, /*ob_size*/ "OSMPBF.ChangeSet", /*tp_name*/ sizeof(ChangeSet), /*tp_basicsize*/ 0, /*tp_itemsize*/ (destructor)ChangeSet_dealloc, /*tp_dealloc*/ 0, /*tp_print*/ 0, /*tp_getattr*/ 0, /*tp_setattr*/ 0, /*tp_compare*/ ChangeSet_repr, /*tp_repr*/ 0, /*tp_as_number*/ 0, /*tp_as_sequence*/ 0, /*tp_as_mapping*/ 0, /*tp_hash */ 0, /*tp_call*/ 0, /*tp_str*/ 0, /*tp_getattro*/ 0, /*tp_setattro*/ 0, /*tp_as_buffer*/ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_RICHCOMPARE, /*tp_flags*/ "ChangeSet objects", /* tp_doc */ 0, /* tp_traverse */ 0, /* tp_clear */ ChangeSet_richcompare, /* tp_richcompare */ 0, /* tp_weaklistoffset */ 0, /* tp_iter */ 0, /* tp_iternext */ ChangeSet_methods, /* tp_methods */ ChangeSet_members, /* tp_members */ ChangeSet_getsetters, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ 0, /* tp_descr_set */ 0, /* tp_dictoffset */ (initproc)ChangeSet_init, /* tp_init */ 0, /* tp_alloc */ ChangeSet_new, /* tp_new */ }; } // Lets try not to pollute the global namespace namespace { // Forward-declaration for recursive structures extern PyTypeObject DenseInfoType; typedef struct { PyObject_HEAD OSMPBF::DenseInfo *protobuf; } DenseInfo; void DenseInfo_dealloc(DenseInfo* self) { delete self->protobuf; self->ob_type->tp_free((PyObject*)self); } PyObject * DenseInfo_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { DenseInfo *self; self = (DenseInfo *)type->tp_alloc(type, 0); self->protobuf = new OSMPBF::DenseInfo(); return (PyObject *)self; } PyObject * DenseInfo_DebugString(DenseInfo* self) { std::string result; Py_BEGIN_ALLOW_THREADS result = self->protobuf->Utf8DebugString(); Py_END_ALLOW_THREADS return PyUnicode_FromStringAndSize(result.data(), result.length()); } PyObject * DenseInfo_SerializeToString(DenseInfo* self) { std::string result; Py_BEGIN_ALLOW_THREADS self->protobuf->SerializeToString(&result); Py_END_ALLOW_THREADS return PyString_FromStringAndSize(result.data(), result.length()); } PyObject * DenseInfo_SerializeMany(void *nothing, PyObject *values) { std::string result; google::protobuf::io::ZeroCopyOutputStream* output = new google::protobuf::io::StringOutputStream(&result); google::protobuf::io::CodedOutputStream* outputStream = new google::protobuf::io::CodedOutputStream(output); PyObject *sequence = PySequence_Fast(values, "The values to serialize must be a sequence."); for (Py_ssize_t i = 0, len = PySequence_Length(sequence); i < len; ++i) { DenseInfo *value = (DenseInfo *)PySequence_Fast_GET_ITEM(sequence, i); Py_BEGIN_ALLOW_THREADS outputStream->WriteVarint32(value->protobuf->ByteSize()); value->protobuf->SerializeToCodedStream(outputStream); Py_END_ALLOW_THREADS } Py_XDECREF(sequence); delete outputStream; delete output; return PyString_FromStringAndSize(result.data(), result.length()); } PyObject * DenseInfo_ParseFromString(DenseInfo* self, PyObject *value) { std::string serialized(PyString_AsString(value), PyString_Size(value)); Py_BEGIN_ALLOW_THREADS self->protobuf->ParseFromString(serialized); Py_END_ALLOW_THREADS Py_RETURN_NONE; } PyObject * DenseInfo_ParseFromLongString(DenseInfo* self, PyObject *value) { google::protobuf::io::ZeroCopyInputStream* input = new google::protobuf::io::ArrayInputStream(PyString_AsString(value), PyString_Size(value)); google::protobuf::io::CodedInputStream* inputStream = new google::protobuf::io::CodedInputStream(input); inputStream->SetTotalBytesLimit(512 * 1024 * 1024, 512 * 1024 * 1024); Py_BEGIN_ALLOW_THREADS self->protobuf->ParseFromCodedStream(inputStream); Py_END_ALLOW_THREADS delete inputStream; delete input; Py_RETURN_NONE; } PyObject * DenseInfo_ParseMany(void* nothing, PyObject *args) { PyObject *value; PyObject *callback; int fail = 0; if (!PyArg_ParseTuple(args, "OO", &value, &callback)) { return NULL; } google::protobuf::io::ZeroCopyInputStream* input = new google::protobuf::io::ArrayInputStream(PyString_AsString(value), PyString_Size(value)); google::protobuf::io::CodedInputStream* inputStream = new google::protobuf::io::CodedInputStream(input); inputStream->SetTotalBytesLimit(512 * 1024 * 1024, 512 * 1024 * 1024); google::protobuf::uint32 bytes; PyObject *single = NULL; while (inputStream->ReadVarint32(&bytes)) { google::protobuf::io::CodedInputStream::Limit messageLimit = inputStream->PushLimit(bytes); if (single == NULL) { single = DenseInfo_new(&DenseInfoType, NULL, NULL); } Py_BEGIN_ALLOW_THREADS ((DenseInfo *)single)->protobuf->ParseFromCodedStream(inputStream); Py_END_ALLOW_THREADS inputStream->PopLimit(messageLimit); PyObject *result = PyObject_CallFunctionObjArgs(callback, single, NULL); if (result == NULL) { fail = 1; break; }; if (single->ob_refcnt != 1) { // If the callback saved a reference to the item, don't re-use it. Py_XDECREF(single); single = NULL; } } if (single != NULL) { Py_XDECREF(single); } delete inputStream; delete input; if (fail) { return NULL; } else { Py_RETURN_NONE; } } PyObject * DenseInfo_getversion(DenseInfo *self, void *closure) { int len = self->protobuf->version_size(); PyObject *tuple = PyTuple_New(len); for (int i = 0; i < len; ++i) { PyObject *value = fastpb_convert5( self->protobuf->version(i)); if (!value) { return NULL; } PyTuple_SetItem(tuple, i, value); } return tuple; } int DenseInfo_setversion(DenseInfo *self, PyObject *input, void *closure) { if (input == NULL || input == Py_None) { self->protobuf->clear_version(); return 0; } if (PyString_Check(input)) { PyErr_SetString(PyExc_TypeError, "The version attribute value must be a sequence"); return -1; } PyObject *sequence = PySequence_Fast(input, "The version attribute value must be a sequence"); self->protobuf->clear_version(); for (Py_ssize_t i = 0, len = PySequence_Length(sequence); i < len; ++i) { PyObject *value = PySequence_Fast_GET_ITEM(sequence, i); ::google::protobuf::int32 protoValue; // int32 if (PyInt_Check(value)) { protoValue = PyInt_AsLong(value); } else { PyErr_SetString(PyExc_TypeError, "The version attribute value must be an integer"); return -1; } self->protobuf->add_version(protoValue); } Py_XDECREF(sequence); return 0; } PyObject * DenseInfo_gettimestamp(DenseInfo *self, void *closure) { int len = self->protobuf->timestamp_size(); PyObject *tuple = PyTuple_New(len); for (int i = 0; i < len; ++i) { PyObject *value = fastpb_convert18( self->protobuf->timestamp(i)); if (!value) { return NULL; } PyTuple_SetItem(tuple, i, value); } return tuple; } int DenseInfo_settimestamp(DenseInfo *self, PyObject *input, void *closure) { if (input == NULL || input == Py_None) { self->protobuf->clear_timestamp(); return 0; } if (PyString_Check(input)) { PyErr_SetString(PyExc_TypeError, "The timestamp attribute value must be a sequence"); return -1; } PyObject *sequence = PySequence_Fast(input, "The timestamp attribute value must be a sequence"); self->protobuf->clear_timestamp(); for (Py_ssize_t i = 0, len = PySequence_Length(sequence); i < len; ++i) { PyObject *value = PySequence_Fast_GET_ITEM(sequence, i); ::google::protobuf::int64 protoValue; // int64 if (PyInt_Check(value)) { protoValue = PyInt_AsLong(value); } else if (PyLong_Check(value)) { protoValue = PyLong_AsLongLong(value); } else { PyErr_SetString(PyExc_TypeError, "The timestamp attribute value must be an integer"); return -1; } self->protobuf->add_timestamp(protoValue); } Py_XDECREF(sequence); return 0; } PyObject * DenseInfo_getchangeset(DenseInfo *self, void *closure) { int len = self->protobuf->changeset_size(); PyObject *tuple = PyTuple_New(len); for (int i = 0; i < len; ++i) { PyObject *value = fastpb_convert18( self->protobuf->changeset(i)); if (!value) { return NULL; } PyTuple_SetItem(tuple, i, value); } return tuple; } int DenseInfo_setchangeset(DenseInfo *self, PyObject *input, void *closure) { if (input == NULL || input == Py_None) { self->protobuf->clear_changeset(); return 0; } if (PyString_Check(input)) { PyErr_SetString(PyExc_TypeError, "The changeset attribute value must be a sequence"); return -1; } PyObject *sequence = PySequence_Fast(input, "The changeset attribute value must be a sequence"); self->protobuf->clear_changeset(); for (Py_ssize_t i = 0, len = PySequence_Length(sequence); i < len; ++i) { PyObject *value = PySequence_Fast_GET_ITEM(sequence, i); ::google::protobuf::int64 protoValue; // int64 if (PyInt_Check(value)) { protoValue = PyInt_AsLong(value); } else if (PyLong_Check(value)) { protoValue = PyLong_AsLongLong(value); } else { PyErr_SetString(PyExc_TypeError, "The changeset attribute value must be an integer"); return -1; } self->protobuf->add_changeset(protoValue); } Py_XDECREF(sequence); return 0; } PyObject * DenseInfo_getuid(DenseInfo *self, void *closure) { int len = self->protobuf->uid_size(); PyObject *tuple = PyTuple_New(len); for (int i = 0; i < len; ++i) { PyObject *value = fastpb_convert17( self->protobuf->uid(i)); if (!value) { return NULL; } PyTuple_SetItem(tuple, i, value); } return tuple; } int DenseInfo_setuid(DenseInfo *self, PyObject *input, void *closure) { if (input == NULL || input == Py_None) { self->protobuf->clear_uid(); return 0; } if (PyString_Check(input)) { PyErr_SetString(PyExc_TypeError, "The uid attribute value must be a sequence"); return -1; } PyObject *sequence = PySequence_Fast(input, "The uid attribute value must be a sequence"); self->protobuf->clear_uid(); for (Py_ssize_t i = 0, len = PySequence_Length(sequence); i < len; ++i) { PyObject *value = PySequence_Fast_GET_ITEM(sequence, i); ::google::protobuf::int32 protoValue; // int32 if (PyInt_Check(value)) { protoValue = PyInt_AsLong(value); } else { PyErr_SetString(PyExc_TypeError, "The uid attribute value must be an integer"); return -1; } self->protobuf->add_uid(protoValue); } Py_XDECREF(sequence); return 0; } PyObject * DenseInfo_getuser_sid(DenseInfo *self, void *closure) { int len = self->protobuf->user_sid_size(); PyObject *tuple = PyTuple_New(len); for (int i = 0; i < len; ++i) { PyObject *value = fastpb_convert17( self->protobuf->user_sid(i)); if (!value) { return NULL; } PyTuple_SetItem(tuple, i, value); } return tuple; } int DenseInfo_setuser_sid(DenseInfo *self, PyObject *input, void *closure) { if (input == NULL || input == Py_None) { self->protobuf->clear_user_sid(); return 0; } if (PyString_Check(input)) { PyErr_SetString(PyExc_TypeError, "The user_sid attribute value must be a sequence"); return -1; } PyObject *sequence = PySequence_Fast(input, "The user_sid attribute value must be a sequence"); self->protobuf->clear_user_sid(); for (Py_ssize_t i = 0, len = PySequence_Length(sequence); i < len; ++i) { PyObject *value = PySequence_Fast_GET_ITEM(sequence, i); ::google::protobuf::int32 protoValue; // int32 if (PyInt_Check(value)) { protoValue = PyInt_AsLong(value); } else { PyErr_SetString(PyExc_TypeError, "The user_sid attribute value must be an integer"); return -1; } self->protobuf->add_user_sid(protoValue); } Py_XDECREF(sequence); return 0; } int DenseInfo_init(DenseInfo *self, PyObject *args, PyObject *kwds) { PyObject *version = NULL; PyObject *timestamp = NULL; PyObject *changeset = NULL; PyObject *uid = NULL; PyObject *user_sid = NULL; static char *kwlist[] = { (char *) "version", (char *) "timestamp", (char *) "changeset", (char *) "uid", (char *) "user_sid", NULL }; if (! PyArg_ParseTupleAndKeywords( args, kwds, "|OOOOO", kwlist, &version,×tamp,&changeset,&uid,&user_sid)) return -1; if (version) { if (DenseInfo_setversion(self, version, NULL) < 0) { return -1; } } if (timestamp) { if (DenseInfo_settimestamp(self, timestamp, NULL) < 0) { return -1; } } if (changeset) { if (DenseInfo_setchangeset(self, changeset, NULL) < 0) { return -1; } } if (uid) { if (DenseInfo_setuid(self, uid, NULL) < 0) { return -1; } } if (user_sid) { if (DenseInfo_setuser_sid(self, user_sid, NULL) < 0) { return -1; } } return 0; } PyObject * DenseInfo_richcompare(PyObject *self, PyObject *other, int op) { PyObject *result = NULL; if (!PyType_IsSubtype(other->ob_type, &DenseInfoType)) { result = Py_NotImplemented; } else { // This is not a particularly efficient implementation since it never short circuits, but it's better // than nothing. It should probably only be used for tests. DenseInfo *selfValue = (DenseInfo *)self; DenseInfo *otherValue = (DenseInfo *)other; std::string selfSerialized; std::string otherSerialized; Py_BEGIN_ALLOW_THREADS selfValue->protobuf->SerializeToString(&selfSerialized); otherValue->protobuf->SerializeToString(&otherSerialized); Py_END_ALLOW_THREADS int cmp = selfSerialized.compare(otherSerialized); bool value = false; switch (op) { case Py_LT: value = cmp < 0; break; case Py_LE: value = cmp <= 0; break; case Py_EQ: value = cmp == 0; break; case Py_NE: value = cmp != 0; break; case Py_GT: value = cmp > 0; break; case Py_GE: value = cmp >= 0; break; } result = value ? Py_True : Py_False; } Py_XINCREF(result); return result; } static PyObject * DenseInfo_repr(PyObject *selfObject) { DenseInfo *self = (DenseInfo *)selfObject; PyObject *member; PyObject *memberRepr; std::stringstream result; result << "DenseInfo("; result << "version="; member = DenseInfo_getversion(self, NULL); memberRepr = PyObject_Repr(member); result << PyString_AsString(memberRepr); Py_XDECREF(memberRepr); Py_XDECREF(member); result << ", "; result << "timestamp="; member = DenseInfo_gettimestamp(self, NULL); memberRepr = PyObject_Repr(member); result << PyString_AsString(memberRepr); Py_XDECREF(memberRepr); Py_XDECREF(member); result << ", "; result << "changeset="; member = DenseInfo_getchangeset(self, NULL); memberRepr = PyObject_Repr(member); result << PyString_AsString(memberRepr); Py_XDECREF(memberRepr); Py_XDECREF(member); result << ", "; result << "uid="; member = DenseInfo_getuid(self, NULL); memberRepr = PyObject_Repr(member); result << PyString_AsString(memberRepr); Py_XDECREF(memberRepr); Py_XDECREF(member); result << ", "; result << "user_sid="; member = DenseInfo_getuser_sid(self, NULL); memberRepr = PyObject_Repr(member); result << PyString_AsString(memberRepr); Py_XDECREF(memberRepr); Py_XDECREF(member); result << ")"; std::string resultString = result.str(); return PyUnicode_Decode(resultString.data(), resultString.length(), "utf-8", NULL); } PyMemberDef DenseInfo_members[] = { {NULL} // Sentinel }; PyGetSetDef DenseInfo_getsetters[] = { {(char *)"version", (getter)DenseInfo_getversion, (setter)DenseInfo_setversion, (char *)"", NULL}, {(char *)"timestamp", (getter)DenseInfo_gettimestamp, (setter)DenseInfo_settimestamp, (char *)"", NULL}, {(char *)"changeset", (getter)DenseInfo_getchangeset, (setter)DenseInfo_setchangeset, (char *)"", NULL}, {(char *)"uid", (getter)DenseInfo_getuid, (setter)DenseInfo_setuid, (char *)"", NULL}, {(char *)"user_sid", (getter)DenseInfo_getuser_sid, (setter)DenseInfo_setuser_sid, (char *)"", NULL}, {NULL} // Sentinel }; PyMethodDef DenseInfo_methods[] = { {"DebugString", (PyCFunction)DenseInfo_DebugString, METH_NOARGS, "Generates a human readable form of this message, useful for debugging and other purposes." }, {"SerializeToString", (PyCFunction)DenseInfo_SerializeToString, METH_NOARGS, "Serializes the protocol buffer to a string." }, {"SerializeMany", (PyCFunction)DenseInfo_SerializeMany, METH_O | METH_CLASS, "Serializes a sequence of protocol buffers to a string." }, {"ParseFromString", (PyCFunction)DenseInfo_ParseFromString, METH_O, "Parses the protocol buffer from a string." }, {"ParseFromLongString", (PyCFunction)DenseInfo_ParseFromLongString, METH_O, "Parses the protocol buffer from a string as large as 512MB." }, {"ParseMany", (PyCFunction)DenseInfo_ParseMany, METH_VARARGS | METH_CLASS, "Parses many protocol buffers of this type from a string." }, {NULL} // Sentinel }; PyTypeObject DenseInfoType = { PyObject_HEAD_INIT(NULL) 0, /*ob_size*/ "OSMPBF.DenseInfo", /*tp_name*/ sizeof(DenseInfo), /*tp_basicsize*/ 0, /*tp_itemsize*/ (destructor)DenseInfo_dealloc, /*tp_dealloc*/ 0, /*tp_print*/ 0, /*tp_getattr*/ 0, /*tp_setattr*/ 0, /*tp_compare*/ DenseInfo_repr, /*tp_repr*/ 0, /*tp_as_number*/ 0, /*tp_as_sequence*/ 0, /*tp_as_mapping*/ 0, /*tp_hash */ 0, /*tp_call*/ 0, /*tp_str*/ 0, /*tp_getattro*/ 0, /*tp_setattro*/ 0, /*tp_as_buffer*/ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_RICHCOMPARE, /*tp_flags*/ "DenseInfo objects", /* tp_doc */ 0, /* tp_traverse */ 0, /* tp_clear */ DenseInfo_richcompare, /* tp_richcompare */ 0, /* tp_weaklistoffset */ 0, /* tp_iter */ 0, /* tp_iternext */ DenseInfo_methods, /* tp_methods */ DenseInfo_members, /* tp_members */ DenseInfo_getsetters, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ 0, /* tp_descr_set */ 0, /* tp_dictoffset */ (initproc)DenseInfo_init, /* tp_init */ 0, /* tp_alloc */ DenseInfo_new, /* tp_new */ }; } // Lets try not to pollute the global namespace namespace { // Forward-declaration for recursive structures extern PyTypeObject HeaderBBoxType; typedef struct { PyObject_HEAD OSMPBF::HeaderBBox *protobuf; } HeaderBBox; void HeaderBBox_dealloc(HeaderBBox* self) { delete self->protobuf; self->ob_type->tp_free((PyObject*)self); } PyObject * HeaderBBox_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { HeaderBBox *self; self = (HeaderBBox *)type->tp_alloc(type, 0); self->protobuf = new OSMPBF::HeaderBBox(); return (PyObject *)self; } PyObject * HeaderBBox_DebugString(HeaderBBox* self) { std::string result; Py_BEGIN_ALLOW_THREADS result = self->protobuf->Utf8DebugString(); Py_END_ALLOW_THREADS return PyUnicode_FromStringAndSize(result.data(), result.length()); } PyObject * HeaderBBox_SerializeToString(HeaderBBox* self) { std::string result; Py_BEGIN_ALLOW_THREADS self->protobuf->SerializeToString(&result); Py_END_ALLOW_THREADS return PyString_FromStringAndSize(result.data(), result.length()); } PyObject * HeaderBBox_SerializeMany(void *nothing, PyObject *values) { std::string result; google::protobuf::io::ZeroCopyOutputStream* output = new google::protobuf::io::StringOutputStream(&result); google::protobuf::io::CodedOutputStream* outputStream = new google::protobuf::io::CodedOutputStream(output); PyObject *sequence = PySequence_Fast(values, "The values to serialize must be a sequence."); for (Py_ssize_t i = 0, len = PySequence_Length(sequence); i < len; ++i) { HeaderBBox *value = (HeaderBBox *)PySequence_Fast_GET_ITEM(sequence, i); Py_BEGIN_ALLOW_THREADS outputStream->WriteVarint32(value->protobuf->ByteSize()); value->protobuf->SerializeToCodedStream(outputStream); Py_END_ALLOW_THREADS } Py_XDECREF(sequence); delete outputStream; delete output; return PyString_FromStringAndSize(result.data(), result.length()); } PyObject * HeaderBBox_ParseFromString(HeaderBBox* self, PyObject *value) { std::string serialized(PyString_AsString(value), PyString_Size(value)); Py_BEGIN_ALLOW_THREADS self->protobuf->ParseFromString(serialized); Py_END_ALLOW_THREADS Py_RETURN_NONE; } PyObject * HeaderBBox_ParseFromLongString(HeaderBBox* self, PyObject *value) { google::protobuf::io::ZeroCopyInputStream* input = new google::protobuf::io::ArrayInputStream(PyString_AsString(value), PyString_Size(value)); google::protobuf::io::CodedInputStream* inputStream = new google::protobuf::io::CodedInputStream(input); inputStream->SetTotalBytesLimit(512 * 1024 * 1024, 512 * 1024 * 1024); Py_BEGIN_ALLOW_THREADS self->protobuf->ParseFromCodedStream(inputStream); Py_END_ALLOW_THREADS delete inputStream; delete input; Py_RETURN_NONE; } PyObject * HeaderBBox_ParseMany(void* nothing, PyObject *args) { PyObject *value; PyObject *callback; int fail = 0; if (!PyArg_ParseTuple(args, "OO", &value, &callback)) { return NULL; } google::protobuf::io::ZeroCopyInputStream* input = new google::protobuf::io::ArrayInputStream(PyString_AsString(value), PyString_Size(value)); google::protobuf::io::CodedInputStream* inputStream = new google::protobuf::io::CodedInputStream(input); inputStream->SetTotalBytesLimit(512 * 1024 * 1024, 512 * 1024 * 1024); google::protobuf::uint32 bytes; PyObject *single = NULL; while (inputStream->ReadVarint32(&bytes)) { google::protobuf::io::CodedInputStream::Limit messageLimit = inputStream->PushLimit(bytes); if (single == NULL) { single = HeaderBBox_new(&HeaderBBoxType, NULL, NULL); } Py_BEGIN_ALLOW_THREADS ((HeaderBBox *)single)->protobuf->ParseFromCodedStream(inputStream); Py_END_ALLOW_THREADS inputStream->PopLimit(messageLimit); PyObject *result = PyObject_CallFunctionObjArgs(callback, single, NULL); if (result == NULL) { fail = 1; break; }; if (single->ob_refcnt != 1) { // If the callback saved a reference to the item, don't re-use it. Py_XDECREF(single); single = NULL; } } if (single != NULL) { Py_XDECREF(single); } delete inputStream; delete input; if (fail) { return NULL; } else { Py_RETURN_NONE; } } PyObject * HeaderBBox_getleft(HeaderBBox *self, void *closure) { if (! self->protobuf->has_left()) { Py_RETURN_NONE; } return fastpb_convert18( self->protobuf->left()); } int HeaderBBox_setleft(HeaderBBox *self, PyObject *input, void *closure) { if (input == NULL || input == Py_None) { self->protobuf->clear_left(); return 0; } PyObject *value = input; ::google::protobuf::int64 protoValue; // int64 if (PyInt_Check(value)) { protoValue = PyInt_AsLong(value); } else if (PyLong_Check(value)) { protoValue = PyLong_AsLongLong(value); } else { PyErr_SetString(PyExc_TypeError, "The left attribute value must be an integer"); return -1; } self->protobuf->set_left(protoValue); return 0; } PyObject * HeaderBBox_getright(HeaderBBox *self, void *closure) { if (! self->protobuf->has_right()) { Py_RETURN_NONE; } return fastpb_convert18( self->protobuf->right()); } int HeaderBBox_setright(HeaderBBox *self, PyObject *input, void *closure) { if (input == NULL || input == Py_None) { self->protobuf->clear_right(); return 0; } PyObject *value = input; ::google::protobuf::int64 protoValue; // int64 if (PyInt_Check(value)) { protoValue = PyInt_AsLong(value); } else if (PyLong_Check(value)) { protoValue = PyLong_AsLongLong(value); } else { PyErr_SetString(PyExc_TypeError, "The right attribute value must be an integer"); return -1; } self->protobuf->set_right(protoValue); return 0; } PyObject * HeaderBBox_gettop(HeaderBBox *self, void *closure) { if (! self->protobuf->has_top()) { Py_RETURN_NONE; } return fastpb_convert18( self->protobuf->top()); } int HeaderBBox_settop(HeaderBBox *self, PyObject *input, void *closure) { if (input == NULL || input == Py_None) { self->protobuf->clear_top(); return 0; } PyObject *value = input; ::google::protobuf::int64 protoValue; // int64 if (PyInt_Check(value)) { protoValue = PyInt_AsLong(value); } else if (PyLong_Check(value)) { protoValue = PyLong_AsLongLong(value); } else { PyErr_SetString(PyExc_TypeError, "The top attribute value must be an integer"); return -1; } self->protobuf->set_top(protoValue); return 0; } PyObject * HeaderBBox_getbottom(HeaderBBox *self, void *closure) { if (! self->protobuf->has_bottom()) { Py_RETURN_NONE; } return fastpb_convert18( self->protobuf->bottom()); } int HeaderBBox_setbottom(HeaderBBox *self, PyObject *input, void *closure) { if (input == NULL || input == Py_None) { self->protobuf->clear_bottom(); return 0; } PyObject *value = input; ::google::protobuf::int64 protoValue; // int64 if (PyInt_Check(value)) { protoValue = PyInt_AsLong(value); } else if (PyLong_Check(value)) { protoValue = PyLong_AsLongLong(value); } else { PyErr_SetString(PyExc_TypeError, "The bottom attribute value must be an integer"); return -1; } self->protobuf->set_bottom(protoValue); return 0; } int HeaderBBox_init(HeaderBBox *self, PyObject *args, PyObject *kwds) { PyObject *left = NULL; PyObject *right = NULL; PyObject *top = NULL; PyObject *bottom = NULL; static char *kwlist[] = { (char *) "left", (char *) "right", (char *) "top", (char *) "bottom", NULL }; if (! PyArg_ParseTupleAndKeywords( args, kwds, "|OOOO", kwlist, &left,&right,&top,&bottom)) return -1; if (left) { if (HeaderBBox_setleft(self, left, NULL) < 0) { return -1; } } if (right) { if (HeaderBBox_setright(self, right, NULL) < 0) { return -1; } } if (top) { if (HeaderBBox_settop(self, top, NULL) < 0) { return -1; } } if (bottom) { if (HeaderBBox_setbottom(self, bottom, NULL) < 0) { return -1; } } return 0; } PyObject * HeaderBBox_richcompare(PyObject *self, PyObject *other, int op) { PyObject *result = NULL; if (!PyType_IsSubtype(other->ob_type, &HeaderBBoxType)) { result = Py_NotImplemented; } else { // This is not a particularly efficient implementation since it never short circuits, but it's better // than nothing. It should probably only be used for tests. HeaderBBox *selfValue = (HeaderBBox *)self; HeaderBBox *otherValue = (HeaderBBox *)other; std::string selfSerialized; std::string otherSerialized; Py_BEGIN_ALLOW_THREADS selfValue->protobuf->SerializeToString(&selfSerialized); otherValue->protobuf->SerializeToString(&otherSerialized); Py_END_ALLOW_THREADS int cmp = selfSerialized.compare(otherSerialized); bool value = false; switch (op) { case Py_LT: value = cmp < 0; break; case Py_LE: value = cmp <= 0; break; case Py_EQ: value = cmp == 0; break; case Py_NE: value = cmp != 0; break; case Py_GT: value = cmp > 0; break; case Py_GE: value = cmp >= 0; break; } result = value ? Py_True : Py_False; } Py_XINCREF(result); return result; } static PyObject * HeaderBBox_repr(PyObject *selfObject) { HeaderBBox *self = (HeaderBBox *)selfObject; PyObject *member; PyObject *memberRepr; std::stringstream result; result << "HeaderBBox("; result << "left="; member = HeaderBBox_getleft(self, NULL); memberRepr = PyObject_Repr(member); result << PyString_AsString(memberRepr); Py_XDECREF(memberRepr); Py_XDECREF(member); result << ", "; result << "right="; member = HeaderBBox_getright(self, NULL); memberRepr = PyObject_Repr(member); result << PyString_AsString(memberRepr); Py_XDECREF(memberRepr); Py_XDECREF(member); result << ", "; result << "top="; member = HeaderBBox_gettop(self, NULL); memberRepr = PyObject_Repr(member); result << PyString_AsString(memberRepr); Py_XDECREF(memberRepr); Py_XDECREF(member); result << ", "; result << "bottom="; member = HeaderBBox_getbottom(self, NULL); memberRepr = PyObject_Repr(member); result << PyString_AsString(memberRepr); Py_XDECREF(memberRepr); Py_XDECREF(member); result << ")"; std::string resultString = result.str(); return PyUnicode_Decode(resultString.data(), resultString.length(), "utf-8", NULL); } PyMemberDef HeaderBBox_members[] = { {NULL} // Sentinel }; PyGetSetDef HeaderBBox_getsetters[] = { {(char *)"left", (getter)HeaderBBox_getleft, (setter)HeaderBBox_setleft, (char *)"", NULL}, {(char *)"right", (getter)HeaderBBox_getright, (setter)HeaderBBox_setright, (char *)"", NULL}, {(char *)"top", (getter)HeaderBBox_gettop, (setter)HeaderBBox_settop, (char *)"", NULL}, {(char *)"bottom", (getter)HeaderBBox_getbottom, (setter)HeaderBBox_setbottom, (char *)"", NULL}, {NULL} // Sentinel }; PyMethodDef HeaderBBox_methods[] = { {"DebugString", (PyCFunction)HeaderBBox_DebugString, METH_NOARGS, "Generates a human readable form of this message, useful for debugging and other purposes." }, {"SerializeToString", (PyCFunction)HeaderBBox_SerializeToString, METH_NOARGS, "Serializes the protocol buffer to a string." }, {"SerializeMany", (PyCFunction)HeaderBBox_SerializeMany, METH_O | METH_CLASS, "Serializes a sequence of protocol buffers to a string." }, {"ParseFromString", (PyCFunction)HeaderBBox_ParseFromString, METH_O, "Parses the protocol buffer from a string." }, {"ParseFromLongString", (PyCFunction)HeaderBBox_ParseFromLongString, METH_O, "Parses the protocol buffer from a string as large as 512MB." }, {"ParseMany", (PyCFunction)HeaderBBox_ParseMany, METH_VARARGS | METH_CLASS, "Parses many protocol buffers of this type from a string." }, {NULL} // Sentinel }; PyTypeObject HeaderBBoxType = { PyObject_HEAD_INIT(NULL) 0, /*ob_size*/ "OSMPBF.HeaderBBox", /*tp_name*/ sizeof(HeaderBBox), /*tp_basicsize*/ 0, /*tp_itemsize*/ (destructor)HeaderBBox_dealloc, /*tp_dealloc*/ 0, /*tp_print*/ 0, /*tp_getattr*/ 0, /*tp_setattr*/ 0, /*tp_compare*/ HeaderBBox_repr, /*tp_repr*/ 0, /*tp_as_number*/ 0, /*tp_as_sequence*/ 0, /*tp_as_mapping*/ 0, /*tp_hash */ 0, /*tp_call*/ 0, /*tp_str*/ 0, /*tp_getattro*/ 0, /*tp_setattro*/ 0, /*tp_as_buffer*/ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_RICHCOMPARE, /*tp_flags*/ "HeaderBBox objects", /* tp_doc */ 0, /* tp_traverse */ 0, /* tp_clear */ HeaderBBox_richcompare, /* tp_richcompare */ 0, /* tp_weaklistoffset */ 0, /* tp_iter */ 0, /* tp_iternext */ HeaderBBox_methods, /* tp_methods */ HeaderBBox_members, /* tp_members */ HeaderBBox_getsetters, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ 0, /* tp_descr_set */ 0, /* tp_dictoffset */ (initproc)HeaderBBox_init, /* tp_init */ 0, /* tp_alloc */ HeaderBBox_new, /* tp_new */ }; } // Lets try not to pollute the global namespace namespace { // Forward-declaration for recursive structures extern PyTypeObject InfoType; typedef struct { PyObject_HEAD OSMPBF::Info *protobuf; } Info; void Info_dealloc(Info* self) { delete self->protobuf; self->ob_type->tp_free((PyObject*)self); } PyObject * Info_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { Info *self; self = (Info *)type->tp_alloc(type, 0); self->protobuf = new OSMPBF::Info(); return (PyObject *)self; } PyObject * Info_DebugString(Info* self) { std::string result; Py_BEGIN_ALLOW_THREADS result = self->protobuf->Utf8DebugString(); Py_END_ALLOW_THREADS return PyUnicode_FromStringAndSize(result.data(), result.length()); } PyObject * Info_SerializeToString(Info* self) { std::string result; Py_BEGIN_ALLOW_THREADS self->protobuf->SerializeToString(&result); Py_END_ALLOW_THREADS return PyString_FromStringAndSize(result.data(), result.length()); } PyObject * Info_SerializeMany(void *nothing, PyObject *values) { std::string result; google::protobuf::io::ZeroCopyOutputStream* output = new google::protobuf::io::StringOutputStream(&result); google::protobuf::io::CodedOutputStream* outputStream = new google::protobuf::io::CodedOutputStream(output); PyObject *sequence = PySequence_Fast(values, "The values to serialize must be a sequence."); for (Py_ssize_t i = 0, len = PySequence_Length(sequence); i < len; ++i) { Info *value = (Info *)PySequence_Fast_GET_ITEM(sequence, i); Py_BEGIN_ALLOW_THREADS outputStream->WriteVarint32(value->protobuf->ByteSize()); value->protobuf->SerializeToCodedStream(outputStream); Py_END_ALLOW_THREADS } Py_XDECREF(sequence); delete outputStream; delete output; return PyString_FromStringAndSize(result.data(), result.length()); } PyObject * Info_ParseFromString(Info* self, PyObject *value) { std::string serialized(PyString_AsString(value), PyString_Size(value)); Py_BEGIN_ALLOW_THREADS self->protobuf->ParseFromString(serialized); Py_END_ALLOW_THREADS Py_RETURN_NONE; } PyObject * Info_ParseFromLongString(Info* self, PyObject *value) { google::protobuf::io::ZeroCopyInputStream* input = new google::protobuf::io::ArrayInputStream(PyString_AsString(value), PyString_Size(value)); google::protobuf::io::CodedInputStream* inputStream = new google::protobuf::io::CodedInputStream(input); inputStream->SetTotalBytesLimit(512 * 1024 * 1024, 512 * 1024 * 1024); Py_BEGIN_ALLOW_THREADS self->protobuf->ParseFromCodedStream(inputStream); Py_END_ALLOW_THREADS delete inputStream; delete input; Py_RETURN_NONE; } PyObject * Info_ParseMany(void* nothing, PyObject *args) { PyObject *value; PyObject *callback; int fail = 0; if (!PyArg_ParseTuple(args, "OO", &value, &callback)) { return NULL; } google::protobuf::io::ZeroCopyInputStream* input = new google::protobuf::io::ArrayInputStream(PyString_AsString(value), PyString_Size(value)); google::protobuf::io::CodedInputStream* inputStream = new google::protobuf::io::CodedInputStream(input); inputStream->SetTotalBytesLimit(512 * 1024 * 1024, 512 * 1024 * 1024); google::protobuf::uint32 bytes; PyObject *single = NULL; while (inputStream->ReadVarint32(&bytes)) { google::protobuf::io::CodedInputStream::Limit messageLimit = inputStream->PushLimit(bytes); if (single == NULL) { single = Info_new(&InfoType, NULL, NULL); } Py_BEGIN_ALLOW_THREADS ((Info *)single)->protobuf->ParseFromCodedStream(inputStream); Py_END_ALLOW_THREADS inputStream->PopLimit(messageLimit); PyObject *result = PyObject_CallFunctionObjArgs(callback, single, NULL); if (result == NULL) { fail = 1; break; }; if (single->ob_refcnt != 1) { // If the callback saved a reference to the item, don't re-use it. Py_XDECREF(single); single = NULL; } } if (single != NULL) { Py_XDECREF(single); } delete inputStream; delete input; if (fail) { return NULL; } else { Py_RETURN_NONE; } } PyObject * Info_getversion(Info *self, void *closure) { if (! self->protobuf->has_version()) { Py_RETURN_NONE; } return fastpb_convert5( self->protobuf->version()); } int Info_setversion(Info *self, PyObject *input, void *closure) { if (input == NULL || input == Py_None) { self->protobuf->clear_version(); return 0; } PyObject *value = input; ::google::protobuf::int32 protoValue; // int32 if (PyInt_Check(value)) { protoValue = PyInt_AsLong(value); } else { PyErr_SetString(PyExc_TypeError, "The version attribute value must be an integer"); return -1; } self->protobuf->set_version(protoValue); return 0; } PyObject * Info_gettimestamp(Info *self, void *closure) { if (! self->protobuf->has_timestamp()) { Py_RETURN_NONE; } return fastpb_convert3( self->protobuf->timestamp()); } int Info_settimestamp(Info *self, PyObject *input, void *closure) { if (input == NULL || input == Py_None) { self->protobuf->clear_timestamp(); return 0; } PyObject *value = input; ::google::protobuf::int64 protoValue; // int64 if (PyInt_Check(value)) { protoValue = PyInt_AsLong(value); } else if (PyLong_Check(value)) { protoValue = PyLong_AsLongLong(value); } else { PyErr_SetString(PyExc_TypeError, "The timestamp attribute value must be an integer"); return -1; } self->protobuf->set_timestamp(protoValue); return 0; } PyObject * Info_getchangeset(Info *self, void *closure) { if (! self->protobuf->has_changeset()) { Py_RETURN_NONE; } return fastpb_convert3( self->protobuf->changeset()); } int Info_setchangeset(Info *self, PyObject *input, void *closure) { if (input == NULL || input == Py_None) { self->protobuf->clear_changeset(); return 0; } PyObject *value = input; ::google::protobuf::int64 protoValue; // int64 if (PyInt_Check(value)) { protoValue = PyInt_AsLong(value); } else if (PyLong_Check(value)) { protoValue = PyLong_AsLongLong(value); } else { PyErr_SetString(PyExc_TypeError, "The changeset attribute value must be an integer"); return -1; } self->protobuf->set_changeset(protoValue); return 0; } PyObject * Info_getuid(Info *self, void *closure) { if (! self->protobuf->has_uid()) { Py_RETURN_NONE; } return fastpb_convert5( self->protobuf->uid()); } int Info_setuid(Info *self, PyObject *input, void *closure) { if (input == NULL || input == Py_None) { self->protobuf->clear_uid(); return 0; } PyObject *value = input; ::google::protobuf::int32 protoValue; // int32 if (PyInt_Check(value)) { protoValue = PyInt_AsLong(value); } else { PyErr_SetString(PyExc_TypeError, "The uid attribute value must be an integer"); return -1; } self->protobuf->set_uid(protoValue); return 0; } PyObject * Info_getuser_sid(Info *self, void *closure) { if (! self->protobuf->has_user_sid()) { Py_RETURN_NONE; } return fastpb_convert13( self->protobuf->user_sid()); } int Info_setuser_sid(Info *self, PyObject *input, void *closure) { if (input == NULL || input == Py_None) { self->protobuf->clear_user_sid(); return 0; } PyObject *value = input; ::google::protobuf::uint32 protoValue; // uint32 if (PyInt_Check(value)) { protoValue = PyInt_AsUnsignedLongMask(value); } else if (PyLong_Check(value)) { protoValue = PyLong_AsUnsignedLong(value); } else { PyErr_SetString(PyExc_TypeError, "The user_sid attribute value must be an integer"); return -1; } self->protobuf->set_user_sid(protoValue); return 0; } int Info_init(Info *self, PyObject *args, PyObject *kwds) { PyObject *version = NULL; PyObject *timestamp = NULL; PyObject *changeset = NULL; PyObject *uid = NULL; PyObject *user_sid = NULL; static char *kwlist[] = { (char *) "version", (char *) "timestamp", (char *) "changeset", (char *) "uid", (char *) "user_sid", NULL }; if (! PyArg_ParseTupleAndKeywords( args, kwds, "|OOOOO", kwlist, &version,×tamp,&changeset,&uid,&user_sid)) return -1; if (version) { if (Info_setversion(self, version, NULL) < 0) { return -1; } } if (timestamp) { if (Info_settimestamp(self, timestamp, NULL) < 0) { return -1; } } if (changeset) { if (Info_setchangeset(self, changeset, NULL) < 0) { return -1; } } if (uid) { if (Info_setuid(self, uid, NULL) < 0) { return -1; } } if (user_sid) { if (Info_setuser_sid(self, user_sid, NULL) < 0) { return -1; } } return 0; } PyObject * Info_richcompare(PyObject *self, PyObject *other, int op) { PyObject *result = NULL; if (!PyType_IsSubtype(other->ob_type, &InfoType)) { result = Py_NotImplemented; } else { // This is not a particularly efficient implementation since it never short circuits, but it's better // than nothing. It should probably only be used for tests. Info *selfValue = (Info *)self; Info *otherValue = (Info *)other; std::string selfSerialized; std::string otherSerialized; Py_BEGIN_ALLOW_THREADS selfValue->protobuf->SerializeToString(&selfSerialized); otherValue->protobuf->SerializeToString(&otherSerialized); Py_END_ALLOW_THREADS int cmp = selfSerialized.compare(otherSerialized); bool value = false; switch (op) { case Py_LT: value = cmp < 0; break; case Py_LE: value = cmp <= 0; break; case Py_EQ: value = cmp == 0; break; case Py_NE: value = cmp != 0; break; case Py_GT: value = cmp > 0; break; case Py_GE: value = cmp >= 0; break; } result = value ? Py_True : Py_False; } Py_XINCREF(result); return result; } static PyObject * Info_repr(PyObject *selfObject) { Info *self = (Info *)selfObject; PyObject *member; PyObject *memberRepr; std::stringstream result; result << "Info("; result << "version="; member = Info_getversion(self, NULL); memberRepr = PyObject_Repr(member); result << PyString_AsString(memberRepr); Py_XDECREF(memberRepr); Py_XDECREF(member); result << ", "; result << "timestamp="; member = Info_gettimestamp(self, NULL); memberRepr = PyObject_Repr(member); result << PyString_AsString(memberRepr); Py_XDECREF(memberRepr); Py_XDECREF(member); result << ", "; result << "changeset="; member = Info_getchangeset(self, NULL); memberRepr = PyObject_Repr(member); result << PyString_AsString(memberRepr); Py_XDECREF(memberRepr); Py_XDECREF(member); result << ", "; result << "uid="; member = Info_getuid(self, NULL); memberRepr = PyObject_Repr(member); result << PyString_AsString(memberRepr); Py_XDECREF(memberRepr); Py_XDECREF(member); result << ", "; result << "user_sid="; member = Info_getuser_sid(self, NULL); memberRepr = PyObject_Repr(member); result << PyString_AsString(memberRepr); Py_XDECREF(memberRepr); Py_XDECREF(member); result << ")"; std::string resultString = result.str(); return PyUnicode_Decode(resultString.data(), resultString.length(), "utf-8", NULL); } PyMemberDef Info_members[] = { {NULL} // Sentinel }; PyGetSetDef Info_getsetters[] = { {(char *)"version", (getter)Info_getversion, (setter)Info_setversion, (char *)"", NULL}, {(char *)"timestamp", (getter)Info_gettimestamp, (setter)Info_settimestamp, (char *)"", NULL}, {(char *)"changeset", (getter)Info_getchangeset, (setter)Info_setchangeset, (char *)"", NULL}, {(char *)"uid", (getter)Info_getuid, (setter)Info_setuid, (char *)"", NULL}, {(char *)"user_sid", (getter)Info_getuser_sid, (setter)Info_setuser_sid, (char *)"", NULL}, {NULL} // Sentinel }; PyMethodDef Info_methods[] = { {"DebugString", (PyCFunction)Info_DebugString, METH_NOARGS, "Generates a human readable form of this message, useful for debugging and other purposes." }, {"SerializeToString", (PyCFunction)Info_SerializeToString, METH_NOARGS, "Serializes the protocol buffer to a string." }, {"SerializeMany", (PyCFunction)Info_SerializeMany, METH_O | METH_CLASS, "Serializes a sequence of protocol buffers to a string." }, {"ParseFromString", (PyCFunction)Info_ParseFromString, METH_O, "Parses the protocol buffer from a string." }, {"ParseFromLongString", (PyCFunction)Info_ParseFromLongString, METH_O, "Parses the protocol buffer from a string as large as 512MB." }, {"ParseMany", (PyCFunction)Info_ParseMany, METH_VARARGS | METH_CLASS, "Parses many protocol buffers of this type from a string." }, {NULL} // Sentinel }; PyTypeObject InfoType = { PyObject_HEAD_INIT(NULL) 0, /*ob_size*/ "OSMPBF.Info", /*tp_name*/ sizeof(Info), /*tp_basicsize*/ 0, /*tp_itemsize*/ (destructor)Info_dealloc, /*tp_dealloc*/ 0, /*tp_print*/ 0, /*tp_getattr*/ 0, /*tp_setattr*/ 0, /*tp_compare*/ Info_repr, /*tp_repr*/ 0, /*tp_as_number*/ 0, /*tp_as_sequence*/ 0, /*tp_as_mapping*/ 0, /*tp_hash */ 0, /*tp_call*/ 0, /*tp_str*/ 0, /*tp_getattro*/ 0, /*tp_setattro*/ 0, /*tp_as_buffer*/ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_RICHCOMPARE, /*tp_flags*/ "Info objects", /* tp_doc */ 0, /* tp_traverse */ 0, /* tp_clear */ Info_richcompare, /* tp_richcompare */ 0, /* tp_weaklistoffset */ 0, /* tp_iter */ 0, /* tp_iternext */ Info_methods, /* tp_methods */ Info_members, /* tp_members */ Info_getsetters, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ 0, /* tp_descr_set */ 0, /* tp_dictoffset */ (initproc)Info_init, /* tp_init */ 0, /* tp_alloc */ Info_new, /* tp_new */ }; } // Lets try not to pollute the global namespace namespace { // Forward-declaration for recursive structures extern PyTypeObject StringTableType; typedef struct { PyObject_HEAD OSMPBF::StringTable *protobuf; } StringTable; void StringTable_dealloc(StringTable* self) { delete self->protobuf; self->ob_type->tp_free((PyObject*)self); } PyObject * StringTable_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { StringTable *self; self = (StringTable *)type->tp_alloc(type, 0); self->protobuf = new OSMPBF::StringTable(); return (PyObject *)self; } PyObject * StringTable_DebugString(StringTable* self) { std::string result; Py_BEGIN_ALLOW_THREADS result = self->protobuf->Utf8DebugString(); Py_END_ALLOW_THREADS return PyUnicode_FromStringAndSize(result.data(), result.length()); } PyObject * StringTable_SerializeToString(StringTable* self) { std::string result; Py_BEGIN_ALLOW_THREADS self->protobuf->SerializeToString(&result); Py_END_ALLOW_THREADS return PyString_FromStringAndSize(result.data(), result.length()); } PyObject * StringTable_SerializeMany(void *nothing, PyObject *values) { std::string result; google::protobuf::io::ZeroCopyOutputStream* output = new google::protobuf::io::StringOutputStream(&result); google::protobuf::io::CodedOutputStream* outputStream = new google::protobuf::io::CodedOutputStream(output); PyObject *sequence = PySequence_Fast(values, "The values to serialize must be a sequence."); for (Py_ssize_t i = 0, len = PySequence_Length(sequence); i < len; ++i) { StringTable *value = (StringTable *)PySequence_Fast_GET_ITEM(sequence, i); Py_BEGIN_ALLOW_THREADS outputStream->WriteVarint32(value->protobuf->ByteSize()); value->protobuf->SerializeToCodedStream(outputStream); Py_END_ALLOW_THREADS } Py_XDECREF(sequence); delete outputStream; delete output; return PyString_FromStringAndSize(result.data(), result.length()); } PyObject * StringTable_ParseFromString(StringTable* self, PyObject *value) { std::string serialized(PyString_AsString(value), PyString_Size(value)); Py_BEGIN_ALLOW_THREADS self->protobuf->ParseFromString(serialized); Py_END_ALLOW_THREADS Py_RETURN_NONE; } PyObject * StringTable_ParseFromLongString(StringTable* self, PyObject *value) { google::protobuf::io::ZeroCopyInputStream* input = new google::protobuf::io::ArrayInputStream(PyString_AsString(value), PyString_Size(value)); google::protobuf::io::CodedInputStream* inputStream = new google::protobuf::io::CodedInputStream(input); inputStream->SetTotalBytesLimit(512 * 1024 * 1024, 512 * 1024 * 1024); Py_BEGIN_ALLOW_THREADS self->protobuf->ParseFromCodedStream(inputStream); Py_END_ALLOW_THREADS delete inputStream; delete input; Py_RETURN_NONE; } PyObject * StringTable_ParseMany(void* nothing, PyObject *args) { PyObject *value; PyObject *callback; int fail = 0; if (!PyArg_ParseTuple(args, "OO", &value, &callback)) { return NULL; } google::protobuf::io::ZeroCopyInputStream* input = new google::protobuf::io::ArrayInputStream(PyString_AsString(value), PyString_Size(value)); google::protobuf::io::CodedInputStream* inputStream = new google::protobuf::io::CodedInputStream(input); inputStream->SetTotalBytesLimit(512 * 1024 * 1024, 512 * 1024 * 1024); google::protobuf::uint32 bytes; PyObject *single = NULL; while (inputStream->ReadVarint32(&bytes)) { google::protobuf::io::CodedInputStream::Limit messageLimit = inputStream->PushLimit(bytes); if (single == NULL) { single = StringTable_new(&StringTableType, NULL, NULL); } Py_BEGIN_ALLOW_THREADS ((StringTable *)single)->protobuf->ParseFromCodedStream(inputStream); Py_END_ALLOW_THREADS inputStream->PopLimit(messageLimit); PyObject *result = PyObject_CallFunctionObjArgs(callback, single, NULL); if (result == NULL) { fail = 1; break; }; if (single->ob_refcnt != 1) { // If the callback saved a reference to the item, don't re-use it. Py_XDECREF(single); single = NULL; } } if (single != NULL) { Py_XDECREF(single); } delete inputStream; delete input; if (fail) { return NULL; } else { Py_RETURN_NONE; } } PyObject * StringTable_gets(StringTable *self, void *closure) { int len = self->protobuf->s_size(); PyObject *tuple = PyTuple_New(len); for (int i = 0; i < len; ++i) { PyObject *value = fastpb_convert12( self->protobuf->s(i)); if (!value) { return NULL; } PyTuple_SetItem(tuple, i, value); } return tuple; } int StringTable_sets(StringTable *self, PyObject *input, void *closure) { if (input == NULL || input == Py_None) { self->protobuf->clear_s(); return 0; } if (PyString_Check(input)) { PyErr_SetString(PyExc_TypeError, "The s attribute value must be a sequence"); return -1; } PyObject *sequence = PySequence_Fast(input, "The s attribute value must be a sequence"); self->protobuf->clear_s(); for (Py_ssize_t i = 0, len = PySequence_Length(sequence); i < len; ++i) { PyObject *value = PySequence_Fast_GET_ITEM(sequence, i); // string if (! PyString_Check(value)) { PyErr_SetString(PyExc_TypeError, "The s attribute value must be a string"); return -1; } std::string protoValue(PyString_AsString(value), PyString_Size(value)); self->protobuf->add_s(protoValue); } Py_XDECREF(sequence); return 0; } int StringTable_init(StringTable *self, PyObject *args, PyObject *kwds) { PyObject *s = NULL; static char *kwlist[] = { (char *) "s", NULL }; if (! PyArg_ParseTupleAndKeywords( args, kwds, "|O", kwlist, &s)) return -1; if (s) { if (StringTable_sets(self, s, NULL) < 0) { return -1; } } return 0; } PyObject * StringTable_richcompare(PyObject *self, PyObject *other, int op) { PyObject *result = NULL; if (!PyType_IsSubtype(other->ob_type, &StringTableType)) { result = Py_NotImplemented; } else { // This is not a particularly efficient implementation since it never short circuits, but it's better // than nothing. It should probably only be used for tests. StringTable *selfValue = (StringTable *)self; StringTable *otherValue = (StringTable *)other; std::string selfSerialized; std::string otherSerialized; Py_BEGIN_ALLOW_THREADS selfValue->protobuf->SerializeToString(&selfSerialized); otherValue->protobuf->SerializeToString(&otherSerialized); Py_END_ALLOW_THREADS int cmp = selfSerialized.compare(otherSerialized); bool value = false; switch (op) { case Py_LT: value = cmp < 0; break; case Py_LE: value = cmp <= 0; break; case Py_EQ: value = cmp == 0; break; case Py_NE: value = cmp != 0; break; case Py_GT: value = cmp > 0; break; case Py_GE: value = cmp >= 0; break; } result = value ? Py_True : Py_False; } Py_XINCREF(result); return result; } static PyObject * StringTable_repr(PyObject *selfObject) { StringTable *self = (StringTable *)selfObject; PyObject *member; PyObject *memberRepr; std::stringstream result; result << "StringTable("; result << "s="; member = StringTable_gets(self, NULL); memberRepr = PyObject_Repr(member); result << PyString_AsString(memberRepr); Py_XDECREF(memberRepr); Py_XDECREF(member); result << ")"; std::string resultString = result.str(); return PyUnicode_Decode(resultString.data(), resultString.length(), "utf-8", NULL); } PyMemberDef StringTable_members[] = { {NULL} // Sentinel }; PyGetSetDef StringTable_getsetters[] = { {(char *)"s", (getter)StringTable_gets, (setter)StringTable_sets, (char *)"", NULL}, {NULL} // Sentinel }; PyMethodDef StringTable_methods[] = { {"DebugString", (PyCFunction)StringTable_DebugString, METH_NOARGS, "Generates a human readable form of this message, useful for debugging and other purposes." }, {"SerializeToString", (PyCFunction)StringTable_SerializeToString, METH_NOARGS, "Serializes the protocol buffer to a string." }, {"SerializeMany", (PyCFunction)StringTable_SerializeMany, METH_O | METH_CLASS, "Serializes a sequence of protocol buffers to a string." }, {"ParseFromString", (PyCFunction)StringTable_ParseFromString, METH_O, "Parses the protocol buffer from a string." }, {"ParseFromLongString", (PyCFunction)StringTable_ParseFromLongString, METH_O, "Parses the protocol buffer from a string as large as 512MB." }, {"ParseMany", (PyCFunction)StringTable_ParseMany, METH_VARARGS | METH_CLASS, "Parses many protocol buffers of this type from a string." }, {NULL} // Sentinel }; PyTypeObject StringTableType = { PyObject_HEAD_INIT(NULL) 0, /*ob_size*/ "OSMPBF.StringTable", /*tp_name*/ sizeof(StringTable), /*tp_basicsize*/ 0, /*tp_itemsize*/ (destructor)StringTable_dealloc, /*tp_dealloc*/ 0, /*tp_print*/ 0, /*tp_getattr*/ 0, /*tp_setattr*/ 0, /*tp_compare*/ StringTable_repr, /*tp_repr*/ 0, /*tp_as_number*/ 0, /*tp_as_sequence*/ 0, /*tp_as_mapping*/ 0, /*tp_hash */ 0, /*tp_call*/ 0, /*tp_str*/ 0, /*tp_getattro*/ 0, /*tp_setattro*/ 0, /*tp_as_buffer*/ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_RICHCOMPARE, /*tp_flags*/ "StringTable objects", /* tp_doc */ 0, /* tp_traverse */ 0, /* tp_clear */ StringTable_richcompare, /* tp_richcompare */ 0, /* tp_weaklistoffset */ 0, /* tp_iter */ 0, /* tp_iternext */ StringTable_methods, /* tp_methods */ StringTable_members, /* tp_members */ StringTable_getsetters, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ 0, /* tp_descr_set */ 0, /* tp_dictoffset */ (initproc)StringTable_init, /* tp_init */ 0, /* tp_alloc */ StringTable_new, /* tp_new */ }; } // Lets try not to pollute the global namespace namespace { // Forward-declaration for recursive structures extern PyTypeObject DenseNodesType; typedef struct { PyObject_HEAD OSMPBF::DenseNodes *protobuf; } DenseNodes; void DenseNodes_dealloc(DenseNodes* self) { delete self->protobuf; self->ob_type->tp_free((PyObject*)self); } PyObject * DenseNodes_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { DenseNodes *self; self = (DenseNodes *)type->tp_alloc(type, 0); self->protobuf = new OSMPBF::DenseNodes(); return (PyObject *)self; } PyObject * DenseNodes_DebugString(DenseNodes* self) { std::string result; Py_BEGIN_ALLOW_THREADS result = self->protobuf->Utf8DebugString(); Py_END_ALLOW_THREADS return PyUnicode_FromStringAndSize(result.data(), result.length()); } PyObject * DenseNodes_SerializeToString(DenseNodes* self) { std::string result; Py_BEGIN_ALLOW_THREADS self->protobuf->SerializeToString(&result); Py_END_ALLOW_THREADS return PyString_FromStringAndSize(result.data(), result.length()); } PyObject * DenseNodes_SerializeMany(void *nothing, PyObject *values) { std::string result; google::protobuf::io::ZeroCopyOutputStream* output = new google::protobuf::io::StringOutputStream(&result); google::protobuf::io::CodedOutputStream* outputStream = new google::protobuf::io::CodedOutputStream(output); PyObject *sequence = PySequence_Fast(values, "The values to serialize must be a sequence."); for (Py_ssize_t i = 0, len = PySequence_Length(sequence); i < len; ++i) { DenseNodes *value = (DenseNodes *)PySequence_Fast_GET_ITEM(sequence, i); Py_BEGIN_ALLOW_THREADS outputStream->WriteVarint32(value->protobuf->ByteSize()); value->protobuf->SerializeToCodedStream(outputStream); Py_END_ALLOW_THREADS } Py_XDECREF(sequence); delete outputStream; delete output; return PyString_FromStringAndSize(result.data(), result.length()); } PyObject * DenseNodes_ParseFromString(DenseNodes* self, PyObject *value) { std::string serialized(PyString_AsString(value), PyString_Size(value)); Py_BEGIN_ALLOW_THREADS self->protobuf->ParseFromString(serialized); Py_END_ALLOW_THREADS Py_RETURN_NONE; } PyObject * DenseNodes_ParseFromLongString(DenseNodes* self, PyObject *value) { google::protobuf::io::ZeroCopyInputStream* input = new google::protobuf::io::ArrayInputStream(PyString_AsString(value), PyString_Size(value)); google::protobuf::io::CodedInputStream* inputStream = new google::protobuf::io::CodedInputStream(input); inputStream->SetTotalBytesLimit(512 * 1024 * 1024, 512 * 1024 * 1024); Py_BEGIN_ALLOW_THREADS self->protobuf->ParseFromCodedStream(inputStream); Py_END_ALLOW_THREADS delete inputStream; delete input; Py_RETURN_NONE; } PyObject * DenseNodes_ParseMany(void* nothing, PyObject *args) { PyObject *value; PyObject *callback; int fail = 0; if (!PyArg_ParseTuple(args, "OO", &value, &callback)) { return NULL; } google::protobuf::io::ZeroCopyInputStream* input = new google::protobuf::io::ArrayInputStream(PyString_AsString(value), PyString_Size(value)); google::protobuf::io::CodedInputStream* inputStream = new google::protobuf::io::CodedInputStream(input); inputStream->SetTotalBytesLimit(512 * 1024 * 1024, 512 * 1024 * 1024); google::protobuf::uint32 bytes; PyObject *single = NULL; while (inputStream->ReadVarint32(&bytes)) { google::protobuf::io::CodedInputStream::Limit messageLimit = inputStream->PushLimit(bytes); if (single == NULL) { single = DenseNodes_new(&DenseNodesType, NULL, NULL); } Py_BEGIN_ALLOW_THREADS ((DenseNodes *)single)->protobuf->ParseFromCodedStream(inputStream); Py_END_ALLOW_THREADS inputStream->PopLimit(messageLimit); PyObject *result = PyObject_CallFunctionObjArgs(callback, single, NULL); if (result == NULL) { fail = 1; break; }; if (single->ob_refcnt != 1) { // If the callback saved a reference to the item, don't re-use it. Py_XDECREF(single); single = NULL; } } if (single != NULL) { Py_XDECREF(single); } delete inputStream; delete input; if (fail) { return NULL; } else { Py_RETURN_NONE; } } PyObject * DenseNodes_getid(DenseNodes *self, void *closure) { int len = self->protobuf->id_size(); PyObject *tuple = PyTuple_New(len); for (int i = 0; i < len; ++i) { PyObject *value = fastpb_convert18( self->protobuf->id(i)); if (!value) { return NULL; } PyTuple_SetItem(tuple, i, value); } return tuple; } int DenseNodes_setid(DenseNodes *self, PyObject *input, void *closure) { if (input == NULL || input == Py_None) { self->protobuf->clear_id(); return 0; } if (PyString_Check(input)) { PyErr_SetString(PyExc_TypeError, "The id attribute value must be a sequence"); return -1; } PyObject *sequence = PySequence_Fast(input, "The id attribute value must be a sequence"); self->protobuf->clear_id(); for (Py_ssize_t i = 0, len = PySequence_Length(sequence); i < len; ++i) { PyObject *value = PySequence_Fast_GET_ITEM(sequence, i); ::google::protobuf::int64 protoValue; // int64 if (PyInt_Check(value)) { protoValue = PyInt_AsLong(value); } else if (PyLong_Check(value)) { protoValue = PyLong_AsLongLong(value); } else { PyErr_SetString(PyExc_TypeError, "The id attribute value must be an integer"); return -1; } self->protobuf->add_id(protoValue); } Py_XDECREF(sequence); return 0; } PyObject * fastpb_convertDenseNodesdenseinfo(const ::google::protobuf::Message &value) { DenseInfo *obj = (DenseInfo *) DenseInfo_new(&DenseInfoType, NULL, NULL); obj->protobuf->MergeFrom(value); return (PyObject *)obj; } PyObject * DenseNodes_getdenseinfo(DenseNodes *self, void *closure) { if (! self->protobuf->has_denseinfo()) { Py_RETURN_NONE; } return fastpb_convertDenseNodesdenseinfo( self->protobuf->denseinfo()); } int DenseNodes_setdenseinfo(DenseNodes *self, PyObject *input, void *closure) { if (input == NULL || input == Py_None) { self->protobuf->clear_denseinfo(); return 0; } PyObject *value = input; if (!PyType_IsSubtype(value->ob_type, &DenseInfoType)) { PyErr_SetString(PyExc_TypeError, "The denseinfo attribute value must be an instance of DenseInfo"); return -1; } // .OSMPBF.DenseInfo ::OSMPBF::DenseInfo *protoValue = ((DenseInfo *) value)->protobuf; self->protobuf->clear_denseinfo(); self->protobuf->mutable_denseinfo()->MergeFrom(*protoValue); return 0; } PyObject * DenseNodes_getlat(DenseNodes *self, void *closure) { int len = self->protobuf->lat_size(); PyObject *tuple = PyTuple_New(len); for (int i = 0; i < len; ++i) { PyObject *value = fastpb_convert18( self->protobuf->lat(i)); if (!value) { return NULL; } PyTuple_SetItem(tuple, i, value); } return tuple; } int DenseNodes_setlat(DenseNodes *self, PyObject *input, void *closure) { if (input == NULL || input == Py_None) { self->protobuf->clear_lat(); return 0; } if (PyString_Check(input)) { PyErr_SetString(PyExc_TypeError, "The lat attribute value must be a sequence"); return -1; } PyObject *sequence = PySequence_Fast(input, "The lat attribute value must be a sequence"); self->protobuf->clear_lat(); for (Py_ssize_t i = 0, len = PySequence_Length(sequence); i < len; ++i) { PyObject *value = PySequence_Fast_GET_ITEM(sequence, i); ::google::protobuf::int64 protoValue; // int64 if (PyInt_Check(value)) { protoValue = PyInt_AsLong(value); } else if (PyLong_Check(value)) { protoValue = PyLong_AsLongLong(value); } else { PyErr_SetString(PyExc_TypeError, "The lat attribute value must be an integer"); return -1; } self->protobuf->add_lat(protoValue); } Py_XDECREF(sequence); return 0; } PyObject * DenseNodes_getlon(DenseNodes *self, void *closure) { int len = self->protobuf->lon_size(); PyObject *tuple = PyTuple_New(len); for (int i = 0; i < len; ++i) { PyObject *value = fastpb_convert18( self->protobuf->lon(i)); if (!value) { return NULL; } PyTuple_SetItem(tuple, i, value); } return tuple; } int DenseNodes_setlon(DenseNodes *self, PyObject *input, void *closure) { if (input == NULL || input == Py_None) { self->protobuf->clear_lon(); return 0; } if (PyString_Check(input)) { PyErr_SetString(PyExc_TypeError, "The lon attribute value must be a sequence"); return -1; } PyObject *sequence = PySequence_Fast(input, "The lon attribute value must be a sequence"); self->protobuf->clear_lon(); for (Py_ssize_t i = 0, len = PySequence_Length(sequence); i < len; ++i) { PyObject *value = PySequence_Fast_GET_ITEM(sequence, i); ::google::protobuf::int64 protoValue; // int64 if (PyInt_Check(value)) { protoValue = PyInt_AsLong(value); } else if (PyLong_Check(value)) { protoValue = PyLong_AsLongLong(value); } else { PyErr_SetString(PyExc_TypeError, "The lon attribute value must be an integer"); return -1; } self->protobuf->add_lon(protoValue); } Py_XDECREF(sequence); return 0; } PyObject * DenseNodes_getkeys_vals(DenseNodes *self, void *closure) { int len = self->protobuf->keys_vals_size(); PyObject *tuple = PyTuple_New(len); for (int i = 0; i < len; ++i) { PyObject *value = fastpb_convert5( self->protobuf->keys_vals(i)); if (!value) { return NULL; } PyTuple_SetItem(tuple, i, value); } return tuple; } int DenseNodes_setkeys_vals(DenseNodes *self, PyObject *input, void *closure) { if (input == NULL || input == Py_None) { self->protobuf->clear_keys_vals(); return 0; } if (PyString_Check(input)) { PyErr_SetString(PyExc_TypeError, "The keys_vals attribute value must be a sequence"); return -1; } PyObject *sequence = PySequence_Fast(input, "The keys_vals attribute value must be a sequence"); self->protobuf->clear_keys_vals(); for (Py_ssize_t i = 0, len = PySequence_Length(sequence); i < len; ++i) { PyObject *value = PySequence_Fast_GET_ITEM(sequence, i); ::google::protobuf::int32 protoValue; // int32 if (PyInt_Check(value)) { protoValue = PyInt_AsLong(value); } else { PyErr_SetString(PyExc_TypeError, "The keys_vals attribute value must be an integer"); return -1; } self->protobuf->add_keys_vals(protoValue); } Py_XDECREF(sequence); return 0; } int DenseNodes_init(DenseNodes *self, PyObject *args, PyObject *kwds) { PyObject *id = NULL; PyObject *denseinfo = NULL; PyObject *lat = NULL; PyObject *lon = NULL; PyObject *keys_vals = NULL; static char *kwlist[] = { (char *) "id", (char *) "denseinfo", (char *) "lat", (char *) "lon", (char *) "keys_vals", NULL }; if (! PyArg_ParseTupleAndKeywords( args, kwds, "|OOOOO", kwlist, &id,&denseinfo,&lat,&lon,&keys_vals)) return -1; if (id) { if (DenseNodes_setid(self, id, NULL) < 0) { return -1; } } if (denseinfo) { if (DenseNodes_setdenseinfo(self, denseinfo, NULL) < 0) { return -1; } } if (lat) { if (DenseNodes_setlat(self, lat, NULL) < 0) { return -1; } } if (lon) { if (DenseNodes_setlon(self, lon, NULL) < 0) { return -1; } } if (keys_vals) { if (DenseNodes_setkeys_vals(self, keys_vals, NULL) < 0) { return -1; } } return 0; } PyObject * DenseNodes_richcompare(PyObject *self, PyObject *other, int op) { PyObject *result = NULL; if (!PyType_IsSubtype(other->ob_type, &DenseNodesType)) { result = Py_NotImplemented; } else { // This is not a particularly efficient implementation since it never short circuits, but it's better // than nothing. It should probably only be used for tests. DenseNodes *selfValue = (DenseNodes *)self; DenseNodes *otherValue = (DenseNodes *)other; std::string selfSerialized; std::string otherSerialized; Py_BEGIN_ALLOW_THREADS selfValue->protobuf->SerializeToString(&selfSerialized); otherValue->protobuf->SerializeToString(&otherSerialized); Py_END_ALLOW_THREADS int cmp = selfSerialized.compare(otherSerialized); bool value = false; switch (op) { case Py_LT: value = cmp < 0; break; case Py_LE: value = cmp <= 0; break; case Py_EQ: value = cmp == 0; break; case Py_NE: value = cmp != 0; break; case Py_GT: value = cmp > 0; break; case Py_GE: value = cmp >= 0; break; } result = value ? Py_True : Py_False; } Py_XINCREF(result); return result; } static PyObject * DenseNodes_repr(PyObject *selfObject) { DenseNodes *self = (DenseNodes *)selfObject; PyObject *member; PyObject *memberRepr; std::stringstream result; result << "DenseNodes("; result << "id="; member = DenseNodes_getid(self, NULL); memberRepr = PyObject_Repr(member); result << PyString_AsString(memberRepr); Py_XDECREF(memberRepr); Py_XDECREF(member); result << ", "; result << "denseinfo="; member = DenseNodes_getdenseinfo(self, NULL); memberRepr = PyObject_Repr(member); result << PyString_AsString(memberRepr); Py_XDECREF(memberRepr); Py_XDECREF(member); result << ", "; result << "lat="; member = DenseNodes_getlat(self, NULL); memberRepr = PyObject_Repr(member); result << PyString_AsString(memberRepr); Py_XDECREF(memberRepr); Py_XDECREF(member); result << ", "; result << "lon="; member = DenseNodes_getlon(self, NULL); memberRepr = PyObject_Repr(member); result << PyString_AsString(memberRepr); Py_XDECREF(memberRepr); Py_XDECREF(member); result << ", "; result << "keys_vals="; member = DenseNodes_getkeys_vals(self, NULL); memberRepr = PyObject_Repr(member); result << PyString_AsString(memberRepr); Py_XDECREF(memberRepr); Py_XDECREF(member); result << ")"; std::string resultString = result.str(); return PyUnicode_Decode(resultString.data(), resultString.length(), "utf-8", NULL); } PyMemberDef DenseNodes_members[] = { {NULL} // Sentinel }; PyGetSetDef DenseNodes_getsetters[] = { {(char *)"id", (getter)DenseNodes_getid, (setter)DenseNodes_setid, (char *)"", NULL}, {(char *)"denseinfo", (getter)DenseNodes_getdenseinfo, (setter)DenseNodes_setdenseinfo, (char *)"", NULL}, {(char *)"lat", (getter)DenseNodes_getlat, (setter)DenseNodes_setlat, (char *)"", NULL}, {(char *)"lon", (getter)DenseNodes_getlon, (setter)DenseNodes_setlon, (char *)"", NULL}, {(char *)"keys_vals", (getter)DenseNodes_getkeys_vals, (setter)DenseNodes_setkeys_vals, (char *)"", NULL}, {NULL} // Sentinel }; PyMethodDef DenseNodes_methods[] = { {"DebugString", (PyCFunction)DenseNodes_DebugString, METH_NOARGS, "Generates a human readable form of this message, useful for debugging and other purposes." }, {"SerializeToString", (PyCFunction)DenseNodes_SerializeToString, METH_NOARGS, "Serializes the protocol buffer to a string." }, {"SerializeMany", (PyCFunction)DenseNodes_SerializeMany, METH_O | METH_CLASS, "Serializes a sequence of protocol buffers to a string." }, {"ParseFromString", (PyCFunction)DenseNodes_ParseFromString, METH_O, "Parses the protocol buffer from a string." }, {"ParseFromLongString", (PyCFunction)DenseNodes_ParseFromLongString, METH_O, "Parses the protocol buffer from a string as large as 512MB." }, {"ParseMany", (PyCFunction)DenseNodes_ParseMany, METH_VARARGS | METH_CLASS, "Parses many protocol buffers of this type from a string." }, {NULL} // Sentinel }; PyTypeObject DenseNodesType = { PyObject_HEAD_INIT(NULL) 0, /*ob_size*/ "OSMPBF.DenseNodes", /*tp_name*/ sizeof(DenseNodes), /*tp_basicsize*/ 0, /*tp_itemsize*/ (destructor)DenseNodes_dealloc, /*tp_dealloc*/ 0, /*tp_print*/ 0, /*tp_getattr*/ 0, /*tp_setattr*/ 0, /*tp_compare*/ DenseNodes_repr, /*tp_repr*/ 0, /*tp_as_number*/ 0, /*tp_as_sequence*/ 0, /*tp_as_mapping*/ 0, /*tp_hash */ 0, /*tp_call*/ 0, /*tp_str*/ 0, /*tp_getattro*/ 0, /*tp_setattro*/ 0, /*tp_as_buffer*/ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_RICHCOMPARE, /*tp_flags*/ "DenseNodes objects", /* tp_doc */ 0, /* tp_traverse */ 0, /* tp_clear */ DenseNodes_richcompare, /* tp_richcompare */ 0, /* tp_weaklistoffset */ 0, /* tp_iter */ 0, /* tp_iternext */ DenseNodes_methods, /* tp_methods */ DenseNodes_members, /* tp_members */ DenseNodes_getsetters, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ 0, /* tp_descr_set */ 0, /* tp_dictoffset */ (initproc)DenseNodes_init, /* tp_init */ 0, /* tp_alloc */ DenseNodes_new, /* tp_new */ }; } // Lets try not to pollute the global namespace namespace { // Forward-declaration for recursive structures extern PyTypeObject HeaderBlockType; typedef struct { PyObject_HEAD OSMPBF::HeaderBlock *protobuf; } HeaderBlock; void HeaderBlock_dealloc(HeaderBlock* self) { delete self->protobuf; self->ob_type->tp_free((PyObject*)self); } PyObject * HeaderBlock_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { HeaderBlock *self; self = (HeaderBlock *)type->tp_alloc(type, 0); self->protobuf = new OSMPBF::HeaderBlock(); return (PyObject *)self; } PyObject * HeaderBlock_DebugString(HeaderBlock* self) { std::string result; Py_BEGIN_ALLOW_THREADS result = self->protobuf->Utf8DebugString(); Py_END_ALLOW_THREADS return PyUnicode_FromStringAndSize(result.data(), result.length()); } PyObject * HeaderBlock_SerializeToString(HeaderBlock* self) { std::string result; Py_BEGIN_ALLOW_THREADS self->protobuf->SerializeToString(&result); Py_END_ALLOW_THREADS return PyString_FromStringAndSize(result.data(), result.length()); } PyObject * HeaderBlock_SerializeMany(void *nothing, PyObject *values) { std::string result; google::protobuf::io::ZeroCopyOutputStream* output = new google::protobuf::io::StringOutputStream(&result); google::protobuf::io::CodedOutputStream* outputStream = new google::protobuf::io::CodedOutputStream(output); PyObject *sequence = PySequence_Fast(values, "The values to serialize must be a sequence."); for (Py_ssize_t i = 0, len = PySequence_Length(sequence); i < len; ++i) { HeaderBlock *value = (HeaderBlock *)PySequence_Fast_GET_ITEM(sequence, i); Py_BEGIN_ALLOW_THREADS outputStream->WriteVarint32(value->protobuf->ByteSize()); value->protobuf->SerializeToCodedStream(outputStream); Py_END_ALLOW_THREADS } Py_XDECREF(sequence); delete outputStream; delete output; return PyString_FromStringAndSize(result.data(), result.length()); } PyObject * HeaderBlock_ParseFromString(HeaderBlock* self, PyObject *value) { std::string serialized(PyString_AsString(value), PyString_Size(value)); Py_BEGIN_ALLOW_THREADS self->protobuf->ParseFromString(serialized); Py_END_ALLOW_THREADS Py_RETURN_NONE; } PyObject * HeaderBlock_ParseFromLongString(HeaderBlock* self, PyObject *value) { google::protobuf::io::ZeroCopyInputStream* input = new google::protobuf::io::ArrayInputStream(PyString_AsString(value), PyString_Size(value)); google::protobuf::io::CodedInputStream* inputStream = new google::protobuf::io::CodedInputStream(input); inputStream->SetTotalBytesLimit(512 * 1024 * 1024, 512 * 1024 * 1024); Py_BEGIN_ALLOW_THREADS self->protobuf->ParseFromCodedStream(inputStream); Py_END_ALLOW_THREADS delete inputStream; delete input; Py_RETURN_NONE; } PyObject * HeaderBlock_ParseMany(void* nothing, PyObject *args) { PyObject *value; PyObject *callback; int fail = 0; if (!PyArg_ParseTuple(args, "OO", &value, &callback)) { return NULL; } google::protobuf::io::ZeroCopyInputStream* input = new google::protobuf::io::ArrayInputStream(PyString_AsString(value), PyString_Size(value)); google::protobuf::io::CodedInputStream* inputStream = new google::protobuf::io::CodedInputStream(input); inputStream->SetTotalBytesLimit(512 * 1024 * 1024, 512 * 1024 * 1024); google::protobuf::uint32 bytes; PyObject *single = NULL; while (inputStream->ReadVarint32(&bytes)) { google::protobuf::io::CodedInputStream::Limit messageLimit = inputStream->PushLimit(bytes); if (single == NULL) { single = HeaderBlock_new(&HeaderBlockType, NULL, NULL); } Py_BEGIN_ALLOW_THREADS ((HeaderBlock *)single)->protobuf->ParseFromCodedStream(inputStream); Py_END_ALLOW_THREADS inputStream->PopLimit(messageLimit); PyObject *result = PyObject_CallFunctionObjArgs(callback, single, NULL); if (result == NULL) { fail = 1; break; }; if (single->ob_refcnt != 1) { // If the callback saved a reference to the item, don't re-use it. Py_XDECREF(single); single = NULL; } } if (single != NULL) { Py_XDECREF(single); } delete inputStream; delete input; if (fail) { return NULL; } else { Py_RETURN_NONE; } } PyObject * fastpb_convertHeaderBlockbbox(const ::google::protobuf::Message &value) { HeaderBBox *obj = (HeaderBBox *) HeaderBBox_new(&HeaderBBoxType, NULL, NULL); obj->protobuf->MergeFrom(value); return (PyObject *)obj; } PyObject * HeaderBlock_getbbox(HeaderBlock *self, void *closure) { if (! self->protobuf->has_bbox()) { Py_RETURN_NONE; } return fastpb_convertHeaderBlockbbox( self->protobuf->bbox()); } int HeaderBlock_setbbox(HeaderBlock *self, PyObject *input, void *closure) { if (input == NULL || input == Py_None) { self->protobuf->clear_bbox(); return 0; } PyObject *value = input; if (!PyType_IsSubtype(value->ob_type, &HeaderBBoxType)) { PyErr_SetString(PyExc_TypeError, "The bbox attribute value must be an instance of HeaderBBox"); return -1; } // .OSMPBF.HeaderBBox ::OSMPBF::HeaderBBox *protoValue = ((HeaderBBox *) value)->protobuf; self->protobuf->clear_bbox(); self->protobuf->mutable_bbox()->MergeFrom(*protoValue); return 0; } PyObject * HeaderBlock_getrequired_features(HeaderBlock *self, void *closure) { int len = self->protobuf->required_features_size(); PyObject *tuple = PyTuple_New(len); for (int i = 0; i < len; ++i) { PyObject *value = fastpb_convert9( self->protobuf->required_features(i)); if (!value) { return NULL; } PyTuple_SetItem(tuple, i, value); } return tuple; } int HeaderBlock_setrequired_features(HeaderBlock *self, PyObject *input, void *closure) { if (input == NULL || input == Py_None) { self->protobuf->clear_required_features(); return 0; } if (PyString_Check(input)) { PyErr_SetString(PyExc_TypeError, "The required_features attribute value must be a sequence"); return -1; } PyObject *sequence = PySequence_Fast(input, "The required_features attribute value must be a sequence"); self->protobuf->clear_required_features(); for (Py_ssize_t i = 0, len = PySequence_Length(sequence); i < len; ++i) { PyObject *value = PySequence_Fast_GET_ITEM(sequence, i); // string bool reallocated = false; if (PyUnicode_Check(value)) { value = PyUnicode_AsEncodedString(value, "utf-8", NULL); reallocated = true; } if (! PyString_Check(value)) { PyErr_SetString(PyExc_TypeError, "The required_features attribute value must be a string"); return -1; } std::string protoValue(PyString_AsString(value), PyString_Size(value)); if (reallocated) { Py_XDECREF(value); } self->protobuf->add_required_features(protoValue); } Py_XDECREF(sequence); return 0; } PyObject * HeaderBlock_getoptional_features(HeaderBlock *self, void *closure) { int len = self->protobuf->optional_features_size(); PyObject *tuple = PyTuple_New(len); for (int i = 0; i < len; ++i) { PyObject *value = fastpb_convert9( self->protobuf->optional_features(i)); if (!value) { return NULL; } PyTuple_SetItem(tuple, i, value); } return tuple; } int HeaderBlock_setoptional_features(HeaderBlock *self, PyObject *input, void *closure) { if (input == NULL || input == Py_None) { self->protobuf->clear_optional_features(); return 0; } if (PyString_Check(input)) { PyErr_SetString(PyExc_TypeError, "The optional_features attribute value must be a sequence"); return -1; } PyObject *sequence = PySequence_Fast(input, "The optional_features attribute value must be a sequence"); self->protobuf->clear_optional_features(); for (Py_ssize_t i = 0, len = PySequence_Length(sequence); i < len; ++i) { PyObject *value = PySequence_Fast_GET_ITEM(sequence, i); // string bool reallocated = false; if (PyUnicode_Check(value)) { value = PyUnicode_AsEncodedString(value, "utf-8", NULL); reallocated = true; } if (! PyString_Check(value)) { PyErr_SetString(PyExc_TypeError, "The optional_features attribute value must be a string"); return -1; } std::string protoValue(PyString_AsString(value), PyString_Size(value)); if (reallocated) { Py_XDECREF(value); } self->protobuf->add_optional_features(protoValue); } Py_XDECREF(sequence); return 0; } PyObject * HeaderBlock_getwritingprogram(HeaderBlock *self, void *closure) { if (! self->protobuf->has_writingprogram()) { Py_RETURN_NONE; } return fastpb_convert9( self->protobuf->writingprogram()); } int HeaderBlock_setwritingprogram(HeaderBlock *self, PyObject *input, void *closure) { if (input == NULL || input == Py_None) { self->protobuf->clear_writingprogram(); return 0; } PyObject *value = input; // string bool reallocated = false; if (PyUnicode_Check(value)) { value = PyUnicode_AsEncodedString(value, "utf-8", NULL); reallocated = true; } if (! PyString_Check(value)) { PyErr_SetString(PyExc_TypeError, "The writingprogram attribute value must be a string"); return -1; } std::string protoValue(PyString_AsString(value), PyString_Size(value)); if (reallocated) { Py_XDECREF(value); } self->protobuf->set_writingprogram(protoValue); return 0; } PyObject * HeaderBlock_getsource(HeaderBlock *self, void *closure) { if (! self->protobuf->has_source()) { Py_RETURN_NONE; } return fastpb_convert9( self->protobuf->source()); } int HeaderBlock_setsource(HeaderBlock *self, PyObject *input, void *closure) { if (input == NULL || input == Py_None) { self->protobuf->clear_source(); return 0; } PyObject *value = input; // string bool reallocated = false; if (PyUnicode_Check(value)) { value = PyUnicode_AsEncodedString(value, "utf-8", NULL); reallocated = true; } if (! PyString_Check(value)) { PyErr_SetString(PyExc_TypeError, "The source attribute value must be a string"); return -1; } std::string protoValue(PyString_AsString(value), PyString_Size(value)); if (reallocated) { Py_XDECREF(value); } self->protobuf->set_source(protoValue); return 0; } int HeaderBlock_init(HeaderBlock *self, PyObject *args, PyObject *kwds) { PyObject *bbox = NULL; PyObject *required_features = NULL; PyObject *optional_features = NULL; PyObject *writingprogram = NULL; PyObject *source = NULL; static char *kwlist[] = { (char *) "bbox", (char *) "required_features", (char *) "optional_features", (char *) "writingprogram", (char *) "source", NULL }; if (! PyArg_ParseTupleAndKeywords( args, kwds, "|OOOOO", kwlist, &bbox,&required_features,&optional_features,&writingprogram,&source)) return -1; if (bbox) { if (HeaderBlock_setbbox(self, bbox, NULL) < 0) { return -1; } } if (required_features) { if (HeaderBlock_setrequired_features(self, required_features, NULL) < 0) { return -1; } } if (optional_features) { if (HeaderBlock_setoptional_features(self, optional_features, NULL) < 0) { return -1; } } if (writingprogram) { if (HeaderBlock_setwritingprogram(self, writingprogram, NULL) < 0) { return -1; } } if (source) { if (HeaderBlock_setsource(self, source, NULL) < 0) { return -1; } } return 0; } PyObject * HeaderBlock_richcompare(PyObject *self, PyObject *other, int op) { PyObject *result = NULL; if (!PyType_IsSubtype(other->ob_type, &HeaderBlockType)) { result = Py_NotImplemented; } else { // This is not a particularly efficient implementation since it never short circuits, but it's better // than nothing. It should probably only be used for tests. HeaderBlock *selfValue = (HeaderBlock *)self; HeaderBlock *otherValue = (HeaderBlock *)other; std::string selfSerialized; std::string otherSerialized; Py_BEGIN_ALLOW_THREADS selfValue->protobuf->SerializeToString(&selfSerialized); otherValue->protobuf->SerializeToString(&otherSerialized); Py_END_ALLOW_THREADS int cmp = selfSerialized.compare(otherSerialized); bool value = false; switch (op) { case Py_LT: value = cmp < 0; break; case Py_LE: value = cmp <= 0; break; case Py_EQ: value = cmp == 0; break; case Py_NE: value = cmp != 0; break; case Py_GT: value = cmp > 0; break; case Py_GE: value = cmp >= 0; break; } result = value ? Py_True : Py_False; } Py_XINCREF(result); return result; } static PyObject * HeaderBlock_repr(PyObject *selfObject) { HeaderBlock *self = (HeaderBlock *)selfObject; PyObject *member; PyObject *memberRepr; std::stringstream result; result << "HeaderBlock("; result << "bbox="; member = HeaderBlock_getbbox(self, NULL); memberRepr = PyObject_Repr(member); result << PyString_AsString(memberRepr); Py_XDECREF(memberRepr); Py_XDECREF(member); result << ", "; result << "required_features="; member = HeaderBlock_getrequired_features(self, NULL); memberRepr = PyObject_Repr(member); result << PyString_AsString(memberRepr); Py_XDECREF(memberRepr); Py_XDECREF(member); result << ", "; result << "optional_features="; member = HeaderBlock_getoptional_features(self, NULL); memberRepr = PyObject_Repr(member); result << PyString_AsString(memberRepr); Py_XDECREF(memberRepr); Py_XDECREF(member); result << ", "; result << "writingprogram="; member = HeaderBlock_getwritingprogram(self, NULL); memberRepr = PyObject_Repr(member); result << PyString_AsString(memberRepr); Py_XDECREF(memberRepr); Py_XDECREF(member); result << ", "; result << "source="; member = HeaderBlock_getsource(self, NULL); memberRepr = PyObject_Repr(member); result << PyString_AsString(memberRepr); Py_XDECREF(memberRepr); Py_XDECREF(member); result << ")"; std::string resultString = result.str(); return PyUnicode_Decode(resultString.data(), resultString.length(), "utf-8", NULL); } PyMemberDef HeaderBlock_members[] = { {NULL} // Sentinel }; PyGetSetDef HeaderBlock_getsetters[] = { {(char *)"bbox", (getter)HeaderBlock_getbbox, (setter)HeaderBlock_setbbox, (char *)"", NULL}, {(char *)"required_features", (getter)HeaderBlock_getrequired_features, (setter)HeaderBlock_setrequired_features, (char *)"", NULL}, {(char *)"optional_features", (getter)HeaderBlock_getoptional_features, (setter)HeaderBlock_setoptional_features, (char *)"", NULL}, {(char *)"writingprogram", (getter)HeaderBlock_getwritingprogram, (setter)HeaderBlock_setwritingprogram, (char *)"", NULL}, {(char *)"source", (getter)HeaderBlock_getsource, (setter)HeaderBlock_setsource, (char *)"", NULL}, {NULL} // Sentinel }; PyMethodDef HeaderBlock_methods[] = { {"DebugString", (PyCFunction)HeaderBlock_DebugString, METH_NOARGS, "Generates a human readable form of this message, useful for debugging and other purposes." }, {"SerializeToString", (PyCFunction)HeaderBlock_SerializeToString, METH_NOARGS, "Serializes the protocol buffer to a string." }, {"SerializeMany", (PyCFunction)HeaderBlock_SerializeMany, METH_O | METH_CLASS, "Serializes a sequence of protocol buffers to a string." }, {"ParseFromString", (PyCFunction)HeaderBlock_ParseFromString, METH_O, "Parses the protocol buffer from a string." }, {"ParseFromLongString", (PyCFunction)HeaderBlock_ParseFromLongString, METH_O, "Parses the protocol buffer from a string as large as 512MB." }, {"ParseMany", (PyCFunction)HeaderBlock_ParseMany, METH_VARARGS | METH_CLASS, "Parses many protocol buffers of this type from a string." }, {NULL} // Sentinel }; PyTypeObject HeaderBlockType = { PyObject_HEAD_INIT(NULL) 0, /*ob_size*/ "OSMPBF.HeaderBlock", /*tp_name*/ sizeof(HeaderBlock), /*tp_basicsize*/ 0, /*tp_itemsize*/ (destructor)HeaderBlock_dealloc, /*tp_dealloc*/ 0, /*tp_print*/ 0, /*tp_getattr*/ 0, /*tp_setattr*/ 0, /*tp_compare*/ HeaderBlock_repr, /*tp_repr*/ 0, /*tp_as_number*/ 0, /*tp_as_sequence*/ 0, /*tp_as_mapping*/ 0, /*tp_hash */ 0, /*tp_call*/ 0, /*tp_str*/ 0, /*tp_getattro*/ 0, /*tp_setattro*/ 0, /*tp_as_buffer*/ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_RICHCOMPARE, /*tp_flags*/ "HeaderBlock objects", /* tp_doc */ 0, /* tp_traverse */ 0, /* tp_clear */ HeaderBlock_richcompare, /* tp_richcompare */ 0, /* tp_weaklistoffset */ 0, /* tp_iter */ 0, /* tp_iternext */ HeaderBlock_methods, /* tp_methods */ HeaderBlock_members, /* tp_members */ HeaderBlock_getsetters, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ 0, /* tp_descr_set */ 0, /* tp_dictoffset */ (initproc)HeaderBlock_init, /* tp_init */ 0, /* tp_alloc */ HeaderBlock_new, /* tp_new */ }; } // Lets try not to pollute the global namespace namespace { // Forward-declaration for recursive structures extern PyTypeObject NodeType; typedef struct { PyObject_HEAD OSMPBF::Node *protobuf; } Node; void Node_dealloc(Node* self) { delete self->protobuf; self->ob_type->tp_free((PyObject*)self); } PyObject * Node_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { Node *self; self = (Node *)type->tp_alloc(type, 0); self->protobuf = new OSMPBF::Node(); return (PyObject *)self; } PyObject * Node_DebugString(Node* self) { std::string result; Py_BEGIN_ALLOW_THREADS result = self->protobuf->Utf8DebugString(); Py_END_ALLOW_THREADS return PyUnicode_FromStringAndSize(result.data(), result.length()); } PyObject * Node_SerializeToString(Node* self) { std::string result; Py_BEGIN_ALLOW_THREADS self->protobuf->SerializeToString(&result); Py_END_ALLOW_THREADS return PyString_FromStringAndSize(result.data(), result.length()); } PyObject * Node_SerializeMany(void *nothing, PyObject *values) { std::string result; google::protobuf::io::ZeroCopyOutputStream* output = new google::protobuf::io::StringOutputStream(&result); google::protobuf::io::CodedOutputStream* outputStream = new google::protobuf::io::CodedOutputStream(output); PyObject *sequence = PySequence_Fast(values, "The values to serialize must be a sequence."); for (Py_ssize_t i = 0, len = PySequence_Length(sequence); i < len; ++i) { Node *value = (Node *)PySequence_Fast_GET_ITEM(sequence, i); Py_BEGIN_ALLOW_THREADS outputStream->WriteVarint32(value->protobuf->ByteSize()); value->protobuf->SerializeToCodedStream(outputStream); Py_END_ALLOW_THREADS } Py_XDECREF(sequence); delete outputStream; delete output; return PyString_FromStringAndSize(result.data(), result.length()); } PyObject * Node_ParseFromString(Node* self, PyObject *value) { std::string serialized(PyString_AsString(value), PyString_Size(value)); Py_BEGIN_ALLOW_THREADS self->protobuf->ParseFromString(serialized); Py_END_ALLOW_THREADS Py_RETURN_NONE; } PyObject * Node_ParseFromLongString(Node* self, PyObject *value) { google::protobuf::io::ZeroCopyInputStream* input = new google::protobuf::io::ArrayInputStream(PyString_AsString(value), PyString_Size(value)); google::protobuf::io::CodedInputStream* inputStream = new google::protobuf::io::CodedInputStream(input); inputStream->SetTotalBytesLimit(512 * 1024 * 1024, 512 * 1024 * 1024); Py_BEGIN_ALLOW_THREADS self->protobuf->ParseFromCodedStream(inputStream); Py_END_ALLOW_THREADS delete inputStream; delete input; Py_RETURN_NONE; } PyObject * Node_ParseMany(void* nothing, PyObject *args) { PyObject *value; PyObject *callback; int fail = 0; if (!PyArg_ParseTuple(args, "OO", &value, &callback)) { return NULL; } google::protobuf::io::ZeroCopyInputStream* input = new google::protobuf::io::ArrayInputStream(PyString_AsString(value), PyString_Size(value)); google::protobuf::io::CodedInputStream* inputStream = new google::protobuf::io::CodedInputStream(input); inputStream->SetTotalBytesLimit(512 * 1024 * 1024, 512 * 1024 * 1024); google::protobuf::uint32 bytes; PyObject *single = NULL; while (inputStream->ReadVarint32(&bytes)) { google::protobuf::io::CodedInputStream::Limit messageLimit = inputStream->PushLimit(bytes); if (single == NULL) { single = Node_new(&NodeType, NULL, NULL); } Py_BEGIN_ALLOW_THREADS ((Node *)single)->protobuf->ParseFromCodedStream(inputStream); Py_END_ALLOW_THREADS inputStream->PopLimit(messageLimit); PyObject *result = PyObject_CallFunctionObjArgs(callback, single, NULL); if (result == NULL) { fail = 1; break; }; if (single->ob_refcnt != 1) { // If the callback saved a reference to the item, don't re-use it. Py_XDECREF(single); single = NULL; } } if (single != NULL) { Py_XDECREF(single); } delete inputStream; delete input; if (fail) { return NULL; } else { Py_RETURN_NONE; } } PyObject * Node_getid(Node *self, void *closure) { if (! self->protobuf->has_id()) { Py_RETURN_NONE; } return fastpb_convert18( self->protobuf->id()); } int Node_setid(Node *self, PyObject *input, void *closure) { if (input == NULL || input == Py_None) { self->protobuf->clear_id(); return 0; } PyObject *value = input; ::google::protobuf::int64 protoValue; // int64 if (PyInt_Check(value)) { protoValue = PyInt_AsLong(value); } else if (PyLong_Check(value)) { protoValue = PyLong_AsLongLong(value); } else { PyErr_SetString(PyExc_TypeError, "The id attribute value must be an integer"); return -1; } self->protobuf->set_id(protoValue); return 0; } PyObject * Node_getkeys(Node *self, void *closure) { int len = self->protobuf->keys_size(); PyObject *tuple = PyTuple_New(len); for (int i = 0; i < len; ++i) { PyObject *value = fastpb_convert13( self->protobuf->keys(i)); if (!value) { return NULL; } PyTuple_SetItem(tuple, i, value); } return tuple; } int Node_setkeys(Node *self, PyObject *input, void *closure) { if (input == NULL || input == Py_None) { self->protobuf->clear_keys(); return 0; } if (PyString_Check(input)) { PyErr_SetString(PyExc_TypeError, "The keys attribute value must be a sequence"); return -1; } PyObject *sequence = PySequence_Fast(input, "The keys attribute value must be a sequence"); self->protobuf->clear_keys(); for (Py_ssize_t i = 0, len = PySequence_Length(sequence); i < len; ++i) { PyObject *value = PySequence_Fast_GET_ITEM(sequence, i); ::google::protobuf::uint32 protoValue; // uint32 if (PyInt_Check(value)) { protoValue = PyInt_AsUnsignedLongMask(value); } else if (PyLong_Check(value)) { protoValue = PyLong_AsUnsignedLong(value); } else { PyErr_SetString(PyExc_TypeError, "The keys attribute value must be an integer"); return -1; } self->protobuf->add_keys(protoValue); } Py_XDECREF(sequence); return 0; } PyObject * Node_getvals(Node *self, void *closure) { int len = self->protobuf->vals_size(); PyObject *tuple = PyTuple_New(len); for (int i = 0; i < len; ++i) { PyObject *value = fastpb_convert13( self->protobuf->vals(i)); if (!value) { return NULL; } PyTuple_SetItem(tuple, i, value); } return tuple; } int Node_setvals(Node *self, PyObject *input, void *closure) { if (input == NULL || input == Py_None) { self->protobuf->clear_vals(); return 0; } if (PyString_Check(input)) { PyErr_SetString(PyExc_TypeError, "The vals attribute value must be a sequence"); return -1; } PyObject *sequence = PySequence_Fast(input, "The vals attribute value must be a sequence"); self->protobuf->clear_vals(); for (Py_ssize_t i = 0, len = PySequence_Length(sequence); i < len; ++i) { PyObject *value = PySequence_Fast_GET_ITEM(sequence, i); ::google::protobuf::uint32 protoValue; // uint32 if (PyInt_Check(value)) { protoValue = PyInt_AsUnsignedLongMask(value); } else if (PyLong_Check(value)) { protoValue = PyLong_AsUnsignedLong(value); } else { PyErr_SetString(PyExc_TypeError, "The vals attribute value must be an integer"); return -1; } self->protobuf->add_vals(protoValue); } Py_XDECREF(sequence); return 0; } PyObject * fastpb_convertNodeinfo(const ::google::protobuf::Message &value) { Info *obj = (Info *) Info_new(&InfoType, NULL, NULL); obj->protobuf->MergeFrom(value); return (PyObject *)obj; } PyObject * Node_getinfo(Node *self, void *closure) { if (! self->protobuf->has_info()) { Py_RETURN_NONE; } return fastpb_convertNodeinfo( self->protobuf->info()); } int Node_setinfo(Node *self, PyObject *input, void *closure) { if (input == NULL || input == Py_None) { self->protobuf->clear_info(); return 0; } PyObject *value = input; if (!PyType_IsSubtype(value->ob_type, &InfoType)) { PyErr_SetString(PyExc_TypeError, "The info attribute value must be an instance of Info"); return -1; } // .OSMPBF.Info ::OSMPBF::Info *protoValue = ((Info *) value)->protobuf; self->protobuf->clear_info(); self->protobuf->mutable_info()->MergeFrom(*protoValue); return 0; } PyObject * Node_getlat(Node *self, void *closure) { if (! self->protobuf->has_lat()) { Py_RETURN_NONE; } return fastpb_convert18( self->protobuf->lat()); } int Node_setlat(Node *self, PyObject *input, void *closure) { if (input == NULL || input == Py_None) { self->protobuf->clear_lat(); return 0; } PyObject *value = input; ::google::protobuf::int64 protoValue; // int64 if (PyInt_Check(value)) { protoValue = PyInt_AsLong(value); } else if (PyLong_Check(value)) { protoValue = PyLong_AsLongLong(value); } else { PyErr_SetString(PyExc_TypeError, "The lat attribute value must be an integer"); return -1; } self->protobuf->set_lat(protoValue); return 0; } PyObject * Node_getlon(Node *self, void *closure) { if (! self->protobuf->has_lon()) { Py_RETURN_NONE; } return fastpb_convert18( self->protobuf->lon()); } int Node_setlon(Node *self, PyObject *input, void *closure) { if (input == NULL || input == Py_None) { self->protobuf->clear_lon(); return 0; } PyObject *value = input; ::google::protobuf::int64 protoValue; // int64 if (PyInt_Check(value)) { protoValue = PyInt_AsLong(value); } else if (PyLong_Check(value)) { protoValue = PyLong_AsLongLong(value); } else { PyErr_SetString(PyExc_TypeError, "The lon attribute value must be an integer"); return -1; } self->protobuf->set_lon(protoValue); return 0; } int Node_init(Node *self, PyObject *args, PyObject *kwds) { PyObject *id = NULL; PyObject *keys = NULL; PyObject *vals = NULL; PyObject *info = NULL; PyObject *lat = NULL; PyObject *lon = NULL; static char *kwlist[] = { (char *) "id", (char *) "keys", (char *) "vals", (char *) "info", (char *) "lat", (char *) "lon", NULL }; if (! PyArg_ParseTupleAndKeywords( args, kwds, "|OOOOOO", kwlist, &id,&keys,&vals,&info,&lat,&lon)) return -1; if (id) { if (Node_setid(self, id, NULL) < 0) { return -1; } } if (keys) { if (Node_setkeys(self, keys, NULL) < 0) { return -1; } } if (vals) { if (Node_setvals(self, vals, NULL) < 0) { return -1; } } if (info) { if (Node_setinfo(self, info, NULL) < 0) { return -1; } } if (lat) { if (Node_setlat(self, lat, NULL) < 0) { return -1; } } if (lon) { if (Node_setlon(self, lon, NULL) < 0) { return -1; } } return 0; } PyObject * Node_richcompare(PyObject *self, PyObject *other, int op) { PyObject *result = NULL; if (!PyType_IsSubtype(other->ob_type, &NodeType)) { result = Py_NotImplemented; } else { // This is not a particularly efficient implementation since it never short circuits, but it's better // than nothing. It should probably only be used for tests. Node *selfValue = (Node *)self; Node *otherValue = (Node *)other; std::string selfSerialized; std::string otherSerialized; Py_BEGIN_ALLOW_THREADS selfValue->protobuf->SerializeToString(&selfSerialized); otherValue->protobuf->SerializeToString(&otherSerialized); Py_END_ALLOW_THREADS int cmp = selfSerialized.compare(otherSerialized); bool value = false; switch (op) { case Py_LT: value = cmp < 0; break; case Py_LE: value = cmp <= 0; break; case Py_EQ: value = cmp == 0; break; case Py_NE: value = cmp != 0; break; case Py_GT: value = cmp > 0; break; case Py_GE: value = cmp >= 0; break; } result = value ? Py_True : Py_False; } Py_XINCREF(result); return result; } static PyObject * Node_repr(PyObject *selfObject) { Node *self = (Node *)selfObject; PyObject *member; PyObject *memberRepr; std::stringstream result; result << "Node("; result << "id="; member = Node_getid(self, NULL); memberRepr = PyObject_Repr(member); result << PyString_AsString(memberRepr); Py_XDECREF(memberRepr); Py_XDECREF(member); result << ", "; result << "keys="; member = Node_getkeys(self, NULL); memberRepr = PyObject_Repr(member); result << PyString_AsString(memberRepr); Py_XDECREF(memberRepr); Py_XDECREF(member); result << ", "; result << "vals="; member = Node_getvals(self, NULL); memberRepr = PyObject_Repr(member); result << PyString_AsString(memberRepr); Py_XDECREF(memberRepr); Py_XDECREF(member); result << ", "; result << "info="; member = Node_getinfo(self, NULL); memberRepr = PyObject_Repr(member); result << PyString_AsString(memberRepr); Py_XDECREF(memberRepr); Py_XDECREF(member); result << ", "; result << "lat="; member = Node_getlat(self, NULL); memberRepr = PyObject_Repr(member); result << PyString_AsString(memberRepr); Py_XDECREF(memberRepr); Py_XDECREF(member); result << ", "; result << "lon="; member = Node_getlon(self, NULL); memberRepr = PyObject_Repr(member); result << PyString_AsString(memberRepr); Py_XDECREF(memberRepr); Py_XDECREF(member); result << ")"; std::string resultString = result.str(); return PyUnicode_Decode(resultString.data(), resultString.length(), "utf-8", NULL); } PyMemberDef Node_members[] = { {NULL} // Sentinel }; PyGetSetDef Node_getsetters[] = { {(char *)"id", (getter)Node_getid, (setter)Node_setid, (char *)"", NULL}, {(char *)"keys", (getter)Node_getkeys, (setter)Node_setkeys, (char *)"", NULL}, {(char *)"vals", (getter)Node_getvals, (setter)Node_setvals, (char *)"", NULL}, {(char *)"info", (getter)Node_getinfo, (setter)Node_setinfo, (char *)"", NULL}, {(char *)"lat", (getter)Node_getlat, (setter)Node_setlat, (char *)"", NULL}, {(char *)"lon", (getter)Node_getlon, (setter)Node_setlon, (char *)"", NULL}, {NULL} // Sentinel }; PyMethodDef Node_methods[] = { {"DebugString", (PyCFunction)Node_DebugString, METH_NOARGS, "Generates a human readable form of this message, useful for debugging and other purposes." }, {"SerializeToString", (PyCFunction)Node_SerializeToString, METH_NOARGS, "Serializes the protocol buffer to a string." }, {"SerializeMany", (PyCFunction)Node_SerializeMany, METH_O | METH_CLASS, "Serializes a sequence of protocol buffers to a string." }, {"ParseFromString", (PyCFunction)Node_ParseFromString, METH_O, "Parses the protocol buffer from a string." }, {"ParseFromLongString", (PyCFunction)Node_ParseFromLongString, METH_O, "Parses the protocol buffer from a string as large as 512MB." }, {"ParseMany", (PyCFunction)Node_ParseMany, METH_VARARGS | METH_CLASS, "Parses many protocol buffers of this type from a string." }, {NULL} // Sentinel }; PyTypeObject NodeType = { PyObject_HEAD_INIT(NULL) 0, /*ob_size*/ "OSMPBF.Node", /*tp_name*/ sizeof(Node), /*tp_basicsize*/ 0, /*tp_itemsize*/ (destructor)Node_dealloc, /*tp_dealloc*/ 0, /*tp_print*/ 0, /*tp_getattr*/ 0, /*tp_setattr*/ 0, /*tp_compare*/ Node_repr, /*tp_repr*/ 0, /*tp_as_number*/ 0, /*tp_as_sequence*/ 0, /*tp_as_mapping*/ 0, /*tp_hash */ 0, /*tp_call*/ 0, /*tp_str*/ 0, /*tp_getattro*/ 0, /*tp_setattro*/ 0, /*tp_as_buffer*/ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_RICHCOMPARE, /*tp_flags*/ "Node objects", /* tp_doc */ 0, /* tp_traverse */ 0, /* tp_clear */ Node_richcompare, /* tp_richcompare */ 0, /* tp_weaklistoffset */ 0, /* tp_iter */ 0, /* tp_iternext */ Node_methods, /* tp_methods */ Node_members, /* tp_members */ Node_getsetters, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ 0, /* tp_descr_set */ 0, /* tp_dictoffset */ (initproc)Node_init, /* tp_init */ 0, /* tp_alloc */ Node_new, /* tp_new */ }; } // Lets try not to pollute the global namespace namespace { // Forward-declaration for recursive structures extern PyTypeObject RelationType; typedef struct { PyObject_HEAD OSMPBF::Relation *protobuf; } Relation; void Relation_dealloc(Relation* self) { delete self->protobuf; self->ob_type->tp_free((PyObject*)self); } PyObject * Relation_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { Relation *self; self = (Relation *)type->tp_alloc(type, 0); self->protobuf = new OSMPBF::Relation(); return (PyObject *)self; } PyObject * Relation_DebugString(Relation* self) { std::string result; Py_BEGIN_ALLOW_THREADS result = self->protobuf->Utf8DebugString(); Py_END_ALLOW_THREADS return PyUnicode_FromStringAndSize(result.data(), result.length()); } PyObject * Relation_SerializeToString(Relation* self) { std::string result; Py_BEGIN_ALLOW_THREADS self->protobuf->SerializeToString(&result); Py_END_ALLOW_THREADS return PyString_FromStringAndSize(result.data(), result.length()); } PyObject * Relation_SerializeMany(void *nothing, PyObject *values) { std::string result; google::protobuf::io::ZeroCopyOutputStream* output = new google::protobuf::io::StringOutputStream(&result); google::protobuf::io::CodedOutputStream* outputStream = new google::protobuf::io::CodedOutputStream(output); PyObject *sequence = PySequence_Fast(values, "The values to serialize must be a sequence."); for (Py_ssize_t i = 0, len = PySequence_Length(sequence); i < len; ++i) { Relation *value = (Relation *)PySequence_Fast_GET_ITEM(sequence, i); Py_BEGIN_ALLOW_THREADS outputStream->WriteVarint32(value->protobuf->ByteSize()); value->protobuf->SerializeToCodedStream(outputStream); Py_END_ALLOW_THREADS } Py_XDECREF(sequence); delete outputStream; delete output; return PyString_FromStringAndSize(result.data(), result.length()); } PyObject * Relation_ParseFromString(Relation* self, PyObject *value) { std::string serialized(PyString_AsString(value), PyString_Size(value)); Py_BEGIN_ALLOW_THREADS self->protobuf->ParseFromString(serialized); Py_END_ALLOW_THREADS Py_RETURN_NONE; } PyObject * Relation_ParseFromLongString(Relation* self, PyObject *value) { google::protobuf::io::ZeroCopyInputStream* input = new google::protobuf::io::ArrayInputStream(PyString_AsString(value), PyString_Size(value)); google::protobuf::io::CodedInputStream* inputStream = new google::protobuf::io::CodedInputStream(input); inputStream->SetTotalBytesLimit(512 * 1024 * 1024, 512 * 1024 * 1024); Py_BEGIN_ALLOW_THREADS self->protobuf->ParseFromCodedStream(inputStream); Py_END_ALLOW_THREADS delete inputStream; delete input; Py_RETURN_NONE; } PyObject * Relation_ParseMany(void* nothing, PyObject *args) { PyObject *value; PyObject *callback; int fail = 0; if (!PyArg_ParseTuple(args, "OO", &value, &callback)) { return NULL; } google::protobuf::io::ZeroCopyInputStream* input = new google::protobuf::io::ArrayInputStream(PyString_AsString(value), PyString_Size(value)); google::protobuf::io::CodedInputStream* inputStream = new google::protobuf::io::CodedInputStream(input); inputStream->SetTotalBytesLimit(512 * 1024 * 1024, 512 * 1024 * 1024); google::protobuf::uint32 bytes; PyObject *single = NULL; while (inputStream->ReadVarint32(&bytes)) { google::protobuf::io::CodedInputStream::Limit messageLimit = inputStream->PushLimit(bytes); if (single == NULL) { single = Relation_new(&RelationType, NULL, NULL); } Py_BEGIN_ALLOW_THREADS ((Relation *)single)->protobuf->ParseFromCodedStream(inputStream); Py_END_ALLOW_THREADS inputStream->PopLimit(messageLimit); PyObject *result = PyObject_CallFunctionObjArgs(callback, single, NULL); if (result == NULL) { fail = 1; break; }; if (single->ob_refcnt != 1) { // If the callback saved a reference to the item, don't re-use it. Py_XDECREF(single); single = NULL; } } if (single != NULL) { Py_XDECREF(single); } delete inputStream; delete input; if (fail) { return NULL; } else { Py_RETURN_NONE; } } PyObject * Relation_getid(Relation *self, void *closure) { if (! self->protobuf->has_id()) { Py_RETURN_NONE; } return fastpb_convert3( self->protobuf->id()); } int Relation_setid(Relation *self, PyObject *input, void *closure) { if (input == NULL || input == Py_None) { self->protobuf->clear_id(); return 0; } PyObject *value = input; ::google::protobuf::int64 protoValue; // int64 if (PyInt_Check(value)) { protoValue = PyInt_AsLong(value); } else if (PyLong_Check(value)) { protoValue = PyLong_AsLongLong(value); } else { PyErr_SetString(PyExc_TypeError, "The id attribute value must be an integer"); return -1; } self->protobuf->set_id(protoValue); return 0; } PyObject * Relation_getkeys(Relation *self, void *closure) { int len = self->protobuf->keys_size(); PyObject *tuple = PyTuple_New(len); for (int i = 0; i < len; ++i) { PyObject *value = fastpb_convert13( self->protobuf->keys(i)); if (!value) { return NULL; } PyTuple_SetItem(tuple, i, value); } return tuple; } int Relation_setkeys(Relation *self, PyObject *input, void *closure) { if (input == NULL || input == Py_None) { self->protobuf->clear_keys(); return 0; } if (PyString_Check(input)) { PyErr_SetString(PyExc_TypeError, "The keys attribute value must be a sequence"); return -1; } PyObject *sequence = PySequence_Fast(input, "The keys attribute value must be a sequence"); self->protobuf->clear_keys(); for (Py_ssize_t i = 0, len = PySequence_Length(sequence); i < len; ++i) { PyObject *value = PySequence_Fast_GET_ITEM(sequence, i); ::google::protobuf::uint32 protoValue; // uint32 if (PyInt_Check(value)) { protoValue = PyInt_AsUnsignedLongMask(value); } else if (PyLong_Check(value)) { protoValue = PyLong_AsUnsignedLong(value); } else { PyErr_SetString(PyExc_TypeError, "The keys attribute value must be an integer"); return -1; } self->protobuf->add_keys(protoValue); } Py_XDECREF(sequence); return 0; } PyObject * Relation_getvals(Relation *self, void *closure) { int len = self->protobuf->vals_size(); PyObject *tuple = PyTuple_New(len); for (int i = 0; i < len; ++i) { PyObject *value = fastpb_convert13( self->protobuf->vals(i)); if (!value) { return NULL; } PyTuple_SetItem(tuple, i, value); } return tuple; } int Relation_setvals(Relation *self, PyObject *input, void *closure) { if (input == NULL || input == Py_None) { self->protobuf->clear_vals(); return 0; } if (PyString_Check(input)) { PyErr_SetString(PyExc_TypeError, "The vals attribute value must be a sequence"); return -1; } PyObject *sequence = PySequence_Fast(input, "The vals attribute value must be a sequence"); self->protobuf->clear_vals(); for (Py_ssize_t i = 0, len = PySequence_Length(sequence); i < len; ++i) { PyObject *value = PySequence_Fast_GET_ITEM(sequence, i); ::google::protobuf::uint32 protoValue; // uint32 if (PyInt_Check(value)) { protoValue = PyInt_AsUnsignedLongMask(value); } else if (PyLong_Check(value)) { protoValue = PyLong_AsUnsignedLong(value); } else { PyErr_SetString(PyExc_TypeError, "The vals attribute value must be an integer"); return -1; } self->protobuf->add_vals(protoValue); } Py_XDECREF(sequence); return 0; } PyObject * fastpb_convertRelationinfo(const ::google::protobuf::Message &value) { Info *obj = (Info *) Info_new(&InfoType, NULL, NULL); obj->protobuf->MergeFrom(value); return (PyObject *)obj; } PyObject * Relation_getinfo(Relation *self, void *closure) { if (! self->protobuf->has_info()) { Py_RETURN_NONE; } return fastpb_convertRelationinfo( self->protobuf->info()); } int Relation_setinfo(Relation *self, PyObject *input, void *closure) { if (input == NULL || input == Py_None) { self->protobuf->clear_info(); return 0; } PyObject *value = input; if (!PyType_IsSubtype(value->ob_type, &InfoType)) { PyErr_SetString(PyExc_TypeError, "The info attribute value must be an instance of Info"); return -1; } // .OSMPBF.Info ::OSMPBF::Info *protoValue = ((Info *) value)->protobuf; self->protobuf->clear_info(); self->protobuf->mutable_info()->MergeFrom(*protoValue); return 0; } PyObject * Relation_getroles_sid(Relation *self, void *closure) { int len = self->protobuf->roles_sid_size(); PyObject *tuple = PyTuple_New(len); for (int i = 0; i < len; ++i) { PyObject *value = fastpb_convert5( self->protobuf->roles_sid(i)); if (!value) { return NULL; } PyTuple_SetItem(tuple, i, value); } return tuple; } int Relation_setroles_sid(Relation *self, PyObject *input, void *closure) { if (input == NULL || input == Py_None) { self->protobuf->clear_roles_sid(); return 0; } if (PyString_Check(input)) { PyErr_SetString(PyExc_TypeError, "The roles_sid attribute value must be a sequence"); return -1; } PyObject *sequence = PySequence_Fast(input, "The roles_sid attribute value must be a sequence"); self->protobuf->clear_roles_sid(); for (Py_ssize_t i = 0, len = PySequence_Length(sequence); i < len; ++i) { PyObject *value = PySequence_Fast_GET_ITEM(sequence, i); ::google::protobuf::int32 protoValue; // int32 if (PyInt_Check(value)) { protoValue = PyInt_AsLong(value); } else { PyErr_SetString(PyExc_TypeError, "The roles_sid attribute value must be an integer"); return -1; } self->protobuf->add_roles_sid(protoValue); } Py_XDECREF(sequence); return 0; } PyObject * Relation_getmemids(Relation *self, void *closure) { int len = self->protobuf->memids_size(); PyObject *tuple = PyTuple_New(len); for (int i = 0; i < len; ++i) { PyObject *value = fastpb_convert18( self->protobuf->memids(i)); if (!value) { return NULL; } PyTuple_SetItem(tuple, i, value); } return tuple; } int Relation_setmemids(Relation *self, PyObject *input, void *closure) { if (input == NULL || input == Py_None) { self->protobuf->clear_memids(); return 0; } if (PyString_Check(input)) { PyErr_SetString(PyExc_TypeError, "The memids attribute value must be a sequence"); return -1; } PyObject *sequence = PySequence_Fast(input, "The memids attribute value must be a sequence"); self->protobuf->clear_memids(); for (Py_ssize_t i = 0, len = PySequence_Length(sequence); i < len; ++i) { PyObject *value = PySequence_Fast_GET_ITEM(sequence, i); ::google::protobuf::int64 protoValue; // int64 if (PyInt_Check(value)) { protoValue = PyInt_AsLong(value); } else if (PyLong_Check(value)) { protoValue = PyLong_AsLongLong(value); } else { PyErr_SetString(PyExc_TypeError, "The memids attribute value must be an integer"); return -1; } self->protobuf->add_memids(protoValue); } Py_XDECREF(sequence); return 0; } PyObject * Relation_gettypes(Relation *self, void *closure) { int len = self->protobuf->types_size(); PyObject *tuple = PyTuple_New(len); for (int i = 0; i < len; ++i) { PyObject *value = fastpb_convert14( self->protobuf->types(i)); if (!value) { return NULL; } PyTuple_SetItem(tuple, i, value); } return tuple; } int Relation_settypes(Relation *self, PyObject *input, void *closure) { if (input == NULL || input == Py_None) { self->protobuf->clear_types(); return 0; } if (PyString_Check(input)) { PyErr_SetString(PyExc_TypeError, "The types attribute value must be a sequence"); return -1; } PyObject *sequence = PySequence_Fast(input, "The types attribute value must be a sequence"); self->protobuf->clear_types(); for (Py_ssize_t i = 0, len = PySequence_Length(sequence); i < len; ++i) { PyObject *value = PySequence_Fast_GET_ITEM(sequence, i); // .OSMPBF.Relation.MemberType ::OSMPBF::Relation::MemberType protoValue; // int32 if (PyInt_Check(value)) { protoValue = (::OSMPBF::Relation::MemberType) PyInt_AsLong(value); } else { PyErr_SetString(PyExc_TypeError, "The types attribute value must be an integer"); return -1; } self->protobuf->add_types(protoValue); } Py_XDECREF(sequence); return 0; } int Relation_init(Relation *self, PyObject *args, PyObject *kwds) { PyObject *id = NULL; PyObject *keys = NULL; PyObject *vals = NULL; PyObject *info = NULL; PyObject *roles_sid = NULL; PyObject *memids = NULL; PyObject *types = NULL; static char *kwlist[] = { (char *) "id", (char *) "keys", (char *) "vals", (char *) "info", (char *) "roles_sid", (char *) "memids", (char *) "types", NULL }; if (! PyArg_ParseTupleAndKeywords( args, kwds, "|OOOOOOO", kwlist, &id,&keys,&vals,&info,&roles_sid,&memids,&types)) return -1; if (id) { if (Relation_setid(self, id, NULL) < 0) { return -1; } } if (keys) { if (Relation_setkeys(self, keys, NULL) < 0) { return -1; } } if (vals) { if (Relation_setvals(self, vals, NULL) < 0) { return -1; } } if (info) { if (Relation_setinfo(self, info, NULL) < 0) { return -1; } } if (roles_sid) { if (Relation_setroles_sid(self, roles_sid, NULL) < 0) { return -1; } } if (memids) { if (Relation_setmemids(self, memids, NULL) < 0) { return -1; } } if (types) { if (Relation_settypes(self, types, NULL) < 0) { return -1; } } return 0; } PyObject * Relation_richcompare(PyObject *self, PyObject *other, int op) { PyObject *result = NULL; if (!PyType_IsSubtype(other->ob_type, &RelationType)) { result = Py_NotImplemented; } else { // This is not a particularly efficient implementation since it never short circuits, but it's better // than nothing. It should probably only be used for tests. Relation *selfValue = (Relation *)self; Relation *otherValue = (Relation *)other; std::string selfSerialized; std::string otherSerialized; Py_BEGIN_ALLOW_THREADS selfValue->protobuf->SerializeToString(&selfSerialized); otherValue->protobuf->SerializeToString(&otherSerialized); Py_END_ALLOW_THREADS int cmp = selfSerialized.compare(otherSerialized); bool value = false; switch (op) { case Py_LT: value = cmp < 0; break; case Py_LE: value = cmp <= 0; break; case Py_EQ: value = cmp == 0; break; case Py_NE: value = cmp != 0; break; case Py_GT: value = cmp > 0; break; case Py_GE: value = cmp >= 0; break; } result = value ? Py_True : Py_False; } Py_XINCREF(result); return result; } static PyObject * Relation_repr(PyObject *selfObject) { Relation *self = (Relation *)selfObject; PyObject *member; PyObject *memberRepr; std::stringstream result; result << "Relation("; result << "id="; member = Relation_getid(self, NULL); memberRepr = PyObject_Repr(member); result << PyString_AsString(memberRepr); Py_XDECREF(memberRepr); Py_XDECREF(member); result << ", "; result << "keys="; member = Relation_getkeys(self, NULL); memberRepr = PyObject_Repr(member); result << PyString_AsString(memberRepr); Py_XDECREF(memberRepr); Py_XDECREF(member); result << ", "; result << "vals="; member = Relation_getvals(self, NULL); memberRepr = PyObject_Repr(member); result << PyString_AsString(memberRepr); Py_XDECREF(memberRepr); Py_XDECREF(member); result << ", "; result << "info="; member = Relation_getinfo(self, NULL); memberRepr = PyObject_Repr(member); result << PyString_AsString(memberRepr); Py_XDECREF(memberRepr); Py_XDECREF(member); result << ", "; result << "roles_sid="; member = Relation_getroles_sid(self, NULL); memberRepr = PyObject_Repr(member); result << PyString_AsString(memberRepr); Py_XDECREF(memberRepr); Py_XDECREF(member); result << ", "; result << "memids="; member = Relation_getmemids(self, NULL); memberRepr = PyObject_Repr(member); result << PyString_AsString(memberRepr); Py_XDECREF(memberRepr); Py_XDECREF(member); result << ", "; result << "types="; member = Relation_gettypes(self, NULL); memberRepr = PyObject_Repr(member); result << PyString_AsString(memberRepr); Py_XDECREF(memberRepr); Py_XDECREF(member); result << ")"; std::string resultString = result.str(); return PyUnicode_Decode(resultString.data(), resultString.length(), "utf-8", NULL); } PyMemberDef Relation_members[] = { {NULL} // Sentinel }; PyGetSetDef Relation_getsetters[] = { {(char *)"id", (getter)Relation_getid, (setter)Relation_setid, (char *)"", NULL}, {(char *)"keys", (getter)Relation_getkeys, (setter)Relation_setkeys, (char *)"", NULL}, {(char *)"vals", (getter)Relation_getvals, (setter)Relation_setvals, (char *)"", NULL}, {(char *)"info", (getter)Relation_getinfo, (setter)Relation_setinfo, (char *)"", NULL}, {(char *)"roles_sid", (getter)Relation_getroles_sid, (setter)Relation_setroles_sid, (char *)"", NULL}, {(char *)"memids", (getter)Relation_getmemids, (setter)Relation_setmemids, (char *)"", NULL}, {(char *)"types", (getter)Relation_gettypes, (setter)Relation_settypes, (char *)"", NULL}, {NULL} // Sentinel }; PyMethodDef Relation_methods[] = { {"DebugString", (PyCFunction)Relation_DebugString, METH_NOARGS, "Generates a human readable form of this message, useful for debugging and other purposes." }, {"SerializeToString", (PyCFunction)Relation_SerializeToString, METH_NOARGS, "Serializes the protocol buffer to a string." }, {"SerializeMany", (PyCFunction)Relation_SerializeMany, METH_O | METH_CLASS, "Serializes a sequence of protocol buffers to a string." }, {"ParseFromString", (PyCFunction)Relation_ParseFromString, METH_O, "Parses the protocol buffer from a string." }, {"ParseFromLongString", (PyCFunction)Relation_ParseFromLongString, METH_O, "Parses the protocol buffer from a string as large as 512MB." }, {"ParseMany", (PyCFunction)Relation_ParseMany, METH_VARARGS | METH_CLASS, "Parses many protocol buffers of this type from a string." }, {NULL} // Sentinel }; PyTypeObject RelationType = { PyObject_HEAD_INIT(NULL) 0, /*ob_size*/ "OSMPBF.Relation", /*tp_name*/ sizeof(Relation), /*tp_basicsize*/ 0, /*tp_itemsize*/ (destructor)Relation_dealloc, /*tp_dealloc*/ 0, /*tp_print*/ 0, /*tp_getattr*/ 0, /*tp_setattr*/ 0, /*tp_compare*/ Relation_repr, /*tp_repr*/ 0, /*tp_as_number*/ 0, /*tp_as_sequence*/ 0, /*tp_as_mapping*/ 0, /*tp_hash */ 0, /*tp_call*/ 0, /*tp_str*/ 0, /*tp_getattro*/ 0, /*tp_setattro*/ 0, /*tp_as_buffer*/ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_RICHCOMPARE, /*tp_flags*/ "Relation objects", /* tp_doc */ 0, /* tp_traverse */ 0, /* tp_clear */ Relation_richcompare, /* tp_richcompare */ 0, /* tp_weaklistoffset */ 0, /* tp_iter */ 0, /* tp_iternext */ Relation_methods, /* tp_methods */ Relation_members, /* tp_members */ Relation_getsetters, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ 0, /* tp_descr_set */ 0, /* tp_dictoffset */ (initproc)Relation_init, /* tp_init */ 0, /* tp_alloc */ Relation_new, /* tp_new */ }; } // Lets try not to pollute the global namespace namespace { // Forward-declaration for recursive structures extern PyTypeObject WayType; typedef struct { PyObject_HEAD OSMPBF::Way *protobuf; } Way; void Way_dealloc(Way* self) { delete self->protobuf; self->ob_type->tp_free((PyObject*)self); } PyObject * Way_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { Way *self; self = (Way *)type->tp_alloc(type, 0); self->protobuf = new OSMPBF::Way(); return (PyObject *)self; } PyObject * Way_DebugString(Way* self) { std::string result; Py_BEGIN_ALLOW_THREADS result = self->protobuf->Utf8DebugString(); Py_END_ALLOW_THREADS return PyUnicode_FromStringAndSize(result.data(), result.length()); } PyObject * Way_SerializeToString(Way* self) { std::string result; Py_BEGIN_ALLOW_THREADS self->protobuf->SerializeToString(&result); Py_END_ALLOW_THREADS return PyString_FromStringAndSize(result.data(), result.length()); } PyObject * Way_SerializeMany(void *nothing, PyObject *values) { std::string result; google::protobuf::io::ZeroCopyOutputStream* output = new google::protobuf::io::StringOutputStream(&result); google::protobuf::io::CodedOutputStream* outputStream = new google::protobuf::io::CodedOutputStream(output); PyObject *sequence = PySequence_Fast(values, "The values to serialize must be a sequence."); for (Py_ssize_t i = 0, len = PySequence_Length(sequence); i < len; ++i) { Way *value = (Way *)PySequence_Fast_GET_ITEM(sequence, i); Py_BEGIN_ALLOW_THREADS outputStream->WriteVarint32(value->protobuf->ByteSize()); value->protobuf->SerializeToCodedStream(outputStream); Py_END_ALLOW_THREADS } Py_XDECREF(sequence); delete outputStream; delete output; return PyString_FromStringAndSize(result.data(), result.length()); } PyObject * Way_ParseFromString(Way* self, PyObject *value) { std::string serialized(PyString_AsString(value), PyString_Size(value)); Py_BEGIN_ALLOW_THREADS self->protobuf->ParseFromString(serialized); Py_END_ALLOW_THREADS Py_RETURN_NONE; } PyObject * Way_ParseFromLongString(Way* self, PyObject *value) { google::protobuf::io::ZeroCopyInputStream* input = new google::protobuf::io::ArrayInputStream(PyString_AsString(value), PyString_Size(value)); google::protobuf::io::CodedInputStream* inputStream = new google::protobuf::io::CodedInputStream(input); inputStream->SetTotalBytesLimit(512 * 1024 * 1024, 512 * 1024 * 1024); Py_BEGIN_ALLOW_THREADS self->protobuf->ParseFromCodedStream(inputStream); Py_END_ALLOW_THREADS delete inputStream; delete input; Py_RETURN_NONE; } PyObject * Way_ParseMany(void* nothing, PyObject *args) { PyObject *value; PyObject *callback; int fail = 0; if (!PyArg_ParseTuple(args, "OO", &value, &callback)) { return NULL; } google::protobuf::io::ZeroCopyInputStream* input = new google::protobuf::io::ArrayInputStream(PyString_AsString(value), PyString_Size(value)); google::protobuf::io::CodedInputStream* inputStream = new google::protobuf::io::CodedInputStream(input); inputStream->SetTotalBytesLimit(512 * 1024 * 1024, 512 * 1024 * 1024); google::protobuf::uint32 bytes; PyObject *single = NULL; while (inputStream->ReadVarint32(&bytes)) { google::protobuf::io::CodedInputStream::Limit messageLimit = inputStream->PushLimit(bytes); if (single == NULL) { single = Way_new(&WayType, NULL, NULL); } Py_BEGIN_ALLOW_THREADS ((Way *)single)->protobuf->ParseFromCodedStream(inputStream); Py_END_ALLOW_THREADS inputStream->PopLimit(messageLimit); PyObject *result = PyObject_CallFunctionObjArgs(callback, single, NULL); if (result == NULL) { fail = 1; break; }; if (single->ob_refcnt != 1) { // If the callback saved a reference to the item, don't re-use it. Py_XDECREF(single); single = NULL; } } if (single != NULL) { Py_XDECREF(single); } delete inputStream; delete input; if (fail) { return NULL; } else { Py_RETURN_NONE; } } PyObject * Way_getid(Way *self, void *closure) { if (! self->protobuf->has_id()) { Py_RETURN_NONE; } return fastpb_convert3( self->protobuf->id()); } int Way_setid(Way *self, PyObject *input, void *closure) { if (input == NULL || input == Py_None) { self->protobuf->clear_id(); return 0; } PyObject *value = input; ::google::protobuf::int64 protoValue; // int64 if (PyInt_Check(value)) { protoValue = PyInt_AsLong(value); } else if (PyLong_Check(value)) { protoValue = PyLong_AsLongLong(value); } else { PyErr_SetString(PyExc_TypeError, "The id attribute value must be an integer"); return -1; } self->protobuf->set_id(protoValue); return 0; } PyObject * Way_getkeys(Way *self, void *closure) { int len = self->protobuf->keys_size(); PyObject *tuple = PyTuple_New(len); for (int i = 0; i < len; ++i) { PyObject *value = fastpb_convert13( self->protobuf->keys(i)); if (!value) { return NULL; } PyTuple_SetItem(tuple, i, value); } return tuple; } int Way_setkeys(Way *self, PyObject *input, void *closure) { if (input == NULL || input == Py_None) { self->protobuf->clear_keys(); return 0; } if (PyString_Check(input)) { PyErr_SetString(PyExc_TypeError, "The keys attribute value must be a sequence"); return -1; } PyObject *sequence = PySequence_Fast(input, "The keys attribute value must be a sequence"); self->protobuf->clear_keys(); for (Py_ssize_t i = 0, len = PySequence_Length(sequence); i < len; ++i) { PyObject *value = PySequence_Fast_GET_ITEM(sequence, i); ::google::protobuf::uint32 protoValue; // uint32 if (PyInt_Check(value)) { protoValue = PyInt_AsUnsignedLongMask(value); } else if (PyLong_Check(value)) { protoValue = PyLong_AsUnsignedLong(value); } else { PyErr_SetString(PyExc_TypeError, "The keys attribute value must be an integer"); return -1; } self->protobuf->add_keys(protoValue); } Py_XDECREF(sequence); return 0; } PyObject * Way_getvals(Way *self, void *closure) { int len = self->protobuf->vals_size(); PyObject *tuple = PyTuple_New(len); for (int i = 0; i < len; ++i) { PyObject *value = fastpb_convert13( self->protobuf->vals(i)); if (!value) { return NULL; } PyTuple_SetItem(tuple, i, value); } return tuple; } int Way_setvals(Way *self, PyObject *input, void *closure) { if (input == NULL || input == Py_None) { self->protobuf->clear_vals(); return 0; } if (PyString_Check(input)) { PyErr_SetString(PyExc_TypeError, "The vals attribute value must be a sequence"); return -1; } PyObject *sequence = PySequence_Fast(input, "The vals attribute value must be a sequence"); self->protobuf->clear_vals(); for (Py_ssize_t i = 0, len = PySequence_Length(sequence); i < len; ++i) { PyObject *value = PySequence_Fast_GET_ITEM(sequence, i); ::google::protobuf::uint32 protoValue; // uint32 if (PyInt_Check(value)) { protoValue = PyInt_AsUnsignedLongMask(value); } else if (PyLong_Check(value)) { protoValue = PyLong_AsUnsignedLong(value); } else { PyErr_SetString(PyExc_TypeError, "The vals attribute value must be an integer"); return -1; } self->protobuf->add_vals(protoValue); } Py_XDECREF(sequence); return 0; } PyObject * fastpb_convertWayinfo(const ::google::protobuf::Message &value) { Info *obj = (Info *) Info_new(&InfoType, NULL, NULL); obj->protobuf->MergeFrom(value); return (PyObject *)obj; } PyObject * Way_getinfo(Way *self, void *closure) { if (! self->protobuf->has_info()) { Py_RETURN_NONE; } return fastpb_convertWayinfo( self->protobuf->info()); } int Way_setinfo(Way *self, PyObject *input, void *closure) { if (input == NULL || input == Py_None) { self->protobuf->clear_info(); return 0; } PyObject *value = input; if (!PyType_IsSubtype(value->ob_type, &InfoType)) { PyErr_SetString(PyExc_TypeError, "The info attribute value must be an instance of Info"); return -1; } // .OSMPBF.Info ::OSMPBF::Info *protoValue = ((Info *) value)->protobuf; self->protobuf->clear_info(); self->protobuf->mutable_info()->MergeFrom(*protoValue); return 0; } PyObject * Way_getrefs(Way *self, void *closure) { int len = self->protobuf->refs_size(); PyObject *tuple = PyTuple_New(len); for (int i = 0; i < len; ++i) { PyObject *value = fastpb_convert18( self->protobuf->refs(i)); if (!value) { return NULL; } PyTuple_SetItem(tuple, i, value); } return tuple; } int Way_setrefs(Way *self, PyObject *input, void *closure) { if (input == NULL || input == Py_None) { self->protobuf->clear_refs(); return 0; } if (PyString_Check(input)) { PyErr_SetString(PyExc_TypeError, "The refs attribute value must be a sequence"); return -1; } PyObject *sequence = PySequence_Fast(input, "The refs attribute value must be a sequence"); self->protobuf->clear_refs(); for (Py_ssize_t i = 0, len = PySequence_Length(sequence); i < len; ++i) { PyObject *value = PySequence_Fast_GET_ITEM(sequence, i); ::google::protobuf::int64 protoValue; // int64 if (PyInt_Check(value)) { protoValue = PyInt_AsLong(value); } else if (PyLong_Check(value)) { protoValue = PyLong_AsLongLong(value); } else { PyErr_SetString(PyExc_TypeError, "The refs attribute value must be an integer"); return -1; } self->protobuf->add_refs(protoValue); } Py_XDECREF(sequence); return 0; } int Way_init(Way *self, PyObject *args, PyObject *kwds) { PyObject *id = NULL; PyObject *keys = NULL; PyObject *vals = NULL; PyObject *info = NULL; PyObject *refs = NULL; static char *kwlist[] = { (char *) "id", (char *) "keys", (char *) "vals", (char *) "info", (char *) "refs", NULL }; if (! PyArg_ParseTupleAndKeywords( args, kwds, "|OOOOO", kwlist, &id,&keys,&vals,&info,&refs)) return -1; if (id) { if (Way_setid(self, id, NULL) < 0) { return -1; } } if (keys) { if (Way_setkeys(self, keys, NULL) < 0) { return -1; } } if (vals) { if (Way_setvals(self, vals, NULL) < 0) { return -1; } } if (info) { if (Way_setinfo(self, info, NULL) < 0) { return -1; } } if (refs) { if (Way_setrefs(self, refs, NULL) < 0) { return -1; } } return 0; } PyObject * Way_richcompare(PyObject *self, PyObject *other, int op) { PyObject *result = NULL; if (!PyType_IsSubtype(other->ob_type, &WayType)) { result = Py_NotImplemented; } else { // This is not a particularly efficient implementation since it never short circuits, but it's better // than nothing. It should probably only be used for tests. Way *selfValue = (Way *)self; Way *otherValue = (Way *)other; std::string selfSerialized; std::string otherSerialized; Py_BEGIN_ALLOW_THREADS selfValue->protobuf->SerializeToString(&selfSerialized); otherValue->protobuf->SerializeToString(&otherSerialized); Py_END_ALLOW_THREADS int cmp = selfSerialized.compare(otherSerialized); bool value = false; switch (op) { case Py_LT: value = cmp < 0; break; case Py_LE: value = cmp <= 0; break; case Py_EQ: value = cmp == 0; break; case Py_NE: value = cmp != 0; break; case Py_GT: value = cmp > 0; break; case Py_GE: value = cmp >= 0; break; } result = value ? Py_True : Py_False; } Py_XINCREF(result); return result; } static PyObject * Way_repr(PyObject *selfObject) { Way *self = (Way *)selfObject; PyObject *member; PyObject *memberRepr; std::stringstream result; result << "Way("; result << "id="; member = Way_getid(self, NULL); memberRepr = PyObject_Repr(member); result << PyString_AsString(memberRepr); Py_XDECREF(memberRepr); Py_XDECREF(member); result << ", "; result << "keys="; member = Way_getkeys(self, NULL); memberRepr = PyObject_Repr(member); result << PyString_AsString(memberRepr); Py_XDECREF(memberRepr); Py_XDECREF(member); result << ", "; result << "vals="; member = Way_getvals(self, NULL); memberRepr = PyObject_Repr(member); result << PyString_AsString(memberRepr); Py_XDECREF(memberRepr); Py_XDECREF(member); result << ", "; result << "info="; member = Way_getinfo(self, NULL); memberRepr = PyObject_Repr(member); result << PyString_AsString(memberRepr); Py_XDECREF(memberRepr); Py_XDECREF(member); result << ", "; result << "refs="; member = Way_getrefs(self, NULL); memberRepr = PyObject_Repr(member); result << PyString_AsString(memberRepr); Py_XDECREF(memberRepr); Py_XDECREF(member); result << ")"; std::string resultString = result.str(); return PyUnicode_Decode(resultString.data(), resultString.length(), "utf-8", NULL); } PyMemberDef Way_members[] = { {NULL} // Sentinel }; PyGetSetDef Way_getsetters[] = { {(char *)"id", (getter)Way_getid, (setter)Way_setid, (char *)"", NULL}, {(char *)"keys", (getter)Way_getkeys, (setter)Way_setkeys, (char *)"", NULL}, {(char *)"vals", (getter)Way_getvals, (setter)Way_setvals, (char *)"", NULL}, {(char *)"info", (getter)Way_getinfo, (setter)Way_setinfo, (char *)"", NULL}, {(char *)"refs", (getter)Way_getrefs, (setter)Way_setrefs, (char *)"", NULL}, {NULL} // Sentinel }; PyMethodDef Way_methods[] = { {"DebugString", (PyCFunction)Way_DebugString, METH_NOARGS, "Generates a human readable form of this message, useful for debugging and other purposes." }, {"SerializeToString", (PyCFunction)Way_SerializeToString, METH_NOARGS, "Serializes the protocol buffer to a string." }, {"SerializeMany", (PyCFunction)Way_SerializeMany, METH_O | METH_CLASS, "Serializes a sequence of protocol buffers to a string." }, {"ParseFromString", (PyCFunction)Way_ParseFromString, METH_O, "Parses the protocol buffer from a string." }, {"ParseFromLongString", (PyCFunction)Way_ParseFromLongString, METH_O, "Parses the protocol buffer from a string as large as 512MB." }, {"ParseMany", (PyCFunction)Way_ParseMany, METH_VARARGS | METH_CLASS, "Parses many protocol buffers of this type from a string." }, {NULL} // Sentinel }; PyTypeObject WayType = { PyObject_HEAD_INIT(NULL) 0, /*ob_size*/ "OSMPBF.Way", /*tp_name*/ sizeof(Way), /*tp_basicsize*/ 0, /*tp_itemsize*/ (destructor)Way_dealloc, /*tp_dealloc*/ 0, /*tp_print*/ 0, /*tp_getattr*/ 0, /*tp_setattr*/ 0, /*tp_compare*/ Way_repr, /*tp_repr*/ 0, /*tp_as_number*/ 0, /*tp_as_sequence*/ 0, /*tp_as_mapping*/ 0, /*tp_hash */ 0, /*tp_call*/ 0, /*tp_str*/ 0, /*tp_getattro*/ 0, /*tp_setattro*/ 0, /*tp_as_buffer*/ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_RICHCOMPARE, /*tp_flags*/ "Way objects", /* tp_doc */ 0, /* tp_traverse */ 0, /* tp_clear */ Way_richcompare, /* tp_richcompare */ 0, /* tp_weaklistoffset */ 0, /* tp_iter */ 0, /* tp_iternext */ Way_methods, /* tp_methods */ Way_members, /* tp_members */ Way_getsetters, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ 0, /* tp_descr_set */ 0, /* tp_dictoffset */ (initproc)Way_init, /* tp_init */ 0, /* tp_alloc */ Way_new, /* tp_new */ }; } // Lets try not to pollute the global namespace namespace { // Forward-declaration for recursive structures extern PyTypeObject PrimitiveGroupType; typedef struct { PyObject_HEAD OSMPBF::PrimitiveGroup *protobuf; } PrimitiveGroup; void PrimitiveGroup_dealloc(PrimitiveGroup* self) { delete self->protobuf; self->ob_type->tp_free((PyObject*)self); } PyObject * PrimitiveGroup_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { PrimitiveGroup *self; self = (PrimitiveGroup *)type->tp_alloc(type, 0); self->protobuf = new OSMPBF::PrimitiveGroup(); return (PyObject *)self; } PyObject * PrimitiveGroup_DebugString(PrimitiveGroup* self) { std::string result; Py_BEGIN_ALLOW_THREADS result = self->protobuf->Utf8DebugString(); Py_END_ALLOW_THREADS return PyUnicode_FromStringAndSize(result.data(), result.length()); } PyObject * PrimitiveGroup_SerializeToString(PrimitiveGroup* self) { std::string result; Py_BEGIN_ALLOW_THREADS self->protobuf->SerializeToString(&result); Py_END_ALLOW_THREADS return PyString_FromStringAndSize(result.data(), result.length()); } PyObject * PrimitiveGroup_SerializeMany(void *nothing, PyObject *values) { std::string result; google::protobuf::io::ZeroCopyOutputStream* output = new google::protobuf::io::StringOutputStream(&result); google::protobuf::io::CodedOutputStream* outputStream = new google::protobuf::io::CodedOutputStream(output); PyObject *sequence = PySequence_Fast(values, "The values to serialize must be a sequence."); for (Py_ssize_t i = 0, len = PySequence_Length(sequence); i < len; ++i) { PrimitiveGroup *value = (PrimitiveGroup *)PySequence_Fast_GET_ITEM(sequence, i); Py_BEGIN_ALLOW_THREADS outputStream->WriteVarint32(value->protobuf->ByteSize()); value->protobuf->SerializeToCodedStream(outputStream); Py_END_ALLOW_THREADS } Py_XDECREF(sequence); delete outputStream; delete output; return PyString_FromStringAndSize(result.data(), result.length()); } PyObject * PrimitiveGroup_ParseFromString(PrimitiveGroup* self, PyObject *value) { std::string serialized(PyString_AsString(value), PyString_Size(value)); Py_BEGIN_ALLOW_THREADS self->protobuf->ParseFromString(serialized); Py_END_ALLOW_THREADS Py_RETURN_NONE; } PyObject * PrimitiveGroup_ParseFromLongString(PrimitiveGroup* self, PyObject *value) { google::protobuf::io::ZeroCopyInputStream* input = new google::protobuf::io::ArrayInputStream(PyString_AsString(value), PyString_Size(value)); google::protobuf::io::CodedInputStream* inputStream = new google::protobuf::io::CodedInputStream(input); inputStream->SetTotalBytesLimit(512 * 1024 * 1024, 512 * 1024 * 1024); Py_BEGIN_ALLOW_THREADS self->protobuf->ParseFromCodedStream(inputStream); Py_END_ALLOW_THREADS delete inputStream; delete input; Py_RETURN_NONE; } PyObject * PrimitiveGroup_ParseMany(void* nothing, PyObject *args) { PyObject *value; PyObject *callback; int fail = 0; if (!PyArg_ParseTuple(args, "OO", &value, &callback)) { return NULL; } google::protobuf::io::ZeroCopyInputStream* input = new google::protobuf::io::ArrayInputStream(PyString_AsString(value), PyString_Size(value)); google::protobuf::io::CodedInputStream* inputStream = new google::protobuf::io::CodedInputStream(input); inputStream->SetTotalBytesLimit(512 * 1024 * 1024, 512 * 1024 * 1024); google::protobuf::uint32 bytes; PyObject *single = NULL; while (inputStream->ReadVarint32(&bytes)) { google::protobuf::io::CodedInputStream::Limit messageLimit = inputStream->PushLimit(bytes); if (single == NULL) { single = PrimitiveGroup_new(&PrimitiveGroupType, NULL, NULL); } Py_BEGIN_ALLOW_THREADS ((PrimitiveGroup *)single)->protobuf->ParseFromCodedStream(inputStream); Py_END_ALLOW_THREADS inputStream->PopLimit(messageLimit); PyObject *result = PyObject_CallFunctionObjArgs(callback, single, NULL); if (result == NULL) { fail = 1; break; }; if (single->ob_refcnt != 1) { // If the callback saved a reference to the item, don't re-use it. Py_XDECREF(single); single = NULL; } } if (single != NULL) { Py_XDECREF(single); } delete inputStream; delete input; if (fail) { return NULL; } else { Py_RETURN_NONE; } } PyObject * fastpb_convertPrimitiveGroupnodes(const ::google::protobuf::Message &value) { Node *obj = (Node *) Node_new(&NodeType, NULL, NULL); obj->protobuf->MergeFrom(value); return (PyObject *)obj; } PyObject * PrimitiveGroup_getnodes(PrimitiveGroup *self, void *closure) { int len = self->protobuf->nodes_size(); PyObject *tuple = PyTuple_New(len); for (int i = 0; i < len; ++i) { PyObject *value = fastpb_convertPrimitiveGroupnodes( self->protobuf->nodes(i)); if (!value) { return NULL; } PyTuple_SetItem(tuple, i, value); } return tuple; } int PrimitiveGroup_setnodes(PrimitiveGroup *self, PyObject *input, void *closure) { if (input == NULL || input == Py_None) { self->protobuf->clear_nodes(); return 0; } if (PyString_Check(input)) { PyErr_SetString(PyExc_TypeError, "The nodes attribute value must be a sequence"); return -1; } PyObject *sequence = PySequence_Fast(input, "The nodes attribute value must be a sequence"); self->protobuf->clear_nodes(); for (Py_ssize_t i = 0, len = PySequence_Length(sequence); i < len; ++i) { PyObject *value = PySequence_Fast_GET_ITEM(sequence, i); if (!PyType_IsSubtype(value->ob_type, &NodeType)) { PyErr_SetString(PyExc_TypeError, "The nodes attribute value must be an instance of Node"); return -1; } // .OSMPBF.Node ::OSMPBF::Node *protoValue = ((Node *) value)->protobuf; self->protobuf->add_nodes()->MergeFrom(*protoValue); } Py_XDECREF(sequence); return 0; } PyObject * fastpb_convertPrimitiveGroupdense(const ::google::protobuf::Message &value) { DenseNodes *obj = (DenseNodes *) DenseNodes_new(&DenseNodesType, NULL, NULL); obj->protobuf->MergeFrom(value); return (PyObject *)obj; } PyObject * PrimitiveGroup_getdense(PrimitiveGroup *self, void *closure) { if (! self->protobuf->has_dense()) { Py_RETURN_NONE; } return fastpb_convertPrimitiveGroupdense( self->protobuf->dense()); } int PrimitiveGroup_setdense(PrimitiveGroup *self, PyObject *input, void *closure) { if (input == NULL || input == Py_None) { self->protobuf->clear_dense(); return 0; } PyObject *value = input; if (!PyType_IsSubtype(value->ob_type, &DenseNodesType)) { PyErr_SetString(PyExc_TypeError, "The dense attribute value must be an instance of DenseNodes"); return -1; } // .OSMPBF.DenseNodes ::OSMPBF::DenseNodes *protoValue = ((DenseNodes *) value)->protobuf; self->protobuf->clear_dense(); self->protobuf->mutable_dense()->MergeFrom(*protoValue); return 0; } PyObject * fastpb_convertPrimitiveGroupways(const ::google::protobuf::Message &value) { Way *obj = (Way *) Way_new(&WayType, NULL, NULL); obj->protobuf->MergeFrom(value); return (PyObject *)obj; } PyObject * PrimitiveGroup_getways(PrimitiveGroup *self, void *closure) { int len = self->protobuf->ways_size(); PyObject *tuple = PyTuple_New(len); for (int i = 0; i < len; ++i) { PyObject *value = fastpb_convertPrimitiveGroupways( self->protobuf->ways(i)); if (!value) { return NULL; } PyTuple_SetItem(tuple, i, value); } return tuple; } int PrimitiveGroup_setways(PrimitiveGroup *self, PyObject *input, void *closure) { if (input == NULL || input == Py_None) { self->protobuf->clear_ways(); return 0; } if (PyString_Check(input)) { PyErr_SetString(PyExc_TypeError, "The ways attribute value must be a sequence"); return -1; } PyObject *sequence = PySequence_Fast(input, "The ways attribute value must be a sequence"); self->protobuf->clear_ways(); for (Py_ssize_t i = 0, len = PySequence_Length(sequence); i < len; ++i) { PyObject *value = PySequence_Fast_GET_ITEM(sequence, i); if (!PyType_IsSubtype(value->ob_type, &WayType)) { PyErr_SetString(PyExc_TypeError, "The ways attribute value must be an instance of Way"); return -1; } // .OSMPBF.Way ::OSMPBF::Way *protoValue = ((Way *) value)->protobuf; self->protobuf->add_ways()->MergeFrom(*protoValue); } Py_XDECREF(sequence); return 0; } PyObject * fastpb_convertPrimitiveGrouprelations(const ::google::protobuf::Message &value) { Relation *obj = (Relation *) Relation_new(&RelationType, NULL, NULL); obj->protobuf->MergeFrom(value); return (PyObject *)obj; } PyObject * PrimitiveGroup_getrelations(PrimitiveGroup *self, void *closure) { int len = self->protobuf->relations_size(); PyObject *tuple = PyTuple_New(len); for (int i = 0; i < len; ++i) { PyObject *value = fastpb_convertPrimitiveGrouprelations( self->protobuf->relations(i)); if (!value) { return NULL; } PyTuple_SetItem(tuple, i, value); } return tuple; } int PrimitiveGroup_setrelations(PrimitiveGroup *self, PyObject *input, void *closure) { if (input == NULL || input == Py_None) { self->protobuf->clear_relations(); return 0; } if (PyString_Check(input)) { PyErr_SetString(PyExc_TypeError, "The relations attribute value must be a sequence"); return -1; } PyObject *sequence = PySequence_Fast(input, "The relations attribute value must be a sequence"); self->protobuf->clear_relations(); for (Py_ssize_t i = 0, len = PySequence_Length(sequence); i < len; ++i) { PyObject *value = PySequence_Fast_GET_ITEM(sequence, i); if (!PyType_IsSubtype(value->ob_type, &RelationType)) { PyErr_SetString(PyExc_TypeError, "The relations attribute value must be an instance of Relation"); return -1; } // .OSMPBF.Relation ::OSMPBF::Relation *protoValue = ((Relation *) value)->protobuf; self->protobuf->add_relations()->MergeFrom(*protoValue); } Py_XDECREF(sequence); return 0; } PyObject * fastpb_convertPrimitiveGroupchangesets(const ::google::protobuf::Message &value) { ChangeSet *obj = (ChangeSet *) ChangeSet_new(&ChangeSetType, NULL, NULL); obj->protobuf->MergeFrom(value); return (PyObject *)obj; } PyObject * PrimitiveGroup_getchangesets(PrimitiveGroup *self, void *closure) { int len = self->protobuf->changesets_size(); PyObject *tuple = PyTuple_New(len); for (int i = 0; i < len; ++i) { PyObject *value = fastpb_convertPrimitiveGroupchangesets( self->protobuf->changesets(i)); if (!value) { return NULL; } PyTuple_SetItem(tuple, i, value); } return tuple; } int PrimitiveGroup_setchangesets(PrimitiveGroup *self, PyObject *input, void *closure) { if (input == NULL || input == Py_None) { self->protobuf->clear_changesets(); return 0; } if (PyString_Check(input)) { PyErr_SetString(PyExc_TypeError, "The changesets attribute value must be a sequence"); return -1; } PyObject *sequence = PySequence_Fast(input, "The changesets attribute value must be a sequence"); self->protobuf->clear_changesets(); for (Py_ssize_t i = 0, len = PySequence_Length(sequence); i < len; ++i) { PyObject *value = PySequence_Fast_GET_ITEM(sequence, i); if (!PyType_IsSubtype(value->ob_type, &ChangeSetType)) { PyErr_SetString(PyExc_TypeError, "The changesets attribute value must be an instance of ChangeSet"); return -1; } // .OSMPBF.ChangeSet ::OSMPBF::ChangeSet *protoValue = ((ChangeSet *) value)->protobuf; self->protobuf->add_changesets()->MergeFrom(*protoValue); } Py_XDECREF(sequence); return 0; } int PrimitiveGroup_init(PrimitiveGroup *self, PyObject *args, PyObject *kwds) { PyObject *nodes = NULL; PyObject *dense = NULL; PyObject *ways = NULL; PyObject *relations = NULL; PyObject *changesets = NULL; static char *kwlist[] = { (char *) "nodes", (char *) "dense", (char *) "ways", (char *) "relations", (char *) "changesets", NULL }; if (! PyArg_ParseTupleAndKeywords( args, kwds, "|OOOOO", kwlist, &nodes,&dense,&ways,&relations,&changesets)) return -1; if (nodes) { if (PrimitiveGroup_setnodes(self, nodes, NULL) < 0) { return -1; } } if (dense) { if (PrimitiveGroup_setdense(self, dense, NULL) < 0) { return -1; } } if (ways) { if (PrimitiveGroup_setways(self, ways, NULL) < 0) { return -1; } } if (relations) { if (PrimitiveGroup_setrelations(self, relations, NULL) < 0) { return -1; } } if (changesets) { if (PrimitiveGroup_setchangesets(self, changesets, NULL) < 0) { return -1; } } return 0; } PyObject * PrimitiveGroup_richcompare(PyObject *self, PyObject *other, int op) { PyObject *result = NULL; if (!PyType_IsSubtype(other->ob_type, &PrimitiveGroupType)) { result = Py_NotImplemented; } else { // This is not a particularly efficient implementation since it never short circuits, but it's better // than nothing. It should probably only be used for tests. PrimitiveGroup *selfValue = (PrimitiveGroup *)self; PrimitiveGroup *otherValue = (PrimitiveGroup *)other; std::string selfSerialized; std::string otherSerialized; Py_BEGIN_ALLOW_THREADS selfValue->protobuf->SerializeToString(&selfSerialized); otherValue->protobuf->SerializeToString(&otherSerialized); Py_END_ALLOW_THREADS int cmp = selfSerialized.compare(otherSerialized); bool value = false; switch (op) { case Py_LT: value = cmp < 0; break; case Py_LE: value = cmp <= 0; break; case Py_EQ: value = cmp == 0; break; case Py_NE: value = cmp != 0; break; case Py_GT: value = cmp > 0; break; case Py_GE: value = cmp >= 0; break; } result = value ? Py_True : Py_False; } Py_XINCREF(result); return result; } static PyObject * PrimitiveGroup_repr(PyObject *selfObject) { PrimitiveGroup *self = (PrimitiveGroup *)selfObject; PyObject *member; PyObject *memberRepr; std::stringstream result; result << "PrimitiveGroup("; result << "nodes="; member = PrimitiveGroup_getnodes(self, NULL); memberRepr = PyObject_Repr(member); result << PyString_AsString(memberRepr); Py_XDECREF(memberRepr); Py_XDECREF(member); result << ", "; result << "dense="; member = PrimitiveGroup_getdense(self, NULL); memberRepr = PyObject_Repr(member); result << PyString_AsString(memberRepr); Py_XDECREF(memberRepr); Py_XDECREF(member); result << ", "; result << "ways="; member = PrimitiveGroup_getways(self, NULL); memberRepr = PyObject_Repr(member); result << PyString_AsString(memberRepr); Py_XDECREF(memberRepr); Py_XDECREF(member); result << ", "; result << "relations="; member = PrimitiveGroup_getrelations(self, NULL); memberRepr = PyObject_Repr(member); result << PyString_AsString(memberRepr); Py_XDECREF(memberRepr); Py_XDECREF(member); result << ", "; result << "changesets="; member = PrimitiveGroup_getchangesets(self, NULL); memberRepr = PyObject_Repr(member); result << PyString_AsString(memberRepr); Py_XDECREF(memberRepr); Py_XDECREF(member); result << ")"; std::string resultString = result.str(); return PyUnicode_Decode(resultString.data(), resultString.length(), "utf-8", NULL); } PyMemberDef PrimitiveGroup_members[] = { {NULL} // Sentinel }; PyGetSetDef PrimitiveGroup_getsetters[] = { {(char *)"nodes", (getter)PrimitiveGroup_getnodes, (setter)PrimitiveGroup_setnodes, (char *)"", NULL}, {(char *)"dense", (getter)PrimitiveGroup_getdense, (setter)PrimitiveGroup_setdense, (char *)"", NULL}, {(char *)"ways", (getter)PrimitiveGroup_getways, (setter)PrimitiveGroup_setways, (char *)"", NULL}, {(char *)"relations", (getter)PrimitiveGroup_getrelations, (setter)PrimitiveGroup_setrelations, (char *)"", NULL}, {(char *)"changesets", (getter)PrimitiveGroup_getchangesets, (setter)PrimitiveGroup_setchangesets, (char *)"", NULL}, {NULL} // Sentinel }; PyMethodDef PrimitiveGroup_methods[] = { {"DebugString", (PyCFunction)PrimitiveGroup_DebugString, METH_NOARGS, "Generates a human readable form of this message, useful for debugging and other purposes." }, {"SerializeToString", (PyCFunction)PrimitiveGroup_SerializeToString, METH_NOARGS, "Serializes the protocol buffer to a string." }, {"SerializeMany", (PyCFunction)PrimitiveGroup_SerializeMany, METH_O | METH_CLASS, "Serializes a sequence of protocol buffers to a string." }, {"ParseFromString", (PyCFunction)PrimitiveGroup_ParseFromString, METH_O, "Parses the protocol buffer from a string." }, {"ParseFromLongString", (PyCFunction)PrimitiveGroup_ParseFromLongString, METH_O, "Parses the protocol buffer from a string as large as 512MB." }, {"ParseMany", (PyCFunction)PrimitiveGroup_ParseMany, METH_VARARGS | METH_CLASS, "Parses many protocol buffers of this type from a string." }, {NULL} // Sentinel }; PyTypeObject PrimitiveGroupType = { PyObject_HEAD_INIT(NULL) 0, /*ob_size*/ "OSMPBF.PrimitiveGroup", /*tp_name*/ sizeof(PrimitiveGroup), /*tp_basicsize*/ 0, /*tp_itemsize*/ (destructor)PrimitiveGroup_dealloc, /*tp_dealloc*/ 0, /*tp_print*/ 0, /*tp_getattr*/ 0, /*tp_setattr*/ 0, /*tp_compare*/ PrimitiveGroup_repr, /*tp_repr*/ 0, /*tp_as_number*/ 0, /*tp_as_sequence*/ 0, /*tp_as_mapping*/ 0, /*tp_hash */ 0, /*tp_call*/ 0, /*tp_str*/ 0, /*tp_getattro*/ 0, /*tp_setattro*/ 0, /*tp_as_buffer*/ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_RICHCOMPARE, /*tp_flags*/ "PrimitiveGroup objects", /* tp_doc */ 0, /* tp_traverse */ 0, /* tp_clear */ PrimitiveGroup_richcompare, /* tp_richcompare */ 0, /* tp_weaklistoffset */ 0, /* tp_iter */ 0, /* tp_iternext */ PrimitiveGroup_methods, /* tp_methods */ PrimitiveGroup_members, /* tp_members */ PrimitiveGroup_getsetters, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ 0, /* tp_descr_set */ 0, /* tp_dictoffset */ (initproc)PrimitiveGroup_init, /* tp_init */ 0, /* tp_alloc */ PrimitiveGroup_new, /* tp_new */ }; } // Lets try not to pollute the global namespace namespace { // Forward-declaration for recursive structures extern PyTypeObject PrimitiveBlockType; typedef struct { PyObject_HEAD OSMPBF::PrimitiveBlock *protobuf; } PrimitiveBlock; void PrimitiveBlock_dealloc(PrimitiveBlock* self) { delete self->protobuf; self->ob_type->tp_free((PyObject*)self); } PyObject * PrimitiveBlock_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { PrimitiveBlock *self; self = (PrimitiveBlock *)type->tp_alloc(type, 0); self->protobuf = new OSMPBF::PrimitiveBlock(); return (PyObject *)self; } PyObject * PrimitiveBlock_DebugString(PrimitiveBlock* self) { std::string result; Py_BEGIN_ALLOW_THREADS result = self->protobuf->Utf8DebugString(); Py_END_ALLOW_THREADS return PyUnicode_FromStringAndSize(result.data(), result.length()); } PyObject * PrimitiveBlock_SerializeToString(PrimitiveBlock* self) { std::string result; Py_BEGIN_ALLOW_THREADS self->protobuf->SerializeToString(&result); Py_END_ALLOW_THREADS return PyString_FromStringAndSize(result.data(), result.length()); } PyObject * PrimitiveBlock_SerializeMany(void *nothing, PyObject *values) { std::string result; google::protobuf::io::ZeroCopyOutputStream* output = new google::protobuf::io::StringOutputStream(&result); google::protobuf::io::CodedOutputStream* outputStream = new google::protobuf::io::CodedOutputStream(output); PyObject *sequence = PySequence_Fast(values, "The values to serialize must be a sequence."); for (Py_ssize_t i = 0, len = PySequence_Length(sequence); i < len; ++i) { PrimitiveBlock *value = (PrimitiveBlock *)PySequence_Fast_GET_ITEM(sequence, i); Py_BEGIN_ALLOW_THREADS outputStream->WriteVarint32(value->protobuf->ByteSize()); value->protobuf->SerializeToCodedStream(outputStream); Py_END_ALLOW_THREADS } Py_XDECREF(sequence); delete outputStream; delete output; return PyString_FromStringAndSize(result.data(), result.length()); } PyObject * PrimitiveBlock_ParseFromString(PrimitiveBlock* self, PyObject *value) { std::string serialized(PyString_AsString(value), PyString_Size(value)); Py_BEGIN_ALLOW_THREADS self->protobuf->ParseFromString(serialized); Py_END_ALLOW_THREADS Py_RETURN_NONE; } PyObject * PrimitiveBlock_ParseFromLongString(PrimitiveBlock* self, PyObject *value) { google::protobuf::io::ZeroCopyInputStream* input = new google::protobuf::io::ArrayInputStream(PyString_AsString(value), PyString_Size(value)); google::protobuf::io::CodedInputStream* inputStream = new google::protobuf::io::CodedInputStream(input); inputStream->SetTotalBytesLimit(512 * 1024 * 1024, 512 * 1024 * 1024); Py_BEGIN_ALLOW_THREADS self->protobuf->ParseFromCodedStream(inputStream); Py_END_ALLOW_THREADS delete inputStream; delete input; Py_RETURN_NONE; } PyObject * PrimitiveBlock_ParseMany(void* nothing, PyObject *args) { PyObject *value; PyObject *callback; int fail = 0; if (!PyArg_ParseTuple(args, "OO", &value, &callback)) { return NULL; } google::protobuf::io::ZeroCopyInputStream* input = new google::protobuf::io::ArrayInputStream(PyString_AsString(value), PyString_Size(value)); google::protobuf::io::CodedInputStream* inputStream = new google::protobuf::io::CodedInputStream(input); inputStream->SetTotalBytesLimit(512 * 1024 * 1024, 512 * 1024 * 1024); google::protobuf::uint32 bytes; PyObject *single = NULL; while (inputStream->ReadVarint32(&bytes)) { google::protobuf::io::CodedInputStream::Limit messageLimit = inputStream->PushLimit(bytes); if (single == NULL) { single = PrimitiveBlock_new(&PrimitiveBlockType, NULL, NULL); } Py_BEGIN_ALLOW_THREADS ((PrimitiveBlock *)single)->protobuf->ParseFromCodedStream(inputStream); Py_END_ALLOW_THREADS inputStream->PopLimit(messageLimit); PyObject *result = PyObject_CallFunctionObjArgs(callback, single, NULL); if (result == NULL) { fail = 1; break; }; if (single->ob_refcnt != 1) { // If the callback saved a reference to the item, don't re-use it. Py_XDECREF(single); single = NULL; } } if (single != NULL) { Py_XDECREF(single); } delete inputStream; delete input; if (fail) { return NULL; } else { Py_RETURN_NONE; } } PyObject * fastpb_convertPrimitiveBlockstringtable(const ::google::protobuf::Message &value) { StringTable *obj = (StringTable *) StringTable_new(&StringTableType, NULL, NULL); obj->protobuf->MergeFrom(value); return (PyObject *)obj; } PyObject * PrimitiveBlock_getstringtable(PrimitiveBlock *self, void *closure) { if (! self->protobuf->has_stringtable()) { Py_RETURN_NONE; } return fastpb_convertPrimitiveBlockstringtable( self->protobuf->stringtable()); } int PrimitiveBlock_setstringtable(PrimitiveBlock *self, PyObject *input, void *closure) { if (input == NULL || input == Py_None) { self->protobuf->clear_stringtable(); return 0; } PyObject *value = input; if (!PyType_IsSubtype(value->ob_type, &StringTableType)) { PyErr_SetString(PyExc_TypeError, "The stringtable attribute value must be an instance of StringTable"); return -1; } // .OSMPBF.StringTable ::OSMPBF::StringTable *protoValue = ((StringTable *) value)->protobuf; self->protobuf->clear_stringtable(); self->protobuf->mutable_stringtable()->MergeFrom(*protoValue); return 0; } PyObject * fastpb_convertPrimitiveBlockprimitivegroup(const ::google::protobuf::Message &value) { PrimitiveGroup *obj = (PrimitiveGroup *) PrimitiveGroup_new(&PrimitiveGroupType, NULL, NULL); obj->protobuf->MergeFrom(value); return (PyObject *)obj; } PyObject * PrimitiveBlock_getprimitivegroup(PrimitiveBlock *self, void *closure) { int len = self->protobuf->primitivegroup_size(); PyObject *tuple = PyTuple_New(len); for (int i = 0; i < len; ++i) { PyObject *value = fastpb_convertPrimitiveBlockprimitivegroup( self->protobuf->primitivegroup(i)); if (!value) { return NULL; } PyTuple_SetItem(tuple, i, value); } return tuple; } int PrimitiveBlock_setprimitivegroup(PrimitiveBlock *self, PyObject *input, void *closure) { if (input == NULL || input == Py_None) { self->protobuf->clear_primitivegroup(); return 0; } if (PyString_Check(input)) { PyErr_SetString(PyExc_TypeError, "The primitivegroup attribute value must be a sequence"); return -1; } PyObject *sequence = PySequence_Fast(input, "The primitivegroup attribute value must be a sequence"); self->protobuf->clear_primitivegroup(); for (Py_ssize_t i = 0, len = PySequence_Length(sequence); i < len; ++i) { PyObject *value = PySequence_Fast_GET_ITEM(sequence, i); if (!PyType_IsSubtype(value->ob_type, &PrimitiveGroupType)) { PyErr_SetString(PyExc_TypeError, "The primitivegroup attribute value must be an instance of PrimitiveGroup"); return -1; } // .OSMPBF.PrimitiveGroup ::OSMPBF::PrimitiveGroup *protoValue = ((PrimitiveGroup *) value)->protobuf; self->protobuf->add_primitivegroup()->MergeFrom(*protoValue); } Py_XDECREF(sequence); return 0; } PyObject * PrimitiveBlock_getgranularity(PrimitiveBlock *self, void *closure) { if (! self->protobuf->has_granularity()) { Py_RETURN_NONE; } return fastpb_convert5( self->protobuf->granularity()); } int PrimitiveBlock_setgranularity(PrimitiveBlock *self, PyObject *input, void *closure) { if (input == NULL || input == Py_None) { self->protobuf->clear_granularity(); return 0; } PyObject *value = input; ::google::protobuf::int32 protoValue; // int32 if (PyInt_Check(value)) { protoValue = PyInt_AsLong(value); } else { PyErr_SetString(PyExc_TypeError, "The granularity attribute value must be an integer"); return -1; } self->protobuf->set_granularity(protoValue); return 0; } PyObject * PrimitiveBlock_getlat_offset(PrimitiveBlock *self, void *closure) { if (! self->protobuf->has_lat_offset()) { Py_RETURN_NONE; } return fastpb_convert3( self->protobuf->lat_offset()); } int PrimitiveBlock_setlat_offset(PrimitiveBlock *self, PyObject *input, void *closure) { if (input == NULL || input == Py_None) { self->protobuf->clear_lat_offset(); return 0; } PyObject *value = input; ::google::protobuf::int64 protoValue; // int64 if (PyInt_Check(value)) { protoValue = PyInt_AsLong(value); } else if (PyLong_Check(value)) { protoValue = PyLong_AsLongLong(value); } else { PyErr_SetString(PyExc_TypeError, "The lat_offset attribute value must be an integer"); return -1; } self->protobuf->set_lat_offset(protoValue); return 0; } PyObject * PrimitiveBlock_getlon_offset(PrimitiveBlock *self, void *closure) { if (! self->protobuf->has_lon_offset()) { Py_RETURN_NONE; } return fastpb_convert3( self->protobuf->lon_offset()); } int PrimitiveBlock_setlon_offset(PrimitiveBlock *self, PyObject *input, void *closure) { if (input == NULL || input == Py_None) { self->protobuf->clear_lon_offset(); return 0; } PyObject *value = input; ::google::protobuf::int64 protoValue; // int64 if (PyInt_Check(value)) { protoValue = PyInt_AsLong(value); } else if (PyLong_Check(value)) { protoValue = PyLong_AsLongLong(value); } else { PyErr_SetString(PyExc_TypeError, "The lon_offset attribute value must be an integer"); return -1; } self->protobuf->set_lon_offset(protoValue); return 0; } PyObject * PrimitiveBlock_getdate_granularity(PrimitiveBlock *self, void *closure) { if (! self->protobuf->has_date_granularity()) { Py_RETURN_NONE; } return fastpb_convert5( self->protobuf->date_granularity()); } int PrimitiveBlock_setdate_granularity(PrimitiveBlock *self, PyObject *input, void *closure) { if (input == NULL || input == Py_None) { self->protobuf->clear_date_granularity(); return 0; } PyObject *value = input; ::google::protobuf::int32 protoValue; // int32 if (PyInt_Check(value)) { protoValue = PyInt_AsLong(value); } else { PyErr_SetString(PyExc_TypeError, "The date_granularity attribute value must be an integer"); return -1; } self->protobuf->set_date_granularity(protoValue); return 0; } int PrimitiveBlock_init(PrimitiveBlock *self, PyObject *args, PyObject *kwds) { PyObject *stringtable = NULL; PyObject *primitivegroup = NULL; PyObject *granularity = NULL; PyObject *lat_offset = NULL; PyObject *lon_offset = NULL; PyObject *date_granularity = NULL; static char *kwlist[] = { (char *) "stringtable", (char *) "primitivegroup", (char *) "granularity", (char *) "lat_offset", (char *) "lon_offset", (char *) "date_granularity", NULL }; if (! PyArg_ParseTupleAndKeywords( args, kwds, "|OOOOOO", kwlist, &stringtable,&primitivegroup,&granularity,&lat_offset,&lon_offset,&date_granularity)) return -1; if (stringtable) { if (PrimitiveBlock_setstringtable(self, stringtable, NULL) < 0) { return -1; } } if (primitivegroup) { if (PrimitiveBlock_setprimitivegroup(self, primitivegroup, NULL) < 0) { return -1; } } if (granularity) { if (PrimitiveBlock_setgranularity(self, granularity, NULL) < 0) { return -1; } } if (lat_offset) { if (PrimitiveBlock_setlat_offset(self, lat_offset, NULL) < 0) { return -1; } } if (lon_offset) { if (PrimitiveBlock_setlon_offset(self, lon_offset, NULL) < 0) { return -1; } } if (date_granularity) { if (PrimitiveBlock_setdate_granularity(self, date_granularity, NULL) < 0) { return -1; } } return 0; } PyObject * PrimitiveBlock_richcompare(PyObject *self, PyObject *other, int op) { PyObject *result = NULL; if (!PyType_IsSubtype(other->ob_type, &PrimitiveBlockType)) { result = Py_NotImplemented; } else { // This is not a particularly efficient implementation since it never short circuits, but it's better // than nothing. It should probably only be used for tests. PrimitiveBlock *selfValue = (PrimitiveBlock *)self; PrimitiveBlock *otherValue = (PrimitiveBlock *)other; std::string selfSerialized; std::string otherSerialized; Py_BEGIN_ALLOW_THREADS selfValue->protobuf->SerializeToString(&selfSerialized); otherValue->protobuf->SerializeToString(&otherSerialized); Py_END_ALLOW_THREADS int cmp = selfSerialized.compare(otherSerialized); bool value = false; switch (op) { case Py_LT: value = cmp < 0; break; case Py_LE: value = cmp <= 0; break; case Py_EQ: value = cmp == 0; break; case Py_NE: value = cmp != 0; break; case Py_GT: value = cmp > 0; break; case Py_GE: value = cmp >= 0; break; } result = value ? Py_True : Py_False; } Py_XINCREF(result); return result; } static PyObject * PrimitiveBlock_repr(PyObject *selfObject) { PrimitiveBlock *self = (PrimitiveBlock *)selfObject; PyObject *member; PyObject *memberRepr; std::stringstream result; result << "PrimitiveBlock("; result << "stringtable="; member = PrimitiveBlock_getstringtable(self, NULL); memberRepr = PyObject_Repr(member); result << PyString_AsString(memberRepr); Py_XDECREF(memberRepr); Py_XDECREF(member); result << ", "; result << "primitivegroup="; member = PrimitiveBlock_getprimitivegroup(self, NULL); memberRepr = PyObject_Repr(member); result << PyString_AsString(memberRepr); Py_XDECREF(memberRepr); Py_XDECREF(member); result << ", "; result << "granularity="; member = PrimitiveBlock_getgranularity(self, NULL); memberRepr = PyObject_Repr(member); result << PyString_AsString(memberRepr); Py_XDECREF(memberRepr); Py_XDECREF(member); result << ", "; result << "lat_offset="; member = PrimitiveBlock_getlat_offset(self, NULL); memberRepr = PyObject_Repr(member); result << PyString_AsString(memberRepr); Py_XDECREF(memberRepr); Py_XDECREF(member); result << ", "; result << "lon_offset="; member = PrimitiveBlock_getlon_offset(self, NULL); memberRepr = PyObject_Repr(member); result << PyString_AsString(memberRepr); Py_XDECREF(memberRepr); Py_XDECREF(member); result << ", "; result << "date_granularity="; member = PrimitiveBlock_getdate_granularity(self, NULL); memberRepr = PyObject_Repr(member); result << PyString_AsString(memberRepr); Py_XDECREF(memberRepr); Py_XDECREF(member); result << ")"; std::string resultString = result.str(); return PyUnicode_Decode(resultString.data(), resultString.length(), "utf-8", NULL); } PyMemberDef PrimitiveBlock_members[] = { {NULL} // Sentinel }; PyGetSetDef PrimitiveBlock_getsetters[] = { {(char *)"stringtable", (getter)PrimitiveBlock_getstringtable, (setter)PrimitiveBlock_setstringtable, (char *)"", NULL}, {(char *)"primitivegroup", (getter)PrimitiveBlock_getprimitivegroup, (setter)PrimitiveBlock_setprimitivegroup, (char *)"", NULL}, {(char *)"granularity", (getter)PrimitiveBlock_getgranularity, (setter)PrimitiveBlock_setgranularity, (char *)"", NULL}, {(char *)"lat_offset", (getter)PrimitiveBlock_getlat_offset, (setter)PrimitiveBlock_setlat_offset, (char *)"", NULL}, {(char *)"lon_offset", (getter)PrimitiveBlock_getlon_offset, (setter)PrimitiveBlock_setlon_offset, (char *)"", NULL}, {(char *)"date_granularity", (getter)PrimitiveBlock_getdate_granularity, (setter)PrimitiveBlock_setdate_granularity, (char *)"", NULL}, {NULL} // Sentinel }; PyMethodDef PrimitiveBlock_methods[] = { {"DebugString", (PyCFunction)PrimitiveBlock_DebugString, METH_NOARGS, "Generates a human readable form of this message, useful for debugging and other purposes." }, {"SerializeToString", (PyCFunction)PrimitiveBlock_SerializeToString, METH_NOARGS, "Serializes the protocol buffer to a string." }, {"SerializeMany", (PyCFunction)PrimitiveBlock_SerializeMany, METH_O | METH_CLASS, "Serializes a sequence of protocol buffers to a string." }, {"ParseFromString", (PyCFunction)PrimitiveBlock_ParseFromString, METH_O, "Parses the protocol buffer from a string." }, {"ParseFromLongString", (PyCFunction)PrimitiveBlock_ParseFromLongString, METH_O, "Parses the protocol buffer from a string as large as 512MB." }, {"ParseMany", (PyCFunction)PrimitiveBlock_ParseMany, METH_VARARGS | METH_CLASS, "Parses many protocol buffers of this type from a string." }, {NULL} // Sentinel }; PyTypeObject PrimitiveBlockType = { PyObject_HEAD_INIT(NULL) 0, /*ob_size*/ "OSMPBF.PrimitiveBlock", /*tp_name*/ sizeof(PrimitiveBlock), /*tp_basicsize*/ 0, /*tp_itemsize*/ (destructor)PrimitiveBlock_dealloc, /*tp_dealloc*/ 0, /*tp_print*/ 0, /*tp_getattr*/ 0, /*tp_setattr*/ 0, /*tp_compare*/ PrimitiveBlock_repr, /*tp_repr*/ 0, /*tp_as_number*/ 0, /*tp_as_sequence*/ 0, /*tp_as_mapping*/ 0, /*tp_hash */ 0, /*tp_call*/ 0, /*tp_str*/ 0, /*tp_getattro*/ 0, /*tp_setattro*/ 0, /*tp_as_buffer*/ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_RICHCOMPARE, /*tp_flags*/ "PrimitiveBlock objects", /* tp_doc */ 0, /* tp_traverse */ 0, /* tp_clear */ PrimitiveBlock_richcompare, /* tp_richcompare */ 0, /* tp_weaklistoffset */ 0, /* tp_iter */ 0, /* tp_iternext */ PrimitiveBlock_methods, /* tp_methods */ PrimitiveBlock_members, /* tp_members */ PrimitiveBlock_getsetters, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ 0, /* tp_descr_set */ 0, /* tp_dictoffset */ (initproc)PrimitiveBlock_init, /* tp_init */ 0, /* tp_alloc */ PrimitiveBlock_new, /* tp_new */ }; } static PyMethodDef module_methods[] = { {NULL} // Sentinel }; #ifndef PyMODINIT_FUNC // Declarations for DLL import/export. #define PyMODINIT_FUNC void #endif PyMODINIT_FUNC initOSMPBF(void) { GOOGLE_PROTOBUF_VERIFY_VERSION; PyObject* m; if (PyType_Ready(&BlobType) < 0) return; if (PyType_Ready(&BlobHeaderType) < 0) return; if (PyType_Ready(&ChangeSetType) < 0) return; if (PyType_Ready(&DenseInfoType) < 0) return; if (PyType_Ready(&HeaderBBoxType) < 0) return; if (PyType_Ready(&InfoType) < 0) return; if (PyType_Ready(&StringTableType) < 0) return; if (PyType_Ready(&DenseNodesType) < 0) return; if (PyType_Ready(&HeaderBlockType) < 0) return; if (PyType_Ready(&NodeType) < 0) return; if (PyType_Ready(&RelationType) < 0) return; if (PyType_Ready(&WayType) < 0) return; if (PyType_Ready(&PrimitiveGroupType) < 0) return; if (PyType_Ready(&PrimitiveBlockType) < 0) return; m = Py_InitModule3("OSMPBF", module_methods, ""); if (m == NULL) return; Py_INCREF(&BlobType); PyModule_AddObject(m, "Blob", (PyObject *)&BlobType); Py_INCREF(&BlobHeaderType); PyModule_AddObject(m, "BlobHeader", (PyObject *)&BlobHeaderType); Py_INCREF(&ChangeSetType); PyModule_AddObject(m, "ChangeSet", (PyObject *)&ChangeSetType); Py_INCREF(&DenseInfoType); PyModule_AddObject(m, "DenseInfo", (PyObject *)&DenseInfoType); Py_INCREF(&HeaderBBoxType); PyModule_AddObject(m, "HeaderBBox", (PyObject *)&HeaderBBoxType); Py_INCREF(&InfoType); PyModule_AddObject(m, "Info", (PyObject *)&InfoType); Py_INCREF(&StringTableType); PyModule_AddObject(m, "StringTable", (PyObject *)&StringTableType); Py_INCREF(&DenseNodesType); PyModule_AddObject(m, "DenseNodes", (PyObject *)&DenseNodesType); Py_INCREF(&HeaderBlockType); PyModule_AddObject(m, "HeaderBlock", (PyObject *)&HeaderBlockType); Py_INCREF(&NodeType); PyModule_AddObject(m, "Node", (PyObject *)&NodeType); Py_INCREF(&RelationType); PyModule_AddObject(m, "Relation", (PyObject *)&RelationType); Py_INCREF(&WayType); PyModule_AddObject(m, "Way", (PyObject *)&WayType); Py_INCREF(&PrimitiveGroupType); PyModule_AddObject(m, "PrimitiveGroup", (PyObject *)&PrimitiveGroupType); Py_INCREF(&PrimitiveBlockType); PyModule_AddObject(m, "PrimitiveBlock", (PyObject *)&PrimitiveBlockType); }imposm.parser-1.0.7/imposm/parser/pbf/parser.py0000644000076500000240000003334012214610523021453 0ustar oltstaff00000000000000# Copyright 2011 Omniscale GmbH & Co. KG # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import with_statement import struct import sys import zlib from marshal import dumps from imposm.parser.pbf import OSMPBF SUPPORTED_FEATURES = set(['OsmSchema-V0.6', 'DenseNodes']) _MEMBERTYPE = {0 : 'node', 1 : 'way', 2 : 'relation'} class PBFParser(object): """ OSM PBF parser. :param xxx_callback: callback functions for coords, nodes, ways and relations. Each callback function gets called with a list of multiple elements. :param xxx_filter: functions that can manipulate the tag dictionary. Nodes and relations without tags will not passed to the callback. :param marshal: return the data as a marshaled string """ def __init__(self, nodes_callback=None, ways_callback=None, relations_callback=None, coords_callback=None, nodes_tag_filter=None, ways_tag_filter=None, relations_tag_filter=None, marshal=False): self.nodes_callback = nodes_callback self.ways_callback = ways_callback self.relations_callback = relations_callback self.coords_callback = coords_callback self.nodes_tag_filter = nodes_tag_filter self.ways_tag_filter = ways_tag_filter self.relations_tag_filter = relations_tag_filter self.marshal = marshal def parse(self, filename, offset, size): """ Parse primitive block from `filename`. :param filename: path to PBF file :param offset: byte offset of the primitive block to parse :param size: size in bytes of the primitive block to parse """ reader = PrimitiveBlockParser(filename, offset, size) if self.nodes_callback or self.coords_callback: self.handle_nodes(reader) if self.ways_callback: self.handle_ways(reader) if self.relations_callback: self.handle_relations(reader) def handle_nodes(self, reader): nodes = [] coords = [] nodes_callback = self.nodes_callback coords_callback = self.coords_callback for node in reader.nodes(): if nodes_callback: if self.nodes_tag_filter: self.nodes_tag_filter(node[1]) if node[1]: if self.marshal: nodes.append((node[0], dumps((node[1], node[2]), 2))) else: nodes.append((node[0], node[1], node[2])) if len(nodes) >= 256: nodes_callback(nodes) nodes = [] if coords_callback: coords.append((node[0], node[2][0], node[2][1])) if len(coords) >= 512: coords_callback(coords) coords = [] if nodes_callback: nodes_callback(nodes) if coords_callback: coords_callback(coords) def handle_ways(self, reader): ways = [] for way in reader.ways(): if self.ways_tag_filter: self.ways_tag_filter(way[1]) # always return ways, might be needed for relations if self.marshal: ways.append((way[0], dumps((way[1], way[2]), 2))) else: ways.append((way[0], way[1], way[2])) if len(ways) >= 256: self.ways_callback(ways) ways = [] self.ways_callback(ways) def handle_relations(self, reader): relations = [] for relation in reader.relations(): if self.relations_tag_filter: self.relations_tag_filter(relation[1]) if not relation[1]: continue if self.marshal: relations.append((relation[0], dumps((relation[1], relation[2]), 2))) else: relations.append((relation[0], relation[1], relation[2])) if len(relations) >= 256: self.relations_callback(relations) relations = [] self.relations_callback(relations) def decoded_stringtable(stringtable): result = [] for s in stringtable: result.append(s.decode('utf-8')) return result class PrimitiveBlockParser(object): """ Low level PBF primitive block parser. Parses a single primitive block and handles OSM PBF internals like dense nodes, delta encoding, stringtables, etc. :param filename: path to PBF file :param offset: byte offset of the primitive block to parse :param size: size in bytes of the primitive block to parse """ def __init__(self, filename, blob_pos, blob_size): self.pos = filename, blob_pos, blob_size data = read_blob_data(filename, blob_pos, blob_size) self.primitive_block = OSMPBF.PrimitiveBlock() self.primitive_block.ParseFromString(data) self.primitivegroup = self.primitive_block.primitivegroup self.stringtable = decoded_stringtable(self.primitive_block.stringtable.s) def __repr__(self): return '' % (self.pos, ) def _get_tags(self, element, pos): tags = {} key = None value = None keyflag = False if pos >= len(element): return {}, pos while True: key_val = element[pos] pos += 1 if key_val == 0: break if not keyflag: key = key_val keyflag = True else: value = key_val tags[self.stringtable[key]] = self.stringtable[value] keyflag = False return tags, pos def nodes(self): """ Return an iterator for all *nodes* in this primitive block. :rtype: iterator of ``(osm_id, tags, (lon, lat))`` tuples """ for group in self.primitivegroup: dense = group.dense if dense: granularity = self.primitive_block.granularity or 100 lat_offset = self.primitive_block.lat_offset or 0 lon_offset = self.primitive_block.lon_offset or 0 coord_scale = 0.000000001 get_tags = self._get_tags ids = dense.id lats = dense.lat lons = dense.lon keys_vals = dense.keys_vals last_id = last_lat = last_lon = last_keysvals_pos = 0 for i in xrange(len(ids)): last_id += ids[i] last_lat += coord_scale * (lat_offset + (granularity * lats[i])) last_lon += coord_scale * (lon_offset + (granularity * lons[i])) tags, last_keysvals_pos = get_tags(keys_vals, last_keysvals_pos) yield (last_id, tags, (last_lon, last_lat)) nodes = group.nodes if nodes: for node in nodes: keys, vals = node.keys, node.vals tags = [] for i in xrange(len(keys)): tags.append((self.stringtable[keys[i]], self.stringtable[vals[i]])) yield (node.id, tags, (node.lon, node.lat)) def ways(self): """ Return an iterator for all *ways* in this primitive block. :rtype: iterator of ``(osm_id, tags, [ref1, ref2, ...])`` tuples """ for group in self.primitivegroup: ways = group.ways if ways: for way in ways: keys = way.keys vals = way.vals delta_refs = way.refs tags = {} for i in xrange(len(keys)): tags[self.stringtable[keys[i]]] = self.stringtable[vals[i]] refs = [] ref = 0 for delta in delta_refs: ref += delta refs.append(ref) yield (way.id, tags, refs) def relations(self): """ Return an iterator for all *relations* in this primitive block. :rtype: iterator of ``(osm_id, tags, [(ref1, type, role), ...])`` tuples """ for group in self.primitivegroup: relations = group.relations if relations: for relation in relations: members = [] memids = relation.memids rel_types = relation.types roles_sids = relation.roles_sid keys = relation.keys vals = relation.vals memid = 0 for i in xrange(len(rel_types)): memid += memids[i] members.append((memid, _MEMBERTYPE[rel_types[i]], self.stringtable[roles_sids[i]])) tags = {} for i in xrange(len(keys)): tags[self.stringtable[keys[i]]] = self.stringtable[vals[i]] yield (relation.id, tags, members) class PBFHeader(object): def __init__(self, filename, blob_pos, blob_size): data = read_blob_data(filename, blob_pos, blob_size) self.header_block = OSMPBF.HeaderBlock() self.header_block.ParseFromString(data) def required_features(self): return set(self.header_block.required_features) def read_blob_data(filename, blob_pos, blob_size): """ Returns the (unzipped) blob data from filename and position. """ with open(filename, 'rb') as f: f.seek(blob_pos) blob_data = f.read(blob_size) blob = OSMPBF.Blob() blob.ParseFromString(blob_data) raw_data = blob.raw if raw_data: return raw_data return zlib.decompress(blob.zlib_data) import time class PBFFile(object): """ OSM PBF file reader. Parses the low-level file structure with header sizes, offsets and blob headers. :param filename: path to the PBF file """ def __init__(self, filename): self.filename = filename self.file = open(filename, 'rb') self.next_blob_pos = self.prev_blob_pos = 0 header_offsets = self._skip_header() self.header = PBFHeader(self.filename, header_offsets['blob_pos'], header_offsets['blob_size']) self.check_features() def check_features(self): missing_features = self.header.required_features().difference(SUPPORTED_FEATURES) if missing_features: raise NotImplementedError( '%s requires features not implemented by this parser: %s' % (self.filename, ', '.join(missing_features)) ) def _skip_header(self): return self.blob_offsets().next() def seek(self, pos): self.next_blob_pos = pos def rewind(self): self.next_blob_pos = self.prev_blob_pos def blob_offsets(self): """ Returns an iterator of the blob offsets in this file. Each offsets is stored in a dictionary with: - `filename` the path of this PBF file. - `blob_pos` the byte offset - `blob_size` the size of this blob in bytes """ while True: self.file.seek(self.next_blob_pos) blob_header_size = self._blob_header_size() if not blob_header_size: break blob_size = self._blob_size(self.file.read(blob_header_size)) blob_pos = self.next_blob_pos + 4 + blob_header_size blob_header_pos=self.next_blob_pos, prev_blob_header_pos = self.prev_blob_pos self.prev_blob_pos = self.next_blob_pos self.next_blob_pos = blob_pos + blob_size yield dict(blob_pos=blob_pos, blob_size=blob_size, blob_header_pos=blob_header_pos, prev_blob_header_pos=prev_blob_header_pos, filename=self.filename) def primitive_block_parsers(self): """ Returns an iterator of PrimitiveBlockParser. """ for pos in self.blob_offsets(): yield PrimitiveBlockParser(self.filename, pos['blob_pos'], pos['blob_size']) def _blob_size(self, data): blob_header = OSMPBF.BlobHeader() blob_header.ParseFromString(data) return blob_header.datasize def _blob_header_size(self): bytes = self.file.read(4) if bytes: return struct.unpack('!i', bytes)[0] return None def read_pbf(filename): pbf = PBFFile(filename) for block in pbf.primitive_block_parsers(): for node in block.nodes(): pass for way in block.ways(): pass for relation in block.relations(): pass if __name__ == '__main__': from timeit import Timer n = 1 r = 1 print "reading %s, number of calls: %d, repeat: %d" %(sys.argv[1],n,r) t = Timer("read_pbf(sys.argv[1])", "from __main__ import read_pbf") times = t.repeat(r,n) avrg_times = [] for time in times: avrg_times.append(time/n) print "avrg time/call: %f" %(min(avrg_times)) imposm.parser-1.0.7/imposm/parser/simple.py0000644000076500000240000001305212012641030020670 0ustar oltstaff00000000000000# Copyright 2011 Omniscale GmbH & Co. KG # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import with_statement import multiprocessing import sys import time from Queue import Empty from imposm.parser.util import default_concurrency, fileinput, setproctitle class OSMParser(object): """ High-level OSM parser. :param concurrency: number of parser processes to start. Defaults to the number of CPUs. :param xxx_callback: callback functions for coords, nodes, ways and relations. Each callback function gets called with a list of multiple elements. See :ref:`callback concepts `. :param xxx_filter: functions that can manipulate the tag dictionary. Nodes and relations without tags will not passed to the callback. See :ref:`tag filter concepts `. """ def __init__(self, concurrency=None, nodes_callback=None, ways_callback=None, relations_callback=None, coords_callback=None, nodes_tag_filter=None, ways_tag_filter=None, relations_tag_filter=None, marshal_elem_data=False): self.concurrency = concurrency or default_concurrency() assert self.concurrency >= 1 self.nodes_callback = nodes_callback self.ways_callback = ways_callback self.relations_callback = relations_callback self.coords_callback = coords_callback self.nodes_tag_filter = nodes_tag_filter self.ways_tag_filter = ways_tag_filter self.relations_tag_filter = relations_tag_filter self.marshal_elem_data = marshal_elem_data def parse(self, filename): """ Parse the given file. Detects the filetype based on the file suffix. Supports ``.pbf``, ``.osm`` and ``.osm.bz2``. """ if filename.endswith('.pbf'): return self.parse_pbf_file(filename) elif filename.endswith(('.osm', '.osm.bz2')): return self.parse_xml_file(filename) else: raise NotImplementedError('unknown file extension') def parse_pbf_file(self, filename): """ Parse a PBF file. """ from imposm.parser.pbf.multiproc import PBFMultiProcParser return self._parse(filename, PBFMultiProcParser) def parse_xml_file(self, filename): """ Parse a XML file. Supports BZip2 compressed files if the filename ends with ``.bz2``. """ from imposm.parser.xml.multiproc import XMLMultiProcParser with fileinput(filename) as input: return self._parse(input, XMLMultiProcParser) def _parse(self, input, parser_class): queues_callbacks = {} if self.coords_callback: queues_callbacks['coords'] = (multiprocessing.JoinableQueue(512), self.coords_callback) if self.nodes_callback: queues_callbacks['nodes'] = (multiprocessing.JoinableQueue(128), self.nodes_callback) if self.ways_callback: queues_callbacks['ways'] = (multiprocessing.JoinableQueue(128), self.ways_callback) if self.relations_callback: queues_callbacks['relations'] = (multiprocessing.JoinableQueue(128), self.relations_callback) def parse_it(): setproctitle('imposm parser') queues = dict([(type, q) for type, (q, c) in queues_callbacks.items()]) parser = parser_class(self.concurrency, ways_queue=queues.get('ways'), coords_queue=queues.get('coords'), nodes_queue=queues.get('nodes'), relations_queue=queues.get('relations'), marshal_elem_data=self.marshal_elem_data ) parser.nodes_tag_filter = self.nodes_tag_filter parser.ways_tag_filter = self.ways_tag_filter parser.relations_tag_filter = self.relations_tag_filter parser.parse(input) for q in queues.values(): q.put(None) proc = multiprocessing.Process(target=parse_it) proc.start() while queues_callbacks: processed = False for items_type, (queue, callback) in queues_callbacks.items(): try: items = None while True: items = queue.get_nowait() if items is None: queue.task_done() del queues_callbacks[items_type] break else: callback(items) if items: processed = True except Empty: pass if not processed: # wait a ms if all queues were empty # to give the parser a chance to fill them up time.sleep(0.001) proc.join() imposm.parser-1.0.7/imposm/parser/test/0000755000076500000240000000000012454223055020020 5ustar oltstaff00000000000000imposm.parser-1.0.7/imposm/parser/test/__init__.py0000644000076500000240000000000011766117447022133 0ustar oltstaff00000000000000imposm.parser-1.0.7/imposm/parser/test/test.osm0000644000076500000240000000144211766117447021534 0ustar oltstaff00000000000000 imposm.parser-1.0.7/imposm/parser/test/test.osm.bz20000644000076500000240000000065511766117447022235 0ustar oltstaff00000000000000BZh91AY&SY[0P@B @8J24 4=hJj i4i#d3 YFp How<\fU[I0,GJ&lN`'ZS4ղ$ˤ&5 9HdȏPd=FEA$sAegyv6lBC$7Kk E782jT)\4ka$%.Xwa@09m` 0U3f"cCDG{j! OB - dʂ:b+2>jZk )&@i ^s;nabb"5فp30p8"PRCC@NRQyǏ8߿rE8P[imposm.parser-1.0.7/imposm/parser/test/test.pbf0000644000076500000240000000051211766117447021502 0ustar oltstaff00000000000000 OSMHeader:.6xS/ NHM 33SrI+NOI-nbs -22521E z OSMDataxbb/I-.I/bKMbK-bre&Urq%&'Urgdg'Vrd$pshk᥇w?jXpx%k~iIj#ۿq2H111(;q6Ly}N/ֆ Al lL, BrR2BRJ|66=b`U``t)"$ g8>} F<7F )_m3;Bimposm.parser-1.0.7/imposm/parser/test/test_simple_parser.py0000644000076500000240000000652411766117447024321 0ustar oltstaff00000000000000# -:- encoding: utf8 -:- # Copyright 2011 Omniscale GmbH & Co. KG # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os from imposm.parser import OSMParser from nose.tools import eq_ class ParserTestBase(object): osm_filename = None ways_filter = None nodes_filter = None relations_filter = None def __init__(self): self.nodes = [] self.coords = [] self.ways = [] self.relations = [] def parse_nodes(self, nodes): self.nodes.extend(nodes) def parse_coords(self, coords): self.coords.extend(coords) def parse_ways(self, ways): self.ways.extend(ways) def parse_relations(self, relations): self.relations.extend(relations) def parse(self): parser = OSMParser(2, nodes_callback=self.parse_nodes, coords_callback=self.parse_coords, ways_callback=self.parse_ways, relations_callback=self.parse_relations, nodes_tag_filter=self.nodes_filter, ways_tag_filter=self.ways_filter, relations_tag_filter=self.relations_filter, ) osm_filename = os.path.join(os.path.dirname(__file__), self.osm_filename) parser.parse(osm_filename) def test_parse_result(self): self.parse() eq_(len(self.nodes), 1) eq_(self.nodes[0], (2, {'name': 'test', 'created_by': 'hand'}, (10.0, 51.0))) eq_(len(self.coords), 2) eq_(self.coords[0], (1, 10.0, 50.0)) eq_(self.coords[1], (2, 10.0, 51.0)) eq_(len(self.ways), 1) eq_(self.ways[0], (3, {'highway': 'primary'}, [1, 2])) eq_(len(self.relations), 1) eq_(self.relations[0], (4, {'name': u'ܵlåû†é'}, [(123, 'way', 'outer'), (124, 'way', 'inner')])) class ParserTestBaseWithFilter(ParserTestBase): def nodes_filter(self, tags): for tag in tags.keys(): if tag != 'name': del tags[tag] ways_filter = nodes_filter def relations_filter(self, tags): tags.clear() def test_parse_result(self): self.parse() eq_(len(self.nodes), 1) eq_(self.nodes[0], (2, {'name': 'test'}, (10.0, 51.0))) eq_(len(self.coords), 2) eq_(self.coords[0], (1, 10.0, 50.0)) eq_(self.coords[1], (2, 10.0, 51.0)) eq_(len(self.ways), 1) eq_(self.ways[0], (3, {}, [1, 2])) eq_(len(self.relations), 0) class TestXML(ParserTestBase): osm_filename = 'test.osm' class TestBZIP2(ParserTestBase): osm_filename = 'test.osm.bz2' class TestPBF(ParserTestBase): osm_filename = 'test.pbf' class TestXMLWithFilter(ParserTestBaseWithFilter): osm_filename = 'test.osm' class TestPBFWithFilter(ParserTestBaseWithFilter): osm_filename = 'test.pbf' imposm.parser-1.0.7/imposm/parser/util.py0000644000076500000240000000300211766117447020377 0ustar oltstaff00000000000000# Copyright 2011 Omniscale GmbH & Co. KG # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import contextlib import multiprocessing import subprocess try: from setproctitle import setproctitle setproctitle except ImportError: setproctitle = lambda x: None def default_concurrency(): return multiprocessing.cpu_count() def bzip_reader(filename): p = subprocess.Popen(['bunzip2', '-c', filename], bufsize=-1, stdout=subprocess.PIPE) return p.stdout @contextlib.contextmanager def fileinput(filename): if filename.endswith('bz2'): yield bzip_reader(filename) else: fh = open(filename, 'rb') yield fh fh.close() def estimate_records(files): records = 0 for f in files: fsize = os.path.getsize(f) if f.endswith('.bz2'): fsize *= 11 # observed bzip2 compression factor on osm data if f.endswith('.pbf'): fsize *= 15 # observed pbf compression factor on osm data records += fsize/200 return int(records)imposm.parser-1.0.7/imposm/parser/xml/0000755000076500000240000000000012454223055017641 5ustar oltstaff00000000000000imposm.parser-1.0.7/imposm/parser/xml/__init__.py0000644000076500000240000000000011766117447021754 0ustar oltstaff00000000000000imposm.parser-1.0.7/imposm/parser/xml/multiproc.py0000644000076500000240000002241612071316555022241 0ustar oltstaff00000000000000# Copyright 2011 Omniscale GmbH & Co. KG # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import mmap import multiprocessing import re from Queue import Empty from imposm.parser.xml.parser import XMLParser from imposm.parser.util import setproctitle KiB = 1024 MiB = 1024*KiB READ_SIZE = 512*KiB class MMapReader(object): def __init__(self, m, size): self.m = m self.m.seek(0) self.size = size def read(self, size=None): if size is None: size = self.size - self.m.tell() else: size = min(self.size - self.m.tell(), size) return self.m.read(size) def readline(self): cur_pos = self.m.tell() if cur_pos >= self.size: return nl_pos = self.m.find('\n') self.m.seek(cur_pos) return self.m.read(nl_pos-cur_pos) def seek(self, n): self.m.seek(n) class XMLParserProcess(XMLParser, multiprocessing.Process): def __init__(self, mmap_pool, mmap_queue, *args, **kw): multiprocessing.Process.__init__(self) XMLParser.__init__(self, *args, **kw) self.daemon = True self.mmap_pool = mmap_pool self.mmap_queue = mmap_queue def run(self): setproctitle('imposm xml parser') while True: mmap_idx, size = self.mmap_queue.get() if mmap_idx is None: self.mmap_queue.task_done() break xml = MMapReader(self.mmap_pool.get(mmap_idx), size) self.parse(xml) self.mmap_queue.task_done() self.mmap_pool.free(mmap_idx) class XMLMultiProcParser(object): nodes_tag_filter = None ways_tag_filter = None relations_tag_filter = None def __init__(self, pool_size, nodes_queue=None, ways_queue=None, relations_queue=None, coords_queue=None, marshal_elem_data=False): self.pool_size = pool_size self.pool = [] self.nodes_callback = nodes_queue.put if nodes_queue else None self.ways_callback = ways_queue.put if ways_queue else None self.relations_callback = relations_queue.put if relations_queue else None self.coords_callback = coords_queue.put if coords_queue else None xml_chunk_size=READ_SIZE self.mmap_pool = MMapPool(pool_size*8, xml_chunk_size*8) self.mmap_queue = multiprocessing.JoinableQueue(8) self.marshal_elem_data = marshal_elem_data def parse(self, stream): assert not self.pool for _ in xrange(self.pool_size): proc = XMLParserProcess(self.mmap_pool, self.mmap_queue, nodes_callback=self.nodes_callback, coords_callback=self.coords_callback, ways_callback=self.ways_callback, relations_callback=self.relations_callback, nodes_tag_filter=self.nodes_tag_filter, ways_tag_filter=self.ways_tag_filter, relations_tag_filter=self.relations_tag_filter, marshal_elem_data=self.marshal_elem_data, ) self.pool.append(proc) proc.start() chunker = XMLChunker(stream, self.mmap_pool, xml_chunk_size=READ_SIZE) chunker.read(self.mmap_queue, coords_callback=self.coords_callback) self.mmap_queue.join() for proc in self.pool: self.mmap_queue.put((None, None)) for proc in self.pool: proc.join() class MMapPool(object): """ Manages multiple mmap files. The mmap files can be read and written in different processes. """ def __init__(self, n, mmap_size): self.n = n self.mmap_size = mmap_size self.pool = [mmap.mmap(-1, mmap_size) for _ in range(n)] self.free_mmaps = set(range(n)) self.free_queue = multiprocessing.JoinableQueue() def new(self): """ Return a free mmap file. :returns: index, mmap file """ if not self.free_mmaps: self.free_mmaps.add(self.free_queue.get()) self.free_queue.task_done() while True: # fetch unless free_queue is empty try: self.free_mmaps.add(self.free_queue.get_nowait()) self.free_queue.task_done() except Empty: break mmap_idx = self.free_mmaps.pop() return mmap_idx, self.pool[mmap_idx] def join(self): while len(self.free_mmaps) < self.n: self.free_mmaps.add(self.free_queue.get()) self.free_queue.task_done() def get(self, idx): """ Return mmap file with `idx`. """ return self.pool[idx] def free(self, idx): """ Mark mmap file with `idx` as free. """ self.free_queue.put(idx) class XMLChunker(object): """ Reads and chunks OSM XML file. Reads OSM XML from `stream` and writes chunks of it into mmap files from the `mmap_pool`. :params xml_chunk_size: chunk XML after this many bytes """ def __init__(self, stream, mmap_pool, xml_chunk_size): self.stream = stream self.size = xml_chunk_size self._last_line = None self.mmap_pool = mmap_pool self.current_mmap_idx = 0 self._skip_header() def _skip_header(self): for line in self.stream: if line.lstrip().startswith('") return stream def _finished_xml_outstream(self, last_line, stream): if '\n') return self.current_mmap_idx, stream.tell() def read(self, mmaps_queue, coords_callback=None): """ Read and chunk all """ coord_node_match = None xml_nodes = self._new_xml_outstream() coords = [] coord_node_re_match = re.compile(r'^\s*').match node_re_match = re.compile(r'^\s*').match xml_nodes.write(self._last_line) split = False line = '' for line in self.stream: if coords_callback: coord_node_match = coord_node_re_match(line) if coord_node_match: osm_id, lat, lon = coord_node_match.groups() coords.append((int(osm_id), float(lon), float(lat))) if len(coords) >= 512: coords_callback(coords) coords = [] else: xml_nodes.write(line) else: xml_nodes.write(line) if split: if (line.rstrip().endswith(('', '', '')) or (coords_callback and coord_node_match) or (not coords_callback and node_re_match(line))): mmaps_queue.put(self._finished_xml_outstream(line, xml_nodes)) xml_nodes = self._new_xml_outstream() split = False elif xml_nodes.tell() > self.size: split = True if coords_callback: coords_callback(coords) # we are at the end of the stream and assume we wrote the end tag # to xml_nodes. we set line to closing tag here to avoid additional # end tag in case the last line(s) is blank line = '= 512: self.coords_callback(coords) coords = [] if len(nodes) >= 128: self.nodes_callback(nodes) nodes = [] if len(relations) >= 128: self.relations_callback(relations) relations = [] if len(ways) >= 128: self.ways_callback(ways) ways = [] root.clear() if self.coords_callback: self.coords_callback(coords) if self.nodes_callback: self.nodes_callback(nodes) if self.ways_callback: self.ways_callback(ways) if self.relations_callback: self.relations_callback(relations)imposm.parser-1.0.7/imposm/parser/xml/util.py0000644000076500000240000000242011766117447021202 0ustar oltstaff00000000000000# Copyright 2011 Omniscale GmbH & Co. KG # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import with_statement from xml.etree import cElementTree as ET from contextlib import contextmanager def iterparse(fileobj): """ Return root object and iterparser for given ``fileobj``. """ context = ET.iterparse(fileobj, events=("start", "end")) context = iter(context) _event, root = context.next() return root, context @contextmanager def log_file_on_exception(xml): try: yield except SyntaxError, ex: import tempfile fd_, filename = tempfile.mkstemp('.osm') xml.seek(0) with open(filename, 'w') as f: f.write(xml.read()) print 'SyntaxError in xml: %s, (stored dump %s)' % (ex, filename)imposm.parser-1.0.7/LICENSE0000644000076500000240000002613611766117447015272 0ustar oltstaff00000000000000 Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. imposm.parser-1.0.7/MANIFEST.in0000644000076500000240000000035512454175343016010 0ustar oltstaff00000000000000include MANIFEST.in include README.rst include CHANGES include LICENSE include setup.py include osm.proto recursive-include imposm/parser/test *.osm *.pbf *.osm.bz2 exclude imposm/parser/pbf/osm.pb.cc exclude imposm/parser/pbf/osm.pb.h imposm.parser-1.0.7/osm.proto0000644000076500000240000002101412454175204016124 0ustar oltstaff00000000000000/** Copyright (c) 2010 Scott A. Crosby. This program is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see . */ option java_package = "crosby.binary"; package OSMPBF; //protoc --java_out=../.. fileformat.proto // // STORAGE LAYER: Storing primitives. // message Blob { optional bytes raw = 1; // No compression optional int32 raw_size = 2; // When compressed, the uncompressed size // Possible compressed versions of the data. optional bytes zlib_data = 3; // PROPOSED feature for LZMA compressed data. SUPPORT IS NOT REQUIRED. optional bytes lzma_data = 4; // Formerly used for bzip2 compressed data. Depreciated in 2010. optional bytes OBSOLETE_bzip2_data = 5 [deprecated=true]; // Don't reuse this tag number. } /* A file contains an sequence of fileblock headers, each prefixed by their length in network byte order, followed by a data block containing the actual data. types staring with a "_" are reserved. */ message BlobHeader { required string type = 1; optional bytes indexdata = 2; required int32 datasize = 3; } /** Copyright (c) 2010 Scott A. Crosby. This program is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see . */ /* OSM Binary file format This is the master schema file of the OSM binary file format. This file is designed to support limited random-access and future extendability. A binary OSM file consists of a sequence of FileBlocks (please see fileformat.proto). The first fileblock contains a serialized instance of HeaderBlock, followed by a sequence of PrimitiveBlock blocks that contain the primitives. Each primitiveblock is designed to be independently parsable. It contains a string table storing all strings in that block (keys and values in tags, roles in relations, usernames, etc.) as well as metadata containing the precision of coordinates or timestamps in that block. A primitiveblock contains a sequence of primitive groups, each containing primitives of the same type (nodes, densenodes, ways, relations). Coordinates are stored in signed 64-bit integers. Lat&lon are measured in units nanodegrees. The default of granularity of 100 nanodegrees corresponds to about 1cm on the ground, and a full lat or lon fits into 32 bits. Converting an integer to a lattitude or longitude uses the formula: $OUT = IN * granularity / 10**9$. Many encoding schemes use delta coding when representing nodes and relations. */ ////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////// /* Contains the file header. */ message HeaderBlock { optional HeaderBBox bbox = 1; /* Additional tags to aid in parsing this dataset */ repeated string required_features = 4; repeated string optional_features = 5; optional string writingprogram = 16; optional string source = 17; // From the bbox field. } /** The bounding box field in the OSM header. BBOX, as used in the OSM header. Units are always in nanodegrees -- they do not obey granularity rules. */ message HeaderBBox { required sint64 left = 1; required sint64 right = 2; required sint64 top = 3; required sint64 bottom = 4; } /////////////////////////////////////////////////////////////////////// /////////////////////////////////////////////////////////////////////// message PrimitiveBlock { required StringTable stringtable = 1; repeated PrimitiveGroup primitivegroup = 2; // Granularity, units of nanodegrees, used to store coordinates in this block optional int32 granularity = 17 [default=100]; // Offset value between the output coordinates coordinates and the granularity grid in unites of nanodegrees. optional int64 lat_offset = 19 [default=0]; optional int64 lon_offset = 20 [default=0]; // Granularity of dates, normally represented in units of milliseconds since the 1970 epoch. optional int32 date_granularity = 18 [default=1000]; // Proposed extension: //optional BBox bbox = XX; } // Group of OSMPrimitives. All primitives in a group must be the same type. message PrimitiveGroup { repeated Node nodes = 1; optional DenseNodes dense = 2; repeated Way ways = 3; repeated Relation relations = 4; repeated ChangeSet changesets = 5; } /** String table, contains the common strings in each block. Note that we reserve index '0' as a delimiter, so the entry at that index in the table is ALWAYS blank and unused. */ message StringTable { repeated bytes s = 1; } /* Optional metadata that may be included into each primitive. */ message Info { optional int32 version = 1 [default = -1]; optional int64 timestamp = 2; optional int64 changeset = 3; optional int32 uid = 4; optional uint32 user_sid = 5; // String IDs } /** Optional metadata that may be included into each primitive. Special dense format used in DenseNodes. */ message DenseInfo { repeated int32 version = 1 [packed = true]; repeated sint64 timestamp = 2 [packed = true]; // DELTA coded repeated sint64 changeset = 3 [packed = true]; // DELTA coded repeated sint32 uid = 4 [packed = true]; // DELTA coded repeated sint32 user_sid = 5 [packed = true]; // String IDs for usernames. DELTA coded } // THIS IS STUB DESIGN FOR CHANGESETS. NOT USED RIGHT NOW. // TODO: REMOVE THIS? message ChangeSet { required int64 id = 1; // // // Parallel arrays. // repeated uint32 keys = 2 [packed = true]; // String IDs. // repeated uint32 vals = 3 [packed = true]; // String IDs. // // optional Info info = 4; // optional int64 created_at = 8; // optional int64 closetime_delta = 9; // optional bool open = 10; // optional HeaderBBox bbox = 11; } message Node { required sint64 id = 1; // Parallel arrays. repeated uint32 keys = 2 [packed = true]; // String IDs. repeated uint32 vals = 3 [packed = true]; // String IDs. optional Info info = 4; // May be omitted in omitmeta required sint64 lat = 8; required sint64 lon = 9; } /* Used to densly represent a sequence of nodes that do not have any tags. We represent these nodes columnwise as five columns: ID's, lats, and lons, all delta coded. When metadata is not omitted, We encode keys & vals for all nodes as a single array of integers containing key-stringid and val-stringid, using a stringid of 0 as a delimiter between nodes. ( ( )* '0' )* */ message DenseNodes { repeated sint64 id = 1 [packed = true]; // DELTA coded //repeated Info info = 4; optional DenseInfo denseinfo = 5; repeated sint64 lat = 8 [packed = true]; // DELTA coded repeated sint64 lon = 9 [packed = true]; // DELTA coded // Special packing of keys and vals into one array. May be empty if all nodes in this block are tagless. repeated int32 keys_vals = 10 [packed = true]; } message Way { required int64 id = 1; // Parallel arrays. repeated uint32 keys = 2 [packed = true]; repeated uint32 vals = 3 [packed = true]; optional Info info = 4; repeated sint64 refs = 8 [packed = true]; // DELTA coded } message Relation { enum MemberType { NODE = 0; WAY = 1; RELATION = 2; } required int64 id = 1; // Parallel arrays. repeated uint32 keys = 2 [packed = true]; repeated uint32 vals = 3 [packed = true]; optional Info info = 4; // Parallel arrays repeated int32 roles_sid = 8 [packed = true]; repeated sint64 memids = 9 [packed = true]; // DELTA encoded repeated MemberType types = 10 [packed = true]; } imposm.parser-1.0.7/PKG-INFO0000644000076500000240000000676412454223055015353 0ustar oltstaff00000000000000Metadata-Version: 1.1 Name: imposm.parser Version: 1.0.7 Summary: Fast and easy OpenStreetMap XML/PBF parser. Home-page: http://imposm.org/docs/imposm.parser/latest/ Author: Oliver Tonnhofer Author-email: olt@omniscale.de License: Apache Software License 2.0 Description: imposm.parser - OpenStreetMap XML/PBF parser for Python ======================================================= ``imposm.parser`` is a Python library that parses OpenStreetMap data in `XML `_ and `PBF `_ format. It has a simple API and it is fast and easy to use. It also works across multiple CPU/cores for extra speed. It is developed and supported by `Omniscale `_ and released under the `Apache Software License 2.0 `_. Example ------- Here is an example that parses an OSM file and counts all ways that are tagged as a highway. :: from imposm.parser import OSMParser # simple class that handles the parsed OSM data. class HighwayCounter(object): highways = 0 def ways(self, ways): # callback method for ways for osmid, tags, refs in ways: if 'highway' in tags: self.highways += 1 # instantiate counter and parser and start parsing counter = HighwayCounter() p = OSMParser(concurrency=4, ways_callback=counter.ways) p.parse('germany.osm.pbf') # done print counter.highways Source and issue tracker ------------------------ Source code and issue tracker are available at ``_. Changelog --------- 1.0.7 2015-01-10 ~~~~~~~~~~~~~~~~ - updated protobuf parser 1.0.6 2014-10-31 ~~~~~~~~~~~~~~~~ - fixed bug in protobuf parser that could cause segfaults 1.0.5 2013-09-13 ~~~~~~~~~~~~~~~~ - support PBF without granularity value - improved support for non-pretty-printed XML 1.0.4 2012-12-10 ~~~~~~~~~~~~~~~~ - improved support for non-pretty-printed XML - fixed dependency check for multiprocessing 1.0.3 2011-07-21 ~~~~~~~~~~~~~~~~ - support for uncompressed PBF - bug fix for PBF without dense nodes 1.0.2 2011-03-10 ~~~~~~~~~~~~~~~~ - improved regexp based XML coord parser - prevent mmap overflow in XMLChunker without coord_callback - successfully parsed whole planet.osm 1.0.0 2011-02-22 ~~~~~~~~~~~~~~~~ - first release Platform: UNKNOWN Classifier: Development Status :: 4 - Beta Classifier: License :: OSI Approved :: Apache Software License Classifier: Operating System :: OS Independent Classifier: Programming Language :: C Classifier: Programming Language :: C++ Classifier: Programming Language :: Python :: 2.5 Classifier: Programming Language :: Python :: 2.6 Classifier: Programming Language :: Python :: 2.7 Classifier: Topic :: Software Development :: Libraries Classifier: Topic :: Scientific/Engineering :: GIS imposm.parser-1.0.7/README.rst0000644000076500000240000000253312061305755015735 0ustar oltstaff00000000000000imposm.parser - OpenStreetMap XML/PBF parser for Python ======================================================= ``imposm.parser`` is a Python library that parses OpenStreetMap data in `XML `_ and `PBF `_ format. It has a simple API and it is fast and easy to use. It also works across multiple CPU/cores for extra speed. It is developed and supported by `Omniscale `_ and released under the `Apache Software License 2.0 `_. Example ------- Here is an example that parses an OSM file and counts all ways that are tagged as a highway. :: from imposm.parser import OSMParser # simple class that handles the parsed OSM data. class HighwayCounter(object): highways = 0 def ways(self, ways): # callback method for ways for osmid, tags, refs in ways: if 'highway' in tags: self.highways += 1 # instantiate counter and parser and start parsing counter = HighwayCounter() p = OSMParser(concurrency=4, ways_callback=counter.ways) p.parse('germany.osm.pbf') # done print counter.highways Source and issue tracker ------------------------ Source code and issue tracker are available at ``_. imposm.parser-1.0.7/setup.cfg0000644000076500000240000000007312454223055016062 0ustar oltstaff00000000000000[egg_info] tag_build = tag_date = 0 tag_svn_revision = 0 imposm.parser-1.0.7/setup.py0000644000076500000240000000502312454207040015747 0ustar oltstaff00000000000000import errno import platform from setuptools import setup, Extension, find_packages from setuptools.command.build_ext import build_ext from distutils.errors import DistutilsPlatformError import subprocess class build_ext_with_protpbuf(build_ext): def run(self): try: proc = subprocess.Popen( ['protoc', '--cpp_out', 'imposm/parser/pbf/', 'osm.proto'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) except OSError, ex: if ex.errno == errno.ENOENT: print ("Could not find protoc command. Make sure protobuf is " "installed and your PATH environment is set.") raise DistutilsPlatformError("Failed to generate protbuf " "CPP files with protoc.") else: raise out = proc.communicate()[0] result = proc.wait() if result != 0: print "Error during protbuf files generation with protoc:" print out raise DistutilsPlatformError("Failed to generate protbuf " "CPP files with protoc.") build_ext.run(self) install_requires = [] if tuple(map(str, platform.python_version_tuple())) < ('2', '6'): install_requires.append('multiprocessing>=2.6') setup( name='imposm.parser', version="1.0.7", description='Fast and easy OpenStreetMap XML/PBF parser.', long_description=open('README.rst').read() + open('CHANGES').read(), author='Oliver Tonnhofer', author_email='olt@omniscale.de', url='http://imposm.org/docs/imposm.parser/latest/', license='Apache Software License 2.0', packages=find_packages(), namespace_packages = ['imposm'], include_package_data=True, package_data = {'': ['*.xml', '*.osm', '*.osm.bz2']}, install_requires=install_requires, classifiers=[ "Development Status :: 4 - Beta", "License :: OSI Approved :: Apache Software License", "Operating System :: OS Independent", "Programming Language :: C", "Programming Language :: C++", "Programming Language :: Python :: 2.5", "Programming Language :: Python :: 2.6", "Programming Language :: Python :: 2.7", "Topic :: Software Development :: Libraries", "Topic :: Scientific/Engineering :: GIS", ], ext_modules=[ Extension("imposm.parser.pbf.OSMPBF", ["imposm/parser/pbf/osm.cc", "imposm/parser/pbf/osm.pb.cc"], libraries=['protobuf']), ], cmdclass={'build_ext':build_ext_with_protpbuf}, )