sourceIterator) {
this.sourceIterator = sourceIterator;
nextValue = null;
nextValueAvailable = false;
}
/**
* {@inheritDoc}
*/
public boolean hasNext() {
return nextValueAvailable || sourceIterator.hasNext();
}
/**
* Returns the next available entity without advancing to the next record.
*
* @return The next available entity.
*/
public T peekNext() {
if (!nextValueAvailable) {
nextValue = sourceIterator.next();
nextValueAvailable = true;
}
return nextValue;
}
/**
* {@inheritDoc}
*/
public T next() {
T result;
result = peekNext();
nextValue = null;
nextValueAvailable = false;
return result;
}
/**
* {@inheritDoc}
*/
public void release() {
sourceIterator.release();
}
/**
* {@inheritDoc}
*/
public void remove() {
throw new UnsupportedOperationException();
}
}
././@LongLink 0000000 0000000 0000000 00000000150 00000000000 011561 L ustar root root openstreetmap-osmosis-b835354/core/src/main/java/org/openstreetmap/osmosis/core/store/ObjectReader.java openstreetmap-osmosis-b835354/core/src/main/java/org/openstreetmap/osmosis/core/store/ObjectReader.j0000664 0001750 0001750 00000000644 11667575673 032673 0 ustar david david // This software is released into the Public Domain. See copying.txt for details.
package org.openstreetmap.osmosis.core.store;
/**
* Implementations provide functionality to deserialise a Storeable
* implementation from a store.
*
* @author Brett Henderson
*/
public interface ObjectReader {
/**
* Reads an object from storage.
*
* @return The re-instantiated object.
*/
Storeable readObject();
}
././@LongLink 0000000 0000000 0000000 00000000175 00000000000 011570 L ustar root root openstreetmap-osmosis-b835354/core/src/main/java/org/openstreetmap/osmosis/core/store/GenericObjectSerializationFactory.java openstreetmap-osmosis-b835354/core/src/main/java/org/openstreetmap/osmosis/core/store/GenericObjectS0000664 0001750 0001750 00000001534 11667575673 032737 0 ustar david david // This software is released into the Public Domain. See copying.txt for details.
package org.openstreetmap.osmosis.core.store;
/**
* An object reader and writer factory providing generic object serialisation
* capabilities capable of storing and loading any Storeable class
* implementations.
*
* @author Brett Henderson
*/
public class GenericObjectSerializationFactory implements ObjectSerializationFactory {
/**
* {@inheritDoc}
*/
@Override
public ObjectReader createObjectReader(StoreReader storeReader, StoreClassRegister storeClassRegister) {
return new GenericObjectReader(storeReader, storeClassRegister);
}
/**
* {@inheritDoc}
*/
@Override
public ObjectWriter createObjectWriter(StoreWriter storeWriter, StoreClassRegister storeClassRegister) {
return new GenericObjectWriter(storeWriter, storeClassRegister);
}
}
././@LongLink 0000000 0000000 0000000 00000000157 00000000000 011570 L ustar root root openstreetmap-osmosis-b835354/core/src/main/java/org/openstreetmap/osmosis/core/store/EndOfStoreException.java openstreetmap-osmosis-b835354/core/src/main/java/org/openstreetmap/osmosis/core/store/EndOfStoreExce0000664 0001750 0001750 00000003021 11667575673 032717 0 ustar david david // This software is released into the Public Domain. See copying.txt for details.
package org.openstreetmap.osmosis.core.store;
import org.openstreetmap.osmosis.core.OsmosisRuntimeException;
/**
* A runtime exception indicating that the end of the store has been reached while reading.
*
* @author Brett Henderson
*/
public class EndOfStoreException extends OsmosisRuntimeException {
private static final long serialVersionUID = 1L;
/**
* Constructs a new exception with null as its detail message.
*/
public EndOfStoreException() {
super();
}
/**
* Constructs a new exception with the specified detail message. The
* cause is not initialized, and may subsequently be initialized by
* a call to {@link #initCause}.
*
* @param message the detail message.
*/
public EndOfStoreException(String message) {
super(message);
}
/**
* Constructs a new exception with the specified cause and a detail
* message of (cause==null ? null : cause.toString()) (which
* typically contains the class and detail message of cause).
*
* @param cause the cause.
*/
public EndOfStoreException(Throwable cause) {
super(cause);
}
/**
* Constructs a new exception with the specified detail message and
* cause.
*
* @param message the detail message.
* @param cause the cause.
*/
public EndOfStoreException(String message, Throwable cause) {
super(message, cause);
}
}
././@LongLink 0000000 0000000 0000000 00000000156 00000000000 011567 L ustar root root openstreetmap-osmosis-b835354/core/src/main/java/org/openstreetmap/osmosis/core/store/ChunkedObjectStore.java openstreetmap-osmosis-b835354/core/src/main/java/org/openstreetmap/osmosis/core/store/ChunkedObjectS0000664 0001750 0001750 00000011311 11667575673 032736 0 ustar david david // This software is released into the Public Domain. See copying.txt for details.
package org.openstreetmap.osmosis.core.store;
import org.openstreetmap.osmosis.core.lifecycle.Completable;
import org.openstreetmap.osmosis.core.lifecycle.ReleasableIterator;
/**
* Adds indexed chunking capabilities to a basic object store allowing groups of
* objects to be written and retrieved later by their chunk index. The number of
* objects and the size of the index is limited only by disk space.
*
* This store is only suitable for single-threaded use because it does not
* provide per-thread readers.
*
* @param
* The class type to be stored.
* @author Brett Henderson
*/
public class ChunkedObjectStore implements Completable {
/**
* Stores all the objects written to this store.
*/
private SegmentedObjectStore objectStore;
/**
* Maintains both the file positions of each chunk and the number of objects
* within each chunk. The file position is written when a new chunk is
* started, and the object count is written when a chunk is completed.
*/
private IndexStore indexStore;
private IndexStoreReader indexStoreReader;
private long chunkCount;
private boolean chunkInProgress;
private long newChunkFilePosition;
private long chunkObjectCount;
/**
* Creates a new instance.
*
* @param serializationFactory
* The factory defining the object serialisation implementation.
* @param storageFilePrefix
* The prefix of the storage file name.
* @param indexFilePrefix
* The prefix of the index file name.
* @param useCompression
* If true, the storage file will be compressed.
*/
public ChunkedObjectStore(
ObjectSerializationFactory serializationFactory,
String storageFilePrefix,
String indexFilePrefix,
boolean useCompression) {
objectStore = new SegmentedObjectStore(serializationFactory, storageFilePrefix, useCompression);
indexStore = new IndexStore(
LongLongIndexElement.class,
new ComparableComparator(),
indexFilePrefix
);
chunkCount = 0;
chunkInProgress = false;
newChunkFilePosition = 0;
chunkObjectCount = 0;
}
/**
* Adds the specified object to the store.
*
* @param data
* The object to be added.
*/
public void add(T data) {
objectStore.add(data);
chunkObjectCount++;
if (!chunkInProgress) {
// Write the file index of the new chunk.
indexStore.write(new LongLongIndexElement((chunkCount * 2), newChunkFilePosition));
chunkInProgress = true;
}
}
/**
* Stops the current writing operation and begins a new one saving the
* current position against a new interval index.
*/
public void closeChunk() {
// We can only close a chunk if one is in progress.
if (chunkInProgress) {
// Create an interval in the underlying object store and note the
// current position.
newChunkFilePosition = objectStore.closeChunk();
// Flag that no chunk is in progress so that the next add will store
// the start file index.
chunkInProgress = false;
// Write then reset the object count of the current chunk.
indexStore.write(new LongLongIndexElement((chunkCount * 2) + 1, chunkObjectCount));
chunkObjectCount = 0;
// Increment the chunk count.
chunkCount++;
}
}
/**
* Returns the number of chunks managed by this store. This count will
* include the in progress chunk if one exists.
*
* @return The number of chunks.
*/
public long getChunkCount() {
// If a chunk is in progress, the chunk count won't have been updated yet.
if (chunkInProgress) {
return chunkCount + 1;
} else {
return chunkCount;
}
}
/**
* Provides access to the contents of this store.
*
* @param chunk
* The chunk to read objects from.
* @return An iterator providing access to contents of the store.
*/
public ReleasableIterator iterate(long chunk) {
complete();
if (indexStoreReader == null) {
indexStoreReader = indexStore.createReader();
}
// Retrieve the file position and number of objects for the specified
// chunk and iterate.
return objectStore.iterate(
indexStoreReader.get(chunk * 2).getValue(),
indexStoreReader.get(chunk * 2 + 1).getValue()
);
}
/**
* {@inheritDoc}
*/
@Override
public void complete() {
// Any outstanding chunks must be closed before we can complete.
closeChunk();
indexStore.complete();
}
/**
* {@inheritDoc}
*/
public void release() {
objectStore.release();
if (indexStoreReader != null) {
indexStoreReader.release();
indexStoreReader = null;
}
indexStore.release();
}
}
././@LongLink 0000000 0000000 0000000 00000000160 00000000000 011562 L ustar root root openstreetmap-osmosis-b835354/core/src/main/java/org/openstreetmap/osmosis/core/store/SegmentedObjectStore.java openstreetmap-osmosis-b835354/core/src/main/java/org/openstreetmap/osmosis/core/store/SegmentedObjec0000664 0001750 0001750 00000023317 11667575673 032772 0 ustar david david // This software is released into the Public Domain. See copying.txt for details.
package org.openstreetmap.osmosis.core.store;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.zip.GZIPOutputStream;
import org.openstreetmap.osmosis.core.OsmosisRuntimeException;
import org.openstreetmap.osmosis.core.lifecycle.Completable;
import org.openstreetmap.osmosis.core.lifecycle.ReleasableIterator;
import org.openstreetmap.osmosis.core.util.MultiMemberGZIPInputStream;
/**
* Provides a store for writing objects to a file for later retrieval. The
* number of objects is limited only by disk space.
*
* This class supports chunking where the stream is broken into segments. This
* is achieved by calling the closeChunk method between add calls.
*
* This store is only suitable for single-threaded use because it does not
* provide per-thread readers.
*
* @param
* The object type to be stored.
* @author Brett Henderson
*/
public class SegmentedObjectStore implements Completable {
private static final Logger LOG = Logger.getLogger(SegmentedObjectStore.class.getName());
private ObjectSerializationFactory serializationFactory;
private StorageStage stage;
private String storageFilePrefix;
private File file;
private FileOutputStream fileOutStream;
private DataOutputStream dataOutStream;
private ByteArrayOutputStream arrayOutStream;
private StoreClassRegister storeClassRegister;
private ObjectWriter objectWriter;
private boolean chunkActive;
private boolean useCompression;
private long fileSize;
/**
* Creates a new instance.
*
* @param serializationFactory
* The factory defining the object serialisation implementation.
* @param storageFilePrefix
* The prefix of the storage file.
* @param useCompression
* If true, the storage file will be compressed.
*/
public SegmentedObjectStore(
ObjectSerializationFactory serializationFactory, String storageFilePrefix, boolean useCompression) {
this.serializationFactory = serializationFactory;
this.storageFilePrefix = storageFilePrefix;
this.useCompression = useCompression;
storeClassRegister = new DynamicStoreClassRegister();
stage = StorageStage.NotStarted;
fileSize = 0;
chunkActive = false;
}
/**
* Adds the specified object to the store.
*
* @param data
* The object to be added.
*/
public void add(T data) {
// We can't add if we've passed the add stage.
if (stage.compareTo(StorageStage.Add) > 0) {
throw new OsmosisRuntimeException("Cannot add to storage in stage " + stage + ".");
}
// If we're not up to the add stage, initialise for adding.
if (stage.compareTo(StorageStage.Add) < 0) {
try {
file = File.createTempFile(storageFilePrefix, null);
fileOutStream = new FileOutputStream(file);
stage = StorageStage.Add;
} catch (IOException e) {
throw new OsmosisRuntimeException("Unable to open temporary file " + file + " for writing.", e);
}
}
// Initialise the current chunk if it isn't already.
if (!chunkActive) {
try {
arrayOutStream = new ByteArrayOutputStream();
if (useCompression) {
dataOutStream = new DataOutputStream(
new BufferedOutputStream(
new GZIPOutputStream(arrayOutStream), 65536));
} else {
dataOutStream = new DataOutputStream(new BufferedOutputStream(arrayOutStream, 65536));
}
objectWriter = serializationFactory.createObjectWriter(
new DataOutputStoreWriter(dataOutStream), storeClassRegister);
chunkActive = true;
} catch (IOException e) {
throw new OsmosisRuntimeException("Unable to create object stream.", e);
}
}
// Write the object to the store.
objectWriter.writeObject(data);
// Update the file position based on the buffer size.
fileSize += arrayOutStream.size();
// Write the buffer to file, and clear the buffer.
try {
arrayOutStream.writeTo(fileOutStream);
arrayOutStream.reset();
} catch (IOException e) {
throw new OsmosisRuntimeException("Unable to write object to file.", e);
}
}
/**
* Closes the current object stream and creates a new one. This allows read
* operations to begin at offsets within the file. This can only be called
* while adding to the store, not once reads are begun. Read operations must
* begin at offsets created by this method.
*
* @return The start position of the new chunk within the file.
*/
public long closeChunk() {
// We can only create an interval if we are in add mode.
if (stage.compareTo(StorageStage.Add) != 0) {
throw new OsmosisRuntimeException("Cannot create interval in stage " + stage + ".");
}
// Nothing needs to be done if the chunk is not yet active.
if (chunkActive) {
try {
dataOutStream.close();
fileSize += arrayOutStream.size();
arrayOutStream.writeTo(fileOutStream);
arrayOutStream.reset();
// Subsequent writes must begin a new object stream.
arrayOutStream = null;
dataOutStream = null;
chunkActive = false;
} catch (IOException e) {
throw new OsmosisRuntimeException("Unable to create a new interval.", e);
}
}
return fileSize;
}
/**
* Configures the state of this object instance for iterating or reading
* mode. If the current state doesn't allow iterating, an exception will be
* thrown.
*
* @return true if data is available, false otherwise.
*/
private boolean initializeIteratingStage() {
// If we've been released, we can't iterate.
if (stage.compareTo(StorageStage.Released) >= 0) {
throw new OsmosisRuntimeException("Cannot iterate over storage in stage " + stage + ".");
}
// If no data was written, an empty iterator should be returned.
if (stage.compareTo(StorageStage.NotStarted) <= 0) {
return false;
}
// If we're in the add stage, close the current chunk and overall file stream.
if (stage.compareTo(StorageStage.Add) == 0) {
closeChunk();
try {
fileOutStream.close();
} catch (IOException e) {
throw new OsmosisRuntimeException("Unable to close output stream.", e);
} finally {
fileOutStream = null;
}
stage = StorageStage.Reading;
}
// Data is available.
return true;
}
/**
* Returns an iterator for reading objects from the underlying data store.
*
* @return An iterator for reading objects from the data store. This
* iterator must be released after use.
*/
public ReleasableIterator iterate() {
return iterate(0, -1);
}
/**
* Returns an iterator for reading objects from the underlying data store.
*
* @param streamOffset
* The location in the underlying stream to begin reading.
* @param maxObjectCount
* The maximum number of objects to be returned, -1 for
* unlimited.
* @return An iterator for reading objects from the data store. This
* iterator must be released after use.
*/
public ReleasableIterator iterate(long streamOffset, long maxObjectCount) {
FileInputStream fileStream = null;
try {
DataInputStream dataInStream;
ObjectReader objectReader;
if (!initializeIteratingStage()) {
return new EmptyIterator();
}
// If we've reached this far, we have a file containing data to be read. Open a file stream on the file.
try {
fileStream = new FileInputStream(file);
} catch (IOException e) {
throw new OsmosisRuntimeException("Unable to open file for reading.", e);
}
// Seek to the required starting point in the file.
if (streamOffset > 0) {
try {
fileStream.skip(streamOffset);
} catch (IOException e) {
throw new OsmosisRuntimeException("Unable to skip to specified location in file.", e);
}
}
// Create the object input stream.
try {
if (useCompression) {
dataInStream = new DataInputStream(
new BufferedInputStream(
new MultiMemberGZIPInputStream(fileStream), 65536));
} else {
dataInStream = new DataInputStream(new BufferedInputStream(fileStream, 65536));
}
} catch (IOException e) {
throw new OsmosisRuntimeException("Unable to open object stream.", e);
}
// The stream will be owned by the caller, therefore we must clear
// the reference now so it isn't closed on method exit.
fileStream = null;
objectReader = serializationFactory.createObjectReader(
new DataInputStoreReader(dataInStream), storeClassRegister);
if (maxObjectCount >= 0) {
return new SubObjectStreamIterator(dataInStream, objectReader, maxObjectCount);
} else {
return new ObjectStreamIterator(dataInStream, objectReader);
}
} finally {
if (fileStream != null) {
try {
fileStream.close();
} catch (IOException e) {
// We are already in an error condition so log and continue.
LOG.log(Level.WARNING, "Unable to close result set.", e);
}
}
}
}
/**
* {@inheritDoc}
*/
@Override
public void complete() {
// Do nothing.
}
/**
* {@inheritDoc}
*/
public void release() {
if (fileOutStream != null) {
try {
fileOutStream.close();
} catch (Exception e) {
// We cannot throw an exception within a release statement.
LOG.log(Level.WARNING, "Unable to file output stream.", e);
}
fileOutStream = null;
}
if (file != null) {
if (!file.delete()) {
LOG.warning("Unable to delete file " + file);
}
file = null;
}
stage = StorageStage.Released;
}
}
././@LongLink 0000000 0000000 0000000 00000000163 00000000000 011565 L ustar root root openstreetmap-osmosis-b835354/core/src/main/java/org/openstreetmap/osmosis/core/store/ObjectDataInputIterator.java openstreetmap-osmosis-b835354/core/src/main/java/org/openstreetmap/osmosis/core/store/ObjectDataInpu0000664 0001750 0001750 00000002503 11667575673 032742 0 ustar david david // This software is released into the Public Domain. See copying.txt for details.
package org.openstreetmap.osmosis.core.store;
import java.util.Iterator;
import java.util.NoSuchElementException;
/**
* Provides functionality common to all object iterators.
*
* @param
* The type of data to be returned by the iterator.
* @author Brett Henderson
*/
public class ObjectDataInputIterator implements Iterator {
private ObjectReader objectReader;
private T nextElement;
/**
* Creates a new instance.
*
* @param objectReader
* The reader containing the objects to be deserialized.
*/
public ObjectDataInputIterator(ObjectReader objectReader) {
this.objectReader = objectReader;
}
/**
* {@inheritDoc}
*/
@SuppressWarnings("unchecked")
public boolean hasNext() {
if (nextElement != null) {
return true;
}
try {
nextElement = (T) objectReader.readObject();
} catch (EndOfStoreException e) {
return false;
}
return true;
}
/**
* {@inheritDoc}
*/
public T next() {
if (hasNext()) {
T result;
result = nextElement;
nextElement = null;
return result;
} else {
throw new NoSuchElementException();
}
}
/**
* {@inheritDoc}
*/
public void remove() {
throw new UnsupportedOperationException();
}
}
././@LongLink 0000000 0000000 0000000 00000000146 00000000000 011566 L ustar root root openstreetmap-osmosis-b835354/core/src/main/java/org/openstreetmap/osmosis/core/store/IndexStore.java openstreetmap-osmosis-b835354/core/src/main/java/org/openstreetmap/osmosis/core/store/IndexStore.jav0000664 0001750 0001750 00000014433 11667575673 032756 0 ustar david david // This software is released into the Public Domain. See copying.txt for details.
package org.openstreetmap.osmosis.core.store;
import java.io.File;
import java.util.Comparator;
import java.util.Iterator;
import org.openstreetmap.osmosis.core.OsmosisRuntimeException;
import org.openstreetmap.osmosis.core.lifecycle.Completable;
import org.openstreetmap.osmosis.core.lifecycle.ReleasableIterator;
import org.openstreetmap.osmosis.core.sort.common.FileBasedSort;
/**
* Writes data into an index file and sorts it if input data is unordered. The
* data must be fixed width to allow index values to be randomly accessed later.
*
* @param
* The index key type.
* @param
* The index element type to be stored.
* @author Brett Henderson
*/
public class IndexStore> implements Completable {
private ObjectSerializationFactory serializationFactory;
private RandomAccessObjectStore indexStore;
private Comparator ordering;
private String tempFilePrefix;
private File indexFile;
private K previousKey;
private boolean sorted;
private long elementCount;
private long elementSize;
private boolean complete;
/**
* Creates a new instance.
*
* @param elementType
* The type of index element to be stored in the index.
* @param ordering
* A comparator that sorts index elements desired index key
* ordering.
* @param indexFile
* The file to use for storing the index.
*/
public IndexStore(Class elementType, Comparator ordering, File indexFile) {
this.ordering = ordering;
this.indexFile = indexFile;
serializationFactory = new SingleClassObjectSerializationFactory(elementType);
indexStore = new RandomAccessObjectStore(serializationFactory, indexFile);
sorted = true;
elementCount = 0;
elementSize = -1;
complete = false;
}
/**
* Creates a new instance.
*
*
* @param elementType
* The type of index element to be stored in the index.
* @param ordering
* A comparator that sorts index elements desired index key
* ordering.
* @param tempFilePrefix
* The prefix of the temporary file.
*/
public IndexStore(Class elementType, Comparator ordering, String tempFilePrefix) {
this.ordering = ordering;
this.tempFilePrefix = tempFilePrefix;
serializationFactory = new SingleClassObjectSerializationFactory(elementType);
indexStore = new RandomAccessObjectStore(serializationFactory, tempFilePrefix);
sorted = true;
elementCount = 0;
elementSize = -1;
complete = false;
}
/**
* Writes the specified element to the index.
*
* @param element
* The index element which includes the identifier when stored.
*/
public void write(T element) {
K key;
long fileOffset;
if (complete) {
throw new OsmosisRuntimeException("Cannot write new data once reading has begun.");
}
fileOffset = indexStore.add(element);
key = element.getKey();
// If the new element contains a key that is not sequential, we need to
// mark the index as unsorted so we can perform a sort prior to reading.
if (previousKey != null) {
if (ordering.compare(previousKey, key) > 0) {
sorted = false;
}
}
previousKey = key;
elementCount++;
// Calculate and verify the element size. This index requires all keys to be the same length
// to allow sorting and searching to occur. The first element has a file offset of 0 so we
// ignore that one. The second element offset will tell us the size of the first element.
// From that point on we verify that all elements have the same size.
if (elementCount == 2) {
elementSize = fileOffset;
} else if (elementCount > 2) {
long expectedOffset;
expectedOffset = (elementCount - 1) * elementSize;
if (expectedOffset != fileOffset) {
throw new OsmosisRuntimeException(
"Inconsistent element sizes, new file offset=" + fileOffset
+ ", expected offset=" + expectedOffset
+ ", element size=" + elementSize
+ ", element count=" + elementCount
);
}
}
}
/**
* Creates a new reader capable of accessing the contents of this store. The
* reader must be explicitly released when no longer required. Readers must
* be released prior to this store.
*
* @return A store reader.
*/
public IndexStoreReader createReader() {
return new IndexStoreReader(indexStore.createReader(), ordering);
}
/**
* {@inheritDoc}
*/
public void complete() {
if (!complete) {
indexStore.complete();
if (!sorted) {
final Comparator keyOrdering = ordering;
FileBasedSort fileSort;
// Create a new file based sort instance ordering elements by their
// identifiers.
fileSort = new FileBasedSort(
serializationFactory,
new Comparator() {
private Comparator elementKeyOrdering = keyOrdering;
@Override
public int compare(T o1, T o2) {
return elementKeyOrdering.compare(o1.getKey(), o2.getKey());
}
},
true
);
try {
RandomAccessObjectStoreReader indexStoreReader;
ReleasableIterator sortIterator;
// Read all data from the index store into the sorting store.
indexStoreReader = indexStore.createReader();
try {
Iterator indexIterator;
indexIterator = indexStoreReader.iterate();
while (indexIterator.hasNext()) {
fileSort.add(indexIterator.next());
}
} finally {
indexStoreReader.release();
}
// Release the existing index store and create a new one.
indexStore.release();
if (indexFile != null) {
indexStore = new RandomAccessObjectStore(serializationFactory, indexFile);
} else {
indexStore = new RandomAccessObjectStore(serializationFactory, tempFilePrefix);
}
// Read all data from the sorting store back into the index store.
sortIterator = fileSort.iterate();
try {
while (sortIterator.hasNext()) {
indexStore.add(sortIterator.next());
}
} finally {
sortIterator.release();
}
} finally {
fileSort.release();
}
}
complete = true;
}
}
/**
* {@inheritDoc}
*/
public void release() {
indexStore.release();
}
}
././@LongLink 0000000 0000000 0000000 00000000146 00000000000 011566 L ustar root root openstreetmap-osmosis-b835354/core/src/main/java/org/openstreetmap/osmosis/core/CorePluginLoader.java openstreetmap-osmosis-b835354/core/src/main/java/org/openstreetmap/osmosis/core/CorePluginLoader.jav0000664 0001750 0001750 00000013463 11667575673 032736 0 ustar david david // This software is released into the Public Domain. See copying.txt for details.
package org.openstreetmap.osmosis.core;
import java.util.HashMap;
import java.util.Map;
import org.openstreetmap.osmosis.core.bound.v0_6.BoundComputerFactory;
import org.openstreetmap.osmosis.core.bound.v0_6.BoundSetterFactory;
import org.openstreetmap.osmosis.core.buffer.v0_6.ChangeBufferFactory;
import org.openstreetmap.osmosis.core.buffer.v0_6.EntityBufferFactory;
import org.openstreetmap.osmosis.core.misc.v0_6.EmptyChangeReaderFactory;
import org.openstreetmap.osmosis.core.misc.v0_6.EmptyReaderFactory;
import org.openstreetmap.osmosis.core.misc.v0_6.NullChangeWriterFactory;
import org.openstreetmap.osmosis.core.misc.v0_6.NullWriterFactory;
import org.openstreetmap.osmosis.core.pipeline.common.TaskManagerFactory;
import org.openstreetmap.osmosis.core.plugin.PluginLoader;
import org.openstreetmap.osmosis.core.progress.v0_6.ChangeProgressLoggerFactory;
import org.openstreetmap.osmosis.core.progress.v0_6.EntityProgressLoggerFactory;
import org.openstreetmap.osmosis.core.report.v0_6.EntityReporterFactory;
import org.openstreetmap.osmosis.core.report.v0_6.IntegrityReporterFactory;
import org.openstreetmap.osmosis.core.sort.v0_6.ChangeForSeekableApplierComparator;
import org.openstreetmap.osmosis.core.sort.v0_6.ChangeForStreamableApplierComparator;
import org.openstreetmap.osmosis.core.sort.v0_6.ChangeSorterFactory;
import org.openstreetmap.osmosis.core.sort.v0_6.ChangeTagSorterFactory;
import org.openstreetmap.osmosis.core.sort.v0_6.EntityByTypeThenIdComparator;
import org.openstreetmap.osmosis.core.sort.v0_6.EntityContainerComparator;
import org.openstreetmap.osmosis.core.sort.v0_6.EntitySorterFactory;
import org.openstreetmap.osmosis.core.sort.v0_6.TagSorterFactory;
import org.openstreetmap.osmosis.core.tee.v0_6.ChangeTeeFactory;
import org.openstreetmap.osmosis.core.tee.v0_6.EntityTeeFactory;
/**
* The plugin loader for the core tasks.
*
* @author Brett Henderson
*/
public class CorePluginLoader implements PluginLoader {
/**
* {@inheritDoc}
*/
@Override
public Map loadTaskFactories() {
Map factoryMap;
EntitySorterFactory entitySorterFactory06;
ChangeSorterFactory changeSorterFactory06;
factoryMap = new HashMap();
// Configure factories that require additional information.
entitySorterFactory06 = new EntitySorterFactory();
entitySorterFactory06.registerComparator("TypeThenId", new EntityContainerComparator(
new EntityByTypeThenIdComparator()), true);
changeSorterFactory06 = new ChangeSorterFactory();
changeSorterFactory06.registerComparator("streamable", new ChangeForStreamableApplierComparator(), true);
changeSorterFactory06.registerComparator("seekable", new ChangeForSeekableApplierComparator(), false);
// Register factories.
factoryMap.put("sort", entitySorterFactory06);
factoryMap.put("s", entitySorterFactory06);
factoryMap.put("sort-change", changeSorterFactory06);
factoryMap.put("sc", changeSorterFactory06);
factoryMap.put("write-null", new NullWriterFactory());
factoryMap.put("wn", new NullWriterFactory());
factoryMap.put("write-null-change", new NullChangeWriterFactory());
factoryMap.put("wnc", new NullChangeWriterFactory());
factoryMap.put("buffer", new EntityBufferFactory());
factoryMap.put("b", new EntityBufferFactory());
factoryMap.put("buffer-change", new ChangeBufferFactory());
factoryMap.put("bc", new ChangeBufferFactory());
factoryMap.put("report-entity", new EntityReporterFactory());
factoryMap.put("re", new EntityReporterFactory());
factoryMap.put("report-integrity", new IntegrityReporterFactory());
factoryMap.put("ri", new IntegrityReporterFactory());
factoryMap.put("log-progress", new EntityProgressLoggerFactory());
factoryMap.put("lp", new EntityProgressLoggerFactory());
factoryMap.put("log-progress-change", new ChangeProgressLoggerFactory());
factoryMap.put("lpc", new ChangeProgressLoggerFactory());
factoryMap.put("tee", new EntityTeeFactory());
factoryMap.put("t", new EntityTeeFactory());
factoryMap.put("tee-change", new ChangeTeeFactory());
factoryMap.put("tc", new ChangeTeeFactory());
factoryMap.put("read-empty", new EmptyReaderFactory());
factoryMap.put("re", new EmptyReaderFactory());
factoryMap.put("read-empty-change", new EmptyChangeReaderFactory());
factoryMap.put("rec", new EmptyChangeReaderFactory());
factoryMap.put("compute-bounding-box", new BoundComputerFactory());
factoryMap.put("cbb", new BoundComputerFactory());
factoryMap.put("set-bounding-box", new BoundSetterFactory());
factoryMap.put("sbb", new BoundSetterFactory());
factoryMap.put("sort-0.6", entitySorterFactory06);
factoryMap.put("sort-change-0.6", changeSorterFactory06);
factoryMap.put("write-null-0.6", new NullWriterFactory());
factoryMap.put("write-null-change-0.6", new NullChangeWriterFactory());
factoryMap.put("buffer-0.6", new EntityBufferFactory());
factoryMap.put("buffer-change-0.6", new ChangeBufferFactory());
factoryMap.put("report-entity-0.6", new EntityReporterFactory());
factoryMap.put("report-integrity-0.6", new IntegrityReporterFactory());
factoryMap.put("log-progress-0.6", new EntityProgressLoggerFactory());
factoryMap.put("log-progress-change-0.6", new ChangeProgressLoggerFactory());
factoryMap.put("tee-0.6", new EntityTeeFactory());
factoryMap.put("tee-change-0.6", new ChangeTeeFactory());
factoryMap.put("read-empty-0.6", new EmptyReaderFactory());
factoryMap.put("read-empty-change-0.6", new EmptyChangeReaderFactory());
factoryMap.put("tag-sort-0.6", new TagSorterFactory());
factoryMap.put("tag-sort-change-0.6", new ChangeTagSorterFactory());
factoryMap.put("compute-bounding-box-0.6", new BoundComputerFactory());
factoryMap.put("set-bounding-box-0.6", new BoundSetterFactory());
return factoryMap;
}
}
openstreetmap-osmosis-b835354/core/src/main/java/org/openstreetmap/osmosis/core/sort/ 0000775 0001750 0001750 00000000000 11667575673 030016 5 ustar david david openstreetmap-osmosis-b835354/core/src/main/java/org/openstreetmap/osmosis/core/sort/v0_6/ 0000775 0001750 0001750 00000000000 11667575673 030570 5 ustar david david ././@LongLink 0000000 0000000 0000000 00000000200 00000000000 011555 L ustar root root openstreetmap-osmosis-b835354/core/src/main/java/org/openstreetmap/osmosis/core/sort/v0_6/SortedHistoryChangePipeValidator.java openstreetmap-osmosis-b835354/core/src/main/java/org/openstreetmap/osmosis/core/sort/v0_6/SortedHist0000664 0001750 0001750 00000004577 11667575673 032620 0 ustar david david // This software is released into the Public Domain. See copying.txt for details.
package org.openstreetmap.osmosis.core.sort.v0_6;
import java.util.Comparator;
import org.openstreetmap.osmosis.core.OsmosisRuntimeException;
import org.openstreetmap.osmosis.core.container.v0_6.ChangeContainer;
import org.openstreetmap.osmosis.core.task.v0_6.ChangeSink;
import org.openstreetmap.osmosis.core.task.v0_6.ChangeSinkChangeSource;
/**
* Validates that change data in a pipeline is sorted by entity type, then id, then version allowing
* full history changes. It accepts input data from a Source and passes all data to a downstream
* Sink.
*
* @author Brett Henderson
*/
public class SortedHistoryChangePipeValidator implements ChangeSinkChangeSource {
private ChangeSink changeSink;
private Comparator comparator;
private ChangeContainer previousChangeContainer;
/**
* Creates a new instance.
*/
public SortedHistoryChangePipeValidator() {
comparator = new ChangeAsEntityComparator(new EntityContainerComparator(
new EntityByTypeThenIdThenVersionComparator()));
}
/**
* {@inheritDoc}
*/
public void complete() {
changeSink.complete();
}
/**
* {@inheritDoc}
*/
public void process(ChangeContainer changeContainer) {
// If this is not the first entity in the pipeline, make sure this
// entity is greater than the previous.
if (previousChangeContainer != null) {
if (comparator.compare(previousChangeContainer, changeContainer) >= 0) {
throw new OsmosisRuntimeException(
"Pipeline entities are not sorted, previous entity type="
+ previousChangeContainer.getEntityContainer().getEntity().getType()
+ ", id=" + previousChangeContainer.getEntityContainer().getEntity().getId()
+ ", version=" + previousChangeContainer.getEntityContainer().getEntity().getVersion()
+ " current entity type=" + changeContainer.getEntityContainer().getEntity().getType()
+ ", id=" + changeContainer.getEntityContainer().getEntity().getId()
+ ", version=" + changeContainer.getEntityContainer().getEntity().getVersion() + "."
);
}
}
changeSink.process(changeContainer);
previousChangeContainer = changeContainer;
}
/**
* {@inheritDoc}
*/
public void release() {
changeSink.release();
}
/**
* {@inheritDoc}
*/
public void setChangeSink(ChangeSink changeSink) {
this.changeSink = changeSink;
}
}
././@LongLink 0000000 0000000 0000000 00000000163 00000000000 011565 L ustar root root openstreetmap-osmosis-b835354/core/src/main/java/org/openstreetmap/osmosis/core/sort/v0_6/StackableComparator.java openstreetmap-osmosis-b835354/core/src/main/java/org/openstreetmap/osmosis/core/sort/v0_6/StackableC0000664 0001750 0001750 00000002330 11667575673 032505 0 ustar david david // This software is released into the Public Domain. See copying.txt for details.
package org.openstreetmap.osmosis.core.sort.v0_6;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
/**
* A comparator implementation that allows multiple comparators to be combined together. It invokes
* each comparator in order, and returns the first non-zero result. If all comparators return 0, the
* result is 0.
*
* @param
* The type of data to be compared.
*/
public class StackableComparator implements Comparator {
private List> comparators;
/**
* Creates a new instance.
*
* @param comparators
* The comparators to use for comparisons.
*/
public StackableComparator(List> comparators) {
this.comparators = new ArrayList>(comparators);
}
/**
* {@inheritDoc}
*/
@Override
public int compare(T o1, T o2) {
// Compare using each comparator in turn. Stop if a comparator detects a difference and
// return the result.
for (Comparator comparator : comparators) {
int result;
result = comparator.compare(o1, o2);
if (result != 0) {
return result;
}
}
return 0;
}
}
././@LongLink 0000000 0000000 0000000 00000000171 00000000000 011564 L ustar root root openstreetmap-osmosis-b835354/core/src/main/java/org/openstreetmap/osmosis/core/sort/v0_6/EntityByVersionComparator.java openstreetmap-osmosis-b835354/core/src/main/java/org/openstreetmap/osmosis/core/sort/v0_6/EntityByVe0000664 0001750 0001750 00000001233 11667575673 032554 0 ustar david david // This software is released into the Public Domain. See copying.txt for details.
package org.openstreetmap.osmosis.core.sort.v0_6;
import java.util.Comparator;
import org.openstreetmap.osmosis.core.domain.v0_6.Entity;
/**
* Compares two entities and sorts them by their version.
*
* @author Brett Henderson
*/
public class EntityByVersionComparator implements Comparator {
/**
* {@inheritDoc}
*/
public int compare(Entity o1, Entity o2) {
long verDiff;
// Compare the version.
verDiff = o1.getVersion() - o2.getVersion();
if (verDiff > 0) {
return 1;
} else if (verDiff < 0) {
return -1;
} else {
return 0;
}
}
}
././@LongLink 0000000 0000000 0000000 00000000207 00000000000 011564 L ustar root root openstreetmap-osmosis-b835354/core/src/main/java/org/openstreetmap/osmosis/core/sort/v0_6/EntityByTypeThenIdThenVersionComparator.java openstreetmap-osmosis-b835354/core/src/main/java/org/openstreetmap/osmosis/core/sort/v0_6/EntityByTy0000664 0001750 0001750 00000002310 11667575673 032573 0 ustar david david // This software is released into the Public Domain. See copying.txt for details.
package org.openstreetmap.osmosis.core.sort.v0_6;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import org.openstreetmap.osmosis.core.domain.v0_6.Entity;
/**
* Orders entities first by their type (bound, node, way and relation), then their identifer, then
* their version.
*
* @author Brett Henderson
*/
public class EntityByTypeThenIdThenVersionComparator implements Comparator {
private Comparator comparator;
/**
* Creates a new instance.
*/
public EntityByTypeThenIdThenVersionComparator() {
List> entityComparators;
// Build the sequence of entity comparisons.
entityComparators = new ArrayList>();
entityComparators.add(new EntityByTypeComparator());
entityComparators.add(new EntityByIdComparator());
entityComparators.add(new EntityByVersionComparator());
// Combine all entity comparisons into a single logical comparison.
comparator = new StackableComparator(entityComparators);
}
/**
* {@inheritDoc}
*/
public int compare(Entity o1, Entity o2) {
return comparator.compare(o1, o2);
}
}
././@LongLink 0000000 0000000 0000000 00000000160 00000000000 011562 L ustar root root openstreetmap-osmosis-b835354/core/src/main/java/org/openstreetmap/osmosis/core/sort/v0_6/TagSorterFactory.java openstreetmap-osmosis-b835354/core/src/main/java/org/openstreetmap/osmosis/core/sort/v0_6/TagSorterF0000664 0001750 0001750 00000001437 11667575673 032540 0 ustar david david // This software is released into the Public Domain. See copying.txt for details.
package org.openstreetmap.osmosis.core.sort.v0_6;
import org.openstreetmap.osmosis.core.pipeline.common.TaskConfiguration;
import org.openstreetmap.osmosis.core.pipeline.common.TaskManager;
import org.openstreetmap.osmosis.core.pipeline.common.TaskManagerFactory;
import org.openstreetmap.osmosis.core.pipeline.v0_6.SinkSourceManager;
/**
* The task manager factory for a tag sorter.
*
* @author Brett Henderson
*/
public class TagSorterFactory extends TaskManagerFactory {
/**
* {@inheritDoc}
*/
@Override
protected TaskManager createTaskManagerImpl(TaskConfiguration taskConfig) {
return new SinkSourceManager(
taskConfig.getId(),
new TagSorter(),
taskConfig.getPipeArgs()
);
}
}
././@LongLink 0000000 0000000 0000000 00000000202 00000000000 011557 L ustar root root openstreetmap-osmosis-b835354/core/src/main/java/org/openstreetmap/osmosis/core/sort/v0_6/ChangeForSeekableApplierComparator.java openstreetmap-osmosis-b835354/core/src/main/java/org/openstreetmap/osmosis/core/sort/v0_6/ChangeForS0000664 0001750 0001750 00000005663 11667575673 032504 0 ustar david david // This software is released into the Public Domain. See copying.txt for details.
package org.openstreetmap.osmosis.core.sort.v0_6;
import java.util.Comparator;
import org.openstreetmap.osmosis.core.container.v0_6.ChangeContainer;
import org.openstreetmap.osmosis.core.OsmosisRuntimeException;
import org.openstreetmap.osmosis.core.domain.v0_6.Entity;
import org.openstreetmap.osmosis.core.domain.v0_6.EntityType;
import org.openstreetmap.osmosis.core.task.common.ChangeAction;
/**
* Orders changes in such a way that they can be applied to a data store while
* maintaining data integrity (ie. a database). For example, the ordering
* prevents a way being added before the underlying nodes are created. The
* changes are ordered as follows:
*
* - Bound creation
* - Node creation
* - Way creation
* - Relation creation
* - Relation modification
* - Way modification
* - Node modification
* - Bound modification
* - Relation deletion
* - Way deletion
* - Node deletion
* - Bound deletion
*
*
* @author Brett Henderson
*/
public class ChangeForSeekableApplierComparator implements Comparator {
/**
* Create a weighting for the change. The weighting is the index into the
* sorting list implemented by this class.
*
* @param changeEntity
* The change to be analysed.
* @return The sort weighting.
*/
private int calculateSortWeight(ChangeContainer changeEntity) {
ChangeAction action = changeEntity.getAction();
Entity entity = changeEntity.getEntityContainer().getEntity();
if (entity.getType().equals(EntityType.Bound)) {
if (action.equals(ChangeAction.Create)) {
return 1;
}
if (action.equals(ChangeAction.Modify)) {
return 8;
}
if (action.equals(ChangeAction.Delete)) {
return 12;
}
} else if (entity.getType().equals(EntityType.Node)) {
if (action.equals(ChangeAction.Create)) {
return 2;
}
if (action.equals(ChangeAction.Modify)) {
return 7;
}
if (action.equals(ChangeAction.Delete)) {
return 11;
}
} else if (entity.getType().equals(EntityType.Way)) {
if (action.equals(ChangeAction.Create)) {
return 3;
}
if (action.equals(ChangeAction.Modify)) {
return 6;
}
if (action.equals(ChangeAction.Delete)) {
return 10;
}
} else if (entity.getType().equals(EntityType.Relation)) {
if (action.equals(ChangeAction.Create)) {
return 4;
}
if (action.equals(ChangeAction.Modify)) {
return 5;
}
if (action.equals(ChangeAction.Delete)) {
return 9;
}
}
throw new OsmosisRuntimeException(
"The change entity with action " + action
+ " type " + entity.getType()
+ " and id " + entity.getId()
+ " was not recognised."
);
}
/**
* {@inheritDoc}
*/
public int compare(ChangeContainer o1, ChangeContainer o2) {
return calculateSortWeight(o1) - calculateSortWeight(o2);
}
}
././@LongLink 0000000 0000000 0000000 00000000157 00000000000 011570 L ustar root root openstreetmap-osmosis-b835354/core/src/main/java/org/openstreetmap/osmosis/core/sort/v0_6/ChangeTagSorter.java openstreetmap-osmosis-b835354/core/src/main/java/org/openstreetmap/osmosis/core/sort/v0_6/ChangeTagS0000664 0001750 0001750 00000004144 11667575673 032462 0 ustar david david // This software is released into the Public Domain. See copying.txt for details.
package org.openstreetmap.osmosis.core.sort.v0_6;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import org.openstreetmap.osmosis.core.container.v0_6.ChangeContainer;
import org.openstreetmap.osmosis.core.container.v0_6.EntityContainer;
import org.openstreetmap.osmosis.core.domain.v0_6.Entity;
import org.openstreetmap.osmosis.core.domain.v0_6.Tag;
import org.openstreetmap.osmosis.core.task.v0_6.ChangeSink;
import org.openstreetmap.osmosis.core.task.v0_6.ChangeSinkChangeSource;
/**
* A data stream filter that sorts tags on changes. This is useful for testing
* to allow two sets of data to be compared for equality.
*
* @author Brett Henderson
*/
public class ChangeTagSorter implements ChangeSinkChangeSource {
private ChangeSink changeSink;
/**
* {@inheritDoc}
*/
public void process(ChangeContainer changeContainer) {
EntityContainer readOnlyContainer;
EntityContainer writeableContainer;
Entity entity;
Collection sortedTags;
readOnlyContainer = changeContainer.getEntityContainer();
writeableContainer = readOnlyContainer.getWriteableInstance();
entity = writeableContainer.getEntity();
sortedTags = sortTags(entity.getTags());
entity.getTags().clear();
entity.getTags().addAll(sortedTags);
changeSink.process(new ChangeContainer(writeableContainer, changeContainer.getAction()));
}
/**
* Sorts the specified tag list.
*
* @param tagList
* The tag list to be sorted.
* @return A new list containing the sorted tags.
*/
private List sortTags(Collection tagList) {
List sortedTagList;
sortedTagList = new ArrayList(tagList);
Collections.sort(sortedTagList);
return sortedTagList;
}
/**
* {@inheritDoc}
*/
public void setChangeSink(ChangeSink changeSink) {
this.changeSink = changeSink;
}
/**
* {@inheritDoc}
*/
public void complete() {
changeSink.complete();
}
/**
* {@inheritDoc}
*/
public void release() {
changeSink.release();
}
}
././@LongLink 0000000 0000000 0000000 00000000163 00000000000 011565 L ustar root root openstreetmap-osmosis-b835354/core/src/main/java/org/openstreetmap/osmosis/core/sort/v0_6/EntitySorterFactory.java openstreetmap-osmosis-b835354/core/src/main/java/org/openstreetmap/osmosis/core/sort/v0_6/EntitySort0000664 0001750 0001750 00000005314 11667575673 032642 0 ustar david david // This software is released into the Public Domain. See copying.txt for details.
package org.openstreetmap.osmosis.core.sort.v0_6;
import java.util.Comparator;
import java.util.HashMap;
import java.util.Map;
import org.openstreetmap.osmosis.core.container.v0_6.EntityContainer;
import org.openstreetmap.osmosis.core.OsmosisRuntimeException;
import org.openstreetmap.osmosis.core.pipeline.common.TaskConfiguration;
import org.openstreetmap.osmosis.core.pipeline.common.TaskManager;
import org.openstreetmap.osmosis.core.pipeline.common.TaskManagerFactory;
import org.openstreetmap.osmosis.core.pipeline.v0_6.SinkSourceManager;
/**
* The task manager factory for an entity sorter.
*
* @author Brett Henderson
*/
public class EntitySorterFactory extends TaskManagerFactory {
private static final String ARG_COMPARATOR_TYPE = "type";
private Map> comparatorMap;
private String defaultComparatorType;
/**
* Creates a new instance.
*/
public EntitySorterFactory() {
comparatorMap = new HashMap>();
}
/**
* Registers a new comparator.
*
* @param comparatorType
* The name of the comparator.
* @param comparator
* The comparator.
* @param setAsDefault
* If true, this will be set to be the default comparator if no
* comparator is specified.
*/
public void registerComparator(
String comparatorType, Comparator comparator, boolean setAsDefault) {
if (comparatorMap.containsKey(comparatorType)) {
throw new OsmosisRuntimeException("Comparator type \"" + comparatorType + "\" already exists.");
}
if (setAsDefault) {
defaultComparatorType = comparatorType;
}
comparatorMap.put(comparatorType, comparator);
}
/**
* Retrieves the comparator identified by the specified type.
*
* @param comparatorType
* The comparator to be retrieved.
* @return The comparator.
*/
private Comparator getComparator(String comparatorType) {
if (!comparatorMap.containsKey(comparatorType)) {
throw new OsmosisRuntimeException("Comparator type " + comparatorType
+ " doesn't exist.");
}
return comparatorMap.get(comparatorType);
}
/**
* {@inheritDoc}
*/
@Override
protected TaskManager createTaskManagerImpl(TaskConfiguration taskConfig) {
Comparator comparator;
// Get the comparator.
comparator = getComparator(
getStringArgument(
taskConfig,
ARG_COMPARATOR_TYPE,
getDefaultStringArgument(taskConfig, defaultComparatorType)
)
);
return new SinkSourceManager(
taskConfig.getId(),
new EntitySorter(comparator),
taskConfig.getPipeArgs()
);
}
}
././@LongLink 0000000 0000000 0000000 00000000204 00000000000 011561 L ustar root root openstreetmap-osmosis-b835354/core/src/main/java/org/openstreetmap/osmosis/core/sort/v0_6/ChangeForStreamableApplierComparator.java openstreetmap-osmosis-b835354/core/src/main/java/org/openstreetmap/osmosis/core/sort/v0_6/ChangeForS0000664 0001750 0001750 00000002423 11667575673 032473 0 ustar david david // This software is released into the Public Domain. See copying.txt for details.
package org.openstreetmap.osmosis.core.sort.v0_6;
import java.util.Comparator;
import org.openstreetmap.osmosis.core.container.v0_6.ChangeContainer;
/**
* Orders changes in such a way that they can be applied to an ordered data
* stream without requiring seeking throughout the data stream. (ie. an xml
* dump). The change action to be performed (eg. Create) doesn't affect the sort
* order. The changes are ordered as follows:
*
* - Nodes ordered by id and version
* - Ways ordered by id and version
* - Relations ordered by id and version
*
*
* @author Brett Henderson
*/
public class ChangeForStreamableApplierComparator implements Comparator {
private Comparator comparator;
/**
* Creates a new instance.
*/
public ChangeForStreamableApplierComparator() {
// We have an existing entity comparator that performs the same ordering so simply adapt it.
comparator = new ChangeAsEntityComparator(new EntityContainerComparator(
new EntityByTypeThenIdThenVersionComparator()));
}
/**
* {@inheritDoc}
*/
public int compare(ChangeContainer o1, ChangeContainer o2) {
return comparator.compare(o1, o2);
}
}
././@LongLink 0000000 0000000 0000000 00000000154 00000000000 011565 L ustar root root openstreetmap-osmosis-b835354/core/src/main/java/org/openstreetmap/osmosis/core/sort/v0_6/EntitySorter.java openstreetmap-osmosis-b835354/core/src/main/java/org/openstreetmap/osmosis/core/sort/v0_6/EntitySort0000664 0001750 0001750 00000003422 11667575673 032640 0 ustar david david // This software is released into the Public Domain. See copying.txt for details.
package org.openstreetmap.osmosis.core.sort.v0_6;
import java.util.Comparator;
import org.openstreetmap.osmosis.core.container.v0_6.EntityContainer;
import org.openstreetmap.osmosis.core.lifecycle.ReleasableIterator;
import org.openstreetmap.osmosis.core.sort.common.FileBasedSort;
import org.openstreetmap.osmosis.core.store.GenericObjectSerializationFactory;
import org.openstreetmap.osmosis.core.task.v0_6.Sink;
import org.openstreetmap.osmosis.core.task.v0_6.SinkSource;
/**
* A data stream filter that sorts entities. The sort order is specified by
* comparator provided during instantiation.
*
* @author Brett Henderson
*/
public class EntitySorter implements SinkSource {
private FileBasedSort fileBasedSort;
private Sink sink;
/**
* Creates a new instance.
*
* @param comparator
* The comparator to use for sorting.
*/
public EntitySorter(Comparator comparator) {
fileBasedSort = new FileBasedSort(new GenericObjectSerializationFactory(), comparator, true);
}
/**
* {@inheritDoc}
*/
public void process(EntityContainer entityContainer) {
fileBasedSort.add(entityContainer);
}
/**
* {@inheritDoc}
*/
public void setSink(Sink sink) {
this.sink = sink;
}
/**
* {@inheritDoc}
*/
public void complete() {
ReleasableIterator iterator = null;
try {
iterator = fileBasedSort.iterate();
while (iterator.hasNext()) {
sink.process(iterator.next());
}
sink.complete();
} finally {
if (iterator != null) {
iterator.release();
}
}
}
/**
* {@inheritDoc}
*/
public void release() {
fileBasedSort.release();
sink.release();
}
}
././@LongLink 0000000 0000000 0000000 00000000174 00000000000 011567 L ustar root root openstreetmap-osmosis-b835354/core/src/main/java/org/openstreetmap/osmosis/core/sort/v0_6/EntityByTypeThenIdComparator.java openstreetmap-osmosis-b835354/core/src/main/java/org/openstreetmap/osmosis/core/sort/v0_6/EntityByTy0000664 0001750 0001750 00000002141 11667575673 032575 0 ustar david david // This software is released into the Public Domain. See copying.txt for details.
package org.openstreetmap.osmosis.core.sort.v0_6;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import org.openstreetmap.osmosis.core.domain.v0_6.Entity;
/**
* Orders entities first by their type (bound, node, way and relation), then their identifer.
*
* @author Brett Henderson
*/
public class EntityByTypeThenIdComparator implements Comparator {
private Comparator comparator;
/**
* Creates a new instance.
*/
public EntityByTypeThenIdComparator() {
List> entityComparators;
// Build the sequence of entity comparisons.
entityComparators = new ArrayList>();
entityComparators.add(new EntityByTypeComparator());
entityComparators.add(new EntityByIdComparator());
// Combine all entity comparisons into a single logical comparison.
comparator = new StackableComparator(entityComparators);
}
/**
* {@inheritDoc}
*/
public int compare(Entity o1, Entity o2) {
return comparator.compare(o1, o2);
}
}
././@LongLink 0000000 0000000 0000000 00000000202 00000000000 011557 L ustar root root openstreetmap-osmosis-b835354/core/src/main/java/org/openstreetmap/osmosis/core/sort/v0_6/SortedDuplicateEntityPipeValidator.java openstreetmap-osmosis-b835354/core/src/main/java/org/openstreetmap/osmosis/core/sort/v0_6/SortedDupl0000664 0001750 0001750 00000004143 11667575673 032602 0 ustar david david // This software is released into the Public Domain. See copying.txt for details.
package org.openstreetmap.osmosis.core.sort.v0_6;
import org.openstreetmap.osmosis.core.OsmosisRuntimeException;
import org.openstreetmap.osmosis.core.container.v0_6.EntityContainer;
import org.openstreetmap.osmosis.core.task.v0_6.Sink;
import org.openstreetmap.osmosis.core.task.v0_6.SinkSource;
/**
* Validates that entity data in a pipeline is sorted by entity type then id. It
* allows duplicates. It accepts input data from a Source and passes all data to
* a downstream Sink.
*
* @author Brett Henderson
*/
public class SortedDuplicateEntityPipeValidator implements SinkSource {
private Sink sink;
private EntityContainerComparator comparator;
private EntityContainer previousEntityContainer;
/**
* Creates a new instance.
*/
public SortedDuplicateEntityPipeValidator() {
comparator = new EntityContainerComparator(new EntityByTypeThenIdComparator());
}
/**
* {@inheritDoc}
*/
public void complete() {
sink.complete();
}
/**
* {@inheritDoc}
*/
public void process(EntityContainer entityContainer) {
// If this is not the first entity in the pipeline, make sure this
// entity is greater than the previous.
if (previousEntityContainer != null) {
if (comparator.compare(previousEntityContainer, entityContainer) > 0) {
throw new OsmosisRuntimeException(
"Pipeline entities are not sorted, previous entity type="
+ previousEntityContainer.getEntity().getType() + ", id="
+ previousEntityContainer.getEntity().getId() + ", version="
+ previousEntityContainer.getEntity().getVersion() + " current entity type="
+ entityContainer.getEntity().getType() + ", id="
+ entityContainer.getEntity().getId() + ", version="
+ entityContainer.getEntity().getVersion() + "."
);
}
}
sink.process(entityContainer);
previousEntityContainer = entityContainer;
}
/**
* {@inheritDoc}
*/
public void release() {
sink.release();
}
/**
* {@inheritDoc}
*/
public void setSink(Sink sink) {
this.sink = sink;
}
}
././@LongLink 0000000 0000000 0000000 00000000171 00000000000 011564 L ustar root root openstreetmap-osmosis-b835354/core/src/main/java/org/openstreetmap/osmosis/core/sort/v0_6/EntityContainerComparator.java openstreetmap-osmosis-b835354/core/src/main/java/org/openstreetmap/osmosis/core/sort/v0_6/EntityCont0000664 0001750 0001750 00000002076 11667575673 032620 0 ustar david david // This software is released into the Public Domain. See copying.txt for details.
package org.openstreetmap.osmosis.core.sort.v0_6;
import java.util.Comparator;
import org.openstreetmap.osmosis.core.container.v0_6.EntityContainer;
import org.openstreetmap.osmosis.core.domain.v0_6.Entity;
/**
* An entity container comparator utilising an inner entity comparator. This allows entity
* containers to be directly compared instead of having to extract the entities first.
*
* @author Brett Henderson
*/
public class EntityContainerComparator implements Comparator {
private Comparator entityComparator;
/**
* Creates a new instance.
*
* @param entityComparator
* The comparator to use for comparing the contained entities.
*/
public EntityContainerComparator(Comparator entityComparator) {
this.entityComparator = entityComparator;
}
/**
* {@inheritDoc}
*/
@Override
public int compare(EntityContainer o1, EntityContainer o2) {
return entityComparator.compare(o1.getEntity(), o2.getEntity());
}
}
././@LongLink 0000000 0000000 0000000 00000000151 00000000000 011562 L ustar root root openstreetmap-osmosis-b835354/core/src/main/java/org/openstreetmap/osmosis/core/sort/v0_6/TagSorter.java openstreetmap-osmosis-b835354/core/src/main/java/org/openstreetmap/osmosis/core/sort/v0_6/TagSorter.0000664 0001750 0001750 00000003452 11667575673 032507 0 ustar david david // This software is released into the Public Domain. See copying.txt for details.
package org.openstreetmap.osmosis.core.sort.v0_6;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import org.openstreetmap.osmosis.core.container.v0_6.EntityContainer;
import org.openstreetmap.osmosis.core.domain.v0_6.Entity;
import org.openstreetmap.osmosis.core.domain.v0_6.Tag;
import org.openstreetmap.osmosis.core.task.v0_6.Sink;
import org.openstreetmap.osmosis.core.task.v0_6.SinkSource;
/**
* A data stream filter that sorts tags on entities. This is useful for testing
* to allow two sets of data to be compared for equality.
*
* @author Brett Henderson
*/
public class TagSorter implements SinkSource {
private Sink sink;
/**
* {@inheritDoc}
*/
public void process(EntityContainer entityContainer) {
EntityContainer writeableContainer;
Entity entity;
Collection sortedTags;
writeableContainer = entityContainer.getWriteableInstance();
entity = writeableContainer.getEntity();
sortedTags = sortTags(entity.getTags());
entity.getTags().clear();
entity.getTags().addAll(sortedTags);
sink.process(writeableContainer);
}
/**
* Sorts the specified tag list.
*
* @param tagList
* The tag list to be sorted.
* @return A new list containing the sorted tags.
*/
private List sortTags(Collection tagList) {
List sortedTagList;
sortedTagList = new ArrayList(tagList);
Collections.sort(sortedTagList);
return sortedTagList;
}
/**
* {@inheritDoc}
*/
public void setSink(Sink sink) {
this.sink = sink;
}
/**
* {@inheritDoc}
*/
public void complete() {
sink.complete();
}
/**
* {@inheritDoc}
*/
public void release() {
sink.release();
}
}
././@LongLink 0000000 0000000 0000000 00000000164 00000000000 011566 L ustar root root openstreetmap-osmosis-b835354/core/src/main/java/org/openstreetmap/osmosis/core/sort/v0_6/EntityByIdComparator.java openstreetmap-osmosis-b835354/core/src/main/java/org/openstreetmap/osmosis/core/sort/v0_6/EntityById0000664 0001750 0001750 00000001236 11667575673 032541 0 ustar david david // This software is released into the Public Domain. See copying.txt for details.
package org.openstreetmap.osmosis.core.sort.v0_6;
import java.util.Comparator;
import org.openstreetmap.osmosis.core.domain.v0_6.Entity;
/**
* Compares two entities and sorts them by their identifier.
*
* @author Brett Henderson
*/
public class EntityByIdComparator implements Comparator {
/**
* {@inheritDoc}
*/
public int compare(Entity o1, Entity o2) {
long idDiff;
// Perform an identifier comparison.
idDiff = o1.getId() - o2.getId();
if (idDiff > 0) {
return 1;
} else if (idDiff < 0) {
return -1;
} else {
return 0;
}
}
}
././@LongLink 0000000 0000000 0000000 00000000176 00000000000 011571 L ustar root root openstreetmap-osmosis-b835354/core/src/main/java/org/openstreetmap/osmosis/core/sort/v0_6/SortedDeltaChangePipeValidator.java openstreetmap-osmosis-b835354/core/src/main/java/org/openstreetmap/osmosis/core/sort/v0_6/SortedDelt0000664 0001750 0001750 00000004661 11667575673 032573 0 ustar david david // This software is released into the Public Domain. See copying.txt for details.
package org.openstreetmap.osmosis.core.sort.v0_6;
import java.util.Comparator;
import org.openstreetmap.osmosis.core.OsmosisRuntimeException;
import org.openstreetmap.osmosis.core.container.v0_6.ChangeContainer;
import org.openstreetmap.osmosis.core.task.v0_6.ChangeSink;
import org.openstreetmap.osmosis.core.task.v0_6.ChangeSinkChangeSource;
/**
* Validates that change data in a pipeline is sorted by entity type then id thus only allowing
* delta style changes (ie. not full history). It accepts input data from a Source and passes all
* data to a downstream Sink.
*
* @author Brett Henderson
*/
public class SortedDeltaChangePipeValidator implements ChangeSinkChangeSource {
private ChangeSink changeSink;
private Comparator comparator;
private ChangeContainer previousChangeContainer;
/**
* Creates a new instance.
*/
public SortedDeltaChangePipeValidator() {
comparator = new ChangeAsEntityComparator(new EntityContainerComparator(new EntityByTypeThenIdComparator()));
}
/**
* {@inheritDoc}
*/
public void complete() {
changeSink.complete();
}
/**
* {@inheritDoc}
*/
public void process(ChangeContainer changeContainer) {
// If this is not the first entity in the pipeline, make sure this
// entity is greater than the previous.
if (previousChangeContainer != null) {
if (comparator.compare(previousChangeContainer, changeContainer) >= 0) {
throw new OsmosisRuntimeException(
"Pipeline entities are not sorted or contain multiple versions of a single entity"
+ ", previous entity type=" + previousChangeContainer.getEntityContainer().getEntity().getType()
+ ", id=" + previousChangeContainer.getEntityContainer().getEntity().getId()
+ ", version=" + previousChangeContainer.getEntityContainer().getEntity().getVersion()
+ " current entity type=" + changeContainer.getEntityContainer().getEntity().getType()
+ ", id=" + changeContainer.getEntityContainer().getEntity().getId()
+ ", version=" + changeContainer.getEntityContainer().getEntity().getVersion() + "."
);
}
}
changeSink.process(changeContainer);
previousChangeContainer = changeContainer;
}
/**
* {@inheritDoc}
*/
public void release() {
changeSink.release();
}
/**
* {@inheritDoc}
*/
public void setChangeSink(ChangeSink changeSink) {
this.changeSink = changeSink;
}
}
././@LongLink 0000000 0000000 0000000 00000000163 00000000000 011565 L ustar root root openstreetmap-osmosis-b835354/core/src/main/java/org/openstreetmap/osmosis/core/sort/v0_6/ChangeSorterFactory.java openstreetmap-osmosis-b835354/core/src/main/java/org/openstreetmap/osmosis/core/sort/v0_6/ChangeSort0000664 0001750 0001750 00000005343 11667575673 032555 0 ustar david david // This software is released into the Public Domain. See copying.txt for details.
package org.openstreetmap.osmosis.core.sort.v0_6;
import java.util.Comparator;
import java.util.HashMap;
import java.util.Map;
import org.openstreetmap.osmosis.core.container.v0_6.ChangeContainer;
import org.openstreetmap.osmosis.core.OsmosisRuntimeException;
import org.openstreetmap.osmosis.core.pipeline.common.TaskConfiguration;
import org.openstreetmap.osmosis.core.pipeline.common.TaskManager;
import org.openstreetmap.osmosis.core.pipeline.common.TaskManagerFactory;
import org.openstreetmap.osmosis.core.pipeline.v0_6.ChangeSinkChangeSourceManager;
/**
* The task manager factory for a change sorter.
*
* @author Brett Henderson
*/
public class ChangeSorterFactory extends TaskManagerFactory {
private static final String ARG_COMPARATOR_TYPE = "type";
private Map> comparatorMap;
private String defaultComparatorType;
/**
* Creates a new instance.
*/
public ChangeSorterFactory() {
comparatorMap = new HashMap>();
}
/**
* Registers a new comparator.
*
* @param comparatorType
* The name of the comparator.
* @param comparator
* The comparator.
* @param setAsDefault
* If true, this will be set to be the default comparator if no
* comparator is specified.
*/
public void registerComparator(
String comparatorType, Comparator comparator, boolean setAsDefault) {
if (comparatorMap.containsKey(comparatorType)) {
throw new OsmosisRuntimeException("Comparator type \"" + comparatorType + "\" already exists.");
}
if (setAsDefault) {
defaultComparatorType = comparatorType;
}
comparatorMap.put(comparatorType, comparator);
}
/**
* Retrieves the comparator identified by the specified type.
*
* @param comparatorType
* The comparator to be retrieved.
* @return The comparator.
*/
private Comparator getComparator(String comparatorType) {
if (!comparatorMap.containsKey(comparatorType)) {
throw new OsmosisRuntimeException("Comparator type " + comparatorType
+ " doesn't exist.");
}
return comparatorMap.get(comparatorType);
}
/**
* {@inheritDoc}
*/
@Override
protected TaskManager createTaskManagerImpl(TaskConfiguration taskConfig) {
Comparator