pax_global_header00006660000000000000000000000064122675237510014524gustar00rootroot0000000000000052 comment=e2ec43dbe7293eb807eb35a967774a8fd2358519 Chronicle-Queue-2.0.3/000077500000000000000000000000001226752375100145165ustar00rootroot00000000000000Chronicle-Queue-2.0.3/.gitignore000066400000000000000000000001251226752375100165040ustar00rootroot00000000000000*.class # Package Files # *.jar *.war *.ear # IntelliJ *.iml .idea # maven target Chronicle-Queue-2.0.3/.idea/000077500000000000000000000000001226752375100154765ustar00rootroot00000000000000Chronicle-Queue-2.0.3/.idea/copyright/000077500000000000000000000000001226752375100175065ustar00rootroot00000000000000Chronicle-Queue-2.0.3/.idea/copyright/Apache_2_0.xml000066400000000000000000000017201226752375100220510ustar00rootroot00000000000000 Chronicle-Queue-2.0.3/README.md000066400000000000000000000011351226752375100157750ustar00rootroot00000000000000# Chronicle Persisted low latency logging and IPC messaging It is available on maven central as
<dependency>
    <groupId>net.openhft</groupId>
    <artifactId>chronicle</artifactId>
    <version>2.0.1</version>
</dependency>
For working examples, have a look at the unit tests and demo module. --- [Chronicle Wiki](https://github.com/OpenHFT/Java-Chronicle/wiki) [Java Chronicle support group](https://groups.google.com/forum/?hl=en-GB#!forum/java-chronicle) [Forum for those learning about OpenHFT](https://groups.google.com/forum/?hl=en-GB#!forum/open-hft) Chronicle-Queue-2.0.3/chronicle-demo/000077500000000000000000000000001226752375100174065ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle-demo/pom.xml000077500000000000000000000050001226752375100207210ustar00rootroot00000000000000 4.0.0 net.openhft Java-Chronicle 2.1-SNAPSHOT ../pom.xml chronicle-demo chronicle-demo 2.1-SNAPSHOT Demo Processing Engine net.openhft chronicle ${project.version} net.openhft chronicle-sandbox ${project.version} net.openhft affinity 2.0.1 org.kohsuke.jetbrains annotations 9.0 compile org.apache.maven.plugins maven-compiler-plugin 3.1 1.6 1.6 UTF-8 Chronicle-Queue-2.0.3/chronicle-demo/src/000077500000000000000000000000001226752375100201755ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle-demo/src/main/000077500000000000000000000000001226752375100211215ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle-demo/src/main/java/000077500000000000000000000000001226752375100220425ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle-demo/src/main/java/vanilla/000077500000000000000000000000001226752375100234705ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle-demo/src/main/java/vanilla/java/000077500000000000000000000000001226752375100244115ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle-demo/src/main/java/vanilla/java/processingengine/000077500000000000000000000000001226752375100277535ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle-demo/src/main/java/vanilla/java/processingengine/GWMain.java000077500000000000000000000227151226752375100317520ustar00rootroot00000000000000/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package vanilla.java.processingengine; import net.openhft.affinity.AffinitySupport; import net.openhft.chronicle.ChronicleConfig; import net.openhft.chronicle.sandbox.VanillaChronicle; import net.openhft.chronicle.sandbox.VanillaChronicleConfig; import net.openhft.chronicle.tools.ChronicleTools; import org.jetbrains.annotations.NotNull; import vanilla.java.processingengine.api.*; import java.io.IOException; import java.util.Arrays; import java.util.concurrent.atomic.AtomicInteger; /** * For a latency test Start first: PEMain Then run: GEMain 2 false When the count down is reached: GEMain 1 false * * @author peter.lawrey */ /* on a dual core i7-4500 laptop Processed 10,000,000 events in and out in 100.2 seconds The latency distribution was 0.5, 0.6/3.4, 322/947 (3,683) us for the 50, 90/99, 99.9/99.99 %tile, (worst) The latency distribution was 0.6, 0.7/3.1, 15/544 (1,856) us for the 50, 90/99, 99.9/99.99 %tile, (worst) Processed 10,000,000 events in and out in 50.1 seconds The latency distribution was 0.6, 0.7/4.0, 132/3563 (6,319) us for the 50, 90/99, 99.9/99.99 %tile, (worst) The latency distribution was 0.5, 0.6/3.2, 58/1557 (4,031) us for the 50, 90/99, 99.9/99.99 %tile, (worst) Processed 10,000,000 events in and out in 20.0 seconds The latency distribution was 0.4, 1.5/32.5, 2420/6440 (9,733) us for the 50, 90/99, 99.9/99.99 %tile, (worst) The latency distribution was 0.6, 0.7/3.3, 17/380 (537) us for the 50, 90/99, 99.9/99.99 %tile, (worst) The latency distribution was 0.5, 1.2/4.3, 226/1800 (2,482) us for the 50, 90/99, 99.9/99.99 %tile, (worst) Processed 10,000,000 events in and out in 10.1 seconds The latency distribution was 31386.8, 80422.8/97564.0, 99054/99780 (100,282) us for the 50, 90/99, 99.9/99.99 %tile, (worst) The latency distribution was 0.5, 40561.4/54889.5, 56752/56876 (56,902) us for the 50, 90/99, 99.9/99.99 %tile, (worst) on a hex core i7 using isolated CPUs (all runs, good and bad) Processed 10,000,000 events in and out in 100.2 seconds The latency distribution was 0.3, 0.3/1.5, 2/13 (6,312) us for the 50, 90/99, 99.9/99.99 %tile, (worst) Processed 100,000,000 events in and out in 1001.0 seconds The latency distribution was 0.3, 0.3/1.6, 2/13 (4,072) us for the 50, 90/99, 99.9/99.99 %tile, (worst) Processed 10,000,000 events in and out in 50.1 seconds The latency distribution was 0.3, 0.3/1.5, 2/11 (91) us for the 50, 90/99, 99.9/99.99 %tile, (worst) Processed 10,000,000 events in and out in 20.0 seconds The latency distribution was 0.3, 0.3/1.6, 2/12 (77) us for the 50, 90/99, 99.9/99.99 %tile, (worst) Processed 100,000,000 events in and out in 200.2 seconds The latency distribution was 0.3, 0.3/1.5, 3/11 (84) us for the 50, 90/99, 99.9/99.99 %tile, (worst) Processed 100,000,000 events in and out in 100.1 seconds The latency distribution was 0.3, 0.9/2.9, 6/25 (2,571) us for the 50, 90/99, 99.9/99.99 %tile, (worst) Processed 100,000,000 events in and out in 50.1 seconds The latency distribution was 27.7, 185.4/598.6, 1815/3830 (4,014) us for the 50, 90/99, 99.9/99.99 %tile, (worst) Processed 100,000,000 events in and out in 50.1 seconds The latency distribution was 1.9, 14.4/38.9, 69/376 (528) us for the 50, 90/99, 99.9/99.99 %tile, (worst) Processed 100,000,000 events in and out in 50.1 seconds The latency distribution was 16.5, 81.7/199.2, 379/581 (623) us for the 50, 90/99, 99.9/99.99 %tile, (worst) */ public class GWMain { public static final int WARMUP = Integer.getInteger("warmup", 100 * 1000); // number of events public static final long EVENT_SPACING = Integer.getInteger("event-spacing", 5 * 1000); public static final int ORDERS = Integer.getInteger("orders", 10 * 1000 * 1000); public static void main(@NotNull String... args) throws IOException, InterruptedException { if (args.length < 2) { System.err.print("java " + GWMain.class.getName() + " [1 or 2] {throughput}"); System.exit(-1); } ChronicleTools.warmup(); final int gwId = Integer.parseInt(args[0]); final boolean throughputTest = Boolean.parseBoolean(args[1]); String tmp = System.getProperty("java.io.tmpdir"); // String tmp = System.getProperty("user.home"); String gw2pePath = tmp + "/demo/gw2pe" + gwId; String pePath = tmp + "/demo/pe"; // setup ChronicleConfig config = ChronicleConfig.DEFAULT.clone(); // config.dataBlockSize(4 * 1024); // config.indexBlockSize(4 * 1024); VanillaChronicle gw2pe = new VanillaChronicle(gw2pePath, VanillaChronicleConfig.DEFAULT); Gw2PeEvents gw2PeWriter = new Gw2PeWriter(gw2pe.createAppender()); VanillaChronicle pe2gw = new VanillaChronicle(pePath, VanillaChronicleConfig.DEFAULT); final long[] times = new long[ORDERS]; final AtomicInteger reportCount = new AtomicInteger(-WARMUP); Pe2GwEvents listener = new Pe2GwEvents() { @Override public void report(@NotNull MetaData metaData, SmallReport smallReport) { if (metaData.sourceId != gwId) return; int count = reportCount.getAndIncrement(); if (!throughputTest) { times[Math.abs(count)] = (metaData.outReadTimestamp - metaData.inWriteTimestamp); } // System.out.println(reportCount); } }; final Pe2GwReader pe2GwReader = new Pe2GwReader(gwId, pe2gw.createTailer(), listener); // synchronize the start. if (gwId > 1) { int startTime = (int) ((System.currentTimeMillis() / 1000 - 5) % 10) + 5; System.out.println("Count down"); for (int i = startTime; i > 0; i--) { System.out.print(i + " "); System.out.flush(); //noinspection BusyWait Thread.sleep(1000); } } // In reality, this would be in the same thread. // A second thread is used here to ensure there is no Co-ordinated Omission // where the producer slows down to suit the consumer which makes delays seem far less significant. Thread t = new Thread(new Runnable() { @Override public void run() { AffinitySupport.setAffinity(1L << 3); while (reportCount.get() < ORDERS) { pe2GwReader.readOne(); } } }); t.start(); Thread t2 = new Thread(new Runnable() { @Override public void run() { int n = 0; while (reportCount.get() < ORDERS) { while (reportCount.get() < n) try { Thread.sleep(100); } catch (InterruptedException e) { throw new AssertionError(e); } int count = reportCount.get(); System.out.println("processed " + count); n += 1000000; } } }); t2.start(); AffinitySupport.setAffinity(1L << 1); // run loop SmallCommand command = new SmallCommand(); @SuppressWarnings("MismatchedQueryAndUpdateOfStringBuilder") StringBuilder clientOrderId = command.clientOrderId; System.out.println("Started"); long start = System.nanoTime(); for (int i = 0; i < ORDERS + WARMUP; i++) { if (i == WARMUP) start = System.nanoTime(); clientOrderId.setLength(0); clientOrderId.append("orderId-"); clientOrderId.append(gwId); clientOrderId.append('-'); clientOrderId.append(i); command.instrument = "XAU/EUR"; command.price = 1209.41; command.quantity = 1000; command.side = (i & 1) == 0 ? Side.BUY : Side.SELL; if (!throughputTest) { long expectedTime = start + i * EVENT_SPACING - 30; while (System.nanoTime() < expectedTime) { // } } gw2PeWriter.small(null, command); } System.out.println("Received " + reportCount.get()); t.join(); long time = System.nanoTime() - start; Arrays.sort(times); System.out.printf("Processed %,d events in and out in %.1f seconds%n", ORDERS, time / 1e9); if (!throughputTest) { System.out.printf("The latency distribution was %.1f, %.1f/%.1f, %d/%d (%,d) us for the 50, 90/99, 99.9/99.99 %%tile, (worst)%n", times[ORDERS / 2] / 1e3, times[ORDERS * 9 / 10] / 1e3, times[ORDERS - ORDERS / 100] / 1e3, times[ORDERS - ORDERS / 1000] / 1000, times[ORDERS - ORDERS / 10000] / 1000, times[ORDERS - 1] / 1000 ); } gw2pe.close(); pe2gw.close(); } } MicroJitterSampler.java000077500000000000000000000070371226752375100343300ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle-demo/src/main/java/vanilla/java/processingengine/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package vanilla.java.processingengine; import net.openhft.affinity.AffinityLock; import org.jetbrains.annotations.NotNull; import java.io.PrintStream; /** * User: peter.lawrey Date: 30/06/13 Time: 13:13 */ public class MicroJitterSampler { static final long[] DELAY = { 2 * 1000, 3 * 1000, 4 * 1000, 6 * 1000, 8 * 1000, 10 * 1000, 14 * 1000, 20 * 1000, 30 * 1000, 40 * 1000, 60 * 1000, 80 * 1000, 100 * 1000, 140 * 1000, 200 * 1000, 300 * 1000, 400 * 1000, 600 * 1000, 800 * 1000, 1000 * 1000, 2 * 1000 * 1000, 5 * 1000 * 1000, 10 * 1000 * 1000, 20 * 1000 * 1000, 50 * 1000 * 1000, 100 * 1000 * 1000 }; static final double UTIL = Double.parseDouble(System.getProperty("util", "50")); static final int CPU = Integer.getInteger("cpu", 0); final int[] count = new int[DELAY.length]; long totalTime = 0; public static void main(String... ignored) throws InterruptedException { AffinityLock al = AffinityLock.acquireLock(); // warmup. new MicroJitterSampler().sample(1000 * 1000 * 1000); MicroJitterSampler microJitterSampler = new MicroJitterSampler(); while (!Thread.currentThread().isInterrupted()) { if (UTIL >= 100) { microJitterSampler.sample(30L * 1000 * 1000 * 1000); } else { long sampleLength = (long) ((1 / (1 - UTIL / 100) - 1) * 1000 * 1000); for (int i = 0; i < 30 * 1000; i += 2) { microJitterSampler.sample(sampleLength); //noinspection BusyWait Thread.sleep(1); } } microJitterSampler.print(System.out); } } @NotNull private static String asString(long timeNS) { return timeNS < 1000 ? timeNS + "ns" : timeNS < 1000000 ? timeNS / 1000 + "us" : timeNS < 1000000000 ? timeNS / 1000000 + "ms" : timeNS / 1000000000 + "sec"; } public void sample(long intervalNS) { long prev = System.nanoTime(); long end = prev + intervalNS; long now; do { now = System.nanoTime(); long time = now - prev; if (time >= DELAY[0]) { int i; for (i = 1; i < DELAY.length; i++) if (time < DELAY[i]) break; count[i - 1]++; } prev = now; } while (now < end); totalTime += intervalNS; } public void print(@NotNull PrintStream ps) { ps.println("After " + totalTime / 1000000000 + " seconds, the average per hour was"); for (int i = 0; i < DELAY.length; i++) { if (count[i] < 1) continue; long countPerHour = (long) Math.ceil(count[i] * 3600e9 / totalTime); ps.println(asString(DELAY[i]) + '\t' + countPerHour); } ps.println(); } } Chronicle-Queue-2.0.3/chronicle-demo/src/main/java/vanilla/java/processingengine/PEMain.java000077500000000000000000000071201226752375100317320ustar00rootroot00000000000000/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package vanilla.java.processingengine; import net.openhft.affinity.AffinitySupport; import net.openhft.chronicle.ExcerptAppender; import net.openhft.chronicle.sandbox.VanillaChronicle; import net.openhft.chronicle.sandbox.VanillaChronicleConfig; import net.openhft.chronicle.tools.ChronicleTools; import org.jetbrains.annotations.NotNull; import vanilla.java.processingengine.api.*; import java.io.IOException; /** * @author peter.lawrey */ public class PEMain { public static void main(String... ignored) throws IOException { ChronicleTools.warmup(); AffinitySupport.setAffinity(1L << 2); String tmp = System.getProperty("java.io.tmpdir"); // String tmp = System.getProperty("user.home"); // ChronicleConfig config = ChronicleConfig.DEFAULT.clone(); // config.dataBlockSize(4 * 1024); // config.indexBlockSize(4 * 1024); String pePath = tmp + "/demo/pe"; ChronicleTools.deleteDirOnExit(pePath); VanillaChronicle pe2gw = new VanillaChronicle(pePath, VanillaChronicleConfig.DEFAULT); ExcerptAppender excerpt = pe2gw.createAppender(); final Pe2GwWriter pe2GwWriter = new Pe2GwWriter(excerpt); Gw2PeEvents listener = new PEEvents(pe2GwWriter); Gw2PeReader[] readers = new Gw2PeReader[1]; VanillaChronicle[] gw2pe = new VanillaChronicle[readers.length]; for (int i = 0; i < readers.length; i++) { int sourceId = i + 1; String gw2pePath = tmp + "/demo/gw2pe" + sourceId; ChronicleTools.deleteDirOnExit(gw2pePath); gw2pe[i] = new VanillaChronicle(gw2pePath, VanillaChronicleConfig.DEFAULT); readers[i] = new Gw2PeReader(sourceId, gw2pe[i].createTailer(), listener); } long prevProcessed = 0, count = 0; //noinspection InfiniteLoopStatement do { boolean readOne = false; for (Gw2PeReader reader : readers) { readOne |= reader.readOne(); } if (readOne) { // did something count = 0; } else if (count++ > 1000000) { // do something else like pause. long processed = excerpt.index() + 1; if (prevProcessed != processed) { System.out.printf("Processed %,d requests%n", processed); prevProcessed = processed; } } } while (true); } static class PEEvents implements Gw2PeEvents { private final Pe2GwWriter pe2GwWriter; private final SmallReport smallReport = new SmallReport(); // int count = 0; public PEEvents(Pe2GwWriter pe2GwWriter) { this.pe2GwWriter = pe2GwWriter; } @Override public void small(@NotNull MetaData metaData, @NotNull SmallCommand command) { smallReport.orderOkay(command.clientOrderId); // System.out.println(++count); pe2GwWriter.report(metaData, smallReport); } } } Chronicle-Queue-2.0.3/chronicle-demo/src/main/java/vanilla/java/processingengine/api/000077500000000000000000000000001226752375100305245ustar00rootroot00000000000000Gw2PeEvents.java000077500000000000000000000013771226752375100334340ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle-demo/src/main/java/vanilla/java/processingengine/api/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package vanilla.java.processingengine.api; /** * @author peter.lawrey */ public interface Gw2PeEvents { void small(MetaData metaData, SmallCommand command); } Gw2PeReader.java000077500000000000000000000043141226752375100333640ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle-demo/src/main/java/vanilla/java/processingengine/api/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package vanilla.java.processingengine.api; import net.openhft.chronicle.ExcerptTailer; import org.jetbrains.annotations.NotNull; /** * @author peter.lawrey */ public class Gw2PeReader { private final int sourceId; private final ExcerptTailer excerpt; private final Gw2PeEvents peEvents; @NotNull private final MetaData metaData; private final SmallCommand smallCommand = new SmallCommand(); public Gw2PeReader(int sourceId, ExcerptTailer excerpt, Gw2PeEvents peEvents) { this.excerpt = excerpt; this.peEvents = peEvents; metaData = new MetaData(true); this.sourceId = sourceId; } public boolean readOne() { if (!excerpt.nextIndex()) { // System.out.println("r- " + excerpt.index()); return false; } long pos = excerpt.position(); // System.out.println("r " + excerpt.index() + " " + excerpt.capacity()); MessageType mt = excerpt.readEnum(MessageType.class); if (mt == null) { // rewind and read again. excerpt.position(pos); System.err.println("Unknown message type " + excerpt.readUTFΔ()); return true; } switch (mt) { case small: { metaData.sourceId = sourceId; metaData.readFromGateway(excerpt); smallCommand.readMarshallable(excerpt); peEvents.small(metaData, smallCommand); break; } default: System.err.println("Unknown message type " + mt); break; } return true; } } Gw2PeWriter.java000077500000000000000000000023751226752375100334430ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle-demo/src/main/java/vanilla/java/processingengine/api/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package vanilla.java.processingengine.api; import net.openhft.chronicle.ExcerptAppender; import org.jetbrains.annotations.NotNull; /** * Gateway to Processing Engine writer * * @author peter.lawrey */ public class Gw2PeWriter implements Gw2PeEvents { private final ExcerptAppender excerpt; public Gw2PeWriter(ExcerptAppender excerpt) { this.excerpt = excerpt; } @Override public void small(MetaData ignored, @NotNull SmallCommand command) { excerpt.startExcerpt(); excerpt.writeEnum(MessageType.small); MetaData.writeForGateway(excerpt); command.writeMarshallable(excerpt); excerpt.finish(); } } MessageType.java000077500000000000000000000013231226752375100335400ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle-demo/src/main/java/vanilla/java/processingengine/api/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package vanilla.java.processingengine.api; /** * @author peter.lawrey */ public enum MessageType { small, report } Chronicle-Queue-2.0.3/chronicle-demo/src/main/java/vanilla/java/processingengine/api/MetaData.java000077500000000000000000000052141226752375100330540ustar00rootroot00000000000000/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package vanilla.java.processingengine.api; import net.openhft.chronicle.ExcerptCommon; import net.openhft.lang.io.RandomDataOutput; import org.jetbrains.annotations.NotNull; /** * @author peter.lawrey */ public class MetaData { boolean targetReader; public int sourceId; public long excerptId; public long writeTimestampMillis; public int inWriteTimestamp; public int inReadTimestamp; public int outWriteTimestamp; public int outReadTimestamp; public MetaData(boolean targetReader) { this.targetReader = targetReader; } private static int fastTime() { return (int) System.nanoTime(); } public static void writeForGateway(@NotNull RandomDataOutput out) { out.writeLong(System.currentTimeMillis()); out.writeInt(fastTime()); out.writeInt(0); } public void readFromGateway(@NotNull ExcerptCommon in) { excerptId = in.index(); writeTimestampMillis = in.readLong(); inWriteTimestamp = in.readInt(); inReadTimestamp = in.readInt(); if (inReadTimestamp == 0 && targetReader) in.writeInt(in.position() - 4, inReadTimestamp = fastTime()); } public void writeForEngine(@NotNull RandomDataOutput out) { out.writeInt(sourceId); out.writeLong(excerptId); out.writeLong(writeTimestampMillis); out.writeInt(inWriteTimestamp); out.writeInt(inReadTimestamp); out.writeInt(fastTime()); out.writeInt(0); } public void readFromEngine(@NotNull ExcerptCommon in, int sourceId) { this.sourceId = in.readInt(); excerptId = in.readLong(); targetReader = sourceId == this.sourceId; writeTimestampMillis = in.readLong(); inWriteTimestamp = in.readInt(); inReadTimestamp = in.readInt(); outWriteTimestamp = in.readInt(); outReadTimestamp = in.readInt(); if (outReadTimestamp == 0 && targetReader) in.writeInt(in.position() - 4, outReadTimestamp = fastTime()); } } Pe2GwEvents.java000077500000000000000000000014561226752375100334320ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle-demo/src/main/java/vanilla/java/processingengine/api/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package vanilla.java.processingengine.api; /** * Processing Engine to Gateway events. * * @author peter.lawrey */ public interface Pe2GwEvents { void report(MetaData metaData, SmallReport smallReport); } Pe2GwReader.java000077500000000000000000000040671226752375100333710ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle-demo/src/main/java/vanilla/java/processingengine/api/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package vanilla.java.processingengine.api; import net.openhft.chronicle.ExcerptTailer; /** * @author peter.lawrey */ public class Pe2GwReader { private final int sourceId; private final ExcerptTailer excerpt; private final Pe2GwEvents gwEvents; private final MetaData metaData = new MetaData(true); private final SmallReport report = new SmallReport(); public Pe2GwReader(int sourceId, ExcerptTailer excerpt, Pe2GwEvents gwEvents) { this.sourceId = sourceId; this.excerpt = excerpt; this.gwEvents = gwEvents; } public boolean readOne() { if (!excerpt.nextIndex()) { // System.out.println("r- " + excerpt.index()); return false; } long pos = excerpt.position(); // System.out.println("Reading " + excerpt.index()); MessageType mt = excerpt.readEnum(MessageType.class); if (mt == null) { // rewind and read again. excerpt.position(pos); System.err.println("Unknown message type " + excerpt.readUTFΔ()); return true; } switch (mt) { case report: { metaData.readFromEngine(excerpt, sourceId); report.readMarshallable(excerpt); gwEvents.report(metaData, report); break; } default: System.err.println("Unknown message type " + mt); break; } return true; } } Pe2GwWriter.java000077500000000000000000000023451226752375100334400ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle-demo/src/main/java/vanilla/java/processingengine/api/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package vanilla.java.processingengine.api; import net.openhft.chronicle.ExcerptAppender; import org.jetbrains.annotations.NotNull; /** * @author peter.lawrey */ public class Pe2GwWriter implements Pe2GwEvents { private final ExcerptAppender excerpt; public Pe2GwWriter(ExcerptAppender excerpt) { this.excerpt = excerpt; } @Override public void report(@NotNull MetaData metaData, @NotNull SmallReport smallReport) { excerpt.startExcerpt(); excerpt.writeEnum(MessageType.report); metaData.writeForEngine(excerpt); smallReport.writeMarshallable(excerpt); excerpt.finish(); } } ReportStatus.java000077500000000000000000000013231226752375100337710ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle-demo/src/main/java/vanilla/java/processingengine/api/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package vanilla.java.processingengine.api; /** * @author peter.lawrey */ public enum ReportStatus { OK, REJECTED } Chronicle-Queue-2.0.3/chronicle-demo/src/main/java/vanilla/java/processingengine/api/Side.java000077500000000000000000000013101226752375100322510ustar00rootroot00000000000000/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package vanilla.java.processingengine.api; /** * @author peter.lawrey */ public enum Side { BUY, SELL } SmallCommand.java000077500000000000000000000033221226752375100336620ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle-demo/src/main/java/vanilla/java/processingengine/api/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package vanilla.java.processingengine.api; import net.openhft.chronicle.ExcerptCommon; import net.openhft.chronicle.ExcerptMarshallable; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; /** * @author peter.lawrey */ public class SmallCommand implements ExcerptMarshallable { public final StringBuilder clientOrderId = new StringBuilder(); @Nullable public String instrument; public double price; public int quantity; @Nullable public Side side; @Override public void readMarshallable(@NotNull ExcerptCommon in) throws IllegalStateException { // changes often. in.readUTFΔ(clientOrderId); // cachable. instrument = in.readEnum(String.class); price = in.readCompactDouble(); quantity = (int) in.readStopBit(); side = in.readEnum(Side.class); } @Override public void writeMarshallable(@NotNull ExcerptCommon out) { out.writeUTFΔ(clientOrderId); out.writeEnum(instrument); out.writeCompactDouble(price); out.writeStopBit(quantity); out.writeEnum(side); } } SmallReport.java000077500000000000000000000040201226752375100335530ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle-demo/src/main/java/vanilla/java/processingengine/api/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package vanilla.java.processingengine.api; import net.openhft.chronicle.ExcerptCommon; import net.openhft.chronicle.ExcerptMarshallable; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; /** * @author peter.lawrey */ public class SmallReport implements ExcerptMarshallable { public CharSequence clientOrderId = new StringBuilder(); @Nullable public ReportStatus status; public CharSequence rejectedReason = new StringBuilder(); public void orderOkay(CharSequence clientOrderId) { this.clientOrderId = clientOrderId; status = ReportStatus.OK; rejectedReason = ""; } public void orderRejected(CharSequence clientOrderId, CharSequence rejectedReason) { this.clientOrderId = clientOrderId; status = ReportStatus.REJECTED; this.rejectedReason = rejectedReason; } @Override public void readMarshallable(@NotNull ExcerptCommon in) throws IllegalStateException { StringBuilder clientOrderId = (StringBuilder) this.clientOrderId; in.readUTFΔ(clientOrderId); status = in.readEnum(ReportStatus.class); StringBuilder rejectedReason = (StringBuilder) this.rejectedReason; in.readUTFΔ(rejectedReason); } @Override public void writeMarshallable(@NotNull ExcerptCommon out) { out.writeUTFΔ(clientOrderId); out.writeEnum(status); out.writeUTFΔ(rejectedReason); } } Chronicle-Queue-2.0.3/chronicle-demo/src/main/java/vanilla/java/processingengine/testing/000077500000000000000000000000001226752375100314305ustar00rootroot00000000000000Histogram.java000077500000000000000000000033011226752375100341510ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle-demo/src/main/java/vanilla/java/processingengine/testing/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package vanilla.java.processingengine.testing; import org.jetbrains.annotations.NotNull; /** * @author peter.lawrey */ public class Histogram { @NotNull private final int[] count; private final int factor; private int underflow; private int overflow; private long total = 0; public Histogram(int counts, int factor) { this.count = new int[counts + 1]; this.factor = factor; } public void sample(long n) { long bucket = (n + factor / 2) / factor; if (bucket < 0) underflow++; else if (bucket >= count.length) overflow++; else count[((int) bucket)]++; total++; } public int percentile(double d) { long num = total - (long) (d * total); if (num <= overflow) return Integer.MAX_VALUE; for (int i = count.length - 1; i > 0; i--) if ((num -= count[i]) <= 0) return i * factor; return 0; } public long underflow() { return underflow; } public long overflow() { return overflow; } } Chronicle-Queue-2.0.3/chronicle-sandbox/000077500000000000000000000000001226752375100201205ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle-sandbox/VanillaChronicle.md000066400000000000000000000062701226752375100236640ustar00rootroot00000000000000Vanilla Chronicle ============ Vanilla Chronicle is a designed for more features rather than just speed. It it is fast enough, it should be easier to use/manage. It supports - rolling files on a daily, weekly or hourly basis. - concurrent writers on the same machine. - concurrent readers on the same machine or across multiple machines via TCP replication. - zero copy serialization and deserialization. - millions of writes/reads per second on commodity hardware.
(~5 M messages / second for 96 byte messages on a i7-4500 laptop) - synchronous persistence as required. (commit to disk before continuing) - reader on source can wait for replication. i.e. source reader sees excerpts after replication acknowledge. - data files have more information for rebuilding indexes. - exact length of entries File Format ------------ The directory structure is as follows.
base-directory /
   {cycle-name} /       - The default format is yyyyMMdd
        index-{n}       - multiple index files from 0 .. {n}
        data-{tid}-{m}  - multiple data files for each thread id (matches the process id) from 0 .. {n}
The index file format is an sequence of 8-byte values which consist of a 16-bit {tid} and the offset in bytes of the start of the record. The data file format has a 4-byte length of record. The length is the inverted bits of the 4-byte value. This is used to avoid seeing regular data as a length and detect corruption. The length always starts of a 4-byte boundary. TCP Replication --------------- Each *source* can have any number of down stream *sinks*. With TCP replication this works well up to 10 consumers, above this you may get scalability issues. When a *sink* connects to a *source*, it sends the last entry it had and the source will send entries from there. The source sends a message for; - each entry, existing or new. - when there is a new cycle. - when the source is in sync. - a heartbeat every 2.5 seconds. Concurrent Producer ------------------- Any number of threads can be writing to the Chronicle at the same time provided - you only append OR - you modify records using a lock or CAS operation Example ExcerptAppender appender = chronicle.createAppender(); // for each record appender.startExcept(); // can be called by any number of threads/processes at once appender.writeXxxx( ... ); // write binary appender.append( ... ); // write text appender.finish(); Concurrent Consumers -------------------- Consumers can work on either a Topic basis (the default) or can be applied on a Queue basis (where only one consumer "gets" a message) ExcerptTailer tailer = chronicle.createTailer(); int threadId = AffinitySupport.getThreadId(); // in a busy loop, check there is an excerpt and this is the only consumer. if (tailer.hasNext() && tailer.compareAndSwapInt(0L, 0, threadId)) { tailer.position(4); // skip the lock. // binary format long v = tailer.readXxxx(); String text = tailer.readEnum(String.class); // read a UTF-8 String using a string pool. // text format long x = tailer.parseLong(); double y = tailer.parseDouble(); tailer.finish(); } Chronicle-Queue-2.0.3/chronicle-sandbox/pom.xml000066400000000000000000000022421226752375100214350ustar00rootroot00000000000000 Java-Chronicle net.openhft 2.1-SNAPSHOT 4.0.0 OpenHFT/Java-Chronicle/chronicle-sandbox chronicle-sandbox net.openhft affinity 2.0.1 net.openhft chronicle 2.1-SNAPSHOT junit junit 4.11 test Chronicle-Queue-2.0.3/chronicle-sandbox/src/000077500000000000000000000000001226752375100207075ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle-sandbox/src/main/000077500000000000000000000000001226752375100216335ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle-sandbox/src/main/java/000077500000000000000000000000001226752375100225545ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle-sandbox/src/main/java/net/000077500000000000000000000000001226752375100233425ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle-sandbox/src/main/java/net/openhft/000077500000000000000000000000001226752375100250055ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle-sandbox/src/main/java/net/openhft/chronicle/000077500000000000000000000000001226752375100267535ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle-sandbox/src/main/java/net/openhft/chronicle/sandbox/000077500000000000000000000000001226752375100304115ustar00rootroot00000000000000DailyRollingChronicle.java000077500000000000000000000314211226752375100354210ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle-sandbox/src/main/java/net/openhft/chronicle/sandbox/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.openhft.chronicle.sandbox; import net.openhft.chronicle.*; import net.openhft.chronicle.tools.MasterIndexFile; import net.openhft.lang.Maths; import net.openhft.lang.io.DirectBytes; import net.openhft.lang.io.DirectStore; import net.openhft.lang.io.NativeBytes; import org.jetbrains.annotations.NotNull; import sun.nio.ch.DirectBuffer; import java.io.File; import java.io.IOException; import java.nio.MappedByteBuffer; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Date; import java.util.List; import java.util.concurrent.CopyOnWriteArrayList; /** * @author peter.lawrey */ public class DailyRollingChronicle implements Chronicle { private static final int INDEX_SAMPLE = 8; // only index every 8th entry. private static final int INDEX_SAMPLE_BITS = Maths.intLog2(INDEX_SAMPLE); // private final ScheduledExecutorService worker; private final File file; private final MasterIndexFile master; private final DailyRollingConfig config; private final SimpleDateFormat dateFormat; private final String currentFilename; private final List filesList; private DirectBytes bytes; private DRCExcerptAppender appender; private long lastWrittenIndex = -1; public DailyRollingChronicle(String filename, DailyRollingConfig config) throws IOException { this.config = config; this.file = new File(filename); file.mkdirs(); if (!file.isDirectory()) throw new IOException("Failed to create directory " + file); master = new MasterIndexFile(new File(file, "master")); bytes = new DirectStore(config.getBytesMarshallerFactory(), config.getMaxEntrySize(), false).createSlice(); dateFormat = new SimpleDateFormat(config.getFileFormat()); dateFormat.setTimeZone(config.getTimeZone()); currentFilename = dateFormat.format(new Date()); int index = master.append(currentFilename); filesList = new CopyOnWriteArrayList(); while (filesList.size() < index) filesList.add(null); DRFiles indexFiles = new DRFiles(file, currentFilename); filesList.add(indexFiles); // worker = Executors.newSingleThreadScheduledExecutor(new NamedThreadFactory(file.getName() + "-worker", true)); } @Override public String name() { return file.getName(); } @NotNull @Override public Excerpt createExcerpt() throws IOException { return new DRCExcerpt(); } @NotNull @Override public ExcerptTailer createTailer() throws IOException { return new DRCExcerptTailer(); } @NotNull @Override public ExcerptAppender createAppender() throws IOException { if (appender == null) appender = new DRCExcerptAppender(); return appender; } @Override public long lastWrittenIndex() { if (lastWrittenIndex < 0) findLastIndex(); return lastWrittenIndex; } private void findLastIndex() { int index = filesList.size() - 1; lastWrittenIndex = index * config.getMaxEntriesPerCycle() + filesList.get(index).findTheLastIndex(); } @Override public long size() { return lastWrittenIndex + 1; } public DailyRollingConfig config() { return config; } @Override public void close() throws IOException { master.close(); for (DRFiles drFiles : filesList) { drFiles.close(); } } synchronized DRFiles acquireDRFile(int fileIndex) { DRFiles drFiles = filesList.get(fileIndex); if (drFiles == null) { String filename = master.filenameFor(fileIndex); try { filesList.set(fileIndex, drFiles = new DRFiles(file, filename)); } catch (IOException e) { throw new IllegalStateException(e); } } return drFiles; } class DRFiles { private final MappedFileCache indexMappedCache, dataMappedCache; private final List indexBuffers = new ArrayList(); private final int indexBlockSize, dataBlockSize; public DRFiles(File dir, String filename) throws IOException { indexBlockSize = config.getIndexBlockSize(); dataBlockSize = config.getDataBlockSize(); this.indexMappedCache = new SingleMappedFileCache(new File(dir, filename + ".index"), indexBlockSize); this.dataMappedCache = new SingleMappedFileCache(new File(dir, filename + ".data"), dataBlockSize); findTheLastIndex(); } private long findTheLastIndex() { long lastBlock = (indexMappedCache.size() - 8) / indexBlockSize; MappedByteBuffer imbb = null; for (; lastBlock >= 0; lastBlock--) { imbb = indexMappedCache.acquireBuffer(lastBlock, false); if (imbb.getLong(0) > 0) break; } long size = 0; long dataOffset = 0; if (lastBlock >= 0 && imbb != null) { int lo = -1, hi = indexBlockSize >>> 3; while (lo + 1 < hi) { int mid = (hi + lo) >>> 1; dataOffset = imbb.getLong(mid << 3); if (dataOffset > 0) lo = mid; else hi = mid; } size = (lastBlock * indexBlockSize >>> 3) + lo; } // follow the offsets to find the entries. long dataBlockIndex = dataOffset / dataBlockSize; MappedByteBuffer dmbb = dataMappedCache.acquireBuffer(dataBlockIndex, false); int offset = (int) (dataOffset % dataBlockSize); while (true) { int excerptSize = dmbb.getInt(offset); if (excerptSize == 0) { lastWrittenIndex = size - 1; break; } if (excerptSize > config.getMaxEntrySize()) throw new AssertionError("Over sized entry of " + (dataBlockIndex * dataBlockSize + offset)); if (excerptSize < 0) throw new AssertionError("Negative size !! at " + (dataBlockIndex * dataBlockSize + offset)); // offsets are placed 4-byte word aligned. offset += (excerptSize + 3) & ~3; size++; if ((size & (INDEX_SAMPLE - 1)) == 0) addIndexEntry(size >>> INDEX_SAMPLE_BITS, dataBlockIndex * dataBlockSize + offset); if (offset > dataBlockSize) { dataBlockIndex++; dmbb = dataMappedCache.acquireBuffer(dataBlockIndex, false); offset -= dataBlockSize; } } return lastWrittenIndex; } private void addIndexEntry(long indexPosition, long dataPositon) { // TODO } public void close() { indexMappedCache.close(); dataMappedCache.close(); } } class AbstractDRCExcerpt extends NativeBytes implements ExcerptCommon { protected long index = -1; private MappedByteBuffer dataMBB; public AbstractDRCExcerpt() { super(bytes); } @Override public boolean wasPadding() { throw new UnsupportedOperationException(); } @Override public long index() { throw new UnsupportedOperationException(); } @Override public long lastWrittenIndex() { return DailyRollingChronicle.this.lastWrittenIndex(); } @Override public long size() { return capacity(); } @Override public ExcerptCommon toEnd() { index(lastWrittenIndex()); return this; } @Override public Chronicle chronicle() { throw new UnsupportedOperationException(); } public boolean index(long l) { if (l < 0) return false; long fileIndex = l / config.getMaxEntriesPerCycle(); if (fileIndex >= filesList.size()) return false; DRFiles drFiles = filesList.get((int) fileIndex); if (drFiles == null) drFiles = acquireDRFile((int) fileIndex); long index2 = l % config.getMaxEntriesPerCycle() >> INDEX_SAMPLE_BITS << 3; MappedByteBuffer indexMBB = drFiles.indexMappedCache.acquireBuffer(index2, false); long dataOffset = indexMBB.getLong((int) index2); int dataBlockSize = config.getDataBlockSize(); for (int indexOffset = (int) (l & (INDEX_SAMPLE - 1)); indexOffset > 0; indexOffset--) { MappedByteBuffer dataMBB = drFiles.dataMappedCache.acquireBuffer(dataOffset / dataBlockSize, false); int size = dataMBB.getInt((int) (dataOffset % dataBlockSize)); dataOffset += (size + 3) & ~3; // 4 - byte alignment. } dataMBB = drFiles.dataMappedCache.acquireBuffer(dataOffset / dataBlockSize, false); long offsetInDataBlock = dataOffset % dataBlockSize + 4; startAddr = positionAddr = ((DirectBuffer) dataMBB).address() + offsetInDataBlock; int size = NativeBytes.UNSAFE.getInt(startAddr - 4); assert offsetInDataBlock + size <= dataBlockSize; limitAddr = startAddr + size; return true; } } class DRCExcerpt extends AbstractDRCExcerpt implements Excerpt { @Override public long findMatch(@NotNull ExcerptComparator comparator) { throw new UnsupportedOperationException(); } @Override public void findRange(@NotNull long[] startEnd, @NotNull ExcerptComparator comparator) { throw new UnsupportedOperationException(); } @NotNull @Override public Excerpt toStart() { throw new UnsupportedOperationException(); } @Override public boolean nextIndex() { throw new UnsupportedOperationException(); } @Override public Excerpt toEnd() { super.toEnd(); return this; } } class DRCExcerptTailer extends AbstractDRCExcerpt implements ExcerptTailer { @Override public boolean nextIndex() { startAddr = (limitAddr + 3) & ~3; int size = NativeBytes.UNSAFE.getInt(startAddr); startAddr = positionAddr = startAddr + 4; limitAddr = startAddr + size; index++; return true; } @NotNull @Override public ExcerptTailer toStart() { index = -1; return this; } @Override public ExcerptTailer toEnd() { super.toEnd(); return this; } } class DRCExcerptAppender extends AbstractDRCExcerpt implements ExcerptAppender { private boolean nextSynchronous = config.isSynchronousWriter(); DRCExcerptAppender() { toEnd(); } @Override public void startExcerpt() { startExcerpt(config.getMaxEntrySize()); } @Override public void startExcerpt(long capacity) { startAddr = positionAddr = ((limitAddr + 3) & ~3) + 4; limitAddr = startAddr + capacity; } @Override public void addPaddedEntry() { // Not implemented. throw new UnsupportedOperationException(); } @Override public boolean nextSynchronous() { return nextSynchronous; } @Override public void nextSynchronous(boolean nextSynchronous) { this.nextSynchronous = nextSynchronous; } @Override public void finish() { super.finish(); long size = positionAddr - startAddr; NativeBytes.UNSAFE.putOrderedInt(null, startAddr - 4, (int) size); nextSynchronous = config.isSynchronousWriter(); } @Override public ExcerptAppender toEnd() { super.toEnd(); return this; } } } DailyRollingConfig.java000077500000000000000000000103441226752375100347210ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle-sandbox/src/main/java/net/openhft/chronicle/sandbox/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.openhft.chronicle.sandbox; import net.openhft.lang.io.serialization.BytesMarshallerFactory; import net.openhft.lang.io.serialization.impl.VanillaBytesMarshallerFactory; import java.util.TimeZone; /** * @author peter.lawrey */ public class DailyRollingConfig implements Cloneable { private TimeZone timeZone; // 128,000 trillion, more than 170 years worth private long maxEntriesPerCycle = 1L << 47; // the default maximum entry size private long maxEntrySize = 1024 * 1024; private BytesMarshallerFactory bytesMarshallerFactory; // is synchronous flushing the default private boolean synchronousWriter = false; // should the reader wait for a flush private boolean synchronousReader = false; // file format to use for each rolled file. private String fileFormat = "yyyyMMdd"; // rolling period private int rollingPeriod = 24 * 60 * 60 * 1000; private int indexBlockSize = 1024 * 1024, dataBlockSize = 32 * 1024 * 1024; public DailyRollingConfig() { setTimeZone(TimeZone.getDefault()); } public TimeZone getTimeZone() { return timeZone; } public DailyRollingConfig setTimeZone(TimeZone timeZone) { if (timeZone.observesDaylightTime()) this.timeZone = timeZone; return this; } public long getMaxEntriesPerCycle() { return maxEntriesPerCycle; } public DailyRollingConfig setMaxEntriesPerCycle(long maxEntriesPerCycle) { this.maxEntriesPerCycle = maxEntriesPerCycle; return this; } public long getMaxEntrySize() { return maxEntrySize; } public DailyRollingConfig setMaxEntrySize(long maxEntrySize) { this.maxEntrySize = maxEntrySize; return this; } @Override protected DailyRollingConfig clone() { try { return (DailyRollingConfig) super.clone(); } catch (CloneNotSupportedException e) { throw new AssertionError(e); } } public BytesMarshallerFactory getBytesMarshallerFactory() { return bytesMarshallerFactory == null ? new VanillaBytesMarshallerFactory() : bytesMarshallerFactory; } public DailyRollingConfig setBytesMarshallerFactory(BytesMarshallerFactory bytesMarshallerFactory) { this.bytesMarshallerFactory = bytesMarshallerFactory; return this; } public boolean isSynchronousWriter() { return synchronousWriter; } public DailyRollingConfig setSynchronousWriter(boolean synchronousWriter) { this.synchronousWriter = synchronousWriter; return this; } public boolean isSynchronousReader() { return synchronousReader; } public DailyRollingConfig setSynchronousReader(boolean synchronousReader) { this.synchronousReader = synchronousReader; return this; } public String getFileFormat() { return fileFormat; } public DailyRollingConfig setFileFormat(String fileFormat) { this.fileFormat = fileFormat; return this; } public int getIndexBlockSize() { return indexBlockSize; } public DailyRollingConfig setIndexBlockSize(int indexBlockSize) { this.indexBlockSize = indexBlockSize; return this; } public int getDataBlockSize() { return dataBlockSize; } public DailyRollingConfig setDataBlockSize(int dataBlockSize) { this.dataBlockSize = dataBlockSize; return this; } public int getRollingPeriod() { return rollingPeriod; } public DailyRollingConfig setRollingPeriod(int rollingPeriod) { this.rollingPeriod = rollingPeriod; return this; } } Chronicle-Queue-2.0.3/chronicle-sandbox/src/main/java/net/openhft/chronicle/sandbox/DateCache.java000066400000000000000000000037541226752375100330660ustar00rootroot00000000000000/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.openhft.chronicle.sandbox; import net.openhft.lang.Maths; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Date; import java.util.TimeZone; public class DateCache { static final int SIZE = 32; static final TimeZone GMT = TimeZone.getTimeZone("GMT"); final SimpleDateFormat format; final DateValue[] values = new DateValue[SIZE]; final int cycleLength; public DateCache(String formatStr, int cycleLength) { this.cycleLength = cycleLength; format = new SimpleDateFormat(formatStr); format.setTimeZone(GMT); } public String formatFor(int cycle) { long millis = (long) cycle * cycleLength; int hash = Maths.hash(millis) & (SIZE - 1); DateValue dv = values[hash]; if (dv == null || dv.millis != millis) { synchronized (format) { String text = format.format(new Date(millis)); values[hash] = new DateValue(millis, text); return text; } } return dv.text; } public long parseCount(String name) throws ParseException { return format.parse(name).getTime() / cycleLength; } static class DateValue { final long millis; final String text; DateValue(long millis, String text) { this.millis = millis; this.text = text; } } } Chronicle-Queue-2.0.3/chronicle-sandbox/src/main/java/net/openhft/chronicle/sandbox/Journal.java000066400000000000000000000022561226752375100326730ustar00rootroot00000000000000/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.openhft.chronicle.sandbox; import net.openhft.chronicle.ExcerptAppender; import net.openhft.chronicle.ExcerptTailer; import org.jetbrains.annotations.NotNull; import java.io.IOException; /** * A Journal is a simplified Chronicle. It is bounded in size and number, does not support random access and is designed for fast writes. */ public interface Journal { @NotNull String name(); @NotNull ExcerptTailer createTailer() throws IOException; @NotNull ExcerptAppender createAppender() throws IOException; // the last written entry. long size(); } JournalConfig.java000066400000000000000000000031051226752375100337340ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle-sandbox/src/main/java/net/openhft/chronicle/sandbox/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.openhft.chronicle.sandbox; public class JournalConfig implements Cloneable { private int indexCount = 4 * 1024; private int dataShift = 8; private int dataAllocations = 1 << 20; public JournalConfig indexCount(int indexCount) { this.indexCount = indexCount; return this; } public int indexCount() { return indexCount; } public JournalConfig dataShift(int dataShift) { this.dataShift = dataShift; return this; } public int dataShift() { return dataShift; } public JournalConfig dataAllocations(int dataAllocations) { this.dataAllocations = dataAllocations; return this; } public int dataAllocations() { return dataAllocations; } @Override protected JournalConfig clone() { try { return (JournalConfig) super.clone(); } catch (CloneNotSupportedException e) { throw new AssertionError(e); } } } Chronicle-Queue-2.0.3/chronicle-sandbox/src/main/java/net/openhft/chronicle/sandbox/JournalFile.java000066400000000000000000000031311226752375100334640ustar00rootroot00000000000000/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.openhft.chronicle.sandbox; public class JournalFile { // format // long magic number static final byte[] MAGIC_NUMBER = "JOURNAL1".getBytes(); static final int MAGIC_OFFSET = 0; // int allowedReaderWriters static final int ALLOWED_READER_WRITERS = MAGIC_OFFSET + 8; // int last reader writer static final int LAST_READER_WRITER = ALLOWED_READER_WRITERS + 4; // int record size static final int RECORD_SIZE = LAST_READER_WRITER + 4; // int record number static final int RECORD_NUMBER = RECORD_SIZE + 4; // int nextAllocate static final int NEXT_ALLOCATE = RECORD_NUMBER + 4; // -- new cache line for each writer // int state static final int STATE = 0; // int writingTo static final int WRITING_TO = STATE + 4; // int processId static final int PROCESS_ID = WRITING_TO + 4; // -- new cache line for each reader // int state // int readingFrom static final int READING_FROM = STATE + 4; // int processId } JournalRecord.java000066400000000000000000000046361226752375100337570ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle-sandbox/src/main/java/net/openhft/chronicle/sandbox/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.openhft.chronicle.sandbox; import net.openhft.lang.io.NativeBytes; import java.util.logging.Logger; public class JournalRecord extends NativeBytes { static final Logger LOGGER = Logger.getLogger(JournalRecord.class.getName()); // int read write lock static final int READ_WRITE_LOCK = -8; // int data size static final int DATA_SIZE = -4; static int HEADER_SIZE = 4 + 4; final int size; public JournalRecord(int size) { super(NativeBytes.NO_PAGE, NativeBytes.NO_PAGE, 0); this.size = size; } public JournalRecord address(long address) { positionAddr = startAddr = address + HEADER_SIZE; limitAddr = address + size; return this; } public boolean writeLock(long timeOutMS) { // first case is the fast path. return compareAndSwapInt(READ_WRITE_LOCK, 0, -1) || writeLock0(timeOutMS); } private boolean writeLock0(long timeOutMS) { // use a decrementing timer as this is more robust to jumps in time e.g. debugging, process stop, hibernation. long last = System.currentTimeMillis(); do { if (readInt(READ_WRITE_LOCK) == -1) return false; // another thread has this lock. if (compareAndSwapInt(READ_WRITE_LOCK, 0, -1)) return true; long now = System.currentTimeMillis(); if (now != last) timeOutMS--; last = now; } while (timeOutMS > 0); LOGGER.warning("Grabbing write lock. count=" + readInt(READ_WRITE_LOCK)); writeOrderedInt(READ_WRITE_LOCK, -1); return true; } public void writeUnlock(int dataSize) { assert dataSize <= size; writeInt(dataSize); compareAndSwapInt(READ_WRITE_LOCK, -1, 0); } public void readLock() { } } PrefetchingMappedFileCache.java000066400000000000000000000164011226752375100363100ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle-sandbox/src/main/java/net/openhft/chronicle/sandbox/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.openhft.chronicle.sandbox; import net.openhft.chronicle.MapUtils; import net.openhft.chronicle.MappedFileCache; import net.openhft.lang.thread.NamedThreadFactory; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import java.io.FileNotFoundException; import java.io.IOException; import java.io.RandomAccessFile; import java.nio.MappedByteBuffer; import java.nio.channels.FileChannel; import java.util.ArrayList; import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.atomic.AtomicLong; /** * User: peter.lawrey Date: 17/08/13 Time: 14:58 */ public class PrefetchingMappedFileCache implements MappedFileCache { public static final AtomicLong totalWait = new AtomicLong(); static final ExecutorService PREFETCHER = Executors.newSingleThreadExecutor(new NamedThreadFactory("mmap-prefetch", true)); @Nullable private static final IndexedMBB NULL_IMBB = new IndexedMBB(Long.MIN_VALUE, null, -1); final String basePath; final FileChannel fileChannel; final int blockSize; long maxIndex = Long.MIN_VALUE; long lastIndex = Long.MIN_VALUE; @Nullable MappedByteBuffer lastMBB = null; @Nullable List allBuffers = null; @NotNull volatile IndexedMBB imbb = NULL_IMBB; public PrefetchingMappedFileCache(String basePath, int blockSize) throws FileNotFoundException { this.basePath = basePath; this.blockSize = blockSize; fileChannel = new RandomAccessFile(basePath, "rw").getChannel(); } public void excerptUsed() { if (allBuffers == null) { allBuffers = new ArrayList(1000); addBuffer(lastIndex, lastMBB); } } private void addBuffer(long index2, MappedByteBuffer mbb) { if (index2 < 0) return; if (index2 >= Integer.MAX_VALUE) throw new AssertionError(); while (allBuffers.size() <= index2) allBuffers.add(null); allBuffers.set((int) index2, mbb); } @NotNull @Override public MappedByteBuffer acquireBuffer(long index, boolean prefetch) { if (allBuffers == null) { if (index == lastIndex) { if (prefetch && index > maxIndex) { prefetch(index); } assert lastMBB != null; return lastMBB; } } else { if (index < allBuffers.size()) { final MappedByteBuffer mbb = allBuffers.get((int) index); if (mbb != null) return mbb; } } // System.out.println(index); // TreeMap timeMap = new TreeMap(); long start = System.nanoTime(); // timeMap.put(start / 1024, "start"); MappedByteBuffer mappedByteBuffer; IndexedMBB indexedMBB = imbb; long index0 = indexedMBB.index; boolean prefetched = index0 == index; try { if (prefetched) { // long waiting = System.nanoTime(); MappedByteBuffer buffer1; while ((buffer1 = indexedMBB.buffer) == null) { Throwable thrown1 = indexedMBB.thrown; if (thrown1 != null) { throw new IllegalStateException(thrown1); } } // timeMap.put(waiting / 1024, "waiting"); // timeMap.put(indexedMBB.created / 1024, "created"); // timeMap.put(indexedMBB.started / 1024, "startMap"); // timeMap.put(indexedMBB.finished / 1024, "finishMap"); mappedByteBuffer = buffer1; } else mappedByteBuffer = MapUtils.getMap(fileChannel, index * blockSize, blockSize); } catch (Exception e) { throw new IllegalStateException(e); } // timeMap.put(System.nanoTime() / 1024, "got"); if (allBuffers == null) { lastIndex = index; lastMBB = mappedByteBuffer; } else { addBuffer(index, mappedByteBuffer); } boolean ascending = index > maxIndex; if (prefetch && ascending) { prefetch(index); } long time = (System.nanoTime() - start); // timeMap.put(end / 1024, "end"); if (index > 0) totalWait.addAndGet(time); // if (prefetched) { // System.out.println(indexedMBB.report()); // } // System.out.println("Took " + time / 1000 + " us to obtain a data chunk, prefetched: " + prefetched + " index0: " + index0 + " index: " + index); /* if (time > 50e3) { long first = timeMap.firstKey(); String sep = ""; for (Map.Entry entry : timeMap.entrySet()) { System.out.print(sep); System.out.print(entry.getValue()); System.out.print(": "); System.out.print(entry.getKey() - first); sep = ", "; } System.out.println(); } */ return mappedByteBuffer; } private void prefetch(long index) { IndexedMBB imbb2 = new IndexedMBB(index + 1, fileChannel, blockSize); this.imbb = imbb2; PREFETCHER.submit(imbb2); maxIndex = index; } @Override public long size() { try { return fileChannel.size(); } catch (IOException e) { return 0; } } @Override public void close() { try { fileChannel.close(); } catch (IOException ignored) { } } static class IndexedMBB implements Runnable { volatile long created, started, finished; long index; volatile MappedByteBuffer buffer; volatile Throwable thrown; private FileChannel fileChannel; private int blockSize; public IndexedMBB(long index, FileChannel fileChannel, int blockSize) { created = System.nanoTime(); this.index = index; this.fileChannel = fileChannel; this.blockSize = blockSize; } @Override public void run() { try { started = System.nanoTime(); buffer = MapUtils.getMap(fileChannel, index * blockSize, blockSize); finished = System.nanoTime(); } catch (Throwable t) { thrown = t; } } @NotNull public String report() { return "started: " + (started - created) / 1000 + ", finished: " + (finished - started) / 1000 + ", pick up: " + (System.nanoTime() - finished) / 1000; } } } RollingChronicle.java000077500000000000000000000147611226752375100344460ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle-sandbox/src/main/java/net/openhft/chronicle/sandbox/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.openhft.chronicle.sandbox; import net.openhft.chronicle.tools.WrappedExcerpt; import org.jetbrains.annotations.NotNull; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; /** * User: peter.lawrey * Date: 26/09/13 * Time: 17:19 */ public class RollingChronicle implements Chronicle { @NotNull private final String basePath; @NotNull private final ChronicleConfig config; @NotNull private final SingleMappedFileCache masterFileCache; @NotNull private final ByteBuffer masterMBB; private int nextIndex; private IndexedChronicleCache chronicleCache; private long lastWriitenIndex; public RollingChronicle(@NotNull String basePath, @NotNull ChronicleConfig config) throws FileNotFoundException { this.basePath = basePath; this.config = config; nextIndex = 0; new File(basePath).mkdirs(); masterFileCache = new SingleMappedFileCache(basePath + "/master", config.indexFileCapacity() * 4); masterMBB = masterFileCache.acquireBuffer(0, false).order(ByteOrder.nativeOrder()); findLastIndex(); rollNewIndexFileData(); } private void findLastIndex() { int indexFileExcerpts = config().indexFileExcerpts(); for (int i = 0; i < masterMBB.capacity() - 3; i += 4) { int used = masterMBB.getInt(i); if (used < indexFileExcerpts) { nextIndex = i / 4; lastWriitenIndex = (i / 4) * indexFileExcerpts + used - 1; return; } } throw new IllegalStateException("The master file has been exhausted."); } private void rollNewIndexFileData() throws FileNotFoundException { chronicleCache = new IndexedChronicleCache(basePath); } @Override public String name() { return basePath; } @NotNull @Override public Excerpt createExcerpt() throws IOException { return new RollingExcerpt(Type.Excerpt); } @NotNull @Override public ExcerptTailer createTailer() throws IOException { return new RollingExcerpt(Type.Tailer); } @NotNull @Override public ExcerptAppender createAppender() throws IOException { return new RollingExcerpt(Type.Appender); } @Override public long lastWrittenIndex() { return lastWriitenIndex; } @Override public long size() { return lastWrittenIndex() + 1; } public ChronicleConfig config() { return config; } @Override public void close() throws IOException { chronicleCache.close(); masterFileCache.close(); } enum Type { Appender, Tailer, Excerpt } class RollingExcerpt extends WrappedExcerpt { private final Type type; private final int indexFileExcerpts = config().indexFileExcerpts(); private long chronicleIndexBase = Long.MIN_VALUE; private IndexedChronicle chronicle; public RollingExcerpt(Type type) { super(null); this.type = type; } @Override public long size() { return RollingChronicle.this.size(); } @Override public boolean nextIndex() { return checkNextChronicle(1); } @Override public Chronicle chronicle() { return RollingChronicle.this; } @Override public boolean index(long index) throws IndexOutOfBoundsException { try { int chronicleIndex0 = (int) (index / indexFileExcerpts); int chronicleIndex1 = (int) (index % indexFileExcerpts); long newBase = (long) chronicleIndex0 * indexFileExcerpts; if (newBase != chronicleIndexBase) { chronicleIndexBase = newBase; chronicle = chronicleCache.acquireChronicle(chronicleIndex0); nextIndex = chronicleIndex0; switch (type) { case Tailer: setExcerpt(chronicle.createTailer()); break; case Excerpt: setExcerpt(chronicle.createExcerpt()); break; case Appender: setExcerpt(chronicle.createAppender()); return true; } } return (type == Type.Appender) || super.index(chronicleIndex1); } catch (IOException e) { throw new AssertionError(e); } } @Override public void startExcerpt() { startExcerpt(config().messageCapacity()); } @Override public void startExcerpt(long capacity) { index(RollingChronicle.this.size()); super.startExcerpt(capacity); } private boolean checkNextChronicle(int n) { if (chronicle == null) { return index(0); } else if (super.index() + n >= indexFileExcerpts) { boolean ret = index(index() + n); nextIndex = (int) (chronicleIndexBase / indexFileExcerpts); return ret; } else if (n > 0) { return super.nextIndex(); } return true; } @Override public void addPaddedEntry() { super.addPaddedEntry(); } @Override public long index() { return chronicleIndexBase + super.index(); } @Override public void finish() { super.finish(); masterMBB.putInt(nextIndex << 2, (int) chronicle.size()); lastWriitenIndex++; } @Override public boolean wasPadding() { return super.wasPadding(); } } } SingleMappedFileCache.java000077500000000000000000000054101226752375100352740ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle-sandbox/src/main/java/net/openhft/chronicle/sandbox/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.openhft.chronicle.sandbox; import net.openhft.chronicle.MapUtils; import net.openhft.chronicle.MappedFileCache; import org.jetbrains.annotations.Nullable; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.io.RandomAccessFile; import java.nio.MappedByteBuffer; import java.nio.channels.FileChannel; import java.util.concurrent.atomic.AtomicLong; /** * User: peter.lawrey Date: 17/08/13 Time: 14:58 */ public class SingleMappedFileCache implements MappedFileCache { public static final AtomicLong totalWait = new AtomicLong(); final FileChannel fileChannel; final int blockSize; long lastIndex = Long.MIN_VALUE; @Nullable MappedByteBuffer lastMBB = null; public SingleMappedFileCache(String basePath, int blockSize) throws FileNotFoundException { this(new File(basePath), blockSize); } public SingleMappedFileCache(File basePath, int blockSize) throws FileNotFoundException { this.blockSize = blockSize; fileChannel = new RandomAccessFile(basePath, "rw").getChannel(); } @Override public void excerptUsed() { // ignored. } @Nullable @Override public MappedByteBuffer acquireBuffer(long index, boolean prefetch) { if (index == lastIndex) return lastMBB; long start = System.nanoTime(); MappedByteBuffer mappedByteBuffer; try { mappedByteBuffer = MapUtils.getMap(fileChannel, index * blockSize, blockSize); } catch (IOException e) { throw new IllegalStateException(e); } lastIndex = index; lastMBB = mappedByteBuffer; long time = (System.nanoTime() - start); if (index > 0) totalWait.addAndGet(time); // System.out.println("Took " + time + " us to obtain a data chunk"); return mappedByteBuffer; } @Override public long size() { try { return fileChannel.size(); } catch (IOException e) { return 0; } } @Override public void close() { try { fileChannel.close(); } catch (IOException ignored) { } } } VanillaChronicle.java000066400000000000000000000440761226752375100344250ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle-sandbox/src/main/java/net/openhft/chronicle/sandbox/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.openhft.chronicle.sandbox; import net.openhft.affinity.AffinitySupport; import net.openhft.chronicle.*; import net.openhft.lang.Maths; import net.openhft.lang.io.DirectBytes; import net.openhft.lang.io.DirectStore; import net.openhft.lang.io.IOTools; import net.openhft.lang.io.NativeBytes; import net.openhft.lang.io.serialization.BytesMarshallerFactory; import net.openhft.lang.io.serialization.impl.VanillaBytesMarshallerFactory; import org.jetbrains.annotations.NotNull; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.lang.ref.WeakReference; /** * Created by peter */ public class VanillaChronicle implements Chronicle { private final String name; private final String basePath; private final VanillaChronicleConfig config; private final ThreadLocal> marshallersCache = new ThreadLocal>(); private final ThreadLocal> tailerCache = new ThreadLocal>(); private final ThreadLocal> appenderCache = new ThreadLocal>(); private final DirectBytes NO_BYTES = DirectStore.allocateLazy(4096).createSlice(); private final VanillaIndexCache indexCache; private final VanillaDataCache dataCache; private final int indexBlockSizeBits, indexBlockSizeMask; private final int indexBlockLongsBits, indexBlockLongsMask; private final int dataBlockSizeBits, dataBlockSizeMask; private final int entriesForCycleBits; private final long entriesForCycleMask; // private volatile int cycle; private volatile long lastWrittenIndex; public VanillaChronicle(String basePath) { this(basePath, VanillaChronicleConfig.DEFAULT); } public VanillaChronicle(String basePath, VanillaChronicleConfig config) { this.basePath = basePath; this.config = config; name = new File(basePath).getName(); DateCache dateCache = new DateCache(config.cycleFormat(), config.cycleLength()); indexBlockSizeBits = Maths.intLog2(config.indexBlockSize()); indexBlockSizeMask = -1 >>> -indexBlockSizeBits; indexCache = new VanillaIndexCache(basePath, indexBlockSizeBits, dateCache); indexBlockLongsBits = indexBlockSizeBits - 3; indexBlockLongsMask = indexBlockSizeMask >>> 3; dataBlockSizeBits = Maths.intLog2(config.dataBlockSize()); dataBlockSizeMask = -1 >>> -dataBlockSizeBits; dataCache = new VanillaDataCache(basePath, dataBlockSizeBits, dateCache); entriesForCycleBits = Maths.intLog2(config.entriesPerCycle()); entriesForCycleMask = -1L >>> -entriesForCycleBits; // cycle = (int) (System.currentTimeMillis() / config.cycleLength()); } @Override public String name() { return name; } @NotNull @Override public Excerpt createExcerpt() throws IOException { return new VanillaExcerpt(); } protected BytesMarshallerFactory acquireBMF() { WeakReference bmfRef = marshallersCache.get(); BytesMarshallerFactory bmf = null; if (bmfRef != null) bmf = bmfRef.get(); if (bmf == null) { bmf = createBMF(); marshallersCache.set(new WeakReference(bmf)); } return bmf; } protected BytesMarshallerFactory createBMF() { return new VanillaBytesMarshallerFactory(); } @NotNull @Override public ExcerptTailer createTailer() throws IOException { WeakReference ref = tailerCache.get(); ExcerptTailer tailer = null; if (ref != null) tailer = ref.get(); if (tailer == null) { tailer = createTailer0(); tailerCache.set(new WeakReference(tailer)); } return tailer; } private ExcerptTailer createTailer0() { return new VanillaTailer(); } @NotNull @Override public ExcerptAppender createAppender() throws IOException { WeakReference ref = appenderCache.get(); ExcerptAppender appender = null; if (ref != null) appender = ref.get(); if (appender == null) { appender = createAppender0(); appenderCache.set(new WeakReference(appender)); } return appender; } private ExcerptAppender createAppender0() { return new VanillaAppender(); } @Override public long lastWrittenIndex() { return lastWrittenIndex; } @Override public long size() { return lastWrittenIndex + 1; } @Override public void close() { indexCache.close(); dataCache.close(); } public void clear() { indexCache.close(); dataCache.close(); IOTools.deleteDir(basePath); } public void checkCounts(int min, int max) { indexCache.checkCounts(min, max); dataCache.checkCounts(min, max); } abstract class AbstractVanillaExcerpt extends NativeBytes implements ExcerptCommon { protected long index = -1; protected VanillaFile dataFile; public AbstractVanillaExcerpt() { super(acquireBMF(), NO_BYTES.startAddr(), NO_BYTES.startAddr(), NO_BYTES.startAddr()); } @Override public boolean wasPadding() { return false; } @Override public long index() { return index; } @Override public long lastWrittenIndex() { return VanillaChronicle.this.lastWrittenIndex(); } @Override public long size() { return lastWrittenIndex() + 1; } @Override public ExcerptCommon toEnd() { throw new UnsupportedOperationException(); } @Override public Chronicle chronicle() { return VanillaChronicle.this; } public int cycle() { return (int) (System.currentTimeMillis() / config.cycleLength()); } private int lastCycle = Integer.MIN_VALUE, lastDailyCount = Integer.MIN_VALUE, lastThreadId = Integer.MIN_VALUE, lastDataCount = Integer.MIN_VALUE; private VanillaFile lastIndexFile = null, lastDataFile = null; public boolean index(long nextIndex) { try { int cycle = (int) (nextIndex >>> entriesForCycleBits); int dailyCount = (int) ((nextIndex & entriesForCycleMask) >>> indexBlockLongsBits); int dailyOffset = (int) (nextIndex & indexBlockLongsMask); long indexValue; try { if (lastCycle != cycle || lastDailyCount != dailyCount) { if (lastIndexFile != null) { lastIndexFile.decrementUsage(); lastIndexFile = null; } lastIndexFile = indexCache.indexFor(cycle, dailyCount, false); assert lastIndexFile.usage() > 1; lastCycle = cycle; lastDailyCount = dailyCount; if (lastDataFile != null) { lastDataFile.decrementUsage(); lastDataFile = null; } } indexValue = lastIndexFile.bytes().readVolatileLong(dailyOffset << 3); } catch (FileNotFoundException e) { return false; } if (indexValue == 0) { return false; } int threadId = (int) (indexValue >>> 48); long dataOffset0 = indexValue & (-1L >>> -48); int dataCount = (int) (dataOffset0 >>> dataBlockSizeBits); int dataOffset = (int) (dataOffset0 & dataBlockSizeMask); if (lastThreadId != threadId || lastDataCount != dataCount) { if (dataFile != null) { dataFile.decrementUsage(); dataFile = null; } } if (dataFile == null) { dataFile = dataCache.dataFor(cycle, threadId, dataCount, false); lastThreadId = threadId; lastDataCount = dataCount; } NativeBytes bytes = dataFile.bytes(); int len = bytes.readVolatileInt(dataOffset - 4); if (len == 0) return false; int len2 = ~len; // invalid if either the top two bits are set, if ((len2 >>> 30) != 0) throw new IllegalStateException("Corrupted length " + Integer.toHexString(len)); startAddr = positionAddr = bytes.startAddr() + dataOffset; limitAddr = startAddr + ~len; index = nextIndex; finished = false; return true; } catch (IOException ioe) { throw new AssertionError(ioe); } } public boolean nextIndex() { if (index < 0) { toStart(); if (index < 0) return false; index--; } long nextIndex = index + 1; while (true) { boolean found = index(nextIndex); if (found) return found; int cycle = (int) (nextIndex / config.entriesPerCycle()); if (cycle >= cycle()) return false; nextIndex = (cycle + 1) * config.entriesPerCycle(); } } @NotNull public ExcerptCommon toStart() { long indexCount = indexCache.firstIndex(); if (indexCount >= 0) { index = indexCount * config.entriesPerCycle(); } return this; } } class VanillaExcerpt extends AbstractVanillaExcerpt implements Excerpt { public long findMatch(@NotNull ExcerptComparator comparator) { long lo = 0, hi = lastWrittenIndex(); while (lo <= hi) { long mid = (hi + lo) >>> 1; if (!index(mid)) { if (mid > lo) index(--mid); else break; } int cmp = comparator.compare((Excerpt) this); finish(); if (cmp < 0) lo = mid + 1; else if (cmp > 0) hi = mid - 1; else return mid; // key found } return ~lo; // -(lo + 1) } public void findRange(@NotNull long[] startEnd, @NotNull ExcerptComparator comparator) { // lower search range long lo1 = 0, hi1 = lastWrittenIndex(); // upper search range long lo2 = 0, hi2 = hi1; boolean both = true; // search for the low values. while (lo1 <= hi1) { long mid = (hi1 + lo1) >>> 1; if (!index(mid)) { if (mid > lo1) index(--mid); else break; } int cmp = comparator.compare((Excerpt) this); finish(); if (cmp < 0) { lo1 = mid + 1; if (both) lo2 = lo1; } else if (cmp > 0) { hi1 = mid - 1; if (both) hi2 = hi1; } else { hi1 = mid - 1; if (both) lo2 = mid + 1; both = false; } } // search for the high values. while (lo2 <= hi2) { long mid = (hi2 + lo2) >>> 1; if (!index(mid)) { if (mid > lo2) index(--mid); else break; } int cmp = comparator.compare((Excerpt) this); finish(); if (cmp <= 0) { lo2 = mid + 1; } else { hi2 = mid - 1; } } startEnd[0] = lo1; // inclusive startEnd[1] = lo2; // exclusive } @NotNull @Override public Excerpt toStart() { super.toStart(); return this; } @Override public Excerpt toEnd() { super.toEnd(); return this; } @Override public void finish() { super.finish(); } } class VanillaAppender extends AbstractVanillaExcerpt implements ExcerptAppender { private int lastCycle = Integer.MIN_VALUE, lastThreadId = Integer.MIN_VALUE; private int appenderCycle, appenderThreadId; private boolean nextSynchronous; private VanillaFile lastIndexFile = null; private VanillaFile appenderFile; VanillaAppender() { } @Override public void startExcerpt() { startExcerpt(config.defaultMessageSize()); } @Override public void startExcerpt(long capacity) { try { appenderCycle = cycle(); appenderThreadId = AffinitySupport.getThreadId(); if (appenderCycle != lastCycle || appenderThreadId != lastThreadId) { if (appenderFile != null) { appenderFile.decrementUsage(); appenderFile = null; } appenderFile = dataCache.dataForLast(appenderCycle, appenderThreadId); lastCycle = appenderCycle; lastThreadId = appenderThreadId; if (lastIndexFile != null) { lastIndexFile.decrementUsage(); lastIndexFile = null; } } if (appenderFile.bytes().remaining() < capacity + 4) { dataCache.incrementLastCount(); appenderFile.decrementUsage(); appenderFile = null; appenderFile = dataCache.dataForLast(appenderCycle, appenderThreadId); } startAddr = positionAddr = appenderFile.bytes().positionAddr() + 4; limitAddr = startAddr + capacity; nextSynchronous = config.synchronous(); finished = false; } catch (IOException e) { throw new AssertionError(e); } } @Override public void addPaddedEntry() { throw new UnsupportedOperationException(); } @Override public boolean nextSynchronous() { return nextSynchronous; } @Override public void nextSynchronous(boolean nextSynchronous) { this.nextSynchronous = nextSynchronous; } @Override public void finish() { super.finish(); int length = ~(int) (positionAddr - startAddr); NativeBytes.UNSAFE.putOrderedInt(null, startAddr - 4, length); // position of the start not the end. int offset = (int) (startAddr - appenderFile.baseAddr()); long dataOffset = appenderFile.indexCount() * config.dataBlockSize() + offset; long indexValue = ((long) appenderThreadId << 48) + dataOffset; try { boolean done = false; if (lastIndexFile != null) { NativeBytes bytes = lastIndexFile.bytes(); while (bytes.remaining() >= 8) { if (bytes.compareAndSwapLong(bytes.position(), 0L, indexValue)) { if (nextSynchronous) lastIndexFile.force(); done = true; break; } bytes.position(bytes.position() + 8); } } if (!done) { if (lastIndexFile != null) { lastIndexFile.decrementUsage(); lastIndexFile = null; } lastIndexFile = indexCache.append(appenderCycle, indexValue, nextSynchronous); } } catch (IOException e) { throw new AssertionError(e); } appenderFile.bytes().positionAddr((positionAddr + 3) & ~3L); if (nextSynchronous) appenderFile.force(); // appenderFile.decrementUsage(); } @Override public ExcerptAppender toEnd() { super.toEnd(); return this; } } class VanillaTailer extends AbstractVanillaExcerpt implements ExcerptTailer { @Override public ExcerptTailer toStart() { super.toStart(); return this; } @Override public ExcerptTailer toEnd() { super.toEnd(); return this; } @Override public void finish() { super.finish(); } } } VanillaChronicleConfig.java000066400000000000000000000051711226752375100355440ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle-sandbox/src/main/java/net/openhft/chronicle/sandbox/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.openhft.chronicle.sandbox; public class VanillaChronicleConfig { public static final VanillaChronicleConfig DEFAULT = new VanillaChronicleConfig(); private String cycleFormat = "yyyyMMdd"; private int cycleLength = 24 * 60 * 60 * 1000; // MILLIS_PER_DAY private long indexBlockSize = 16L << 20; // 16 MB private long dataBlockSize = 64L << 20; // 16 MB private int defaultMessageSize = 128 << 10; // 128 KB. private long entriesPerCycle = 1L << 40; // one trillion per day or per hour. private boolean synchronous = false; public VanillaChronicleConfig cycleFormat(String cycleFormat) { this.cycleFormat = cycleFormat; return this; } public String cycleFormat() { return cycleFormat; } public VanillaChronicleConfig cycleFormat(int cycleLength) { this.cycleLength = cycleLength; return this; } public int cycleLength() { return cycleLength; } public VanillaChronicleConfig indexBlockSize(int indexBlockSize) { this.indexBlockSize = indexBlockSize; return this; } public long indexBlockSize() { return indexBlockSize; } public long dataBlockSize() { return dataBlockSize; } public VanillaChronicleConfig dataBlockSize(int dataBlockSize) { this.dataBlockSize = dataBlockSize; return this; } public VanillaChronicleConfig entriesPerCycle(long entriesPerCycle) { this.entriesPerCycle = entriesPerCycle; return this; } public long entriesPerCycle() { return entriesPerCycle; } public VanillaChronicleConfig defaultMessageSize(int defaultMessageSize) { this.defaultMessageSize = defaultMessageSize; return this; } public int defaultMessageSize() { return defaultMessageSize; } public VanillaChronicleConfig synchronous(boolean synchronous) { this.synchronous = synchronous; return this; } public boolean synchronous() { return synchronous; } } VanillaChronicleMonitor.java000066400000000000000000000013431226752375100357630ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle-sandbox/src/main/java/net/openhft/chronicle/sandbox/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.openhft.chronicle.sandbox; public interface VanillaChronicleMonitor { public void onNewCycle(int cycle, int threadId); } VanillaChronicleReader.java000066400000000000000000000150251226752375100355400ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle-sandbox/src/main/java/net/openhft/chronicle/sandbox/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.openhft.chronicle.sandbox; import net.openhft.lang.io.ByteBufferBytes; import java.io.*; import java.nio.ByteOrder; import java.nio.MappedByteBuffer; import java.nio.channels.FileChannel; import java.text.SimpleDateFormat; import java.util.Date; import java.util.LinkedHashMap; import java.util.Map; import java.util.TimeZone; public class VanillaChronicleReader { public static final int MILLI_PER_DAY = 24 * 60 * 60 * 1000; private final String baseDir; private final String format; private final int cycleLength; private final SimpleDateFormat sdf; public VanillaChronicleReader(String baseDir) { this(baseDir, "yyyyMMdd", MILLI_PER_DAY); } public VanillaChronicleReader(String baseDir, String format, int cycleLength) { this.baseDir = baseDir; this.format = format; this.cycleLength = cycleLength; sdf = new SimpleDateFormat(format); sdf.setTimeZone(TimeZone.getTimeZone("GMT")); } public void dump(int cycle, Writer writer) throws IOException { Map fileMap = new LinkedHashMap(128, 0.7f, true) { @Override protected boolean removeEldestEntry(Map.Entry eldest) { boolean remove = size() > 64; if (remove) { try { eldest.getValue().close(); } catch (IOException e) { // ignored } } return remove; } }; PrintWriter pw = writer instanceof PrintWriter ? (PrintWriter) writer : new PrintWriter(writer, true); File dir = new File(baseDir + "/" + sdf.format(new Date((long) cycle * cycleLength))); if (!dir.exists()) { pw.println("Directory " + dir + " does not exist."); return; } File[] files = dir.listFiles(); int dataSize = -1; if (files != null) { for (File file : files) { if (file.getName().startsWith("data-")) { long size = file.length(); if (size > 0 && size != dataSize) { if (dataSize == -1) { dataSize = (int) size; } else { pw.println("ERROR: Multiple data file sizes " + dataSize + " and " + size); return; } } } } } if (dataSize <= 0) { pw.println("ERROR: Unable to determine data file size"); return; } long count = 0; for (int i = 0; i < 10000; i++) { File file2 = new File(dir, "index-" + i); if (!file2.exists()) { pw.println(file2.getName() + " not present."); break; } FileChannel fc = new FileInputStream(file2).getChannel(); ByteBufferBytes bbb = new ByteBufferBytes(fc.map(FileChannel.MapMode.READ_ONLY, 0, fc.size()).order(ByteOrder.nativeOrder())); while (bbb.remaining() > 0) { long index = bbb.readLong(); if (index == 0) { pw.println(count + ": unset"); return; } int threadId = (int) (index >>> 48); long offset = index & (-1L >>> -48); Long fileKey = index / dataSize; int fileOffset = (int) (index % dataSize); StringBuilder mesg = new StringBuilder(); try { ReaderFile readerFile = fileMap.get(fileKey); if (readerFile == null) { fileMap.put(fileKey, readerFile = new ReaderFile(new FileInputStream(new File(dir, "data-" + threadId + "-" + (offset / dataSize))).getChannel())); } MappedByteBuffer mbb = readerFile.mbb; int length = ~mbb.getInt(fileOffset - 4); for (int j = 0; j < length; j++) { byte b = mbb.get(fileOffset + j); if (b < ' ' || b >= 127) { switch (b) { case '\t': mesg.append("\\t"); continue; case '\r': mesg.append("\\r"); continue; case '\n': mesg.append("\\n"); continue; } b = '.'; } mesg.append((char) b); } } catch (IOException e) { mesg.append(e); } pw.println(count + ": " + threadId + " @" + offset + " | " + mesg); count++; } fc.close(); } for (ReaderFile file : fileMap.values()) { file.close(); } fileMap.clear(); pw.flush(); } public static void main(String... args) throws IOException { int cycle = (int) (System.currentTimeMillis() / MILLI_PER_DAY); Writer out = new OutputStreamWriter(System.out); new VanillaChronicleReader(args[0]).dump(cycle, out); System.out.println("Done."); } static class ReaderFile implements Closeable { final FileChannel channel; final MappedByteBuffer mbb; ReaderFile(FileChannel channel) throws IOException { this.channel = channel; mbb = channel.map(FileChannel.MapMode.READ_ONLY, 0, channel.size()); mbb.order(ByteOrder.nativeOrder()); } @Override public void close() throws IOException { channel.close(); } } } VanillaDataCache.java000066400000000000000000000120671226752375100343070ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle-sandbox/src/main/java/net/openhft/chronicle/sandbox/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.openhft.chronicle.sandbox; import net.openhft.lang.io.NativeBytes; import java.io.Closeable; import java.io.File; import java.io.IOException; import java.util.LinkedHashMap; import java.util.Map; public class VanillaDataCache implements Closeable { private static final int MAX_SIZE = 32; private final String basePath; private final DataKey key = new DataKey(); private final int blockBits; private final DateCache dateCache; private final Map dataKeyVanillaFileMap = new LinkedHashMap(MAX_SIZE, 1.0f, true) { @Override protected boolean removeEldestEntry(Map.Entry eldest) { return size() >= MAX_SIZE; } }; public VanillaDataCache(String basePath, int blockBits, DateCache dateCache) { this.basePath = basePath; this.blockBits = blockBits; this.dateCache = dateCache; } public synchronized VanillaFile dataFor(int cycle, int threadId, int dataCount, boolean forWrite) throws IOException { key.cycle = cycle; key.threadId = threadId; key.dataCount = dataCount; VanillaFile vanillaFile = dataKeyVanillaFileMap.get(key); if (vanillaFile == null) { String cycleStr = dateCache.formatFor(cycle); dataKeyVanillaFileMap.put(key.clone(), vanillaFile = new VanillaFile(basePath, cycleStr, "data-" + threadId + "-" + dataCount, dataCount, 1L << blockBits, forWrite)); findEndOfData(vanillaFile); } vanillaFile.incrementUsage(); return vanillaFile; } private void findEndOfData(VanillaFile vanillaFile) { NativeBytes bytes = vanillaFile.bytes(); for (int i = 0, max = 1 << blockBits; i < max; i += 4) { int len = bytes.readInt(bytes.position()); if (len == 0) { return; } int len2 = nextWordAlignment(~len); if (len2 < 0) { throw new IllegalStateException("Corrupted length in " + vanillaFile.file() + " " + Integer.toHexString(len)); } bytes.position(bytes.position() + len2 + 4); } throw new AssertionError(); } public int nextWordAlignment(int len) { return (len + 3) & ~3; } @Override public synchronized void close() { for (VanillaFile vanillaFile : dataKeyVanillaFileMap.values()) { vanillaFile.close(); } dataKeyVanillaFileMap.clear(); } int lastCycle = -1; int lastCount = -1; public VanillaFile dataForLast(int cycle, int threadId) throws IOException { String cycleStr = dateCache.formatFor(cycle); String dataPrefix = basePath + "/" + cycleStr + "/data-" + threadId + "-"; if (lastCycle != cycle) { int maxCount = 0; File[] files = new File(dataPrefix).listFiles(); if (files != null) for (File file : files) { if (file.getName().startsWith(dataPrefix)) { int count = Integer.parseInt(file.getName().substring(dataPrefix.length())); if (maxCount < count) maxCount = count; } } lastCycle = cycle; lastCount = maxCount; } return dataFor(cycle, threadId, lastCount, true); } public void incrementLastCount() { lastCount++; } public synchronized void checkCounts(int min, int max) { for (VanillaFile file : dataKeyVanillaFileMap.values()) { if (file.usage() < min || file.usage() > max) throw new IllegalStateException(file.file() + " has a count of " + file.usage()); } } static class DataKey implements Cloneable { int cycle; int threadId; int dataCount; @Override public int hashCode() { return threadId * 10191 + cycle * 17 + dataCount; } @Override public boolean equals(Object obj) { if (!(obj instanceof DataKey)) return false; DataKey key = (DataKey) obj; return dataCount == key.dataCount && threadId == key.threadId && cycle == key.cycle; } @Override protected DataKey clone() { try { return (DataKey) super.clone(); } catch (CloneNotSupportedException e) { throw new AssertionError(e); } } } } Chronicle-Queue-2.0.3/chronicle-sandbox/src/main/java/net/openhft/chronicle/sandbox/VanillaFile.java000066400000000000000000000067451226752375100334560ustar00rootroot00000000000000/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.openhft.chronicle.sandbox; import net.openhft.lang.io.NativeBytes; import sun.nio.ch.DirectBuffer; import java.io.*; import java.nio.ByteOrder; import java.nio.MappedByteBuffer; import java.nio.channels.FileChannel; import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Level; import java.util.logging.Logger; public class VanillaFile implements Closeable { private final Logger logger; private final File file; private final FileChannel fc; private final MappedByteBuffer map; private final long baseAddr; private final NativeBytes bytes; private final AtomicInteger usage = new AtomicInteger(1); private final int indexCount; private volatile boolean closed = false; public VanillaFile(String basePath, String cycleStr, String name, int indexCount, long size, boolean forAppend) throws IOException { logger = Logger.getLogger(VanillaFile.class.getName() + "." + name); File dir = new File(basePath, cycleStr); this.indexCount = indexCount; if (!dir.isDirectory()) { boolean created = dir.mkdirs(); if (logger.isLoggable(Level.FINE)) logger.fine("Created " + dir + " is " + created); } file = new File(dir, name); if (file.exists()) { if (logger.isLoggable(Level.FINE)) logger.fine("Opening " + file); } else if (forAppend) { if (logger.isLoggable(Level.FINE)) logger.fine("Creating " + file); } else { throw new FileNotFoundException(file.getAbsolutePath()); } fc = new RandomAccessFile(file, "rw").getChannel(); map = fc.map(FileChannel.MapMode.READ_WRITE, 0, size); map.order(ByteOrder.nativeOrder()); baseAddr = ((DirectBuffer) map).address(); bytes = new NativeBytes(null, baseAddr, baseAddr, baseAddr + size); } public File file() { return file; } public NativeBytes bytes() { return bytes; } public void incrementUsage() { usage.incrementAndGet(); } public void decrementUsage() { // new Throwable("dec " + this +" as "+usage).printStackTrace(); if (usage.decrementAndGet() <= 0 && closed) close0(); } public int indexCount() { return indexCount; } public int usage() { return usage.get(); } public long baseAddr() { return baseAddr; } private void close0() { Logger logger = Logger.getLogger(getClass().getName()); if (logger.isLoggable(Level.FINE)) logger.fine("... Closing " + file); try { fc.close(); } catch (IOException e) { throw new AssertionError(e); } } @Override public void close() { closed = true; decrementUsage(); } public void force() { map.force(); } } VanillaIndexCache.java000066400000000000000000000126701226752375100345050ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle-sandbox/src/main/java/net/openhft/chronicle/sandbox/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.openhft.chronicle.sandbox; import net.openhft.lang.io.NativeBytes; import java.io.Closeable; import java.io.File; import java.io.IOException; import java.text.ParseException; import java.util.LinkedHashMap; import java.util.Map; public class VanillaIndexCache implements Closeable { private static final int MAX_SIZE = 32; public static final String INDEX = "index-"; private final String basePath; private final File baseFile; private final IndexKey key = new IndexKey(); private final int blockBits; private final DateCache dateCache; private final Map indexKeyVanillaFileMap = new LinkedHashMap(MAX_SIZE, 1.0f, true) { @Override protected boolean removeEldestEntry(Map.Entry eldest) { boolean removed = size() >= MAX_SIZE; if (removed) { VanillaFile file = eldest.getValue(); file.decrementUsage(); file.close(); } return removed; } }; public VanillaIndexCache(String basePath, int blockBits, DateCache dateCache) { this.basePath = basePath; baseFile = new File(basePath); this.blockBits = blockBits; this.dateCache = dateCache; } public synchronized VanillaFile indexFor(int cycle, int indexCount, boolean forAppend) throws IOException { key.cycle = cycle; key.indexCount = indexCount << blockBits; VanillaFile vanillaFile = indexKeyVanillaFileMap.get(key); if (vanillaFile == null) { String cycleStr = dateCache.formatFor(cycle); indexKeyVanillaFileMap.put(key.clone(), vanillaFile = new VanillaFile(basePath, cycleStr, INDEX + indexCount, indexCount, 1L << blockBits, forAppend)); } vanillaFile.incrementUsage(); // new Throwable("IndexFor " + vanillaFile + " as " + vanillaFile.usage()).printStackTrace(); return vanillaFile; } @Override public synchronized void close() { for (VanillaFile vanillaFile : indexKeyVanillaFileMap.values()) { vanillaFile.close(); } indexKeyVanillaFileMap.clear(); } public int lastIndexFile(int cycle) { int maxIndex = 0; File[] files = new File(dateCache.formatFor(cycle)).listFiles(); if (files != null) for (File file : files) { String name = file.getName(); if (name.startsWith(INDEX)) { int index = Integer.parseInt(name.substring(INDEX.length())); if (maxIndex < index) maxIndex = index; } } return maxIndex; } public VanillaFile append(int cycle, long indexValue, boolean synchronous) throws IOException { for (int indexCount = lastIndexFile(cycle); indexCount < 10000; indexCount++) { VanillaFile file = indexFor(cycle, indexCount, true); NativeBytes bytes = file.bytes(); while (bytes.remaining() >= 8) { if (bytes.compareAndSwapLong(bytes.position(), 0L, indexValue)) { if (synchronous) file.force(); return file; } bytes.position(bytes.position() + 8); } file.decrementUsage(); } throw new AssertionError(); } public long firstIndex() { File[] files = baseFile.listFiles(); if (files == null) return -1; long firstDate = Long.MAX_VALUE; for (File file : files) { try { long date = dateCache.parseCount(file.getName()); if (firstDate > date) firstDate = date; } catch (ParseException ignored) { // ignored } } return firstDate; } public synchronized void checkCounts(int min, int max) { for (VanillaFile file : indexKeyVanillaFileMap.values()) { if (file.usage() < min || file.usage() > max) throw new IllegalStateException(file.file() + " has a count of " + file.usage()); } } static class IndexKey implements Cloneable { int cycle; int indexCount; @Override public int hashCode() { return cycle * 10191 ^ indexCount; } @Override public boolean equals(Object obj) { if (!(obj instanceof IndexKey)) return false; IndexKey key = (IndexKey) obj; return indexCount == key.indexCount && cycle == key.cycle; } @Override protected IndexKey clone() { try { return (IndexKey) super.clone(); } catch (CloneNotSupportedException e) { throw new AssertionError(e); } } } } VanillaJournal.java000066400000000000000000000017471226752375100341270ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle-sandbox/src/main/java/net/openhft/chronicle/sandbox/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.openhft.chronicle.sandbox; /** * File structure * 64 byte header of * long magic number: JOURNAL\\u001 * int header size: 64 * int index count: e.g. 4096 * int data shift: e.g. 0 * int data allocations: e.g. 16 M * padding * rotating indexes * data */ public class VanillaJournal { public VanillaJournal(String fileName, JournalConfig config) { } } Chronicle-Queue-2.0.3/chronicle-sandbox/src/main/java/net/openhft/chronicle/sandbox/tcp/000077500000000000000000000000001226752375100311775ustar00rootroot00000000000000VanillaChronicleSink.java000077500000000000000000000214361226752375100360360ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle-sandbox/src/main/java/net/openhft/chronicle/sandbox/tcp/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.openhft.chronicle.sandbox.tcp; import net.openhft.chronicle.*; import net.openhft.chronicle.sandbox.VanillaChronicle; import net.openhft.chronicle.tcp.TcpUtil; import net.openhft.chronicle.tools.WrappedExcerpt; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import java.io.EOFException; import java.io.IOException; import java.io.StreamCorruptedException; import java.net.InetSocketAddress; import java.net.SocketAddress; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.channels.SocketChannel; import java.util.logging.Level; import java.util.logging.Logger; /** * This listens to a ChronicleSource and copies new entries. This SInk can be any number of excerpt behind the source * and can be restart many times without losing data. *

* Can be used as a component with lower over head than ChronicleSink * * @author peter.lawrey */ public class VanillaChronicleSink implements Chronicle { @NotNull private final VanillaChronicle chronicle; @NotNull private final SocketAddress address; private final ExcerptAppender excerpt; private final Logger logger; private volatile boolean closed = false; public VanillaChronicleSink(@NotNull VanillaChronicle chronicle, String hostname, int port) throws IOException { this.chronicle = chronicle; this.address = new InetSocketAddress(hostname, port); logger = Logger.getLogger(getClass().getName() + '.' + chronicle); excerpt = chronicle.createAppender(); readBuffer = TcpUtil.createBuffer(256 * 1024, ByteOrder.nativeOrder()); } @Override public String name() { return chronicle.name(); } @NotNull @Override public Excerpt createExcerpt() throws IOException { return new SinkExcerpt(chronicle.createExcerpt()); } @NotNull @Override public ExcerptTailer createTailer() throws IOException { return new SinkExcerpt(chronicle.createTailer()); } @NotNull @Override public ExcerptAppender createAppender() throws IOException { throw new UnsupportedOperationException(); } @Override public long lastWrittenIndex() { return chronicle.lastWrittenIndex(); } @Override public long size() { return chronicle.size(); } private class SinkExcerpt extends WrappedExcerpt { @SuppressWarnings("unchecked") public SinkExcerpt(ExcerptCommon excerpt) throws IOException { super(excerpt); } @Override public boolean nextIndex() { return super.nextIndex() || readNext() && super.nextIndex(); } @Override public boolean index(long index) throws IndexOutOfBoundsException { if (super.index(index)) return true; return index >= 0 && readNext() && super.index(index); } } @Nullable private SocketChannel sc = null; private boolean scFirst = true; boolean readNext() { if (sc == null || !sc.isOpen()) { sc = createConnection(); scFirst = true; } return sc != null && readNextExcerpt(sc); } @Nullable private SocketChannel createConnection() { while (!closed) { try { readBuffer.clear(); readBuffer.limit(0); SocketChannel sc = SocketChannel.open(address); sc.socket().setReceiveBufferSize(256 * 1024); logger.info("Connected to " + address); ByteBuffer bb = ByteBuffer.allocate(8); bb.putLong(0, chronicle.lastWrittenIndex()); TcpUtil.writeAllOrEOF(sc, bb); return sc; } catch (IOException e) { if (logger.isLoggable(Level.FINE)) logger.log(Level.FINE, "Failed to connect to " + address + " retrying", e); else if (logger.isLoggable(Level.INFO)) logger.log(Level.INFO, "Failed to connect to " + address + " retrying " + e); } try { Thread.sleep(500); } catch (InterruptedException e) { Thread.currentThread().interrupt(); return null; } } return null; } private final ByteBuffer readBuffer; // minimum size private boolean readNextExcerpt(@NotNull SocketChannel sc) { try { if (closed) return false; /* if (readBuffer.remaining() < (scFirst ? TcpUtil.HEADER_SIZE : 4)) { if (readBuffer.remaining() == 0) readBuffer.clear(); else readBuffer.compact(); int minSize = scFirst ? 8 + 4 + 8 : 4 + 8; while (readBuffer.position() < minSize) { if (sc.read(readBuffer) < 0) { sc.close(); return false; } } readBuffer.flip(); } //*/ System.out.println("rb " + readBuffer); if (scFirst) { long scIndex = readBuffer.getLong(); // System.out.println("ri " + scIndex); if (scIndex != chronicle.size()) throw new StreamCorruptedException("Expected index " + chronicle.size() + " but got " + scIndex); scFirst = false; } int size = readBuffer.getInt(); /* switch (size) { case InProcessChronicleSource.IN_SYNC_LEN: // System.out.println("... received inSync"); return false; case InProcessChronicleSource.PADDED_LEN: // System.out.println("... received padded"); excerpt.startExcerpt(((IndexedChronicle) chronicle).config().dataBlockSize() - 1); return true; default: break; }*/ // System.out.println("size=" + size + " rb " + readBuffer); if (size > 128 << 20 || size < 0) throw new StreamCorruptedException("size was " + size); excerpt.startExcerpt(size); // perform a progressive copy of data. long remaining = size; int limit = readBuffer.limit(); int size2 = (int) Math.min(readBuffer.remaining(), remaining); remaining -= size2; readBuffer.limit(readBuffer.position() + size2); excerpt.write(readBuffer); // reset the limit; readBuffer.limit(limit); // needs more than one read. while (remaining > 0) { // System.out.println("++ read remaining "+remaining +" rb "+readBuffer); readBuffer.clear(); int size3 = (int) Math.min(readBuffer.capacity(), remaining); readBuffer.limit(size3); // System.out.println("... reading"); if (sc.read(readBuffer) < 0) throw new EOFException(); readBuffer.flip(); // System.out.println("r " + ChronicleTools.asString(bb)); remaining -= readBuffer.remaining(); excerpt.write(readBuffer); } excerpt.finish(); // System.out.println(" ri: " + excerpt.index()); } catch (IOException e) { if (logger.isLoggable(Level.FINE)) logger.log(Level.FINE, "Lost connection to " + address + " retrying", e); else if (logger.isLoggable(Level.INFO)) logger.log(Level.INFO, "Lost connection to " + address + " retrying " + e); try { sc.close(); } catch (IOException ignored) { } } return true; } void closeSocket(@Nullable SocketChannel sc) { if (sc != null) try { sc.close(); } catch (IOException e) { logger.warning("Error closing socket " + e); } } @Override public void close() { closed = true; closeSocket(sc); // chronicle.close(); } public ChronicleConfig config() { throw new UnsupportedOperationException(); } } VanillaChronicleSource.java000066400000000000000000000247631226752375100363750ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle-sandbox/src/main/java/net/openhft/chronicle/sandbox/tcp/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.openhft.chronicle.sandbox.tcp; import net.openhft.chronicle.sandbox.VanillaChronicle; import net.openhft.chronicle.tcp.TcpUtil; import net.openhft.chronicle.tools.WrappedExcerpt; import net.openhft.lang.thread.NamedThreadFactory; import org.jetbrains.annotations.NotNull; import java.io.EOFException; import java.io.IOException; import java.net.InetSocketAddress; import java.net.SocketException; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.channels.ServerSocketChannel; import java.nio.channels.SocketChannel; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.logging.Level; import java.util.logging.Logger; public class VanillaChronicleSource implements Chronicle { static final int IN_SYNC_LEN = -128; static final int PADDED_LEN = -127; static final long HEARTBEAT_INTERVAL_MS = 2500; private static final int MAX_MESSAGE = 128; private final VanillaChronicle chronicle; private final ServerSocketChannel server; @NotNull private final String name; @NotNull private final ExecutorService service; private final Logger logger; private final Object notifier = new Object(); private long busyWaitTimeNS = 100 * 1000; private volatile boolean closed = false; private long lastUnpausedNS = 0; public VanillaChronicleSource(@NotNull VanillaChronicle chronicle, int port) throws IOException { this.chronicle = chronicle; server = ServerSocketChannel.open(); server.socket().setReuseAddress(true); server.socket().bind(new InetSocketAddress(port)); name = chronicle.name() + "@" + port; logger = Logger.getLogger(getClass().getName() + "." + name); service = Executors.newCachedThreadPool(new NamedThreadFactory(name, true)); service.execute(new Acceptor()); } private void pauseReset() { lastUnpausedNS = System.nanoTime(); } protected void pause() { if (lastUnpausedNS + busyWaitTimeNS > System.nanoTime()) return; try { synchronized (notifier) { notifier.wait(HEARTBEAT_INTERVAL_MS / 2); } } catch (InterruptedException ie) { logger.warning("Interrupt ignored"); } } void wakeSessionHandlers() { synchronized (notifier) { notifier.notifyAll(); } } @Override public String name() { return chronicle.name(); } @NotNull @Override public Excerpt createExcerpt() throws IOException { return new SourceExcerpt(chronicle.createExcerpt()); } @NotNull @Override public ExcerptTailer createTailer() throws IOException { return new SourceExcerpt(chronicle.createTailer()); } @NotNull @Override public ExcerptAppender createAppender() throws IOException { return new SourceExcerpt(chronicle.createAppender()); } @Override public long lastWrittenIndex() { return chronicle.lastWrittenIndex(); } @Override public long size() { return chronicle.size(); } @Override public void close() { closed = true; try { chronicle.close(); server.close(); } catch (IOException e) { logger.warning("Error closing server port " + e); } } public ChronicleConfig config() { throw new UnsupportedOperationException(); } public int getLocalPort() { return server.socket().getLocalPort(); } public void clear() { chronicle.clear(); } private class Acceptor implements Runnable { @Override public void run() { Thread.currentThread().setName(name + "-acceptor"); try { while (!closed) { SocketChannel socket = server.accept(); service.execute(new Handler(socket)); } } catch (IOException e) { if (!closed) logger.log(Level.SEVERE, "Acceptor dying", e); } finally { service.shutdown(); } } } class Handler implements Runnable { @NotNull private final SocketChannel socket; public Handler(@NotNull SocketChannel socket) throws SocketException { this.socket = socket; socket.socket().setSendBufferSize(256 * 1024); socket.socket().setTcpNoDelay(true); } @Override public void run() { try { long index = readIndex(socket); ExcerptTailer excerpt = chronicle.createTailer(); ByteBuffer bb = TcpUtil.createBuffer(1, ByteOrder.nativeOrder()); // minimum size long sendInSync = 0; boolean first = true; OUTER: while (!closed) { while (!excerpt.index(index)) { long now = System.currentTimeMillis(); if (excerpt.wasPadding()) { if (index >= 0) { bb.clear(); if (first) { bb.putLong(excerpt.index()); first = false; } bb.putInt(PADDED_LEN); bb.flip(); TcpUtil.writeAll(socket, bb); sendInSync = now + HEARTBEAT_INTERVAL_MS; } index++; continue; } // System.out.println("Waiting for " + index); if (sendInSync <= now && !first) { bb.clear(); bb.putInt(IN_SYNC_LEN); bb.flip(); TcpUtil.writeAll(socket, bb); sendInSync = now + HEARTBEAT_INTERVAL_MS; } pause(); if (closed) break OUTER; } pauseReset(); // System.out.println("Writing " + index); final long size = excerpt.capacity(); long remaining; bb.clear(); if (first) { // System.out.println("wi " + index); bb.putLong(excerpt.index()); first = false; remaining = size; // + TcpUtil.HEADER_SIZE; } else { remaining = size + 4; } bb.putInt((int) size); // for large objects send one at a time. if (size > bb.capacity() / 2) { while (remaining > 0) { int size2 = (int) Math.min(remaining, bb.capacity()); bb.limit(size2); excerpt.read(bb); bb.flip(); // System.out.println("w " + ChronicleTools.asString(bb)); remaining -= bb.remaining(); TcpUtil.writeAll(socket, bb); } } else { bb.limit((int) remaining); excerpt.read(bb); int count = 1; while (excerpt.index(index + 1) && count++ < MAX_MESSAGE) { if (excerpt.wasPadding()) { index++; continue; } if (excerpt.remaining() + 4 >= bb.capacity() - bb.position()) break; // if there is free space, copy another one. int size2 = (int) excerpt.capacity(); // System.out.println("W+ "+size); bb.limit(bb.position() + size2 + 4); bb.putInt(size2); excerpt.read(bb); index++; } bb.flip(); // System.out.println("W " + size + " wb " + bb); TcpUtil.writeAll(socket, bb); } if (bb.remaining() > 0) throw new EOFException("Failed to send index=" + index); index++; sendInSync = 0; // if (index % 20000 == 0) // System.out.println(System.currentTimeMillis() + ": wrote " + index); } } catch (Exception e) { if (!closed) { String msg = e.getMessage(); if (msg != null && (msg.contains("reset by peer") || msg.contains("Broken pipe") || msg.contains("was aborted by"))) logger.log(Level.INFO, "Connect " + socket + " closed from the other end " + e); else logger.log(Level.INFO, "Connect " + socket + " died", e); } } } private long readIndex(@NotNull SocketChannel socket) throws IOException { ByteBuffer bb = ByteBuffer.allocate(8); TcpUtil.readFullyOrEOF(socket, bb); return bb.getLong(0); } } class SourceExcerpt extends WrappedExcerpt { public SourceExcerpt(ExcerptCommon excerptCommon) { super(excerptCommon); } @Override public void finish() { super.finish(); wakeSessionHandlers(); // System.out.println("Wrote " + index()); } } } VanillaChronicleTCPCommon.java000066400000000000000000000012611226752375100367200ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle-sandbox/src/main/java/net/openhft/chronicle/sandbox/tcp/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.openhft.chronicle.sandbox.tcp; public class VanillaChronicleTCPCommon { } Chronicle-Queue-2.0.3/chronicle-sandbox/src/test/000077500000000000000000000000001226752375100216665ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle-sandbox/src/test/java/000077500000000000000000000000001226752375100226075ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle-sandbox/src/test/java/net/000077500000000000000000000000001226752375100233755ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle-sandbox/src/test/java/net/openhft/000077500000000000000000000000001226752375100250405ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle-sandbox/src/test/java/net/openhft/chronicle/000077500000000000000000000000001226752375100270065ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle-sandbox/src/test/java/net/openhft/chronicle/sandbox/000077500000000000000000000000001226752375100304445ustar00rootroot00000000000000DailyRollingChronicleTest.java000077500000000000000000000120731226752375100363160ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle-sandbox/src/test/java/net/openhft/chronicle/sandbox/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.openhft.chronicle.sandbox; import net.openhft.chronicle.Excerpt; import net.openhft.chronicle.ExcerptAppender; import net.openhft.chronicle.ExcerptTailer; import net.openhft.chronicle.tools.ChronicleTools; import net.openhft.chronicle.tools.DailingRollingIndexReader; import net.openhft.chronicle.tools.DailingRollingReader; import org.junit.Assert; import org.junit.Ignore; import org.junit.Test; import java.io.*; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.channels.FileChannel; import java.text.SimpleDateFormat; import java.util.Date; import static org.junit.Assert.assertEquals; /** * @author peter.lawrey */ public class DailyRollingChronicleTest { public static final int SIZE = 450; @Test public void testCheckIndex() throws IOException { File tempDir = new File(System.getProperty("java.io.tmpdir") + "/deleteme" + Long.toString(System.nanoTime(), 36)); ChronicleTools.deleteDirOnExit(tempDir.toString()); tempDir.mkdirs(); String yyyyMMdd = new SimpleDateFormat("yyyyMMdd").format(new Date()); FileOutputStream fos = new FileOutputStream(tempDir + "/" + yyyyMMdd + ".data"); ByteBuffer bb = ByteBuffer.allocateDirect(10000).order(ByteOrder.nativeOrder()); int pos = 0; for (int i = 0; i < SIZE; i++) { int size = 16 + i % 5; bb.putInt(pos, size); pos += (size + 3) & ~3; } bb.limit(pos); fos.getChannel().write(bb); fos.close(); DailyRollingConfig config = new DailyRollingConfig(); DailyRollingChronicle chronicle = new DailyRollingChronicle(tempDir.getAbsolutePath(), config); Assert.assertEquals(tempDir.getName() + ": 1\n" + "\t0\t" + yyyyMMdd + "\n", DailingRollingIndexReader.masterToString(tempDir).replace("\r", "")); assertEquals(SIZE, chronicle.size()); chronicle.close(); // check the index is rebuilt. FileInputStream fis = new FileInputStream(tempDir + "/" + yyyyMMdd + ".index"); ByteBuffer bb2 = fis.getChannel().map(FileChannel.MapMode.READ_ONLY, 0, fis.getChannel().size()).order(ByteOrder.nativeOrder()); int pos2 = 0; for (int i = 0; i < SIZE; i++) { if ((i & 7) == 0) { long pos3 = bb2.getLong(); Assert.assertEquals(pos2, pos3); } int size = 16 + i % 5; pos += (size + 3) & ~3; } fis.close(); } @Test @Ignore public void writeOneAtATime() throws IOException { File tempDir = new File(System.getProperty("java.io.tmpdir") + "/deleteme" + Long.toString(System.nanoTime(), 36)); ChronicleTools.deleteDirOnExit(tempDir.toString()); tempDir.mkdirs(); String yyyyMMdd = new SimpleDateFormat("yyyyMMdd").format(new Date()); DailyRollingConfig config = new DailyRollingConfig(); DailyRollingChronicle chronicle = new DailyRollingChronicle(tempDir.getAbsolutePath(), config); ExcerptAppender appender = chronicle.createAppender(); DailyRollingChronicle chronicle2 = new DailyRollingChronicle(tempDir.getAbsolutePath(), config); Excerpt excerpt = chronicle2.createExcerpt(); ExcerptTailer tailer = chronicle2.createTailer(); for (int index = 0; index < 100; index++) { appender.startExcerpt(); appender.writeInt(~index); for (int i = 0; i < index; i++) appender.write(-1); appender.finish(); DailingRollingReader.dumpData(tempDir + "/" + yyyyMMdd + ".data", new PrintWriter(System.out)); Assert.assertTrue(excerpt.index(index)); Assert.assertEquals(index + 4, excerpt.size()); Assert.assertEquals(~index, excerpt.readInt()); Assert.assertEquals(index, excerpt.remaining()); excerpt.finish(); Assert.assertTrue(tailer.nextIndex()); Assert.assertEquals(index + 4, tailer.size()); Assert.assertEquals(~index, tailer.readInt()); Assert.assertEquals(index, tailer.remaining()); tailer.finish(); // check we can jump to this index. tailer.index(index - 1); Assert.assertTrue(tailer.nextIndex()); Assert.assertEquals(~index, tailer.readInt()); Assert.assertEquals(index, tailer.remaining()); tailer.finish(); } chronicle2.close(); chronicle.close(); } } DateCacheTest.java000066400000000000000000000031741226752375100336760ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle-sandbox/src/test/java/net/openhft/chronicle/sandbox/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.openhft.chronicle.sandbox; import org.junit.Test; import java.text.SimpleDateFormat; import java.util.Date; import java.util.TimeZone; import static org.junit.Assert.assertEquals; public class DateCacheTest { @Test public void testFormat() { DateCache dc = new DateCache("yyyyMMdd", 86400000); String str = dc.formatFor(16067); assertEquals("20131228", str); String str1 = dc.formatFor(1); assertEquals("19700102", str1); } @Test public void testFormatMillis() { String format = "yyyyMMddHHmmss"; SimpleDateFormat sdf = new SimpleDateFormat(format); sdf.setTimeZone(TimeZone.getTimeZone("GMT")); DateCache dc = new DateCache(format, 1000); int now = (int) (System.currentTimeMillis() / 1000); for (int i = 0; i < 10000; i++) { int now2 = now + i; String str2 = sdf.format(new Date(now2 * 1000L)); String str = dc.formatFor(now2); assertEquals("i: " + i, str2, str); } } } RollingChronicleTest.java000066400000000000000000000045471226752375100353370ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle-sandbox/src/test/java/net/openhft/chronicle/sandbox/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.openhft.chronicle.sandbox; import net.openhft.chronicle.ChronicleConfig; import net.openhft.chronicle.ExcerptAppender; import net.openhft.chronicle.ExcerptTailer; import net.openhft.chronicle.tools.ChronicleTools; import org.junit.Assert; import org.junit.Test; import java.io.IOException; /** * User: peter.lawrey * Date: 26/09/13 * Time: 17:48 */ public class RollingChronicleTest { @Test public void testAppending() throws IOException { int counter = 0; String basePath = System.getProperty("java.io.tmpdir") + "/testAppending"; ChronicleTools.deleteDirOnExit(basePath); ChronicleConfig test = ChronicleConfig.TEST.clone(); test.indexFileExcerpts(256); for (int k = 0; k < 500; k++) { RollingChronicle rc = new RollingChronicle(basePath, test); ExcerptAppender appender = rc.createAppender(); Assert.assertEquals("k: " + k, (long) counter, appender.size()); for (int i = 0; i < 255 /* ChronicleConfig.TEST.indexFileExcerpts() * 2 / 7 */; i++) { appender.startExcerpt(); appender.writeInt(counter++); appender.finish(); Assert.assertEquals("k: " + k + ", i: " + i, (long) counter, appender.size()); } appender.close(); rc.close(); // ChronicleMasterReader.main(basePath + "/master"); } // counter = 8192*2; RollingChronicle rc = new RollingChronicle(basePath, test); ExcerptTailer tailer = rc.createTailer(); for (int i = 0; i < counter; i++) { Assert.assertTrue("i: " + i, tailer.nextIndex()); Assert.assertEquals(i, tailer.readInt()); tailer.finish(); } rc.close(); } } VanillaChronicleSourceTest.java000066400000000000000000000052451226752375100364740ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle-sandbox/src/test/java/net/openhft/chronicle/sandbox/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.openhft.chronicle.sandbox; import net.openhft.chronicle.ExcerptAppender; import net.openhft.chronicle.ExcerptTailer; import net.openhft.chronicle.sandbox.tcp.VanillaChronicleSink; import net.openhft.chronicle.sandbox.tcp.VanillaChronicleSource; import org.junit.Test; import java.io.IOException; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; public class VanillaChronicleSourceTest { @Test public void testReplication() throws IOException { int RUNS = 100; String basePath = System.getProperty("java.io.tmpdir") + "/testReplication"; VanillaChronicleSource chronicle = new VanillaChronicleSource(new VanillaChronicle(basePath + "-source"), 0); int localPort = chronicle.getLocalPort(); VanillaChronicleSink chronicle2 = new VanillaChronicleSink(new VanillaChronicle(basePath + "-sink"), "localhost", localPort); try { ExcerptAppender appender = chronicle.createAppender(); ExcerptTailer tailer = chronicle2.createTailer(); assertEquals(-1L, tailer.index()); for (int i = 0; i < RUNS; i++) { // if ((i & 65535) == 0) // System.err.println("i: " + i); // if (i == 88000) // Thread.yield(); assertFalse(tailer.nextIndex()); appender.startExcerpt(); int value = 1000000000 + i; appender.append(value).append(' '); appender.finish(); // chronicle.checkCounts(1, 2); assertTrue("i: " + i, tailer.nextIndex()); // chronicle2.checkCounts(1, 2); assertTrue("i: " + i + " remaining: " + tailer.remaining(), tailer.remaining() > 0); assertEquals("i: " + i, value, tailer.parseLong()); assertEquals("i: " + i, 0, tailer.remaining()); tailer.finish(); // chronicle2.checkCounts(1, 2); } } finally { chronicle2.close(); chronicle.clear(); } } } VanillaChronicleTest.java000066400000000000000000000213131226752375100353050ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle-sandbox/src/test/java/net/openhft/chronicle/sandbox/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.openhft.chronicle.sandbox; import net.openhft.chronicle.ExcerptAppender; import net.openhft.chronicle.ExcerptTailer; import org.junit.Test; import java.io.IOException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import static org.junit.Assert.*; public class VanillaChronicleTest { private static final int N_THREADS = 4; @Test public void testAppend() throws IOException { final int RUNS = 1000; String baseDir = System.getProperty("java.io.tmpdir") + "/testAppend"; VanillaChronicleConfig config = new VanillaChronicleConfig(); config.defaultMessageSize(128); config.indexBlockSize(1024); config.dataBlockSize(1024); VanillaChronicle chronicle = new VanillaChronicle(baseDir, config); chronicle.clear(); ExcerptAppender appender = chronicle.createAppender(); for (int i = 0; i < RUNS; i++) { // System.err.println("i: " + i); // if (i == 256) // Thread.yield(); appender.startExcerpt(); appender.append(1000000000 + i); appender.finish(); chronicle.checkCounts(1, 2); } chronicle.close(); chronicle.clear(); } @Test public void testAppend4() throws IOException, InterruptedException { final int RUNS = 20000; String baseDir = System.getProperty("java.io.tmpdir") + "/testAppend4"; VanillaChronicleConfig config = new VanillaChronicleConfig(); config.defaultMessageSize(128); // config.indexBlockSize(1024); // config.dataBlockSize(1024); long start = System.nanoTime(); final VanillaChronicle chronicle = new VanillaChronicle(baseDir, config); chronicle.clear(); ExecutorService es = Executors.newFixedThreadPool(N_THREADS); for (int t = 0; t < N_THREADS; t++) { final int finalT = t; es.submit(new Runnable() { @Override public void run() { try { ExcerptAppender appender = chronicle.createAppender(); for (int i = 0; i < RUNS; i++) { appender.startExcerpt(); appender.append(finalT).append("/").append(i).append('\n'); appender.finish(); Thread.yield(); } } catch (Exception e) { e.printStackTrace(); } } }); } es.shutdown(); es.awaitTermination(2, TimeUnit.SECONDS); long time = System.nanoTime() - start; chronicle.close(); chronicle.clear(); System.out.printf("Took an average of %.1f us per entry%n", time / 1e3 / (RUNS * N_THREADS)); } @Test public void testTailer() throws IOException { final int RUNS = 1000000; // 5000000; String baseDir = System.getProperty("java.io.tmpdir") + "/" + "testTailer"; VanillaChronicleConfig config = new VanillaChronicleConfig(); config.defaultMessageSize(128); config.indexBlockSize(256 << 10); config.dataBlockSize(512 << 10); VanillaChronicle chronicle = new VanillaChronicle(baseDir, config); chronicle.clear(); VanillaChronicle chronicle2 = new VanillaChronicle(baseDir, config); try { ExcerptAppender appender = chronicle.createAppender(); ExcerptTailer tailer = chronicle2.createTailer(); assertEquals(-1L, tailer.index()); for (int i = 0; i < RUNS; i++) { // if ((i & 65535) == 0) // System.err.println("i: " + i); // if (i == 88000) // Thread.yield(); assertFalse(tailer.nextIndex()); appender.startExcerpt(); int value = 1000000000 + i; appender.append(value).append(' '); appender.finish(); chronicle.checkCounts(1, 2); assertTrue("i: " + i, tailer.nextIndex()); chronicle2.checkCounts(1, 2); assertTrue("i: " + i + " remaining: " + tailer.remaining(), tailer.remaining() > 0); assertEquals("i: " + i, value, tailer.parseLong()); assertEquals("i: " + i, 0, tailer.remaining()); tailer.finish(); chronicle2.checkCounts(1, 2); } } finally { chronicle2.close(); chronicle.clear(); } } @Test public void testTailerPerf() throws IOException { final int WARMUP = 50000; final int RUNS = 5000000; String baseDir = System.getProperty("java.io.tmpdir") + "/testTailerPerf"; VanillaChronicle chronicle = new VanillaChronicle(baseDir); chronicle.clear(); try { ExcerptAppender appender = chronicle.createAppender(); ExcerptTailer tailer = chronicle.createTailer(); long start = 0; assertEquals(-1L, tailer.index()); for (int i = -WARMUP; i < RUNS; i++) { if (i == 0) start = System.nanoTime(); boolean condition0 = tailer.nextIndex(); if (condition0) assertFalse("i: " + i, condition0); appender.startExcerpt(); int value = 1000000000 + i; appender.append(value).append(' '); appender.finish(); boolean condition = tailer.nextIndex(); long actual = tailer.parseLong(); if (i < 0) { assertTrue("i: " + i, condition); assertEquals("i: " + i, value, actual); } tailer.finish(); } long time = System.nanoTime() - start; System.out.printf("Average write/read times was %.3f us%n", time / RUNS / 1e3); } finally { chronicle.close(); chronicle.clear(); } } @Test public void testTailerPerf2() throws IOException, InterruptedException { final int WARMUP = 100000; final int RUNS = 4000000; final int BYTES = 96; String baseDir = System.getProperty("java.io.tmpdir") + "/testTailerPerf"; final VanillaChronicle chronicle = new VanillaChronicle(baseDir); chronicle.clear(); try { Thread t = new Thread(new Runnable() { @Override public void run() { for (int i = -WARMUP; i < RUNS; i++) { ExcerptTailer tailer = null; try { tailer = chronicle.createTailer(); } catch (IOException e) { e.printStackTrace(); } while (!tailer.nextIndex()) ; long actual = -1; for (int j = 0; j < BYTES; j += 8) actual = tailer.readLong(); if (i < 0) { int value = 1000000000 + i; assertEquals("i: " + i, value, actual); } tailer.finish(); } } }); t.start(); ExcerptAppender appender = chronicle.createAppender(); long start = 0; for (int i = -WARMUP; i < RUNS; i++) { if (i == 0) start = System.nanoTime(); appender.startExcerpt(); int value = 1000000000 + i; for (int j = 0; j < BYTES; j += 8) appender.writeLong(value); appender.finish(); } t.join(); long time = System.nanoTime() - start; System.out.printf("Average write/read times was %.3f us%n", time / RUNS / 1e3); } finally { chronicle.close(); chronicle.clear(); } } } VanillaDataCacheTest.java000066400000000000000000000075201226752375100352000ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle-sandbox/src/test/java/net/openhft/chronicle/sandbox/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.openhft.chronicle.sandbox; import net.openhft.affinity.AffinitySupport; import org.junit.Test; import java.io.File; import static org.junit.Assert.*; public class VanillaDataCacheTest { @Test public void testDataFor() throws Exception { File dir = new File(System.getProperty("java.io.tmpdir"), "testDataFor"); DateCache dateCache = new DateCache("yyyyMMddHHmmss", 1000); VanillaDataCache cache = new VanillaDataCache(dir.getAbsolutePath(), 10 + 6, dateCache); int cycle = (int) (System.currentTimeMillis() / 1000); VanillaFile vanillaFile0 = cache.dataFor(cycle, AffinitySupport.getThreadId(), 0, true); vanillaFile0.bytes().writeLong(0, 0x12345678); File file0 = vanillaFile0.file(); assertEquals(64 << 10, file0.length()); assertEquals(0x12345678L, vanillaFile0.bytes().readLong(0)); vanillaFile0.decrementUsage(); VanillaFile vanillaFile1 = cache.dataFor(cycle, AffinitySupport.getThreadId(), 1, true); File file1 = vanillaFile1.file(); assertEquals(64 << 10, file1.length()); vanillaFile1.decrementUsage(); assertNotEquals(vanillaFile1.file(), vanillaFile0.file()); VanillaFile vanillaFile2 = cache.dataFor(cycle, AffinitySupport.getThreadId(), 2, true); File file2 = vanillaFile2.file(); assertEquals(64 << 10, file2.length()); vanillaFile2.decrementUsage(); assertNotEquals(vanillaFile2.file(), vanillaFile0.file()); assertNotEquals(vanillaFile2.file(), vanillaFile1.file()); cache.close(); assertEquals(0, vanillaFile0.usage()); assertEquals(0, vanillaFile1.usage()); assertEquals(0, vanillaFile2.usage()); // check you can delete after closing. assertTrue(file0.delete()); assertTrue(file1.delete()); assertTrue(file2.delete()); assertTrue(file0.getParentFile().delete()); dir.delete(); } @Test public void testDataForPerf() throws Exception { File dir = new File(System.getProperty("java.io.tmpdir"), "testDataForPerf"); DateCache dateCache = new DateCache("yyyyMMddHHmmss", 1000); VanillaDataCache cache = new VanillaDataCache(dir.getAbsolutePath(), 10 + 7, dateCache); int cycle = (int) (System.currentTimeMillis() / 1000); File file0 = null; for (int j = 0; j < 5; j++) { long start = System.nanoTime(); int runs = 10000; for (int i = 0; i < runs; i++) { VanillaFile vanillaFile0 = cache.dataFor(cycle, AffinitySupport.getThreadId(), i, true); vanillaFile0.bytes().writeLong(0, 0x12345678); file0 = vanillaFile0.file(); assertEquals(128 << 10, file0.length()); assertEquals(0x12345678L, vanillaFile0.bytes().readLong(0)); vanillaFile0.decrementUsage(); vanillaFile0.close(); assertTrue(file0.delete()); } file0.getParentFile().getParentFile().delete(); long time = System.nanoTime() - start; System.out.printf("The average time was %,d us%n", time / runs / 1000); } assertTrue(file0.getParentFile().delete()); dir.delete(); } } VanillaIndexCacheTest.java000066400000000000000000000046621226752375100354020ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle-sandbox/src/test/java/net/openhft/chronicle/sandbox/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.openhft.chronicle.sandbox; import org.junit.Test; import java.io.File; import static org.junit.Assert.*; public class VanillaIndexCacheTest { @Test public void testIndexFor() throws Exception { File dir = new File(System.getProperty("java.io.tmpdir"), "testIndexFor"); DateCache dateCache = new DateCache("yyyyMMddHHmmss", 1000); VanillaIndexCache cache = new VanillaIndexCache(dir.getAbsolutePath(), 10 + 3, dateCache); int cycle = (int) (System.currentTimeMillis() / 1000); VanillaFile vanillaFile0 = cache.indexFor(cycle, 0, true); vanillaFile0.bytes().writeLong(0, 0x12345678); File file0 = vanillaFile0.file(); assertEquals(8 << 10, file0.length()); assertEquals(0x12345678L, vanillaFile0.bytes().readLong(0)); vanillaFile0.decrementUsage(); VanillaFile vanillaFile1 = cache.indexFor(cycle, 1, true); File file1 = vanillaFile1.file(); assertEquals(8 << 10, file1.length()); vanillaFile1.decrementUsage(); assertNotEquals(vanillaFile1.file(), vanillaFile0.file()); VanillaFile vanillaFile2 = cache.indexFor(cycle, 2, true); File file2 = vanillaFile2.file(); assertEquals(8 << 10, file2.length()); vanillaFile2.decrementUsage(); assertNotEquals(vanillaFile2.file(), vanillaFile0.file()); assertNotEquals(vanillaFile2.file(), vanillaFile1.file()); cache.close(); assertEquals(0, vanillaFile0.usage()); assertEquals(0, vanillaFile1.usage()); assertEquals(0, vanillaFile2.usage()); // check you can delete after closing. assertTrue(file0.delete()); assertTrue(file1.delete()); assertTrue(file2.delete()); assertTrue(file0.getParentFile().delete()); file0.getParentFile().getParentFile().delete(); } } Chronicle-Queue-2.0.3/chronicle-sandbox/src/test/java/net/openhft/chronicle/sandbox/VdsoTest.java000066400000000000000000000040071226752375100330630ustar00rootroot00000000000000/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.openhft.chronicle.sandbox; import net.openhft.lang.io.NativeBytes; import org.junit.Test; import java.io.BufferedReader; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStreamReader; public class VdsoTest { @Test public void printVdso() throws IOException, InterruptedException { long start = 0; long end = 0; BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream("/proc/self/maps"))); try { for (String line; (line = br.readLine()) != null; ) { if (line.endsWith("[vdso]")) { String[] parts = line.split("[- ]"); start = Long.parseLong(parts[0], 16); end = Long.parseLong(parts[1], 16); } // System.out.println(line); } } catch (IOException ioe) { br.close(); throw ioe; } System.out.printf("vdso %x to %x %n", start, end); NativeBytes nb = new NativeBytes(start, start, end); long[] longs = new long[(int) ((end - start) / 8)]; for (int i = 0; i < longs.length; i++) longs[i] = nb.readLong(i * 8); Thread.sleep(1); for (int i = 0; i < longs.length; i++) { long l = nb.readLong(i * 8); if (l != longs[i]) System.out.printf("%d: %d %x%n", i, l, l); } } } Chronicle-Queue-2.0.3/chronicle/000077500000000000000000000000001226752375100164645ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle/pom.xml000077500000000000000000000203461226752375100200110ustar00rootroot00000000000000 4.0.0 net.openhft chronicle 2.0.3-SNAPSHOT bundle OpenHTF/Java-Chronicle Java library for persisted low latency messaging (Java 6+) UTF-8 3.2.2 3.3.0 1.6.0 net.openhft lang 6.1.4-SNAPSHOT net.openhft affinity 2.0.1 test org.kohsuke.jetbrains annotations 9.0 compile org.easymock easymock 3.1 test junit junit 4.11 test org.ops4j.pax.exam pax-exam-container-native ${pax.exam.version} test org.ops4j.pax.exam pax-exam-junit4 ${pax.exam.version} test org.ops4j.pax.exam pax-exam-link-mvn ${pax.exam.version} test org.ops4j.pax.url pax-url-aether ${pax.url.version} test org.apache.felix org.apache.felix.framework ${felix.version} test ch.qos.logback logback-core 0.9.6 test ch.qos.logback logback-classic 0.9.6 test net.java.dev.jna jna 4.0.0 test net.java.dev.jna platform 3.5.2 test net.sf.trove4j trove4j 3.0.3 test org.apache.maven.plugins maven-compiler-plugin 3.1 1.6 1.6 UTF-8 org.apache.maven.plugins maven-source-plugin 2.2 attach-sources verify jar org.apache.maven.plugins maven-javadoc-plugin 2.9 attach-javadocs verify jar public true org.apache.felix maven-bundle-plugin 2.3.7 true ${project.groupId}.${project.artifactId} ${project.artifactId} net.openhft.chronicle.* http://www.openhft.net The Apache Software License, Version 2.0 http://www.apache.org/licenses/LICENSE-2.0.txt repo A business-friendly OSS license Peter Lawrey peter.lawrey@higherfrequencytrading.com scm:git:https://github.com/OpenHFT/Java-Chronicle.git Chronicle-Queue-2.0.3/chronicle/src/000077500000000000000000000000001226752375100172535ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle/src/main/000077500000000000000000000000001226752375100201775ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle/src/main/java/000077500000000000000000000000001226752375100211205ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle/src/main/java/net/000077500000000000000000000000001226752375100217065ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle/src/main/java/net/openhft/000077500000000000000000000000001226752375100233515ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle/src/main/java/net/openhft/chronicle/000077500000000000000000000000001226752375100253175ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle/src/main/java/net/openhft/chronicle/AbstractNativeExcerpt.java000077500000000000000000000231451226752375100324370ustar00rootroot00000000000000/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.openhft.chronicle; import net.openhft.lang.io.MappedMemory; import net.openhft.lang.io.NativeBytes; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import java.io.IOException; /** * @author peter.lawrey */ public abstract class AbstractNativeExcerpt extends NativeBytes implements ExcerptCommon { @NotNull protected final IndexedChronicle chronicle; final int cacheLineMask; final int dataBlockSize; final int indexBlockSize; final int indexEntriesPerLine; final int indexEntriesPerBlock; private final int cacheLineSize; @Nullable @SuppressWarnings("FieldCanBeLocal") MappedMemory indexBuffer; @Nullable @SuppressWarnings("FieldCanBeLocal") MappedMemory dataBuffer; long index = -1; // relatively static // the start of the index block, as an address long indexStartAddr; // which index does this refer to? long indexStartOffset; // the offset in data referred to the start of the line long indexBaseForLine; // the start of the data block, as an address long dataStartAddr; // which offset does this refer to. long dataStartOffset; // the position currently writing to in the index. long indexPositionAddr; boolean padding = true; // the start of this entry // inherited - long startAddr; // inherited - long positionAddr; // inherited - long limitAddr; public AbstractNativeExcerpt(@NotNull IndexedChronicle chronicle) throws IOException { super(0, 0, 0); this.chronicle = chronicle; cacheLineSize = chronicle.config.cacheLineSize(); cacheLineMask = (cacheLineSize - 1); dataBlockSize = chronicle.config.dataBlockSize(); indexBlockSize = chronicle.config.indexBlockSize(); indexEntriesPerLine = (cacheLineSize - 8) / 4; indexEntriesPerBlock = indexBlockSize * indexEntriesPerLine / cacheLineSize; loadIndexBuffer(); loadDataBuffer(); finished = true; } @Override public long index() { return index; } public ExcerptCommon toStart() { index = -1; return this; } protected boolean indexForRead(long l) throws IOException { if (l < 0) { setIndexBuffer(0, true); index = -1; padding = true; return false; } long indexLookup = l / indexEntriesPerBlock; setIndexBuffer(indexLookup, true); long indexLookupMod = l % indexEntriesPerBlock; int indexLineEntry = (int) (indexLookupMod % indexEntriesPerLine); int indexLineStart = (int) (indexLookupMod / indexEntriesPerLine * cacheLineSize); int inLine = (indexLineEntry << 2) + 8; int dataOffsetEnd = UNSAFE.getInt(indexStartAddr + indexLineStart + inLine); indexBaseForLine = UNSAFE.getLong(indexStartAddr + indexLineStart); indexPositionAddr = indexStartAddr + indexLineStart + inLine; long dataOffsetStart = inLine == 0 ? indexBaseForLine : (indexBaseForLine + Math.abs(UNSAFE.getInt(indexPositionAddr - 4))); long dataLookup = dataOffsetStart / dataBlockSize; long dataLookupMod = dataOffsetStart % dataBlockSize; setDataBuffer(dataLookup); startAddr = positionAddr = dataStartAddr + dataLookupMod; index = l; if (dataOffsetEnd > 0) { limitAddr = dataStartAddr + (indexBaseForLine + dataOffsetEnd - dataLookup * dataBlockSize); indexPositionAddr += 4; padding = false; return true; } else if (dataOffsetEnd == 0) { limitAddr = startAddr; padding = false; return false; } else /* if (dataOffsetEnd < 0) */ { padding = true; return false; } } private void setIndexBuffer(long index, boolean prefetch) throws IOException { MappedMemory.release(indexBuffer); indexBuffer = chronicle.indexFileCache.acquire(index, prefetch); indexPositionAddr = indexStartAddr = indexBuffer.address(); } protected void indexForAppender(long l) throws IOException { if (l < 0) { throw new IndexOutOfBoundsException("index: " + l); } else if (l == 0) { indexStartOffset = 0; loadIndexBuffer(); dataStartOffset = 0; loadDataBuffer(); return; } // We need the end of the previous Excerpt l--; long indexLookup = l / indexEntriesPerBlock; setIndexBuffer(indexLookup, true); long indexLookupMod = l % indexEntriesPerBlock; int indexLineEntry = (int) (indexLookupMod % indexEntriesPerLine); int indexLineStart = (int) (indexLookupMod / indexEntriesPerLine * cacheLineSize); int inLine = (indexLineEntry << 2) + 8; indexStartOffset = indexLookup * indexBlockSize + indexLineStart; indexBaseForLine = UNSAFE.getLong(indexStartAddr + indexLineStart); long dataOffsetEnd = indexBaseForLine + Math.abs(UNSAFE.getInt(indexStartAddr + indexLineStart + inLine)); long dataLookup = dataOffsetEnd / dataBlockSize; long dataLookupMod = dataOffsetEnd % dataBlockSize; setDataBuffer(dataLookup); dataStartOffset = dataLookup * dataBlockSize; startAddr = positionAddr = dataStartAddr + dataLookupMod; index = l + 1; indexPositionAddr = indexStartAddr + indexLineStart + inLine + 4; } private void setDataBuffer(long dataLookup) throws IOException { MappedMemory.release(dataBuffer); dataBuffer = chronicle.dataFileCache.acquire(dataLookup, true); dataStartAddr = dataBuffer.address(); } @Override public boolean wasPadding() { return padding; } @Override public long lastWrittenIndex() { return chronicle.lastWrittenIndex(); } @Override public long size() { return chronicle.size(); } @NotNull @Override public Chronicle chronicle() { return chronicle; } void loadNextIndexBuffer() throws IOException { indexStartOffset += indexBlockSize; loadIndexBuffer(); } void loadNextDataBuffer() throws IOException { dataStartOffset += dataBlockSize; loadDataBuffer(); } void loadNextDataBuffer(long offsetInThisBuffer) throws IOException { dataStartOffset += offsetInThisBuffer / dataBlockSize * dataBlockSize; loadDataBuffer(); } void loadDataBuffer() throws IOException { setDataBuffer(dataStartOffset / dataBlockSize); startAddr = positionAddr = limitAddr = dataStartAddr; } void loadIndexBuffer() throws IOException { setIndexBuffer(indexStartOffset / indexBlockSize, true); } public boolean index(long index) { throw new UnsupportedOperationException(); } public long findMatch(@NotNull ExcerptComparator comparator) { long lo = 0, hi = lastWrittenIndex(); while (lo <= hi) { long mid = (hi + lo) >>> 1; if (!index(mid)) { if (mid > lo) index(--mid); else break; } int cmp = comparator.compare((Excerpt) this); finish(); if (cmp < 0) lo = mid + 1; else if (cmp > 0) hi = mid - 1; else return mid; // key found } return ~lo; // -(lo + 1) } public void findRange(@NotNull long[] startEnd, @NotNull ExcerptComparator comparator) { // lower search range long lo1 = 0, hi1 = lastWrittenIndex(); // upper search range long lo2 = 0, hi2 = hi1; boolean both = true; // search for the low values. while (lo1 <= hi1) { long mid = (hi1 + lo1) >>> 1; if (!index(mid)) { if (mid > lo1) index(--mid); else break; } int cmp = comparator.compare((Excerpt) this); finish(); if (cmp < 0) { lo1 = mid + 1; if (both) lo2 = lo1; } else if (cmp > 0) { hi1 = mid - 1; if (both) hi2 = hi1; } else { hi1 = mid - 1; if (both) lo2 = mid + 1; both = false; } } // search for the high values. while (lo2 <= hi2) { long mid = (hi2 + lo2) >>> 1; if (!index(mid)) { if (mid > lo2) index(--mid); else break; } int cmp = comparator.compare((Excerpt) this); finish(); if (cmp <= 0) { lo2 = mid + 1; } else { hi2 = mid - 1; } } startEnd[0] = lo1; // inclusive startEnd[1] = lo2; // exclusive } } Chronicle-Queue-2.0.3/chronicle/src/main/java/net/openhft/chronicle/Chronicle.java000077500000000000000000000020671226752375100301000ustar00rootroot00000000000000/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.openhft.chronicle; import org.jetbrains.annotations.NotNull; import java.io.Closeable; import java.io.IOException; /** * @author peter.lawrey */ public interface Chronicle extends Closeable { String name(); @NotNull Excerpt createExcerpt() throws IOException; @NotNull ExcerptTailer createTailer() throws IOException; @NotNull ExcerptAppender createAppender() throws IOException; long lastWrittenIndex(); long size(); } Chronicle-Queue-2.0.3/chronicle/src/main/java/net/openhft/chronicle/ChronicleConfig.java000077500000000000000000000117171226752375100312300ustar00rootroot00000000000000/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.openhft.chronicle; import net.openhft.lang.Jvm; import org.jetbrains.annotations.NotNull; import java.nio.ByteOrder; /** * @author peter.lawrey */ public class ChronicleConfig implements Cloneable { // 16 billion max, or one per day for 11 years. public static final ChronicleConfig SMALL = new ChronicleConfig(4 * 1024, 2 * 1024 * 1024, true, 16 * 1024 * 1024); // 256 billion max public static final ChronicleConfig MEDIUM = new ChronicleConfig(16 * 1024, 16 * 1024 * 1024, false, 128 * 1024 * 1024); // 4 trillion max public static final ChronicleConfig LARGE = new ChronicleConfig(64 * 1024, 64 * 1024 * 1024, false, 512 * 1024 * 1024); // 1 quadrillion max public static final ChronicleConfig HUGE = new ChronicleConfig(4 * 1024 * 1024, 256 * 1024 * 1024, false, 512 * 1024 * 1024); // maximise overhead for testing purposes public static final ChronicleConfig TEST = new ChronicleConfig(1024 * 1024, 8 * 1024, true, 8 * 1024); // default used by Chronicle if not specified. public static final ChronicleConfig DEFAULT = Jvm.is64Bit() ? MEDIUM : SMALL; private int indexFileCapacity; private int indexFileExcerpts; private boolean minimiseFootprint; // optional parameters, turn on for benchmarks. private boolean useUnsafe = false; private boolean synchronousMode = false; private ByteOrder byteOrder = ByteOrder.nativeOrder(); private int cacheLineSize = 64; private int dataBlockSize; private int indexBlockSize; private int messageCapacity = 128 * 1024; public ChronicleConfig(int indexFileCapacity, int indexFileExcerpts, boolean minimiseFootprint, int dataBlockSize) { this.indexFileCapacity = indexFileCapacity; this.indexFileExcerpts = indexFileExcerpts; this.minimiseFootprint = minimiseFootprint; this.dataBlockSize = dataBlockSize; indexBlockSize = Math.max(4096, this.dataBlockSize / 4); } public ChronicleConfig indexFileCapacity(int indexFileCapacity) { this.indexFileCapacity = indexFileCapacity; return this; } public int indexFileCapacity() { return indexFileCapacity; } public ChronicleConfig indexFileExcerpts(int indexFileExcerpts) { this.indexFileExcerpts = indexFileExcerpts; return this; } public int indexFileExcerpts() { return indexFileExcerpts; } public void minimiseFootprint(boolean minimiseFootprint) { this.minimiseFootprint = minimiseFootprint; } public boolean minimiseFootprint() { return minimiseFootprint; } public ChronicleConfig useUnsafe(boolean useUnsafe) { this.useUnsafe = useUnsafe; return this; } public boolean useUnsafe() { return useUnsafe; } public ChronicleConfig synchronousMode(boolean synchronousMode) { this.synchronousMode = synchronousMode; return this; } public boolean synchronousMode() { return synchronousMode; } public ChronicleConfig byteOrder(ByteOrder byteOrder) { this.byteOrder = byteOrder; return this; } public ByteOrder byteOrder() { return byteOrder; } public ChronicleConfig cacheLineSize(int cacheLineSize) { this.cacheLineSize = cacheLineSize; return this; } public int cacheLineSize() { return cacheLineSize; } public ChronicleConfig dataBlockSize(int dataBlockSize) { this.dataBlockSize = dataBlockSize; if (messageCapacity > dataBlockSize / 2) messageCapacity = dataBlockSize / 2; return this; } public int dataBlockSize() { return dataBlockSize; } public ChronicleConfig indexBlockSize(int indexBlockSize) { this.indexBlockSize = indexBlockSize; return this; } public int indexBlockSize() { return indexBlockSize; } public ChronicleConfig messageCapacity(int messageCapacity) { this.messageCapacity = messageCapacity; return this; } public int messageCapacity() { return messageCapacity; } @NotNull @SuppressWarnings("CloneDoesntDeclareCloneNotSupportedException") @Override public ChronicleConfig clone() { try { return (ChronicleConfig) super.clone(); } catch (CloneNotSupportedException e) { throw new AssertionError(e); } } } Chronicle-Queue-2.0.3/chronicle/src/main/java/net/openhft/chronicle/Excerpt.java000077500000000000000000000037151226752375100276050ustar00rootroot00000000000000/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.openhft.chronicle; import org.jetbrains.annotations.NotNull; /** * @author peter.lawrey */ public interface Excerpt extends ExcerptTailer { /** * Find any entry which return a match i.e. 0, or a negative value which is the boundary between -1 and +1 * * @param comparator to use for comparison. * @return 0 to size()-1 for a match, -1 to -size()-1 for index of closest match */ long findMatch(@NotNull ExcerptComparator comparator); /** * Find entries which return a match. This is a combined search which is usually faster than two binary search. * This doesn't access the same index two. The best case is one additional comparison and the worst case is the same. * * @param startEnd start (inclusive) to end (enclusive). Will be equal if no exact match is found. * @param comparator to use for comparison. */ void findRange(@NotNull long[] startEnd, @NotNull ExcerptComparator comparator); /** * Randomly select an Excerpt. * * @param l index to look up * @return true if this is a valid entries and not padding. */ boolean index(long l); /** * Replay from the start. * * @return this Excerpt */ @NotNull Excerpt toStart(); /** * Wind to the end. * * @return this Excerpt */ @NotNull Excerpt toEnd(); } Chronicle-Queue-2.0.3/chronicle/src/main/java/net/openhft/chronicle/ExcerptAppender.java000077500000000000000000000040521226752375100312570ustar00rootroot00000000000000/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.openhft.chronicle; import org.jetbrains.annotations.NotNull; /** * @author peter.lawrey */ public interface ExcerptAppender extends ExcerptCommon { /** * Start an excerpt with the default message capacity of 128K (can be configured) * This can waste up to 0.1% of disk space, unless you have sparse file support like Linux, when you will waste far less. */ void startExcerpt(); /** * Ensure there is enough capacity for a new entry of up to the size given. If there is not enough space left in * the chunk of memory mapped file, a padded entry is added and a new entry at the start of a new chunk is * commenced. The capacity can be more than you need as finish() will shrink wrap the entry. It is onl a waste if * you trigger a padded entry when none was required. * * @param capacity to allow for, but not exceed. */ void startExcerpt(long capacity); /** * Skip to the last index. * * @return this Excerpt */ @NotNull ExcerptAppender toEnd(); /** * Add a padded entry to keep the index in sync with a master source. */ void addPaddedEntry(); /** * The default value is ChronicleConfig.synchronousMode() * * @return will the next write be synchronous */ boolean nextSynchronous(); /** * @param nextSynchronous make the next write synchronous or not. */ void nextSynchronous(boolean nextSynchronous); } Chronicle-Queue-2.0.3/chronicle/src/main/java/net/openhft/chronicle/ExcerptCommon.java000077500000000000000000000031051226752375100307470ustar00rootroot00000000000000/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.openhft.chronicle; import net.openhft.lang.io.Bytes; /** * @author peter.lawrey */ public interface ExcerptCommon extends Bytes { /** * @return true if the last index(long) looked up as padding. */ boolean wasPadding(); /** * @return this appender is pointing. */ long index(); /** * @return the index last written to including padded entries. */ long lastWrittenIndex(); /** * This is an upper bound for the number of entires available. This includes padded entries. * * @return lastWrittenIndex() + 1 */ long size(); /** * Wind to the end. * * @return this Excerpt */ ExcerptCommon toEnd(); /** * @return the chronicle associated with this Excerpt */ Chronicle chronicle(); /** * Finish reading or writing. This checks there was not a buffer overflow and shrink wraps new entries and adds * them to the index */ void finish(); } Chronicle-Queue-2.0.3/chronicle/src/main/java/net/openhft/chronicle/ExcerptComparator.java000066400000000000000000000020221226752375100316200ustar00rootroot00000000000000/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.openhft.chronicle; /** * For a binary search, provide a comparison of Excerpts */ public interface ExcerptComparator { /** * Given some criteria, deterime if the entry is -1 = below range, +1 = above range and 0 in range * Can be used for exact matches or a range of values. * * @param excerpt to check * @return -1 below, 0 = in range, +1 above range. */ int compare(Excerpt excerpt); } Chronicle-Queue-2.0.3/chronicle/src/main/java/net/openhft/chronicle/ExcerptMarshallable.java000077500000000000000000000020671226752375100321140ustar00rootroot00000000000000/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.openhft.chronicle; /** * @author peter.lawrey */ public interface ExcerptMarshallable { /** * read an object from an excerpt * * @param in to read from * @throws IllegalStateException if the object could not be read. */ void readMarshallable(ExcerptCommon in) throws IllegalStateException; /** * write an object to an excerpt * * @param out to write to */ void writeMarshallable(ExcerptCommon out); } Chronicle-Queue-2.0.3/chronicle/src/main/java/net/openhft/chronicle/ExcerptTailer.java000077500000000000000000000026751226752375100307520ustar00rootroot00000000000000/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.openhft.chronicle; import org.jetbrains.annotations.NotNull; /** * @author peter.lawrey */ public interface ExcerptTailer extends ExcerptCommon { /** * Randomly select an Excerpt. * * @param l index to look up * @return true if this is a valid entries and not padding. */ boolean index(long l); /** * Wind to the next entry, no matter how many padded index entries you need to skip. If there is padding, the * index() can change even though this method might return false * * @return Is there a valid entry to be read. */ boolean nextIndex(); /** * Replay from the start. * * @return this Excerpt */ @NotNull ExcerptTailer toStart(); /** * Wind to the end. * * @return this Excerpt */ @NotNull ExcerptTailer toEnd(); } Chronicle-Queue-2.0.3/chronicle/src/main/java/net/openhft/chronicle/IndexedChronicle.java000077500000000000000000000104221226752375100313730ustar00rootroot00000000000000/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.openhft.chronicle; import net.openhft.lang.io.MappedFile; import org.jetbrains.annotations.NotNull; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.nio.ByteOrder; import java.nio.MappedByteBuffer; /** * @author peter.lawrey */ public class IndexedChronicle implements Chronicle { @NotNull final MappedFile indexFileCache; @NotNull final MappedFile dataFileCache; @NotNull final ChronicleConfig config; private final String basePath; // todo consider making volatile to help detect bugs in calling code. private long lastWrittenIndex = -1; public IndexedChronicle(@NotNull String basePath) throws FileNotFoundException { this(basePath, ChronicleConfig.DEFAULT); } public IndexedChronicle(@NotNull String basePath, @NotNull ChronicleConfig config) throws FileNotFoundException { this.basePath = basePath; this.config = config.clone(); File parentFile = new File(basePath).getParentFile(); if (parentFile != null) parentFile.mkdirs(); this.indexFileCache = new MappedFile(basePath + ".index", config.indexBlockSize()); this.dataFileCache = new MappedFile(basePath + ".data", config.dataBlockSize()); findTheLastIndex(); } public ChronicleConfig config() { return config; } public long findTheLastIndex() { return lastWrittenIndex = findTheLastIndex0(); } private long findTheLastIndex0() { long size = indexFileCache.size(); if (size <= 0) { return -1; } int indexBlockSize = config.indexBlockSize(); for (long block = size / indexBlockSize - 1; block >= 0; block--) { MappedByteBuffer mbb = null; try { mbb = indexFileCache.acquire(block).buffer(); } catch (IOException e) { continue; } mbb.order(ByteOrder.nativeOrder()); if (block > 0 && mbb.getLong(0) == 0) { continue; } int cacheLineSize = config.cacheLineSize(); for (int pos = 0; pos < indexBlockSize; pos += cacheLineSize) { // if the next line is blank if (pos + cacheLineSize >= indexBlockSize || mbb.getLong(pos + cacheLineSize) == 0) { // last cache line. int pos2 = 8; for (pos2 = 8; pos2 < cacheLineSize; pos2 += 4) { if (mbb.getInt(pos + pos2) == 0) break; } return (block * indexBlockSize + pos) / cacheLineSize * (cacheLineSize / 4 - 2) + pos2 / 4 - 3; } } return (block + 1) * indexBlockSize / cacheLineSize * (cacheLineSize / 4 - 2); } return -1; } @Override public long size() { return lastWrittenIndex + 1; } @Override public String name() { return basePath; } @Override public void close() throws IOException { this.indexFileCache.close(); this.dataFileCache.close(); } @NotNull @Override public Excerpt createExcerpt() throws IOException { return new NativeExcerpt(this); } @NotNull @Override public ExcerptTailer createTailer() throws IOException { return new NativeExcerptTailer(this); } @NotNull @Override public ExcerptAppender createAppender() throws IOException { return new NativeExcerptAppender(this); } @Override public long lastWrittenIndex() { return lastWrittenIndex; } void incrSize() { lastWrittenIndex++; } } Chronicle-Queue-2.0.3/chronicle/src/main/java/net/openhft/chronicle/IndexedChronicleCache.java000066400000000000000000000027071226752375100323230ustar00rootroot00000000000000/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.openhft.chronicle; import java.io.FileNotFoundException; import java.io.IOException; /** * User: peter.lawrey * Date: 26/09/13 * Time: 18:08 */ public class IndexedChronicleCache { private final String basePath; private IndexedChronicle chronicle; private int chronicleIndex = -1; public IndexedChronicleCache(String basePath) { this.basePath = basePath; } public IndexedChronicle acquireChronicle(int index) throws FileNotFoundException { if (index == chronicleIndex) return chronicle; chronicleIndex = index; String basePath2 = basePath + "/" + index; // System.out.println("Opening " + basePath2); return chronicle = new IndexedChronicle(basePath2); } public void close() throws IOException { if (chronicle != null) chronicle.close(); } } Chronicle-Queue-2.0.3/chronicle/src/main/java/net/openhft/chronicle/MapUtils.java000077500000000000000000000021201226752375100277160ustar00rootroot00000000000000/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.openhft.chronicle; import net.openhft.lang.io.MappedFile; import org.jetbrains.annotations.NotNull; import java.io.IOException; import java.nio.MappedByteBuffer; import java.nio.channels.FileChannel; /** * User: peter.lawrey Date: 13/08/13 Time: 18:58 */ public class MapUtils { @Deprecated() public static MappedByteBuffer getMap(@NotNull FileChannel fileChannel, long start, int size) throws IOException { return MappedFile.getMap(fileChannel, start, size); } } Chronicle-Queue-2.0.3/chronicle/src/main/java/net/openhft/chronicle/MappedFileCache.java000077500000000000000000000017051226752375100311220ustar00rootroot00000000000000/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.openhft.chronicle; import org.jetbrains.annotations.NotNull; import java.io.Closeable; import java.nio.MappedByteBuffer; /** * @author peter.lawrey */ public interface MappedFileCache extends Closeable { @NotNull MappedByteBuffer acquireBuffer(long index, boolean prefetch); void excerptUsed(); long size(); void close(); } Chronicle-Queue-2.0.3/chronicle/src/main/java/net/openhft/chronicle/NativeExcerpt.java000077500000000000000000000105341226752375100307510ustar00rootroot00000000000000/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.openhft.chronicle; import org.jetbrains.annotations.NotNull; import java.io.IOException; /** * @author peter.lawrey */ public class NativeExcerpt extends AbstractNativeExcerpt implements Excerpt { private boolean padding = true; public NativeExcerpt(@NotNull IndexedChronicle chronicle) throws IOException { super(chronicle); } public void startExcerpt(long capacity) { // if the capacity is to large, roll the previous entry, and there was one if (positionAddr + capacity > dataStartAddr + dataBlockSize) { // check we are the start of a block. checkNewIndexLine(); writePaddedEntry(); try { loadNextDataBuffer(); } catch (IOException e) { throw new IllegalStateException(e); } } // check we are the start of a block. checkNewIndexLine(); // update the soft limitAddr startAddr = positionAddr; limitAddr = positionAddr + capacity; finished = false; } private void writePaddedEntry() { int size = (int) (dataBlockSize + dataStartOffset - indexBaseForLine); assert size >= 0; if (size == 0) return; checkNewIndexLine(); writePaddingIndexEntry(size); indexPositionAddr += 4; } private void writePaddingIndexEntry(int size) { UNSAFE.putInt(indexPositionAddr, -size); } @Override public boolean index(long l) { try { return indexForRead(l); } catch (IOException e) { throw new IllegalStateException(e); } } @Override public void finish() { super.finish(); if (chronicle.config.synchronousMode()) { dataBuffer.force(); indexBuffer.force(); } } void checkNewIndexLine() { switch ((int) (indexPositionAddr & cacheLineMask)) { case 0: newIndexLine(); break; case 4: throw new AssertionError(); } } void newIndexLine() { // check we have a valid index if (indexPositionAddr >= indexStartAddr + indexBlockSize) { try { loadNextIndexBuffer(); } catch (IOException e) { throw new IllegalStateException(e); } } // sets the base address indexBaseForLine = positionAddr - dataStartAddr + dataStartOffset; assert indexBaseForLine >= 0 && indexBaseForLine < 1L << 48 : "dataPositionAtStartOfLine out of bounds, was " + indexBaseForLine; appendToIndex(); // System.out.println(Long.toHexString(indexPositionAddr - indexStartAddr + indexStart) + "=== " + dataPositionAtStartOfLine); indexPositionAddr += 8; } private void appendToIndex() { UNSAFE.putLong(indexPositionAddr, indexBaseForLine); } @NotNull @Override public Excerpt toStart() { index = -1; return this; } @NotNull @Override public Excerpt toEnd() { index = chronicle().size(); try { indexForRead(index); } catch (IOException e) { throw new IllegalStateException(e); } return this; } @Override public boolean nextIndex() { try { long index2 = index; if (indexForRead(index() + 1)) { return true; } else { // rewind on a failure index = index2; } if (wasPadding()) { index++; return indexForRead(index() + 1); } return false; } catch (IOException e) { return false; } } } Chronicle-Queue-2.0.3/chronicle/src/main/java/net/openhft/chronicle/NativeExcerptAppender.java000077500000000000000000000143511226752375100324310ustar00rootroot00000000000000/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.openhft.chronicle; import org.jetbrains.annotations.NotNull; import java.io.IOException; import java.util.ConcurrentModificationException; /** * @author peter.lawrey */ public class NativeExcerptAppender extends AbstractNativeExcerpt implements ExcerptAppender { private boolean nextSynchronous; public NativeExcerptAppender(@NotNull IndexedChronicle chronicle) throws IOException { super(chronicle); toEnd(); } @Override public void startExcerpt() { startExcerpt(chronicle.config().messageCapacity()); } public void startExcerpt(long capacity) { // in case there is more than one appender :P if (index != size()) { toEnd(); } if (capacity >= chronicle.config.dataBlockSize()) throw new IllegalArgumentException("Capacity too large " + capacity + " >= " + chronicle.config.dataBlockSize()); // if the capacity is to large, roll the previous entry, and there was one if (positionAddr + capacity > dataStartAddr + dataBlockSize) { // check we are the start of a block. checkNewIndexLine(); writePaddedEntry(); try { loadNextDataBuffer(); } catch (IOException e) { throw new IllegalStateException(e); } } // check we are the start of a block. checkNewIndexLine(); // update the soft limitAddr startAddr = positionAddr; limitAddr = positionAddr + capacity; finished = false; nextSynchronous = chronicle.config.synchronousMode(); } public void nextSynchronous(boolean nextSynchronous) { this.nextSynchronous = nextSynchronous; } public boolean nextSynchronous() { return nextSynchronous; } @Override public void addPaddedEntry() { // in case there is more than one appender :P if (index != lastWrittenIndex()) { toEnd(); } // check we are the start of a block. checkNewIndexLine(); writePaddedEntry(); try { loadNextDataBuffer(); } catch (IOException e) { throw new IllegalStateException(e); } // check we are the start of a block. checkNewIndexLine(); finished = true; } private void writePaddedEntry() { int size = (int) (dataBlockSize + dataStartOffset - indexBaseForLine); assert size >= 0; if (size == 0) return; appendIndexPaddingEntry(size); indexPositionAddr += 4; index++; chronicle.incrSize(); } private void appendIndexPaddingEntry(int size) { assert index < this.indexEntriesPerLine || UNSAFE.getLong(indexPositionAddr & ~cacheLineMask) != 0 : "index: " + index + ", no start of line set."; UNSAFE.putInt(indexPositionAddr, -size); } @Override public void finish() { super.finish(); if (index != chronicle.size()) throw new ConcurrentModificationException("Chronicle appended by more than one Appender at the same time, index=" + index + ", size=" + chronicle().size()); // push out the entry is available. This is what the reader polls. // System.out.println(Long.toHexString(indexPositionAddr - indexStartAddr + indexStart) + "= " + (int) (dataPosition() - dataPositionAtStartOfLine)); long offsetInBlock = positionAddr - dataStartAddr; assert offsetInBlock >= 0 && offsetInBlock <= dataBlockSize; int relativeOffset = (int) (dataStartOffset + offsetInBlock - indexBaseForLine); assert relativeOffset > 0; writeIndexEntry(relativeOffset); indexPositionAddr += 4; index++; chronicle.incrSize(); if ((indexPositionAddr & cacheLineMask) == 0 && indexPositionAddr - indexStartAddr < indexBlockSize) { indexBaseForLine += relativeOffset; appendStartOfLine(); } if (nextSynchronous) { assert dataBuffer != null; dataBuffer.force(); assert indexBuffer != null; indexBuffer.force(); } } private void writeIndexEntry(int relativeOffset) { UNSAFE.putOrderedInt(null, indexPositionAddr, relativeOffset); } @NotNull @Override public ExcerptAppender toEnd() { index = chronicle().size(); try { indexForAppender(index); } catch (IOException e) { throw new IllegalStateException(e); } return this; } void checkNewIndexLine() { switch ((int) (indexPositionAddr & cacheLineMask)) { case 0: newIndexLine(); break; case 4: throw new AssertionError(); } } void newIndexLine() { // check we have a valid index if (indexPositionAddr >= indexStartAddr + indexBlockSize) { try { loadNextIndexBuffer(); } catch (IOException e) { throw new IllegalStateException(e); } } // sets the base address indexBaseForLine = positionAddr - dataStartAddr + dataStartOffset; assert (index == 0 || indexBaseForLine > 0) && indexBaseForLine < 1L << 48 : "dataPositionAtStartOfLine out of bounds, was " + indexBaseForLine; appendStartOfLine(); } private void appendStartOfLine() { UNSAFE.putLong(indexPositionAddr, indexBaseForLine); // System.out.println(Long.toHexString(indexPositionAddr - indexStartAddr + indexStart) + "=== " + dataPositionAtStartOfLine); indexPositionAddr += 8; } } Chronicle-Queue-2.0.3/chronicle/src/main/java/net/openhft/chronicle/NativeExcerptTailer.java000077500000000000000000000110111226752375100321010ustar00rootroot00000000000000/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.openhft.chronicle; import org.jetbrains.annotations.NotNull; import java.io.IOException; /** * @author peter.lawrey */ public class NativeExcerptTailer extends AbstractNativeExcerpt implements ExcerptTailer { public static final long UNSIGNED_INT_MASK = 0xFFFFFFFFL; public NativeExcerptTailer(@NotNull IndexedChronicle chronicle) throws IOException { super(chronicle); } @Override public boolean index(long l) { try { return indexForRead(l); } catch (IOException e) { throw new IllegalStateException(e); } } @NotNull @Override public ExcerptTailer toEnd() { index = chronicle().size(); try { indexForRead(index); } catch (IOException e) { throw new IllegalStateException(e); } return this; } @NotNull @Override public ExcerptTailer toStart() { super.toStart(); return this; } public boolean nextIndex() { checkNextLine(); long offset = UNSAFE.getInt(null, indexPositionAddr); if (offset == 0) offset = UNSAFE.getIntVolatile(null, indexPositionAddr); // System.out.println(Long.toHexString(indexPositionAddr - indexStartAddr + indexStart) + " was " + offset); if (offset == 0) { return false; } index++; return nextIndex0(offset) || nextIndex1(); } private boolean nextIndex1() { long offset; checkNextLine(); offset = UNSAFE.getInt(null, indexPositionAddr); if (offset == 0) offset = UNSAFE.getIntVolatile(null, indexPositionAddr); // System.out.println(Long.toHexString(indexPositionAddr - indexStartAddr + indexStart) + " was " + offset); if (offset == 0) { return false; } index++; return nextIndex0(offset); } private void checkNextLine() { switch ((int) (indexPositionAddr & cacheLineMask)) { case 0: newIndexLine(); // skip the base until we have the offset. indexPositionAddr += 8; break; case 4: throw new AssertionError(); } } private void newIndexLine() { if (indexPositionAddr >= indexStartAddr + indexBlockSize) { try { loadNextIndexBuffer(); } catch (IOException e) { throw new IllegalStateException(e); } } } private boolean nextIndex0(long offset) { boolean present = true; padding = (offset < 0); if (padding) { present = false; offset = -offset; } checkNewIndexLine2(); startAddr = positionAddr = limitAddr; setLmitAddr(offset); assert limitAddr >= startAddr || (!present && limitAddr == startAddr); indexPositionAddr += 4; return present; } private void setLmitAddr(long offset) { long offsetInThisBuffer = indexBaseForLine + offset - dataStartOffset; if (offsetInThisBuffer > dataBlockSize) { try { loadNextDataBuffer(offsetInThisBuffer); } catch (IOException e) { throw new IllegalStateException(e); } offsetInThisBuffer = indexBaseForLine + offset - dataStartOffset; } assert offsetInThisBuffer >= 0 && offsetInThisBuffer <= dataBlockSize : "index: " + index + ", offsetInThisBuffer: " + offsetInThisBuffer; limitAddr = dataStartAddr + offsetInThisBuffer; } void checkNewIndexLine2() { if ((indexPositionAddr & cacheLineMask) == 8) { indexBaseForLine = UNSAFE.getLongVolatile(null, indexPositionAddr - 8); assert index <= indexEntriesPerLine || indexBaseForLine > 0 : "index: " + index + " indexBaseForLine: " + indexBaseForLine; setLmitAddr(0); } } } Chronicle-Queue-2.0.3/chronicle/src/main/java/net/openhft/chronicle/tcp/000077500000000000000000000000001226752375100261055ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle/src/main/java/net/openhft/chronicle/tcp/InProcessChronicleSink.java000077500000000000000000000212471226752375100333420ustar00rootroot00000000000000/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.openhft.chronicle.tcp; import net.openhft.chronicle.*; import net.openhft.chronicle.tools.WrappedExcerpt; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import java.io.EOFException; import java.io.IOException; import java.io.StreamCorruptedException; import java.net.InetSocketAddress; import java.net.SocketAddress; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.channels.SocketChannel; import java.util.logging.Level; import java.util.logging.Logger; /** * This listens to a ChronicleSource and copies new entries. This SInk can be any number of excerpt behind the source * and can be restart many times without losing data. *

* Can be used as a component with lower over head than ChronicleSink * * @author peter.lawrey */ public class InProcessChronicleSink implements Chronicle { @NotNull private final Chronicle chronicle; @NotNull private final SocketAddress address; private final ExcerptAppender excerpt; private final Logger logger; private volatile boolean closed = false; public InProcessChronicleSink(@NotNull Chronicle chronicle, String hostname, int port) throws IOException { this.chronicle = chronicle; this.address = new InetSocketAddress(hostname, port); logger = Logger.getLogger(getClass().getName() + '.' + chronicle); excerpt = chronicle.createAppender(); readBuffer = TcpUtil.createBuffer(256 * 1024, ByteOrder.nativeOrder()); } @Override public String name() { return chronicle.name(); } @NotNull @Override public Excerpt createExcerpt() throws IOException { return new SinkExcerpt(chronicle.createExcerpt()); } @NotNull @Override public ExcerptTailer createTailer() throws IOException { return new SinkExcerpt(chronicle.createTailer()); } @NotNull @Override public ExcerptAppender createAppender() throws IOException { throw new UnsupportedOperationException(); } @Override public long lastWrittenIndex() { return chronicle.lastWrittenIndex(); } @Override public long size() { return chronicle.size(); } private class SinkExcerpt extends WrappedExcerpt { @SuppressWarnings("unchecked") public SinkExcerpt(ExcerptCommon excerpt) throws IOException { super(excerpt); } @Override public boolean nextIndex() { return super.nextIndex() || readNext() && super.nextIndex(); } @Override public boolean index(long index) throws IndexOutOfBoundsException { if (super.index(index)) return true; return index >= 0 && readNext() && super.index(index); } } @Nullable private SocketChannel sc = null; private boolean scFirst = true; boolean readNext() { if (sc == null || !sc.isOpen()) { sc = createConnection(); scFirst = true; } return sc != null && readNextExcerpt(sc); } @Nullable private SocketChannel createConnection() { while (!closed) { try { readBuffer.clear(); readBuffer.limit(0); SocketChannel sc = SocketChannel.open(address); sc.socket().setReceiveBufferSize(256 * 1024); logger.info("Connected to " + address); ByteBuffer bb = ByteBuffer.allocate(8); bb.putLong(0, chronicle.lastWrittenIndex()); TcpUtil.writeAllOrEOF(sc, bb); return sc; } catch (IOException e) { if (logger.isLoggable(Level.FINE)) logger.log(Level.FINE, "Failed to connect to " + address + " retrying", e); else if (logger.isLoggable(Level.INFO)) logger.log(Level.INFO, "Failed to connect to " + address + " retrying " + e); } try { Thread.sleep(500); } catch (InterruptedException e) { Thread.currentThread().interrupt(); return null; } } return null; } private final ByteBuffer readBuffer; // minimum size private boolean readNextExcerpt(@NotNull SocketChannel sc) { try { if (closed) return false; if (readBuffer.remaining() < (scFirst ? TcpUtil.HEADER_SIZE : 4)) { if (readBuffer.remaining() == 0) readBuffer.clear(); else readBuffer.compact(); int minSize = scFirst ? 8 + 4 + 8 : 4 + 8; while (readBuffer.position() < minSize) { if (sc.read(readBuffer) < 0) { sc.close(); return false; } } readBuffer.flip(); } // System.out.println("rb " + readBuffer); if (scFirst) { long scIndex = readBuffer.getLong(); // System.out.println("ri " + scIndex); if (scIndex != chronicle.size()) throw new StreamCorruptedException("Expected index " + chronicle.size() + " but got " + scIndex); scFirst = false; } int size = readBuffer.getInt(); switch (size) { case InProcessChronicleSource.IN_SYNC_LEN: // System.out.println("... received inSync"); return false; case InProcessChronicleSource.PADDED_LEN: // System.out.println("... received padded"); excerpt.startExcerpt(((IndexedChronicle) chronicle).config().dataBlockSize() - 1); return true; default: break; } // System.out.println("size=" + size + " rb " + readBuffer); if (size > 128 << 20 || size < 0) throw new StreamCorruptedException("size was " + size); excerpt.startExcerpt(size); // perform a progressive copy of data. long remaining = size; int limit = readBuffer.limit(); int size2 = (int) Math.min(readBuffer.remaining(), remaining); remaining -= size2; readBuffer.limit(readBuffer.position() + size2); excerpt.write(readBuffer); // reset the limit; readBuffer.limit(limit); // needs more than one read. while (remaining > 0) { // System.out.println("++ read remaining "+remaining +" rb "+readBuffer); readBuffer.clear(); int size3 = (int) Math.min(readBuffer.capacity(), remaining); readBuffer.limit(size3); // System.out.println("... reading"); if (sc.read(readBuffer) < 0) throw new EOFException(); readBuffer.flip(); // System.out.println("r " + ChronicleTools.asString(bb)); remaining -= readBuffer.remaining(); excerpt.write(readBuffer); } excerpt.finish(); // System.out.println(" ri: " + excerpt.index()); } catch (IOException e) { if (logger.isLoggable(Level.FINE)) logger.log(Level.FINE, "Lost connection to " + address + " retrying", e); else if (logger.isLoggable(Level.INFO)) logger.log(Level.INFO, "Lost connection to " + address + " retrying " + e); try { sc.close(); } catch (IOException ignored) { } } return true; } void closeSocket(@Nullable SocketChannel sc) { if (sc != null) try { sc.close(); } catch (IOException e) { logger.warning("Error closing socket " + e); } } @Override public void close() { closed = true; closeSocket(sc); // chronicle.close(); } public ChronicleConfig config() { return ((IndexedChronicle) chronicle).config(); } } InProcessChronicleSource.java000077500000000000000000000251101226752375100336100ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle/src/main/java/net/openhft/chronicle/tcp/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.openhft.chronicle.tcp; import net.openhft.chronicle.*; import net.openhft.chronicle.tools.WrappedExcerpt; import net.openhft.lang.thread.NamedThreadFactory; import org.jetbrains.annotations.NotNull; import java.io.EOFException; import java.io.IOException; import java.net.InetSocketAddress; import java.net.SocketException; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.channels.ServerSocketChannel; import java.nio.channels.SocketChannel; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.logging.Level; import java.util.logging.Logger; /** * A Chronicle as a service to be replicated to any number of clients. Clients can restart from where ever they are up * to. *

* Can be used an in process component which wraps the underlying Chronicle and offers lower overhead than using * ChronicleSource * * @author peter.lawrey */ public class InProcessChronicleSource implements Chronicle { static final int IN_SYNC_LEN = -128; static final int PADDED_LEN = -127; static final long HEARTBEAT_INTERVAL_MS = 2500; private static final int MAX_MESSAGE = 128; @NotNull private final Chronicle chronicle; private final ServerSocketChannel server; @NotNull private final String name; @NotNull private final ExecutorService service; private final Logger logger; private final Object notifier = new Object(); private long busyWaitTimeNS = 100 * 1000; private volatile boolean closed = false; private long lastUnpausedNS = 0; public InProcessChronicleSource(@NotNull Chronicle chronicle, int port) throws IOException { this.chronicle = chronicle; server = ServerSocketChannel.open(); server.socket().setReuseAddress(true); server.socket().bind(new InetSocketAddress(port)); name = chronicle.name() + "@" + port; logger = Logger.getLogger(getClass().getName() + "." + name); service = Executors.newCachedThreadPool(new NamedThreadFactory(name, true)); service.execute(new Acceptor()); } private void pauseReset() { lastUnpausedNS = System.nanoTime(); } protected void pause() { if (lastUnpausedNS + busyWaitTimeNS > System.nanoTime()) return; try { synchronized (notifier) { notifier.wait(HEARTBEAT_INTERVAL_MS / 2); } } catch (InterruptedException ie) { logger.warning("Interrupt ignored"); } } void wakeSessionHandlers() { synchronized (notifier) { notifier.notifyAll(); } } @Override public String name() { return chronicle.name(); } @NotNull @Override public Excerpt createExcerpt() throws IOException { return new SourceExcerpt(chronicle.createExcerpt()); } @NotNull @Override public ExcerptTailer createTailer() throws IOException { return new SourceExcerpt(chronicle.createTailer()); } @NotNull @Override public ExcerptAppender createAppender() throws IOException { return new SourceExcerpt(chronicle.createAppender()); } @Override public long lastWrittenIndex() { return chronicle.lastWrittenIndex(); } @Override public long size() { return chronicle.size(); } @Override public void close() { closed = true; try { chronicle.close(); server.close(); } catch (IOException e) { logger.warning("Error closing server port " + e); } } public ChronicleConfig config() { return ((IndexedChronicle) chronicle).config(); } private class Acceptor implements Runnable { @Override public void run() { Thread.currentThread().setName(name + "-acceptor"); try { while (!closed) { SocketChannel socket = server.accept(); service.execute(new Handler(socket)); } } catch (IOException e) { if (!closed) logger.log(Level.SEVERE, "Acceptor dying", e); } finally { service.shutdown(); } } } class Handler implements Runnable { @NotNull private final SocketChannel socket; public Handler(@NotNull SocketChannel socket) throws SocketException { this.socket = socket; socket.socket().setSendBufferSize(256 * 1024); socket.socket().setTcpNoDelay(true); } @Override public void run() { try { long index = readIndex(socket); ExcerptTailer excerpt = chronicle.createTailer(); ByteBuffer bb = TcpUtil.createBuffer(1, ByteOrder.nativeOrder()); // minimum size long sendInSync = 0; boolean first = true; OUTER: while (!closed) { while (!excerpt.index(index)) { long now = System.currentTimeMillis(); if (excerpt.wasPadding()) { if (index >= 0) { bb.clear(); if (first) { bb.putLong(excerpt.index()); first = false; } bb.putInt(PADDED_LEN); bb.flip(); TcpUtil.writeAll(socket, bb); sendInSync = now + HEARTBEAT_INTERVAL_MS; } index++; continue; } // System.out.println("Waiting for " + index); if (sendInSync <= now && !first) { bb.clear(); bb.putInt(IN_SYNC_LEN); bb.flip(); TcpUtil.writeAll(socket, bb); sendInSync = now + HEARTBEAT_INTERVAL_MS; } pause(); if (closed) break OUTER; } pauseReset(); // System.out.println("Writing " + index); final long size = excerpt.capacity(); long remaining; bb.clear(); if (first) { // System.out.println("wi " + index); bb.putLong(excerpt.index()); first = false; remaining = size + TcpUtil.HEADER_SIZE; } else { remaining = size + 4; } bb.putInt((int) size); // for large objects send one at a time. if (size > bb.capacity() / 2) { while (remaining > 0) { int size2 = (int) Math.min(remaining, bb.capacity()); bb.limit(size2); excerpt.read(bb); bb.flip(); // System.out.println("w " + ChronicleTools.asString(bb)); remaining -= bb.remaining(); TcpUtil.writeAll(socket, bb); } } else { bb.limit((int) remaining); excerpt.read(bb); int count = 1; while (excerpt.index(index + 1) && count++ < MAX_MESSAGE) { if (excerpt.wasPadding()) { index++; continue; } if (excerpt.remaining() + 4 >= bb.capacity() - bb.position()) break; // if there is free space, copy another one. int size2 = (int) excerpt.capacity(); // System.out.println("W+ "+size); bb.limit(bb.position() + size2 + 4); bb.putInt(size2); excerpt.read(bb); index++; } bb.flip(); // System.out.println("W " + size + " wb " + bb); TcpUtil.writeAll(socket, bb); } if (bb.remaining() > 0) throw new EOFException("Failed to send index=" + index); index++; sendInSync = 0; // if (index % 20000 == 0) // System.out.println(System.currentTimeMillis() + ": wrote " + index); } } catch (Exception e) { if (!closed) { String msg = e.getMessage(); if (msg != null && (msg.contains("reset by peer") || msg.contains("Broken pipe") || msg.contains("was aborted by"))) logger.log(Level.INFO, "Connect " + socket + " closed from the other end " + e); else logger.log(Level.INFO, "Connect " + socket + " died", e); } } } private long readIndex(@NotNull SocketChannel socket) throws IOException { ByteBuffer bb = ByteBuffer.allocate(8); TcpUtil.readFullyOrEOF(socket, bb); return bb.getLong(0); } } class SourceExcerpt extends WrappedExcerpt { public SourceExcerpt(ExcerptCommon excerptCommon) { super(excerptCommon); } @Override public void finish() { super.finish(); wakeSessionHandlers(); // System.out.println("Wrote " + index()); } } } Chronicle-Queue-2.0.3/chronicle/src/main/java/net/openhft/chronicle/tcp/TcpUtil.java000066400000000000000000000040361226752375100303370ustar00rootroot00000000000000/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.openhft.chronicle.tcp; import org.jetbrains.annotations.NotNull; import java.io.EOFException; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.channels.SocketChannel; /** * @author peter.lawrey */ public enum TcpUtil { ; static final int HEADER_SIZE = 12; static final int INITIAL_BUFFER_SIZE = 64 * 1024; public static ByteBuffer createBuffer(int minSize, ByteOrder byteOrder) { int newSize = (minSize + INITIAL_BUFFER_SIZE - 1) / INITIAL_BUFFER_SIZE * INITIAL_BUFFER_SIZE; return ByteBuffer.allocateDirect(newSize).order(byteOrder); } public static void writeAllOrEOF(@NotNull SocketChannel sc, @NotNull ByteBuffer bb) throws IOException { writeAll(sc, bb); if (bb.remaining() > 0) throw new EOFException(); } public static void writeAll(@NotNull SocketChannel sc, @NotNull ByteBuffer bb) throws IOException { while (bb.remaining() > 0) if (sc.write(bb) < 0) break; } public static void readFullyOrEOF(@NotNull SocketChannel socket, @NotNull ByteBuffer bb) throws IOException { readAvailable(socket, bb); if (bb.remaining() > 0) throw new EOFException(); } public static void readAvailable(@NotNull SocketChannel socket, @NotNull ByteBuffer bb) throws IOException { while (bb.remaining() > 0) if (socket.read(bb) < 0) break; } } Chronicle-Queue-2.0.3/chronicle/src/main/java/net/openhft/chronicle/tools/000077500000000000000000000000001226752375100264575ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle/src/main/java/net/openhft/chronicle/tools/ChronicleIndexReader.java000066400000000000000000000042461226752375100333510ustar00rootroot00000000000000/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.openhft.chronicle.tools; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.channels.FileChannel; /** * User: peter.lawrey Date: 16/08/13 Time: 12:30 */ public class ChronicleIndexReader { static final boolean HEX = Boolean.getBoolean("hex"); public static void main(String... args) throws IOException { int zeros = 0; FileChannel fc; try { fc = new FileInputStream(args[0]).getChannel(); } catch (FileNotFoundException e) { System.err.println(e); return; } int count = 0; ByteBuffer buffer = ByteBuffer.allocateDirect(4096).order(ByteOrder.nativeOrder()); while (fc.read(buffer) > 0) { for (int i = 0; i < buffer.capacity(); i += 4 * 16) { long indexStart = buffer.getLong(i); if (indexStart == 0 && zeros++ > 2) { continue; } System.out.print(count + ": "); count += 14; System.out.print(HEX ? Long.toHexString(indexStart) : String.valueOf(indexStart)); for (int j = i + 8; j < i + 64; j += 4) { System.out.print(' '); int offset = buffer.getInt(j); System.out.print(HEX ? Integer.toHexString(offset) : String.valueOf(offset)); } System.out.println(); } buffer.clear(); } fc.close(); } } Chronicle-Queue-2.0.3/chronicle/src/main/java/net/openhft/chronicle/tools/ChronicleMasterReader.java000066400000000000000000000041341226752375100335310ustar00rootroot00000000000000/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.openhft.chronicle.tools; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.channels.FileChannel; /** * User: peter.lawrey Date: 16/08/13 Time: 12:30 */ public class ChronicleMasterReader { static final boolean HEX = Boolean.getBoolean("hex"); public static void main(String... args) throws IOException { int zeros = 0; FileChannel fc; try { fc = new FileInputStream(args[0]).getChannel(); } catch (FileNotFoundException e) { System.err.println(e); return; } ByteBuffer buffer = ByteBuffer.allocateDirect(4096).order(ByteOrder.nativeOrder()); while (fc.read(buffer) > 0) { for (int i = 0; i < buffer.capacity(); i += 4 * 16) { long indexStart = buffer.getInt(i) & 0xFFFFFFFFL; if (indexStart == 0 && zeros++ > 2) { continue; } System.out.print(HEX ? Long.toHexString(indexStart) : String.valueOf(indexStart)); for (int j = i + 4; j < i + 64; j += 4) { System.out.print(' '); long offset = buffer.getInt(j) & 0xFFFFFFFFL; System.out.print(HEX ? Long.toHexString(offset) : String.valueOf(offset)); } System.out.println(); } buffer.clear(); } fc.close(); } } Chronicle-Queue-2.0.3/chronicle/src/main/java/net/openhft/chronicle/tools/ChronicleReader.java000077500000000000000000000046551226752375100323700ustar00rootroot00000000000000/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.openhft.chronicle.tools; import net.openhft.chronicle.ExcerptTailer; import net.openhft.chronicle.IndexedChronicle; import org.jetbrains.annotations.NotNull; import java.io.IOException; /** * Display records in a Chronicle in a text form. * * @author peter.lawrey */ public enum ChronicleReader { ; public static void main(@NotNull String... args) throws IOException, InterruptedException { if (args.length < 1) { System.err.println("Usage: java " + ChronicleReader.class.getName() + " {chronicle-base-path} [from-index]"); System.exit(-1); } String basePath = args[0]; long index = args.length > 1 ? Long.parseLong(args[1]) : 0L; IndexedChronicle ic = new IndexedChronicle(basePath); ExcerptTailer excerpt = ic.createTailer(); //noinspection InfiniteLoopStatement while (true) { while (!excerpt.nextIndex()) //noinspection BusyWait Thread.sleep(50); System.out.print(index + ": "); int nullCount = 0; while (excerpt.remaining() > 0) { char ch = (char) excerpt.readUnsignedByte(); if (ch == 0) { nullCount++; continue; } if (nullCount > 0) System.out.print(" " + nullCount + "*\\0"); nullCount = 0; if (ch < ' ') System.out.print("^" + (char) (ch + '@')); else if (ch > 126) System.out.print("\\x" + Integer.toHexString(ch)); else System.out.print(ch); } if (nullCount > 0) System.out.print(" " + nullCount + "*\\0"); System.out.println(); index++; } } } Chronicle-Queue-2.0.3/chronicle/src/main/java/net/openhft/chronicle/tools/ChronicleTools.java000077500000000000000000000131061226752375100322550ustar00rootroot00000000000000/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.openhft.chronicle.tools; import net.openhft.chronicle.*; import net.openhft.lang.io.IOTools; import org.jetbrains.annotations.NotNull; import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; import java.util.LinkedHashSet; import java.util.Set; /** * @author peter.lawrey */ public enum ChronicleTools { ; /** * Delete a chronicle now and on exit, for testing * * @param basePath of the chronicle */ public static void deleteOnExit(String basePath) { for (String name : new String[]{basePath + ".data", basePath + ".index"}) { File file = new File(name); // noinspection ResultOfMethodCallIgnored file.delete(); file.deleteOnExit(); } } public static void deleteDirOnExit(String dirPath) { DeleteStatic.INSTANCE.add(dirPath); } @NotNull public static String asString(@NotNull ByteBuffer bb) { StringBuilder sb = new StringBuilder(); for (int i = bb.position(); i < bb.limit(); i++) { byte b = bb.get(i); if (b < ' ') { int h = b & 0xFF; if (h < 16) sb.append('0'); sb.append(Integer.toHexString(h)); } else { sb.append(' ').append((char) b); } } return sb.toString(); } /** * Take a text copy of the contents of the Excerpt without changing it's position. Can be called in the debugger. * * @param excerpt to get text from * @return 256 bytes as text with `.` replacing special bytes. */ @NotNull public static String asString(@NotNull ExcerptCommon excerpt) { return asString(excerpt, excerpt.position()); } /** * Take a text copy of the contents of the Excerpt without changing it's position. Can be called in the debugger. * * @param excerpt to get text from * @param position the position to get text from * @return up to 1024 bytes as text with `.` replacing special bytes. */ @NotNull public static String asString(@NotNull ExcerptCommon excerpt, long position) { return asString(excerpt, position, 1024); } /** * Take a text copy of the contents of the Excerpt without changing it's position. Can be called in the debugger. * * @param excerpt to get text from * @param position the position to get text from * @param length the maximum length * @return length bytes as text with `.` replacing special bytes. */ @NotNull public static String asString(@NotNull ExcerptCommon excerpt, long position, long length) { long limit = Math.min(position + length, excerpt.capacity()); StringBuilder sb = new StringBuilder((int) (limit - position)); for (long i = position; i < limit; i++) { char ch = (char) excerpt.readUnsignedByte(i); if (ch < ' ' || ch > 127) ch = '.'; sb.append(ch); } return sb.toString(); } enum DeleteStatic { INSTANCE; @SuppressWarnings("TypeMayBeWeakened") final Set toDeleteList = new LinkedHashSet(); { Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() { @Override public void run() { for (String dir : toDeleteList) { // TODO no idea why the // is needed. Appears to be a bug in the JVM. System.out.println("Deleting " + dir.replaceAll("/", "//")); IOTools.deleteDir(dir); } } })); } synchronized void add(String dirPath) { IOTools.deleteDir(dirPath); toDeleteList.add(dirPath); } } public static void warmup() { boolean done = ChronicleWarmup.DONE; } } class ChronicleWarmup { public static final boolean DONE; public static final int WARMUP_ITER = 200000; private static final String TMP = System.getProperty("java.io.tmpdir"); static { ChronicleConfig cc = ChronicleConfig.DEFAULT.clone(); cc.dataBlockSize(64); cc.indexBlockSize(64); String basePath = TMP + "/warmup-" + Math.random(); ChronicleTools.deleteOnExit(basePath); try { IndexedChronicle ic = new IndexedChronicle(basePath, cc); ExcerptAppender appender = ic.createAppender(); ExcerptTailer tailer = ic.createTailer(); for (int i = 0; i < WARMUP_ITER; i++) { appender.startExcerpt(); appender.writeInt(i); appender.finish(); boolean b = tailer.nextIndex() || tailer.nextIndex(); tailer.readInt(); tailer.finish(); } ic.close(); System.gc(); DONE = true; } catch (IOException e) { throw new AssertionError(); } } } DailingRollingIndexReader.java000077500000000000000000000026751226752375100342710ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle/src/main/java/net/openhft/chronicle/tools/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.openhft.chronicle.tools; import java.io.File; import java.io.PrintWriter; import java.io.StringWriter; /** * @author peter.lawrey */ public enum DailingRollingIndexReader { ; public static void dumpMaster(File dir, PrintWriter writer) { MasterIndexFile mif = new MasterIndexFile(new File(dir, "master")); writer.println(dir.getName() + ": " + mif.size()); for (int i = 0; i < mif.size(); i++) writer.println("\t" + i + "\t" + mif.filenameFor(i)); mif.close(); } public static String masterToString(File dir) { StringWriter sw = new StringWriter(); dumpMaster(dir, new PrintWriter(sw)); return sw.toString(); } public static void main(String... args) { PrintWriter pw = new PrintWriter(System.out); dumpMaster(new File(args[0]), pw); } } Chronicle-Queue-2.0.3/chronicle/src/main/java/net/openhft/chronicle/tools/DailingRollingReader.java000077500000000000000000000053211226752375100333470ustar00rootroot00000000000000/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.openhft.chronicle.tools; import java.io.FileInputStream; import java.io.IOException; import java.io.PrintWriter; import java.io.Writer; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.channels.FileChannel; /** * @author peter.lawrey */ public class DailingRollingReader { public static void main(String[] args) throws IOException { dumpData(args[0], new PrintWriter(System.out)); } public static void dumpData(String filename, Writer writer) throws IOException { PrintWriter pw = writer instanceof PrintWriter ? ((PrintWriter) writer) : new PrintWriter(writer); FileChannel fc = new FileInputStream(filename).getChannel(); try { ByteBuffer bb = ByteBuffer.allocateDirect(4096).order(ByteOrder.nativeOrder()); long index = 0; int size = 0; while (true) { bb.clear(); fc.read(bb); bb.flip(); while (bb.remaining() > 0) { // write the remaining data. while (bb.remaining() > 0 && size > 0) { byte b = bb.get(); if (b < ' ' || b > 126) b = '.'; pw.print((char) b); size--; } if (size == 0) { // skip the padding. while ((bb.position() & 3) != 0) bb.get(); if (bb.remaining() >= 4) { size = bb.getInt(); pw.println(); if (size == 0) { pw.println("Exceprts " + index); return; } pw.print(index++); pw.print(":"); pw.print(size); pw.print(":"); } } } } } finally { fc.close(); } } } Chronicle-Queue-2.0.3/chronicle/src/main/java/net/openhft/chronicle/tools/MasterIndexFile.java000077500000000000000000000042401226752375100323500ustar00rootroot00000000000000/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.openhft.chronicle.tools; import java.io.*; import java.util.ArrayList; import java.util.List; /** * @author peter.lawrey */ public class MasterIndexFile { final File file; final List filenames = new ArrayList(); public MasterIndexFile(File file) { this.file = file; loadFile(); } private void loadFile() { try { if (!file.exists()) return; BufferedReader br = new BufferedReader(new FileReader(file)); try { for (String line; (line = br.readLine()) != null; ) { filenames.add(line.trim().isEmpty() ? null : line); } } finally { br.close(); } } catch (FileNotFoundException fnfe) { throw new AssertionError(fnfe); } catch (IOException ioe) { ioe.printStackTrace(); } } public String filenameFor(int index) { if (filenames.size() <= index) return null; return filenames.get(index); } public int append(String filenameAdded) throws IOException { int index = filenames.indexOf(filenameAdded); if (index >= 0) return index; filenames.add(filenameAdded); FileWriter fileWriter = new FileWriter(file); fileWriter.append(filenameAdded).append("\n"); fileWriter.close(); return filenames.size() - 1; } public void close() { // nothing to do. } public int size() { return filenames.size(); } } Chronicle-Queue-2.0.3/chronicle/src/main/java/net/openhft/chronicle/tools/WrappedExcerpt.java000066400000000000000000000573721226752375100322750ustar00rootroot00000000000000/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.openhft.chronicle.tools; import net.openhft.chronicle.*; import net.openhft.lang.io.*; import net.openhft.lang.io.serialization.BytesMarshallerFactory; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import java.io.InputStream; import java.io.OutputStream; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.util.Collection; import java.util.Map; /** * @author peter.lawrey */ public class WrappedExcerpt implements ExcerptTailer, ExcerptAppender, Excerpt { private ExcerptTailer tailer; private ExcerptAppender appender; private ExcerptCommon common; private Excerpt excerpt; public WrappedExcerpt(ExcerptCommon excerptCommon) { setExcerpt(excerptCommon); } protected void setExcerpt(ExcerptCommon excerptCommon) { tailer = excerptCommon instanceof ExcerptTailer ? (ExcerptTailer) excerptCommon : null; appender = excerptCommon instanceof ExcerptAppender ? (ExcerptAppender) excerptCommon : null; excerpt = excerptCommon instanceof Excerpt ? (Excerpt) excerptCommon : null; common = excerptCommon; } @Override public > E parseEnum(@NotNull Class eClass, @NotNull StopCharTester tester) { return common.parseEnum(eClass, tester); } @Override public void reset() { common.reset(); } @Override public void readObject(Object object, int start, int end) { common.readObject(object, start, end); } @Override public void writeObject(Object object, int start, int end) { common.writeObject(object, start, end); } @Override public Chronicle chronicle() { return common.chronicle(); } @Override public long size() { return common.size(); } @Override public boolean nextIndex() { return tailer.nextIndex(); } @Override public boolean index(long index) throws IndexOutOfBoundsException { return excerpt == null ? tailer.index(index) : excerpt.index(index); } @Override public void startExcerpt() { appender.startExcerpt(); } @Override public void startExcerpt(long capacity) { appender.startExcerpt(capacity); } @Override public void addPaddedEntry() { appender.addPaddedEntry(); } @Override public boolean nextSynchronous() { return appender.nextSynchronous(); } @Override public void nextSynchronous(boolean nextSynchronous) { appender.nextSynchronous(); } @Override public void finish() { common.finish(); } @Override public long index() { return common.index(); } @Override public long position() { return common.position(); } @Override public Boolean parseBoolean(@NotNull StopCharTester tester) { return common.parseBoolean(tester); } @Override public long capacity() { return common.capacity(); } @Override public long remaining() { return common.remaining(); } @Override public void readFully(@NotNull byte[] bytes) { common.readFully(bytes); } @Override public int skipBytes(int n) { return common.skipBytes(n); } @Override public void readFully(@NotNull byte[] b, int off, int len) { common.readFully(b, off, len); } @Override public boolean readBoolean() { return common.readBoolean(); } @Override public boolean readBoolean(long offset) { return common.readBoolean(offset); } @Override public int readUnsignedByte() { return common.readUnsignedByte(); } @Override public int readUnsignedByte(long offset) { return common.readUnsignedByte(offset); } @Override public int readUnsignedShort() { return common.readUnsignedShort(); } @Override public int readUnsignedShort(long offset) { return common.readUnsignedShort(offset); } @Override public String readLine() { return common.readLine(); } @NotNull @Override public String readUTF() { return common.readUTF(); } @Nullable @Override public String readUTFΔ() { return common.readUTFΔ(); } @Nullable @Override public String readUTFΔ(long offset) throws IllegalStateException { return common.readUTFΔ(offset); } @Override public boolean readUTFΔ(@NotNull StringBuilder stringBuilder) { return common.readUTFΔ(stringBuilder); } @NotNull @Override public String parseUTF(@NotNull StopCharTester tester) { return common.parseUTF(tester); } @Override public void parseUTF(@NotNull StringBuilder builder, @NotNull StopCharTester tester) { common.parseUTF(builder, tester); } @Override public short readCompactShort() { return common.readCompactShort(); } @Override public int readCompactUnsignedShort() { return common.readCompactUnsignedShort(); } @Override public int readInt24() { return common.readInt24(); } @Override public int readInt24(long offset) { return common.readInt24(offset); } @Override public long readUnsignedInt() { return common.readUnsignedInt(); } @Override public long readUnsignedInt(long offset) { return common.readUnsignedInt(offset); } @Override public int readCompactInt() { return common.readCompactInt(); } @Override public long readCompactUnsignedInt() { return common.readCompactUnsignedInt(); } @Override public long readInt48() { return common.readInt48(); } @Override public long readInt48(long offset) { return common.readInt48(offset); } @Override public long readCompactLong() { return common.readCompactLong(); } @Override public long readStopBit() { return common.readStopBit(); } @Override public double readCompactDouble() { return common.readCompactDouble(); } @Override public void read(@NotNull ByteBuffer bb) { common.read(bb); } @Override public void write(byte[] bytes) { common.write(bytes); } @Override public void writeBoolean(boolean v) { common.writeBoolean(v); } @Override public void writeBoolean(long offset, boolean v) { common.writeBoolean(offset, v); } @Override public void writeBytes(@NotNull String s) { common.writeBytes(s); } @Override public void writeChars(@NotNull String s) { common.writeChars(s); } @Override public void writeUTF(@NotNull String s) { common.writeUTF(s); } @Override public void writeUTFΔ(CharSequence str) { common.writeUTFΔ(str); } @Override public void writeUTFΔ(long offset, int maxSize, @Nullable CharSequence s) throws IllegalStateException { common.writeUTFΔ(offset, maxSize, s); } @Override public void writeByte(int v) { common.writeByte(v); } @Override public void writeUnsignedByte(int v) { common.writeUnsignedByte(v); } @Override public void writeUnsignedByte(long offset, int v) { common.writeUnsignedByte(offset, v); } @Override public void write(long offset, byte[] bytes) { common.write(offset, bytes); } @Override public void write(byte[] bytes, int off, int len) { common.write(bytes, off, len); } @Override public void writeUnsignedShort(int v) { common.writeUnsignedShort(v); } @Override public void writeUnsignedShort(long offset, int v) { common.writeUnsignedShort(offset, v); } @Override public void writeCompactShort(int v) { common.writeCompactShort(v); } @Override public void writeCompactUnsignedShort(int v) { common.writeCompactUnsignedShort(v); } @Override public void writeInt24(int v) { common.writeInt24(v); } @Override public void writeInt24(long offset, int v) { common.writeInt24(offset, v); } @Override public void writeUnsignedInt(long v) { common.writeUnsignedInt(v); } @Override public void writeUnsignedInt(long offset, long v) { common.writeUnsignedInt(offset, v); } @Override public void writeCompactInt(int v) { common.writeCompactInt(v); } @Override public void writeCompactUnsignedInt(long v) { common.writeCompactUnsignedInt(v); } @Override public void writeInt48(long v) { common.writeInt48(v); } @Override public void writeInt48(long offset, long v) { common.writeInt48(offset, v); } @Override public void writeCompactLong(long v) { common.writeCompactLong(v); } @Override public void writeCompactDouble(double v) { common.writeCompactDouble(v); } @Override public void write(@NotNull ByteBuffer bb) { common.write(bb); } @NotNull @Override public ByteStringAppender append(@NotNull CharSequence s) { common.append(s); return this; } @NotNull @Override public ByteStringAppender append(@NotNull CharSequence s, int start, int end) { common.append(s, start, end); return this; } @NotNull @Override public ByteStringAppender append(@NotNull Enum value) { common.append(value); return this; } @NotNull @Override public ByteStringAppender append(boolean b) { common.append(b); return this; } @NotNull @Override public ByteStringAppender append(char c) { common.append(c); return this; } @NotNull @Override public ByteStringAppender append(int num) { common.append(num); return this; } @NotNull @Override public ByteStringAppender append(long num) { common.append(num); return this; } @NotNull @Override public ByteStringAppender append(double d) { common.append(d); return this; } @NotNull @Override public ByteStringAppender append(double d, int precision) { common.append(d, precision); return this; } @NotNull @Override public ByteStringAppender append(@NotNull MutableDecimal md) { common.append(md); return this; } @Override public double parseDouble() { return common.parseDouble(); } @Override public long parseLong() { return common.parseLong(); } @NotNull @Override public InputStream inputStream() { return common.inputStream(); } @NotNull @Override public OutputStream outputStream() { return common.outputStream(); } @Override public void writeEnum(E o) { common.writeEnum(o); } @Override public E readEnum(@NotNull Class aClass) { return common.readEnum(aClass); } @Override public void writeMap(@NotNull Map map) { common.writeMap(map); } @Override public Map readMap(@NotNull Map map, @NotNull Class kClass, @NotNull Class vClass) { return common.readMap(map, kClass, vClass); } @Override public byte readByte() { return common.readByte(); } @Override public byte readByte(long offset) { return common.readByte(offset); } @Override public short readShort() { return common.readShort(); } @Override public short readShort(long offset) { return common.readShort(offset); } @Override public char readChar() { return common.readChar(); } @Override public char readChar(long offset) { return common.readChar(offset); } @Override public int readInt() { return common.readInt(); } @Override public int readInt(long offset) { return common.readInt(offset); } @Override public long readLong() { return common.readLong(); } @Override public long readLong(long offset) { return common.readLong(offset); } @Override public float readFloat() { return common.readFloat(); } @Override public float readFloat(long offset) { return common.readFloat(offset); } @Override public double readDouble() { return common.readDouble(); } @Override public double readDouble(long offset) { return common.readDouble(offset); } @Override public void write(int b) { common.write(b); } @Override public void writeByte(long offset, int b) { common.writeByte(offset, b); } @Override public void writeShort(int v) { common.writeShort(v); } @Override public void writeShort(long offset, int v) { common.writeShort(offset, v); } @Override public void writeChar(int v) { common.writeChar(v); } @Override public void writeChar(long offset, int v) { common.writeChar(offset, v); } @Override public void writeInt(int v) { common.writeInt(v); } @Override public void writeInt(long offset, int v) { common.writeInt(offset, v); } @Override public void writeLong(long v) { common.writeLong(v); } @Override public void writeLong(long offset, long v) { common.writeLong(offset, v); } @Override public void writeStopBit(long n) { common.writeStopBit(n); } @Override public void writeFloat(float v) { common.writeFloat(v); } @Override public void writeFloat(long offset, float v) { common.writeFloat(offset, v); } @Override public void writeDouble(double v) { common.writeDouble(v); } @Override public void writeDouble(long offset, double v) { common.writeDouble(offset, v); } @Nullable @Override public Object readObject() { return common.readObject(); } @Override public T readObject(Class tClass) throws IllegalStateException { return common.readObject(tClass); } @Override public int read() { return common.read(); } @Override public int read(@NotNull byte[] bytes) { return common.read(bytes); } @Override public int read(@NotNull byte[] bytes, int off, int len) { return common.read(bytes, off, len); } @Override public long skip(long n) { return common.skip(n); } @Override public int available() { return common.available(); } @Override public void close() { try { common.close(); } catch (Exception ignored) { } } @Override public void writeObject(Object obj) { common.writeObject(obj); } @Override public void flush() { common.flush(); } @Override public void writeList(@NotNull Collection list) { common.writeList(list); } @Override public void readList(@NotNull Collection list, @NotNull Class eClass) { common.readList(list, eClass); } @Override public long lastWrittenIndex() { return common.lastWrittenIndex(); } @Override public boolean stepBackAndSkipTo(@NotNull StopCharTester tester) { return common.stepBackAndSkipTo(tester); } @Override public boolean skipTo(@NotNull StopCharTester tester) { return common.skipTo(tester); } @NotNull @Override public MutableDecimal parseDecimal(@NotNull MutableDecimal decimal) { return common.parseDecimal(decimal); } @NotNull @Override public Excerpt toStart() { if (tailer == null) excerpt.toStart(); else tailer.toStart(); return this; } @NotNull @Override public WrappedExcerpt toEnd() { common.toEnd(); return this; } @Override public boolean isFinished() { return common.isFinished(); } @Override public boolean wasPadding() { return common.wasPadding(); } @NotNull @Override public ByteStringAppender appendTimeMillis(long timeInMS) { common.appendTimeMillis(timeInMS); return this; } @NotNull @Override public ByteStringAppender appendDateMillis(long timeInMS) { common.appendDateMillis(timeInMS); return this; } @NotNull @Override public ByteStringAppender appendDateTimeMillis(long timeInMS) { common.appendDateTimeMillis(timeInMS); return this; } @NotNull @Override public ByteStringAppender append(@NotNull Iterable list, @NotNull CharSequence seperator) { common.append(list, seperator); return this; } @Override public int readVolatileInt() { return common.readVolatileInt(); } @Override public int readVolatileInt(long offset) { return common.readVolatileInt(offset); } @Override public long readVolatileLong() { return common.readVolatileLong(); } @Override public long readVolatileLong(long offset) { return common.readVolatileLong(offset); } @Override public void writeOrderedInt(int v) { common.writeOrderedInt(v); } @Override public void writeOrderedInt(long offset, int v) { common.writeOrderedInt(offset, v); } @Override public boolean compareAndSwapInt(long offset, int expected, int x) { return common.compareAndSwapInt(offset, expected, x); } @Override public int getAndAdd(long offset, int delta) { return common.getAndAdd(offset, delta); } @Override public int addAndGetInt(long offset, int delta) { return common.addAndGetInt(offset, delta); } @Override public void writeOrderedLong(long v) { common.writeOrderedLong(v); } @Override public void writeOrderedLong(long offset, long v) { common.writeOrderedLong(offset, v); } @Override public boolean compareAndSwapLong(long offset, long expected, long x) { return common.compareAndSwapLong(offset, expected, x); } @Override public void position(long position) { common.position(position); } @NotNull @Override public ByteOrder byteOrder() { return common.byteOrder(); } @NotNull @Override public BytesMarshallerFactory bytesMarshallerFactory() { return common.bytesMarshallerFactory(); } @Override public void checkEndOfBuffer() throws IndexOutOfBoundsException { common.checkEndOfBuffer(); } @Override public boolean tryLockInt(long offset) { return common.tryLockInt(offset); } @Override public boolean tryLockNanosInt(long offset, long nanos) { return common.tryLockNanosInt(offset, nanos); } @Override public void busyLockInt(long offset) throws InterruptedException, IllegalStateException { common.busyLockInt(offset); } @Override public void unlockInt(long offset) throws IllegalStateException { common.unlockInt(offset); } @Override public boolean tryLockLong(long offset) { return common.tryLockLong(offset); } @Override public boolean tryLockNanosLong(long offset, long nanos) { return common.tryLockNanosLong(offset, nanos); } @Override public void busyLockLong(long offset) throws InterruptedException, IllegalStateException { common.busyLockLong(offset); } @Override public void unlockLong(long offset) throws IllegalStateException { common.unlockLong(offset); } @Override public float readVolatileFloat(long offset) { return common.readVolatileFloat(offset); } @Override public double readVolatileDouble(long offset) { return common.readVolatileDouble(offset); } @Override public void writeOrderedFloat(long offset, float v) { common.writeOrderedFloat(offset, v); } @Override public void writeOrderedDouble(long offset, double v) { common.writeOrderedDouble(offset, v); } @Override public byte addByte(long offset, byte b) { return common.addByte(offset, b); } @Override public int addUnsignedByte(long offset, int i) { return common.addUnsignedByte(offset, i); } @Override public short addShort(long offset, short s) { return common.addShort(offset, s); } @Override public int addUnsignedShort(long offset, int i) { return common.addUnsignedShort(offset, i); } @Override public int addInt(long offset, int i) { return common.addInt(offset, i); } @Override public long addUnsignedInt(long offset, long i) { return common.addUnsignedInt(offset, i); } @Override public long addLong(long offset, long i) { return common.addLong(offset, i); } @Override public float addFloat(long offset, float f) { return common.addFloat(offset, f); } @Override public double addDouble(long offset, double d) { return common.addDouble(offset, d); } @Override public int addAtomicInt(long offset, int i) { return common.addAtomicInt(offset, i); } @Override public long addAtomicLong(long offset, long l) { return common.addAtomicLong(offset, l); } @Override public float addAtomicFloat(long offset, float f) { return common.addAtomicFloat(offset, f); } @Override public double addAtomicDouble(long offset, double d) { return common.addAtomicDouble(offset, d); } @Override public long findMatch(@NotNull ExcerptComparator comparator) { return excerpt.findMatch(comparator); } @Override public void findRange(@NotNull long[] startEnd, @NotNull ExcerptComparator comparator) { excerpt.findRange(startEnd, comparator); } @Deprecated @Override public void writeStartToPosition(Bytes bytes) { common.writeStartToPosition(bytes); } @NotNull @Override public ByteStringAppender append(long l, int base) { common.append(l, base); return this; } @Override public long parseLong(int base) { return common.parseLong(base); } @Override public void write(BytesCommon bytes, long position, long length) { common.write(bytes, position, length); } @Override public void readMarshallable(@NotNull Bytes in) throws IllegalStateException { common.readMarshallable(in); } @Override public void writeMarshallable(@NotNull Bytes out) { common.writeMarshallable(out); } @Override public int length() { return common.length(); } @Override public char charAt(int index) { return common.charAt(index); } @Override public CharSequence subSequence(int start, int end) { return common.subSequence(start, end); } } Chronicle-Queue-2.0.3/chronicle/src/test/000077500000000000000000000000001226752375100202325ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle/src/test/java/000077500000000000000000000000001226752375100211535ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle/src/test/java/net/000077500000000000000000000000001226752375100217415ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle/src/test/java/net/openhft/000077500000000000000000000000001226752375100234045ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle/src/test/java/net/openhft/chronicle/000077500000000000000000000000001226752375100253525ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle/src/test/java/net/openhft/chronicle/AssertionErrorNextIndexTest.java000066400000000000000000000077611226752375100337000ustar00rootroot00000000000000/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.openhft.chronicle; import net.openhft.chronicle.tools.ChronicleIndexReader; import net.openhft.chronicle.tools.ChronicleTools; import org.junit.Ignore; import org.junit.Test; import java.io.IOException; import java.util.Random; import static net.openhft.chronicle.IndexedChronicle1Test.assertEquals; import static org.junit.Assert.assertTrue; /** * @author Alex Koon */ public class AssertionErrorNextIndexTest { private static final String CHRONICLE = System.getProperty("java.io.tmpdir") + System.getProperty("file.separator") + "AssertionErrorNextIndexTest"; private static final Random R = new Random(1); private static void writeToChronicle(ExcerptAppender a, int i) { a.startExcerpt(1024); final int size = R.nextInt((int) a.remaining() - 8) + 8; a.writeInt(i); a.writeInt(size); a.position(size); a.finish(); } private static int readFromChronicle(ExcerptTailer t) { int n = t.readInt(); int size = t.readInt(); assertEquals(size, t.capacity()); t.finish(); return n; } @Test @Ignore public void startTest() throws IOException, InterruptedException { ChronicleTools.deleteOnExit(CHRONICLE); // shrink the chronicle chunks to trigger error earlier final ChronicleConfig config = ChronicleConfig.TEST; config.indexBlockSize(1024 * 1024); config.dataBlockSize(4 * 1024); Chronicle chronicle1 = new IndexedChronicle(CHRONICLE, config); ExcerptAppender appender = chronicle1.createAppender(); for (int i = 0; i < 100; i++) { writeToChronicle(appender, i); } chronicle1.close(); { Chronicle chronicle = new IndexedChronicle(CHRONICLE, config); ExcerptTailer tailer = chronicle.createTailer(); int counter = 0; while (tailer.nextIndex()) { // System.out.println(counter+": " +tailer.index()); assertTrue("Capacity: " + tailer.capacity(), tailer.capacity() <= 1024); int i = readFromChronicle(tailer); assertEquals(counter, i); counter++; } chronicle.close(); } ChronicleIndexReader.main(CHRONICLE + ".index"); // Let the writer start writing first long lastIndex = 0; long counter = 0; while (counter < 100) { Chronicle chronicle = new IndexedChronicle(CHRONICLE, config); ExcerptTailer tailer = chronicle.createTailer(); System.out.println("index(" + (lastIndex - 1) + ")"); boolean ok = tailer.index(lastIndex - 1); if (ok) { assertTrue("Capacity: " + tailer.capacity(), tailer.capacity() <= 1024); int i = readFromChronicle(tailer); assertEquals(counter - 1, i); } int count = 10; while (tailer.nextIndex() && count-- > 0 && counter < 100) { System.out.println(counter + ": " + tailer.index()); assertTrue("counter: " + counter + ", Capacity: " + tailer.capacity(), tailer.capacity() <= 1024); int i = readFromChronicle(tailer); assertEquals(counter, i); counter++; } lastIndex = tailer.index(); chronicle.close(); } } }ChainedInProcessChronicleTest.java000066400000000000000000000045701226752375100340140ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle/src/test/java/net/openhft/chronicle/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.openhft.chronicle; import net.openhft.chronicle.tcp.InProcessChronicleSink; import net.openhft.chronicle.tcp.InProcessChronicleSource; import net.openhft.chronicle.tools.ChronicleTools; import org.junit.Test; import java.io.IOException; /** * @author peter.lawrey */ public class ChainedInProcessChronicleTest { private static final String TMP = System.getProperty("java.io.tmpdir"); @Test public void testChained() throws IOException { ChronicleTools.deleteOnExit(TMP + "/chronicle1"); Chronicle chronicle1 = new IndexedChronicle(TMP + "/chronicle1"); InProcessChronicleSource source1 = new InProcessChronicleSource(chronicle1, 61111); ChronicleTools.deleteOnExit(TMP + "/chronicle2"); Chronicle chronicle2 = new IndexedChronicle(TMP + "/chronicle2"); InProcessChronicleSource source2 = new InProcessChronicleSource(chronicle2, 62222); InProcessChronicleSink sink2 = new InProcessChronicleSink(source2, "localhost", 61111); ChronicleTools.deleteOnExit(TMP + "/chronicle3"); Chronicle chronicle3 = new IndexedChronicle(TMP + "/chronicle3"); InProcessChronicleSink sink3 = new InProcessChronicleSink(chronicle3, "localhost", 62222); ExcerptAppender excerpt1 = source1.createAppender(); ExcerptTailer excerpt2 = sink2.createTailer(); ExcerptTailer excerpt3 = sink3.createTailer(); for (int i = 1; i < 20; i++) { excerpt1.startExcerpt(); excerpt1.writeLong(System.nanoTime()); excerpt1.finish(); while (excerpt2.size() < i) excerpt2.nextIndex(); while (excerpt3.size() < i) excerpt3.nextIndex(); } sink3.close(); sink2.close(); source1.close(); } } Chronicle-Queue-2.0.3/chronicle/src/test/java/net/openhft/chronicle/InProcessChronicleTest.java000066400000000000000000000375101226752375100326170ustar00rootroot00000000000000/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.openhft.chronicle; import net.openhft.chronicle.tcp.InProcessChronicleSink; import net.openhft.chronicle.tcp.InProcessChronicleSource; import net.openhft.chronicle.tools.ChronicleTools; import net.openhft.lang.io.StopCharTesters; import org.jetbrains.annotations.NotNull; import org.junit.Test; import java.io.*; import java.util.ArrayList; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; import static org.junit.Assert.assertEquals; /** * @author peter.lawrey */ public class InProcessChronicleTest { public static final int PORT = 12345; @Test public void testOverTCP() throws IOException, InterruptedException { String baseDir = System.getProperty("java.io.tmpdir"); String srcBasePath = baseDir + "/IPCT.testOverTCP.source"; ChronicleTools.deleteOnExit(srcBasePath); // NOTE: the sink and source must have different chronicle files. // TODO, make more robust. final int messages = 5 * 1000 * 1000; ChronicleConfig config = ChronicleConfig.DEFAULT.clone(); // config.dataBlockSize(4096); // config.indexBlockSize(4096); final IndexedChronicle underlying = new IndexedChronicle(srcBasePath/*, config*/); final Chronicle source = new InProcessChronicleSource(underlying, PORT + 1); Thread t = new Thread(new Runnable() { @Override public void run() { try { // PosixJNAAffinity.INSTANCE.setAffinity(1 << 1); ExcerptAppender excerpt = source.createAppender(); for (int i = 1; i <= messages; i++) { // use a size which will cause mis-alignment. excerpt.startExcerpt(); excerpt.writeLong(i); excerpt.append(' '); excerpt.append(i); excerpt.append('\n'); excerpt.finish(); } System.out.println(System.currentTimeMillis() + ": Finished writing messages"); } catch (Exception e) { throw new AssertionError(e); } } }); // PosixJNAAffinity.INSTANCE.setAffinity(1 << 2); String snkBasePath = baseDir + "/IPCT.testOverTCP.sink"; ChronicleTools.deleteOnExit(snkBasePath); final IndexedChronicle underlying2 = new IndexedChronicle(snkBasePath/*, config*/); Chronicle sink = new InProcessChronicleSink(underlying2, "localhost", PORT + 1); long start = System.nanoTime(); t.start(); ExcerptTailer excerpt = sink.createTailer(); int count = 0; for (int i = 1; i <= messages; i++) { while (!excerpt.nextIndex()) count++; long n = excerpt.readLong(); String text = excerpt.parseUTF(StopCharTesters.CONTROL_STOP); if (i != n) assertEquals('\'' + text + '\'', i, n); excerpt.finish(); } sink.close(); System.out.println("There were " + count + " isSync messages"); t.join(); source.close(); long time = System.nanoTime() - start; System.out.printf("Messages per second %,d%n", (int) (messages * 1e9 / time)); } @Test public void testPricePublishing() throws IOException, InterruptedException { String baseDir = System.getProperty("java.io.tmpdir"); String sourceName = baseDir + "/price.source"; ChronicleTools.deleteOnExit(sourceName); Chronicle source = new InProcessChronicleSource(new IndexedChronicle(sourceName), PORT + 2); PriceWriter pw = new PriceWriter(source.createAppender()); String sinkName = baseDir + "/price.sink"; ChronicleTools.deleteOnExit(sinkName); Chronicle sink = new InProcessChronicleSink(new IndexedChronicle(sinkName), "localhost", PORT + 2); final AtomicInteger count = new AtomicInteger(); PriceReader reader = new PriceReader(sink.createTailer(), new PriceListener() { @Override public void onPrice(long timeInMicros, String symbol, double bp, int bq, double ap, int aq) { count.incrementAndGet(); } }); pw.onPrice(1, "symbol", 99.9, 1, 100.1, 2); reader.read(); long start = System.nanoTime(); int prices = 12000000; for (int i = 1; i <= prices; i++) { pw.onPrice(i, "symbol", 99.9, i, 100.1, i + 1); } long mid = System.nanoTime(); while (count.get() < prices) reader.read(); long end = System.nanoTime(); System.out.printf("Took an average of %.2f us to write and %.2f us to read%n", (mid - start) / prices / 1e3, (end - mid) / prices / 1e3); source.close(); sink.close(); } @Test public void testPricePublishing2() throws IOException, InterruptedException { String baseDir = System.getProperty("java.io.tmpdir"); String sourceName = baseDir + "/price3.source"; ChronicleTools.deleteOnExit(sourceName); Chronicle source = new InProcessChronicleSource(new IndexedChronicle(sourceName), PORT + 2); PriceWriter pw = new PriceWriter(source.createAppender()); String sinkName = baseDir + "/price3.sink"; ChronicleTools.deleteOnExit(sinkName); Chronicle sink = new InProcessChronicleSink(new IndexedChronicle(sinkName), "localhost", PORT + 2); final AtomicInteger count = new AtomicInteger(); PriceReader reader = new PriceReader(sink.createTailer(), new PriceListener() { @Override public void onPrice(long timeInMicros, String symbol, double bp, int bq, double ap, int aq) { count.incrementAndGet(); } }); pw.onPrice(1, "symbol", 99.9, 1, 100.1, 2); assertEquals(-1, reader.excerpt.index()); reader.read(); assertEquals(0, reader.excerpt.index()); long start = System.nanoTime(); int prices = 2 * 1000 * 1000; for (int i = 1; i <= prices; i++) { pw.onPrice(i, "symbol", 99.9, i, 100.1, i + 1); } long mid = System.nanoTime(); while (count.get() < prices) reader.read(); long end = System.nanoTime(); System.out.printf("Took an average of %.2f us to write and %.2f us to read using Excerpt%n", (mid - start) / prices / 1e3, (end - mid) / prices / 1e3); source.close(); sink.close(); } @Test public void testPricePublishing3() throws IOException, InterruptedException { String baseDir = System.getProperty("java.io.tmpdir"); String sourceName = baseDir + "/price2.source"; ChronicleTools.deleteOnExit(sourceName); Chronicle source = new InProcessChronicleSource(new IndexedChronicle(sourceName), PORT + 2); PriceWriter pw = new PriceWriter(source.createAppender()); String sinkName = baseDir + "/price2.sink"; ChronicleTools.deleteOnExit(sinkName); Chronicle sink = new InProcessChronicleSink(new IndexedChronicle(sinkName), "localhost", PORT + 2); final AtomicInteger count = new AtomicInteger(); PriceReader reader = new PriceReader(sink.createTailer(), new PriceListener() { @Override public void onPrice(long timeInMicros, String symbol, double bp, int bq, double ap, int aq) { count.incrementAndGet(); } }); pw.onPrice(1, "symbol", 99.9, 1, 100.1, 2); assertEquals(-1, reader.excerpt.index()); reader.read(); assertEquals(0, reader.excerpt.index()); long start = System.nanoTime(); int prices = 2 * 1000 * 1000; for (int i = 1; i <= prices; i++) { pw.onPrice(i, "symbol", 99.9, i, 100.1, i + 1); } long mid = System.nanoTime(); while (count.get() < prices) reader.read(); long end = System.nanoTime(); System.out.printf("Took an average of %.2f us to write and %.2f us to read using Tailer%n", (mid - start) / prices / 1e3, (end - mid) / prices / 1e3); source.close(); sink.close(); } // Took an average of 2.8 us to write and 7.6 us to read (Java 7) @Test public void testSerializationPerformance() throws IOException, ClassNotFoundException { List bytes = new ArrayList(); long start = System.nanoTime(); int prices = 200 * 1000; for (int i = 0; i < prices; i++) { ByteArrayOutputStream baos = new ByteArrayOutputStream(); ObjectOutputStream oos = new ObjectOutputStream(baos); PriceUpdate pu = new PriceUpdate(1 + i, "symbol", 99.9, i + 1, 100.1, i + 2); oos.writeObject(pu); oos.close(); bytes.add(baos.toByteArray()); } long mid = System.nanoTime(); for (byte[] bs : bytes) { ObjectInputStream ois = new ObjectInputStream(new ByteArrayInputStream(bs)); PriceUpdate pu = (PriceUpdate) ois.readObject(); } long end = System.nanoTime(); System.out.printf("Took an average of %.1f us to write and %.1f us to read%n", (mid - start) / prices / 1e3, (end - mid) / prices / 1e3); } // Took an average of 0.42 us to write and 0.61 us to read (Java 6) // Took an average of 0.35 us to write and 0.59 us to read (Java 7) interface PriceListener { void onPrice(long timeInMicros, String symbol, double bp, int bq, double ap, int aq); } static class PriceWriter implements PriceListener { private final ExcerptAppender excerpt; PriceWriter(ExcerptAppender excerpt) { this.excerpt = excerpt; } @Override public void onPrice(long timeInMicros, @NotNull String symbol, double bp, int bq, double ap, int aq) { excerpt.startExcerpt(); excerpt.writeByte('P'); // code for a price excerpt.writeLong(timeInMicros); excerpt.writeEnum(symbol); excerpt.writeDouble(bp); excerpt.writeInt(bq); excerpt.writeDouble(ap); excerpt.writeInt(aq); excerpt.finish(); } } static class PriceReader { private final ExcerptTailer excerpt; private final PriceListener listener; PriceReader(ExcerptTailer excerpt, PriceListener listener) { this.excerpt = excerpt; this.listener = listener; } public boolean read() { if (!excerpt.nextIndex()) return false; // System.out.println("ei: "+excerpt.index()); char ch = (char) excerpt.readByte(); switch (ch) { case 'P': { long timeInMicros = excerpt.readLong(); String symbol = excerpt.readEnum(String.class); double bp = excerpt.readDouble(); int bq = excerpt.readInt(); double ap = excerpt.readDouble(); int aq = excerpt.readInt(); listener.onPrice(timeInMicros, symbol, bp, bq, ap, aq); break; } default: throw new AssertionError("Unexpected code " + ch); } return true; } } /* @Test @Ignore public void testOverTCPRolling() throws IOException, InterruptedException { String baseDir = System.getProperty("java.io.tmpdir"); String srcBasePath = baseDir + "/IPCTR.testOverTCP.source"; ChronicleTools.deleteDirOnExit(srcBasePath); // NOTE: the sink and source must have different chronicle files. // TODO, make more robust. final int messages = 2 * 1000 * 1000; ChronicleConfig config = ChronicleConfig.TEST.clone(); config.indexFileExcerpts(512); // config.dataBlockSize(4096); // config.indexBlockSize(4096); final Chronicle source = new InProcessChronicleSource(new RollingChronicle(srcBasePath, config), PORT + 1); Thread t = new Thread(new Runnable() { @Override public void run() { try { // PosixJNAAffinity.INSTANCE.setAffinity(1 << 1); ExcerptAppender excerpt = source.createAppender(); for (int i = 1; i <= messages; i++) { // use a size which will cause mis-alignment. excerpt.startExcerpt(); excerpt.writeLong(i); excerpt.append(' '); excerpt.append(i); excerpt.append('\n'); excerpt.finish(); } System.out.println(System.currentTimeMillis() + ": Finished writing messages"); } catch (Exception e) { throw new AssertionError(e); } } }); // PosixJNAAffinity.INSTANCE.setAffinity(1 << 2); String snkBasePath = baseDir + "/IPCTR.testOverTCP.sink"; ChronicleTools.deleteDirOnExit(snkBasePath); Chronicle sink = new InProcessChronicleSink(new RollingChronicle(snkBasePath, config), "localhost", PORT + 1); long start = System.nanoTime(); t.start(); ExcerptTailer excerpt = sink.createTailer(); int count = 0; for (int i = 1; i <= messages; i++) { while (!excerpt.nextIndex()) count++; long n = excerpt.readLong(); String text = excerpt.parseUTF(StopCharTesters.CONTROL_STOP); if (i != n) assertEquals('\'' + text + '\'', i, n); excerpt.finish(); System.out.println(i); } sink.close(); System.out.println("There were " + count + " isSync messages"); t.join(); source.close(); long time = System.nanoTime() - start; System.out.printf("Messages per second %,d%n", (int) (messages * 1e9 / time)); } */ static class PriceUpdate implements Externalizable, Serializable { private long timeInMicros; private String symbol; private double bp; private int bq; private double ap; private int aq; public PriceUpdate() { } PriceUpdate(long timeInMicros, String symbol, double bp, int bq, double ap, int aq) { this.timeInMicros = timeInMicros; this.symbol = symbol; this.bp = bp; this.bq = bq; this.ap = ap; this.aq = aq; } // @Override public void writeExternal(@NotNull ObjectOutput out) throws IOException { out.writeLong(timeInMicros); out.writeUTF(symbol); out.writeDouble(bp); out.writeInt(bq); out.writeDouble(ap); out.writeInt(aq); } // @Override public void readExternal(@NotNull ObjectInput in) throws IOException, ClassNotFoundException { timeInMicros = in.readLong(); symbol = in.readUTF(); bp = in.readDouble(); bq = in.readInt(); ap = in.readDouble(); aq = in.readInt(); } } } Chronicle-Queue-2.0.3/chronicle/src/test/java/net/openhft/chronicle/IndexedChronicle1Test.java000077500000000000000000000272351226752375100323610ustar00rootroot00000000000000/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.openhft.chronicle; import net.openhft.chronicle.tools.ChronicleIndexReader; import net.openhft.chronicle.tools.ChronicleTools; import org.jetbrains.annotations.Nullable; import org.junit.Assert; import org.junit.Ignore; import org.junit.Test; import java.io.File; import java.io.IOException; import java.math.BigDecimal; import java.math.BigInteger; import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; /** * @author peter.lawrey */ public class IndexedChronicle1Test { static final String TMP = System.getProperty("java.io.tmpdir"); static void assertEquals(long a, long b) { if (a != b) Assert.assertEquals(a, b); } static void assertEquals(@Nullable T a, @Nullable T b) { if (a == null) { if (b == null) return; } else if (a.equals(b)) { return; } Assert.assertEquals(a, b); } @Test public void testSerializationPerformance() throws IOException, ClassNotFoundException, InterruptedException { String testPath = TMP + File.separator + "chronicle-object"; IndexedChronicle tsc = new IndexedChronicle(testPath); ChronicleTools.deleteOnExit(testPath); // tsc.clear(); ExcerptAppender appender = tsc.createAppender(); int objects = 1000000; long start = System.nanoTime(); for (int i = 0; i < objects; i++) { appender.startExcerpt(); appender.writeObject(BigDecimal.valueOf(i % 1000)); appender.finish(); } ExcerptTailer tailer = tsc.createTailer(); for (int i = 0; i < objects; i++) { assertTrue(tailer.nextIndex() || tailer.nextIndex()); BigDecimal bd = (BigDecimal) tailer.readObject(); assertEquals(i % 1000, bd.longValue()); tailer.finish(); } // System.out.println("waiting"); // Thread.sleep(20000); // System.out.println("waited"); // System.gc(); tsc.close(); long time = System.nanoTime() - start; System.out.printf("The average time to write and read a BigDecimal was %,d ns%n", time / objects); // tsc = null; // System.gc(); // Thread.sleep(10000); } @Test @Ignore public void rewritibleEntries() throws IOException { // boolean[] booleans = {false, true}; // for (boolean useUnsafe : booleans) // for (boolean minimiseByteBuffers : booleans) // for (boolean synchronousMode : booleans) // doRewriteableEntries(useUnsafe, minimiseByteBuffers, synchronousMode); doRewriteableEntries(true, true, false); } private void doRewriteableEntries(boolean useUnsafe, boolean minimiseByteBuffers, boolean synchronousMode) throws IOException { String basePath = TMP + File.separator + "doRewriteableEntries.ict"; IndexedChronicle tsc = new IndexedChronicle(basePath); // tsc.useUnsafe(useUnsafe); ChronicleTools.deleteOnExit(basePath); // tsc.clear(); ExcerptAppender excerpt = tsc.createAppender(); int counter = 1; for (int i = 0; i < 1024; i++) { excerpt.startExcerpt(); for (int j = 0; j < 128; j += 8) excerpt.writeLong(counter++); excerpt.write(-1); excerpt.finish(); } int counter2 = 1; ExcerptTailer excerpt2 = tsc.createTailer(); while (excerpt2.nextIndex()) { for (int j = 0; j < 128; j += 8) { long actual = excerpt2.readLong(); long expected = counter2++; if (expected != actual) assertEquals(expected, actual); } assertEquals(-1, excerpt2.readByte()); excerpt2.finish(); } assertEquals(counter, counter2); // assertFalse(excerpt2.index(1024)); tsc.close(); } /** * Tests that IndexedChronicle.close() does not blow up (anymore) when you reopen an existing chronicle * due to the null data buffers created internally. * * @throws IOException if opening chronicle fails */ @Test public void testCloseWithNullBuffers() throws IOException { String basePath = TMP + File.separator + "deleteme.ict"; ChronicleTools.deleteOnExit(basePath); IndexedChronicle tsc = new IndexedChronicle(basePath); // tsc.clear(); ExcerptAppender excerpt = tsc.createAppender(); for (int i = 0; i < 512; i++) { excerpt.startExcerpt(); excerpt.writeByte(1); excerpt.finish(); } // used to throw NPE if you have finished already. excerpt.close(); tsc.close(); tsc = new IndexedChronicle(basePath); tsc.createAppender().close(); tsc.close(); // used to throw an exception. } @Test @Ignore public void testTimeTenMillion() throws IOException { int repeats = 3; for (int j = 0; j < repeats; j++) { long start = System.nanoTime(); String basePath = TMP + File.separator + "testTimeTenMillion"; ChronicleTools.deleteOnExit(basePath); int records = 10 * 1000 * 1000; { IndexedChronicle ic = new IndexedChronicle(basePath); // ic.useUnsafe(true); // ic.clear(); ExcerptAppender excerpt = ic.createAppender(); for (int i = 1; i <= records; i++) { excerpt.startExcerpt(); excerpt.writeLong(i); excerpt.writeDouble(i); excerpt.finish(); } ic.close(); } { IndexedChronicle ic = new IndexedChronicle(basePath); // ic.useUnsafe(true); ExcerptTailer excerpt = ic.createTailer(); for (int i = 1; i <= records; i++) { boolean found = excerpt.nextIndex() || excerpt.nextIndex(); if (!found) assertTrue(found); long l = excerpt.readLong(); double d = excerpt.readDouble(); if (l != i) assertEquals(i, l); if (d != i) assertEquals((double) i, d); excerpt.finish(); } ic.close(); } long time = System.nanoTime() - start; System.out.printf("Time taken %,d ms%n", time / 1000000); } } /** * https://github.com/peter-lawrey/Java-Chronicle/issues/9 * * @author AndrasMilassin */ @Test public void test_boolean() throws Exception { String testPath = TMP + File.separator + "chroncle-bool-test"; ChronicleTools.deleteOnExit(testPath); IndexedChronicle tsc = new IndexedChronicle(testPath); // tsc.useUnsafe(false); ExcerptAppender excerpt = tsc.createAppender(); excerpt.startExcerpt(); excerpt.writeBoolean(false); excerpt.writeBoolean(true); excerpt.finish(); ExcerptTailer tailer = tsc.createTailer(); tailer.nextIndex(); boolean one = tailer.readBoolean(); boolean onetwo = tailer.readBoolean(); tsc.close(); Assert.assertEquals(false, one); Assert.assertEquals(true, onetwo); } @Test public void testStopBitEncoded() throws IOException { boolean ok = false; String testPath = TMP + File.separator + "chronicle-stop-bit"; ChronicleTools.deleteOnExit(testPath); IndexedChronicle tsc = new IndexedChronicle(testPath); // ChronicleIndexReader.main(testPath); try { ExcerptAppender writer = tsc.createAppender(); ExcerptTailer reader = tsc.createTailer(); long[] longs = {Long.MIN_VALUE, Integer.MIN_VALUE, Short.MIN_VALUE, Character.MIN_VALUE, Byte.MIN_VALUE, Long.MAX_VALUE, Integer.MAX_VALUE, Short.MAX_VALUE, Character.MAX_CODE_POINT, Character.MAX_VALUE, Byte.MAX_VALUE}; for (long l : longs) { writer.startExcerpt(); writer.writeChar('T'); writer.writeStopBit(l); writer.finish(); // System.out.println("finished"); reader.nextIndex(); reader.readChar(); long l2 = reader.readStopBit(); reader.finish(); assertEquals(l, l2); } writer.startExcerpt(longs.length * 10); writer.writeChar('t'); for (long l : longs) writer.writeStopBit(l); writer.finish(); reader.nextIndex(); reader.readChar(); for (long l : longs) { long l2 = reader.readStopBit(); assertEquals(l, l2); } assertEquals(0, reader.remaining()); reader.finish(); tsc.close(); ok = true; } finally { if (!ok) ChronicleIndexReader.main(testPath); } } @Test public void testEnum() throws IOException { String testPath = TMP + File.separator + "chroncle-bool-enum"; IndexedChronicle tsc = new IndexedChronicle(testPath); // tsc.useUnsafe(false); ChronicleTools.deleteOnExit(testPath); // tsc.clear(); ExcerptAppender excerpt = tsc.createAppender(); excerpt.startExcerpt(); excerpt.writeEnum(AccessMode.EXECUTE); excerpt.writeEnum(AccessMode.READ); excerpt.writeEnum(AccessMode.WRITE); excerpt.writeEnum(BigInteger.ONE); excerpt.writeEnum(BigInteger.TEN); excerpt.writeEnum(BigInteger.ZERO); excerpt.writeEnum(BigInteger.ONE); excerpt.writeEnum(BigInteger.TEN); excerpt.writeEnum(BigInteger.ZERO); excerpt.finish(); // System.out.println("size=" + excerpt.position()); ExcerptTailer tailer = tsc.createTailer(); tailer.nextIndex(); AccessMode e = tailer.readEnum(AccessMode.class); AccessMode r = tailer.readEnum(AccessMode.class); AccessMode w = tailer.readEnum(AccessMode.class); BigInteger one = tailer.readEnum(BigInteger.class); BigInteger ten = tailer.readEnum(BigInteger.class); BigInteger zero = tailer.readEnum(BigInteger.class); BigInteger one2 = tailer.readEnum(BigInteger.class); BigInteger ten2 = tailer.readEnum(BigInteger.class); BigInteger zero2 = tailer.readEnum(BigInteger.class); tsc.close(); assertSame(AccessMode.EXECUTE, e); assertSame(AccessMode.READ, r); assertSame(AccessMode.WRITE, w); assertEquals(BigInteger.ONE, one); assertEquals(BigInteger.TEN, ten); assertEquals(BigInteger.ZERO, zero); assertSame(one, one2); assertSame(ten, ten2); assertSame(zero, zero2); } enum AccessMode { EXECUTE, READ, WRITE } } Chronicle-Queue-2.0.3/chronicle/src/test/java/net/openhft/chronicle/IndexedChronicleTest.java000077500000000000000000000521761226752375100323020ustar00rootroot00000000000000/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.openhft.chronicle; //import vanilla.java.processingengine.affinity.PosixJNAAffinity; import net.openhft.chronicle.tools.ChronicleIndexReader; import net.openhft.chronicle.tools.ChronicleTools; import net.openhft.lang.io.StopCharTesters; import org.jetbrains.annotations.NotNull; import org.junit.Ignore; import org.junit.Test; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.util.*; import static org.junit.Assert.*; /** * @author peter.lawrey */ public class IndexedChronicleTest { static { ChronicleTools.warmup(); } public static final String TMP = System.getProperty("java.io.tmpdir"); private static final long WARMUP = 20000; static void validateExcerpt(@NotNull ExcerptCommon r, int i, int expected) { if (expected > r.remaining() || 8 * expected < r.remaining()) assertEquals("index: " + r.index(), expected, r.remaining()); if (expected > r.capacity() || 8 * expected < r.capacity()) assertEquals("index: " + r.index(), expected, r.capacity()); assertEquals(0, r.position()); long l = r.readLong(); assertEquals(i, l); assertEquals(8, r.position()); if (expected - 8 != r.remaining()) assertEquals("index: " + r.index(), expected - 8, r.remaining()); double d = r.readDouble(); assertEquals(i, d, 0.0); if (0 != r.remaining()) assertEquals("index: " + r.index(), 0, r.remaining()); r.position(0); long l2 = r.readLong(); assertEquals(i, l2); r.position(expected); r.finish(); } static void testSearchRange(List ints, Excerpt excerpt, MyExcerptComparator mec, long[] startEnd) { int elo = Collections.binarySearch(ints, mec.lo); if (elo < 0) elo = ~elo; int ehi = Collections.binarySearch(ints, mec.hi); if (ehi < 0) ehi = ~ehi; else ehi++; excerpt.findRange(startEnd, mec); assertEquals("lo: " + mec.lo + ", hi: " + mec.hi, "[" + elo + ", " + ehi + "]", Arrays.toString(startEnd)); } @Test @Ignore public void testWasPadding() throws IOException { final String basePath = TMP + "/singleThreaded"; ChronicleTools.deleteOnExit(basePath); ChronicleConfig config = ChronicleConfig.TEST.clone(); config.dataBlockSize(128); config.indexBlockSize(128); IndexedChronicle chronicle1 = new IndexedChronicle(basePath, config); ExcerptAppender appender = chronicle1.createAppender(); IndexedChronicle chronicle2 = new IndexedChronicle(basePath, config); ExcerptTailer tailer = chronicle2.createTailer(); assertEquals(-1, tailer.index()); assertTrue(tailer.wasPadding()); assertFalse(tailer.index(-1)); assertTrue(tailer.wasPadding()); appender.startExcerpt(48); appender.position(48); appender.finish(); assertTrue(tailer.nextIndex()); assertFalse(tailer.wasPadding()); assertEquals(0, tailer.index()); assertTrue(tailer.index(0)); assertFalse(tailer.wasPadding()); // rewind it to the start - issue # 12 assertFalse(tailer.index(-1)); assertEquals(-1, tailer.index()); assertTrue(tailer.nextIndex()); assertFalse(tailer.wasPadding()); assertEquals(0, tailer.index()); // end of issue # 12; assertFalse(tailer.nextIndex()); assertFalse(tailer.wasPadding()); assertEquals(0, tailer.index()); assertFalse(tailer.index(1)); assertFalse(tailer.wasPadding()); appender.startExcerpt(48); appender.position(48); appender.finish(); assertTrue(tailer.nextIndex()); assertFalse(tailer.wasPadding()); assertEquals(2, tailer.index()); assertTrue(tailer.index(1)); assertFalse(tailer.wasPadding()); assertEquals(1, tailer.index()); assertFalse(tailer.nextIndex()); assertFalse(tailer.wasPadding()); assertEquals(1, tailer.index()); assertFalse(tailer.index(2)); assertFalse(tailer.wasPadding()); assertEquals(2, tailer.index()); // doesn't fit. appender.startExcerpt(48); appender.position(48); appender.finish(); assertFalse(tailer.index(2)); assertTrue(tailer.wasPadding()); assertEquals(2, tailer.index()); assertTrue(tailer.index(1)); assertTrue(tailer.nextIndex()); assertFalse(tailer.wasPadding()); assertEquals(3, tailer.index()); assertFalse(tailer.index(2)); assertTrue(tailer.wasPadding()); assertEquals(2, tailer.index()); assertTrue(tailer.index(3)); assertFalse(tailer.wasPadding()); assertEquals(3, tailer.index()); assertFalse(tailer.index(4)); assertFalse(tailer.wasPadding()); assertEquals(4, tailer.index()); chronicle1.close(); chronicle2.close(); } @Test public void singleThreaded() throws IOException { final String basePath = TMP + "/singleThreaded"; ChronicleTools.deleteOnExit(basePath); ChronicleConfig config = ChronicleConfig.TEST.clone(); // TODO fix for 4096 !!! int dataBlockSize = 4 * 1024; config.dataBlockSize(dataBlockSize); config.indexBlockSize(128 * 1024); IndexedChronicle chronicle = new IndexedChronicle(basePath, config); int i = 0; try { ExcerptAppender w = chronicle.createAppender(); ExcerptTailer r = chronicle.createTailer(); Excerpt e = chronicle.createExcerpt(); Random rand = new Random(1); // finish just at the end of the first page. int idx = 0; for (i = 0; i < 50000; i++) { // System.out.println(i + " " + idx); // if (i == 28) // ChronicleIndexReader.main(basePath + ".index"); assertFalse("i: " + i, r.nextIndex()); assertFalse("i: " + i, e.index(idx)); int capacity = 16 * (1 + rand.nextInt(7)); w.startExcerpt(capacity); assertEquals(0, w.position()); w.writeLong(i); assertEquals(8, w.position()); w.writeDouble(i); int expected = 16; assertEquals(expected, w.position()); assertEquals(capacity - expected, w.remaining()); w.finish(); // ChronicleIndexReader.main(basePath + ".index"); // if (i >= 5542) // ChronicleIndexReader.main(basePath + ".index"); if (!r.nextIndex()) { assertTrue(r.nextIndex()); } validateExcerpt(r, i, expected); if (!e.index(idx++)) { assertTrue(e.wasPadding()); assertTrue(e.index(idx++)); } validateExcerpt(e, i, expected); } w.close(); r.close(); } finally { chronicle.close(); System.out.println("i: " + i); // ChronicleIndexReader.main(basePath + ".index"); // ChronicleTools.deleteOnExit(basePath); } } @Test public void multiThreaded() throws IOException, InterruptedException { // for (int i = 0; i < 20; i++) System.out.println(); if (Runtime.getRuntime().availableProcessors() < 2) { System.err.println("Test requires 2 CPUs, skipping"); return; } final String basePath = TMP + "/multiThreaded"; ChronicleTools.deleteOnExit(basePath); final ChronicleConfig config = ChronicleConfig.DEFAULT.clone(); int dataBlockSize = 1 << 26; config.dataBlockSize(dataBlockSize); config.indexBlockSize(dataBlockSize / 4); IndexedChronicle chronicle = new IndexedChronicle(basePath, config); final ExcerptTailer r = chronicle.createTailer(); // shorten the test for a build server. final long words = 50L * 1000 * 1000; final int size = 4; long start = System.nanoTime(); Thread t = new Thread(new Runnable() { @Override public void run() { try { IndexedChronicle chronicle = new IndexedChronicle(basePath, config); final ExcerptAppender w = chronicle.createAppender(); for (int i = 0; i < words; i += size) { w.startExcerpt(); for (int s = 0; s < size; s++) w.writeInt(1 + i); // w.position(4L * size); w.finish(); // System.out.println(i); } w.close(); // chronicle.close(); } catch (Throwable e) { e.printStackTrace(); } } }); t.start(); long maxDelay = 0, maxJitter = 0; for (long i = 0; i < words; i += size) { if (!r.nextIndex()) { long start0 = System.nanoTime(); long last = start0; while (!r.nextIndex()) { long now = System.nanoTime(); long jitter = now - last; if (i > WARMUP && maxJitter < jitter) maxJitter = jitter; long delay0 = now - start0; if (delay0 > 100e6) throw new AssertionError("delay: " + delay0 / 1000000 + ", index: " + r.index()); if (i > WARMUP && maxDelay < delay0) maxDelay = delay0; last = now; } } try { for (int s = 0; s < size; s++) { int j = r.readInt(); if (j != i + 1) { ChronicleIndexReader.main(basePath + ".index"); throw new AssertionError(j + " != " + (i + 1)); } } r.finish(); } catch (Exception e) { System.err.println("i= " + i); e.printStackTrace(); break; } } r.close(); long rate = words / size * 10 * 1000L / (System.nanoTime() - start); System.out.println("Rate = " + rate / 10.0 + " Mmsg/sec for " + size * 4 + " byte messages, " + "maxJitter: " + maxJitter / 1000 + " us, " + "maxDelay: " + maxDelay / 1000 + " us," + ""); // "totalWait: " + (PrefetchingMappedFileCache.totalWait.longValue() + SingleMappedFileCache.totalWait.longValue()) / 1000 + " us"); Thread.sleep(200); ChronicleTools.deleteOnExit(basePath); } @Test @Ignore public void multiThreaded2() throws IOException, InterruptedException { if (Runtime.getRuntime().availableProcessors() < 3) { System.err.println("Test requires 3 CPUs, skipping"); return; } final String basePath = TMP + "/multiThreaded"; final String basePath2 = TMP + "/multiThreaded2"; ChronicleTools.deleteOnExit(basePath); ChronicleTools.deleteOnExit(basePath2); final ChronicleConfig config = ChronicleConfig.DEFAULT.clone(); // config.dataBlockSize(4*1024); // config.indexBlockSize(4 * 1024); final int runs = 100 * 1000 * 1000; final int size = 4; long start = System.nanoTime(); Thread t = new Thread(new Runnable() { @Override public void run() { try { IndexedChronicle chronicle = new IndexedChronicle(basePath, config); final ExcerptAppender w = chronicle.createAppender(); for (int i = 0; i < runs; i += size) { w.startExcerpt(); for (int s = 0; s < size; s++) w.writeInt(1 + i); w.finish(); } w.close(); // chronicle.close(); } catch (IOException e) { e.printStackTrace(); } } }, "t1"); t.start(); Thread t2 = new Thread(new Runnable() { @Override public void run() { try { IndexedChronicle chronicle = new IndexedChronicle(basePath, config); final ExcerptTailer r = chronicle.createTailer(); IndexedChronicle chronicle2 = null; try { chronicle2 = new IndexedChronicle(basePath2, config); } catch (FileNotFoundException e) { System.in.read(); } final ExcerptAppender w = chronicle2.createAppender(); for (int i = 0; i < runs; i += size) { do { } while (!r.nextIndex()); w.startExcerpt(); for (int s = 0; s < size; s++) w.writeInt(r.readInt()); r.finish(); w.finish(); } w.close(); // chronicle.close(); // chronicle2.close(); } catch (IOException e) { e.printStackTrace(); } } }, "t2"); t2.start(); IndexedChronicle chronicle = new IndexedChronicle(basePath2, config); final ExcerptTailer r = chronicle.createTailer(); for (int i = 0; i < runs; i += size) { do { } while (!r.nextIndex()); try { for (int s = 0; s < size; s++) { long l = r.readInt(); if (l != i + 1) throw new AssertionError(); } r.finish(); } catch (Exception e) { System.err.println("i= " + i); e.printStackTrace(); break; } } r.close(); long rate = 2 * runs / size * 10000L / (System.nanoTime() - start); System.out.println("Rate = " + rate / 10.0 + " Mmsg/sec"); chronicle.close(); Thread.sleep(200); ChronicleTools.deleteOnExit(basePath); ChronicleTools.deleteOnExit(basePath2); } @Test public void testOneAtATime() throws IOException { ChronicleConfig config = ChronicleConfig.TEST.clone(); config.indexBlockSize(128); // very small config.dataBlockSize(128); // very small final String basePath = TMP + "/testOneAtATime"; ChronicleTools.deleteOnExit(basePath); File indexFile = new File(basePath + ".index"); for (int i = 0; i < 1000; i++) { if (i % 10 == 0) System.out.println("i: " + i); long indexFileSize = indexFile.length(); IndexedChronicle chronicle = new IndexedChronicle(basePath, config); assertEquals("Index should not grow on open (i=" + i + ")", indexFileSize, indexFile.length()); if (i == 0) { ExcerptTailer tailer = chronicle.createTailer(); assertFalse(tailer.nextIndex()); Excerpt excerpt = chronicle.createExcerpt(); assertFalse(excerpt.index(0)); } ExcerptAppender appender = chronicle.createAppender(); appender.startExcerpt(); appender.writeDouble(i); appender.finish(); // ChronicleIndexReader.main(basePath+".index"); ExcerptTailer tailer = chronicle.createTailer(); long[] indexes = new long[i + 1]; long lastIndex = -1; for (int j = 0; j <= i; j++) { assertTrue(tailer.nextIndex()); assertTrue(tailer.index() + " > " + lastIndex, tailer.index() > lastIndex); lastIndex = tailer.index(); double d = tailer.readDouble(); assertEquals(j, d, 0.0); assertEquals(0, tailer.remaining()); indexes[j] = tailer.index(); tailer.finish(); } assertFalse(tailer.nextIndex()); Excerpt excerpt = chronicle.createExcerpt(); // forward for (int j = 0; j < i; j++) { assertTrue(excerpt.index(indexes[j])); double d = excerpt.readDouble(); assertEquals(j, d, 0.0); assertEquals(0, excerpt.remaining()); excerpt.finish(); } assertFalse(excerpt.index(indexes[indexes.length - 1] + 1)); // backward for (int j = i - 1; j >= 0; j--) { assertTrue(excerpt.index(indexes[j])); double d = excerpt.readDouble(); assertEquals(j, d, 0.0); assertEquals(0, excerpt.remaining()); excerpt.finish(); } assertFalse(excerpt.index(-1)); chronicle.close(); } } @Test public void testFindRange() throws IOException { final String basePath = TMP + "/testFindRange"; ChronicleTools.deleteOnExit(basePath); IndexedChronicle chronicle = new IndexedChronicle(basePath); ExcerptAppender appender = chronicle.createAppender(); List ints = new ArrayList(); for (int i = 0; i < 1000; i += 10) { appender.startExcerpt(); appender.writeInt(i); appender.finish(); ints.add(i); } Excerpt excerpt = chronicle.createExcerpt(); final MyExcerptComparator mec = new MyExcerptComparator(); // exact matches at a the start mec.lo = mec.hi = -1; assertEquals(~0, excerpt.findMatch(mec)); mec.lo = mec.hi = 0; assertEquals(0, excerpt.findMatch(mec)); mec.lo = mec.hi = 9; assertEquals(~1, excerpt.findMatch(mec)); mec.lo = mec.hi = 10; assertEquals(1, excerpt.findMatch(mec)); // exact matches at a the end mec.lo = mec.hi = 980; assertEquals(98, excerpt.findMatch(mec)); mec.lo = mec.hi = 981; assertEquals(~99, excerpt.findMatch(mec)); mec.lo = mec.hi = 990; assertEquals(99, excerpt.findMatch(mec)); mec.lo = mec.hi = 1000; assertEquals(~100, excerpt.findMatch(mec)); // range match near the start long[] startEnd = new long[2]; mec.lo = 0; mec.hi = 3; excerpt.findRange(startEnd, mec); assertEquals("[0, 1]", Arrays.toString(startEnd)); mec.lo = 21; mec.hi = 29; excerpt.findRange(startEnd, mec); assertEquals("[3, 3]", Arrays.toString(startEnd)); /* mec.lo = 129; mec.hi = 631; testSearchRange(ints, excerpt, mec, startEnd); */ Random rand = new Random(1); for (int i = 0; i < 1000; i++) { int x = rand.nextInt(1010) - 5; int y = rand.nextInt(1010) - 5; mec.lo = Math.min(x, y); mec.hi = Math.max(x, y); testSearchRange(ints, excerpt, mec, startEnd); } chronicle.close(); } @Test public void testParseLines() throws IOException { final String basePath = TMP + "/testParseLines"; ChronicleTools.deleteOnExit(basePath); IndexedChronicle chronicle = new IndexedChronicle(basePath); ExcerptAppender appender = chronicle.createAppender(); int runs = 10000; for (int i = 0; i < runs; i++) { appender.startExcerpt(); appender.append("Hello world ").append(i).append("\n"); appender.finish(); } ExcerptTailer tailer = chronicle.createTailer(); for (int i = 0; i < runs; i++) { assertTrue(tailer.nextIndex()); String s = tailer.parseUTF(StopCharTesters.CONTROL_STOP); // System.out.println(s); assertEquals("Hello world " + i, s); tailer.finish(); } chronicle.close(); } static class MyExcerptComparator implements ExcerptComparator { int lo, hi; @Override public int compare(Excerpt excerpt) { final int x = excerpt.readInt(); return x < lo ? -1 : x > hi ? +1 : 0; } } } Chronicle-Queue-2.0.3/chronicle/src/test/java/net/openhft/chronicle/NewNumberAppendTest.java000077500000000000000000000100311226752375100321050ustar00rootroot00000000000000/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.openhft.chronicle; import net.openhft.chronicle.tools.ChronicleTools; import org.jetbrains.annotations.NotNull; import org.junit.Before; import org.junit.Test; import sun.misc.Unsafe; import java.io.File; import java.io.IOException; import java.util.Random; import static org.junit.Assert.fail; /** * @author andrew.bissell */ public class NewNumberAppendTest { static final String TMP = System.getProperty("java.io.tmpdir"); static final Class[] NUMBER_TYPES = {long.class, double.class}; static final Class[] EXCERPT_TYPES = {/*ByteBuffer.class,*/ Unsafe.class}; private static final int NUM_ENTRIES_PER_RECORD = 20; private static final int NUM_RECORDS = 50 * 1000; private static final int NUM_WARMUP_RECORDS = 10 * 1000; private static final int TOTAL_RECORDS = NUM_RECORDS + NUM_WARMUP_RECORDS; private static final long[][] RANDOM_LONGS = new long[TOTAL_RECORDS][NUM_ENTRIES_PER_RECORD]; private static final double[][] RANDOM_DOUBLES = new double[TOTAL_RECORDS][NUM_ENTRIES_PER_RECORD]; private static final int MAX_PRECISION = 8; private static void timeAppends( @NotNull Class excerptType, Class numType) throws IOException { String newPath = TMP + File.separator + excerptType.getSimpleName() + "Ic"; ChronicleTools.deleteOnExit(newPath); IndexedChronicle newIc = new IndexedChronicle(newPath); // newIc.useUnsafe(excerptType == Unsafe.class); ExcerptAppender excerpt = newIc.createAppender(); long start = 0; for (int i = -NUM_WARMUP_RECORDS; i < TOTAL_RECORDS; i++) { if (i == 0) start = System.nanoTime(); int precision = Math.abs(i) % MAX_PRECISION + 1; excerpt.startExcerpt(); if (numType == long.class) { long[] longs = RANDOM_LONGS[Math.abs(i)]; for (int j = 0; j < NUM_ENTRIES_PER_RECORD; j++) { excerpt.append(longs[j]); excerpt.append(' '); // need something in between. } } else if (numType == double.class) { double[] doubles = RANDOM_DOUBLES[Math.abs(i)]; for (int j = 0; j < NUM_ENTRIES_PER_RECORD; j++) { excerpt.append(doubles[j], precision); excerpt.append(' '); // need something in between. } } else { fail(); } excerpt.finish(); } newIc.close(); long time = System.nanoTime() - start; System.out.printf("%s %s average time taken %d ns%n", numType, excerptType, time / TOTAL_RECORDS / NUM_ENTRIES_PER_RECORD); } @Before public void fillRandoms() { Random random = new Random(); for (int i = 0; i < TOTAL_RECORDS; i++) { for (int j = 0; j < NUM_ENTRIES_PER_RECORD; j++) { RANDOM_LONGS[i][j] = random.nextLong(); RANDOM_DOUBLES[i][j] = random.nextDouble() * 1e10; } RANDOM_LONGS[i][0] = Long.MIN_VALUE; RANDOM_LONGS[i][1] = Long.MAX_VALUE; RANDOM_LONGS[i][2] = 0; RANDOM_DOUBLES[i][0] = 0; } } @Test public void testNumberAppends() throws IOException { // for (int i = 0; i < 3; i++) { for (Class type : NUMBER_TYPES) for (Class excerptType : EXCERPT_TYPES) timeAppends(excerptType, type); // } } } Chronicle-Queue-2.0.3/chronicle/src/test/java/net/openhft/chronicle/examples/000077500000000000000000000000001226752375100271705ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle/src/test/java/net/openhft/chronicle/examples/CachePerfMain.java000066400000000000000000000170101226752375100324570ustar00rootroot00000000000000/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.openhft.chronicle.examples; import gnu.trove.map.TIntIntMap; import gnu.trove.map.hash.TIntIntHashMap; import net.openhft.chronicle.Chronicle; import net.openhft.chronicle.Excerpt; import net.openhft.chronicle.ExcerptAppender; import net.openhft.chronicle.IndexedChronicle; import net.openhft.chronicle.tools.ChronicleTools; import net.openhft.lang.io.Bytes; import net.openhft.lang.io.IOTools; import net.openhft.lang.io.serialization.BytesMarshallable; import org.jetbrains.annotations.NotNull; import java.io.IOException; import java.util.Random; /** * @author ygokirmak */ public class CachePerfMain { private static final String TMP = System.getProperty("java.io.tmpdir"); private static int[] keyArray; @NotNull private final Chronicle chronicle; @NotNull private final Excerpt randomAccessor; @NotNull private final ExcerptAppender appender; private final TIntIntMap keyIndex = new TIntIntHashMap() { @Override public int getNoEntryValue() { return -1; } }; private final int _maxObjSize; public CachePerfMain(String basePath, int maxObjSize) throws IOException { chronicle = new IndexedChronicle(basePath); appender = chronicle.createAppender(); randomAccessor = chronicle.createExcerpt(); _maxObjSize = maxObjSize; } static final int keys = Integer.getInteger("keys", 10000000); public static void main(String... ignored) throws IOException { String basePath = TMP + "/ExampleCacheMain"; ChronicleTools.deleteOnExit(basePath); CachePerfMain map = new CachePerfMain(basePath, 64); long duration; buildkeylist(keys); StringBuilder name = new StringBuilder(); StringBuilder surname = new StringBuilder(); Person person = new Person(name, surname, 0); for (int i = 0; i < 2; i++) { duration = putTest(keys, "base", map); System.out.printf(i + "th iter: Took %.3f secs to put seq %,d entries%n", duration / 1e9, keys); } for (int i = 0; i < 2; i++) { duration = getTest(keys, map); System.out.printf(i + "th iter: Took %.3f secs to get seq %,d entries%n", duration / 1e9, keys); } System.out.println("before shuffle"); shufflelist(); System.out.println("after shuffle"); for (int i = 0; i < 2; i++) { System.gc(); duration = getTest(keys, map); System.out.printf(i + "th iter: Took %.3f secs to get random %,d entries%n", duration / 1e9, keys); } for (int i = 0; i < 2; i++) { duration = putTest(keys, "modif", map); System.out .printf(i + "th iter: Took %.3f secs to update random %,d entries%n", duration / 1e9, keys); } } static long putTest(int keycount, String prefix, CachePerfMain map) { long start = System.nanoTime(); StringBuilder name = new StringBuilder(); StringBuilder surname = new StringBuilder(); Person person = new Person(name, surname, 0); for (int i = 0; i < keys; i++) { name.setLength(0); name.append(prefix); name.append("name"); name.append(i); surname.setLength(0); surname.append(prefix); surname.append("surname"); surname.append(i); person.set_age(i % 100); map.put(i, person); } return System.nanoTime() - start; } static void shufflelist() { Random rnd = new Random(); int size = keyArray.length; for (int i = size; i > 1; i--) swap(keyArray, i - 1, rnd.nextInt(i)); } private static void swap(int[] ints, int x, int y) { int t = ints[x]; ints[x] = ints[y]; ints[y] = t; } static void buildkeylist(int keycount) { keyArray = new int[keycount]; for (int i = 0; i < keycount; i++) { keyArray[i] = i; } } static long getTest(int keycount, CachePerfMain map) { long start = System.nanoTime(); Person person = new Person(); for (int i = 0; i < keycount; i++) { map.get(keyArray[i], person); } return System.nanoTime() - start; } public void get(int key, Person person) { // Change reader position randomAccessor.index(keyIndex.get(key)); // Read contents into byte buffer person.readMarshallable(randomAccessor); // validate reading was correct randomAccessor.finish(); } public void put(int key, Person person) { if (keyIndex.containsKey(key)) { // update existing record // Change accessor index to record. randomAccessor.index(keyIndex.get(key)); // Override existing person.writeMarshallable(randomAccessor); } else { // Start an excerpt with given chunksize appender.startExcerpt(_maxObjSize); // Write the object bytes person.writeMarshallable(appender); // pad it for later. appender.position(_maxObjSize); // Get the position of the excerpt for further access. long index = appender.index(); // finish works as "commit" consider transactional // consistency between putting key to map and putting object to chronicle appender.finish(); // Put the position of the excerpt with its key to a map. keyIndex.put(key, (int) index); } } public void close() { IOTools.close(chronicle); } // Took 5.239 secs to add 10,000,000 entries static class Person implements BytesMarshallable { private StringBuilder _name; private StringBuilder _surname; private int _age; public Person() { this(new StringBuilder(), new StringBuilder(), 0); } public Person(StringBuilder name, StringBuilder surname, int age) { _name = name; _surname = surname; _age = age; } public StringBuilder get_name() { return _name; } public StringBuilder get_surname() { return _surname; } public int get_age() { return _age; } public void set_age(int age) { _age = age; } @Override public void writeMarshallable(@NotNull Bytes out) { out.writeUTFΔ(_name); out.writeUTFΔ(_surname); out.writeStopBit(_age); } @Override public void readMarshallable(@NotNull Bytes in) throws IllegalStateException { in.readUTFΔ(_name); in.readUTFΔ(_surname); _age = (int) in.readStopBit(); } } }Chronicle-Queue-2.0.3/chronicle/src/test/java/net/openhft/chronicle/examples/ExampleCache2Main.java000066400000000000000000000130111226752375100332350ustar00rootroot00000000000000/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.openhft.chronicle.examples; import gnu.trove.map.TLongLongMap; import gnu.trove.map.hash.TLongLongHashMap; import net.openhft.chronicle.*; import net.openhft.chronicle.tools.ChronicleTools; import net.openhft.lang.io.Bytes; import net.openhft.lang.io.IOTools; import net.openhft.lang.io.serialization.BytesMarshallable; import org.jetbrains.annotations.NotNull; import java.io.IOException; /** * @author ygokirmak *

* This is just a simple try for a cache implementation * Future Improvements * 1- Test multiple writer concurrency and performance * 2- Support variable size objects. */ public class ExampleCache2Main { private static final String TMP = System.getProperty("java.io.tmpdir"); @NotNull private final Chronicle chronicle; @NotNull private final Excerpt reader; @NotNull private final ExcerptAppender appender; private final TLongLongMap keyIndex = new TLongLongHashMap() { @Override public long getNoEntryValue() { return -1L; } }; private final int _maxObjSize; public ExampleCache2Main(String basePath) throws IOException { this(basePath, 128 * 1024); } public ExampleCache2Main(String basePath, int maxObjSize) throws IOException { ChronicleConfig config = ChronicleConfig.DEFAULT.clone(); chronicle = new IndexedChronicle(basePath, config); appender = chronicle.createAppender(); reader = chronicle.createExcerpt(); _maxObjSize = maxObjSize; } public static void main(String... ignored) throws IOException { String basePath = TMP + "/ExampleCacheMain"; ChronicleTools.deleteOnExit(basePath); ExampleCache2Main map = new ExampleCache2Main(basePath); long start = System.nanoTime(); int keys = 10000000; StringBuilder name = new StringBuilder(); StringBuilder surname = new StringBuilder(); Person person = new Person(name, surname, 0); for (int i = 0; i < keys; i++) { name.setLength(0); name.append("name"); name.append(i); surname.setLength(0); surname.append("surname"); surname.append(i); person.set_age(i); map.put(i, person); } Person person2 = new Person(); for (int i = 0; i < keys; i++) { map.get(i, person); // System.out.println(p.get_name() + " " + p.get_surname() + " " // + p.get_age()); } long time = System.nanoTime() - start; System.out.printf("Took %.3f secs to add %,d entries%n", time / 1e9, keys); } public void get(long key, Person person) { // Get the excerpt position for the given key from keyIndex map long position = keyIndex.get(key); // Change reader position reader.index(position); // Read contents into byte buffer person.readMarshallable(reader); // validate reading was correct reader.finish(); } public void put(long key, Person person) { // Start an excerpt with given chunksize appender.startExcerpt(_maxObjSize); // Write the object bytes person.writeMarshallable(appender); // pad it for later. appender.position(_maxObjSize); // Get the position of the excerpt for further access. long index = appender.index(); // TODO Does finish works as "commit" consider transactional // consistency between putting key to map and putting object to // chronicle appender.finish(); // Put the position of the excerpt with its key to a map. keyIndex.put(key, index); } public void close() { IOTools.close(chronicle); } // Took 5.239 secs to add 10,000,000 entries static class Person implements BytesMarshallable { private StringBuilder _name; private StringBuilder _surname; private int _age; public Person() { this(new StringBuilder(), new StringBuilder(), 0); } public Person(StringBuilder name, StringBuilder surname, int age) { _name = name; _surname = surname; _age = age; } public StringBuilder get_name() { return _name; } public StringBuilder get_surname() { return _surname; } public int get_age() { return _age; } public void set_age(int age) { _age = age; } @Override public void writeMarshallable(@NotNull Bytes out) { out.writeUTFΔ(_name); out.writeUTFΔ(_surname); out.writeInt(_age); } @Override public void readMarshallable(@NotNull Bytes in) throws IllegalStateException { in.readUTFΔ(_name); in.readUTFΔ(_surname); _age = in.readInt(); } } } Chronicle-Queue-2.0.3/chronicle/src/test/java/net/openhft/chronicle/examples/ExampleCacheMain.java000066400000000000000000000120471226752375100331630ustar00rootroot00000000000000/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.openhft.chronicle.examples; import gnu.trove.map.TLongLongMap; import gnu.trove.map.hash.TLongLongHashMap; import net.openhft.chronicle.*; import net.openhft.chronicle.tools.ChronicleTools; import net.openhft.lang.io.IOTools; import org.jetbrains.annotations.NotNull; import java.io.Externalizable; import java.io.IOException; import java.io.ObjectInput; import java.io.ObjectOutput; /** * @author ygokirmak *

* This is just a simple try for a cache implementation * Future Improvements * 1- Test multiple writer concurrency and performance * 2- Support variable size objects. */ public class ExampleCacheMain { private static final String TMP = System.getProperty("java.io.tmpdir"); @NotNull private final Chronicle chronicle; @NotNull private final Excerpt reader; @NotNull private final ExcerptAppender appender; private final TLongLongMap keyIndex = new TLongLongHashMap() { @Override public long getNoEntryValue() { return -1L; } }; private final int _maxObjSize; public ExampleCacheMain(String basePath) throws IOException { this(basePath, 128 * 1024); } public ExampleCacheMain(String basePath, int maxObjSize) throws IOException { ChronicleConfig config = ChronicleConfig.DEFAULT.clone(); chronicle = new IndexedChronicle(basePath, config); appender = chronicle.createAppender(); reader = chronicle.createExcerpt(); _maxObjSize = maxObjSize; } public static void main(String... ignored) throws IOException { String basePath = TMP + "/ExampleCacheMain"; ChronicleTools.deleteOnExit(basePath); ExampleCacheMain map = new ExampleCacheMain(basePath, 200); long start = System.nanoTime(); int keys = 1000000; for (int i = 0; i < keys; i++) { map.put(i, new Person("name" + i, "surname" + i, i)); } for (int i = 0; i < keys; i++) { Person p = (Person) map.get(i); // System.out.println(p.get_name() + " " + p.get_surname() + " " // + p.get_age()); } long time = System.nanoTime() - start; System.out.printf("Took %.3f secs to add %,d entries%n", time / 1e9, keys); } public Object get(long key) { // Get the excerpt position for the given key from keyIndex map long position = keyIndex.get(key); // Change reader position reader.index(position); // Read contents into byte buffer Object ret = reader.readObject(); // validate reading was correct reader.finish(); return ret; } public void put(long key, Object value) { // Start an excerpt with given chunksize appender.startExcerpt(_maxObjSize); // Write the object bytes appender.writeObject(value); // pad it for later. appender.position(_maxObjSize); // Get the position of the excerpt for further access. long index = appender.index(); // TODO Does finish works as "commit" consider transactional // consistency between putting key to map and putting object to // chronicle appender.finish(); // Put the position of the excerpt with its key to a map. keyIndex.put(key, index); } public void close() { IOTools.close(chronicle); } // Took 7.727 secs to add 1,000,000 entries with Serializable // Took 2.599 secs to add 1,000,000 entries with Externalizable static class Person implements Externalizable { private static final long serialVersionUID = 1L; private String _name; private String _surname; private int _age; public Person(String name, String surname, int age) { _name = name; _surname = name; _age = age; } public String get_name() { return _name; } public String get_surname() { return _surname; } public int get_age() { return _age; } public void writeExternal(ObjectOutput out) throws IOException { out.writeUTF(_name); out.writeUTF(_surname); out.writeInt(_age); } public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { _name = in.readUTF(); _surname = in.readUTF(); _age = in.readInt(); } } } ExampleKeyedExcerptMain.java000077500000000000000000000106441226752375100345010ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle/src/test/java/net/openhft/chronicle/examples/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.openhft.chronicle.examples; import gnu.trove.map.TObjectLongMap; import gnu.trove.map.hash.TObjectLongHashMap; import net.openhft.chronicle.*; import net.openhft.chronicle.tools.ChronicleTools; import net.openhft.lang.io.IOTools; import org.jetbrains.annotations.NotNull; import java.io.IOException; import java.util.Collections; import java.util.LinkedHashMap; import java.util.Map; import static org.junit.Assert.assertEquals; /** * @author peter.lawrey */ public class ExampleKeyedExcerptMain { private static final String TMP = System.getProperty("java.io.tmpdir"); @NotNull private final Chronicle chronicle; @NotNull private final Excerpt reader; @NotNull private final ExcerptTailer tailer; @NotNull private final ExcerptAppender appender; private final TObjectLongMap keyToExcerpt = new TObjectLongHashMap() { @Override public long getNoEntryValue() { return -1; } }; public ExampleKeyedExcerptMain(String basePath) throws IOException { ChronicleConfig config = ChronicleConfig.DEFAULT.clone(); // config.indexBlockSize(4*1024); // config.dataBlockSize(4*1024); chronicle = new IndexedChronicle(basePath, config); tailer = chronicle.createTailer(); appender = chronicle.createAppender(); reader = chronicle.createExcerpt(); } public static void main(String... ignored) throws IOException { String basePath = TMP + "/ExampleKeyedExcerptMain"; ChronicleTools.deleteOnExit(basePath); ExampleKeyedExcerptMain map = new ExampleKeyedExcerptMain(basePath); map.load(); long start = System.nanoTime(); int keys = 1000000; for (int i = 0; i < keys; i++) { Map props = new LinkedHashMap(); props.put("a", Integer.toString(i)); // an int. props.put("b", "value-" + i); // String props.put("c", Double.toString(i / 1000.0)); // a double map.putMapFor(Integer.toHexString(i), props); } map.close(); ExampleKeyedExcerptMain map2 = new ExampleKeyedExcerptMain(basePath); map2.load(); long start2 = System.nanoTime(); for (int i = 0; i < keys; i++) { Map props = new LinkedHashMap(); props.put("a", Integer.toString(i)); // an int. props.put("b", "value-" + i); // String props.put("c", Double.toString(i / 1000.0)); // a double Map props2 = map2.getMapFor(Integer.toHexString(i)); assertEquals("i: " + i, props, props2); } map2.close(); long time = System.nanoTime() - start; long time2 = System.nanoTime() - start2; System.out.printf("Took an average of %,d ns to write and read each entry, an average of %,d ns to lookup%n", time / keys, time2 / keys); } public void load() { while (tailer.nextIndex()) { String key = tailer.readUTFΔ(); keyToExcerpt.put(key, tailer.index()); tailer.finish(); } } public void putMapFor(String key, @NotNull Map map) { appender.startExcerpt(); appender.writeUTFΔ(key); appender.writeMap(map); appender.finish(); } @NotNull public Map getMapFor(String key) { long value = keyToExcerpt.get(key); if (value < 0) return Collections.emptyMap(); reader.index(value); // skip the key reader.skip(reader.readStopBit()); Map map = new LinkedHashMap(); reader.readMap(map, String.class, String.class); reader.finish(); return map; } public void close() { IOTools.close(chronicle); } } Chronicle-Queue-2.0.3/chronicle/src/test/java/net/openhft/chronicle/examples/ExampleRewriteMain.java000077500000000000000000000111471226752375100336040ustar00rootroot00000000000000/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.openhft.chronicle.examples; import net.openhft.affinity.AffinityLock; import net.openhft.affinity.AffinityStrategies; import net.openhft.chronicle.ExcerptAppender; import net.openhft.chronicle.ExcerptTailer; import net.openhft.chronicle.IndexedChronicle; import net.openhft.chronicle.tools.ChronicleTools; import java.io.File; import java.io.IOException; import java.util.Arrays; /** * @author peter.lawrey *

* Chronicle 1.7: 50.0% took 0.3 µs, 90.0% took 0.4 µs, 99.0% took 33.5 µs, 99.9% took 66.9 µs, 99.99% took * 119.7 µs, worst took 183 µs Chronicle 2.0: 50.0% took 0.13 µs, 90.0% took 0.15 µs, 99.0% took 0.44 µs, 99.9% * took 14.37 µs, 99.99% took 22.16 µs, worst took 40 µs */ public class ExampleRewriteMain { public static void main(String... ignored) throws IOException { ChronicleTools.warmup(); final String basePath = System.getProperty("java.io.tmpdir") + File.separator + "test"; ChronicleTools.deleteOnExit(basePath); final int[] consolidates = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}; final int warmup = 50000; final int repeats = 5 * 1000 * 1000; final int rate = 1 * 1000 * 1000; final AffinityLock al = AffinityLock.acquireLock(); //Write Thread t = new Thread(new Runnable() { @Override public void run() { try { al.acquireLock(AffinityStrategies.DIFFERENT_CORE).bind(); final IndexedChronicle chronicle = new IndexedChronicle(basePath); // chronicle.useUnsafe(true); // for benchmarks. final ExcerptAppender excerpt = chronicle.createAppender(); for (int i = -warmup; i < repeats; i++) { doSomeThinking(); // start writing an new entry excerpt.startExcerpt(); excerpt.writeLong(System.nanoTime()); excerpt.writeUnsignedShort(consolidates.length); for (final int consolidate : consolidates) { excerpt.writeStopBit(consolidate); } excerpt.finish(); } chronicle.close(); } catch (Exception e) { e.printStackTrace(); } } long last = System.nanoTime(); private void doSomeThinking() { // real programs do some work between messages // this has an impact on the worst case latencies. while (System.nanoTime() - last < 1e9 / rate) ; last = System.nanoTime(); } }); t.start(); //Read final IndexedChronicle chronicle = new IndexedChronicle(basePath); // chronicle.useUnsafe(true); // for benchmarks. final ExcerptTailer excerpt = chronicle.createTailer(); int[] times = new int[repeats]; for (int count = -warmup; count < repeats; count++) { do { /* busy wait */ } while (!excerpt.nextIndex()); final long timestamp = excerpt.readLong(); long time = System.nanoTime() - timestamp; if (count >= 0) times[count] = (int) time; final int nbConsolidates = excerpt.readUnsignedShort(); assert nbConsolidates == consolidates.length; for (int i = 0; i < nbConsolidates; i++) { excerpt.readStopBit(); } excerpt.finish(); } Arrays.sort(times); System.out.printf("After writing %,d excerpts, ", repeats); for (double perc : new double[]{50, 90, 99, 99.9, 99.99}) { System.out.printf("%s%% took %.2f µs, ", perc, times[((int) (repeats * perc / 100))] / 1000.0); } System.out.printf("worst took %d µs%n", times[times.length - 1] / 1000); chronicle.close(); } } ExampleSimpleWriteReadMain.java000077500000000000000000000112321226752375100351370ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle/src/test/java/net/openhft/chronicle/examples/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.openhft.chronicle.examples; import net.openhft.affinity.AffinityLock; import net.openhft.affinity.AffinityStrategies; import net.openhft.chronicle.ExcerptAppender; import net.openhft.chronicle.ExcerptTailer; import net.openhft.chronicle.IndexedChronicle; import net.openhft.chronicle.tools.ChronicleTools; import java.io.IOException; import java.util.Random; /** * @author peter.lawrey */ public class ExampleSimpleWriteReadMain { public static void main(String... args) throws IOException { ChronicleTools.warmup(); final int warmup = 50 * 1000; final int runs = 5 * 1000 * 1000; final int rate = 1000 * 1000; long start = System.nanoTime(); final String basePath = System.getProperty("user.home") + "/ExampleSimpleWriteReadMain"; ChronicleTools.deleteOnExit(basePath); final AffinityLock al = AffinityLock.acquireLock(); new Thread(new Runnable() { @Override public void run() { try { al.acquireLock(AffinityStrategies.DIFFERENT_CORE).bind(); IndexedChronicle ic = new IndexedChronicle(basePath); // ic.useUnsafe(true); // for benchmarks Random random = new Random(); ExcerptAppender excerpt = ic.createAppender(); long next = System.nanoTime(); for (int i = 1; i <= runs; i++) { double v = random.nextDouble(); excerpt.startExcerpt(); excerpt.writeUnsignedByte('M'); // message type excerpt.writeLong(next); // write time stamp excerpt.writeLong(0L); // read time stamp excerpt.writeDouble(v); excerpt.finish(); next += 1e9 / rate - 30; while (System.nanoTime() < next) ; } ic.close(); } catch (IOException e) { throw new AssertionError(e); } } }).start(); IndexedChronicle ic = new IndexedChronicle(basePath); // ic.useUnsafe(true); // for benchmarks int time1 = 0, time3 = 0, time10 = 0, time30 = 0, time100 = 0; ExcerptTailer excerpt = ic.createTailer(); for (int i = 1; i <= runs; i++) { while (!excerpt.nextIndex()) { // busy wait } char ch = (char) excerpt.readUnsignedByte(); long writeTS = excerpt.readLong(); excerpt.writeLong(System.nanoTime()); double d = excerpt.readDouble(); } excerpt = ic.createTailer(); while (excerpt.nextIndex()) { if (excerpt.index() < warmup) continue; excerpt.readUnsignedByte(); long writeTS = excerpt.readLong(); long readTS = excerpt.readLong(); if (readTS <= 0) throw new AssertionError(); long time = readTS - writeTS; if (time > 1000) { if (time > 3000) time3++; if (time > 10000) time10++; if (time > 30000) time30++; if (time > 100000) time100++; time1++; } excerpt.finish(); } ic.close(); long time = System.nanoTime() - start; System.out.printf("Took %.2f seconds to write and read %,d entries%n", time / 1e9, runs); System.out.printf("Time 1us: %.3f%% 3us: %.3f%% 10us: %.3f%% 30us: %.3f%% 100us: %.3f%%%n", time1 * 100.0 / runs, time3 * 100.0 / runs, time10 * 100.0 / runs, time30 * 100.0 / runs, time100 * 100.0 / runs); } } /* On an i7 desktop Took 17.45 to write and read 30,050,000 entries Time 1us: 13.05% 3us: 2.88% 10us: 0.13% 30us: 0.004% 100us: 0.004% On an i5 laptop Took 6.98 to write and read 30,050,000 entries Time 2us: 2.06% 10us: 1.45% 100us: 0.47% 1ms: 0.07% 10ms: 0.000% */ TestManyUpdates2Main.java000077500000000000000000000034301226752375100337400ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle/src/test/java/net/openhft/chronicle/examples/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.openhft.chronicle.examples; import net.openhft.chronicle.ExcerptAppender; import net.openhft.chronicle.IndexedChronicle; import net.openhft.chronicle.tools.ChronicleTools; import java.io.IOException; /** * 68,000,000 inserts took 7.710 seconds on i7-4500 laptop with SSD. * * @author peter.lawrey */ public class TestManyUpdates2Main { public static void main(String... ignored) throws IOException { ChronicleTools.warmup(); String basePath = System.getProperty("java.io.tmpdir") + "/updates"; ChronicleTools.deleteOnExit(basePath); long start = System.nanoTime(); IndexedChronicle chronicle = new IndexedChronicle(basePath); int count = 68 * 1000 * 1000; for (ExcerptAppender e = chronicle.createAppender(); e.index() < count; ) { e.startExcerpt(); e.writeUTFΔ("Message Type."); e.writeLong(System.currentTimeMillis()); for (int i = 0; i < 6; i++) e.writeCompactDouble(i); e.finish(); } chronicle.close(); long time = System.nanoTime() - start; System.out.printf("%,d inserts took %.3f seconds%n", count, time / 1e9); } } TestManyUpdatesMain.java000077500000000000000000000032401226752375100336550ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle/src/test/java/net/openhft/chronicle/examples/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.openhft.chronicle.examples; import net.openhft.chronicle.ExcerptAppender; import net.openhft.chronicle.IndexedChronicle; import net.openhft.chronicle.tools.ChronicleTools; import java.io.IOException; /** * @author peter.lawrey */ public class TestManyUpdatesMain { public static void main(String... ignored) throws IOException { String basePath = System.getProperty("java.io.tmpdir") + "/updates"; ChronicleTools.deleteOnExit(basePath); long start = System.nanoTime(); IndexedChronicle chronicle = new IndexedChronicle(basePath); int count = 10 * 1000 * 1000; for (ExcerptAppender e = chronicle.createAppender(); e.index() < count; ) { e.startExcerpt(); e.appendTimeMillis(System.currentTimeMillis()); e.append(", id=").append(e.index()); e.append(", name=lyj").append(e.index()); e.finish(); } chronicle.close(); long time = System.nanoTime() - start; System.out.printf("%,d inserts took %.3f seconds%n", count, time / 1e9); } } Chronicle-Queue-2.0.3/chronicle/src/test/java/net/openhft/chronicle/osgi/000077500000000000000000000000001226752375100263135ustar00rootroot00000000000000Chronicle-Queue-2.0.3/chronicle/src/test/java/net/openhft/chronicle/osgi/ChronicleBundleTest.java000066400000000000000000000054561226752375100330700ustar00rootroot00000000000000/* * Copyright 2013 Peter Lawrey * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.openhft.chronicle.osgi; import ch.qos.logback.classic.Level; import ch.qos.logback.classic.Logger; import org.junit.Test; import org.junit.runner.RunWith; import org.ops4j.pax.exam.Configuration; import org.ops4j.pax.exam.Option; import org.ops4j.pax.exam.junit.PaxExam; import org.osgi.framework.Bundle; import org.osgi.framework.BundleContext; import org.slf4j.LoggerFactory; import javax.inject.Inject; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.ops4j.pax.exam.CoreOptions.*; /** * @author lburgazzoli *

* Thank for adding OSGi testing to Chronicle. */ @RunWith(PaxExam.class) public class ChronicleBundleTest { @Inject BundleContext context; @Configuration public Option[] config() { Logger root = (Logger) LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME); root.setLevel(Level.INFO); return options( //systemProperty("org.osgi.framework.storage").value("/tmp/felix-cache"), systemProperty("org.osgi.framework.storage.clean").value("true"), systemProperty("org.ops4j.pax.logging.DefaultServiceLog.level").value("WARN"), mavenBundle("net.openhft", "lang", "6.1.1"), mavenBundle("net.openhft", "chronicle", "2.0.2"), junitBundles(), systemPackage("sun.misc"), systemPackage("sun.nio.ch"), cleanCaches() ); } @Test public void checkInject() { assertNotNull(context); } @Test public void checkHelloBundle() { Boolean bundleChronicleFound = false; Boolean bundleChronicleActive = false; Bundle[] bundles = context.getBundles(); for (Bundle bundle : bundles) { if (bundle != null) { if (bundle.getSymbolicName().equals("net.openhft.chronicle")) { bundleChronicleFound = true; if (bundle.getState() == Bundle.ACTIVE) { bundleChronicleActive = true; } } } } assertTrue(bundleChronicleFound); assertTrue(bundleChronicleActive); } } Chronicle-Queue-2.0.3/docs/000077500000000000000000000000001226752375100154465ustar00rootroot00000000000000Chronicle-Queue-2.0.3/docs/FAQ.md000066400000000000000000000262641226752375100164110ustar00rootroot00000000000000# Frequently Asked Questions about Chronicle ## What is Chronicle designed for? Chronicle is design to be a record everything of interest logger and persisted IPC. A key requirement of low latency systems is transparency and you should be able to record enough information that a monitoring system can recreate the state of the system monitored. This allows downstream systems to record any information they need and perform queries without needed to touch the critical system as they can record anything they might need later. Chronicle works best in SEDA style event driven systems, where latency is critical and you need a record of exactly what was performed when. (Without expensive network sniffing/recording systems) ## What was the library originally designed for? The original design was for a low latency trading system which required persistence of everything in and out for a complete record of what happened, when and for deterministic testing. The target round trip time for persisting the request, processing and persisting the response was a micro-second. Key principles are; ultra-low GC (less than one object per event), lock-less, cache friendly data structures. ## What was not in the originally design? The marshalling, de-marshalling and handling of thread safe off heap memory has been added more recently and moving into the Java-Lang module. This library now supports low latency/GC-less writing and reading/parsing or text as well as binary. ## How fast is fast? Chronicle is design to persist messages and replay them in micro-second time. Simple messages are as low as 0.1 micro-seconds. Complex messages might take 10 micro-seconds to write and read. Chronicle is designed to sustain millions of inserts and updates per second. For burst of up to 10% of your main memory, you can sustain rates of 1 - 3 GB/second written. e.g. A laptop with 8 GB of memory might handle bursts of 800 MB at a rate of 1 GB per second. A server with 64 GB of memory might handle a burst of 6.5 GB at a rate of 3 GB per second. If your key system is not measuring latency in micro-seconds and throughput in thousands per second, it is not that fast. It may well be fast enough however. ;) ## How does it scale? It scales vertically. Many distributed systems can scale by adding more boxes. They are designed to handle between 100 and 1000 transactions per second per node. Chronicle is design to handle more transaction per node, in the order of 100K to 1M transactions per second. This means you need far less nodes, between 10 and 100 times less. Vertical scalability is essential for low latency as having more nodes usually increases latency. Having one node which can handle the load of data centre also save money and power consumption. ## What if I have a slow consumer? Chronicle has an advantage over other queuing systems that the consumer can be any amount behind the producer (up to the free space on your disk) Chronicle has been tested where the consumer was more than main memory behind the producer. This reduced the maximum throughput by about half. Most systems, in Java, where the queue exceed the main memory cause the machine to become unusable. Note: the Consumer can stop, restart and continue with minimal impact to the producer, if the data is still in main memory. Having a faster disk sub-system helps in extreme conditions like these. Chronicle has been tested on a laptop with and HDD with a write speed of 12 MB/s and an over-clocked hex core i7 PCI-SSD card which sustained write speed of 900 MB/s. ## What types of Excerpt are there? Chronicle 2.x has three types of excerpt optimised for different purposes. Chronicle chronicle = new IndexedChronicle(basePath); ExcerptAppender appender = chronicle.createAppender(); // sequential writes. ExcerptTailer tailer = chronicle.createTailer(); // sequential reads. Excerpt excerpt = chronicle.createExcerpt(); // random access to existing excerpts. The plain Except is slower so only use this is to need random access. ## How does writing work? You start by making sure there is enough free spaces with appender.startExcerpt(capacity); It doesn't matter to much if the capacity is more than you need as the entry is shrink wrapped at the end. Making the capacity too large only matters at the end of a chunk as it can trigger a new chunk to be allocated when perhaps the end of the previous one would have been enough. Making the capacity a few KB more than it needs to be will have little measurable difference. Then you write the text or binary to the excerpt with the variety of RandomDataOutput (binary) or ByteStringAppender (text) methods. These are all designed to operate without creating garbage. appender.writeXxxx(xxx); // write binary appender.append(value); // write text Note: Serializable objects are supported for compatibility and are ok in small doses. Java Serialization creates a lot of garbage for writing and reading. Say you want to write to an excerpt later and you don't want shrink wrapping // don't do this unless you want to pad the excerpts. appender.position(capacity); // tell the appender the whole capacity is needed. At this point if the program crashes, the entry is lost. On restart, the entry will be ignored. To finish writing (or reading) call finish(); appender.finish(); // update the index. When finish() returns, the data will be written to disk even if the program crashes. By crash I mean a JVM crash not just an Error or Exception thrown. ## What do I do if I really don't know the size? If you have no control of the size you have a concern that you don't control the system. If this is a rare event, and you have to, you can create a large NativeBytes off heap buffer of any size (even > 2 GB) You can serialize into this off heap memory and write the Bytes produced with the actual size. Note: huge memory to memory copies do not have safe points, which means a 1 GB raw memory copy can prevent a GC from starting until it is finished. ## How does reading work? When you read an excerpt, it first checks that index entry is there (the last thing written) if(tailer.nextIndex()) { // read the new excerpt Type value = tailer.readXxxx(); // read a data type. } Finally you must call finish() to perform bounds check. Chronicle doesn't bounds check every access to reduce runtime overhead. tailer.finish(); ## How is disk space managed? A key assumption is that disk space is cheap, or at least it should be. Some organizations have amazing unrealistic (almost unprofessional) internal charging rates, but you should be able to get 100 GB for about one hour of your time. This assumes retail costs for disk compares with minimum wage. The organizational cost of disk is often 10-100x the real cost, but so is your cost to the business. In essence, disk should be cheap and you can record a week to a month of continuous data on one cheap drive. Never the less, there is less maintenance overhead if the chronicle logs rotate themselves and there is work being done to implement this for Chronicle 2.1. Initially, chronicle files will be rotated when they reach a specific number of entries. ## I want to use Chronicle as an off heap cache. What do I do? Chronicle is designed for replay. While it can, and has been used as an off heap persisted cache, it doesn't do this very easily. An old library called HugeCollections will be resurrected to handle collections more cleanly. # Thread safety ## Can I have multiple readers? A given Chronicle can safely have many readers, both inside and outside of the process creating it. To have multiple readers of a Chronicle, you should generally create a new Chronicle per reader pointing at the same underlying Journal. On each of these Chronicles, you will call createTailer and get a new tailer that can be used to read it. These Tailers should never be shared. A less performant option to this is to share a single Chronicle and Tailer and lock access with synchronized or ReentrantLock. Only one Tailer should ever be active at the same time. ## Can I have multiple writers? A given Chronicle should only have a single writer. It is not threadsafe for multiple threads to write to the same Chronicle. Multiple writers in the same process will cause a performance degradation. If you still want to, you need to use some form of external locking. Either synchronized or ReentrantLock may be suitable. You cannot safely write to the same Chronicle from multiple processes. # Replication ## Does Chronicle support replication? Yes, you can wrap the source (single master) with InProcessChronicleSource and the copies with InProcessChronicleSink. This supports TCP replication and means a copy is stored on each client. When file rolling is supported, this will make it easier to delete old files. ## Does Chronicle support UDP replication? No, Chronicle is designed to be both reliable and deterministic. UDP is not designed for this. A hybrid UDP/TCP system is possible is the future. ## How do I know the consumer is up to date? For the tailer, either replicated or not, you can assume you are up to date when nextIndex() returns false for the first time. # Infrequently Asked Questions ## Can records be updated? They can be updated at any time, but you lose any event driven notification to readers at this point. It might be practical to have multiple chronicles, one which stores large updated records, and another for small notifications. ## I want to store large messages, what is the limit. The theoretic limit is about 1 GB as Chronicle 2.x still uses Java's memory mappings. The practical limit without tuning the configuration is about 64 MB. At this point you get significant inefficiencies unless you increase the data allocation chunk size. ## I get an Exception writing or finish()ing an excerpt. What does this mean? Most often this means you wrote more than the capcity allowed. The quaility of the messages is improving? ## I get an Exception attempting to read an Excerpt. What does this mean? Most likely your read code doesn't match your write code. I suggest making your reader and writer separate, stand alone and well tested so you can pick up such errors. ## How does the byte order work with replication? The byte order doesn't change in replication. This means it will work best in a byte endian homogeneous systems. e.g. Windows/Linux x86/x64/ARM. Chronicle may support changing the byte order in future. ## Does chronicle support other serialization libraries? Chronicle supports ObjectInput, ObjectOutput, Appendable, OutputStream and InputStream APIs. It also has a fast copy to/from a byte[]. Chronicle is designed to be faster with persistence than other serialization libraries are without persistence. To date, I haven't found a faster library for serialization without a standardized format. e.g. Chronicle doesn't support JSON or XML yet. Where XML or JSon is neede down stream, I suggest writing in binary format and have the reader incur the overhead of the conversion rather than slow the producer. ## Does Chronicle support a synchronous mode? It does, clone() a ChronicleConfig you like, e.g. DEFAULT and set synchronousMode(true). This will force() a persistence for every finish(). What this does is likely to be OS platform dependant. Chronicle-Queue-2.0.3/docs/HowItWorks.md000066400000000000000000000151301226752375100200500ustar00rootroot00000000000000## How Chronicle Works Chronicle is a Java project focused on building a persisted low latency messaging framework for high performance and critical applications. ## What is Different? In first glance it can be seen as **yet another queue implementation** but it has major design choices that should be emphasized. Using non-heap storage options(RandomAccessFile) Chronicle provides a processing environment where applications does not suffer from GarbageCollection. While implementing high performance and memory-intensive applications ( you heard the fancy term "bigdata"?) in Java; one of the biggest problem is GarbageCollection. GarbageCollection (GC) may slow down your critical operations non-deterministically at any time.. In order to avoid non-determinism and escape from GC delays off-heap memory solutions are addressed. The main idea is to manage your memory manually so does not suffer from GC. Chronicle behaves like a management interface over off-heap memory so you can build your own solutions over it. Chronicle uses RandomAccessFiles while managing memory and this choice brings lots of possibility. Random access files permit non-sequential, or random, access to a file's contents. To access a file randomly, you open the file, seek a particular location, and read from or write to that file. RandomAccessFiles can be seen as "large" C-type byte arrays that you can access any random index "directly" using pointers. File portions can be used as ByteBuffers if the portion is mapped into memory. ### What is the effect of page faults when we have a huge Chronicle and not enough RAM ? Pages are swapped in and out by the OS on demand. Writes are performed asynchronously and under moderate loads don't impact the latency of writes. Sequential reads are also read uses look ahead ie before you ask for them. Random reads perform best when the data accessed is already in memory The size of data stored can exceed the amount of memory you have, provided the amount you use is less than the main memory size, you see little impact. If you exceed main memory size, you will see measurable performance degradation but it is dramatically more graceful than using too much heap. Using the same amount of heap can cause the whole machine to fail. ## Building Blocks Chronicle has three main concepts; Tailer (sequential reads), Excerpt (random reads) and Appender (sequential writes). Chronicle is the main interface for management and can be seen as the Collection class of Chronicle environment. You will reserve a portion of memory and then put/fetch/update records using Chronicle interface. Current version of Chronicle contains IndexedChronicle implementation. IndexedChronicle is a single writer multiple(?) reader Chronicle that you can put huge numbers of objects having different sizes. For each record, IndexedChronicle holds the memory-offset in another index cache for random access. This means IndexedChronicle "knows" where the 3rd object resides in memory this is why it named as "Indexed". But this index is just sequential index, first object has index 0, second object has index 1... If you want to access objects with other logical keys ( i.e via some value of object ) you have to manage your own mapping from logical key to index. Excerpt is the main data container in a Chronicle, each Chronicle is composed of Excerpts. Putting data to a chronicle means starting a new Excerpt, writing data into it and finishing Excerpt at the end. Appender is something like Iterator in Chronicle environment. You add data appending the current chronicle. ### In examples TLongLongMap is used for key mapping. What if we use a performant b+tree implementation like [BTreeMap](https://github.com/jankotek/MapDB/blob/master/src/main/java/org/mapdb/BTreeMap.java) ) HashMap is more performant and lower GC than a B-Tree. The advantage of a b-Tree is that it is sorted. If you don't need this, use a hash map of some kind. ## How it Really Works Lets see Chronicle in action with an example. In this example we simply will: Create a Chronicle, Put a record to chronicle and Read the record from chronicle. package net.openhft.chronicle.examples; import java.io.IOException; import net.openhft.chronicle.ChronicleConfig; import net.openhft.chronicle.Excerpt; import net.openhft.chronicle.ExcerptAppender; import net.openhft.chronicle.IndexedChronicle; import net.openhft.chronicle.tools.ChronicleTools; public class ExampleCacheMain { public static void main(String... ignored) throws IOException { String basePath = System.getProperty("java.io.tmpdir") + "/SimpleChronicle"; ChronicleTools.deleteOnExit(basePath); IndexedChronicle chronicle = new IndexedChronicle(basePath); // write one object ExcerptAppender appender = chronicle.createAppender(); appender.startExcerpt(100); // an upper limit to how much space in bytes this message should need. appender.writeObject("TestMessage"); appender.finish(); // read one object ExcerptTailer reader = chronicle.createTailer(); Object ret = reader.readObject(); reader.finish(); System.out.println(ret); } } Create a chronicle giving Java_temp_directory/SimpleChronicle as the base folder. String basePath = System.getProperty("java.io.tmpdir") + "/SimpleChronicle"; ChronicleTools.deleteOnExit(basePath); IndexedChronicle chronicle = new IndexedChronicle(basePath); IndexedChronicle creates two RandomAccessFile one for indexes and one for data having names relatively: Java_temp_directory/SimpleChronicle.index Java_temp_directory/SimpleChronicle.data Create appender and reader ExcerptAppender appender = chronicle.createAppender(); ExcerptTailer reader = chronicle.createTailer(); NativeExcerptAppender.startExcerpt method does some checks and calculates startAddr and limitAddr(startAddr+100) for this excerpt appender.startExcerpt(100); writeObject method copies the contents of the object int excerpt appender.writeObject("TestMessage"); in finish method object offset is written to index cache. This method acts like a commit, without writing this offset to cache you put data to datacache but not persist it. appender.finish(); In order to read data from data cache, you first need to get physical start address of the data from index cache. Reader.index(0) method does the calculation for you. You read the data and finish reading operation. reader.index(0); // optional as it is at the start already Object ret = reader.readObject(); reader.finish(); End of simple put/get example. Chronicle-Queue-2.0.3/pom.xml000077500000000000000000000024741226752375100160450ustar00rootroot00000000000000 4.0.0 net.openhft Java-Chronicle 2.1-SNAPSHOT pom Chronicle Parent High Performance Logging and Persisted Messaging chronicle chronicle-demo chronicle-sandbox