openmessaging / benchmark Goto Github PK
View Code? Open in Web Editor NEWOpenMessaging Benchmark Framework
License: Apache License 2.0
OpenMessaging Benchmark Framework
License: Apache License 2.0
"drivers" : [ "driver-pulsar/pulsar.yaml" ],
"workers" : [ "http://benchmark-worker-0.benchmark-worker:8080", "http://benchmark-worker-1.benchmark-worker:8080", "http://benchmark-worker-2.benchmark-worker:8080", "http://benchmark-worker-3.benchmark-worker:8080", "http://benchmark-worker-4.benchmark-worker:8080", "http://benchmark-worker-5.benchmark-worker:8080", "http://benchmark-worker-6.benchmark-worker:8080", "http://benchmark-worker-7.benchmark-worker:8080" ],
"workersFile" : null,
"workloads" : [ "workloads/1-topic-16-partitions-1kb.yaml" ]
}
11:51:06.690 [main] INFO - Workloads: {
"1-topic-16-partitions-1kb" : {
"name" : "1 producer / 1 consumers on 1 topic",
"topics" : 1,
"partitionsPerTopic" : 16,
"keyDistributor" : "NO_KEY",
"messageSize" : 1024,
"payloadFile" : "payload/payload-1Kb.data",
"subscriptionsPerTopic" : 1,
"producersPerTopic" : 1,
"consumerPerSubscription" : 1,
"producerRate" : 50000,
"consumerBacklogSizeGB" : 0,
"testDurationMinutes" : 15
}
}
11:51:08.303 [main] INFO - Workers list - producers: [http://benchmark-worker-0.benchmark-worker:8080, http://benchmark-worker-1.benchmark-worker:8080, http://benchmark-worker-2.benchmark-worker:8080, http://benchmark-worker-3.benchmark-worker:8080]
11:51:08.305 [main] INFO - Workers list - consumers: [http://benchmark-worker-4.benchmark-worker:8080, http://benchmark-worker-5.benchmark-worker:8080, http://benchmark-worker-6.benchmark-worker:8080, http://benchmark-worker-7.benchmark-worker:8080]
11:51:08.345 [main] INFO - --------------- WORKLOAD : 1 producer / 1 consumers on 1 topic --- DRIVER : Pulsar---------------
11:51:09.491 [AsyncHttpClient-2-1] ERROR - Failed to do HTTP post request to http://benchmark-worker-0.benchmark-worker:8080/stop-all -- code: 500
11:51:09.494 [AsyncHttpClient-2-2] ERROR - Failed to do HTTP post request to http://benchmark-worker-3.benchmark-worker:8080/stop-all -- code: 500
11:51:09.506 [AsyncHttpClient-2-2] ERROR - Failed to do HTTP post request to http://benchmark-worker-5.benchmark-worker:8080/stop-all -- code: 500
11:51:09.509 [AsyncHttpClient-2-2] ERROR - Failed to do HTTP post request to http://benchmark-worker-1.benchmark-worker:8080/stop-all -- code: 500
11:51:09.510 [AsyncHttpClient-2-1] ERROR - Failed to do HTTP post request to http://benchmark-worker-2.benchmark-worker:8080/stop-all -- code: 500
11:51:09.520 [AsyncHttpClient-2-1] ERROR - Failed to do HTTP post request to http://benchmark-worker-6.benchmark-worker:8080/stop-all -- code: 500
11:51:09.573 [AsyncHttpClient-2-1] ERROR - Failed to do HTTP post request to http://benchmark-worker-4.benchmark-worker:8080/stop-all -- code: 500
11:51:09.577 [main] ERROR - Failed to run the workload '1 producer / 1 consumers on 1 topic' for driver 'driver-pulsar/pulsar.yaml'
java.util.concurrent.CompletionException: java.lang.IllegalArgumentException
at java.util.concurrent.CompletableFuture.encodeThrowable(CompletableFuture.java:273) ~[?:1.8.0_171]
at java.util.concurrent.CompletableFuture.completeThrowable(CompletableFuture.java:280) ~[?:1.8.0_171]
at java.util.concurrent.CompletableFuture.uniApply(CompletableFuture.java:604) ~[?:1.8.0_171]
at java.util.concurrent.CompletableFuture$UniApply.tryFire(CompletableFuture.java:577) ~[?:1.8.0_171]
at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:474) ~[?:1.8.0_171]
at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:1962) ~[?:1.8.0_171]
at org.asynchttpclient.netty.NettyResponseFuture.getContent(NettyResponseFuture.java:226) ~[org.asynchttpclient-async-http-client-2.1.0-alpha26.jar:?]
at org.asynchttpclient.netty.NettyResponseFuture.done(NettyResponseFuture.java:261) ~[org.asynchttpclient-async-http-client-2.1.0-alpha26.jar:?]
at org.asynchttpclient.netty.handler.AsyncHttpClientHandler.finishUpdate(AsyncHttpClientHandler.java:239) ~[org.asynchttpclient-async-http-client-2.1.0-alpha26.jar:?]
at org.asynchttpclient.netty.handler.HttpHandler.handleChunk(HttpHandler.java:120) ~[org.asynchttpclient-async-http-client-2.1.0-alpha26.jar:?]
at org.asynchttpclient.netty.handler.HttpHandler.handleRead(HttpHandler.java:149) ~[org.asynchttpclient-async-http-client-2.1.0-alpha26.jar:?]
at org.asynchttpclient.netty.handler.AsyncHttpClientHandler.channelRead(AsyncHttpClientHandler.java:76) ~[org.asynchttpclient-async-http-client-2.1.0-alpha26.jar:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[io.netty-netty-all-4.1.12.Final.jar:4.1.12.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[io.netty-netty-all-4.1.12.Final.jar:4.1.12.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[io.netty-netty-all-4.1.12.Final.jar:4.1.12.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.channelRead(ChannelInboundHandlerAdapter.java:86) ~[io.netty-netty-all-4.1.12.Final.jar:4.1.12.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[io.netty-netty-all-4.1.12.Final.jar:4.1.12.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[io.netty-netty-all-4.1.12.Final.jar:4.1.12.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[io.netty-netty-all-4.1.12.Final.jar:4.1.12.Final]
at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102) ~[io.netty-netty-all-4.1.12.Final.jar:4.1.12.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[io.netty-netty-all-4.1.12.Final.jar:4.1.12.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[io.netty-netty-all-4.1.12.Final.jar:4.1.12.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[io.netty-netty-all-4.1.12.Final.jar:4.1.12.Final]
at io.netty.channel.CombinedChannelDuplexHandler$DelegatingChannelHandlerContext.fireChannelRead(CombinedChannelDuplexHandler.java:438) ~[io.netty-netty-all-4.1.12.Final.jar:4.1.12.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:310) ~[io.netty-netty-all-4.1.12.Final.jar:4.1.12.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:284) ~[io.netty-netty-all-4.1.12.Final.jar:4.1.12.Final]
at io.netty.channel.CombinedChannelDuplexHandler.channelRead(CombinedChannelDuplexHandler.java:253) ~[io.netty-netty-all-4.1.12.Final.jar:4.1.12.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[io.netty-netty-all-4.1.12.Final.jar:4.1.12.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[io.netty-netty-all-4.1.12.Final.jar:4.1.12.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[io.netty-netty-all-4.1.12.Final.jar:4.1.12.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1334) ~[io.netty-netty-all-4.1.12.Final.jar:4.1.12.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[io.netty-netty-all-4.1.12.Final.jar:4.1.12.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[io.netty-netty-all-4.1.12.Final.jar:4.1.12.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:926) ~[io.netty-netty-all-4.1.12.Final.jar:4.1.12.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:134) ~[io.netty-netty-all-4.1.12.Final.jar:4.1.12.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:644) ~[io.netty-netty-all-4.1.12.Final.jar:4.1.12.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:579) ~[io.netty-netty-all-4.1.12.Final.jar:4.1.12.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:496) ~[io.netty-netty-all-4.1.12.Final.jar:4.1.12.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:458) ~[io.netty-netty-all-4.1.12.Final.jar:4.1.12.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:858) ~[io.netty-netty-all-4.1.12.Final.jar:4.1.12.Final]
at io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:138) ~[io.netty-netty-all-4.1.12.Final.jar:4.1.12.Final]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_171]
Caused by: java.lang.IllegalArgumentException
at com.google.common.base.Preconditions.checkArgument(Preconditions.java:121) ~[com.google.guava-guava-23.3-jre.jar:?]
at io.openmessaging.benchmark.worker.DistributedWorkersEnsemble.lambda$sendPost$6(DistributedWorkersEnsemble.java:268) ~[io.openmessaging.benchmark-benchmark-framework-0.0.1-SNAPSHOT.jar:?]
at java.util.concurrent.CompletableFuture.uniApply(CompletableFuture.java:602) ~[?:1.8.0_171]
... 39 more
All benchmark workers are running in a digitalocean kubernetes cluster with 3 nodes. When I try to run the the benchmark using the command in deployment/kubernetes/helm/README.md
it fails with the above error. All the benchmark-workers are returning HTTP 500 status codes.
23:56:18.846 [kafka-producer-network-thread | producer-3] WARN - Write error on message
java.util.concurrent.CompletionException: org.apache.kafka.common.errors.TimeoutException: Expiring 125 record(s) for test-topic-O1HToMc-0000-11: 30129 ms has passed since batch creation plus linger time
at java.util.concurrent.CompletableFuture.encodeThrowable(CompletableFuture.java:292) ~[?:1.8.0_161]
at java.util.concurrent.CompletableFuture.completeThrowable(CompletableFuture.java:308) ~[?:1.8.0_161]
at java.util.concurrent.CompletableFuture.uniRun(CompletableFuture.java:700) ~[?:1.8.0_161]
at java.util.concurrent.CompletableFuture$UniRun.tryFire(CompletableFuture.java:687) ~[?:1.8.0_161]
at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:474) ~[?:1.8.0_161]
at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:1977) ~[?:1.8.0_161]
at io.openmessaging.benchmark.driver.kafka.KafkaBenchmarkProducer.lambda$sendAsync$0(KafkaBenchmarkProducer.java:46) ~[io.openmessaging.benchmark-driver-kafka-0.0.1-SNAPSHOT.jar:?]
at org.apache.kafka.clients.producer.internals.ProducerBatch.completeFutureAndFireCallbacks(ProducerBatch.java:204) [org.apache.kafka-kafka-clients-1.0.0.jar:?]
at org.apache.kafka.clients.producer.internals.ProducerBatch.done(ProducerBatch.java:187) [org.apache.kafka-kafka-clients-1.0.0.jar:?]
at org.apache.kafka.clients.producer.internals.Sender.failBatch(Sender.java:627) [org.apache.kafka-kafka-clients-1.0.0.jar:?]
at org.apache.kafka.clients.producer.internals.Sender.sendProducerData(Sender.java:287) [org.apache.kafka-kafka-clients-1.0.0.jar:?]
at org.apache.kafka.clients.producer.internals.Sender.run(Sender.java:238) [org.apache.kafka-kafka-clients-1.0.0.jar:?]
at org.apache.kafka.clients.producer.internals.Sender.run(Sender.java:176) [org.apache.kafka-kafka-clients-1.0.0.jar:?]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
Pulsar 2.6.0 released a new feature where you can advertise multiple addresses. This also comes with the requirement that the consumer must specify the listener name that it wants to use for communication between it and the broker, like below:
PulsarClient.builder()
.serviceUrl(url)
.listenerName("internal")
.build();
Currently since the listener name cannot be set the multiple addresses feature is unusable and the consumer cannot connect to a pulsar cluster that requires multiple advertised addresses.
Pulsar 2.6.0 release docs:
https://pulsar.apache.org/blog/2020/06/18/Apache-Pulsar-2-6-0/
Fails to run with more than 4 local workers.
17:39:46.988 [main] INFO - Workers list - producers: [http://10.1.1.10:8080, http://0.0.0.0:8080, http://10.1.1.8:8080]
17:39:46.989 [main] INFO - Workers list - consumers: [http://10.1.1.19:8080, http://10.1.1.16:8080, http://10.1.1.17:8080]
17:39:46.996 [main] INFO - --------------- WORKLOAD : 1 producer / 1 consumers on 1 topic --- DRIVER : Pulsar---------------
17:39:47.609 [main] INFO - Created 4 topics in 18.547497 ms
17:39:47.612 [main] ERROR - Failed to run the workload '1 producer / 1 consumers on 1 topic' for driver 'driver-pulsar/pulsar.yaml'
java.lang.IndexOutOfBoundsException: Index: 3, Size: 3
at java.util.ArrayList$SubList.rangeCheck(ArrayList.java:1225) ~[?:1.8.0_191]
at java.util.ArrayList$SubList.get(ArrayList.java:1042) ~[?:1.8.0_191]
at io.openmessaging.benchmark.worker.DistributedWorkersEnsemble.createConsumers(DistributedWorkersEnsemble.java:166) ~[classes/:?]
at io.openmessaging.benchmark.WorkloadGenerator.createConsumers(WorkloadGenerator.java:289) ~[classes/:?]
at io.openmessaging.benchmark.WorkloadGenerator.run(WorkloadGenerator.java:76) ~[classes/:?]
at io.openmessaging.benchmark.Benchmark.lambda$null$0(Benchmark.java:142) ~[classes/:?]
at java.util.ArrayList.forEach(ArrayList.java:1257) ~[?:1.8.0_191]
at io.openmessaging.benchmark.Benchmark.lambda$main$1(Benchmark.java:127) ~[classes/:?]
at java.util.TreeMap.forEach(TreeMap.java:1005) [?:1.8.0_191]
at io.openmessaging.benchmark.Benchmark.main(Benchmark.java:126) [classes/:?]
I try to benchmark MQs, such as Rocketmq,Kafka, etc. , I found two possibe and correlated little problems, as follows:
The two problems may lead the benchmark same like blocking , actually it's waiting the messages back. the messages may never back if we failed to produce in the beginning or other server side problems .
SendAsync is waiting for confirm each time and the publish rate is 3000msg/s in 1-topic-1-partion-1kb workload
Since Pulsar has already released, would like to update Pulsar client to latest verison
NSQ is a realtime distributed messaging platform, And I think providing a nsq driver is meaningful.
Driver pulsar returns an error in benchmark-worker if the TLS authentication field is empty in pulsar.yaml.
Currently, the deployments are different, for example the rabbitmq is one client and three broker while the rocketmq is four client and two broker. And what's more , rocketmq is deployed on alicloud.
If we want to compare their performance, should it be a analogous deployment?
Nats is a simple, high performance open source messaging system for cloud native applications, IoT messaging, and microservices architectures. And I think providing a nats driver is meaningful
The following is an error I'm getting while attempting to run against workers I've deployed in kubernetes. Is there a different way I should be listing out the workers or am I missing something else?
java.lang.IllegalArgumentException: 192.168.66.0:8080/stop-all could not be parsed into a proper Uri, missing scheme
at org.asynchttpclient.uri.Uri.create(Uri.java:40) ~[org.asynchttpclient-async-http-client-2.1.0-alpha26.jar:?]
at org.asynchttpclient.uri.Uri.create(Uri.java:32) ~[org.asynchttpclient-async-http-client-2.1.0-alpha26.jar:?]
at org.asynchttpclient.RequestBuilderBase.setUrl(RequestBuilderBase.java:151) ~[org.asynchttpclient-async-http-client-2.1.0-alpha26.jar:?]
at org.asynchttpclient.DefaultAsyncHttpClient.requestBuilder(DefaultAsyncHttpClient.java:269) ~[org.asynchttpclient-async-http-client-2.1.0-alpha26.jar:?]
at org.asynchttpclient.DefaultAsyncHttpClient.preparePost(DefaultAsyncHttpClient.java:147) ~[org.asynchttpclient-async-http-client-2.1.0-alpha26.jar:?]
at io.openmessaging.benchmark.worker.DistributedWorkersEnsemble.sendPost(DistributedWorkersEnsemble.java:264) ~[io.openmessaging.benchmark-benchmark-framework-0.0.1-SNAPSHOT.jar:?]
at io.openmessaging.benchmark.worker.DistributedWorkersEnsemble.lambda$sendPost$5(DistributedWorkersEnsemble.java:260) ~[io.openmessaging.benchmark-benchmark-framework-0.0.1-SNAPSHOT.jar:?]
at java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193) ~[?:1.8.0_171]
at java.util.ArrayList$ArrayListSpliterator.forEachRemaining(ArrayList.java:1382) ~[?:1.8.0_171]
at java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:481) ~[?:1.8.0_171]
at java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:471) ~[?:1.8.0_171]
at java.util.stream.ReduceOps$ReduceOp.evaluateSequential(ReduceOps.java:708) ~[?:1.8.0_171]
at java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234) ~[?:1.8.0_171]
at java.util.stream.ReferencePipeline.collect(ReferencePipeline.java:499) ~[?:1.8.0_171]
at io.openmessaging.benchmark.worker.DistributedWorkersEnsemble.sendPost(DistributedWorkersEnsemble.java:260) ~[io.openmessaging.benchmark-benchmark-framework-0.0.1-SNAPSHOT.jar:?]
at io.openmessaging.benchmark.worker.DistributedWorkersEnsemble.stopAll(DistributedWorkersEnsemble.java:143) ~[io.openmessaging.benchmark-benchmark-framework-0.0.1-SNAPSHOT.jar:?]
at io.openmessaging.benchmark.Benchmark.lambda$null$0(Benchmark.java:136) ~[io.openmessaging.benchmark-benchmark-framework-0.0.1-SNAPSHOT.jar:?]
at java.util.ArrayList.forEach(ArrayList.java:1257) ~[?:1.8.0_171]
at io.openmessaging.benchmark.Benchmark.lambda$main$1(Benchmark.java:127) ~[io.openmessaging.benchmark-benchmark-framework-0.0.1-SNAPSHOT.jar:?]
at java.util.TreeMap.forEach(TreeMap.java:1005) [?:1.8.0_171]
at io.openmessaging.benchmark.Benchmark.main(Benchmark.java:126) [io.openmessaging.benchmark-benchmark-framework-0.0.1-SNAPSHOT.jar:?]```
In recent weeks, i try to benchmark several open source Message Queue implementations,such as Kafka,RocketMQ etc and compare them.
Openmesaging benchmark only provides a asyncSend method on BenchmarkProducer now. In proudction, sync and async proudce are equally important . The performance of async and async are signifacant different. Kafka is more effecient on async send message ,but others may outperform kafka on sync send message.
Kafka sync and async prodce benchmark result as following:
-----WORKLOAD : 1 topic / 5 partition / 1Kb / async 10P1C --- DRIVER : Kafka----------
Pub rate 34454.7 msg/s / 33.6 Mb/s | Cons rate 34454.7 msg/s / 33.6 Mb/s | Backlog: 0.0 K | Pub Latency (ms) avg: 1053.9 - 50%: 1014.5 - 99%: 1506.5 - 99.9%: 1510.4 - Max: 1653.3
Pub rate 28774.8 msg/s / 28.1 Mb/s | Cons rate 28774.8 msg/s / 28.1 Mb/s | Backlog: 0.0 K | Pub Latency (ms) avg: 1068.6 - 50%: 1046.5 - 99%: 1659.1 - 99.9%: 1660.8 - Max: 1864.9
Pub rate 30446.1 msg/s / 29.7 Mb/s | Cons rate 30434.7 msg/s / 29.7 Mb/s | Backlog: 0.1 K | Pub Latency (ms) avg: 1051.0 - 50%: 1060.0 - 99%: 1290.5 - 99.9%: 1340.7 - Max: 1449.9
Pub rate 30124.1 msg/s / 29.4 Mb/s | Cons rate 30135.5 msg/s / 29.4 Mb/s | Backlog: 0.0 K | Pub Latency (ms) avg: 1026.9 - 50%: 1029.1 - 99%: 1227.6 - 99.9%: 1267.6 - Max: 1288.0
Pub rate 31099.2 msg/s / 30.4 Mb/s | Cons rate 31099.2 msg/s / 30.4 Mb/s | Backlog: 0.0 K | Pub Latency (ms) avg: 1018.7 - 50%: 1009.4 - 99%: 1195.4 - 99.9%: 1288.1 - Max: 1317.7
Pub rate 27860.3 msg/s / 27.2 Mb/s | Cons rate 27860.3 msg/s / 27.2 Mb/s | Backlog: 0.0 K | Pub Latency (ms) avg: 1114.4 - 50%: 1029.7 - 99%: 1898.3 - 99.9%: 1977.4 - Max: 1979.9
Aggregated Pub Latency (ms) avg: 1055.6 - 50%: 1031.3 - 95%: 1337.8 - 99%: 1821.4 - 99.9%: 1901.6 - 99.99%: 1977.6 - Max: 1979.9
Pub rate 967.3 msg/s / 0.9 Mb/s | Cons rate 967.3 msg/s / 0.9 Mb/s | Backlog: 0.0 K | Pub Latency (ms) avg: 3.1 - 50%: 2.1 - 99%: 18.4 - 99.9%: 22.4 - Max: 35.3
Pub rate 917.1 msg/s / 0.9 Mb/s | Cons rate 917.2 msg/s / 0.9 Mb/s | Backlog: -0.0 K | Pub Latency (ms) avg: 3.3 - 50%: 2.1 - 99%: 20.9 - 99.9%: 24.2 - Max: 27.8
Pub rate 865.6 msg/s / 0.8 Mb/s | Cons rate 865.8 msg/s / 0.8 Mb/s | Backlog: -0.0 K | Pub Latency (ms) avg: 3.5 - 50%: 2.1 - 99%: 23.1 - 99.9%: 27.1 - Max: 35.2
Pub rate 802.7 msg/s / 0.8 Mb/s | Cons rate 802.4 msg/s / 0.8 Mb/s | Backlog: 0.0 K | Pub Latency (ms) avg: 3.7 - 50%: 2.1 - 99%: 24.6 - 99.9%: 34.0 - Max: 368.7
Pub rate 758.1 msg/s / 0.7 Mb/s | Cons rate 758.1 msg/s / 0.7 Mb/s | Backlog: 0.0 K | Pub Latency (ms) avg: 4.0 - 50%: 2.1 - 99%: 26.8 - 99.9%: 31.9 - Max: 33.9
Pub rate 935.0 msg/s / 0.9 Mb/s | Cons rate 935.0 msg/s / 0.9 Mb/s | Backlog: 0.0 K | Pub Latency (ms) avg: 3.2 - 50%: 2.1 - 99%: 26.7 - 99.9%: 32.8 - Max: 34.6
Aggregated Pub Latency (ms) avg: 3.4 - 50%: 2.1 - 95%: 19.1 - 99%: 25.6 - 99.9%: 30.7 - 99.99%: 48.3 - Max: 368.7
Maybe , we could provide a syncSend method and with a extra config key:syncProducer=true/false on payload config file,which decide whether sync or async send message
Environment:
docker-compose:1.23.2
zookeeper:3.5.5
bookkeeper:4.8.2
distributedlog-core-shaded:4.8.2
For openmessaging bookkeeper benchmark when i set producersPerTopic with value 1 all looks fine.
But when i set greater than 1 i got errors:
dlshade.org.apache.zookeeper.KeeperException$BadVersionException: KeeperErrorCode = BadVersion
at dlshade.org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[distributedlog-core-shaded-4.8.2.jar:4.8.2]
at org.apache.distributedlog.util.Utils$5.processResult(Utils.java:395) [distributedlog-core-shaded-4.8.2.jar:4.8.2]
at dlshade.org.apache.bookkeeper.zookeeper.ZooKeeperClient$22$1.processResult(ZooKeeperClient.java:1091) [distributedlog-core-shaded-4.8.2.jar:4.8.2]
at dlshade.org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:566) [distributedlog-core-shaded-4.8.2.jar:4.8.2]
at dlshade.org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:508) [distributedlog-core-shaded-4.8.2.jar:4.8.2]
14:02:54.222 [local-worker-1-1] WARN - Write error on message
java.util.concurrent.CompletionException: org.apache.distributedlog.exceptions.WriteException: Write rejected because stream test-stream-Ir8h5B8-0000 has encountered an error : writer has been closed due to error. at java.util.concurrent.CompletableFuture.encodeThrowable(CompletableFuture.java:331) ~[?:?]
at java.util.concurrent.CompletableFuture.completeThrowable(CompletableFuture.java:346) ~[?:?]
at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:870) [?:?]
at java.util.concurrent.CompletableFuture.uniWhenCompleteStage(CompletableFuture.java:883) [?:?]
at java.util.concurrent.CompletableFuture.whenComplete(CompletableFuture.java:2251) [?:?]
at dlshade.org.apache.bookkeeper.common.concurrent.FutureUtils.stats(FutureUtils.java:365) ~[distributedlog-core-shaded-4.8.2.jar:4.8.2]
at org.apache.distributedlog.BKAsyncLogWriter.write(BKAsyncLogWriter.java:415) ~[distributedlog-core-shaded-4.8.2.jar:4.8.2]
at io.openmessaging.benchmark.driver.bookkeeper.DlogBenchmarkProducer.sendAsync(DlogBenchmarkProducer.java:50) ~[driver-bookkeeper-0.0.1-SNAPSHOT.jar:?]
at io.openmessaging.benchmark.worker.LocalWorker.lambda$submitProducersToExecutor$10(LocalWorker.java:219) [classes/:?]
at java.util.ArrayList.forEach(ArrayList.java:1540) [?:?]
at io.openmessaging.benchmark.worker.LocalWorker.lambda$submitProducersToExecutor$11(LocalWorker.java:216) [classes/:?]
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) [?:?]
at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?]
at io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:138) [netty-all-4.1.12.Final.jar:4.1.12.Final]
at java.lang.Thread.run(Thread.java:834) [?:?]
Caused by: org.apache.distributedlog.exceptions.WriteException: Write rejected because stream test-stream-Ir8h5B8-0000 has encountered an error : writer has been closed due to error.
at org.apache.distributedlog.BKAsyncLogWriter.getCachedLogSegmentWriter(BKAsyncLogWriter.java:189) ~[distributedlog-core-shaded-4.8.2.jar:4.8.2]
at org.apache.distributedlog.BKAsyncLogWriter.asyncWrite(BKAsyncLogWriter.java:280) ~[distributedlog-core-shaded-4.8.2.jar:4.8.2]
at org.apache.distributedlog.BKAsyncLogWriter.write(BKAsyncLogWriter.java:416) ~[distributedlog-core-shaded-4.8.2.jar:4.8.2]
... 10 more
openmessaging output:
14:02:52.937 [main] INFO - Starting benchmark with config: {
"drivers" : [ "driver-bookkeeper/bookkeeper.yaml" ],
"workers" : null,
"workersFile" : null,
"workloads" : [ "workloads/1-topic-1-partition-100b.yaml" ]
}
14:02:52.975 [main] INFO - Workloads: {
"1-topic-1-partition-100b" : {
"name" : "1 topic / 1 partition / 100b",
"topics" : 1,
"partitionsPerTopic" : 1,
"keyDistributor" : "NO_KEY",
"messageSize" : 100,
"payloadFile" : "payload/payload-100b.data",
"subscriptionsPerTopic" : 1,
"producersPerTopic" : 2,
"consumerPerSubscription" : 1,
"producerRate" : 1000,
"consumerBacklogSizeGB" : 0,
"testDurationMinutes" : 1
}
}
14:02:53.026 [main] INFO - --------------- WORKLOAD : 1 topic / 1 partition / 100b --- DRIVER : BookKeeper---------------
14:02:53.197 [main] INFO - Driver: {
"name" : "BookKeeper",
"driverClass" : "io.openmessaging.benchmark.driver.bookkeeper.DlogBenchmarkDriver"
}
14:02:53.337 [main] INFO - DistributedLog NamespaceDriverManager initialized
14:02:53.337 [main] INFO - DistributedLog NamespaceDriverManager initialized
14:02:53.355 [main] INFO - No feature provider is set. All features are disabled now.
14:02:53.388 [main] INFO - Created shared zooKeeper client builder dlzk:distributedlog://zookeeper:2181/distributedlog:factory_writer_shared: zkServers = zookeeper:2181, numRetries = 0, sessionTimeout = 30000, retryBackoff = 100, maxRetryBackoff = 200, zkAclId = null.
14:02:53.409 [main] INFO - Client environment:zookeeper.version=3.4.13-2d71af4dbe22557fda74f9a9b4309b15a7487f03, built on 06/29/2018 00:39 GMT
14:02:53.409 [main] INFO - Client environment:host.name=5e95860dad58
14:02:53.409 [main] INFO - Client environment:java.version=11.0.6
14:02:53.409 [main] INFO - Client environment:java.vendor=Oracle Corporation
14:02:53.409 [main] INFO - Client environment:java.home=/usr/local/openjdk-11
14:02:53.411 [main] INFO - Client environment:java.library.path=/usr/java/packages/lib:/usr/lib64:/lib64:/lib:/usr/lib
14:02:53.411 [main] INFO - Client environment:java.io.tmpdir=/tmp
14:02:53.411 [main] INFO - Client environment:java.compiler=<NA>
14:02:53.411 [main] INFO - Client environment:os.name=Linux
14:02:53.411 [main] INFO - Client environment:os.arch=amd64
14:02:53.411 [main] INFO - Client environment:os.version=3.10.0-957.el7.x86_64
14:02:53.411 [main] INFO - Client environment:user.name=root
14:02:53.411 [main] INFO - Client environment:user.home=/root
14:02:53.411 [main] INFO - Client environment:user.dir=/opt/openmessaging-benchmark
14:02:53.414 [main] INFO - Initiating client connection, connectString=zookeeper:2181 sessionTimeout=30000 watcher=dlshade.org.apache.bookkeeper.zookeeper.ZooKeeperWatcherBase@6de7778f
14:02:53.431 [main-SendThread(zookeeper:2181)] INFO - Opening socket connection to server zookeeper/172.22.0.2:2181. Will not attempt to authenticate using SASL (unknown error)
14:02:53.439 [main-SendThread(zookeeper:2181)] INFO - Socket connection established to zookeeper/172.22.0.2:2181, initiating session
14:02:53.444 [main-SendThread(zookeeper:2181)] INFO - Session establishment complete on server zookeeper/172.22.0.2:2181, sessionid = 0x1021228d8370018, negotiated timeout = 30000
14:02:53.446 [main-EventThread] INFO - ZooKeeper client is connected now.
14:02:53.539 [main] INFO - Created shared client builder bk:distributedlog://zookeeper:2181/distributedlog:factory_writer_shared : zkServers = zookeeper:2181, ledgersPath = /ledgers, numIOThreads = 8
14:02:53.541 [main] INFO - Propagate BKDLConfig to DLConfig : encodeRegionID = false, firstLogSegmentSequenceNumber = 1, createStreamIfNotExists = true, isFederated = false.
14:02:53.550 [main] INFO - Initialized BK namespace driver: clientId = 5e95860dad58/172.22.0.3, regionId = 0, federated = false.
14:02:53.568 [main] INFO - Log segment cache is enabled = true
14:02:53.568 [main] INFO - Initialized distributedlog namespace at distributedlog://zookeeper:2181/distributedlog
14:02:53.606 [ForkJoinPool.commonPool-worker-3] INFO - Successfully create topic test-stream-Ir8h5B8-0000 with 1 partitions
14:02:53.606 [main] INFO - Created 1 topics in 35.403847 ms
14:02:53.606 [main] INFO - Created 1 topics in 35.751094 ms
14:02:53.619 [ForkJoinPool.commonPool-worker-3] INFO - Open stream test-stream-Ir8h5B8-0000 for consumer
14:02:53.621 [dlog-benchmark-reader-thread-0] INFO - Create sync reader starting from DLSN{logSegmentSequenceNo=1, entryId=0, slotId=0}
14:02:53.627 [main] INFO - Created 1 consumers in 19.870688 ms
14:02:53.627 [main] INFO - Created 1 consumers in 19.997035 ms
14:02:53.635 [ForkJoinPool.commonPool-worker-3] INFO - Open stream test-stream-Ir8h5B8-0000 for producer
14:02:53.636 [ForkJoinPool.commonPool-worker-5] INFO - Open stream test-stream-Ir8h5B8-0000 for producer
14:02:53.645 [main-EventThread] INFO - Ledger allocator for /distributedlog/test-stream-Ir8h5B8-0000/<default>/allocation moved version from -1 to 0.
14:02:53.647 [dlog-benchmark-reader-thread-0] INFO - Successfully open log reader for stream test-stream-Ir8h5B8-0000 at DLSN{logSegmentSequenceNo=1, entryId=0, slotId=0}
14:02:53.658 [main-EventThread] INFO - Ledger allocator for /distributedlog/test-stream-Ir8h5B8-0000/<default>/allocation moved version from -1 to 0.
14:02:53.668 [main-EventThread] INFO - Starting the readahead entry reader for test-stream-Ir8h5B8-0000:<default> : segments = []
14:02:53.668 [main-EventThread] INFO - Initiating Recovery For test-stream-Ir8h5B8-0000:<default> : []
14:02:53.669 [main-EventThread] INFO - Initiating Recovery For test-stream-Ir8h5B8-0000:<default> : []
14:02:53.670 [main] INFO - Created 2 producers in 43.022277 ms
14:02:53.670 [main] INFO - Created 2 producers in 43.173542 ms
14:02:53.671 [main] INFO - Waiting for consumers to be ready
14:02:53.690 [main] INFO - Initiating Recovery For test-stream-Ir8h5B8-0000:<default> : []
14:02:53.693 [DLM-/distributedlog-OrderedScheduler-0-0] INFO - Ledger allocator /distributedlog/test-stream-Ir8h5B8-0000/<default>/allocation moved to phase ALLOCATING : version = 0.
14:02:53.695 [main] INFO - Initiating Recovery For test-stream-Ir8h5B8-0000:<default> : []
14:02:53.696 [DLM-/distributedlog-OrderedScheduler-3-0] INFO - Ledger allocator /distributedlog/test-stream-Ir8h5B8-0000/<default>/allocation moved to phase ALLOCATING : version = 0.
14:02:53.711 [DLM-/distributedlog-OrderedScheduler-0-0] INFO - Initiating client connection, connectString=zookeeper:2181 sessionTimeout=60000 watcher=dlshade.org.apache.bookkeeper.zookeeper.ZooKeeperWatcherBase@4b17568f
14:02:53.713 [DLM-/distributedlog-OrderedScheduler-0-0-SendThread(zookeeper:2181)] INFO - Opening socket connection to server zookeeper/172.22.0.2:2181. Will not attempt to authenticate using SASL (unknown error)
14:02:53.713 [DLM-/distributedlog-OrderedScheduler-0-0-SendThread(zookeeper:2181)] INFO - Socket connection established to zookeeper/172.22.0.2:2181, initiating session
14:02:53.715 [DLM-/distributedlog-OrderedScheduler-0-0-SendThread(zookeeper:2181)] INFO - Session establishment complete on server zookeeper/172.22.0.2:2181, sessionid = 0x1021228d8370019, negotiated timeout = 40000
14:02:53.715 [DLM-/distributedlog-OrderedScheduler-0-0-EventThread] INFO - ZooKeeper client is connected now.
14:02:53.733 [DLM-/distributedlog-OrderedScheduler-0-0] INFO - BookKeeper metadata driver manager initialized
14:02:53.733 [DLM-/distributedlog-OrderedScheduler-0-0] INFO - BookKeeper metadata driver manager initialized
14:02:53.734 [DLM-/distributedlog-OrderedScheduler-0-0] INFO - BookKeeper metadata driver manager initialized
14:02:53.735 [DLM-/distributedlog-OrderedScheduler-0-0] INFO - Initialize zookeeper metadata driver with external zookeeper client : ledgersRootPath = /ledgers.
14:02:53.753 [DLM-/distributedlog-OrderedScheduler-0-0] INFO - Initialize rackaware ensemble placement policy @ <Bookie:172.22.0.3:0> @ /default-region/default-rack : org.apache.distributedlog.net.DNSResolverForRacks.
14:02:53.754 [DLM-/distributedlog-OrderedScheduler-0-0] INFO - Not weighted
14:02:53.758 [DLM-/distributedlog-OrderedScheduler-0-0] INFO - Weighted ledger placement is not enabled
14:02:53.767 [BookKeeperClientScheduler-OrderedScheduler-0-0] INFO - Adding a new node: /default-region/default-rack/172.22.0.6:3181
14:02:53.769 [BookKeeperClientScheduler-OrderedScheduler-0-0] INFO - Adding a new node: /default-region/default-rack/172.22.0.5:3181
14:02:53.769 [BookKeeperClientScheduler-OrderedScheduler-0-0] INFO - Adding a new node: /default-region/default-rack/172.22.0.7:3181
14:02:53.773 [DLM-/distributedlog-OrderedScheduler-0-0] WARN - Failed to instantiate ledger manager factory org.apache.bookkeeper.meta.HierarchicalLedgerManagerFactory, trying its shaded class dlshade.org.apac
he.bookkeeper.meta.HierarchicalLedgerManagerFactory
14:02:53.782 [DLM-/distributedlog-OrderedScheduler-0-0] INFO - BookKeeper Client created bk:distributedlog://zookeeper:2181/distributedlog:factory_writer_shared with its own ZK Client : ledgersPath = /ledgers,
numRetries = 20, sessionTimeout = 60000, backoff = 100, maxBackoff = 200, dnsResolver =
14:02:53.901 [DLM-/distributedlog-OrderedScheduler-0-0-EventThread] INFO - Ensemble: [172.22.0.7:3181, 172.22.0.5:3181, 172.22.0.6:3181] for ledger: 8
14:02:53.903 [DLM-/distributedlog-OrderedScheduler-0-0-EventThread] INFO - Ensemble: [172.22.0.6:3181, 172.22.0.5:3181, 172.22.0.7:3181] for ledger: 9
14:02:53.905 [main-EventThread] INFO - Ledger allocator for /distributedlog/test-stream-Ir8h5B8-0000/<default>/allocation moved version from 0 to 1.
14:02:53.905 [main-EventThread] INFO - Ledger allocator /distributedlog/test-stream-Ir8h5B8-0000/<default>/allocation moved to phase ALLOCATED : version = 1.
14:02:53.906 [main-EventThread] INFO - Ledger allocator /distributedlog/test-stream-Ir8h5B8-0000/<default>/allocation moved to phase HANDING_OVER : version = 1.
14:02:53.907 [main-EventThread] INFO - No max ledger sequence number found while creating log segment 1 for test-stream-Ir8h5B8-0000:<default>.
14:02:53.913 [main-EventThread] INFO - Ledger allocator /distributedlog/test-stream-Ir8h5B8-0000/<default>/allocation moved to phase ERROR : version = 0.
14:02:53.918 [main-EventThread] ERROR - Fail mark ledger 9 as allocated under /distributedlog/test-stream-Ir8h5B8-0000/<default>/allocation :
dlshade.org.apache.zookeeper.KeeperException$BadVersionException: KeeperErrorCode = BadVersion
dlshade.org.apache.zookeeper.KeeperException$BadVersionException: KeeperErrorCode = BadVersion
at dlshade.org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[distributedlog-core-shaded-4.8.2.jar:4.8.2]
at org.apache.distributedlog.util.Utils$5.processResult(Utils.java:395) [distributedlog-core-shaded-4.8.2.jar:4.8.2]
at dlshade.org.apache.bookkeeper.zookeeper.ZooKeeperClient$22$1.processResult(ZooKeeperClient.java:1091) [distributedlog-core-shaded-4.8.2.jar:4.8.2]
at dlshade.org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:566) [distributedlog-core-shaded-4.8.2.jar:4.8.2]
at dlshade.org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:508) [distributedlog-core-shaded-4.8.2.jar:4.8.2]
14:02:53.929 [main-EventThread] INFO - Ledger allocator /distributedlog/test-stream-Ir8h5B8-0000/<default>/allocation moved to phase HANDED_OVER : version = 1.
14:02:53.929 [main-EventThread] INFO - Ledger allocator for /distributedlog/test-stream-Ir8h5B8-0000/<default>/allocation moved version from 1 to 2.
14:02:53.929 [main-EventThread] INFO - Ledger allocator /distributedlog/test-stream-Ir8h5B8-0000/<default>/allocation moved to phase ALLOCATING : version = 2.
14:02:53.932 [main-EventThread] INFO - test-stream-Ir8h5B8-0000 added log segment (inprogress_000000000000000001 : [LogSegmentId:8, firstTxId:1583503373695, lastTxId:-999, version:VERSION_V5_SEQUENCE_ID, completionTime:0, recordCount:0, regionId:0, status:0, logSegmentSequenceNumber:1, lastEntryId:-1, lastSlotId:-1, inprogress:true, minActiveDLSN:DLSN{logSegmentSequenceNo=1, entryId=0, slotId=0}, startSequenceId:-1]) to cache.
14:02:53.933 [DLM-/distributedlog-OrderedScheduler-1-0] INFO - test-stream-Ir8h5B8-0000 added log segment (inprogress_000000000000000001 : [LogSegmentId:8, firstTxId:1583503373695, lastTxId:-999, version:VERSION_V5_SEQUENCE_ID, completionTime:0, recordCount:0, regionId:0, status:0, logSegmentSequenceNumber:1, lastEntryId:-1, lastSlotId:-1, inprogress:true, minActiveDLSN:DLSN{logSegmentSequenceNo=1, entryId=0, slotId=0}, startSequenceId:-1]) to cache.
14:02:53.934 [DLM-/distributedlog-OrderedScheduler-1-0] INFO - segments is updated with [[LogSegmentId:8, firstTxId:1583503373695, lastTxId:-999, version:VERSION_V5_SEQUENCE_ID, completionTime:0, recordCount:0, regionId:0, status:0, logSegmentSequenceNumber:1, lastEntryId:-1, lastSlotId:-1, inprogress:true, minActiveDLSN:DLSN{logSegmentSequenceNo=1, entryId=0, slotId=0}, startSequenceId:0]]
14:02:53.942 [DLM-/distributedlog-OrderedScheduler-0-0-EventThread] INFO - Ensemble: [172.22.0.7:3181, 172.22.0.6:3181, 172.22.0.5:3181] for ledger: 10
14:02:53.944 [main-EventThread] INFO - Ledger allocator for /distributedlog/test-stream-Ir8h5B8-0000/<default>/allocation moved version from 2 to 3.
14:02:53.944 [main-EventThread] INFO - Ledger allocator /distributedlog/test-stream-Ir8h5B8-0000/<default>/allocation moved to phase ALLOCATED : version = 3.
14:02:53.963 [main-EventThread] INFO - Deleting log segments older than 1583330573960 for test-stream-Ir8h5B8-0000:<default> : []
14:02:53.993 [BookKeeperClientWorker-OrderedExecutor-8-0] INFO - Closing the per channel bookie client for 172.22.0.7:3181
14:02:54.073 [DL-io-2] INFO - Successfully connected to bookie: [id: 0xde3e7c61, L:/172.22.0.3:53182 - R:builds_bookie1_1.builds_default/172.22.0.6:3181]
14:02:54.073 [DL-io-1] INFO - Successfully connected to bookie: [id: 0x5815edbc, L:/172.22.0.3:49790 - R:172.22.0.7/172.22.0.7:3181]
14:02:54.073 [DL-io-0] INFO - Successfully connected to bookie: [id: 0x58af16d5, L:/172.22.0.3:43310 - R:builds_bookie2_1.builds_default/172.22.0.5:3181]
14:02:54.082 [DL-io-2] INFO - Successfully connected to bookie: 172.22.0.6:3181
14:02:54.082 [DL-io-1] INFO - Successfully connected to bookie: 172.22.0.7:3181
14:02:54.082 [DL-io-0] INFO - Successfully connected to bookie: 172.22.0.5:3181
14:02:54.101 [DL-io-0] INFO - connection [id: 0x58af16d5, L:/172.22.0.3:43310 - R:builds_bookie2_1.builds_default/172.22.0.5:3181] authenticated as BookKeeperPrincipal{ANONYMOUS}
14:02:54.101 [DL-io-2] INFO - connection [id: 0xde3e7c61, L:/172.22.0.3:53182 - R:builds_bookie1_1.builds_default/172.22.0.6:3181] authenticated as BookKeeperPrincipal{ANONYMOUS}
14:02:54.102 [DL-io-1] INFO - connection [id: 0x5815edbc, L:/172.22.0.3:49790 - R:172.22.0.7/172.22.0.7:3181] authenticated as BookKeeperPrincipal{ANONYMOUS}
14:02:54.168 [BookKeeperClientWorker-OrderedExecutor-8-0] INFO - ReadAhead for test-stream-Ir8h5B8-0000:<default> is caught up
14:02:54.196 [main] INFO - All consumers are ready
14:02:54.198 [main] INFO - ----- Starting warm-up traffic ------
14:02:54.222 [local-worker-1-1] WARN - Write error on message
java.util.concurrent.CompletionException: org.apache.distributedlog.exceptions.WriteException: Write rejected because stream test-stream-Ir8h5B8-0000 has encountered an error : writer has been closed due to er
ror.
at java.util.concurrent.CompletableFuture.encodeThrowable(CompletableFuture.java:331) ~[?:?]
at java.util.concurrent.CompletableFuture.completeThrowable(CompletableFuture.java:346) ~[?:?]
at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:870) [?:?]
at java.util.concurrent.CompletableFuture.uniWhenCompleteStage(CompletableFuture.java:883) [?:?]
at java.util.concurrent.CompletableFuture.whenComplete(CompletableFuture.java:2251) [?:?]
at dlshade.org.apache.bookkeeper.common.concurrent.FutureUtils.stats(FutureUtils.java:365) ~[distributedlog-core-shaded-4.8.2.jar:4.8.2]
at org.apache.distributedlog.BKAsyncLogWriter.write(BKAsyncLogWriter.java:415) ~[distributedlog-core-shaded-4.8.2.jar:4.8.2]
at io.openmessaging.benchmark.driver.bookkeeper.DlogBenchmarkProducer.sendAsync(DlogBenchmarkProducer.java:50) ~[driver-bookkeeper-0.0.1-SNAPSHOT.jar:?]
at io.openmessaging.benchmark.worker.LocalWorker.lambda$submitProducersToExecutor$10(LocalWorker.java:219) [classes/:?]
at java.util.ArrayList.forEach(ArrayList.java:1540) [?:?]
at io.openmessaging.benchmark.worker.LocalWorker.lambda$submitProducersToExecutor$11(LocalWorker.java:216) [classes/:?]
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) [?:?]
at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?]
at io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:138) [netty-all-4.1.12.Final.jar:4.1.12.Final]
at java.lang.Thread.run(Thread.java:834) [?:?]
Caused by: org.apache.distributedlog.exceptions.WriteException: Write rejected because stream test-stream-Ir8h5B8-0000 has encountered an error : writer has been closed due to error.
at org.apache.distributedlog.BKAsyncLogWriter.getCachedLogSegmentWriter(BKAsyncLogWriter.java:189) ~[distributedlog-core-shaded-4.8.2.jar:4.8.2]
at org.apache.distributedlog.BKAsyncLogWriter.asyncWrite(BKAsyncLogWriter.java:280) ~[distributedlog-core-shaded-4.8.2.jar:4.8.2]
at org.apache.distributedlog.BKAsyncLogWriter.write(BKAsyncLogWriter.java:416) ~[distributedlog-core-shaded-4.8.2.jar:4.8.2]
... 10 more
For example:
This is more of a wishlist item, but it would make the results easier to consume and publish in a standard way. Perhaps you can have an entire section where people can contribute notebooks back to the project and share results, etc
Rather than have the users choose from a list of pre-defined configurations, it might be useful to have a simple command line tools that will take in all of these values as parameters, and generate the YAML configuration file automatically.
e.g. ./bin/generate-workload --num-topics=3 --msg-size=100kb --num-producers=5 ....etc.
I tried pulsar 2.1.1 and 2.2.1, both with problem.
what i have done:
use alicloud instead of aws (no extra disk)
when running Simple Workload (with duration 1 min), ERROR log:
09:51:09.563 [main] INFO - --------------- WORKLOAD : Simple Workload 1 producer on 1 topic --- DRIVER : Pulsar---------------
09:51:11.089 [main] INFO - Created 1 topics in 1036.930529 ms
09:51:11.256 [main] INFO - Created 1 consumers in 165.190064 ms
09:51:11.284 [main] INFO - Created 1 producers in 27.391998 ms
09:51:11.284 [main] INFO - Waiting for consumers to be ready
09:51:12.350 [main] INFO - All consumers are ready
09:51:12.351 [main] INFO - ----- Starting warm-up traffic ------
09:51:22.734 [main] INFO - Pub rate 71.2 msg/s / 0.1 Mb/s | Cons rate 34.0 msg/s / 0.0 Mb/s | Backlog: 0.4 K | Pub Latency (ms) avg: 4726.2 - 50%: 5703.2 - 99%: 9423.6 - 99.9%: 9423.6 - Max: 9423.6
09:51:32.965 [main] INFO - Pub rate 112.0 msg/s / 0.1 Mb/s | Cons rate 124.4 msg/s / 0.1 Mb/s | Backlog: 0.3 K | Pub Latency (ms) avg: 15270.5 - 50%: 14684.5 - 99%: 19451.4 - 99.9%: 19451.4 - Max: 19451.4
09:51:43.156 [main] INFO - Pub rate 121.9 msg/s / 0.1 Mb/s | Cons rate 79.6 msg/s / 0.1 Mb/s | Backlog: 0.7 K | Pub Latency (ms) avg: 26262.7 - 50%: 28144.1 - 99%: 30383.2 - 99.9%: 30383.2 - Max: 30383.2
09:51:53.355 [main] INFO - Pub rate 25.1 msg/s / 0.0 Mb/s | Cons rate 97.1 msg/s / 0.1 Mb/s | Backlog: -0.0 K | Pub Latency (ms) avg: 32207.5 - 50%: 32135.9 - 99%: 32279.4 - 99.9%: 32279.4 - Max: 32279.4
09:52:03.701 [main] INFO - Pub rate 0.0 msg/s / 0.0 Mb/s | Cons rate 149.8 msg/s / 0.1 Mb/s | Backlog: -1.6 K | Pub Latency (ms) avg: 0.0 - 50%: 0.0 - 99%: 0.0 - 99.9%: 0.0 - Max: 0.0
09:52:13.869 [main] INFO - Pub rate 0.0 msg/s / 0.0 Mb/s | Cons rate 100.5 msg/s / 0.1 Mb/s | Backlog: -2.6 K | Pub Latency (ms) avg: 0.0 - 50%: 0.0 - 99%: 0.0 - 99.9%: 0.0 - Max: 0.0
09:52:15.016 [main] INFO - ----- Aggregated Pub Latency (ms) avg: 18300.7 - 50%: 18709.5 - 95%: 32135.8 - 99%: 32279.3 - 99.9%: 32279.4 - 99.99%: 32279.4 - Max: 32279.4
09:52:15.171 [main] INFO - ----- Starting benchmark traffic ------
09:52:27.821 [main] INFO - Pub rate 0.0 msg/s / 0.0 Mb/s | Cons rate 90.1 msg/s / 0.1 Mb/s | Backlog: -3.8 K | Pub Latency (ms) avg: 0.0 - 50%: 0.0 - 99%: 0.0 - 99.9%: 0.0 - Max: 0.0
09:52:37.986 [main] INFO - Pub rate 0.0 msg/s / 0.0 Mb/s | Cons rate 126.2 msg/s / 0.1 Mb/s | Backlog: -5.0 K | Pub Latency (ms) avg: 0.0 - 50%: 0.0 - 99%: 0.0 - 99.9%: 0.0 - Max: 0.0
09:52:48.165 [main] INFO - Pub rate 0.0 msg/s / 0.0 Mb/s | Cons rate 107.1 msg/s / 0.1 Mb/s | Backlog: -6.1 K | Pub Latency (ms) avg: 0.0 - 50%: 0.0 - 99%: 0.0 - 99.9%: 0.0 - Max: 0.0
09:52:59.490 [main] INFO - Pub rate 0.0 msg/s / 0.0 Mb/s | Cons rate 120.5 msg/s / 0.1 Mb/s | Backlog: -7.5 K | Pub Latency (ms) avg: 0.0 - 50%: 0.0 - 99%: 0.0 - 99.9%: 0.0 - Max: 0.0
09:53:09.626 [main] INFO - Pub rate 0.0 msg/s / 0.0 Mb/s | Cons rate 135.4 msg/s / 0.1 Mb/s | Backlog: -8.9 K | Pub Latency (ms) avg: 0.0 - 50%: 0.0 - 99%: 0.0 - 99.9%: 0.0 - Max: 0.0
09:53:19.750 [main] INFO - Pub rate 0.0 msg/s / 0.0 Mb/s | Cons rate 76.2 msg/s / 0.1 Mb/s | Backlog: -9.6 K | Pub Latency (ms) avg: 0.0 - 50%: 0.0 - 99%: 0.0 - 99.9%: 0.0 - Max: 0.0
09:53:19.979 [main] INFO - ----- Aggregated Pub Latency (ms) avg: 0.0 - 50%: 0.0 - 95%: 0.0 - 99%: 0.0 - 99.9%: 0.0 - 99.99%: 0.0 - Max: 0.0
09:54:20.098 [main] ERROR - Failed to run the workload 'Simple Workload 1 producer on 1 topic' for driver 'driver-pulsar/pulsar.yaml'
java.util.concurrent.CompletionException: java.util.concurrent.TimeoutException: Request timeout to 10.79.113.174/10.79.113.174:8080 after 60000 ms
at java.util.concurrent.CompletableFuture.encodeThrowable(CompletableFuture.java:292) ~[?:1.8.0_191]
at java.util.concurrent.CompletableFuture.completeThrowable(CompletableFuture.java:308) ~[?:1.8.0_191]
at java.util.concurrent.CompletableFuture.uniApply(CompletableFuture.java:593) ~[?:1.8.0_191]
at java.util.concurrent.CompletableFuture$UniApply.tryFire(CompletableFuture.java:577) ~[?:1.8.0_191]
at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:474) ~[?:1.8.0_191]
at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:1977) ~[?:1.8.0_191]
at org.asynchttpclient.netty.NettyResponseFuture.abort(NettyResponseFuture.java:277) ~[org.asynchttpclient-async-http-client-2.1.0-alpha26.jar:?]
at org.asynchttpclient.netty.request.NettyRequestSender.abort(NettyRequestSender.java:462) ~[org.asynchttpclient-async-http-client-2.1.0-alpha26.jar:?]
at org.asynchttpclient.netty.timeout.TimeoutTimerTask.expire(TimeoutTimerTask.java:44) ~[org.asynchttpclient-async-http-client-2.1.0-alpha26.jar:?]
at org.asynchttpclient.netty.timeout.RequestTimeoutTimerTask.run(RequestTimeoutTimerTask.java:51) ~[org.asynchttpclient-async-http-client-2.1.0-alpha26.jar:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:663) ~[io.netty-netty-all-4.1.12.Final.jar:4.1.12.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:738) ~[io.netty-netty-all-4.1.12.Final.jar:4.1.12.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:466) ~[io.netty-netty-all-4.1.12.Final.jar:4.1.12.Final]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_191]
Caused by: java.util.concurrent.TimeoutException: Request timeout to 10.79.113.174/10.79.113.174:8080 after 60000 ms
... 6 more
benchmark-worker log on 10.79.113.174
Jan 17 09:52:45 benchmark-worker[23932]: 09:52:43.952 [pulsar-client-io-11-1] WARN - Write error on message
Jan 17 09:52:45 benchmark-worker[23932]: java.util.concurrent.CompletionException: org.apache.pulsar.client.api.PulsarClientException$TimeoutException: Could not send message to broker wit
Jan 17 09:52:45 benchmark-worker[23932]: at java.util.concurrent.CompletableFuture.encodeThrowable(CompletableFuture.java:292) ~[?:1.8.0_191]
Jan 17 09:52:45 benchmark-worker[23932]: at java.util.concurrent.CompletableFuture.completeThrowable(CompletableFuture.java:308) ~[?:1.8.0_191]
Jan 17 09:52:45 benchmark-worker[23932]: at java.util.concurrent.CompletableFuture.uniApply(CompletableFuture.java:593) ~[?:1.8.0_191]
Jan 17 09:52:45 benchmark-worker[23932]: at java.util.concurrent.CompletableFuture$UniApply.tryFire(CompletableFuture.java:577) ~[?:1.8.0_191]
Jan 17 09:52:45 benchmark-worker[23932]: at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:474) ~[?:1.8.0_191]
Jan 17 09:52:45 benchmark-worker[23932]: at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:1977) ~[?:1.8.0_191]
Jan 17 09:52:45 benchmark-worker[23932]: at org.apache.pulsar.client.impl.ProducerImpl$1.sendComplete(ProducerImpl.java:263) ~[org.apache.pulsar-pulsar-client-original-2.2.1.jar:2.2.1]
Jan 17 09:52:45 benchmark-worker[23932]: at org.apache.pulsar.client.impl.ProducerImpl.lambda$failPendingMessages$9(ProducerImpl.java:1184) ~[org.apache.pulsar-pulsar-client-original-2.2.1
Jan 17 09:52:45 benchmark-worker[23932]: at java.lang.Iterable.forEach(Iterable.java:75) ~[?:1.8.0_191]
Jan 17 09:52:45 benchmark-worker[23932]: at org.apache.pulsar.client.impl.ProducerImpl.failPendingMessages(ProducerImpl.java:1179) ~[org.apache.pulsar-pulsar-client-original-2.2.1.jar:2.2.
Jan 17 09:52:45 benchmark-worker[23932]: at org.apache.pulsar.client.impl.ProducerImpl.lambda$failPendingMessages$10(ProducerImpl.java:1203) ~[org.apache.pulsar-pulsar-client-original-2.2.
Jan 17 09:52:45 benchmark-worker[23932]: at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:163) [io.netty-netty-all-4.1.12.Final.jar:4.1.12.Final]
Jan 17 09:52:45 benchmark-worker[23932]: at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:403) [io.netty-netty-all-4.1.12.Final.jar:4.1.12.F
Jan 17 09:52:45 benchmark-worker[23932]: at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:311) [io.netty-netty-all-4.1.12.Final.jar:4.1.12.Final]
Jan 17 09:52:45 benchmark-worker[23932]: at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:858) [io.netty-netty-all-4.1.12.Final.jar:4.1.12.Final]
Jan 17 09:52:45 benchmark-worker[23932]: at io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:138) [io.netty-netty-all-4.1.12.Final.jar:4
Jan 17 09:52:45 benchmark-worker[23932]: at java.lang.Thread.run(Thread.java:748) [?:1.8.0_191]
Jan 17 09:52:45 benchmark-worker[23932]: Caused by: org.apache.pulsar.client.api.PulsarClientException$TimeoutException: Could not send message to broker within given timeout
Jan 17 09:52:45 benchmark-worker[23932]: at org.apache.pulsar.client.impl.ProducerImpl.run(ProducerImpl.java:1156) ~[org.apache.pulsar-pulsar-client-original-2.2.1.jar:2.2.1]
Jan 17 09:52:45 benchmark-worker[23932]: at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:663) ~[io.netty-netty-all-4.1.12.Final.jar:4.1.12.Final]
Jan 17 09:52:45 benchmark-worker[23932]: at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:738) ~[io.netty-netty-all-4.1.12.Final.jar:4.1.12.Final]
Jan 17 09:52:45 benchmark-worker[23932]: at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:466) ~[io.netty-netty-all-4.1.12.Final.jar:4.1.12.Final]
Jan 17 09:52:45 benchmark-worker[23932]: ... 2 more
Jan 17 09:54:20 benchmark-worker[23932]: 09:54:20.282 [qtp1569754439-27] INFO - Pulsar benchmark driver successfully shut down
Jan 17 09:54:20 benchmark-worker[23932]: 09:54:20.284 [qtp1569754439-29] WARN - Uncaught exception
Jan 17 09:54:20 benchmark-worker[23932]: java.lang.RuntimeException: java.util.ConcurrentModificationException
Jan 17 09:54:20 benchmark-worker[23932]: at io.openmessaging.benchmark.worker.LocalWorker.stopAll(LocalWorker.java:365) ~[io.openmessaging.benchmark-benchmark-framework-0.0.1-SNAPSHOT.jar:
Jan 17 09:54:20 benchmark-worker[23932]: at io.openmessaging.benchmark.worker.WorkerHandler.handleStopAll(WorkerHandler.java:124) ~[io.openmessaging.benchmark-benchmark-framework-0.0.1-SNA
Jan 17 09:54:20 benchmark-worker[23932]: at io.javalin.core.JavalinServlet.service(JavalinServlet.kt:61) [io.javalin-javalin-1.3.0.jar:?]
Jan 17 09:54:20 benchmark-worker[23932]: at io.javalin.embeddedserver.jetty.EmbeddedJettyServer$start$httpHandler$1.doHandle(EmbeddedJettyServer.kt:42) [io.javalin-javalin-1.3.0.jar:?]
Jan 17 09:54:20 benchmark-worker[23932]: at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:168) [org.eclipse.jetty-jetty-server-9.4.8.v20171121.jar:9.4.8.v2017
Jan 17 09:54:20 benchmark-worker[23932]: at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) [org.eclipse.jetty-jetty-servlet-9.4.5.v20170502.jar:9.4.5.v20170502]
Jan 17 09:54:20 benchmark-worker[23932]: at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) [org.eclipse.jetty-jetty-server-9.4.8.v20171121.jar:9.4.8.v201
Jan 17 09:54:20 benchmark-worker[23932]: at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:166) [org.eclipse.jetty-jetty-server-9.4.8.v20171121.jar:9.4.8.v2017
Jan 17 09:54:20 benchmark-worker[23932]: at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1155) [org.eclipse.jetty-jetty-server-9.4.8.v20171121.jar:9.4.8.v201
Jan 17 09:54:20 benchmark-worker[23932]: at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) [org.eclipse.jetty-jetty-server-9.4.8.v20171121.jar:9.4.8.v2017112
Jan 17 09:54:20 benchmark-worker[23932]: at org.eclipse.jetty.server.handler.HandlerList.handle(HandlerList.java:61) [org.eclipse.jetty-jetty-server-9.4.8.v20171121.jar:9.4.8.v20171121]
Jan 17 09:54:20 benchmark-worker[23932]: at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) [org.eclipse.jetty-jetty-server-9.4.8.v20171121.jar:9.4.8.v20171
Jan 17 09:54:20 benchmark-worker[23932]: at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) [org.eclipse.jetty-jetty-server-9.4.8.v20171121.jar:9.4.8.v20171
Jan 17 09:54:20 benchmark-worker[23932]: at org.eclipse.jetty.server.Server.handle(Server.java:530) [org.eclipse.jetty-jetty-server-9.4.8.v20171121.jar:9.4.8.v20171121]
Jan 17 09:54:20 benchmark-worker[23932]: at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:347) [org.eclipse.jetty-jetty-server-9.4.8.v20171121.jar:9.4.8.v20171121]
Jan 17 09:54:20 benchmark-worker[23932]: at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:256) [org.eclipse.jetty-jetty-server-9.4.8.v20171121.jar:9.4.8.v20171121]
Jan 17 09:54:20 benchmark-worker[23932]: at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:279) [org.eclipse.jetty-jetty-io-9.4.8.v20171121.jar:9.4.
Jan 17 09:54:20 benchmark-worker[23932]: at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) [org.eclipse.jetty-jetty-io-9.4.8.v20171121.jar:9.4.8.v20171121]
Jan 17 09:54:20 benchmark-worker[23932]: at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:124) [org.eclipse.jetty-jetty-io-9.4.8.v20171121.jar:9.4.8.v20171121]
Jan 17 09:54:20 benchmark-worker[23932]: at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:247) [org.eclipse.jetty-jetty-util-9.4.8.v20171121.jar:9.4.8
Jan 17 09:54:20 benchmark-worker[23932]: at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140) [org.eclipse.jetty-jetty-util-9.4.8.v20171121.jar:9.4.8.v
Jan 17 09:54:20 benchmark-worker[23932]: at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131) [org.eclipse.jetty-jetty-util-9.4.8.v20171121.jar:9.4.8.v2017
Jan 17 09:54:20 benchmark-worker[23932]: at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:382) [org.eclipse.jetty-jetty-util-9.4.8.v20
Jan 17 09:54:20 benchmark-worker[23932]: at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708) [org.eclipse.jetty-jetty-util-9.4.8.v20171121.jar:9.4.8.v201711
Jan 17 09:54:20 benchmark-worker[23932]: at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) [org.eclipse.jetty-jetty-util-9.4.8.v20171121.jar:9.4.8.v2017112
Jan 17 09:54:20 benchmark-worker[23932]: at java.lang.Thread.run(Thread.java:748) [?:1.8.0_191]
Jan 17 09:54:20 benchmark-worker[23932]: Caused by: java.util.ConcurrentModificationException
Jan 17 09:54:20 benchmark-worker[23932]: at java.util.ArrayList$Itr.checkForComodification(ArrayList.java:909) ~[?:1.8.0_191]
Jan 17 09:54:20 benchmark-worker[23932]: at java.util.ArrayList$Itr.next(ArrayList.java:859) ~[?:1.8.0_191]
Jan 17 09:54:20 benchmark-worker[23932]: at io.openmessaging.benchmark.worker.LocalWorker.stopAll(LocalWorker.java:350) ~[io.openmessaging.benchmark-benchmark-framework-0.0.1-SNAPSHOT.jar:
Jan 17 09:54:20 benchmark-worker[23932]: ... 25 more
Consumers are in different consumer groups
Driver-NSQ Benchmark Driver.java has extra bracket at the bottom which prevents a complete build with master.
my system is Ubuntu running as docker instance ..in my windows setup ..
Compile worked fine.
Terrform deploy worked fine ..
./terraform-inventory --inventory lists all the servers it sees in the aws
root@c0636b234b79:/openmessaging-benchmark/driver-kafka/deploy# ansible-playbook --user ec2-user --inventory ./terraform-inventory deploy.yaml
ERROR! no action detected in task
The error appears to have been in '/openmessaging-benchmark/driver-kafka/deploy/deploy.yaml': line 89, column 7, but may
be elsewhere in the file depending on the exact syntax problem.
The offending line appears to be:
dest: "/etc/systemd/system/zookeeper.service"
- systemd:
^ here
root@c0636b234b79:/openmessaging-benchmark/driver-kafka/deploy# ansible-playbook --user ec2-user --inventory which ./terraform-inventory deploy.yaml
[WARNING]: provided hosts list is empty, only localhost is available
Unexpected Exception: 'utf8' codec can't decode byte 0x80 in position 24: invalid start byte
to see the full traceback, use -vvv
root@c0636b234b79:/openmessaging-benchmark/driver-kafka/deploy# ansible-playbook --user ec2-user --inventory terraform-inventory deploy.yaml
ERROR! no action detected in task
The error appears to have been in '/openmessaging-benchmark/driver-kafka/deploy/deploy.yaml': line 89, column 7, but may
be elsewhere in the file depending on the exact syntax problem.
The offending line appears to be:
dest: "/etc/systemd/system/zookeeper.service"
- systemd:
^ here
[ERROR] Failed to execute goal com.mycila:license-maven-plugin:3.0:check (default) on project driver-pulsar: Some files do not have the expected license header -> [Help 1]
bookkeeper.version: 4.7.0-SNAPSHOT
distributedlog-core-shaded: 4.7.0-SNAPSHOT
Couldn't find the artifacts of these two versions in central maven.
RabbitMqBenchmarkDriver initialize get empty config structure.
I could not tell how the client side was immune to coordinated omission. I wonder if this is an issue or if not how do you guaranty or work around the problem ?
from here
Load testing. Load generators generate requests at a specified rate. Response times are logged for each request, put in a bucket, and percentiles are generated.
This approach works only if all responses fit within the collection bucket interval. Let’s say requests are supposed to be sent every second, but a response takes a second and half. The generator will wait to send the next request.
So something problematic is missed and is not being measured. There's a kind of hidden coordination to avoid looking at a problem.
The guest account is used by default but the guest account does not have remote access rights.
how to use this framework?
It would be useful to show the structure of the JSON output from these tests, so people would know how to interpret them.
KafkaBenchmarkConsumer#KafkaBenchmarkConsumer use a new thread to run consumer logic, the consumer will stop if exception happens.
Upgrade to version 4.3.0 can fix this
This line seems not working anymore https://github.com/openmessaging/openmessaging-benchmark/blob/master/driver-pulsar/deploy/deploy.yaml#L68.
I navigate to http://apache.osuosl.org/pulsar/ and only found the latest version 2.5.0
whereas Bookkeeper and Zookeeper keep their previous released versions. Is it on purpose to delete the old Pulsar versions?
Thanks
Hello,
Can you tell me how we can configure your tools to connect in SSL to KAFKA brokers?
Thank you
I'm trying to get a RabbitMQ cluster and encountering an error along these lines on each RabbitMQ host, during the rabbitmq status
task:
fatal: [52.11.238.169]: FAILED! => {"changed": true, "cmd": "rabbitmqctl cluster_status", "delta": "0:00:00.651314", "end": "2018-01-08 17:20:26.023578", "msg": "non-zero return code", "rc": 69, "start": "2018-01-08 17:20:25.372264", "stderr": "Error: unable to perform an operation on node 'rabbit@ip-10-0-0-223'. Please see diagnostics information and suggestions below.\n\nMost common reasons for this are:\n\n * Target node is unreachable (e.g. due to hostname resolution, TCP connection or firewall issues)\n * CLI tool fails to authenticate with the server (e.g. due to CLI tool's Erlang cookie not matching that of the server)\n * Target node is not running\n\nIn addition to the diagnostics info below:\n\n * See the CLI, clustering and networking guides on http://rabbitmq.com/documentation.html to learn more\n * Consult server logs on node rabbit@ip-10-0-0-223\n\nDIAGNOSTICS\n===========\n\nattempted to contact: ['rabbit@ip-10-0-0-223']\n\nrabbit@ip-10-0-0-223:\n * connected to epmd (port 4369) on ip-10-0-0-223\n * epmd reports: node 'rabbit' not running at all\n no other nodes on ip-10-0-0-223\n * suggestion: start the node\n\nCurrent node details:\n * node name: 'rabbitmqcli37@ip-10-0-0-223'\n * effective user's home directory: /var/lib/rabbitmq\n * Erlang cookie hash: 7L2Y4WVlOaJal7q3JAgYfg==", "stderr_lines": ["Error: unable to perform an operation on node 'rabbit@ip-10-0-0-223'. Please see diagnostics information and suggestions below.", "", "Most common reasons for this are:", "", " * Target node is unreachable (e.g. due to hostname resolution, TCP connection or firewall issues)", " * CLI tool fails to authenticate with the server (e.g. due to CLI tool's Erlang cookie not matching that of the server)", " * Target node is not running", "", "In addition to the diagnostics info below:", "", " * See the CLI, clustering and networking guides on http://rabbitmq.com/documentation.html to learn more", " * Consult server logs on node rabbit@ip-10-0-0-223", "", "DIAGNOSTICS", "===========", "", "attempted to contact: ['rabbit@ip-10-0-0-223']", "", "rabbit@ip-10-0-0-223:", " * connected to epmd (port 4369) on ip-10-0-0-223", " * epmd reports: node 'rabbit' not running at all", " no other nodes on ip-10-0-0-223", " * suggestion: start the node", "", "Current node details:", " * node name: 'rabbitmqcli37@ip-10-0-0-223'", " * effective user's home directory: /var/lib/rabbitmq", " * Erlang cookie hash: 7L2Y4WVlOaJal7q3JAgYfg=="], "stdout": "", "stdout_lines": []}
After following the instruction on http://openmessaging.cloud/docs/benchmarks/pulsar/, I got an error when running a benchmark (any benchmark will do for ex. workloads/simple_workload.yaml).
I checked all the instances are up and communicating well.
Has anyone experienced the same?
Any hints are greatly appreciated.
Thanks,
Indra
20:49:56.371 [main] INFO - Workers list - producers: [http://10.0.0.244:8080, http://10.0.0.175:8080]
20:49:56.371 [main] INFO - Workers list - consumers: [http://10.0.0.225:8080, http://10.0.0.11:8080]
20:49:56.375 [main] INFO - --------------- WORKLOAD : 1 producer / 3 consumers on 100 topics --- DRIVER : Pulsar---------------
20:49:57.999 [main] INFO - Created 100 topics in 31.2175 ms
20:49:58.526 [AsyncHttpClient-2-4] ERROR - Failed to do HTTP post request to http://10.0.0.11:8080/create-consumers -- code: 500
20:49:58.592 [AsyncHttpClient-2-3] ERROR - Failed to do HTTP post request to http://10.0.0.225:8080/create-consumers -- code: 500
20:49:58.593 [main] ERROR - Failed to run the workload '1 producer / 3 consumers on 100 topics' for driver 'driver-pulsar/pulsar.yaml'
java.util.concurrent.CompletionException: java.lang.IllegalArgumentException
at java.util.concurrent.CompletableFuture.encodeThrowable(CompletableFuture.java:273) ~[?:1.8.0_181]
at java.util.concurrent.CompletableFuture.completeThrowable(CompletableFuture.java:280) ~[?:1.8.0_181]
...... DELETED
WorkloadGenerator#createConsumers use same subscription name to subscribe different topic. Some MQ not allow to do this.
Class KafkaBenchmarkDriver did not close adminClient when stop benchmark test. This will cause a lot of tcp links not released.
I've notice that KafkaBenchmarkConsumer ignores enable.auto.commit=true, may lead commit the same offset twice.
And another possible problem is the OffsetAndMetadata we committed don't have the same semantic with Kafka automatically commit offset, which may lead consume the last committed message again when rebalance
I tried but get some problem. (pity no detail info 😂)
I ran the max-rate-1-topic-1-partition-100b workload with the Kafka driver and get 1 msg/s throughput.
sudo bin/benchmark --drivers driver-kafka/kafka.yaml workloads/max-rate-1-topic-1-partition-100b.yaml
The results looked like:
{
"workload" : "Max rate 1 producer on 1 topic / 1 partition",
"driver" : "Kafka",
"publishRate" : [ 0.9940110866817811, 0.9938639587565928, 0.9934661518438835, 0.9900274464041732, 0.9935629196582744, 0.9907865442494922, 0.9933999872757184, 0.9894087670954211, 0.9933890456698582, 0.9890120683057304, 1.0928478989996409, 0.9896480145627102, 0.9934658212073704, 0.9898219180752204, 0.9934650420904388, 0.9936716152101688, 0.9909047311183474, 0.9921966493817206, 0.9897291937660128, 0.9930860717078047, 0.9899467545219122, 0.9944877125340724, 1.0934709288228361, 0.9939319407413475, 0.9939900236080046, 0.9937668056078406, 0.9935306294789762, 0.988986869099298, 0.9933570714031665, 0.9902347583407627 ],
"consumeRate" : [ 0.9940110866817811, 0.9938639587565928, 0.9934661518438835, 0.9900274464041732, 0.9935629196582744, 0.9907865442494922, 0.9933999872757184, 0.9894087670954211, 0.9933890456698582, 0.9890120683057304, 1.0928478989996409, 0.9896480145627102, 0.9934658212073704, 0.9898219180752204, 0.9934650420904388, 0.9936716152101688, 0.9909047311183474, 0.9921966493817206, 0.9897291937660128, 0.9930860717078047, 0.9899467545219122, 0.9944877125340724, 1.0934709288228361, 0.9939319407413475, 0.9939900236080046, 0.9937668056078406, 0.9935306294789762, 0.988986869099298, 0.9933570714031665, 0.9902347583407627 ], ...
I changed it to producerRate: 10000000 and got the high throughput I expected.
So somehow I don't think the producerRate 0 is being passed on correctly.
There is a broken link on main README page...the RocketMQ link; https://github.com/openmessaging/openmessaging-benchmark/blob/master/driver-rocketmq/README.md returns a 404....
Environment:
docker-compose:1.23.2
zookeeper:3.5.5
bookkeeper:4.8.2
distributedlog-core-shaded:4.8.2
When i use 1 master and 4 worker nodes with producerRate=1000, i see only 500 , i.e. only 1st producer and 1st consumer is worked.
Start openmessaging:
docker exec -i builds_bk-omb_1 /opt/openmessaging-benchmark/bin/benchmark -d driver-bookkeeper/bookkeeper.yaml -w 'http://builds_bk-omb_2:8383,http://builds_bk-omb_3:8383,http://builds_bk-omb_4:8383,http://builds_bk-omb_5:8383' workloads/1-topic-1-partition-100b.yaml
Test output:
SLF4J: Found binding in [jar:file:/root/.m2/repository/org/apache/logging/log4j/log4j-slf4j-impl/2.9.1/log4j-slf4j-impl-2.9.1.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: Found binding in [jar:file:/root/.m2/repository/ch/qos/logback/logback-classic/1.0.13/logback-classic-1.0.13.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an explanation.
SLF4J: Actual binding is of type [org.apache.logging.slf4j.Log4jLoggerFactory]
14:53:14.018 [main] INFO - Starting benchmark with config: {
"drivers" : [ "driver-bookkeeper/bookkeeper.yaml" ],
"workers" : [ "http://builds_bk-omb_2:8383", "http://builds_bk-omb_3:8383", "http://builds_bk-omb_4:8383", "http://builds_bk-omb_5:8383" ],
"workersFile" : null,
"workloads" : [ "workloads/1-topic-1-partition-100b.yaml" ]
}
14:53:14.064 [main] INFO - Workloads: {
"1-topic-1-partition-100b" : {
"name" : "1 topic / 1 partition / 100b",
"topics" : 1,
"partitionsPerTopic" : 1,
"keyDistributor" : "NO_KEY",
"messageSize" : 100,
"payloadFile" : "payload/payload-100b.data",
"subscriptionsPerTopic" : 1,
"producersPerTopic" : 1,
"consumerPerSubscription" : 1,
"producerRate" : 1000,
"consumerBacklogSizeGB" : 0,
"testDurationMinutes" : 1
}
}
WARNING: An illegal reflective access operation has occurred
WARNING: Illegal reflective access by io.netty.util.internal.ReflectionUtil (file:/root/.m2/repository/io/netty/netty-all/4.1.12.Final/netty-all-4.1.12.Final.jar) to constructor java.nio.DirectByteBuffer(long,
int)
WARNING: Please consider reporting this to the maintainers of io.netty.util.internal.ReflectionUtil
WARNING: Use --illegal-access=warn to enable warnings of further illegal reflective access operations
WARNING: All illegal access operations will be denied in a future release
14:53:14.399 [main] INFO - Workers list - producers: [http://builds_bk-omb_2:8383, http://builds_bk-omb_3:8383]
14:53:14.399 [main] INFO - Workers list - consumers: [http://builds_bk-omb_4:8383, http://builds_bk-omb_5:8383]
14:53:14.405 [main] INFO - --------------- WORKLOAD : 1 topic / 1 partition / 100b --- DRIVER : BookKeeper---------------
14:53:14.965 [main] INFO - Created 1 topics in 78.950927 ms
14:53:15.022 [main] INFO - Created 1 consumers in 51.625864 ms
4:53:15.129 [main] INFO - Created 1 producers in 106.395072 ms
14:53:15.129 [main] INFO - Waiting for consumers to be ready
14:53:15.931 [main] INFO - All consumers are ready
14:53:15.932 [main] INFO - ----- Starting warm-up traffic ------
14:53:26.208 [main] INFO - Pub rate 538.7 msg/s / 0.1 Mb/s | Cons rate 500.1 msg/s / 0.0 Mb/s | Backlog: 0.4 K | Pub Latency (ms) avg: 6.3 - 50%: 3.3 - 99%: 48.7 - 99.9%: 50.1 - Max: 50.3
14:53:36.387 [main] INFO - Pub rate 502.2 msg/s / 0.0 Mb/s | Cons rate 541.7 msg/s / 0.1 Mb/s | Backlog: -0.0 K | Pub Latency (ms) avg: 3.1 - 50%: 3.0 - 99%: 8.6 - 99.9%: 15.1 - Max: 22.1
14:53:46.589 [main] INFO - Pub rate 498.6 msg/s / 0.0 Mb/s | Cons rate 461.3 msg/s / 0.0 Mb/s | Backlog: 0.4 K | Pub Latency (ms) avg: 2.9 - 50%: 3.0 - 99%: 6.3 - 99.9%: 8.1 - Max: 9.0
14:53:56.783 [main] INFO - Pub rate 500.7 msg/s / 0.0 Mb/s | Cons rate 494.8 msg/s / 0.0 Mb/s | Backlog: 0.4 K | Pub Latency (ms) avg: 3.0 - 50%: 3.0 - 99%: 7.0 - 99.9%: 14.2 - Max: 18.3
14:54:06.992 [main] INFO - Pub rate 499.7 msg/s / 0.0 Mb/s | Cons rate 541.3 msg/s / 0.1 Mb/s | Backlog: 0.0 K | Pub Latency (ms) avg: 2.8 - 50%: 3.0 - 99%: 5.4 - 99.9%: 7.8 - Max: 10.0
14:54:17.216 [main] INFO - Pub rate 498.2 msg/s / 0.0 Mb/s | Cons rate 500.1 msg/s / 0.0 Mb/s | Backlog: -0.0 K | Pub Latency (ms) avg: 2.9 - 50%: 3.0 - 99%: 5.8 - 99.9%: 11.0 - Max: 19.2
14:54:17.393 [main] INFO - ----- Aggregated Pub Latency (ms) avg: 3.5 - 50%: 3.0 - 95%: 5.0 - 99%: 30.9 - 99.9%: 49.4 - 99.99%: 50.1 - Max: 50.3
14:54:17.531 [main] INFO - ----- Starting benchmark traffic ------
14:54:27.659 [main] INFO - Pub rate 520.8 msg/s / 0.0 Mb/s | Cons rate 506.3 msg/s / 0.0 Mb/s | Backlog: 0.1 K | Pub Latency (ms) avg: 2.8 - 50%: 3.0 - 99%: 5.7 - 99.9%: 8.5 - Max: 14.1
14:54:37.785 [main] INFO - Pub rate 500.4 msg/s / 0.0 Mb/s | Cons rate 474.5 msg/s / 0.0 Mb/s | Backlog: 0.4 K | Pub Latency (ms) avg: 3.0 - 50%: 3.0 - 99%: 7.3 - 99.9%: 10.4 - Max: 11.2
14:54:47.944 [main] INFO - Pub rate 500.3 msg/s / 0.0 Mb/s | Cons rate 491.9 msg/s / 0.0 Mb/s | Backlog: 0.5 K | Pub Latency (ms) avg: 2.8 - 50%: 3.0 - 99%: 5.1 - 99.9%: 8.3 - Max: 11.3
14:54:58.081 [main] INFO - Pub rate 498.3 msg/s / 0.0 Mb/s | Cons rate 509.5 msg/s / 0.0 Mb/s | Backlog: 0.4 K | Pub Latency (ms) avg: 2.9 - 50%: 3.0 - 99%: 6.3 - 99.9%: 12.6 - Max: 16.6
14:55:08.225 [main] INFO - Pub rate 499.9 msg/s / 0.0 Mb/s | Cons rate 535.8 msg/s / 0.1 Mb/s | Backlog: 0.0 K | Pub Latency (ms) avg: 2.9 - 50%: 3.0 - 99%: 6.8 - 99.9%: 21.7 - Max: 31.3
14:55:18.347 [main] INFO - Pub rate 501.1 msg/s / 0.0 Mb/s | Cons rate 491.8 msg/s / 0.0 Mb/s | Backlog: 0.1 K | Pub Latency (ms) avg: 2.9 - 50%: 3.0 - 99%: 6.5 - 99.9%: 9.3 - Max: 10.9
14:55:18.480 [main] INFO - ----- Aggregated Pub Latency (ms) avg: 2.9 - 50%: 3.0 - 95%: 4.3 - 99%: 6.3 - 99.9%: 10.5 - 99.99%: 25.0 - Max: 31.3
14:55:18.702 [main] INFO - Writing test result into 1-topic-1-partition-100b-BookKeeper-2020-03-06-14-55-18.json
Docker stats during the test (only 2nd and 4th omb instances are processed)
And another example with 1 master and 6 workers (333 msg/s instead of 1000 msg/s):
A declarative, efficient, and flexible JavaScript library for building user interfaces.
🖖 Vue.js is a progressive, incrementally-adoptable JavaScript framework for building UI on the web.
TypeScript is a superset of JavaScript that compiles to clean JavaScript output.
An Open Source Machine Learning Framework for Everyone
The Web framework for perfectionists with deadlines.
A PHP framework for web artisans
Bring data to life with SVG, Canvas and HTML. 📊📈🎉
JavaScript (JS) is a lightweight interpreted programming language with first-class functions.
Some thing interesting about web. New door for the world.
A server is a program made to process requests and deliver data to clients.
Machine learning is a way of modeling and interpreting data that allows a piece of software to respond intelligently.
Some thing interesting about visualization, use data art
Some thing interesting about game, make everyone happy.
We are working to build community through open source technology. NB: members must have two-factor auth.
Open source projects and samples from Microsoft.
Google ❤️ Open Source for everyone.
Alibaba Open Source for everyone
Data-Driven Documents codes.
China tencent open source team.