In a clustered ODL setup with 3 Controllers and 3 Computes, when we triggered browbeat/rally tests with concurrencies of 8, 16, 32 and times set to 500, we observed the following exception in karaf.

Note: some related discussion can be found in bug#9038

2017-09-14 03:54:13,184 | ERROR | ntLoopGroup-7-76 | TransactionChainManager | 304 - org.opendaylight.openflowplugin.common - 0.4.2.SNAPSHOT | Exception during INITIAL transaction submitting.
java.util.concurrent.TimeoutException: Timeout waiting for task.
at com.google.common.util.concurrent.AbstractFuture$Sync.get(AbstractFuture.java:269)[65:com.google.guava:18.0.0]
at com.google.common.util.concurrent.AbstractFuture.get(AbstractFuture.java:96)[65:com.google.guava:18.0.0]
at com.google.common.util.concurrent.ForwardingFuture.get(ForwardingFuture.java:69)[65:com.google.guava:18.0.0]
at org.opendaylight.openflowplugin.common.txchain.TransactionChainManager.submitTransaction(TransactionChainManager.java:180)[304:org.opendaylight.openflowplugin.common:0.4.2.SNAPSHOT]
at org.opendaylight.openflowplugin.common.txchain.TransactionChainManager.initialSubmitWriteTransaction(TransactionChainManager.java:85)[304:org.opendaylight.openflowplugin.common:0.4.2.SNAPSHOT]
at org.opendaylight.openflowplugin.impl.device.DeviceContextImpl.initialSubmitTransaction(DeviceContextImpl.java:225)[307:org.opendaylight.openflowplugin.impl:0.4.2.SNAPSHOT]
at org.opendaylight.openflowplugin.impl.statistics.StatisticsContextImpl$4.onSuccess(StatisticsContextImpl.java:362)[307:org.opendaylight.openflowplugin.impl:0.4.2.SNAPSHOT]
at org.opendaylight.openflowplugin.impl.statistics.StatisticsContextImpl$4.onSuccess(StatisticsContextImpl.java:354)[307:org.opendaylight.openflowplugin.impl:0.4.2.SNAPSHOT]
at com.google.common.util.concurrent.Futures$6.run(Futures.java:1319)[65:com.google.guava:18.0.0]
at com.google.common.util.concurrent.MoreExecutors$DirectExecutor.execute(MoreExecutors.java:457)[65:com.google.guava:18.0.0]
at com.google.common.util.concurrent.ExecutionList.executeListener(ExecutionList.java:156)[65:com.google.guava:18.0.0]
at com.google.common.util.concurrent.ExecutionList.execute(ExecutionList.java:145)[65:com.google.guava:18.0.0]
at com.google.common.util.concurrent.AbstractFuture.set(AbstractFuture.java:185)[65:com.google.guava:18.0.0]
at com.google.common.util.concurrent.Futures$ChainingListenableFuture$1.run(Futures.java:918)[65:com.google.guava:18.0.0]
at com.google.common.util.concurrent.MoreExecutors$DirectExecutor.execute(MoreExecutors.java:457)[65:com.google.guava:18.0.0]
at com.google.common.util.concurrent.ExecutionList.executeListener(ExecutionList.java:156)[65:com.google.guava:18.0.0]
at com.google.common.util.concurrent.ExecutionList.execute(ExecutionList.java:145)[65:com.google.guava:18.0.0]
at com.google.common.util.concurrent.AbstractFuture.set(AbstractFuture.java:185)[65:com.google.guava:18.0.0]
at com.google.common.util.concurrent.Futures$ChainingListenableFuture$1.run(Futures.java:918)[65:com.google.guava:18.0.0]
at com.google.common.util.concurrent.MoreExecutors$DirectExecutor.execute(MoreExecutors.java:457)[65:com.google.guava:18.0.0]
at com.google.common.util.concurrent.ExecutionList.executeListener(ExecutionList.java:156)[65:com.google.guava:18.0.0]
at com.google.common.util.concurrent.ExecutionList.execute(ExecutionList.java:145)[65:com.google.guava:18.0.0]
at com.google.common.util.concurrent.AbstractFuture.set(AbstractFuture.java:185)[65:com.google.guava:18.0.0]
at com.google.common.util.concurrent.Futures$ChainingListenableFuture$1.run(Futures.java:918)[65:com.google.guava:18.0.0]
at com.google.common.util.concurrent.MoreExecutors$DirectExecutor.execute(MoreExecutors.java:457)[65:com.google.guava:18.0.0]
at com.google.common.util.concurrent.Futures$ImmediateFuture.addListener(Futures.java:106)[65:com.google.guava:18.0.0]
at com.google.common.util.concurrent.Futures$ChainingListenableFuture.run(Futures.java:914)[65:com.google.guava:18.0.0]
at com.google.common.util.concurrent.MoreExecutors$DirectExecutor.execute(MoreExecutors.java:457)[65:com.google.guava:18.0.0]
at com.google.common.util.concurrent.ExecutionList.executeListener(ExecutionList.java:156)[65:com.google.guava:18.0.0]
at com.google.common.util.concurrent.ExecutionList.execute(ExecutionList.java:145)[65:com.google.guava:18.0.0]
at com.google.common.util.concurrent.AbstractFuture.set(AbstractFuture.java:185)[65:com.google.guava:18.0.0]
at com.google.common.util.concurrent.SettableFuture.set(SettableFuture.java:53)[65:com.google.guava:18.0.0]
at org.opendaylight.openflowplugin.impl.rpc.AbstractRequestContext.setResult(AbstractRequestContext.java:32)[307:org.opendaylight.openflowplugin.impl:0.4.2.SNAPSHOT]
at org.opendaylight.openflowplugin.impl.device.listener.MultiMsgCollectorImpl.endCollecting(MultiMsgCollectorImpl.java:70)[307:org.opendaylight.openflowplugin.impl:0.4.2.SNAPSHOT]
at org.opendaylight.openflowplugin.impl.device.listener.MultiMsgCollectorImpl.addMultipartMsg(MultiMsgCollectorImpl.java:58)[307:org.opendaylight.openflowplugin.impl:0.4.2.SNAPSHOT]
at org.opendaylight.openflowplugin.impl.services.AbstractMultipartRequestCallback.onSuccess(AbstractMultipartRequestCallback.java:54)[307:org.opendaylight.openflowplugin.impl:0.4.2.SNAPSHOT]
at org.opendaylight.openflowplugin.impl.services.AbstractMultipartRequestCallback.onSuccess(AbstractMultipartRequestCallback.java:22)[307:org.opendaylight.openflowplugin.impl:0.4.2.SNAPSHOT]
at org.opendaylight.openflowjava.protocol.impl.core.connection.OutboundQueueEntry.complete(OutboundQueueEntry.java:121)[296:org.opendaylight.openflowjava.openflow-protocol-impl:0.9.2.SNAPSHOT]
at org.opendaylight.openflowjava.protocol.impl.core.connection.StackedSegment.completeEntry(StackedSegment.java:113)[296:org.opendaylight.openflowjava.openflow-protocol-impl:0.9.2.SNAPSHOT]
at org.opendaylight.openflowjava.protocol.impl.core.connection.StackedSegment.pairRequest(StackedSegment.java:153)[296:org.opendaylight.openflowjava.openflow-protocol-impl:0.9.2.SNAPSHOT]
at org.opendaylight.openflowjava.protocol.impl.core.connection.AbstractStackedOutboundQueue.pairRequest(AbstractStackedOutboundQueue.java:203)[296:org.opendaylight.openflowjava.openflow-protocol-impl:0.9.2.SNAPSHOT]
at org.opendaylight.openflowjava.protocol.impl.core.connection.AbstractOutboundQueueManager.onMessage(AbstractOutboundQueueManager.java:208)[296:org.opendaylight.openflowjava.openflow-protocol-impl:0.9.2.SNAPSHOT]
at org.opendaylight.openflowjava.protocol.impl.core.connection.ConnectionAdapterImpl.consumeDeviceMessage(ConnectionAdapterImpl.java:143)[296:org.opendaylight.openflowjava.openflow-protocol-impl:0.9.2.SNAPSHOT]
at org.opendaylight.openflowjava.protocol.impl.core.connection.AbstractConnectionAdapterStatistics.consume(AbstractConnectionAdapterStatistics.java:66)[296:org.opendaylight.openflowjava.openflow-protocol-impl:0.9.2.SNAPSHOT]
at org.opendaylight.openflowjava.protocol.impl.core.connection.ConnectionAdapterImpl.consume(ConnectionAdapterImpl.java:44)[296:org.opendaylight.openflowjava.openflow-protocol-impl:0.9.2.SNAPSHOT]
at org.opendaylight.openflowjava.protocol.impl.core.DelegatingInboundHandler.channelRead(DelegatingInboundHandler.java:43)[296:org.opendaylight.openflowjava.openflow-protocol-impl:0.9.2.SNAPSHOT]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:363)[146:io.netty.transport:4.1.8.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:349)[146:io.netty.transport:4.1.8.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:341)[146:io.netty.transport:4.1.8.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.channelRead(ChannelInboundHandlerAdapter.java:86)[146:io.netty.transport:4.1.8.Final]
at org.opendaylight.openflowjava.protocol.impl.core.connection.AbstractOutboundQueueManager.channelRead(AbstractOutboundQueueManager.java:195)[296:org.opendaylight.openflowjava.openflow-protocol-impl:0.9.2.SNAPSHOT]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:363)[146:io.netty.transport:4.1.8.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:349)[146:io.netty.transport:4.1.8.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:341)[146:io.netty.transport:4.1.8.Final]
at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102)[148:io.netty.codec:4.1.8.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:363)[146:io.netty.transport:4.1.8.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:349)[146:io.netty.transport:4.1.8.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:341)[146:io.netty.transport:4.1.8.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:293)[148:io.netty.codec:4.1.8.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:267)[148:io.netty.codec:4.1.8.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:363)[146:io.netty.transport:4.1.8.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:349)[146:io.netty.transport:4.1.8.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:341)[146:io.netty.transport:4.1.8.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:293)[148:io.netty.codec:4.1.8.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:267)[148:io.netty.codec:4.1.8.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:363)[146:io.netty.transport:4.1.8.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:349)[146:io.netty.transport:4.1.8.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:341)[146:io.netty.transport:4.1.8.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:287)[150:io.netty.handler:4.1.8.Final]
at org.opendaylight.openflowjava.protocol.impl.core.IdleHandler.channelRead(IdleHandler.java:39)[296:org.opendaylight.openflowjava.openflow-protocol-impl:0.9.2.SNAPSHOT]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:363)[146:io.netty.transport:4.1.8.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:349)[146:io.netty.transport:4.1.8.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:341)[146:io.netty.transport:4.1.8.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1334)[146:io.netty.transport:4.1.8.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:363)[146:io.netty.transport:4.1.8.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:349)[146:io.netty.transport:4.1.8.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:926)[146:io.netty.transport:4.1.8.Final]
at io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:1018)[151:io.netty.transport-native-epoll:4.1.8.Final]
at io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:394)[151:io.netty.transport-native-epoll:4.1.8.Final]
at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:288)[151:io.netty.transport-native-epoll:4.1.8.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:858)[145:io.netty.common:4.1.8.Final]
at io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:144)[145:io.netty.common:4.1.8.Final]
at java.lang.Thread.run(Thread.java:748)[:1.8.0_141]
2017-09-14 03:54:13,185 | WARN | ntLoopGroup-7-76 | ContextChainHolderImpl | 307 - org.opendaylight.openflowplugin.impl - 0.4.2.SNAPSHOT | Not able to set MASTER role on device openflow:848556169412, reason: Initial transaction cannot be submitted.
2