We've been having issues recently in our production environment that our has caused our software to stop functioning properly. As you can see below, there is none of our code showing up in the stack trace, and it's just a recursive call within Netty. When this happens, it appears that all future connections get denied, and the CPU is maxed out on a few cores. (Not all, but some), which is strange, because since there's no traffic anymore since all of it just got disconnected, what on earth is being processed?
As mentioned previously, this only happens in our production environment. We see this issue arise on peak days more frequently when we have more traffic, although we run multiple instances with 100s of connections each 24h a day, and we still only see this issue maybe 2 times a week, so unfortunately trying to get information regarding this issue is a painful process. We also have reason to believe that when there are poor connections between our servers that the likelihood of this issue is increased.
I've not worked much before with Netty and as most of this isn't my code, I'm slightly clueless on where to even look, so felt seeking help would be the best option.
Here is the code in our channel initialiser:
@Override
protected void initChannel(SocketChannel ch) {
//Final handler in the pipeline. Deals with the objects once and hands them off to the rest of the code
MessageHandler handler = new MessageHandler(client);
//Converts the raw bytes into objects that we can deal with
CodecsHandler codecs = new CodecsHandler(client, ProtocolType.HANDSHAKE.getProtocol());
//Splits byte streams up into their packets
FramingHandler framing = new FramingHandler();
try {
ch.config().setOption(ChannelOption.IP_TOS, 0x18);
} catch (ChannelException ex) {
log.warn("Kernel lacks support for IP_TOS");
}
ch.config().setAllocator(PooledByteBufAllocator.DEFAULT);
ch.pipeline()
.addLast("idle_timeout", new IdleStateHandler(READ_IDLE_TIMEOUT, WRITE_IDLE_TIMEOUT, 0))
.addLast("framing", framing)
//The Noop handler does nothing (These parts of the pipeline are placed later)
.addLast("compression", NoopHandler.INSTANCE)
.addLast("codecs", codecs)
.addLast("handler", handler);
}
Here is our message handler class:
import com.flowpowered.network.Message;
import com.flowpowered.network.session.Session;
import io.netty.channel.Channel;
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.SimpleChannelInboundHandler;
import lombok.extern.slf4j.Slf4j;
import mycode.Client;
import java.util.concurrent.atomic.AtomicReference;
@Slf4j
public final class MessageHandler extends SimpleChannelInboundHandler<Message> {
private final AtomicReference<Session> session = new AtomicReference<>(null);
private final Client client;
public MessageHandler(Client client) {
this.client = client;
}
@Override
public void channelActive(ChannelHandlerContext ctx) {
Channel c = ctx.channel();
//Sends some packets packet back to the server and sets the channel object in the client to the channel object above.
//The returned object is an extended class of https://github.com/OverCaste/flow-networking/blob/master/src/main/java/com/flowpowered/networking/session/Session.java
Session s = client.newSession(c);
if (!session.compareAndSet(null, s)) {
throw new IllegalStateException("Session may not be set more than once");
}
s.onReady();
}
@Override
public void channelInactive(ChannelHandlerContext ctx) {
Session session = this.session.get();
if(session != null) {
session.onDisconnect();
} else {
log.warn("Child session was null so could not disconnect");
}
}
@Override
protected void channelRead0(ChannelHandlerContext ctx, Message i) {
//Passes the message off to our session object
session.get().messageReceived(i);
}
@Override
public void userEventTriggered(ChannelHandlerContext ctx, Object evt) {
}
@Override
public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {
session.get().onInboundThrowable(cause);
}
public AtomicReference<Session> getSession() {
return session;
}
}
Here is our Codecs Handler
import com.flowpowered.network.Codec;
import com.flowpowered.network.Message;
import com.flowpowered.network.util.ByteBufUtils;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.Unpooled;
import io.netty.channel.ChannelHandlerContext;
import io.netty.handler.codec.EncoderException;
import io.netty.handler.codec.MessageToMessageCodec;
import lombok.extern.slf4j.Slf4j;
import mycode.Client;
import mycode.CustomProtocol;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.List;
@Slf4j
public final class CodecsHandler extends MessageToMessageCodec<ByteBuf, Message> {
private final CustomProtocol protocol;
private final Client client;
public CodecsHandler(Client client, CustomProtocol protocol) {
this.protocol = protocol;
this.client = client;
}
@Override
protected void encode(ChannelHandlerContext ctx, Message msg, List<Object> out) throws Exception {
// find codec
Class<? extends Message> clazz = msg.getClass();
//Codec registration is handled by https://github.com/OverCaste/flow-networking/blob/master/src/main/java/com/flowpowered/networking/Codec.java
Codec.CodecRegistration reg = protocol.getCodecRegistration(clazz);
if (reg == null) {
throw new EncoderException("Unknown message type: " + clazz);
}
// write header
ByteBuf headerBuf = ctx.alloc().buffer(8);
ByteBufUtils.writeVarInt(headerBuf, reg.getOpcode());
// write body
ByteBuf messageBuf = ctx.alloc().buffer();
messageBuf = reg.getCodec().encode(messageBuf, msg);
out.add(Unpooled.wrappedBuffer(headerBuf, messageBuf));
}
@Override
protected void decode(ChannelHandlerContext ctx, ByteBuf msg, List<Object> out) throws Exception {
// find codec and read header
//https://github.com/OverCaste/flow-networking/blob/master/src/main/java/com/flowpowered/networking/Codec.java
Codec<?> codec = protocol.newReadHeader(msg);
// read body
Message decoded = codec.decode(msg);
if (msg.readableBytes() > 0) {
log.warn("Leftover bytes ({}) after decoding: {}", msg.readableBytes(), decoded);
}
out.add(decoded);
}
@Override
public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {
log.error("Codec throwable caught", cause);
//We do some other stuff around other parts of the code here
}
}
Here is our FramingHandler
import com.flowpowered.network.util.ByteBufUtils;
import io.netty.buffer.ByteBuf;
import io.netty.channel.ChannelHandlerContext;
import io.netty.handler.codec.ByteToMessageCodec;
import java.util.List;
public final class FramingHandler extends ByteToMessageCodec<ByteBuf> {
private static boolean readableVarInt(ByteBuf buf) {
if (buf.readableBytes() > 5) {
// maximum varint size
return true;
}
int idx = buf.readerIndex();
byte in;
do {
if (buf.readableBytes() < 1) {
buf.readerIndex(idx);
return false;
}
in = buf.readByte();
} while ((in & 0x80) != 0);
buf.readerIndex(idx);
return true;
}
@Override
protected void encode(ChannelHandlerContext ctx, ByteBuf msg, ByteBuf out) {
ByteBufUtils.writeVarInt(out, msg.readableBytes());
out.writeBytes(msg);
}
@Override
protected void decode(ChannelHandlerContext ctx, ByteBuf in, List<Object> out) throws Exception {
// check for length field readability
in.markReaderIndex();
if (!readableVarInt(in)) {
return;
}
// check for contents readability
int length = ByteBufUtils.readVarInt(in);
if (in.readableBytes() < length) {
in.resetReaderIndex();
return;
}
// read contents into buf
ByteBuf buf = ctx.alloc().buffer(length);
in.readBytes(buf, length);
out.add(buf);
}
}
Here is the compression handler
import com.flowpowered.network.util.ByteBufUtils;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.Unpooled;
import io.netty.channel.ChannelHandlerContext;
import io.netty.handler.codec.DecoderException;
import io.netty.handler.codec.EncoderException;
import io.netty.handler.codec.MessageToMessageCodec;
import java.util.List;
import java.util.zip.Deflater;
import java.util.zip.Inflater;
public final class CompressionHandler extends MessageToMessageCodec<ByteBuf, ByteBuf> {
private static final int MAX_INFLATED_BYTES = 1_000_000;
private static final int COMPRESSION_LEVEL = Deflater.DEFAULT_COMPRESSION;
private final int threshold;
private final Inflater inflater;
private final Deflater deflater;
public CompressionHandler(int threshold) {
this.threshold = threshold;
inflater = new Inflater();
deflater = new Deflater(COMPRESSION_LEVEL);
}
@Override
protected void encode(ChannelHandlerContext ctx, ByteBuf msg, List<Object> out) {
ByteBuf prefixBuf = ctx.alloc().buffer(5);
ByteBuf contentsBuf;
try {
if (msg.readableBytes() >= threshold){
// message should be compressed
int index = msg.readerIndex();
int length = msg.readableBytes();
byte[] sourceData = new byte[length];
msg.readBytes(sourceData);
deflater.setInput(sourceData);
deflater.finish();
ByteBuf result = msg.alloc().buffer(length);
byte[] byteArray = new byte[8192];
int totalBytes = 0;
while (!deflater.finished()){
int compressedLength = deflater.deflate(byteArray);
result.writeBytes(byteArray, 0, compressedLength);
totalBytes += compressedLength;
}
deflater.reset();
if (totalBytes == 0){
// compression failed in some weird way
throw new EncoderException("Failed to compress message of size " + length);
} else if (totalBytes >= length){
// compression increased the size. threshold is probably too low
// send as an uncompressed packet
result.release();
ByteBufUtils.writeVarInt(prefixBuf, 0);
msg.readerIndex(index);
msg.retain();
contentsBuf = msg;
} else {
// all is well
ByteBufUtils.writeVarInt(prefixBuf, length);
contentsBuf = result;
}
} else {
// message should be sent through
ByteBufUtils.writeVarInt(prefixBuf, 0);
msg.retain();
contentsBuf = msg;
}
} catch (Exception e){
prefixBuf.release();
throw e;
}
out.add(Unpooled.wrappedBuffer(prefixBuf, contentsBuf));
}
@Override
protected void decode(ChannelHandlerContext ctx, ByteBuf msg, List<Object> out) throws Exception {
int index = msg.readerIndex();
int uncompressedSize = ByteBufUtils.readVarInt(msg);
if (uncompressedSize == 0) {
// message is uncompressed
int length = msg.readableBytes();
if (length >= threshold) {
// invalid
throw new DecoderException("Received uncompressed message of size " + length + " greater than threshold " + threshold);
}
msg.retain();
out.add(msg);
} else {
if (uncompressedSize > MAX_INFLATED_BYTES)
//Don't trust this - this is a very big and may come with malicious intent
throw new DecoderException("Resulting uncompressed size is too large for us to handle safely");
// message is compressed
byte[] sourceData = new byte[msg.readableBytes()];
msg.readBytes(sourceData);
inflater.setInput(sourceData);
byte[] destData = new byte[8192];
ByteBuf result = msg.alloc().buffer(uncompressedSize);
int totalBytes = 0;
while (!inflater.finished()){
int resultLength = inflater.inflate(destData);
result.writeBytes(destData, 0, resultLength);
totalBytes += resultLength;
if (totalBytes > uncompressedSize)
throw new DecoderException("Received compressed message claiming to be of size " + uncompressedSize + " but actually larger");
}
inflater.reset();
if (totalBytes == 0) {
// might be a leftover from before compression was enabled (no compression header)
// uncompressedSize is likely to be < threshold
result.release();
msg.readerIndex(index);
msg.retain();
out.add(msg);
} else if (totalBytes != uncompressedSize) {
throw new DecoderException("Received compressed message claiming to be of size " + uncompressedSize + " but actually " + totalBytes);
} else {
out.add(result);
}
}
}
}
Here is our NoopHandler (This is literally just left as a placeholder for parts of the pipeline that aren't doing anything yet)
import io.netty.channel.ChannelHandler;
import io.netty.channel.ChannelHandlerAdapter;
@ChannelHandler.Sharable
public class NoopHandler extends ChannelHandlerAdapter {
public static final NoopHandler INSTANCE = new NoopHandler();
private NoopHandler() {}
}
Due to the scale of the project, and the fact that I'm not allowed to release all the code, I've stripped out a lot of the unnecessary code and refactored class names.
java.lang.StackOverflowError: null
at org.apache.logging.slf4j.Log4jLogger.log(Log4jLogger.java:371) ~[log4j-slf4j-impl-2.11.1.jar:2.11.1]
at io.netty.util.internal.logging.LocationAwareSlf4JLogger.log(LocationAwareSlf4JLogger.java:42) ~[netty-common-4.1.31.Final.jar:4.1.31.Final]
at io.netty.util.internal.logging.LocationAwareSlf4JLogger.warn(LocationAwareSlf4JLogger.java:198) ~[netty-common-4.1.31.Final.jar:4.1.31.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:294) ~[netty-transport-4.1.31.Final.jar:4.1.31.Final]
at io.netty.channel.AbstractChannelHandlerContext.notifyHandlerException(AbstractChannelHandlerContext.java:856) ~[netty-transport-4.1.31.Final.jar:4.1.31.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeFlush0(AbstractChannelHandlerContext.java:778) ~[netty-transport-4.1.31.Final.jar:4.1.31.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeFlush(AbstractChannelHandlerContext.java:768) ~[netty-transport-4.1.31.Final.jar:4.1.31.Final]
at io.netty.channel.AbstractChannelHandlerContext.flush(AbstractChannelHandlerContext.java:749) ~[netty-transport-4.1.31.Final.jar:4.1.31.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeFlush(AbstractChannelHandlerContext.java:770) ~[netty-transport-4.1.31.Final.jar:4.1.31.Final]
at io.netty.channel.AbstractChannelHandlerContext.flush(AbstractChannelHandlerContext.java:749) ~[netty-transport-4.1.31.Final.jar:4.1.31.Final]
at io.netty.channel.ChannelDuplexHandler.flush(ChannelDuplexHandler.java:117) ~[netty-transport-4.1.31.Final.jar:4.1.31.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeFlush0(AbstractChannelHandlerContext.java:776) ~[netty-transport-4.1.31.Final.jar:4.1.31.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeFlush(AbstractChannelHandlerContext.java:768) ~[netty-transport-4.1.31.Final.jar:4.1.31.Final]
at io.netty.channel.AbstractChannelHandlerContext.flush(AbstractChannelHandlerContext.java:749) ~[netty-transport-4.1.31.Final.jar:4.1.31.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeFlush(AbstractChannelHandlerContext.java:770) ~[netty-transport-4.1.31.Final.jar:4.1.31.Final]
at io.netty.channel.AbstractChannelHandlerContext.flush(AbstractChannelHandlerContext.java:749) ~[netty-transport-4.1.31.Final.jar:4.1.31.Final]
at io.netty.channel.ChannelDuplexHandler.flush(ChannelDuplexHandler.java:117) ~[netty-transport-4.1.31.Final.jar:4.1.31.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeFlush0(AbstractChannelHandlerContext.java:776) ~[netty-transport-4.1.31.Final.jar:4.1.31.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeFlush(AbstractChannelHandlerContext.java:768) ~[netty-transport-4.1.31.Final.jar:4.1.31.Final]
at io.netty.channel.AbstractChannelHandlerContext.flush(AbstractChannelHandlerContext.java:749) ~[netty-transport-4.1.31.Final.jar:4.1.31.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeFlush(AbstractChannelHandlerContext.java:770) ~[netty-transport-4.1.31.Final.jar:4.1.31.Final]
at io.netty.channel.AbstractChannelHandlerContext.flush(AbstractChannelHandlerContext.java:749) ~[netty-transport-4.1.31.Final.jar:4.1.31.Final]
at io.netty.channel.ChannelDuplexHandler.flush(ChannelDuplexHandler.java:117) ~[netty-transport-4.1.31.Final.jar:4.1.31.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeFlush0(AbstractChannelHandlerContext.java:776) ~[netty-transport-4.1.31.Final.jar:4.1.31.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeFlush(AbstractChannelHandlerContext.java:768) ~[netty-transport-4.1.31.Final.jar:4.1.31.Final]
at io.netty.channel.AbstractChannelHandlerContext.flush(AbstractChannelHandlerContext.java:749) ~[netty-transport-4.1.31.Final.jar:4.1.31.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeFlush(AbstractChannelHandlerContext.java:770) ~[netty-transport-4.1.31.Final.jar:4.1.31.Final]
at io.netty.channel.AbstractChannelHandlerContext.flush(AbstractChannelHandlerContext.java:749) ~[netty-transport-4.1.31.Final.jar:4.1.31.Final]
at io.netty.channel.ChannelDuplexHandler.flush(ChannelDuplexHandler.java:117) ~[netty-transport-4.1.31.Final.jar:4.1.31.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeFlush0(AbstractChannelHandlerContext.java:776) ~[netty-transport-4.1.31.Final.jar:4.1.31.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeFlush(AbstractChannelHandlerContext.java:768) ~[netty-transport-4.1.31.Final.jar:4.1.31.Final]
at io.netty.channel.AbstractChannelHandlerContext.flush(AbstractChannelHandlerContext.java:749) ~[netty-transport-4.1.31.Final.jar:4.1.31.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeFlush(AbstractChannelHandlerContext.java:770) ~[netty-transport-4.1.31.Final.jar:4.1.31.Final]
at io.netty.channel.AbstractChannelHandlerContext.flush(AbstractChannelHandlerContext.java:749) ~[netty-transport-4.1.31.Final.jar:4.1.31.Final]
at io.netty.channel.ChannelDuplexHandler.flush(ChannelDuplexHandler.java:117) ~[netty-transport-4.1.31.Final.jar:4.1.31.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeFlush0(AbstractChannelHandlerContext.java:776) ~[netty-transport-4.1.31.Final.jar:4.1.31.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeFlush(AbstractChannelHandlerContext.java:768) ~[netty-transport-4.1.31.Final.jar:4.1.31.Final]
at io.netty.channel.AbstractChannelHandlerContext.flush(AbstractChannelHandlerContext.java:749) ~[netty-transport-4.1.31.Final.jar:4.1.31.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeFlush(AbstractChannelHandlerContext.java:770) ~[netty-transport-4.1.31.Final.jar:4.1.31.Final]
at io.netty.channel.AbstractChannelHandlerContext.flush(AbstractChannelHandlerContext.java:749) ~[netty-transport-4.1.31.Final.jar:4.1.31.Final]
at io.netty.channel.ChannelDuplexHandler.flush(ChannelDuplexHandler.java:117) ~[netty-transport-4.1.31.Final.jar:4.1.31.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeFlush0(AbstractChannelHandlerContext.java:776) ~[netty-transport-4.1.31.Final.jar:4.1.31.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeFlush(AbstractChannelHandlerContext.java:768) ~[netty-transport-4.1.31.Final.jar:4.1.31.Final]
at io.netty.channel.AbstractChannelHandlerContext.flush(AbstractChannelHandlerContext.java:749) ~[netty-transport-4.1.31.Final.jar:4.1.31.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeFlush(AbstractChannelHandlerContext.java:770) ~[netty-transport-4.1.31.Final.jar:4.1.31.Final]
at io.netty.channel.AbstractChannelHandlerContext.flush(AbstractChannelHandlerContext.java:749) ~[netty-transport-4.1.31.Final.jar:4.1.31.Final]
at io.netty.channel.ChannelDuplexHandler.flush(ChannelDuplexHandler.java:117) ~[netty-transport-4.1.31.Final.jar:4.1.31.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeFlush0(AbstractChannelHandlerContext.java:776) ~[netty-transport-4.1.31.Final.jar:4.1.31.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeFlush(AbstractChannelHandlerContext.java:768) ~[netty-transport-4.1.31.Final.jar:4.1.31.Final]
at io.netty.channel.AbstractChannelHandlerContext.flush(AbstractChannelHandlerContext.java:749) ~[netty-transport-4.1.31.Final.jar:4.1.31.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeFlush(AbstractChannelHandlerContext.java:770) ~[netty-transport-4.1.31.Final.jar:4.1.31.Final]
at io.netty.channel.AbstractChannelHandlerContext.flush(AbstractChannelHandlerContext.java:749) ~[netty-transport-4.1.31.Final.jar:4.1.31.Final]
at io.netty.channel.ChannelDuplexHandler.flush(ChannelDuplexHandler.java:117) ~[netty-transport-4.1.31.Final.jar:4.1.31.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeFlush0(AbstractChannelHandlerContext.java:776) ~[netty-transport-4.1.31.Final.jar:4.1.31.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeFlush(AbstractChannelHandlerContext.java:768) ~[netty-transport-4.1.31.Final.jar:4.1.31.Final]
at io.netty.channel.AbstractChannelHandlerContext.flush(AbstractChannelHandlerContext.java:749) ~[netty-transport-4.1.31.Final.jar:4.1.31.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeFlush(AbstractChannelHandlerContext.java:770) ~[netty-transport-4.1.31.Final.jar:4.1.31.Final]
at io.netty.channel.AbstractChannelHandlerContext.flush(AbstractChannelHandlerContext.java:749) ~[netty-transport-4.1.31.Final.jar:4.1.31.Final]
at io.netty.channel.ChannelDuplexHandler.flush(ChannelDuplexHandler.java:117) ~[netty-transport-4.1.31.Final.jar:4.1.31.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeFlush0(AbstractChannelHandlerContext.java:776) ~[netty-transport-4.1.31.Final.jar:4.1.31.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeFlush(AbstractChannelHandlerContext.java:768) ~[netty-transport-4.1.31.Final.jar:4.1.31.Final]
at io.netty.channel.AbstractChannelHandlerContext.flush(AbstractChannelHandlerContext.java:749) ~[netty-transport-4.1.31.Final.jar:4.1.31.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeFlush(AbstractChannelHandlerContext.java:770) ~[netty-transport-4.1.31.Final.jar:4.1.31.Final]
at io.netty.channel.AbstractChannelHandlerContext.flush(AbstractChannelHandlerContext.java:749) ~[netty-transport-4.1.31.Final.jar:4.1.31.Final]
at io.netty.channel.ChannelDuplexHandler.flush(ChannelDuplexHandler.java:117) ~[netty-transport-4.1.31.Final.jar:4.1.31.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeFlush0(AbstractChannelHandlerContext.java:776) ~[netty-transport-4.1.31.Final.jar:4.1.31.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeFlush(AbstractChannelHandlerContext.java:768) ~[netty-transport-4.1.31.Final.jar:4.1.31.Final]
at io.netty.channel.AbstractChannelHandlerContext.flush(AbstractChannelHandlerContext.java:749) ~[netty-transport-4.1.31.Final.jar:4.1.31.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeFlush(AbstractChannelHandlerContext.java:770) ~[netty-transport-4.1.31.Final.jar:4.1.31.Final]
at io.netty.channel.AbstractChannelHandlerContext.flush(AbstractChannelHandlerContext.java:749) ~[netty-transport-4.1.31.Final.jar:4.1.31.Final]
at io.netty.channel.ChannelDuplexHandler.flush(ChannelDuplexHandler.java:117) ~[netty-transport-4.1.31.Final.jar:4.1.31.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeFlush0(AbstractChannelHandlerContext.java:776) ~[netty-transport-4.1.31.Final.jar:4.1.31.Final]
... ... ...