fix: OF-2652 - prevent parser.read being called by changing isMaxBuff…

…erSizeExceeded logic

xmppserver/src/main/java/org/jivesoftware/openfire/nio/NettyXMPPDecoder.java

@@ -35,12 +35,10 @@ protected void decode(ChannelHandlerContext ctx, ByteBuf in, List<Object> out) t
         // Get the XML parser from the channel
         XMLLightweightParser parser = ctx.channel().attr(NettyConnectionHandler.XML_PARSER).get();
 
-        // Check that the buffer is not bigger than 1 Megabyte. For security reasons
+        // Check that the stanza constructed by the parser is not bigger than 1 Megabyte. For security reasons
         // we will abort parsing when 1 Mega of queued chars was found.
         if (parser.isMaxBufferSizeExceeded()) {
-            if (in.refCnt() > 0) { // prevent IllegalReferenceCountException if the ByteBuf has already been deallocated.
-                in.release();
-            }
+            in.release();
             return;
         }
 

xmppserver/src/main/java/org/jivesoftware/openfire/nio/XMLLightweightParser.java

@@ -155,7 +155,7 @@ protected void foundMsg(String msg) throws XMLNotWellFormedException {
     }
 
     public boolean isMaxBufferSizeExceeded() {
-        return maxBufferSizeExceeded;
+        return maxBufferSizeExceeded || buffer.length() > maxBufferSize;
     }
 
     public void read(char[] buf) throws Exception {