1 2 Previous Next 24 Replies Latest reply on Aug 19, 2014 5:15 AM by rsinghal

    Memory leak with clustered JMS Server

    rsinghal

      I have configured clustered JMS Server with following configuration:

       

      • JMS Server 1 (Live) on machine 1 using standalone-full-ha.xml profile.
      • JMS Server 2 (Live) on machine 2 using standalone-full-ha.xml profile.
      • Deployed MDBs on machine 3 using standalone-full-ha.xml profile.

       

      We are using JBoss AS 7.1.2.Final. There are multiple MDBs and some of them are for Topics and some of them are for Queues. After some time JBoss Server where MDBs are deployed goes out of memory. I have analysed heap dump and found that HornetQRAActivation objects holding numerous HorneQMessageHandler objects which are for queue MDBs. Please let me know if I am missing something in configuration. We have following configuration in jboss-ejb3.xml for MDBs:

       

      <jee:message-driven>

                                    <jee:ejb-name>TestMDBTopic</jee:ejb-name>

                                    <jee:activation-config>

                                      <jee:activation-config-property>

                                              <jee:activation-config-property-name>reconnectAttempts</jee:activation-config-property-name>

                                              <jee:activation-config-property-value>-1</jee:activation-config-property-value>

                                      </jee:activation-config-property>

                                      <jee:activation-config-property>

                                              <jee:activation-config-property-name>setupAttempts</jee:activation-config-property-name>

                                              <jee:activation-config-property-value>-1</jee:activation-config-property-value>

                                      </jee:activation-config-property>

                                      <jee:activation-config-property>

                                              <jee:activation-config-property-name>destination</jee:activation-config-property-name>

                                              <jee:activation-config-property-value>test.topic</jee:activation-config-property-value>

                                      </jee:activation-config-property>

                                      <jee:activation-config-property>

                                              <jee:activation-config-property-name>connectorClassName</jee:activation-config-property-name>

                                              <jee:activation-config-property-value>${jboss.connection.classname}</jee:activation-config-property-value>

                                      </jee:activation-config-property>

                                      <jee:activation-config-property>

                                              <jee:activation-config-property-name>connectionParameters</jee:activation-config-property-name>

                                              <jee:activation-config-property-value>${jboss.connection.params}</jee:activation-config-property-value>

                                      </jee:activation-config-property>

                                     <jee:activation-config-property>

                     <jee:activation-config-property-name>hA</jee:activation-config-property-name>

                     <jee:activation-config-property-value>true</jee:activation-config-property-value>

                  </jee:activation-config-property>

                  <jee:activation-config-property>

                     <jee:activation-config-property-name>subscriptionDurability</jee:activation-config-property-name>

                     <jee:activation-config-property-value>Durable</jee:activation-config-property-value>

                  </jee:activation-config-property>

                  <jee:activation-config-property>

                     <jee:activation-config-property-name>clientID</jee:activation-config-property-name>

                     <jee:activation-config-property-value>${app.name}_test.topic.da6e11a2-278e-4575-9a3f-2014c7140d98</jee:activation-config-property-value>

                  </jee:activation-config-property>

                  <jee:activation-config-property>

                     <jee:activation-config-property-name>subscriptionName</jee:activation-config-property-name>

                     <jee:activation-config-property-value>${app.name}_test.topic.da6e11a2-278e-4575-9a3f-2014c7140d98</jee:activation-config-property-value>

                  </jee:activation-config-property>

               </jee:activation-config>

                          </jee:message-driven>

       

      <jee:message-driven>

                                    <jee:ejb-name>TestMDBQueue</jee:ejb-name>

                                    <jee:activation-config>

                                      <jee:activation-config-property>

                                              <jee:activation-config-property-name>reconnectAttempts</jee:activation-config-property-name>

                                              <jee:activation-config-property-value>-1</jee:activation-config-property-value>

                                      </jee:activation-config-property>

                                      <jee:activation-config-property>

                                              <jee:activation-config-property-name>setupAttempts</jee:activation-config-property-name>

                                              <jee:activation-config-property-value>-1</jee:activation-config-property-value>

                                      </jee:activation-config-property>

                                      <jee:activation-config-property>

                                              <jee:activation-config-property-name>destination</jee:activation-config-property-name>

                                              <jee:activation-config-property-value>test.queue</jee:activation-config-property-value>

                                      </jee:activation-config-property>

                                      <jee:activation-config-property>

                                              <jee:activation-config-property-name>connectorClassName</jee:activation-config-property-name>

                                              <jee:activation-config-property-value>${jboss.connection.classname}</jee:activation-config-property-value>

                                      </jee:activation-config-property>

                                      <jee:activation-config-property>

                                              <jee:activation-config-property-name>connectionParameters</jee:activation-config-property-name>

                                              <jee:activation-config-property-value>${jboss.connection.params}</jee:activation-config-property-value>

                                      </jee:activation-config-property>

                                     <jee:activation-config-property>

                     <jee:activation-config-property-name>hA</jee:activation-config-property-name>

                     <jee:activation-config-property-value>true</jee:activation-config-property-value>

                  </jee:activation-config-property>

                  <jee:activation-config-property>

                     <jee:activation-config-property-name>clientID</jee:activation-config-property-name>

                     <jee:activation-config-property-value>${app.name}_test.queue.49230413-fb63-4001-bda2-fa3b8af77c96</jee:activation-config-property-value>

                  </jee:activation-config-property>

                  <jee:activation-config-property>

                     <jee:activation-config-property-name>subscriptionName</jee:activation-config-property-name>

                     <jee:activation-config-property-value>${app.name}_test.queue.49230413-fb63-4001-bda2-fa3b8af77c96</jee:activation-config-property-value>

                  </jee:activation-config-property>

               </jee:activation-config>

                          </jee:message-driven>

        • 1. Re: Memory leak with clustered JMS Server
          rsinghal

          I am stuck here please share the configuration if anyone has configured similar JMS cluster.

          • 2. Re: Memory leak with clustered JMS Server
            mnovak

            Hi Ravi,

             

            could you attach your standalone-full-ha.xml(s) from your servers, please? (important is messaging subsystem)

             

            Also could you attach sources of your MDBs or describe what they're doing?

             

            We're testing this test this scenario. What needs to be configured is following:

             

            1. Deploy all queues, topics to server1 and server2

            2. Set <clustered>true</clustered> and provide  the same <cluster-password>${jboss.messaging.cluster.password:CHANGE ME!!}</cluster-password> for server1 and server2 if is security is enabled.

            3. Configure HornetQ RA on server3 in this way:

                                <pooled-connection-factory name="hornetq-ra">

                                    <transaction mode="xa"/>

                                    <connectors>

                                        <connector-ref connector-name="connector-to-remote-server"/>

                                    </connectors>

                                    <entries>

                                        <entry name="java:/JmsXA"/>

                                    </entries>

                                </pooled-connection-factory>

             

            where connector is:

            <connectors>

                                <netty-connector name="connector-to-remote-server" socket-binding="socket-to-remote-server"/>

            </connectors>

             

            where socket-binding is:

            <outbound-socket-binding name="mail-smtp">

                        <remote-destination host="server1_ip" port="5445"/>

            </outbound-socket-binding>

             

            Cheers,

             

            Mirek

            • 3. Re: Memory leak with clustered JMS Server
              rsinghal

              Thanks Mirek, for quick response.

               

              System goes OOM even if I leave my deployment idle. Here is messaging subsystem which we are using on Machine 3 where MDBs are deployed. It is same on Machine 1 and 2. Additionally. machine 1 and 2 have topics and queue deployed in messaging subsystem.

               

              <subsystem xmlns="urn:jboss:domain:messaging:1.2">

                          <hornetq-server>

                              <clustered>true</clustered>

                              <persistence-enabled>false</persistence-enabled>

                              <security-enabled>false</security-enabled>

                              <journal-file-size>102400</journal-file-size>

                              <journal-min-files>2</journal-min-files>

                              <connectors>

                                  <netty-connector name="netty" socket-binding="messaging"/>

                                  <netty-connector name="netty-throughput" socket-binding="messaging-throughput">

                                      <param key="batch-delay" value="50"/>

                                  </netty-connector>

                                  <in-vm-connector name="in-vm" server-id="0"/>

                              </connectors>

                              <acceptors>

                                  <netty-acceptor name="netty" socket-binding="messaging">

                                      <param key="use-nio" value="true"/>

                                  </netty-acceptor>

                                  <netty-acceptor name="netty-throughput" socket-binding="messaging-throughput">

                                      <param key="batch-delay" value="50"/>

                                      <param key="direct-deliver" value="false"/>

                                                                          <param key="use-nio" value="true"/>

                                  </netty-acceptor>

                                  <in-vm-acceptor name="in-vm" server-id="0"/>

                              </acceptors>

                              <broadcast-groups>

                                  <broadcast-group name="bg-group1">

                                      <socket-binding>messaging-group</socket-binding>

                                      <broadcast-period>5000</broadcast-period>

                                      <connector-ref>netty</connector-ref>

                                  </broadcast-group>

                              </broadcast-groups>

                              <discovery-groups>

                                  <discovery-group name="dg-group1">

                                      <socket-binding>messaging-group</socket-binding>

                                      <refresh-timeout>10000</refresh-timeout>

                                  </discovery-group>

                              </discovery-groups>

                              <cluster-connections>

                                  <cluster-connection name="my-cluster">

                                      <address>jms</address>

                                      <connector-ref>netty</connector-ref>

                                      <discovery-group-ref discovery-group-name="dg-group1"/>

                                  </cluster-connection>

                              </cluster-connections>

                              <security-settings>

                                  <security-setting match="#">

                                      <permission type="send" roles="guest"/>

                                      <permission type="consume" roles="guest"/>

                                      <permission type="createNonDurableQueue" roles="guest"/>

                                      <permission type="deleteNonDurableQueue" roles="guest"/>

                                  </security-setting>

                              </security-settings>

                              <address-settings>

                                  <!--default for catch all-->

                                  <address-setting match="#">

                                      <dead-letter-address>jms.queue.DLQ</dead-letter-address>

                                      <expiry-address>jms.queue.ExpiryQueue</expiry-address>

                                      <redelivery-delay>0</redelivery-delay>

                                                                          <redistribution-delay>1000</redistribution-delay>

                                      <max-size-bytes>10485760</max-size-bytes>

                                      <address-full-policy>BLOCK</address-full-policy>

                                      <message-counter-history-day-limit>10</message-counter-history-day-limit>

                                  </address-setting>

                              </address-settings>

                              <jms-connection-factories>

                                  <connection-factory name="InVmConnectionFactory">

                                      <connectors>

                                          <connector-ref connector-name="in-vm"/>

                                      </connectors>

                                      <entries>

                                          <entry name="java:/ConnectionFactory"/>

                                      </entries>

                                  </connection-factory>

                                  <connection-factory name="RemoteConnectionFactory">

                                      <connectors>

                                          <connector-ref connector-name="netty"/>

                                      </connectors>

                                      <entries>

                                          <entry name="java:jboss/exported/jms/RemoteConnectionFactory"/>

                                      </entries>

                                  </connection-factory>

                                  <pooled-connection-factory name="hornetq-ra">

                                      <transaction mode="xa"/>

                                      <connectors>

                                          <connector-ref connector-name="in-vm"/>

                                      </connectors>

                                      <entries>

                                          <entry name="java:/JmsXA"/>

                                      </entries>

                                  </pooled-connection-factory>

                              </jms-connection-factories>

                          </hornetq-server>

                      </subsystem>

              • 4. Re: Memory leak with clustered JMS Server
                mnovak

                Hi Ravi,

                 

                I can't find anything what could cause OOM. Only what comes to my mind is standalone.conf file and whether you modified setting memory for JVM. By default there should be something like:

                 

                if [ "x$JAVA_OPTS" = "x" ]; then

                   JAVA_OPTS="-Xms1303m -Xmx1303m -XX:MaxPermSize=256m -Djava.net.preferIPv4Stack=true"

                   JAVA_OPTS="$JAVA_OPTS -Djboss.modules.system.pkgs=$JBOSS_MODULES_SYSTEM_PKGS -Djava.awt.headless=true"

                else

                   echo "JAVA_OPTS already set in environment; overriding default settings with values: $JAVA_OPTS"

                fi

                 

                During start of the server you can check what is set in JAVA_OPTS variable. I can see:

                 

                =========================================================================

                 

                  JBoss Bootstrap Environment

                 

                  JBOSS_HOME: /home/mnovak/tmp/hornetq_eap6_dev/internal/eap6.1.0.ER8/server1/jboss-eap-6.1

                 

                  JAVA: /home/mnovak/Downloads/jdk1.7.0_15/bin/java

                 

                  JAVA_OPTS:  -server -XX:+UseCompressedOops -Xms1303m -Xmx1303m -XX:MaxPermSize=256m -Djava.net.preferIPv4Stack=true -Djboss.modules.system.pkgs=org.jboss.byteman -Djava.awt.headless=true

                 

                =========================================================================

                 

                 

                In our test we set memory for AS 7 server to:

                -Xms64m -Xmx788m -XX:MaxPermSize=256m

                 

                Are you using virtual machines? Is there enough memory for them?

                 

                Cheers,

                 

                Mirek

                • 5. Re: Memory leak with clustered JMS Server
                  rsinghal

                  Yes, we are using VMs and there is enough memory for servers. Following is the output at the beginning of JBoss server log on Application Server:

                   

                  "JAVA_OPTS already set in environment; overriding default settings with values:  -server -Xrs -Xms512m -Xmx512m -XX:+DisableExplicitGC -Xss192K -XX:MaxPermSize=256m -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=C:\logs\ -DAPP_SERVER_NAME=TestServer -DAPP_SERVER_PORT=4447 -Dservertype=cs  -Djboss.connection.params=host=JMSServer1;port=5445,host=JMSServer2;port=5445 -Djboss.connection.classname=org.hornetq.core.remoting.impl.netty.NettyConnectorFactory,org.hornetq.core.remoting.impl.netty.NettyConnectorFactory -Dapp.name=TestServer"

                  ===============================================================================

                   

                   

                    JBoss Bootstrap Environment

                   

                   

                    JBOSS_HOME: C:\jboss-as-7.1.2.Final

                   

                   

                    JAVA: C:\Program Files\Java\jdk1.7.0_04\bin\java

                   

                   

                    JAVA_OPTS: -Dprogram.name=standalone.bat  -server -Xrs -Xms512m -Xmx512m -XX:+DisableExplicitGC -Xss192K -XX:MaxPermSize=256m -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=C:\logs\  -DAPP_SERVER_NAME=TestServer -DAPP_SERVER_PORT=4447 -Dservertype=cs  -Djboss.connection.params=host=JMSServer1;port=5445,host=JMSServer2;port=5445 -Djboss.connection.classname=org.hornetq.core.remoting.impl.netty.NettyConnectorFactory,org.hornetq.core.remoting.impl.netty.NettyConnectorFactory -Dapp.name=TestServer

                   

                   

                  ===============================================================================

                   

                  In current test -Xms and -Xmx are set to 512MB but it goes OOM for 1024m also.

                  • 6. Re: Memory leak with clustered JMS Server
                    rsinghal

                    I tried by passing missing JAVA_OPTS but same result. I have uploaded heapdump at this location if it helps:

                     

                    https://docs.google.com/file/d/0BzcNdKDAu5f4Ujd0d3V0NXNGVGs/edit?usp=sharing

                    • 7. Re: Memory leak with clustered JMS Server
                      mnovak

                      Another thought...could you try to set reconnectAttempts and setupAttempts to 10 in activation-config property.

                      1 of 1 people found this helpful
                      • 8. Re: Memory leak with clustered JMS Server
                        rsinghal

                        It seems that changing reconnectAttempts and setupAttempts to 10 has fixed this problem. But we want MDBs to keep on retrying in case of network disconnect.

                        • 9. Re: Memory leak with clustered JMS Server
                          mnovak

                          Ok, it appears there is a leak in HornetQ resource apdapter when setupAttempts is set to -1 and it's indefinitely trying to create JMS connection to cluster. I've spent some time reproducing it with AS 7.2.0/EAP 6.1 but without success. Can you see warnings/errors in server.log, like  connection to cluster is unsuccessful.

                          • 10. Re: Memory leak with clustered JMS Server
                            rsinghal

                            We are using JBoss As 7.1.2 and we are observing it only for MDBs which are for queues. I do not see message like "Connection to cluster is unsuccessful" but I do see following message getting logged after couple of hours starting the application:

                             

                            06:36:33,054 INFO  [org.jboss.as] (Controller Boot Thread) JBAS015874: JBoss AS 7.1.2.Final "Steropes" started in 65423ms - Started 2655 of 3202 services (546 services are passive or on-demand)

                            06:38:45,094 WARN  [org.hornetq.core.protocol.core.impl.RemotingConnectionImpl] (hornetq-failure-check-thread) Connection failure has been detected: Did not receive data from /10.10.**.***:54108. It is likely the client has exited or crashed without closing its connection, or the network between the server and client has failed. You also might have configured connection-ttl and client-failure-check-period incorrectly. Please check user manual for more information. The connection will now be closed. [code=3]

                            06:54:09,517 WARN  [org.hornetq.core.protocol.core.impl.RemotingConnectionImpl] (hornetq-failure-check-thread) Connection failure has been detected: Did not receive data from /10.10.**.***:60308. It is likely the client has exited or crashed without closing its connection, or the network between the server and client has failed. You also might have configured connection-ttl and client-failure-check-period incorrectly. Please check user manual for more information. The connection will now be closed. [code=3]

                            09:56:57,698 INFO  [org.jboss.as.naming] (Remoting "****761l" task-3) JBAS011806: Channel end notification received, closing channel Channel ID 278d8f0f (inbound) of Remoting connection 12fe447e to /10.10.**.***:55638

                            09:57:19,633 ERROR [org.jboss.as.naming] (pool-5-thread-7) JBAS011809: Failed to send exception response to client: org.jboss.remoting3.NotOpenException: Writes closed

                                      at org.jboss.remoting3.remote.RemoteConnectionChannel.openOutboundMessage(RemoteConnectionChannel.java:107) [jboss-remoting-3.2.7.GA.jar:3.2.7.GA]

                                      at org.jboss.remoting3.remote.RemoteConnectionChannel.writeMessage(RemoteConnectionChannel.java:296) [jboss-remoting-3.2.7.GA.jar:3.2.7.GA]

                                      at org.jboss.naming.remote.protocol.v1.WriteUtil.write(WriteUtil.java:59)

                                      at org.jboss.naming.remote.protocol.v1.WriteUtil.writeExceptionResponse(WriteUtil.java:81)

                                      at org.jboss.naming.remote.protocol.v1.RemoteNamingServerV1$MessageReciever$1.sendIOException(RemoteNamingServerV1.java:97)

                                      at org.jboss.naming.remote.protocol.v1.RemoteNamingServerV1$MessageReciever$1.run(RemoteNamingServerV1.java:86)

                                      at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110) [rt.jar:1.7.0_04]

                                      at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603) [rt.jar:1.7.0_04]

                                      at java.lang.Thread.run(Thread.java:722) [rt.jar:1.7.0_04]

                             

                            • 11. Re: Memory leak with clustered JMS Server
                              mnovak

                              I did some testing with AS 7.1.2 but still no success. Based on the WARN ...Connection failure has been detected: Did not receive... I suspect there could be network failure problem in  environment or there is something in MDB what is causing the memory leak.

                               

                              INFO log ..  Channel end notification received, closing channel Channel ID 278d8f0f (inbound) of Remoting connection 12fe447e.. seems to be already resolved in [1]

                               

                              Some other thoughts what could be done:

                              • upgrade to AS 7.2.0/EAP 6.1 and give it a try (there are bugfixes for HQ RA in it)
                              • if it does not help then provide a reproducer or better description/source what MDB is doing
                              • (try to debug the server when MDB is deployed and see why HorneQMessageHandler instances are not garbage collected)

                               

                              [1] https://issues.jboss.org/browse/AS7-4508

                              • 12. Re: Memory leak with clustered JMS Server
                                rsinghal

                                Thanks Mirek, I'll try out with AS 7.2.0 and will let you know. I did debug the MDBs and MDBs are not doing anything at all when I leave my system idle.

                                • 13. Re: Memory leak with clustered JMS Server
                                  rsinghal

                                  Problem still exists with JBoss AS 7.2.0. I again checked heapdump and I could see that it is creating thousands of instances of MessageQHandler object for MDBs which are for queue. It is creating exactly 16 instances of MessageQHandler for MDBs which are for topic. It seems that it is not closing the instance properly in case of queue MDBs I can see following attributes for MessageQHandler instance:

                                   

                                  Type   |Name      |Value

                                  --------------------------------------------------------------------------------------

                                  ref    |cf        |org.hornetq.core.client.impl.ClientSessionFactoryImpl @ 0xff80b298

                                  ref    |tm        |com.arjuna.ats.jbossatx.jta.TransactionManagerDelegate @ 0xe146ab80

                                  int    |sessionNr |2

                                  boolean|useXA     |false

                                  boolean|transacted|false

                                  boolean|useLocalTx|false

                                  ref    |activation|org.hornetq.ra.inflow.HornetQActivation @ 0xe87c2170

                                  ref    |endpoint  |null

                                  ref    |consumer  |org.hornetq.core.client.impl.ClientConsumerImpl @ 0xff80ff90

                                  ref    |session   |org.hornetq.core.client.impl.DelegatingSession @ 0xff80ddf0

                                  --------------------------------------------------------------------------------------

                                   

                                  Also, sessionNr is either 0,1 or 2 for all these instances while for topic MDBs these are 0-15. Something is wrong for queue MDBs. Please advise how I can enable extra logging in HornetQActivation class to check what it is doing there.

                                   

                                  Thanks,

                                  Ravi

                                  • 14. Re: Memory leak with clustered JMS Server
                                    mnovak

                                    Just replace logging subsystem by:

                                     

                                    <subsystem xmlns="urn:jboss:domain:logging:1.2">
                                                <console-handler name="CONSOLE">
                                                    <level name="DEBUG"/>
                                                    <formatter>
                                                        <pattern-formatter pattern="%d{HH:mm:ss,SSS} %-5p [%c] (%t) %s%E%n"/>
                                                    </formatter>
                                                </console-handler>
                                                <periodic-rotating-file-handler name="FILE">
                                                    <formatter>
                                                        <pattern-formatter pattern="%d{HH:mm:ss,SSS} %-5p [%c] (%t) %s%E%n"/>
                                                    </formatter>
                                                    <file relative-to="jboss.server.log.dir" path="server.log"/>
                                                    <suffix value=".yyyy-MM-dd"/>
                                                    <append value="true"/>
                                                </periodic-rotating-file-handler>
                                                <logger category="com.arjuna">
                                                    <level name="INFO"/>
                                                </logger>
                                                <logger category="org.apache.tomcat.util.modeler">
                                                    <level name="WARN"/>
                                                </logger>
                                                <logger category="sun.rmi">
                                                    <level name="WARN"/>
                                                </logger>
                                                <logger category="jacorb">
                                                    <level name="WARN"/>
                                                </logger>
                                                <logger category="jacorb.config">
                                                    <level name="ERROR"/>
                                                </logger>
                                    
                                                <logger category="org.hornetq">
                                                    <level name="TRACE"/>
                                                </logger>
                                    
                                            <logger category="org.jboss.netty">
                                                    <level name="TRACE"/>
                                                </logger>
                                    
                                                <logger category="org.jboss">
                                                    <level name="INFO"/>
                                                </logger>
                                    
                                                <root-logger>
                                                    <level name="TRACE"/>
                                                    <handlers>
                                                        <handler name="CONSOLE"/>
                                                        <handler name="FILE"/>
                                                    </handlers>
                                                </root-logger>
                                    </subsystem>
                                    

                                     

                                    This will provide TRACE logs for org.hornetq and org.jboss.netty packages.

                                    1 2 Previous Next