0 Replies Latest reply on Jun 10, 2015 5:18 AM by Giuseppe Nespolino

    Continuos warning with clustered HA HornetQ instances

    Giuseppe Nespolino Newbie

      Hello,

       

      I'm quite new to hornetQ and I did my best to read all documentation I can to achieve my goal, but I still got some issue.

      I need a clustered highly available messaging system, so since my application is developed on wildfly8, my choice for messaging was the embedded HornetQ.

       

      I choose to use 2 server each with a collocated setup.

      Relevant standalone.xml configuration part follow:

       

      SERVER 1

      <subsystem xmlns="urn:jboss:domain:messaging:2.0">
          <hornetq-server>
        <persistence-enabled>true</persistence-enabled>
        <security-enabled>false</security-enabled>
        <cluster-password>password</cluster-password>
        <shared-store>false</shared-store>
        <journal-type>ASYNCIO</journal-type>
        <journal-file-size>102400</journal-file-size>
        <journal-min-files>2</journal-min-files>
        <backup-group-name>group1</backup-group-name>
      
      
        <connectors>
           <netty-connector name="netty" socket-binding="messaging"/>
           <netty-connector name="netty-throughput" socket-binding="messaging-throughput">
        <param key="batch-delay" value="50"/>
           </netty-connector>
           <in-vm-connector name="in-vm" server-id="0"/>
        </connectors>
      
      
        <acceptors>
           <netty-acceptor name="netty" socket-binding="messaging"/>
           <netty-acceptor name="netty-throughput" socket-binding="messaging-throughput">
        <param key="batch-delay" value="50"/>
        <param key="direct-deliver" value="false"/>
           </netty-acceptor>
           <in-vm-acceptor name="in-vm" server-id="0"/>
        </acceptors>
      
      
        <broadcast-groups>
           <broadcast-group name="bg-group1">
        <group-address>231.7.7.7</group-address>
        <group-port>9876</group-port>
        <broadcast-period>5000</broadcast-period>
        <connector-ref>
           netty
        </connector-ref>
           </broadcast-group>
        </broadcast-groups>
      
      
        <discovery-groups>
           <discovery-group name="dg-group1">
        <group-address>231.7.7.7</group-address>
        <group-port>9876</group-port>
        <refresh-timeout>10000</refresh-timeout>
           </discovery-group>
        </discovery-groups>
      
      
        <cluster-connections>
           <cluster-connection name="my-cluster">
        <address>jms</address>
        <connector-ref>netty</connector-ref>
        <discovery-group-ref discovery-group-name="dg-group1"/>
           </cluster-connection>
        </cluster-connections>
      
      
        <security-settings>
           <security-setting match="#">
        <permission type="send" roles="guest"/>
        <permission type="consume" roles="guest"/>
        <permission type="createNonDurableQueue" roles="guest"/>
        <permission type="deleteNonDurableQueue" roles="guest"/>
           </security-setting>
        </security-settings>
      
      
        <address-settings>
           <address-setting match="#">
        <dead-letter-address>jms.queue.DLQ</dead-letter-address>
        <expiry-address>jms.queue.ExpiryQueue</expiry-address>
        <redelivery-delay>0</redelivery-delay>
        <max-size-bytes>10485760</max-size-bytes>
        <address-full-policy>BLOCK</address-full-policy>
        <message-counter-history-day-limit>10</message-counter-history-day-limit>
           </address-setting>
        </address-settings>
      
      
        <jms-connection-factories>
           <connection-factory name="InVmConnectionFactory">
        <connectors>
           <connector-ref connector-name="in-vm"/>
        </connectors>
        <entries>
           <entry name="java:/ConnectionFactory"/>
        </entries>
           </connection-factory>
           <connection-factory name="RemoteConnectionFactory">
        <connectors>
           <connector-ref connector-name="netty"/>
        </connectors>
        <entries>
           <entry name="java:jboss/exported/jms/RemoteConnectionFactory"/>
        </entries>
        <ha>true</ha>
        <block-on-acknowledge>true</block-on-acknowledge>
        <retry-interval>2000</retry-interval>
        <retry-interval-multiplier>1</retry-interval-multiplier>
        <reconnect-attempts>-1</reconnect-attempts>
           </connection-factory>
           <pooled-connection-factory name="hornetq-ra">
        <transaction mode="xa"/>
        <connectors>
           <connector-ref connector-name="in-vm"/>
        </connectors>
        <entries>
           <entry name="java:/JmsXA"/>
           <entry name="java:jboss/DefaultJMSConnectionFactory"/>
        </entries>
        <ha>true</ha>
        <block-on-acknowledge>true</block-on-acknowledge>
        <retry-interval>2000</retry-interval>
        <retry-interval-multiplier>1</retry-interval-multiplier>
        <reconnect-attempts>-1</reconnect-attempts>
           </pooled-connection-factory>
        </jms-connection-factories>
      
      
        <jms-destinations>
           <jms-queue name="ExpiryQueue">
        <entry name="java:/jms/queue/ExpiryQueue"/>
           </jms-queue>
           <jms-queue name="DLQ">
        <entry name="java:/jms/queue/DLQ"/>
           </jms-queue>
           <jms-queue name="InvioFattureQueue">
        <entry name="java:/jboss/exported/jms/queue/InvioFattureQueue"/>
        <durable>true</durable>
           </jms-queue>
           <jms-queue name="RicezioneNotificheQueue">
        <entry name="java:/jboss/exported/jms/queue/RicezioneNotificheQueue"/>
        <durable>true</durable>
           </jms-queue>
        </jms-destinations>
          </hornetq-server>
      
          <hornetq-server name="backup-server">
        <persistence-enabled>true</persistence-enabled>
        <security-enabled>false</security-enabled>
        <cluster-password>password</cluster-password>
        <backup>true</backup>
        <shared-store>false</shared-store>
        <journal-type>ASYNCIO</journal-type>
        <journal-file-size>102400</journal-file-size>
        <journal-min-files>2</journal-min-files>
        <backup-group-name>group2</backup-group-name>
        <paging-directory path="messagingpaging2"/>
        <bindings-directory path="messagingbindings2"/>
        <journal-directory path="messagingjournal2"/>
        <large-messages-directory path="messaginglargemessages2"/>
      
      
        <connectors>
           <netty-connector name="netty" socket-binding="messaging-backup"/>
           <in-vm-connector name="in-vm" server-id="0"/>
        </connectors>
      
      
        <acceptors>
           <netty-acceptor name="netty" socket-binding="messaging-backup"/>
        </acceptors>
      
      
        <broadcast-groups>
           <broadcast-group name="bg-group1">
        <group-address>231.7.7.7</group-address>
        <group-port>9876</group-port>
        <broadcast-period>5000</broadcast-period>
        <connector-ref>
           netty
        </connector-ref>
           </broadcast-group>
        </broadcast-groups>
      
      
        <discovery-groups>
           <discovery-group name="dg-group1">
        <group-address>231.7.7.7</group-address>
        <group-port>9876</group-port>
        <refresh-timeout>10000</refresh-timeout>
           </discovery-group>
        </discovery-groups>
      
      
        <cluster-connections>
           <cluster-connection name="my-cluster">
        <address>jms</address>
        <connector-ref>netty</connector-ref>
        <discovery-group-ref discovery-group-name="dg-group1"/>
           </cluster-connection>
        </cluster-connections>
      </hornetq-server>
      </subsystem>
      ......
      
              <socket-binding name="messaging" port="5445"/>
              <socket-binding name="messaging-backup" port="5545"/>
              <socket-binding name="messaging-throughput" port="5455"/>
      


      Server 2

      <subsystem xmlns="urn:jboss:domain:messaging:2.0">
          <hornetq-server>
        <persistence-enabled>true</persistence-enabled>
        <security-enabled>false</security-enabled>
        <cluster-password>password</cluster-password>
        <shared-store>false</shared-store>
        <journal-type>ASYNCIO</journal-type>
        <journal-file-size>102400</journal-file-size>
        <journal-min-files>2</journal-min-files>
        <backup-group-name>group2</backup-group-name>
      
      
        <connectors>
           <netty-connector name="netty" socket-binding="messaging"/>
           <netty-connector name="netty-throughput" socket-binding="messaging-throughput">
        <param key="batch-delay" value="50"/>
           </netty-connector>
           <in-vm-connector name="in-vm" server-id="0"/>
        </connectors>
      
      
        <acceptors>
           <netty-acceptor name="netty" socket-binding="messaging"/>
           <netty-acceptor name="netty-throughput" socket-binding="messaging-throughput">
        <param key="batch-delay" value="50"/>
        <param key="direct-deliver" value="false"/>
           </netty-acceptor>
           <in-vm-acceptor name="in-vm" server-id="0"/>
        </acceptors>
      
      
        <broadcast-groups>
           <broadcast-group name="bg-group1">
        <group-address>231.7.7.7</group-address>
        <group-port>9876</group-port>
        <broadcast-period>5000</broadcast-period>
        <connector-ref>
           netty
        </connector-ref>
           </broadcast-group>
        </broadcast-groups>
      
      
        <discovery-groups>
           <discovery-group name="dg-group1">
        <group-address>231.7.7.7</group-address>
        <group-port>9876</group-port>
        <refresh-timeout>10000</refresh-timeout>
           </discovery-group>
        </discovery-groups>
      
      
        <cluster-connections>
           <cluster-connection name="my-cluster">
        <address>jms</address>
        <connector-ref>netty</connector-ref>
        <discovery-group-ref discovery-group-name="dg-group1"/>
           </cluster-connection>
        </cluster-connections>
      
      
        <security-settings>
           <security-setting match="#">
        <permission type="send" roles="guest"/>
        <permission type="consume" roles="guest"/>
        <permission type="createNonDurableQueue" roles="guest"/>
        <permission type="deleteNonDurableQueue" roles="guest"/>
           </security-setting>
        </security-settings>
      
      
        <address-settings>
           <address-setting match="#">
        <dead-letter-address>jms.queue.DLQ</dead-letter-address>
        <expiry-address>jms.queue.ExpiryQueue</expiry-address>
        <redelivery-delay>0</redelivery-delay>
        <max-size-bytes>10485760</max-size-bytes>
        <address-full-policy>BLOCK</address-full-policy>
        <message-counter-history-day-limit>10</message-counter-history-day-limit>
           </address-setting>
        </address-settings>
      
      
        <jms-connection-factories>
           <connection-factory name="InVmConnectionFactory">
        <connectors>
           <connector-ref connector-name="in-vm"/>
        </connectors>
        <entries>
           <entry name="java:/ConnectionFactory"/>
        </entries>
           </connection-factory>
           <connection-factory name="RemoteConnectionFactory">
        <connectors>
           <connector-ref connector-name="netty"/>
        </connectors>
        <entries>
           <entry name="java:jboss/exported/jms/RemoteConnectionFactory"/>
        </entries>
        <ha>true</ha>
        <block-on-acknowledge>true</block-on-acknowledge>
        <retry-interval>2000</retry-interval>
        <retry-interval-multiplier>1</retry-interval-multiplier>
        <reconnect-attempts>-1</reconnect-attempts>
           </connection-factory>
           <pooled-connection-factory name="hornetq-ra">
        <transaction mode="xa"/>
        <connectors>
           <connector-ref connector-name="in-vm"/>
        </connectors>
        <entries>
           <entry name="java:/JmsXA"/>
           <entry name="java:jboss/DefaultJMSConnectionFactory"/>
        </entries>
        <ha>true</ha>
        <block-on-acknowledge>true</block-on-acknowledge>
        <retry-interval>2000</retry-interval>
        <retry-interval-multiplier>1</retry-interval-multiplier>
        <reconnect-attempts>-1</reconnect-attempts>
           </pooled-connection-factory>
        </jms-connection-factories>
      
      
        <jms-destinations>
           <jms-queue name="ExpiryQueue">
        <entry name="java:/jms/queue/ExpiryQueue"/>
           </jms-queue>
           <jms-queue name="DLQ">
        <entry name="java:/jms/queue/DLQ"/>
           </jms-queue>
           <jms-queue name="InvioFattureQueue">
        <entry name="java:/jboss/exported/jms/queue/InvioFattureQueue"/>
        <durable>true</durable>
           </jms-queue>
           <jms-queue name="RicezioneNotificheQueue">
        <entry name="java:/jboss/exported/jms/queue/RicezioneNotificheQueue"/>
        <durable>true</durable>
           </jms-queue>
        </jms-destinations>
          </hornetq-server>
          <hornetq-server name="backup-server">
        <persistence-enabled>true</persistence-enabled>
        <security-enabled>false</security-enabled>
        <cluster-password>password</cluster-password>
        <backup>true</backup>
        <shared-store>false</shared-store>
        <journal-type>ASYNCIO</journal-type>
        <journal-file-size>102400</journal-file-size>
        <journal-min-files>2</journal-min-files>
        <backup-group-name>group1</backup-group-name>
        <paging-directory path="messagingpaging2"/>
        <bindings-directory path="messagingbindings2"/>
        <journal-directory path="messagingjournal2"/>
        <large-messages-directory path="messaginglargemessages2"/>
      
      
        <connectors>
           <netty-connector name="netty" socket-binding="messaging-backup"/>
           <in-vm-connector name="in-vm" server-id="0"/>
        </connectors>
      
      
        <acceptors>
           <netty-acceptor name="netty" socket-binding="messaging-backup"/>
        </acceptors>
      
      
        <broadcast-groups>
           <broadcast-group name="bg-group1">
        <group-address>231.7.7.7</group-address>
        <group-port>9876</group-port>
        <broadcast-period>5000</broadcast-period>
        <connector-ref>
           netty
        </connector-ref>
           </broadcast-group>
        </broadcast-groups>
      
      
        <discovery-groups>
           <discovery-group name="dg-group1">
        <group-address>231.7.7.7</group-address>
        <group-port>9876</group-port>
        <refresh-timeout>10000</refresh-timeout>
           </discovery-group>
        </discovery-groups>
      
      
        <cluster-connections>
           <cluster-connection name="my-cluster">
        <address>jms</address>
        <connector-ref>netty</connector-ref>
        <discovery-group-ref discovery-group-name="dg-group1"/>
           </cluster-connection>
        </cluster-connections>
      </hornetq-server>
      </subsystem>
              <socket-binding name="messaging" port="5445"/>
              <socket-binding name="messaging-backup" port="5545"/>
              <socket-binding name="messaging-throughput" port="5455"/>
      

       

      When I startup both nodes everything seems to work like a charm, i.e. messages are received in a round-robin like fashion by MDB deployed on both nodes.

      If I stop server1 journal is replicated on server2.

      But when I restart server1, it seems like the corresponding hornetq backup instance on server2 stay alive, infact I see many log like the following on both nodes:

      2015-06-10 10:25:43,317 WARN  [org.hornetq.core.client] (hornetq-discovery-group-thread-dg-group1) HQ212034: There are more than one servers on the network broadcasting the same node id. You will see this message exactly once (per node) if a node is restarted, in which case it can be safely ignored. But if it is logged continuously it means you really do have more than one node on the same network active concurrently with the same node id. This could occur if you have a backup node active at the same time as its live node. nodeID=d304f0bc-0eaf-11e5-ab3e-9dee7c4cb5a2
      2015-06-10 10:25:47,486 WARN  [org.hornetq.core.client] (hornetq-discovery-group-thread-dg-group1) HQ212034: There are more than one servers on the network broadcasting the same node id. You will see this message exactly once (per node) if a node is restarted, in which case it can be safely ignored. But if it is logged continuously it means you really do have more than one node on the same network active concurrently with the same node id. This could occur if you have a backup node active at the same time as its live node. nodeID=d304f0bc-0eaf-11e5-ab3e-9dee7c4cb5a2
      2015-06-10 10:25:47,486 WARN  [org.hornetq.core.client] (hornetq-discovery-group-thread-dg-group1) HQ212034: There are more than one servers on the network broadcasting the same node id. You will see this message exactly once (per node) if a node is restarted, in which case it can be safely ignored. But if it is logged continuously it means you really do have more than one node on the same network active concurrently with the same node id. This could occur if you have a backup node active at the same time as its live node. nodeID=d304f0bc-0eaf-11e5-ab3e-9dee7c4cb5a2
      

       

      moreover stopping and restarting server1 and server2 many times result in many data directories been created with sequential numbers like the following:

       

      [root@server1 ~]# ll /opt/standalone/standalone_1/standalone/data/
      totale 272
      drwxr-xr-x 4 wildfly wildfly 4096  9 giu 16:35 content
      drwxr-xr-x 2 wildfly wildfly 4096 10 giu 09:28 messagingbindings
      drwxr-xr-x 2 wildfly wildfly 4096 10 giu 09:36 messagingbindings2
      drwxr-xr-x 2 wildfly wildfly 4096  9 giu 15:55 messagingbindings21
      drwxr-xr-x 2 wildfly wildfly 4096  9 giu 16:58 messagingbindings210
      drwxr-xr-x 2 wildfly wildfly 4096 10 giu 09:24 messagingbindings211
      drwxr-xr-x 2 wildfly wildfly 4096 10 giu 09:28 messagingbindings212
      drwxr-xr-x 2 wildfly wildfly 4096 10 giu 09:33 messagingbindings213
      drwxr-xr-x 2 wildfly wildfly 4096  9 giu 16:28 messagingbindings22
      drwxr-xr-x 2 wildfly wildfly 4096  9 giu 16:31 messagingbindings23
      drwxr-xr-x 2 wildfly wildfly 4096  9 giu 16:31 messagingbindings24
      drwxr-xr-x 2 wildfly wildfly 4096  9 giu 16:33 messagingbindings25
      drwxr-xr-x 2 wildfly wildfly 4096  9 giu 16:37 messagingbindings26
      drwxr-xr-x 2 wildfly wildfly 4096  9 giu 16:41 messagingbindings27
      drwxr-xr-x 2 wildfly wildfly 4096  9 giu 16:47 messagingbindings28
      drwxr-xr-x 2 wildfly wildfly 4096  9 giu 16:56 messagingbindings29
      drwxr-xr-x 2 wildfly wildfly 4096  9 giu 15:56 messagingbindings3
      drwxr-xr-x 2 wildfly wildfly 4096 10 giu 09:37 messagingjournal
      drwxr-xr-x 2 wildfly wildfly 4096  9 giu 15:55 messagingjournal1
      drwxr-xr-x 2 wildfly wildfly 4096 10 giu 09:37 messagingjournal2
      drwxr-xr-x 2 wildfly wildfly 4096  9 giu 15:56 messagingjournal21
      drwxr-xr-x 2 wildfly wildfly 4096 10 giu 09:24 messagingjournal210
      drwxr-xr-x 2 wildfly wildfly 4096 10 giu 09:26 messagingjournal211
      drwxr-xr-x 2 wildfly wildfly 4096 10 giu 09:31 messagingjournal212
      drwxr-xr-x 2 wildfly wildfly 4096 10 giu 09:36 messagingjournal213
      drwxr-xr-x 2 wildfly wildfly 4096  9 giu 16:30 messagingjournal22
      drwxr-xr-x 2 wildfly wildfly 4096  9 giu 16:31 messagingjournal23
      drwxr-xr-x 2 wildfly wildfly 4096  9 giu 16:31 messagingjournal24
      drwxr-xr-x 2 wildfly wildfly 4096  9 giu 16:37 messagingjournal25
      drwxr-xr-x 2 wildfly wildfly 4096  9 giu 16:37 messagingjournal26
      drwxr-xr-x 2 wildfly wildfly 4096  9 giu 16:41 messagingjournal27
      drwxr-xr-x 2 wildfly wildfly 4096  9 giu 16:47 messagingjournal28
      drwxr-xr-x 2 wildfly wildfly 4096  9 giu 16:58 messagingjournal29
      drwxr-xr-x 2 wildfly wildfly 4096  9 giu 16:17 messagingjournal3
      drwxr-xr-x 2 wildfly wildfly 4096  9 giu 16:18 messaginglargemessages
      drwxr-xr-x 2 wildfly wildfly 4096 10 giu 09:36 messaginglargemessages2
      drwxr-xr-x 2 wildfly wildfly 4096  9 giu 15:55 messaginglargemessages21
      drwxr-xr-x 2 wildfly wildfly 4096  9 giu 16:58 messaginglargemessages210
      drwxr-xr-x 2 wildfly wildfly 4096 10 giu 09:24 messaginglargemessages211
      drwxr-xr-x 2 wildfly wildfly 4096 10 giu 09:26 messaginglargemessages212
      drwxr-xr-x 2 wildfly wildfly 4096 10 giu 09:33 messaginglargemessages213
      drwxr-xr-x 2 wildfly wildfly 4096  9 giu 16:28 messaginglargemessages22
      drwxr-xr-x 2 wildfly wildfly 4096  9 giu 16:30 messaginglargemessages23
      drwxr-xr-x 2 wildfly wildfly 4096  9 giu 16:31 messaginglargemessages24
      drwxr-xr-x 2 wildfly wildfly 4096  9 giu 16:33 messaginglargemessages25
      drwxr-xr-x 2 wildfly wildfly 4096  9 giu 16:37 messaginglargemessages26
      drwxr-xr-x 2 wildfly wildfly 4096  9 giu 16:40 messaginglargemessages27
      drwxr-xr-x 2 wildfly wildfly 4096  9 giu 16:46 messaginglargemessages28
      drwxr-xr-x 2 wildfly wildfly 4096  9 giu 16:56 messaginglargemessages29
      drwxr-xr-x 2 wildfly wildfly 4096  9 giu 15:56 messaginglargemessages3
      drwxr-xr-x 8 wildfly wildfly 4096  9 giu 16:31 messagingpaging
      drwxr-xr-x 8 wildfly wildfly 4096 10 giu 09:36 messagingpaging2
      drwxr-xr-x 2 wildfly wildfly 4096  9 giu 15:55 messagingpaging21
      drwxr-xr-x 2 wildfly wildfly 4096  9 giu 16:58 messagingpaging210
      drwxr-xr-x 2 wildfly wildfly 4096 10 giu 09:24 messagingpaging211
      drwxr-xr-x 2 wildfly wildfly 4096 10 giu 09:26 messagingpaging212
      drwxr-xr-x 8 wildfly wildfly 4096 10 giu 09:33 messagingpaging213
      drwxr-xr-x 2 wildfly wildfly 4096  9 giu 16:28 messagingpaging22
      drwxr-xr-x 2 wildfly wildfly 4096  9 giu 16:30 messagingpaging23
      drwxr-xr-x 8 wildfly wildfly 4096  9 giu 16:31 messagingpaging24
      drwxr-xr-x 2 wildfly wildfly 4096  9 giu 16:33 messagingpaging25
      drwxr-xr-x 9 wildfly wildfly 4096  9 giu 16:37 messagingpaging26
      drwxr-xr-x 9 wildfly wildfly 4096  9 giu 16:41 messagingpaging27
      drwxr-xr-x 9 wildfly wildfly 4096  9 giu 16:47 messagingpaging28
      drwxr-xr-x 2 wildfly wildfly 4096  9 giu 16:56 messagingpaging29
      drwxr-xr-x 7 wildfly wildfly 4096  9 giu 15:56 messagingpaging3
      drwxr-xr-x 2 wildfly wildfly 4096  9 giu 15:55 timer-service-data
      drwxr-xr-x 3 wildfly wildfly 4096  9 giu 15:55 tx-object-store
      

       

      So even if everything seems to work, I feel like something wrong is appening with my configuration.

       

      Could someone please help me with this scenario or point me to specific examples where this architecture is showed in detail ?

       

      Thank you

      Giuseppe