1 Reply Latest reply on Nov 7, 2017 1:34 AM by mnovak

External ArtemisMQ 2.3 cluster doesn't work

fioacnoaesn Nov 6, 2017 7:41 AM

Ubuntu 15 Desktop, java 8

Symmetric cluster

External ArtemisMQ 1.5.4 cluster (2 nodes with 1.5.4) works except redistribution functionality (redistribution-delay 0)

External ArtemisMQ 2.3 cluster (2 nodes with 2.3.0) doesn't work. It sends messages to another nodes but another node denies to accept them (stack trace below).

14:36:53,595 DEBUG [org.apache.activemq.artemis.core.postoffice.impl.PostOfficeImpl] Couldn't find any bindings for address=jms.queue.NewItem on message=CoreMessage[messageID=491,durable=true,userID=f75cff43-c292-11e7-bfb0-ef1bf8dd85f9,priority=4, timestamp=Mon Nov 04 14:36:52 MSK 2017,expiration=0, durable=true, address=jms.queue.NewItem,properties=TypedProperties[__AMQ_CID=f6d4955f-c292-11e7-bfb0-ef1bf8dd85f9,type=IMAGE,counter=1,_AMQ_ROUTE_TO=[0000 0000 0000 000C),bytesAsLongs(12],url=http://icons.iconarchive.com/icons/seanau/email/256/Clear-icon.png]]@247622711

14:36:53,595 DEBUG [org.apache.activemq.artemis.core.postoffice.impl.PostOfficeImpl] Message CoreMessage[messageID=491,durable=true,userID=f75cff43-c292-11e7-bfb0-ef1bf8dd85f9,priority=4, timestamp=Mon Nov 04 14:36:52 MSK 2017,expiration=0, durable=true,
address=jms.queue.NewItem,properties=TypedProperties[__AMQ_CID=f6d4955f-c292-11e7-bfb0-ef1bf8dd85f9,type=IMAGE,counter=1,_AMQ_ROUTE_TO=[0000 0000 0000 000C),bytesAsLongs(12],url=http://icons.iconarchive.com/icons/seanau/email/256/Clear-icon.png]]@247622711 is not going anywhere as it didn't have a binding on address:jms.queue.NewItem

broker.xml for ver.2.3:

<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<configuration xmlns="urn:activemq"
               xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
               xsi:schemaLocation="urn:activemq /schema/artemis-configuration.xsd">

   <core xmlns="urn:activemq:core" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="urn:activemq:core ">
      <name>192.168.0.85</name>
      <persistence-enabled>true</persistence-enabled>
      <journal-type>NIO</journal-type>
      <paging-directory>./data/paging</paging-directory>
      <bindings-directory>./data/bindings</bindings-directory>
      <journal-directory>./data/journal</journal-directory>
      <large-messages-directory>./data/large-messages</large-messages-directory>
      <journal-datasync>true</journal-datasync>
      <journal-min-files>2</journal-min-files>
      <journal-pool-files>-1</journal-pool-files>
      <journal-buffer-timeout>5896000</journal-buffer-timeout>

      <!-- how often we are looking for how many bytes are being used on the disk in ms -->
      <disk-scan-period>5000</disk-scan-period>

      <!-- once the disk hits this limit the system will block, or close the connection in certain protocols
           that won't support flow control. -->
      <max-disk-usage>90</max-disk-usage>
      <!-- should the broker detect dead locks and other issues -->
      <critical-analyzer>true</critical-analyzer>
      <critical-analyzer-timeout>120000</critical-analyzer-timeout>
      <critical-analyzer-check-period>60000</critical-analyzer-check-period>
      <critical-analyzer-policy>HALT</critical-analyzer-policy>

      <!-- Connectors -->
      <connectors>
         <connector name="netty-connector">tcp://192.168.0.85:61616</connector>
         <!-- connector to the server86 -->
         <!-- <connector name="server86-connector">tcp://192.168.0.86:61616</connector> -->
      </connectors>
      <!-- Acceptors -->
      <acceptors>
         <acceptor name="netty-acceptor">tcp://192.168.0.85:61616</acceptor>
      </acceptors>
      <!-- Clustering configuration -->
      <broadcast-groups>
         <broadcast-group name="my-broadcast-group">
            <group-address>${udp-address:231.7.7.7}</group-address>
            <group-port>9876</group-port>
            <broadcast-period>100</broadcast-period>
            <connector-ref>netty-connector</connector-ref>
         </broadcast-group>
      </broadcast-groups>
      <discovery-groups>
         <discovery-group name="my-discovery-group">
            <group-address>${udp-address:231.7.7.7}</group-address>
            <group-port>9876</group-port>
            <refresh-timeout>10000</refresh-timeout>
         </discovery-group>
      </discovery-groups>
      <!-- Clustering configuration -->
      <cluster-connections>
         <cluster-connection name="my-cluster">
            <connector-ref>netty-connector</connector-ref>
            <retry-interval>500</retry-interval>
            <use-duplicate-detection>true</use-duplicate-detection>
            <!-- ON_DEMAND STRICT -->
            <message-load-balancing>STRICT</message-load-balancing>
            <max-hops>1</max-hops>
            <discovery-group-ref discovery-group-name="my-discovery-group"/>
            <!-- <static-connectors>
               <connector-ref>server86-connector</connector-ref>
            </static-connectors> -->
         </cluster-connection>
      </cluster-connections>
      <!-- Other config -->
      <security-settings>
         <!--security for example topic-->
         <security-setting match="exampleTopic">
            <permission roles="amq" type="createDurableQueue"/>
            <permission roles="amq" type="deleteDurableQueue"/>
            <permission roles="amq" type="createAddress"/>
            <permission roles="amq" type="deleteAddress"/>
            <permission roles="amq" type="createNonDurableQueue"/>
            <permission roles="amq" type="deleteNonDurableQueue"/>
            <permission roles="amq" type="consume"/>
            <permission roles="amq" type="send"/>
            <permission type="browse" roles="amq"/>
            <permission type="manage" roles="amq"/>
         </security-setting>
         <security-setting match="#">
            <permission roles="amq" type="createDurableQueue"/>
            <permission roles="amq" type="deleteDurableQueue"/>
            <permission roles="amq" type="createNonDurableQueue"/>
            <permission roles="amq" type="deleteNonDurableQueue"/>
            <permission roles="amq" type="consume"/>
            <permission roles="amq" type="send"/>
            <permission type="browse" roles="amq"/>
            <permission type="manage" roles="amq"/>
         </security-setting>
      </security-settings>
      <address-settings>
         <!-- if you define auto-create on certain queues, management has to be auto-create -->
         <address-setting match="activemq.management#">
            <dead-letter-address>DLQ</dead-letter-address>
            <expiry-address>ExpiryQueue</expiry-address>
            <redelivery-delay>0</redelivery-delay>
            <!-- with -1 only the global-max-size is in use for limiting -->
            <max-size-bytes>-1</max-size-bytes>
            <message-counter-history-day-limit>10</message-counter-history-day-limit>
            <address-full-policy>PAGE</address-full-policy>
            <auto-create-queues>true</auto-create-queues>
            <auto-create-addresses>true</auto-create-addresses>
            <auto-create-jms-queues>true</auto-create-jms-queues>
            <auto-create-jms-topics>true</auto-create-jms-topics>
         </address-setting>
         <!--default for catch all-->
         <address-setting match="#">
            <dead-letter-address>DLQ</dead-letter-address>
            <expiry-address>ExpiryQueue</expiry-address>
            <redelivery-delay>0</redelivery-delay>
            <!-- with -1 only the global-max-size is in use for limiting -->
            <max-size-bytes>-1</max-size-bytes>
            <message-counter-history-day-limit>10</message-counter-history-day-limit>
            <address-full-policy>PAGE</address-full-policy>
            <auto-create-queues>true</auto-create-queues>
            <auto-create-addresses>true</auto-create-addresses>
            <auto-create-jms-queues>true</auto-create-jms-queues>
            <auto-create-jms-topics>true</auto-create-jms-topics>
            <redistribution-delay>0</redistribution-delay>
         </address-setting>
      </address-settings>
      <addresses>
         <address name="DLQ">
            <anycast>
               <queue name="DLQ" />
            </anycast>
         </address>
         <address name="ExpiryQueue">
            <anycast>
               <queue name="ExpiryQueue" />
            </anycast>
         </address>
         <address name="NewItem">
            <anycast>
               <queue name="NewItem"/>
            </anycast>
         </address>
         <address name="DeleteItem">
            <anycast>
               <queue name="DeleteItem"/>
            </anycast>
         </address>
         <address name="InaccessibleItem">
            <anycast>
               <queue name="InaccessibleItem"/>
            </anycast>
         </address>
         <!-- <address name="jms.queue.NewItem">
            <anycast>
               <queue name="NewItem"/>
            </anycast>
         </address>
         <address name="jms.queue.DeleteItem">
            <anycast>
               <queue name="DeleteItem"/>
            </anycast>
         </address>
         <address name="jms.queue.InaccessibleItem">
            <anycast>
               <queue name="InaccessibleItem"/>
            </anycast>
         </address> -->
      </addresses>
   </core>
</configuration>

1. Re: External ArtemisMQ 2.3 cluster doesn't work

mnovak Nov 7, 2017 1:34 AM (in response to fioacnoaesn)

Hi,

the reason why redistribution does not work in Artemis 1.5.4 is in <cluster-connection ...>:

...
<message-load-balancing>STRICT</message-load-balancing>
...

Option STRICT disables redistribution. Redistribution will work only with ON_DEMAND. It's somewhat ambiguous configuration of Artemis that it mixes 2 different things. Server side message load balancing (what happens when message is send to artemis cluster) and message redistribution (what to do when I have a message in queue but no local consumer).

I'm not sure what might be wrong with Artemis 2.3 that you're getting this exception. Is it possible that you've re-used journal directory from Artemis 1.5.4 and ran it with Artemis 2.3? In this case it's possible that there are addresses of queues set messages which no longer exists. At least it looks like so. I did a try with following config on 2 node cluster and it works for me:

<configuration xmlns="urn:activemq"
               xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
               xsi:schemaLocation="urn:activemq /schema/artemis-configuration.xsd">

   <core xmlns="urn:activemq:core" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="urn:activemq:core ">

      <name>0.0.0.0</name>

      <persistence-enabled>true</persistence-enabled>

      <journal-type>ASYNCIO</journal-type>

      <paging-directory>./data/paging</paging-directory>

      <bindings-directory>./data/bindings</bindings-directory>

      <journal-directory>./data/journal</journal-directory>

      <large-messages-directory>./data/large-messages</large-messages-directory>

      <journal-datasync>true</journal-datasync>

      <journal-min-files>2</journal-min-files>

      <journal-pool-files>-1</journal-pool-files>

      <journal-buffer-timeout>1412000</journal-buffer-timeout>

      <!-- how often we are looking for how many bytes are being used on the disk in ms -->
      <disk-scan-period>5000</disk-scan-period>

      <!-- once the disk hits this limit the system will block, or close the connection in certain protocols
           that won't support flow control. -->
      <max-disk-usage>90</max-disk-usage>

      <!-- should the broker detect dead locks and other issues -->
      <critical-analyzer>true</critical-analyzer>

      <critical-analyzer-timeout>120000</critical-analyzer-timeout>

      <critical-analyzer-check-period>60000</critical-analyzer-check-period>

      <critical-analyzer-policy>HALT</critical-analyzer-policy>
      <acceptors>

         <!-- useEpoll means: it will use Netty epoll if you are on a system (Linux) that supports it -->
         <!-- amqpCredits: The number of credits sent to AMQP producers -->
         <!-- amqpLowCredits: The server will send the # credits specified at amqpCredits at this low mark -->

         <!-- Acceptor for every supported protocol -->
         <acceptor name="artemis">tcp://0.0.0.0:61616?tcpSendBufferSize=1048576;tcpReceiveBufferSize=1048576;protocols=CORE,AMQP,STOMP,HORNETQ,MQTT,OPENWIRE;useEpoll=true;amqpCredits=1000;amqpLowCredits=300</acceptor>

         <!-- AMQP Acceptor.  Listens on default AMQP port for AMQP traffic.-->
         <acceptor name="amqp">tcp://0.0.0.0:5672?tcpSendBufferSize=1048576;tcpReceiveBufferSize=1048576;protocols=AMQP;useEpoll=true;amqpCredits=1000;amqpMinCredits=300</acceptor>

         <!-- STOMP Acceptor. -->
         <acceptor name="stomp">tcp://0.0.0.0:61613?tcpSendBufferSize=1048576;tcpReceiveBufferSize=1048576;protocols=STOMP;useEpoll=true</acceptor>

         <!-- HornetQ Compatibility Acceptor.  Enables HornetQ Core and STOMP for legacy HornetQ clients. -->
         <acceptor name="hornetq">tcp://0.0.0.0:5445?protocols=HORNETQ,STOMP;useEpoll=true</acceptor>

         <!-- MQTT Acceptor -->
         <acceptor name="mqtt">tcp://0.0.0.0:1883?tcpSendBufferSize=1048576;tcpReceiveBufferSize=1048576;protocols=MQTT;useEpoll=true</acceptor>

      </acceptors>

<connectors>
   <connector name="netty">tcp://localhost:61616</connector>
</connectors>

<broadcast-groups>
   <broadcast-group name="my-broadcast-group">
      <group-address>231.7.7.7</group-address>
      <group-port>9876</group-port>
      <broadcast-period>2000</broadcast-period>
      <connector-ref>netty</connector-ref>
   </broadcast-group>
</broadcast-groups>
<discovery-groups>
   <discovery-group name="my-discovery-group">
      <group-address>231.7.7.7</group-address>
      <group-port>9876</group-port>
      <refresh-timeout>10000</refresh-timeout>
   </discovery-group>
</discovery-groups>
<cluster-connections>
   <cluster-connection name="my-cluster">

      <connector-ref>netty</connector-ref>
      <check-period>1000</check-period>
      <connection-ttl>5000</connection-ttl>
      <initial-connect-attempts>-1</initial-connect-attempts>
      <reconnect-attempts>-1</reconnect-attempts>
      <use-duplicate-detection>true</use-duplicate-detection>
      <message-load-balancing>ON_DEMAND</message-load-balancing>
      <max-hops>1</max-hops>
      <discovery-group-ref discovery-group-name="my-discovery-group"/>
   </cluster-connection>
</cluster-connections>

      <security-settings>
         <security-setting match="#">
            <permission type="createNonDurableQueue" roles="amq"/>
            <permission type="deleteNonDurableQueue" roles="amq"/>
            <permission type="createDurableQueue" roles="amq"/>
            <permission type="deleteDurableQueue" roles="amq"/>
            <permission type="createAddress" roles="amq"/>
            <permission type="deleteAddress" roles="amq"/>
            <permission type="consume" roles="amq"/>
            <permission type="browse" roles="amq"/>
            <permission type="send" roles="amq"/>
            <!-- we need this otherwise ./artemis data imp wouldn't work -->
            <permission type="manage" roles="amq"/>
         </security-setting>
      </security-settings>

      <address-settings>
         <!-- if you define auto-create on certain queues, management has to be auto-create -->
         <address-setting match="activemq.management#">
            <dead-letter-address>DLQ</dead-letter-address>
            <expiry-address>ExpiryQueue</expiry-address>
            <redelivery-delay>0</redelivery-delay>
            <!-- with -1 only the global-max-size is in use for limiting -->
            <max-size-bytes>-1</max-size-bytes>
            <message-counter-history-day-limit>10</message-counter-history-day-limit>
            <address-full-policy>PAGE</address-full-policy>
            <auto-create-queues>true</auto-create-queues>
            <auto-create-addresses>true</auto-create-addresses>
            <auto-create-jms-queues>true</auto-create-jms-queues>
            <auto-create-jms-topics>true</auto-create-jms-topics>
         </address-setting>
         <!--default for catch all-->
         <address-setting match="#">
            <dead-letter-address>DLQ</dead-letter-address>
            <expiry-address>ExpiryQueue</expiry-address>
            <redelivery-delay>0</redelivery-delay>
            <!-- with -1 only the global-max-size is in use for limiting -->
            <max-size-bytes>-1</max-size-bytes>
            <message-counter-history-day-limit>10</message-counter-history-day-limit>
            <address-full-policy>PAGE</address-full-policy>
            <auto-create-queues>true</auto-create-queues>
            <auto-create-addresses>true</auto-create-addresses>
            <auto-create-jms-queues>true</auto-create-jms-queues>
            <auto-create-jms-topics>true</auto-create-jms-topics>
         </address-setting>
      </address-settings>

      <addresses>
         <address name="DLQ">
            <anycast>
               <queue name="DLQ" />
            </anycast>
         </address>
         <address name="ExpiryQueue">
            <anycast>
               <queue name="ExpiryQueue" />
            </anycast>
         </address>
    <address name="InQueue">
            <anycast>
               <queue name="InQueue" />
            </anycast>
         </address>
      </addresses>
   </core>
</configuration>