Wildfly 10 - Messages distributed by clusters

[paulocsouza1]

Hi!

I am trying to configure wildfly 10 to distribute messages between 2 clusters running as active-active. Can someone help me with the problem? Messages are being consumed by only one cluster. Here are the settings:

            <subsystem xmlns="urn:jboss:domain:messaging-activemq:1.0">
                <server name="jms-master-live">
                    <security enabled="false"/>
                    <cluster password="${jboss.messaging.cluster.password:pwd@2017}"/>
                    <shared-store-master failover-on-server-shutdown="true"/>
                    <bindings-directory path="${path-live}/bindings" relative-to="jms.path.live"/>
                    <journal-directory path="${path-live}/journal" relative-to="jms.path.live"/>
                    <large-messages-directory path="${path-live}/largemessages" relative-to="jms.path.live"/>
                    <paging-directory path="${path-live}/paging" relative-to="jms.path.live"/>
                    <security-setting name="#">
                        <role name="guest" send="true" consume="true" create-non-durable-queue="true" delete-non-durable-queue="true"/>
                    </security-setting>
                    <address-setting name="#" dead-letter-address="jms.queue.DLQ" expiry-address="jms.queue.ExpiryQueue" max-size-bytes="10485760" page-size-bytes="2097152" message-counter-history-day-limit="10" redistribution-delay="0"/>
                    <http-connector name="http-connector" socket-binding="http" endpoint="http-acceptor"/>
                    <http-connector name="http-connector-throughput" socket-binding="http" endpoint="http-acceptor-throughput">
                        <param name="batch-delay" value="50"/>
                    </http-connector>
                    <remote-connector name="netty" socket-binding="messaging">
                        <param name="use-nio" value="true"/>
                        <param name="use-nio-global-worker-pool" value="true"/>
                    </remote-connector>
                    <in-vm-connector name="in-vm" server-id="0"/>
                    <http-acceptor name="http-acceptor" http-listener="default"/>
                    <http-acceptor name="http-acceptor-throughput" http-listener="default">
                        <param name="batch-delay" value="50"/>
                        <param name="direct-deliver" value="false"/>
                    </http-acceptor>
                    <in-vm-acceptor name="in-vm" server-id="0"/>
                    <broadcast-group name="bg-desen-group" connectors="http-connector" socket-binding="messaging-group" broadcast-period="1000"/>
                    <discovery-group name="dg-desen-group" socket-binding="messaging-group-disc" refresh-timeout="500"/>
                    <cluster-connection name="desev-cluster" address="10,DEV,DES" connector-name="http-connector" confirmation-window-size="1" discovery-group="dg-desen-group"/>
                    <jms-queue name="ExpiryQueue" entries="java:/jms/queue/ExpiryQueue"/>
                    <jms-queue name="DLQ" entries="java:/jms/queue/DLQ"/>
                    <jms-queue name="TSupervisorSupervisaoQueue" entries="java:/queue/tSupervisorSupervisaoQueue java:jboss/exported/jms/queue/tSupervisorSupervisaoQueue"/>
                    <jms-queue name="TSupervisorAgenteQueue" entries="java:/queue/tSupervisorAgenteQueue java:jboss/exported/jms/queue/tSupervisorAgenteQueue"/>
                    <jms-queue name="tSupervisorMonitoringScheduledQueue" entries="java:/queue/tSupervisorMonitoringScheduledQueue java:jboss/exported/jms/queue/tSupervisorMonitoringScheduledQueue"/>
                    <connection-factory name="InVmConnectionFactory" entries="java:/ConnectionFactory" connectors="http-connector" consumer-window-size="0"/>
                    <connection-factory name="RemoteConnectionFactory" entries="java:jboss/exported/jms/RemoteConnectionFactory" connectors="http-connector" ha="true" consumer-window-size="0" block-on-acknowledge="true" reconnect-attempts="-1"/>
                    <pooled-connection-factory name="activemq-ra" entries="java:/JmsXA java:jboss/DefaultJMSConnectionFactory" connectors="http-connector" transaction="xa" consumer-window-size="0"/>
                </server>
                <server name="jms-master-backup">
                    <security enabled="false"/>
                    <cluster password="contax@2016"/>
                    <shared-store-slave failover-on-server-shutdown="true"/>
                    <bindings-directory path="${path-bkp}/bindings" relative-to="jms.path.live"/>
                    <journal-directory path="${path-bkp}/journal" relative-to="jms.path.live"/>
                    <large-messages-directory path="${path-bkp}/largemessages" relative-to="jms.path.live"/>
                    <paging-directory path="${path-bkp}/paging" relative-to="jms.path.live"/>
                    <security-setting name="#">
                        <role name="guest" send="true" consume="true" create-durable-queue="true" delete-durable-queue="true" create-non-durable-queue="true" delete-non-durable-queue="true" manage="true"/>
                    </security-setting>
                    <address-setting name="#" dead-letter-address="jms.queue.DLQ" expiry-address="jms.queue.ExpiryQueue" max-size-bytes="10485760" page-size-bytes="2097152" message-counter-history-day-limit="10" redistribution-delay="0"/>
                    <http-connector name="http-connector" socket-binding="http" endpoint="http-acceptor"/>
                    <http-connector name="http-connector-throughput" socket-binding="http" endpoint="http-acceptor-throughput">
                        <param name="batch-delay" value="50"/>
                    </http-connector>
                    <remote-connector name="netty" socket-binding="messaging-backup">
                        <param name="use-nio" value="true"/>
                        <param name="use-nio-global-worker-pool" value="true"/>
                    </remote-connector>
                    <remote-acceptor name="netty" socket-binding="messaging-backup"/>  

                    <broadcast-group name="bg-desen-group" broadcast-period="1000" connectors="netty" socket-binding="messaging-group-bkp"/>
                    <discovery-group name="dg-desen-group" refresh-timeout="500" socket-binding="messaging-group-disc-bkp"/>
                    <cluster-connection name="desev-cluster" address="10,DEV,DES" connector-name="netty" confirmation-window-size="1" discovery-group="dg-desen-group"/>
                </server>
            </subsystem>
...
        <interface name="messenger">
            <any-address/>
        </interface>
...
            <socket-binding name="messaging-group"          interface="messenger" port="5871" multicast-address="${jboss.messaging.group.address:230.0.0.4}" multicast-port="${jboss.messaging.group.port:5971}"/>
            <socket-binding name="messaging-group-disc"     interface="messenger" port="5872" multicast-address="${jboss.messaging.group.address:230.0.0.4}" multicast-port="${jboss.messaging.group.port:5972}"/>
            <socket-binding name="messaging-group-bkp"      interface="messenger" port="5873" multicast-address="${jboss.messaging.group.address:230.0.0.4}" multicast-port="${jboss.messaging.group.port:5973}"/>
            <socket-binding name="messaging-group-disc-bkp" interface="messenger" port="5874" multicast-address="${jboss.messaging.group.address:230.0.0.4}" multicast-port="${jboss.messaging.group.port:5974}"/>
            <socket-binding name="messaging" interface="messenger" port="5446"/>
            <socket-binding name="messaging-backup" interface="messenger" port="5447"/>

[jbertram]

I am trying to configure wildfly 10 to distribute messages between 2 clusters running as active-active.

Can you clarify this a bit?  Are you really attempting to link 2 individual clusters together or are you just running 2 nodes in a single cluster?

Can someone help me with the problem? Messages are being consumed by only one cluster.

Can you elaborate on this?  Typically clients consume messages, not clusters.

[paulocsouza1]

I attempting to link 2 individual clusters. When a say "Messages are being consumed by only one cluster" it means that there is no load balance between the clusters, All messages are being processed by the same cluster that receives them.

[jbertram]

I attempting to link 2 individual clusters.

I don't really understand this.  Conceptually speaking you link individual nodes together into a cluster.  You typically don't talk about linking clusters together.  Please elaborate on your use-case here.

When a say "Messages are being consumed by only one cluster" it means that there is no load balance between the clusters, All messages are being processed by the same cluster that receives them.

Can you elaborate on your consumers and producers (e.g. how many you have, where they are connected, etc.).  The default <message-load-balancing> is ON_DEMAND which means messages will only be moved between nodes if there is proper demand.  You can specify STRICT if you want messages to be strictly distributed round-robin around the cluster.  See the documentation for more details.

[paulocsouza1]

I have two independent servers, not instances inside the wildfly. The requests are distributed by a third server called Alteon.

One of the purposes of these servers is to process a file of 60000 lines on average, each line takes an average of 20 seconds to process. To shorten processing time, each row is inserted into the JMS queue and consumed by MDBs.

I have a similar architecture working in JBoss EAP 6.2 and Active MQ with persistence in the database and when importing the file there is distribution of messages between the servers.
The other use is to include a single message sent by the user, and another user can retrieve that message through any of the servers.

[jbertram]

The requests are distributed by a third server called Alteon.

Is the Alteon server processing JMS message or requests of some other type?  If JMS messages how are they being distributed.  If some other type at what point do JMS messages come into the picture?

Please provide more detail specifically about the JMS pieces of your architecture (e.g. where the MDBs run, how many of them there are, who is producing messages, where the producers live, etc.) or if you could simplify your use-case down to the bare essentials to reproduce the problem that would probably be better. At this point you just haven't provided enough details.

[paulocsouza1]

Alteon server do not process JMS Message, It just does the load balance of HTTP requests, like mod_cluster.

The user logs on to the application that is installed on both servers, which will depend on Alteon's load balance.

The user uploads the file, the application validates the file and saves each row in the queue (1 producer). For each line an MDB is generated, which must be the default. In ActiveMQ (not Artemis) it is 1000. What to pass from this value should be sent to the other server (redistributed).

The configuration to set the amount of consumers is defined in "<connection-factory ... consumer-window-size =" 0 "/>" but as you can see it is zero and according to the documentation the buffer is off and can help With consumers and determine distribution among multiple consumers.

[jbertram]

For each line an MDB is generated, which must be the default.

Do you mean that for each line a message is generated?  Typically MDBs aren't generated automatically.  In fact, I've not ever heard of that before.  Or do you mean that for every line a message is generated and that an MDB consumes the message?  Please clarify.

Also, what do you mean by, "which must be the default."  What default are you talking about?

In ActiveMQ (not Artemis) it is 1000.

What exactly is 1000?  I'm not clear on this point.

What to pass from this value should be sent to the other server (redistributed).

I mentioned this earlier in the thread.  You should set the <message-load-balancing> to STRICT to force messages to be load balanced round-robin around the cluster.

There is also another option.  You can leave the <message-load-balancing> as default (i.e. ON_DEMAND) and tune the consumerWindowSize of the MDBs (e.g. set it to 0).  Every consumer (MDBs included) has an internal buffer of messages it keeps and in certain circumstances this can cause starvation among other consumers.  If you reduce or eliminate the size of this buffer (via consumerWindowSize) then you can solve some starvation issues.

[paulocsouza1]

I changed the message-load-balancing value to STRICT as you requested. I encountered other issues in my setup that prevented shared use:

The cluster configuration was to use UDP but my server was not configured to use this protocol. I changed the jgroups-stack to use TCP.

I was also using IP 0.0.0.0 in the interface settings, Now it is using the <multicast> and <subnet-match ... />.

The problem now that the wildfly is logging the trace below:

2017-05-22 13:16:25,644 INFO  [org.apache.activemq.artemis.core.server] (Thread-9 (ActiveMQ-server-org.apache.activemq.artemis.core.server.impl.ActiveMQServerImpl$2@2595555a-282792407)) AMQ221027: Bridge ClusterConnectionBridge@5bc02572 [name=sf.desev-cluster.b11a846a-3f09-11e7-b6ad-8f3628e6c16f, queue=QueueImpl[name=sf.desev-cluster.b11a846a-3f09-11e7-b6ad-8f3628e6c16f, postOffice=PostOfficeImpl [server=ActiveMQServerImpl::serverUUID=b11a846b-3f09-11e7-b6ad-8f3628e6c16f]]@3b7918d1 targetConnector=ServerLocatorImpl (identity=(Cluster-connection-bridge::ClusterConnectionBridge@5bc02572 [name=sf.desev-cluster.b11a846a-3f09-11e7-b6ad-8f3628e6c16f, queue=QueueImpl[name=sf.desev-cluster.b11a846a-3f09-11e7-b6ad-8f3628e6c16f, postOffice=PostOfficeImpl [server=ActiveMQServerImpl::serverUUID=b11a846b-3f09-11e7-b6ad-8f3628e6c16f]]@3b7918d1 targetConnector=ServerLocatorImpl [initialConnectors=[TransportConfiguration(name=in-vm, factory=org-apache-activemq-artemis-core-remoting-impl-invm-InVMConnectorFactory) ?serverId=0], discoveryGroupConfiguration=null]]::ClusterConnectionImpl@1818062455[nodeUUID=b11a846b-3f09-11e7-b6ad-8f3628e6c16f, connector=TransportConfiguration(name=http-connector, factory=org-apache-activemq-artemis-core-remoting-impl-netty-NettyConnectorFactory) ?httpUpgradeEnabled=true&httpPpgradeEndpoint=http-acceptor&port=8480&host=10-201-36-103, address=10,dev,des,jms, server=ActiveMQServerImpl::serverUUID=b11a846b-3f09-11e7-b6ad-8f3628e6c16f])) [initialConnectors=[TransportConfiguration(name=in-vm, factory=org-apache-activemq-artemis-core-remoting-impl-invm-InVMConnectorFactory) ?serverId=0], discoveryGroupConfiguration=null]] is connected
2017-05-22 13:16:52,695 ERROR [org.infinispan.CLUSTER] (transport-thread--p16-t3) ISPN000196: Failed to recover cluster state after the current node became the coordinator (or after merge): java.util.concurrent.ExecutionException: org.infinispan.remoting.transport.jgroups.SuspectException: Cache not running on node desen-server-domain-widfly:10.171.193.102-jms-live
  at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:357)
  at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:1915)
  at org.infinispan.util.concurrent.CompletableFutures.await(CompletableFutures.java:100)
  at org.infinispan.topology.ClusterTopologyManagerImpl.executeOnClusterSync(ClusterTopologyManagerImpl.java:567)
  at org.infinispan.topology.ClusterTopologyManagerImpl.recoverClusterStatus(ClusterTopologyManagerImpl.java:437)
  at org.infinispan.topology.ClusterTopologyManagerImpl.handleClusterView(ClusterTopologyManagerImpl.java:358)
  at org.infinispan.topology.ClusterTopologyManagerImpl$ClusterViewListener.lambda$handleViewChange$0(ClusterTopologyManagerImpl.java:692)
  at java.util.concurrent.FutureTask.run(FutureTask.java:266)
  at org.infinispan.executors.SemaphoreCompletionService$QueueingTask.runInternal(SemaphoreCompletionService.java:172)
  at org.infinispan.executors.SemaphoreCompletionService$QueueingTask.run(SemaphoreCompletionService.java:151)
  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  at org.jboss.as.clustering.infinispan.ClassLoaderThreadFactory.lambda$newThread$12(ClassLoaderThreadFactory.java:48)
  at java.lang.Thread.run(Thread.java:748)
Caused by: org.infinispan.remoting.transport.jgroups.SuspectException: Cache not running on node desen-server-domain-widfly:10.171.193.102-jms-live
  at org.infinispan.remoting.transport.AbstractTransport.checkResponse(AbstractTransport.java:46)
  at org.infinispan.remoting.transport.jgroups.JGroupsTransport.checkRsp(JGroupsTransport.java:795)
  at org.infinispan.remoting.transport.jgroups.JGroupsTransport.lambda$invokeRemotelyAsync$1(JGroupsTransport.java:642)
  at java.util.concurrent.CompletableFuture.uniApply(CompletableFuture.java:602)
  at java.util.concurrent.CompletableFuture$UniApply.tryFire(CompletableFuture.java:577)
  at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:474)
  at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:1962)
  at org.infinispan.remoting.transport.jgroups.RspListFuture.futureDone(RspListFuture.java:31)
  at org.jgroups.blocks.Request.checkCompletion(Request.java:152)
  at org.jgroups.blocks.GroupRequest.receiveResponse(GroupRequest.java:116)
  at org.jgroups.blocks.RequestCorrelator.dispatch(RequestCorrelator.java:427)
  at org.jgroups.blocks.RequestCorrelator.receiveMessage(RequestCorrelator.java:357)
  at org.jgroups.blocks.RequestCorrelator.receive(RequestCorrelator.java:245)
  at org.jgroups.blocks.MessageDispatcher$ProtocolAdapter.up(MessageDispatcher.java:664)
  at org.jgroups.JChannel.up(JChannel.java:738)
  at org.jgroups.fork.ForkProtocolStack.up(ForkProtocolStack.java:120)
  at org.jgroups.stack.Protocol.up(Protocol.java:380)
  at org.jgroups.protocols.FORK.up(FORK.java:114)
  at org.jgroups.protocols.FRAG2.up(FRAG2.java:165)
  at org.jgroups.protocols.FlowControl.up(FlowControl.java:390)
  at org.jgroups.protocols.pbcast.GMS.up(GMS.java:1040)
  at org.jgroups.protocols.pbcast.STABLE.up(STABLE.java:234)
  at org.jgroups.protocols.UNICAST3.deliverMessage(UNICAST3.java:1070)
  at org.jgroups.protocols.UNICAST3.handleDataReceived(UNICAST3.java:785)
  at org.jgroups.protocols.UNICAST3.up(UNICAST3.java:426)
  at org.jgroups.protocols.pbcast.NAKACK2.up(NAKACK2.java:649)
  at org.jgroups.protocols.VERIFY_SUSPECT.up(VERIFY_SUSPECT.java:155)
  at org.jgroups.protocols.FD.up(FD.java:260)
  at org.jgroups.protocols.FD_SOCK.up(FD_SOCK.java:310)
  at org.jgroups.protocols.MERGE3.up(MERGE3.java:285)
  at org.jgroups.protocols.Discovery.up(Discovery.java:296)
  at org.jgroups.protocols.MPING.up(MPING.java:178)
  at org.jgroups.protocols.TP.passMessageUp(TP.java:1601)
  at org.jgroups.protocols.TP$SingleMessageHandler.run(TP.java:1817)
  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  at org.jboss.as.clustering.jgroups.ClassLoaderThreadFactory.lambda$newThread$4(ClassLoaderThreadFactory.java:52)
  ... 1 more

[jbertram]

As far as I can tell this latest problem completely unrelated to Artemis.  It looks like a problem with Infinispan.  I recommend you create a new forum thread to address this issue.

[paulocsouza1]

OK! Thank you!