0 Replies Latest reply on May 8, 2018 1:34 PM by Vamshi Krishna

    Messaging-ActiveMQ Artemis HA (Not working)

    Vamshi Krishna Newbie

      Good Afternoon !

       

      I am configuring messaging-activemq artemis to support high availability using shared-store(live-backup pair) use-case scenario out of the two options available (Data replication and shared-store).

       

      Here is my configuration for messaging default server and backup server.

       

                  <subsystem xmlns="urn:jboss:domain:messaging-activemq:2.0">

                      <server name="default">

                          <cluster user="admin" password="activemq"/>

                          <shared-store-master failover-on-server-shutdown="true"/>

                          <bindings-directory path="D:\mnt\activemq\sharefs\group-${live-group}\bindings"/>

                          <journal-directory path="D:\mnt\activemq\sharefs\group-${live-group}\journal"/>

                          <large-messages-directory path="D:\mnt\activemq\sharefs\group-${live-group}\largemessages"/>

                          <paging-directory path="D:\mnt\activemq\sharefs\group-${live-group}\paging"/>

                          <security-setting name="#">

                              <role name="guest" send="true" consume="true" create-non-durable-queue="true" delete-non-durable-queue="true"/>

                          </security-setting>

                          <address-setting name="#" dead-letter-address="jms.queue.DLQ" expiry-address="jms.queue.ExpiryQueue" max-size-bytes="10485760" page-size-bytes="2097152" message-counter-history-day-limit="10" redistribution-delay="1000"/>

                          <http-connector name="http-connector" socket-binding="http" endpoint="http-acceptor"/>

                          <http-connector name="http-connector-throughput" socket-binding="http" endpoint="http-acceptor-throughput">

                              <param name="batch-delay" value="50"/>

                          </http-connector>

                          <remote-connector name="netty" socket-binding="messaging">

                              <param name="use-nio" value="true"/>

                              <param name="use-nio-global-worker-pool" value="true"/>

                          </remote-connector>

                          <in-vm-connector name="in-vm" server-id="0">

                              <param name="buffer-pooling" value="false"/>

                          </in-vm-connector>

                          <http-acceptor name="http-acceptor" http-listener="default"/>

                          <http-acceptor name="http-acceptor-throughput" http-listener="default">

                              <param name="batch-delay" value="50"/>

                              <param name="direct-deliver" value="false"/>

                          </http-acceptor>

                          <remote-acceptor name="netty" socket-binding="messaging">

                              <param name="use-nio" value="true"/>

                          </remote-acceptor>

                          <in-vm-acceptor name="in-vm" server-id="0">

                              <param name="buffer-pooling" value="false"/>

                          </in-vm-acceptor>

                          <broadcast-group name="bg-group1" jgroups-channel="activemq-cluster" connectors="netty"/>

                          <discovery-group name="dg-group1" jgroups-channel="activemq-cluster"/>

                          <cluster-connection name="amq-cluster" address="jms" connector-name="netty" discovery-group="dg-group1"/>

                          <jms-queue name="ExpiryQueue" entries="java:/jms/queue/ExpiryQueue"/>

                          <jms-queue name="DLQ" entries="java:/jms/queue/DLQ"/>

                          <jms-queue name="testQueue2" entries="java:/jms/queue/testQueue2 java:/jboss/exported/jms/queue/testQueue2"/>

                          <connection-factory name="InVmConnectionFactory" entries="java:/ConnectionFactory" connectors="in-vm"/>

                          <connection-factory name="RemoteConnectionFactory" entries="java:jboss/exported/jms/RemoteConnectionFactory" connectors="netty" ha="true" block-on-acknowledge="true" reconnect-attempts="-1"/>

                          <pooled-connection-factory name="activemq-ra" entries="java:/JmsXA java:jboss/DefaultJMSConnectionFactory" connectors="in-vm" transaction="xa"/>

                      </server>

       

                      <server name="backup">

                          <cluster user="admin" password="password"/>

                          <shared-store-slave allow-failback="true" failover-on-server-shutdown="true" restart-backup="false"/>

                          <bindings-directory path="D:\mnt\activemq\sharefs\group-${backup-group}\bindings"/>

                          <journal-directory path="D:\mnt\activemq\sharefs\group-${backup-group}\journal"/>

                          <large-messages-directory path="D:\mnt\activemq\sharefs\group-${backup-group}\largemessages"/>

                          <paging-directory path="D:\mnt\activemq\sharefs\group-${backup-group}\paging"/>

                          <remote-connector name="netty-backup" socket-binding="messaging-backup">

                              <param name="use-nio" value="true"/>

                          </remote-connector>

                          <remote-acceptor name="netty-backup" socket-binding="messaging-backup">

                              <param name="use-nio" value="true"/>

                          </remote-acceptor>

                          <broadcast-group name="bg-group-backup" jgroups-channel="activemq-cluster" broadcast-period="1000" connectors="netty-backup"/>

                          <discovery-group name="dg-group-backup" jgroups-channel="activemqcluster" refresh-timeout="1000"/>

                          <cluster-connection name="amq-cluster" address="jms" connector-name=" netty-backup" discovery-group="dg-group-backup"/>

                      </server>

       

      Here is my configuration for the live-backup pair from my host.xml

       

      Server1:(LIVE)

           <system-properties>

                <property name="live-group" value="1"/>

                <property name="backup-group" value="2"/>

           </system-properties>

       

      Server2: (BACKUP)

           <system-properties>

                <property name="live-group" value="2"/>

                <property name="backup-group" value="1"/>

           </system-properties>

       

      Socket-bindings:

       

              <socket-binding-group name="full-ha-sockets" default-interface="public">

                  <socket-binding name="ajp" port="${jboss.ajp.port:8009}"/>

                  <socket-binding name="http" port="${jboss.http.port:8080}"/>

                  <socket-binding name="https" port="${jboss.https.port:8443}"/>

                  <socket-binding name="iiop" interface="unsecure" port="3528"/>

                  <socket-binding name="iiop-ssl" interface="unsecure" port="3529"/>

                  <socket-binding name="jgroups-mping" interface="private" port="0" multicast-address="${jboss.default.multicast.address:230.0.0.4}" multicast-port="45700"/>

                  <socket-binding name="jgroups-tcp" interface="private" port="7600"/>

                  <socket-binding name="jgroups-udp" interface="private" port="55200" multicast-address="${jboss.default.multicast.address:230.0.0.4}" multicast-port="45688"/>

                  <socket-binding name="modcluster" port="0" multicast-address="${jboss.modcluster.multicast.address:224.0.1.105}" multicast-port="23364"/>

                  <socket-binding name="txn-recovery-environment" port="4712"/>

                  <socket-binding name="txn-status-manager" port="4713"/>

                  <socket-binding name="messaging" port="7500"/>

                  <socket-binding name="messaging-backup" port="7501"/>

                  <outbound-socket-binding name="mail-smtp">

                      <remote-destination host="localhost" port="25"/>

                  </outbound-socket-binding>

              </socket-binding-group>

       

      When I start the JVMs on both server1(master) and server2(slave), server1 starts and server2 loads most of the server and then waits indefinitely and times out after 30,000 ms. Also I get error on Server1(master) for initialising the backup server

       

       

      2018-05-08 11:52:47,456 ERROR [org.apache.activemq.artemis.core.server] (AMQ119000: Activation for server ActiveMQServerImpl::serverUUID=5198bf0d-4fe5-11e8-a905-005056a9098d) AMQ224000: Failure in initialisation: java.lang.NullPointerException

      at org.apache.activemq.artemis.core.server.impl.SharedStoreBackupActivation$FailbackChecker.<init>(SharedStoreBackupActivation.java:193)

      at org.apache.activemq.artemis.core.server.impl.SharedStoreBackupActivation.startFailbackChecker(SharedStoreBackupActivation.java:185)

      at org.apache.activemq.artemis.core.server.impl.SharedStoreBackupActivation.run(SharedStoreBackupActivation.java:118)

      at org.apache.activemq.artemis.core.server.impl.ActiveMQServerImpl$ActivationThread.run(ActiveMQServerImpl.java:2496)

       

       

      Server2:

      2018-05-08 11:54:57,959 INFO  [org.apache.activemq.artemis.core.server] (Thread-1 (ActiveMQ-server-org.apache.activemq.artemis.core.server.impl.ActiveMQServerImpl$3@ce3606a)) AMQ221031: backup announced

      2018-05-08 11:59:52,885 ERROR [org.jboss.as.controller.management-operation] (Controller Boot Thread) WFLYCTL0348: Timeout after [300] seconds waiting for service container stability. Operation will roll back. Step that first updated the service container was 'add' at address '[("interface" => "unsecure")]'

      2018-05-08 11:59:53,016 INFO  [org.jboss.as.server.deployment] (MSC service thread 1-4) WFLYSRV0028: Stopped deployment DemoJSM.war (runtime-name: DemoJSM.war) in 125ms

      2018-05-08 11:59:53,026 INFO  [org.jboss.as.server.deployment] (MSC service thread 1-2) WFLYSRV0028: Stopped deployment jboss-as-helloworld.war (runtime-name: jboss-as-helloworld.war) in 135ms

      2018-05-08 11:59:53,036 INFO  [org.wildfly.extension.undertow] (MSC service thread 1-4) WFLYUT0008: Undertow HTTPS listener https suspending

      2018-05-08 11:59:53,037 INFO  [org.wildfly.extension.undertow] (MSC service thread 1-4) WFLYUT0007: Undertow HTTPS listener https stopped, was bound to 127.0.0.2:8643

      2018-05-08 11:59:53,102 INFO  [org.infinispan.remoting.transport.jgroups.JGroupsTransport] (MSC service thread 1-4) ISPN000080: Disconnecting JGroups channel ee

      2018-05-08 11:59:53,102 INFO  [org.infinispan.remoting.transport.jgroups.JGroupsTransport] (MSC service thread 1-4) ISPN000082: Stopping the RpcDispatcher for channel ee

      2018-05-08 11:59:53,106 INFO  [org.infinispan.remoting.transport.jgroups.JGroupsTransport] (MSC service thread 1-4) ISPN000080: Disconnecting JGroups channel ee

      2018-05-08 11:59:53,106 INFO  [org.infinispan.remoting.transport.jgroups.JGroupsTransport] (MSC service thread 1-4) ISPN000082: Stopping the RpcDispatcher for channel ee

      2018-05-08 11:59:53,110 INFO  [org.infinispan.remoting.transport.jgroups.JGroupsTransport] (MSC service thread 1-4) ISPN000080: Disconnecting JGroups channel ee

      2018-05-08 11:59:53,110 INFO  [org.infinispan.remoting.transport.jgroups.JGroupsTransport] (MSC service thread 1-4) ISPN000082: Stopping the RpcDispatcher for channel ee

      2018-05-08 11:59:53,174 INFO  [org.jboss.as.clustering.infinispan] (ServerService Thread Pool -- 78) WFLYCLINF0003: Stopped client-mappings cache from ejb container

      2018-05-08 11:59:53,195 INFO  [org.infinispan.remoting.transport.jgroups.JGroupsTransport] (MSC service thread 1-1) ISPN000080: Disconnecting JGroups channel ee

      2018-05-08 11:59:53,195 INFO  [org.infinispan.remoting.transport.jgroups.JGroupsTransport] (MSC service thread 1-1) ISPN000082: Stopping the RpcDispatcher for channel ee

      2018-05-08 11:59:53,208 INFO  [org.apache.activemq.artemis.core.server] (ServerService Thread Pool -- 77) AMQ221002: Apache ActiveMQ Artemis Message Broker version 1.5.5.008-redhat-1 [4ed050b9-4fe5-11e8-94e8-005056a9098d] stopped, uptime 4 minutes

      2018-05-08 11:59:58,095 ERROR [org.jboss.as.controller.management-operation] (Controller Boot Thread) WFLYCTL0190: Step handler org.jboss.as.controller.AbstractControllerService$ModelControllerServiceInitializationBootStepHandler$1@78e9d45 for operation boottime-controller-initializer-step at address [] failed handling operation rollback -- java.util.concurrent.TimeoutException: java.util.concurrent.TimeoutException

      at org.jboss.as.controller.OperationContextImpl.waitForRemovals(OperationContextImpl.java:522)

      at org.jboss.as.controller.AbstractOperationContext$Step.handleResult(AbstractOperationContext.java:1485)

      at org.jboss.as.controller.AbstractOperationContext$Step.finalizeInternal(AbstractOperationContext.java:1439)

      at org.jboss.as.controller.AbstractOperationContext$Step.finalizeStep(AbstractOperationContext.java:1422)

      at org.jboss.as.controller.AbstractOperationContext$Step.access$400(AbstractOperationContext.java:1286)

      at org.jboss.as.controller.AbstractOperationContext.executeResultHandlerPhase(AbstractOperationContext.java:859)

      at org.jboss.as.controller.AbstractOperationContext.processStages(AbstractOperationContext.java:709)

      at org.jboss.as.controller.AbstractOperationContext.executeOperation(AbstractOperationContext.java:450)

      at org.jboss.as.controller.OperationContextImpl.executeOperation(OperationContextImpl.java:1402)

      at org.jboss.as.controller.ModelControllerImpl.boot(ModelControllerImpl.java:516)

      at org.jboss.as.controller.AbstractControllerService.boot(AbstractControllerService.java:468)

      at org.jboss.as.controller.AbstractControllerService.boot(AbstractControllerService.java:430)

      at org.jboss.as.server.ServerService.boot(ServerService.java:437)

      at org.jboss.as.server.ServerService.boot(ServerService.java:396)

      at org.jboss.as.controller.AbstractControllerService$1.run(AbstractControllerService.java:370)

      at java.lang.Thread.run(Thread.java:748)

       

       

      I have attached the server.logs for both the JVMs.

       

      I want both of my servers to run and take traffic, process messages, etc... by utilizing the shared-store use-case-model ... is this achievable in shared-store model ..? Also why does my slave(server2) times out without waiting for the live server going down.

       

       

      Any help would be appreciated, many thanks !!!