ARJUNA016129: Could not end XA resource Subordinate XAResource at remote://localhost:31099: javax.transaction.xa.XAException: WFTXN0034: Failed to acquire a connection for this operation
reshmakoganti1 Dec 6, 2018 4:04 AMHello All,
I am having some difficultly in using Remote EJB between two Wildfly11 servers. Note: I am not using "elytron" subsystem.
Currently we are migrating from JBOSS EAP 6.2 to Wildfly11.
I am able to connect to the JMS using ActiveMq on port 5445. But having trouble in connected to port 31099 from server 2. Kindly help
Server 1 : Standalone.xml
<subsystem xmlns="urn:jboss:domain:messaging-activemq:2.0">
<server name="default" persistence-enabled="true" >
<security enabled="true" domain="myApp"/>
<security-setting name="#">
<role name="Subscriber" send="false" consume="true" create-durable-queue="true" delete-durable-queue="true"
create-non-durable-queue="true" delete-non-durable-queue="true"/>
<role name="Publisher" send="true" consume="true" create-durable-queue="false" delete-durable-queue="false"
create-non-durable-queue="false" delete-non-durable-queue="false"/>
</security-setting>
<address-setting name="#" dead-letter-address="jms.queue.DLQ" expiry-address="jms.queue.ExpiryQueue"
redelivery-delay="2000" max-delivery-attempts="5" max-size-bytes="10485760" page-size-bytes="2097152" address-full-policy="PAGE" message-counter-history-day-limit="10"/>
<http-connector name="http-connector" socket-binding="http" endpoint="http-acceptor"/>
<http-connector name="http-connector-throughput" socket-binding="http" endpoint="http-acceptor-throughput">
<param name="batch-delay" value="50"/>
</http-connector>
<remote-connector name="remoting-connector" socket-binding="messaging" />
<in-vm-connector name="in-vm" server-id="0"/>
<http-acceptor name="http-acceptor" http-listener="default"/>
<http-acceptor name="http-acceptor-throughput" http-listener="default">
<param name="batch-delay" value="50"/>
<param name="direct-deliver" value="false"/>
</http-acceptor>
<remote-acceptor name="remoting-acceptor" socket-binding="messaging"/>
<in-vm-acceptor name="in-vm" server-id="0"/>
<jms-queue name="ExpiryQueue" entries="java:/jms/queue/ExpiryQueue"/>
<jms-queue name="DLQ" entries="java:/jms/queue/DLQ"/>
<jms-topic name="Topic" entries="topic/POTopic java:jboss/exported/jms/topic/Topic"/>
<connection-factory name="InVmConnectionFactory" entries="java:/ConnectionFactory" connectors="in-vm"/>
<connection-factory name="RemoteConnectionFactory" entries="java:jboss/exported/jms/RemoteConnectionFactory" connectors="remoting-connector" ha="true"/>
<pooled-connection-factory name="activemq-ra" entries="java:/JmsXA" connectors="in-vm" transaction="xa"/>
</server>
</subsystem>
<subsystem xmlns="urn:jboss:domain:ejb3:5.0">
<session-bean>
<stateless>
<bean-instance-pool-ref pool-name="slsb-strict-max-pool"/>
</stateless>
<stateful default-access-timeout="5000" cache-ref="simple" passivation-disabled-cache-ref="simple"/>
<singleton default-access-timeout="5000"/>
</session-bean>
<mdb>
<resource-adapter-ref resource-adapter-name="${ejb.resource-adapter-name:RaActivatoractivemq-ra.rar}"/>
<bean-instance-pool-ref pool-name="mdb-strict-max-pool"/>
</mdb>
<pools>
<bean-instance-pools>
<strict-max-pool name="slsb-strict-max-pool" instance-acquisition-timeout="5"
instance-acquisition-timeout-unit="MINUTES"/>
<strict-max-pool name="mdb-strict-max-pool" instance-acquisition-timeout="5"
instance-acquisition-timeout-unit="MINUTES"/>
</bean-instance-pools>
</pools>
<caches>
<cache name="simple" aliases="NoPassivationCache"/>
<cache name="passivating" passivation-store-ref="file" aliases="SimpleStatefulCache"/>
</caches>
<passivation-stores>
<file-passivation-store name="file"/>
</passivation-stores>
<async thread-pool-name="default"/>
<timer-service thread-pool-name="default" default-data-store="default-file-store">
<data-stores>
<file-data-store name="default-file-store" path="timer-service-data" relative-to="jboss.server.data.dir"/>
</data-stores>
</timer-service>
<remote connector-ref="remoting-connector" thread-pool-name="default">
<channel-creation-options>
<option name="READ_TIMEOUT" value="${prop.remoting-connector.read.timeout:20}" type="xnio"/>
<option name="MAX_OUTBOUND_MESSAGES" value="1234" type="remoting"/>
</channel-creation-options>
</remote>
<thread-pools>
<thread-pool name="default">
<max-threads count="10"/>
<keepalive-time time="100" unit="milliseconds"/>
</thread-pool>
</thread-pools>
<default-security-domain value="other"/>
<default-missing-method-permissions-deny-access value="true"/>
<log-system-exceptions value="true"/>
</subsystem>
<subsystem xmlns="urn:jboss:domain:remoting:4.0">
<endpoint/>
<http-connector name="remoting-connector" connector-ref="default" security-realm="myApp"/>
<connector name="local-web" socket-binding="remoting"
security-realm="myApp"/>
</subsystem>
<subsystem xmlns="urn:jboss:domain:security:2.0">
<security-domains>
<security-domain name="myApp" cache-type="default">
<authentication>
<login-module code="Remoting" flag="optional">
<module-option name="password-stacking" value="useFirstPass"/>
</login-module>
</authentication>
</security-domain>
</subsystem>
[......]
<socket-binding name="remoting" port="31099"/>
<socket-binding name="messaging" port="5445"/>
Server 2 : Standalone.xml
<subsystem xmlns="urn:jboss:domain:ejb3:5.0">
<session-bean>
<stateless>
<bean-instance-pool-ref pool-name="slsb-strict-max-pool"/>
</stateless>
<stateful default-access-timeout="5000" cache-ref="simple" passivation-disabled-cache-ref="simple"/>
<singleton default-access-timeout="5000"/>
</session-bean>
<pools>
<bean-instance-pools>
<strict-max-pool name="slsb-strict-max-pool" derive-size="from-worker-pools" instance-acquisition-timeout="5"
instance-acquisition-timeout-unit="MINUTES"/>
<strict-max-pool name="mdb-strict-max-pool" derive-size="from-cpu-count" instance-acquisition-timeout="5"
instance-acquisition-timeout-unit="MINUTES"/>
</bean-instance-pools>
</pools>
<caches>
<cache name="simple" aliases="NoPassivationCache"/>
<cache name="passivating" passivation-store-ref="file" aliases="SimpleStatefulCache"/>
</caches>
<mdb>
<resource-adapter-ref resource-adapter-name="${ejb.resource-adapter-name:RaActivatoractivemq-ra.rar}"/>
<bean-instance-pool-ref pool-name="mdb-strict-max-pool"/>
</mdb>
<passivation-stores>
<file-passivation-store name="file"/>
</passivation-stores>
<async thread-pool-name="default"/>
<timer-service thread-pool-name="default" default-data-store="default-file-store">
<data-stores>
<file-data-store name="default-file-store" path="timer-service-data" relative-to="jboss.server.data.dir"/>
</data-stores>
</timer-service>
<remote connector-ref="remoting-connector" thread-pool-name="default"/>
<thread-pools>
<thread-pool name="default">
<max-threads count="10"/>
<keepalive-time time="100" unit="milliseconds"/>
</thread-pool>
</thread-pools>
<default-security-domain value="other"/>
<default-missing-method-permissions-deny-access value="false"/>
<log-system-exceptions value="true"/>
</subsystem>
<subsystem xmlns="urn:jboss:domain:remoting:4.0">
<endpoint/>
<http-connector name="http-remoting-connector" connector-ref="default" security-realm="ApplicationRealm"/>
<connector name="remoting-connector" socket-binding="remoting"
security-realm="ApplicationRealm"/>
</subsystem>[......]
<socket-binding name="remoting" port="4447"/>
[......]
Code:
final Properties jndiProps = new Properties();
jndiProps.put(Context.INITIAL_CONTEXT_FACTORY, "org.wildfly.naming.client.WildFlyInitialContextFactory");
jndiProps.put(Context.URL_PKG_PREFIXES, "org.jboss.ejb.client.naming");
jndiProps.put(Context.PROVIDER_URL, "remote://" + PaEarPropertyReader.getLookUpServerInfo());
jndiProps.put(Context.SECURITY_PRINCIPAL, PaEarPropertyReader.getLookUpServerPoPrincipal());
jndiProps.put(Context.SECURITY_CREDENTIALS, PaEarPropertyReader.getLookUpServerPoCredentials());
jndiProps.put("jboss.naming.client.ejb.context", true);
jndiProps.put("jboss.naming.client.connect.options.org.xnio.Options.SASL_POLICY_NOPLAINTEXT", "false");
jndiProps.put("jboss.naming.client.connect.options.org.xnio.Options.SASL_POLICY_NOANONYMOUS", "true"); // added later for testing but did not help
jndiProps.put("jboss.naming.client.connect.options.org.xnio.Options." +"SASL_DISALLOWED_MECHANISMS", "JBOSS-LOCAL-USER"); // added later for testing but did not help
ctx = new InitialContext(jndiProps);
Full Error Stack trace from Server 2:
08:20:53,196 INFO [org.jboss.as.remoting] (MSC service thread 1-3) WFLYRMT0001: Listening on 127.0.0.1:9998
08:20:53,361 INFO [org.jboss.as.remoting] (MSC service thread 1-7) WFLYRMT0001: Listening on 127.0.0.1:4447
22:21:32,148 INFO [gov.nih.nci.pa.service.util.JmsMessageReceiver] (Thread-JmsMessageReceiver) Connecting to JMS Topic jms/topic/Topic at localhost as Client local1 with Subscription MYAPP using Factory jms/RemoteConnectionFactory
22:21:32,158 INFO [org.wildfly.naming] (Thread-JmsMessageReceiver) WildFly Naming version 1.0.7.Final
22:21:32,811 INFO [gov.nih.nci.pa.service.util.JmsMessageReceiver] (Thread-JmsMessageReceiver) Applied JBoss ConnectionFactory host name fix: host=localhost, port=5445
22:21:33,304 INFO [gov.nih.nci.pa.service.util.JmsMessageReceiver] (Thread-JmsMessageReceiver) Connected to JMS Topic jms/topic/Topic at localhost as Client local1 with Subscription MYAPP using Factory jms/RemoteConnectionFactory
22:22:37,446 WARN [com.arjuna.ats.jta] (QuartzScheduler_Worker-2) ARJUNA016045: attempted rollback of < formatId=131077, gtrid_length=29, bqual_length=36, tx_uid=0:ffffc0a801fe:30057cea:5c07441f:18d, node_name=1, branch_uid=0:ffffc0a801fe:30057cea:5c07441f:1a4, subordinatenodename=null, eis_name=unknown eis name > (Subordinate XAResource at remote://localhost:31099) failed with exception code XAException.XAER_RMFAIL: javax.transaction.xa.XAException: WFTXN0034: Failed to acquire a connection for this operation
at org.wildfly.transaction.client.provider.remoting.RemotingRemoteTransactionPeer.getPeerIdentityXA(RemotingRemoteTransactionPeer.java:94)
at org.wildfly.transaction.client.provider.remoting.RemotingRemoteTransactionPeer$1.rollback(RemotingRemoteTransactionPeer.java:148)
at org.wildfly.transaction.client.SubordinateXAResource.rollback(SubordinateXAResource.java:147)
at com.arjuna.ats.internal.jta.resources.arjunacore.XAResourceRecord.topLevelAbort(XAResourceRecord.java:330)
at com.arjuna.ats.arjuna.coordinator.BasicAction.doAbort(BasicAction.java:3023)
at com.arjuna.ats.arjuna.coordinator.BasicAction.doAbort(BasicAction.java:3002)
at com.arjuna.ats.arjuna.coordinator.BasicAction.Abort(BasicAction.java:1674)
at com.arjuna.ats.arjuna.coordinator.BasicAction.Abort(BasicAction.java:1615)
at com.arjuna.ats.arjuna.coordinator.TwoPhaseCoordinator.end(TwoPhaseCoordinator.java:99)
at com.arjuna.ats.arjuna.AtomicAction.commit(AtomicAction.java:162)
at com.arjuna.ats.internal.jta.transaction.arjunacore.TransactionImple.commitAndDisassociate(TransactionImple.java:1289)
at com.arjuna.ats.internal.jta.transaction.arjunacore.BaseTransaction.commit(BaseTransaction.java:126)
at com.arjuna.ats.jbossatx.BaseTransactionManagerDelegate.commit(BaseTransactionManagerDelegate.java:89)
at org.wildfly.transaction.client.LocalTransaction.commitAndDissociate(LocalTransaction.java:73)
at org.wildfly.transaction.client.ContextTransactionManager.commit(ContextTransactionManager.java:71)
at org.jboss.as.ejb3.tx.CMTTxInterceptor.endTransaction(CMTTxInterceptor.java:92)
at org.jboss.as.ejb3.tx.CMTTxInterceptor.invokeInOurTx(CMTTxInterceptor.java:279)
at org.jboss.as.ejb3.tx.CMTTxInterceptor.required(CMTTxInterceptor.java:332)
at org.jboss.as.ejb3.tx.CMTTxInterceptor.processInvocation(CMTTxInterceptor.java:240)
at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:422)
at org.jboss.invocation.InterceptorContext$Invocation.proceed(InterceptorContext.java:509)
at org.jboss.weld.ejb.AbstractEJBRequestScopeActivationInterceptor.aroundInvoke(AbstractEJBRequestScopeActivationInterceptor.java:73)
at org.jboss.as.weld.ejb.EjbRequestScopeActivationInterceptor.processInvocation(EjbRequestScopeActivationInterceptor.java:89)
at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:422)
at org.jboss.as.ejb3.component.interceptors.CurrentInvocationContextInterceptor.processInvocation(CurrentInvocationContextInterceptor.java:41)
at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:422)
at org.jboss.as.ejb3.component.invocationmetrics.WaitTimeInterceptor.processInvocation(WaitTimeInterceptor.java:47)
at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:422)
at org.jboss.as.ejb3.security.SecurityContextInterceptor.processInvocation(SecurityContextInterceptor.java:100)
at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:422)
at org.jboss.as.ejb3.deployment.processors.StartupAwaitInterceptor.processInvocation(StartupAwaitInterceptor.java:22)
at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:422)
at org.jboss.as.ejb3.component.interceptors.ShutDownInterceptorFactory$1.processInvocation(ShutDownInterceptorFactory.java:64)
at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:422)
at org.jboss.as.ejb3.component.interceptors.LoggingInterceptor.processInvocation(LoggingInterceptor.java:67)
at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:422)
at org.jboss.as.ee.component.NamespaceContextInterceptor.processInvocation(NamespaceContextInterceptor.java:50)
at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:422)
at org.jboss.as.ejb3.component.interceptors.AdditionalSetupInterceptor.processInvocation(AdditionalSetupInterceptor.java:54)
at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:422)
at org.jboss.invocation.ContextClassLoaderInterceptor.processInvocation(ContextClassLoaderInterceptor.java:60)
at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:422)
at org.jboss.invocation.InterceptorContext.run(InterceptorContext.java:438)
at org.wildfly.security.manager.WildFlySecurityManager.doChecked(WildFlySecurityManager.java:609)
at org.jboss.invocation.AccessCheckingInterceptor.processInvocation(AccessCheckingInterceptor.java:57)
at org.jboss.invocation.InterceptorContext.proceed(InterceptorContext.java:422)
at org.jboss.invocation.ChainedInterceptor.processInvocation(ChainedInterceptor.java:53)
at org.jboss.as.ee.component.ViewService$View.invoke(ViewService.java:198)
at org.jboss.as.ee.component.ViewDescription$1.processInvocation(ViewDescription.java:185)
at org.jboss.as.ee.component.ProxyInvocationHandler.invoke(ProxyInvocationHandler.java:81)
at gov.nih.nci.pa.service.util.FamilyProgramCodeServiceLocal$$$view132.populate(Unknown Source)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at gov.nih.nci.pa.util.EJBInvokerJob.execute(EJBInvokerJob.java:209)
at org.quartz.core.JobRunShell.run(JobRunShell.java:202)
at org.quartz.simpl.SimpleThreadPool$WorkerThread.run(SimpleThreadPool.java:525)
Caused by: org.wildfly.security.auth.AuthenticationException: JBREM000308: Authentication failed (no mechanisms left), tried: (none)
at org.jboss.remoting3.ConnectionPeerIdentityContext.doAuthenticate(ConnectionPeerIdentityContext.java:368)
at org.jboss.remoting3.ConnectionPeerIdentityContext.authenticate(ConnectionPeerIdentityContext.java:174)
at org.jboss.remoting3.EndpointImpl$3.handleDone(EndpointImpl.java:505)
at org.jboss.remoting3.EndpointImpl$3.handleDone(EndpointImpl.java:494)
at org.xnio.IoFuture$HandlingNotifier.notify(IoFuture.java:208)
at org.xnio.AbstractIoFuture$NotifierRunnable.run(AbstractIoFuture.java:720)
at org.xnio.IoUtils$2.execute(IoUtils.java:71)
at org.xnio.AbstractIoFuture.runNotifier(AbstractIoFuture.java:693)
at org.xnio.AbstractIoFuture$CompleteState.withNotifier(AbstractIoFuture.java:132)
at org.xnio.AbstractIoFuture.addNotifier(AbstractIoFuture.java:570)
at org.jboss.remoting3.EndpointImpl.doGetConnection(EndpointImpl.java:494)
at org.jboss.remoting3.EndpointImpl.getConnectedIdentity(EndpointImpl.java:433)
at org.wildfly.transaction.client.provider.remoting.RemotingRemoteTransactionPeer.getPeerIdentity(RemotingRemoteTransactionPeer.java:87)
at org.wildfly.transaction.client.provider.remoting.RemotingRemoteTransactionPeer.getPeerIdentityXA(RemotingRemoteTransactionPeer.java:92)
... 57 more
server 1 Logs :
-----------------
Wildfly11 PO logs
06:46:43,465 INFO [org.jboss.as.server.deployment] (MSC service thread 1-1) WFLYSRV0027: Starting deployment of "MYAPP.ear" (runtime-name: "MYAPP.ear")
06:46:43,477 INFO [org.jboss.as.remoting] (MSC service thread 1-8) WFLYRMT0001: Listening on 127.0.0.1:31099
06:46:43,819 INFO [org.apache.activemq.artemis.core.server] (ServerService Thread Pool -- 62) AMQ221043: Protocol module found: [artemis-server]. Adding protocol support for: CORE
06:46:43,832 INFO [org.apache.activemq.artemis.core.server] (ServerService Thread Pool -- 62) AMQ221043: Protocol module found: [artemis-amqp-protocol]. Adding protocol support for: AMQP
06:46:43,833 INFO [org.apache.activemq.artemis.core.server] (ServerService Thread Pool -- 62) AMQ221043: Protocol module found: [artemis-hornetq-protocol]. Adding protocol support for: HORNETQ
06:46:43,835 INFO [org.apache.activemq.artemis.core.server] (ServerService Thread Pool -- 62) AMQ221043: Protocol module found: [artemis-stomp-protocol]. Adding protocol support for: STOMP
06:46:44,091 INFO [org.wildfly.extension.messaging-activemq] (MSC service thread 1-3) WFLYMSGAMQ0016: Registered HTTP upgrade for activemq-remoting protocol handled by http-acceptor acceptor
06:46:44,090 INFO [org.wildfly.extension.messaging-activemq] (MSC service thread 1-5) WFLYMSGAMQ0016: Registered HTTP upgrade for activemq-remoting protocol handled by http-acceptor-throughput acceptor
06:46:44,090 INFO [org.wildfly.extension.messaging-activemq] (MSC service thread 1-4) WFLYMSGAMQ0016: Registered HTTP upgrade for activemq-remoting protocol handled by http-acceptor acceptor
06:46:44,090 INFO [org.wildfly.extension.messaging-activemq] (MSC service thread 1-1) WFLYMSGAMQ0016: Registered HTTP upgrade for activemq-remoting protocol handled by http-acceptor-throughput acceptor
06:46:44,288 INFO [org.apache.activemq.artemis.core.server] (ServerService Thread Pool -- 62) AMQ221020: Started Acceptor at 127.0.0.1:5445 for protocols [CORE,AMQP,HORNETQ,STOMP]
06:46:44,290 INFO [org.apache.activemq.artemis.core.server] (ServerService Thread Pool -- 62) AMQ221007: Server is now live
06:46:44,290 INFO [org.apache.activemq.artemis.core.server] (ServerService Thread Pool -- 62) AMQ221001: Apache ActiveMQ Artemis Message Broker version 1.5.5.jbossorg-008 [default, nodeID=18d3c3b3-d7af-11e8-90e9-8c8590208ad3]
06:46:44,294 INFO [org.apache.activemq.artemis.core.server] (ServerService Thread Pool -- 62) AMQ221003: Deploying queue jms.queue.ExpiryQueue
06:46:44,330 INFO [org.wildfly.extension.messaging-activemq] (ServerService Thread Pool -- 66) WFLYMSGAMQ0002: Bound messaging object to jndi name java:/ConnectionFactory
06:46:44,332 INFO [org.apache.activemq.artemis.core.server] (ServerService Thread Pool -- 64) AMQ221052: Deploying topic jms.topic.Topic
06:46:44,340 INFO [org.wildfly.extension.messaging-activemq] (ServerService Thread Pool -- 65) WFLYMSGAMQ0002: Bound messaging object to jndi name java:jboss/exported/jms/RemoteConnectionFactory
06:46:44,340 INFO [org.apache.activemq.artemis.core.server] (ServerService Thread Pool -- 63) AMQ221003: Deploying queue jms.queue.DLQ
06:46:44,407 INFO [org.jboss.as.connector.deployment] (MSC service thread 1-8) WFLYJCA0007: Registered connection factory java:/JmsXA
06:46:44,483 INFO [org.apache.activemq.artemis.ra] (MSC service thread 1-8) Resource adaptor started
06:46:44,483 INFO [org.jboss.as.connector.services.resourceadapters.ResourceAdapterActivatorService$ResourceAdapterActivator] (MSC service thread 1-8) IJ020002: Deployed: file://RaActivatoractivemq-ra
06:46:44,485 INFO [org.jboss.as.connector.deployment] (MSC service thread 1-2) WFLYJCA0002: Bound JCA ConnectionFactory [java:/JmsXA]
Please suggest and help.
Thank you in advance.
Reshma