I've tried upgrading to CR3 (both the server and the client JAR), and now under load we get the following exception on the client (it happens anywhere from 20 seconds to a couple of minutes after processing starts):
2006-07-31 11:05:10,829 ERROR org.jboss.jms.client.container.ExceptionInterceptor - Caught Exception:
org.jboss.aop.NotFoundInDispatcherException: Object with oid: -2147483420 was not found in the Dispatcher
at org.jboss.aop.Dispatcher.invoke(Dispatcher.java:85)
at org.jboss.jms.server.remoting.JMSServerInvocationHandler.invoke(JMSServerInvocationHandler.java:126)
at org.jboss.remoting.ServerInvoker.invoke(ServerInvoker.java:842)
at org.jboss.remoting.ServerInvoker.invoke(ServerInvoker.java:691)
at org.jboss.remoting.transport.socket.ServerThread.processInvocation(ServerThread.java:443)
at org.jboss.remoting.transport.socket.ServerThread.dorun(ServerThread.java:530)
at org.jboss.remoting.transport.socket.ServerThread.run(ServerThread.java:253)
at org.jboss.remoting.MicroRemoteClientInvoker.invoke(MicroRemoteClientInvoker.java:163)
at org.jboss.remoting.Client.invoke(Client.java:610)
at org.jboss.remoting.Client.invoke(Client.java:602)
at org.jboss.jms.client.delegate.DelegateSupport.invoke(DelegateSupport.java:112)
at org.jboss.jms.client.delegate.ClientSessionDelegate$send_N3028277934545793941.invokeNext(ClientSessionDelegate$send_N3028277934545793941.java)
at org.jboss.jms.client.container.TransactionAspect.handleSend(TransactionAspect.java:170)
at org.jboss.aop.advice.org.jboss.jms.client.container.TransactionAspect10.invoke(TransactionAspect10.java)
at org.jboss.jms.client.delegate.ClientSessionDelegate$send_N3028277934545793941.invokeNext(ClientSessionDelegate$send_N3028277934545793941.java)
at org.jboss.jms.client.container.ClosedInterceptor.invoke(ClosedInterceptor.java:134)
at org.jboss.aop.advice.PerInstanceInterceptor.invoke(PerInstanceInterceptor.java:117)
at org.jboss.jms.client.delegate.ClientSessionDelegate$send_N3028277934545793941.invokeNext(ClientSessionDelegate$send_N3028277934545793941.java)
at org.jboss.jms.client.container.ExceptionInterceptor.invoke(ExceptionInterceptor.java:69)
at org.jboss.jms.client.delegate.ClientSessionDelegate$send_N3028277934545793941.invokeNext(ClientSessionDelegate$send_N3028277934545793941.java)
at org.jboss.jms.client.container.ClientLogInterceptor.invoke(ClientLogInterceptor.java:107)
at org.jboss.jms.client.delegate.ClientSessionDelegate$send_N3028277934545793941.invokeNext(ClientSessionDelegate$send_N3028277934545793941.java)
at org.jboss.jms.client.delegate.ClientSessionDelegate.send(ClientSessionDelegate.java)
at org.jboss.jms.client.container.ProducerAspect.handleSend(ProducerAspect.java:237)
at sun.reflect.GeneratedMethodAccessor504.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:324)
at org.jboss.aop.advice.PerInstanceAdvice.invoke(PerInstanceAdvice.java:130)
at org.jboss.jms.client.delegate.ClientProducerDelegate$send_3961598017717988886.invokeNext(ClientProducerDelegate$send_3961598017717988886.java)
at org.jboss.jms.client.container.ClosedInterceptor.invoke(ClosedInterceptor.java:134)
at org.jboss.aop.advice.PerInstanceInterceptor.invoke(PerInstanceInterceptor.java:117)
at org.jboss.jms.client.delegate.ClientProducerDelegate$send_3961598017717988886.invokeNext(ClientProducerDelegate$send_3961598017717988886.java)
at org.jboss.jms.client.container.ExceptionInterceptor.invoke(ExceptionInterceptor.java:69)
at org.jboss.jms.client.delegate.ClientProducerDelegate$send_3961598017717988886.invokeNext(ClientProducerDelegate$send_3961598017717988886.java)
at org.jboss.jms.client.container.ClientLogInterceptor.invoke(ClientLogInterceptor.java:107)
at org.jboss.jms.client.delegate.ClientProducerDelegate$send_3961598017717988886.invokeNext(ClientProducerDelegate$send_3961598017717988886.java)
at org.jboss.jms.client.delegate.ClientProducerDelegate.send(ClientProducerDelegate.java)
at org.jboss.jms.client.JBossMessageProducer.send(JBossMessageProducer.java:164)
at org.jboss.jms.client.JBossMessageProducer.send(JBossMessageProducer.java:208)
at org.jboss.jms.client.JBossMessageProducer.send(JBossMessageProducer.java:145)
at org.jboss.jms.client.JBossMessageProducer.send(JBossMessageProducer.java:136)
at org.springframework.jms.core.JmsTemplate.doSend(JmsTemplate.java:671)
at org.springframework.jms.core.JmsTemplate.doSend(JmsTemplate.java:653)
at org.springframework.jms.core.JmsTemplate$3.doInJms(JmsTemplate.java:638)
at org.springframework.jms.core.JmsTemplate.execute(JmsTemplate.java:585)
at org.springframework.jms.core.JmsTemplate.send(JmsTemplate.java:635)
at com.platefood.pfp.integration.messaging.JmsTemplateMessageDispatcher.dispatchAsynchronous(JmsTemplateMessageDispatcher.java:67)
at com.platefood.pfp.core.integration.publishing.MulePublisherInvoker.invokeAsynchronous(MulePublisherInvoker.java:116)
at com.platefood.pfp.core.integration.publishing.PublisherIntegrationServiceImpl.send(PublisherIntegrationServiceImpl.java:985)
at com.platefood.pfp.core.integration.publishing.PublisherIntegrationServiceImpl.validateAndSend(PublisherIntegrationServiceImpl.java:1686)
at com.platefood.pfp.core.integration.publishing.PublisherIntegrationServiceImpl.updateListingKeyword(PublisherIntegrationServiceImpl.java:1595)
at sun.reflect.GeneratedMethodAccessor508.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:324)
at org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:292)
at org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:155)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:122)
at org.springframework.transaction.interceptor.TransactionInterceptor.invoke(TransactionInterceptor.java:96)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:144)
at com.platefood.pfp.core.business.common.dao.OptimisticLockInterceptor.invoke(OptimisticLockInterceptor.java:46)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:144)
at com.platefood.pfp.core.business.aop.PerformanceLoggerInterceptor.invoke(PerformanceLoggerInterceptor.java:377)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:144)
at org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:174)
at $Proxy146.updateListingKeyword(Unknown Source)
at sun.reflect.GeneratedMethodAccessor508.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:324)
at com.platefood.pfp.core.integration.publishing.PublisherIntegrationReceiver.onMessage(PublisherIntegrationReceiver.java:190)
at com.platefood.pfp.integration.spring.jms.JMSReceiverManager$2.onMessage(JMSReceiverManager.java:226)
at org.jboss.jms.client.remoting.MessageCallbackHandler.callOnMessage(MessageCallbackHandler.java:82)
at org.jboss.jms.client.remoting.MessageCallbackHandler$ClientDeliveryRunnable.run(MessageCallbackHandler.java:751)
at EDU.oswego.cs.dl.util.concurrent.QueuedExecutor$RunLoop.run(Unknown Source)
at java.lang.Thread.run(Thread.java:534)
The log is then filled with this exception, and no messages are processed anymore. I still have the code in there to close and re-open the session every sixty seconds but that doesn't fix the problem.
This didn't happen with CR2 and I haven't changed our application apart from the jboss-messaging-client.jar. There are only two warnings in the JBoss server.log:
2006-07-31 11:03:57,257 WARN [org.jboss.jms.server.endpoint.DeliveryRunnable] Failed to deliver the message to the client.
2006-07-31 11:03:57,267 WARN [org.jboss.jms.server.connectionmanager.SimpleConnectionManager] A problem has been detected with the connection to remote client 5c4o25-8eg1pp-eqa511ib-1-eqa51w8f-h It is possible the client has exited without closing its connection(s) or there is a network problem. All connection resources corresponding to that client process will now be removed.
I will try to update my test harness class above to include some load and see if I can re-produce the problem.