Concurrent TimeoutException
Hello,
We are using infinispan 9.2 and following are high level application running params. We moved to 9.2 because of split brain issues encountered in the past with 6.2 around 1 year ago. Now we occasionally get issues related to cache as mentioned in the log file below.
Java : 1.8
OS: GNU/Linux
infinispan 9.2
Spring, Hibernate and Jersey
Sample infinispan xml
<jgroups> <stack-file name="configurationFile" path="jgroups-tcp.xml"/> </jgroups> <cache-container default-cache="default" statistics="true" shutdown-hook="DONT_REGISTER"> <transport lock-timeout="240000" stack="configurationFile"></transport> <jmx duplicate-domains="true" /> <replicated-cache name="default" mode="SYNC" remote-timeout="20000" statistics="true"> <locking isolation="REPEATABLE_READ" acquire-timeout="200000" write-skew="false" concurrency-level="500" striping="false"/> <indexing index="ALL"> <property name="default.directory_provider">local-heap</property> <property name="default.indexmanager">near-real-time</property> <property name="hibernate.search.lucene_version">LUCENE_5_5_5</property> <property name="default.exclusive_index_use">true</property> <property name="default.sharding_strategy.nbr_of_shards">1</property> <property name="default.chunk_size">12800</property> <property name="default.indexwriter.merge_max_size">1024</property> <property name="default.worker.execution">async</property> <property name="default.worker.thread_pool.size">10</property> <property name="default.indexwriter.term_index_interval">default</property> <property name="default.indexwriter.merge_factor">20</property> <property name="default.max_queue_length">50000</property> <property name="default.indexwriter.ram_buffer_size">2048</property> </indexing> </replicated-cache>
Log File 1
2019-09-10 13:24:07.242 +0000 ERROR http-bio-10110-exec-119 org.infinispan.interceptors.impl.InvocationContextInterceptor ISPN000136: Error executing command SizeCommand, writing keys []
org.infinispan.util.concurrent.TimeoutException: ISPN000476: Timed out waiting for responses for request 2675740 from <server>
at org.infinispan.remoting.transport.impl.SingleTargetRequest.onTimeout(SingleTargetRequest.java:65)
at org.infinispan.remoting.transport.AbstractRequest.call(AbstractRequest.java:87)
at org.infinispan.remoting.transport.AbstractRequest.call(AbstractRequest.java:22)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
Log File 2
2019-09-10 13:39:44.893 +0000 ERROR http-bio-10110-exec-245 org.infinispan.interceptors.impl.InvocationContextInterceptor ISPN000136: Error executing command SizeCommand, writing keys []
org.infinispan.util.concurrent.TimeoutException
at org.infinispan.stream.impl.AbstractCacheStream.performOperationRehashAware(AbstractCacheStream.java:313)
at org.infinispan.stream.impl.AbstractCacheStream.performOperation(AbstractCacheStream.java:232)
at org.infinispan.stream.impl.DistributedCacheStream.count(DistributedCacheStream.java:426)
at org.infinispan.commands.read.SizeCommand.perform(SizeCommand.java:45)
at org.infinispan.commands.read.SizeCommand.perform(SizeCommand.java:20)
at org.infinispan.interceptors.impl.CallInterceptor.visitCommand(CallInterceptor.java:29)
at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNext(BaseAsyncInterceptor.java:56)
at org.infinispan.interceptors.DDAsyncInterceptor.handleDefault(DDAsyncInterceptor.java:54)
at org.infinispan.interceptors.DDAsyncInterceptor.visitSizeCommand(DDAsyncInterceptor.java:100)
at org.infinispan.interceptors.distribution.BaseDistributionInterceptor.visitSizeCommand(BaseDistributionInterceptor.java:115)
at org.infinispan.commands.read.SizeCommand.acceptVisitor(SizeCommand.java:35)
at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNext(BaseAsyncInterceptor.java:54)
at org.infinispan.interceptors.DDAsyncInterceptor.handleDefault(DDAsyncInterceptor.java:54)
at org.infinispan.interceptors.DDAsyncInterceptor.visitSizeCommand(DDAsyncInterceptor.java:100)
at org.infinispan.commands.read.SizeCommand.acceptVisitor(SizeCommand.java:35)
at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNext(BaseAsyncInterceptor.java:54)
at org.infinispan.interceptors.DDAsyncInterceptor.handleDefault(DDAsyncInterceptor.java:54)
at org.infinispan.interceptors.DDAsyncInterceptor.visitSizeCommand(DDAsyncInterceptor.java:100)
at org.infinispan.commands.read.SizeCommand.acceptVisitor(SizeCommand.java:35)
at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNext(BaseAsyncInterceptor.java:54)
at org.infinispan.interceptors.DDAsyncInterceptor.handleDefault(DDAsyncInterceptor.java:54)
at org.infinispan.interceptors.DDAsyncInterceptor.visitSizeCommand(DDAsyncInterceptor.java:100)
at org.infinispan.commands.read.SizeCommand.acceptVisitor(SizeCommand.java:35)
at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNext(BaseAsyncInterceptor.java:54)
at org.infinispan.statetransfer.StateTransferInterceptor.handleDefault(StateTransferInterceptor.java:347)
at org.infinispan.interceptors.DDAsyncInterceptor.visitSizeCommand(DDAsyncInterceptor.java:100)
at org.infinispan.commands.read.SizeCommand.acceptVisitor(SizeCommand.java:35)
at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNext(BaseAsyncInterceptor.java:54)
at org.infinispan.interceptors.DDAsyncInterceptor.handleDefault(DDAsyncInterceptor.java:54)
at org.infinispan.interceptors.DDAsyncInterceptor.visitSizeCommand(DDAsyncInterceptor.java:100)
at org.infinispan.commands.read.SizeCommand.acceptVisitor(SizeCommand.java:35)
at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNextAndExceptionally(BaseAsyncInterceptor.java:123)
at org.infinispan.interceptors.impl.InvocationContextInterceptor.visitCommand(InvocationContextInterceptor.java:90)
at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNext(BaseAsyncInterceptor.java:56)
at org.infinispan.interceptors.DDAsyncInterceptor.handleDefault(DDAsyncInterceptor.java:54)
at org.infinispan.interceptors.DDAsyncInterceptor.visitSizeCommand(DDAsyncInterceptor.java:100)
at org.infinispan.commands.read.SizeCommand.acceptVisitor(SizeCommand.java:35)
at org.infinispan.interceptors.DDAsyncInterceptor.visitCommand(DDAsyncInterceptor.java:50)
at org.infinispan.interceptors.impl.AsyncInterceptorChainImpl.invoke(AsyncInterceptorChainImpl.java:248)
at org.infinispan.cache.impl.CacheImpl.size(CacheImpl.java:449)
at org.infinispan.cache.impl.CacheImpl.size(CacheImpl.java:444)
at org.infinispan.cache.impl.AbstractDelegatingCache.size(AbstractDelegatingCache.java:313)
at com.gs.ibd.atlas.services.AtlasHelperService.getHealthStatus(AtlasHelperService.java:56)
at sun.reflect.GeneratedMethodAccessor184.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:483)
at org.glassfish.jersey.server.model.internal.ResourceMethodInvocationHandlerFactory$1.invoke(ResourceMethodInvocationHandlerFactory.java:81)
at org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher$1.run(AbstractJavaResourceMethodDispatcher.java:151)
at org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher.invoke(AbstractJavaResourceMethodDispatcher.java:171)
at org.glassfish.jersey.server.model.internal.JavaResourceMethodDispatcherProvider$ResponseOutInvoker.doDispatch(JavaResourceMethodDispatcherProvider.java:152)
at org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher.dispatch(AbstractJavaResourceMethodDispatcher.java:104)
at org.glassfish.jersey.server.model.ResourceMethodInvoker.invoke(ResourceMethodInvoker.java:387)
at org.glassfish.jersey.server.model.ResourceMethodInvoker.apply(ResourceMethodInvoker.java:331)
at org.glassfish.jersey.server.model.ResourceMethodInvoker.apply(ResourceMethodInvoker.java:103)
at org.glassfish.jersey.server.ServerRuntime$1.run(ServerRuntime.java:271)
at org.glassfish.jersey.internal.Errors$1.call(Errors.java:271)
at org.glassfish.jersey.internal.Errors$1.call(Errors.java:267)
at org.glassfish.jersey.internal.Errors.process(Errors.java:315)
at org.glassfish.jersey.internal.Errors.process(Errors.java:297)
at org.glassfish.jersey.internal.Errors.process(Errors.java:267)
at org.glassfish.jersey.process.internal.RequestScope.runInScope(RequestScope.java:297)
at org.glassfish.jersey.server.ServerRuntime.process(ServerRuntime.java:254)
at org.glassfish.jersey.server.ApplicationHandler.handle(ApplicationHandler.java:1028)
at org.glassfish.jersey.servlet.WebComponent.service(WebComponent.java:372)
at org.glassfish.jersey.servlet.ServletContainer.service(ServletContainer.java:382)
at org.glassfish.jersey.servlet.ServletContainer.doFilter(ServletContainer.java:535)
at org.glassfish.jersey.servlet.ServletContainer.doFilter(ServletContainer.java:483)
at org.glassfish.jersey.servlet.ServletContainer.doFilter(ServletContainer.java:420)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
is this related to [ISPN-10229] NullPointerException when enabling cache statistics in Hibernate Cache - JBoss Issue Tracker ? and fixed in 10.X? as one of the comment says on the jira that its still issue in 9.x?
