2 Replies Latest reply on Dec 2, 2016 4:41 AM by binu_driver

    ISPN000136: Error executing command SizeCommand, writing keys

    binu_driver

      Hi,

       

      I am getting below error in my distributed cluster environment which is spanned across 10 infinispan nodes spanned across 2 physical hosts which belongs to the same subnet. We have loaded around 1 Lakh entries in cache. Then we bring down one instance by killing the process. And then we are trying to fetch the count in cache (or) new entries in cahce (or) view the data in cache. All the request failed with below exception.

       

      Kindly assist regarding this.

        

      ******************************************************************************************************

      ERROR InvocationContextInterceptor:140 - ISPN000136: Error executing command SizeCommand, writing keys []

      org.infinispan.util.concurrent.TimeoutException

              at org.infinispan.stream.impl.AbstractCacheStream.performOperationRehashAware(AbstractCacheStream.java:338)

              at org.infinispan.stream.impl.AbstractCacheStream.performOperation(AbstractCacheStream.java:258)

              at org.infinispan.stream.impl.AbstractCacheStream.performOperation(AbstractCacheStream.java:247)

              at org.infinispan.stream.impl.DistributedCacheStream.count(DistributedCacheStream.java:403)

              at org.infinispan.commands.read.SizeCommand.perform(SizeCommand.java:54)

              at org.infinispan.commands.read.SizeCommand.perform(SizeCommand.java:25)

              at org.infinispan.interceptors.CallInterceptor.handleDefault(CallInterceptor.java:125)

              at org.infinispan.commands.AbstractVisitor.visitSizeCommand(AbstractVisitor.java:80)

              at org.infinispan.commands.read.SizeCommand.acceptVisitor(SizeCommand.java:35)

              at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:99)

              at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:113)

              at org.infinispan.commands.AbstractVisitor.visitSizeCommand(AbstractVisitor.java:80)

              at org.infinispan.commands.read.SizeCommand.acceptVisitor(SizeCommand.java:35)

              at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:99)

              at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:113)

              at org.infinispan.commands.AbstractVisitor.visitSizeCommand(AbstractVisitor.java:80)

              at org.infinispan.commands.read.SizeCommand.acceptVisitor(SizeCommand.java:35)

              at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:99)

              at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:113)

              at org.infinispan.commands.AbstractVisitor.visitSizeCommand(AbstractVisitor.java:80)

              at org.infinispan.commands.read.SizeCommand.acceptVisitor(SizeCommand.java:35)

              at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:99)

              at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:113)

              at org.infinispan.commands.AbstractVisitor.visitSizeCommand(AbstractVisitor.java:80)

              at org.infinispan.commands.read.SizeCommand.acceptVisitor(SizeCommand.java:35)

              at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:99)

              at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:113)

              at org.infinispan.commands.AbstractVisitor.visitSizeCommand(AbstractVisitor.java:80)

              at org.infinispan.commands.read.SizeCommand.acceptVisitor(SizeCommand.java:35)

              at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:99)

              at org.infinispan.statetransfer.StateTransferInterceptor.handleDefault(StateTransferInterceptor.java:379)

              at org.infinispan.commands.AbstractVisitor.visitSizeCommand(AbstractVisitor.java:80)

              at org.infinispan.commands.read.SizeCommand.acceptVisitor(SizeCommand.java:35)

              at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:99)

              at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:113)

              at org.infinispan.commands.AbstractVisitor.visitSizeCommand(AbstractVisitor.java:80)

              at org.infinispan.commands.read.SizeCommand.acceptVisitor(SizeCommand.java:35)

              at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:99)

              at org.infinispan.interceptors.InvocationContextInterceptor.handleAll(InvocationContextInterceptor.java:114)

              at org.infinispan.interceptors.InvocationContextInterceptor.handleDefault(InvocationContextInterceptor.java:83)

              at org.infinispan.commands.AbstractVisitor.visitSizeCommand(AbstractVisitor.java:80)

              at org.infinispan.commands.read.SizeCommand.acceptVisitor(SizeCommand.java:35)

              at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:99)

              at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:113)

              at org.infinispan.commands.AbstractVisitor.visitSizeCommand(AbstractVisitor.java:80)

              at org.infinispan.commands.read.SizeCommand.acceptVisitor(SizeCommand.java:35)

              at org.infinispan.interceptors.InterceptorChain.invoke(InterceptorChain.java:335)

              at org.infinispan.cache.impl.CacheImpl.size(CacheImpl.java:370)

              at org.infinispan.cache.impl.CacheImpl.size(CacheImpl.java:365)

              at com.tecnotree.eia.caching.imdg.bd.impl.IMDGBusinessDeligateImpl.getCount(IMDGBusinessDeligateImpl.java:2479)

              at com.tecnotree.eia.caching.imdg.cache.CachingInterface.getCount(CachingInterface.java:788)

              at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)

              at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)

              at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)

              at java.lang.reflect.Method.invoke(Method.java:498)

              at com.sun.jersey.spi.container.JavaMethodInvokerFactory$1.invoke(JavaMethodInvokerFactory.java:60)

             at com.sun.jersey.server.impl.model.method.dispatch.AbstractResourceMethodDispatchProvider$TypeOutInvoker._dispatch(AbstractResourceMethodDispatchProvider.java:185)

              at com.sun.jersey.server.impl.model.method.dispatch.ResourceJavaMethodDispatcher.dispatch(ResourceJavaMethodDispatcher.java:75)

              at com.sun.jersey.server.impl.uri.rules.HttpMethodRule.accept(HttpMethodRule.java:302)

              at com.sun.jersey.server.impl.uri.rules.RightHandPathRule.accept(RightHandPathRule.java:147)

              at com.sun.jersey.server.impl.uri.rules.ResourceClassRule.accept(ResourceClassRule.java:108)

              at com.sun.jersey.server.impl.uri.rules.RightHandPathRule.accept(RightHandPathRule.java:147)

              at com.sun.jersey.server.impl.uri.rules.RootResourceClassesRule.accept(RootResourceClassesRule.java:84)

              at com.sun.jersey.server.impl.application.WebApplicationImpl._handleRequest(WebApplicationImpl.java:1542)

              at com.sun.jersey.server.impl.application.WebApplicationImpl._handleRequest(WebApplicationImpl.java:1473)

              at com.sun.jersey.server.impl.application.WebApplicationImpl.handleRequest(WebApplicationImpl.java:1419)

              at com.sun.jersey.server.impl.application.WebApplicationImpl.handleRequest(WebApplicationImpl.java:1409)

              at com.sun.jersey.spi.container.servlet.WebComponent.service(WebComponent.java:409)

              at com.sun.jersey.spi.container.servlet.ServletContainer.service(ServletContainer.java:558)

              at com.sun.jersey.spi.container.servlet.ServletContainer.service(ServletContainer.java:733)

              at javax.servlet.http.HttpServlet.service(HttpServlet.java:731)

              at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:303)

              at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)

              at org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:52)

              at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)

              at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)

              at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:218)

              at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:122)

              at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:505)

              at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:169)

              at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:103)

              at org.apache.catalina.valves.AccessLogValve.invoke(AccessLogValve.java:956)

              at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:116)

              at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:442)

              at org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:1082)

              at org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:623)

              at org.apache.tomcat.util.net.JIoEndpoint$SocketProcessor.run(JIoEndpoint.java:316)

              at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)

              at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)

              at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)

              at java.lang.Thread.run(Thread.java:745)

      ******************************************************************************************************

       

      Regards

      Binu

        • 1. Re: ISPN000136: Error executing command SizeCommand, writing keys
          william.burns

          What kind of configuration do you have?  The size command will have to read all of the contents of each node to verify how many live elements there are (remove expired).  This can be a slow process especially when you have a cache store.

           

          Also depending on how long it took JGroups to suspect this member, it could be that it took a bit longer than normal before JGroups removed it from the view (compounded with how long the size takes)

           

          These remote calls will timeout by default with the replication timeout that can be configured under the clustering configuration [1].  You can also optionally set this timeout on an invocation basis using the stream API by calling [2].  If you do the second the code could look like:

           

          cache.keySet().stream().timeout(2. TimeUnit.MINUTES).count();
          

           

          [1] ClusteringConfigurationBuilder (Infinispan JavaDoc 9.0.0.Alpha4 API)

          [2] CacheStream (Infinispan JavaDoc 9.0.0.Alpha4 API)

          • 2. Re: ISPN000136: Error executing command SizeCommand, writing keys
            binu_driver

            Hi,

             

              Regarding configuration, We are using Distributed Cache with below configuration.

             

              As part of POC, we needs to get the number of element in cache[collectively, not individually]. So, we have used the below code to get the cache size.

              cache1.size()

             

              Regarding remote-timeout, we are using the default remote-timeout[15000ms]. we will try testing after increasing the timeout value.

             

            ******************************************************************************************************

            <?xml version="1.0" encoding="UTF-8"?>

            <infinispan xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"

              xsi:schemaLocation="urn:infinispan:config:8.2 http://www.infinispan.org/schemas/infinispan-config-8.2.xsd"

              xmlns="urn:infinispan:config:8.2">

             

              <jgroups>

              <stack-file name="udp" path="jgroups-udp.xml" />

              </jgroups>

             

              <cache-container default-cache="default">

              <transport stack="udp" node-name="TOMCAT-1-1" cluster="cluster1" />

             

              <distributed-cache name="cache1" mode="SYNC" owners="2" remote-timeout="300000" >

              <state-transfer timeout="300000" await-initial-transfer="true" />

              <locking acquire-timeout="60000" striping="false" isolation="READ_COMMITTED" />

              <transaction mode ="NONE" />

              <indexing index="LOCAL" auto-config="true" >

              <property name="default.directory_provider">ram</property>

              <property name="lucene_version">LUCENE_CURRENT</property>

              </indexing>

              </distributed-cache>

              <!-- List of other caches with same configuration-->

             

             

              </cache-container>

            </infinispan>

            ******************************************************************************************************

            jgroups-udp.xml:

             

             

            <?xml version="1.0" encoding="UTF-8"?>

            <config xmlns="urn:org:jgroups" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"

              xsi:schemaLocation="urn:org:jgroups http://www.jgroups.org/schema/JGroups-3.0.xsd">

              <UDP

              mcast_port="${jgroups.udp.mcast_port:45588}" tos="8"

              ucast_recv_buf_size="20M" ucast_send_buf_size="640K"

              mcast_recv_buf_size="25M" mcast_send_buf_size="640K" loopback="true"

              max_bundle_size="64K" max_bundle_timeout="30" ip_ttl="${jgroups.udp.ip_ttl:2}"

              enable_diagnostics="true" thread_naming_pattern="cl" timer_type="new"

              timer.min_threads="4" timer.max_threads="10" timer.keep_alive_time="3000"

              timer.queue_max_size="500" thread_pool.enabled="true"

              thread_pool.min_threads="2" thread_pool.max_threads="8"

              thread_pool.keep_alive_time="5000" thread_pool.queue_enabled="true"

              thread_pool.queue_max_size="10000" thread_pool.rejection_policy="discard"

              oob_thread_pool.enabled="true" oob_thread_pool.min_threads="1"

              oob_thread_pool.max_threads="8" oob_thread_pool.keep_alive_time="5000"

              oob_thread_pool.queue_enabled="false" oob_thread_pool.queue_max_size="100"

              oob_thread_pool.rejection_policy="Run" />

              <PING timeout="2000" num_initial_members="3" />

              <MERGE2 max_interval="30000" min_interval="10000" />

              <FD_SOCK />

              <FD_ALL />

              <VERIFY_SUSPECT timeout="1500" />

              <BARRIER />

              <CENTRAL_LOCK num_backups="2"/>

              <pbcast.NAKACK use_mcast_xmit="true" retransmit_timeout="300,600,1200"

              discard_delivered_msgs="true" />

              <UNICAST />

              <pbcast.STABLE stability_delay="1000" desired_avg_gossip="50000"

              max_bytes="4M" />

              <pbcast.GMS print_local_addr="true" join_timeout="3000"

              view_bundling="true" />

              <UFC max_credits="2M" min_threshold="0.4" />

              <MFC max_credits="2M" min_threshold="0.4" />

              <FRAG2 frag_size="60K" />

              <pbcast.STATE />

            </config>

            ******************************************************************************************************

             

            Regards,

            Binu