8 Replies Latest reply on Sep 6, 2017 11:53 AM by galder.zamarreno

    Receiving "failed unmarshalling buffer into return value" error during clusters connection

    kikosha

      Hi

       

      I'm using distributions:

       

      infinispan-7.0.2.Final-all

      infinispan-server-7.0.2.Final-bin

       

      I've started two nodes, one as a standalone server and one as an embedded server, both running on the same machine.

       

      The standalone node is started using:

       

      bin/standalone.sh -c clustered.xml -Djboss.node.name=nodeB


      and clustered.xml content is:

       

      
      
      <?xml version="1.0" ?>
      
      <server xmlns="urn:jboss:domain:2.1">
          <extensions>
              <extension module="org.infinispan.server.endpoint" />
              <extension module="org.jboss.as.clustering.infinispan" />
              <extension module="org.jboss.as.clustering.jgroups" />
              <extension module="org.jboss.as.connector" />
              <extension module="org.jboss.as.deployment-scanner" />
              <extension module="org.jboss.as.jdr" />
              <extension module="org.jboss.as.jmx" />
              <extension module="org.jboss.as.logging" />
              <extension module="org.jboss.as.modcluster" />
              <extension module="org.jboss.as.naming" />
              <extension module="org.jboss.as.remoting" />
              <extension module="org.jboss.as.security" />
              <extension module="org.jboss.as.threads" />
              <extension module="org.jboss.as.transactions" />
              <extension module="org.wildfly.extension.io" />
              <extension module="org.wildfly.extension.undertow" />
          </extensions>
          <management>
              <security-realms>
                  <security-realm name="ManagementRealm">
                      <authentication>
                          <local default-user="$local" skip-group-loading="true" />
                          <properties path="mgmt-users.properties" relative-to="jboss.server.config.dir" />
                      </authentication>
                      <authorization map-groups-to-roles="false">
                          <properties path="mgmt-groups.properties" relative-to="jboss.server.config.dir" />
                      </authorization>
                  </security-realm>
                  <security-realm name="ApplicationRealm">
                      <authentication>
                          <local default-user="$local" allowed-users="*"
                              skip-group-loading="true" />
                          <properties path="application-users.properties"
                              relative-to="jboss.server.config.dir" />
                      </authentication>
                      <authorization>
                          <properties path="application-roles.properties"
                              relative-to="jboss.server.config.dir" />
                      </authorization>
                  </security-realm>
              </security-realms>
              <audit-log>
                  <formatters>
                      <json-formatter name="json-formatter" />
                  </formatters>
                  <handlers>
                      <file-handler name="file" formatter="json-formatter"
                          relative-to="jboss.server.data.dir" path="audit-log.log" />
                  </handlers>
                  <logger log-boot="true" log-read-only="false" enabled="true">
                      <handlers>
                          <handler name="file" />
                      </handlers>
                  </logger>
              </audit-log>
              <management-interfaces>
                  <http-interface security-realm="ManagementRealm"
                      http-upgrade-enabled="true">
                      <socket-binding http="management-http" />
                  </http-interface>
              </management-interfaces>
              <access-control provider="simple">
                  <role-mapping>
                      <role name="SuperUser">
                          <include>
                              <user name="$local" />
                          </include>
                      </role>
                  </role-mapping>
              </access-control>
          </management>
          <profile>
              <subsystem xmlns="urn:jboss:domain:logging:2.0">
                  <console-handler name="CONSOLE">
                      <level name="INFO" />
                      <formatter>
                          <named-formatter name="COLOR-PATTERN" />
                      </formatter>
                  </console-handler>
                  <periodic-rotating-file-handler name="FILE"
                      autoflush="true">
                      <formatter>
                          <named-formatter name="PATTERN" />
                      </formatter>
                      <file relative-to="jboss.server.log.dir" path="server.log" />
                      <suffix value=".yyyy-MM-dd" />
                      <append value="true" />
                  </periodic-rotating-file-handler>
                  <logger category="com.arjuna">
                      <level name="DEBUG" />
                  </logger>
                  <logger category="org.apache.tomcat.util.modeler">
                      <level name="WARN" />
                  </logger>
                  <logger category="org.jboss.as.config">
                      <level name="DEBUG" />
                  </logger>
                  <logger category="sun.rmi">
                      <level name="WARN" />
                  </logger>
                  <logger category="jacorb">
                      <level name="WARN" />
                  </logger>
                  <logger category="jacorb.config">
                      <level name="DEBUG" />
                  </logger>
                  <root-logger>
                      <level name="DEBUG" />
                      <handlers>
                          <handler name="CONSOLE" />
                          <handler name="FILE" />
                      </handlers>
                  </root-logger>
                  <formatter name="PATTERN">
                      <pattern-formatter
                          pattern="%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p [%c] (%t) %s%E%n" />
                  </formatter>
                  <formatter name="COLOR-PATTERN">
                      <pattern-formatter pattern="%K{level}%d{HH:mm:ss,SSS} %-5p [%c] (%t) %s%E%n" />
                  </formatter>
              </subsystem>
              <subsystem xmlns="urn:jboss:domain:deployment-scanner:2.0">
                  <deployment-scanner path="deployments"
                      relative-to="jboss.server.base.dir" scan-interval="5000"
                      runtime-failure-causes-rollback="${jboss.deployment.scanner.rollback.on.failure:false}" />
              </subsystem>
              <subsystem xmlns="urn:infinispan:server:endpoint:7.0">
                  <hotrod-connector socket-binding="hotrod"
                      cache-container="clustered">
                      <topology-state-transfer lazy-retrieval="true"
                          lock-timeout="1000" replication-timeout="5000" />
                  </hotrod-connector>
              </subsystem>
              <subsystem xmlns="urn:jboss:domain:datasources:2.0">
                  <datasources />
              </subsystem>
              <subsystem xmlns="urn:infinispan:server:core:7.0"
                  default-cache-container="clustered">
                  <cache-container name="clustered" default-cache="myDistCache"
                      statistics="true">
                      <transport executor="infinispan-transport" lock-timeout="60000" />
                      <distributed-cache name="myDistCache" mode="ASYNC"
                          segments="20" owners="1" remote-timeout="30000" start="LAZY">
                          <locking acquire-timeout="30000" concurrency-level="1000"
                              striping="false" />
                          <transaction mode="NONE" />
                      </distributed-cache>
                  </cache-container>
                  <cache-container name="security" />
              </subsystem>
              <subsystem xmlns="urn:jboss:domain:io:1.1">
                  <worker name="default" />
                  <buffer-pool name="default" />
              </subsystem>
              <subsystem xmlns="urn:jboss:domain:jca:2.0">
                  <archive-validation enabled="true" fail-on-error="true"
                      fail-on-warn="false" />
                  <bean-validation enabled="true" />
                  <default-workmanager>
                      <short-running-threads>
                          <core-threads count="50" />
                          <queue-length count="50" />
                          <max-threads count="50" />
                          <keepalive-time time="10" unit="seconds" />
                      </short-running-threads>
                      <long-running-threads>
                          <core-threads count="50" />
                          <queue-length count="50" />
                          <max-threads count="50" />
                          <keepalive-time time="10" unit="seconds" />
                      </long-running-threads>
                  </default-workmanager>
                  <cached-connection-manager />
              </subsystem>
              <subsystem xmlns="urn:jboss:domain:jdr:1.0" />
              <subsystem xmlns="urn:infinispan:server:jgroups:7.0"
                  default-stack="${jboss.default.jgroups.stack:tcp}">
                  <stack name="udp">
                      <transport type="UDP" socket-binding="jgroups-udp" />
                      <protocol type="PING" />
                      <protocol type="MERGE3" />
                      <protocol type="FD_SOCK" socket-binding="jgroups-udp-fd" />
                      <protocol type="FD_ALL" />
                      <protocol type="VERIFY_SUSPECT" />
                      <protocol type="pbcast.NAKACK2" />
                      <protocol type="UNICAST3" />
                      <protocol type="pbcast.STABLE" />
                      <protocol type="pbcast.GMS" />
                      <protocol type="UFC" />
                      <protocol type="MFC" />
                      <protocol type="FRAG2" />
                      <protocol type="RSVP" />
                  </stack>
                  <stack name="tcp">
                      <transport type="TCP" socket-binding="jgroups-tcp" />
                      <protocol type="MPING" socket-binding="jgroups-mping" />
                      <protocol type="MERGE3" />
                      <protocol type="FD_SOCK" socket-binding="jgroups-tcp-fd" />
                      <protocol type="FD" />
                      <protocol type="VERIFY_SUSPECT" />
                      <protocol type="pbcast.NAKACK2">
                          <property name="use_mcast_xmit">false</property>
                      </protocol>
                      <protocol type="UNICAST3" />
                      <protocol type="pbcast.STABLE" />
                      <protocol type="pbcast.GMS" />
                      <protocol type="MFC" />
                      <protocol type="FRAG2" />
                      <protocol type="RSVP" />
                  </stack>
                  <stack name="s3">
                      <transport type="TCP" socket-binding="jgroups-tcp" />
                      <protocol type="S3_PING">
                          <property name="location">${jgroups.s3.bucket}</property>
                          <property name="access_key">${jgroups.s3.access_key}</property>
                          <property name="secret_access_key">${jgroups.s3.secret_access_key}</property>
                          <property name="pre_signed_delete_url">${jgroups.s3.pre_signed_delete_url}
                          </property>
                          <property name="pre_signed_put_url">${jgroups.s3.pre_signed_put_url}</property>
                          <property name="prefix">${jgroups.s3.prefix}</property>
                      </protocol>
                      <protocol type="MERGE3" />
                      <protocol type="FD_SOCK" socket-binding="jgroups-tcp-fd" />
                      <protocol type="FD" />
                      <protocol type="VERIFY_SUSPECT" />
                      <protocol type="pbcast.NAKACK2" />
                      <protocol type="UNICAST3" />
                      <protocol type="pbcast.STABLE" />
                      <protocol type="pbcast.GMS" />
                      <protocol type="MFC" />
                      <protocol type="FRAG2" />
                      <protocol type="RSVP" />
                  </stack>
              </subsystem>
              <subsystem xmlns="urn:jboss:domain:jmx:1.3">
                  <expose-resolved-model />
                  <expose-expression-model />
                  <remoting-connector />
              </subsystem>
              <subsystem xmlns="urn:jboss:domain:modcluster:1.2">
                  <mod-cluster-config advertise-socket="modcluster"
                      connector="ajp">
                      <dynamic-load-provider>
                          <load-metric type="cpu" />
                      </dynamic-load-provider>
                  </mod-cluster-config>
              </subsystem>
              <subsystem xmlns="urn:jboss:domain:naming:2.0">
                  <remote-naming />
              </subsystem>
              <subsystem xmlns="urn:jboss:domain:remoting:2.0">
                  <endpoint worker="default" />
                  <http-connector name="http-remoting-connector"
                      connector-ref="default" security-realm="ApplicationRealm" />
              </subsystem>
              <subsystem xmlns="urn:jboss:domain:security:1.2">
                  <security-domains>
                      <security-domain name="other" cache-type="default">
                          <authentication>
                              <login-module code="Remoting" flag="optional">
                                  <module-option name="password-stacking" value="useFirstPass" />
                              </login-module>
                              <login-module code="RealmDirect" flag="required">
                                  <module-option name="password-stacking" value="useFirstPass" />
                              </login-module>
                          </authentication>
                      </security-domain>
                      <security-domain name="jboss-web-policy" cache-type="default">
                          <authorization>
                              <policy-module code="Delegating" flag="required" />
                          </authorization>
                      </security-domain>
                  </security-domains>
              </subsystem>
              <subsystem xmlns="urn:jboss:domain:threads:1.1">
                  <thread-factory name="infinispan-factory" group-name="infinispan"
                      priority="5" />
                  <unbounded-queue-thread-pool name="infinispan-transport">
                      <max-threads count="25" />
                      <keepalive-time time="0" unit="milliseconds" />
                      <thread-factory name="infinispan-factory" />
                  </unbounded-queue-thread-pool>
              </subsystem>
              <subsystem xmlns="urn:jboss:domain:transactions:2.0">
                  <core-environment>
                      <process-id>
                          <uuid />
                      </process-id>
                  </core-environment>
                  <recovery-environment socket-binding="txn-recovery-environment"
                      status-socket-binding="txn-status-manager" />
              </subsystem>
              <subsystem xmlns="urn:jboss:domain:undertow:1.1">
                  <buffer-cache name="default" />
                  <server name="default-server">
                      <http-listener name="default" socket-binding="http" />
                      <host name="default-host" alias="localhost">
                          <filter-ref name="server-header" />
                          <filter-ref name="x-powered-by-header" />
                      </host>
                  </server>
                  <servlet-container name="default" />
                  <handlers />
                  <filters>
                      <response-header name="server-header" header-name="Server"
                          header-value="WildFly/8" />
                      <response-header name="x-powered-by-header"
                          header-name="X-Powered-By" header-value="Undertow/1" />
                  </filters>
              </subsystem>
          </profile>
          <interfaces>
              <interface name="management">
                  <inet-address value="${jboss.bind.address.management:127.0.0.1}" />
              </interface>
              <interface name="public">
                  <inet-address value="${jboss.bind.address:127.0.0.1}" />
              </interface>
          </interfaces>
          <socket-binding-group name="standard-sockets"
              default-interface="public" port-offset="${jboss.socket.binding.port-offset:0}">
              <socket-binding name="management-http" interface="management"
                  port="${jboss.management.http.port:9990}" />
              <socket-binding name="management-https" interface="management"
                  port="${jboss.management.https.port:9993}" />
              <socket-binding name="ajp" port="${jboss.ajp.port:8009}" />
              <socket-binding name="hotrod" port="11222" />
              <socket-binding name="http" port="${jboss.http.port:8080}" />
              <socket-binding name="https" port="${jboss.https.port:8443}" />
              <socket-binding name="jgroups-mping" port="0"
                  multicast-address="${jboss.default.multicast.address:228.2.4.6}"
                  multicast-port="43366" />
              <socket-binding name="jgroups-tcp" port="7600" />
              <socket-binding name="jgroups-tcp-fd" port="57600" />
              <socket-binding name="jgroups-udp" port="55200"
                  multicast-address="${jboss.default.multicast.address:234.99.54.14}"
                  multicast-port="45688" />
              <socket-binding name="jgroups-udp-fd" port="54200" />
              <socket-binding name="memcached" port="11211" />
              <socket-binding name="modcluster" port="0"
                  multicast-address="224.0.1.115" multicast-port="23364" />
              <socket-binding name="txn-recovery-environment" port="4712" />
              <socket-binding name="txn-status-manager" port="4713" />
              <socket-binding name="websocket" port="8181" />
          </socket-binding-group>
      </server>
      
      
      


      The embedded node is started from within Eclipse, using the following code:


      public static void main(String[] args)
          {
              Configuration config = new ConfigurationBuilder().clustering().cacheMode(CacheMode.DIST_ASYNC).build();
             
              EmbeddedCacheManager manager;
              try
              {
                  manager = new DefaultCacheManager("sample.xml");
             
                  manager.defineConfiguration("myDistCache", config);
                  manager.addListener(new ListenToAll());
                  Cache<String, String> defaultCache = manager.getCache("myDistCache");
                  defaultCache.addListener(new ListenToAll());
                  System.out.println("Got cache: " + defaultCache.getName());
                 
                  defaultCache.start();
                 
                  getAndPrint("key13", defaultCache);
                  defaultCache.put("key13", "value13");
                  getAndPrint("key13", defaultCache);
              }
              catch (IOException e)
              {
                  // TODO Auto-generated catch block
                  e.printStackTrace();
              }
          }
      
      


      samle.xml configuration file content is:

       

      
      
      <?xml version="1.0" encoding="UTF-8"?>
      <infinispan xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
          xsi:schemaLocation="urn:infinispan:config:7.0 http://www.infinispan.org/schemas/infinispan-config-7.0.xsd"
          xmlns="urn:infinispan:config:7.0">
      
          <jgroups>
              <stack-file name="tcp" path="jgroups-tcp.xml" />
          </jgroups>
      
          <cache-container name="SampleCacheManager" statistics="true"
              default-cache="myDistCache" shutdown-hook="DEFAULT">
      
              <transport stack="tcp" cluster="clustered" node-name="Node-A"
                  machine="m1" rack="r1" site="LON" />
      
              <serialization marshaller="org.infinispan.marshall.core.VersionAwareMarshaller"
                  version="1.0">
              </serialization>
      
              <jmx domain="org.infinispan" />
      
              <invalidation-cache name="invalidatedWithClusterCacheLoaderCache"
                  mode="SYNC" remote-timeout="20000">
                  <persistence>
                      <cluster-loader remote-timeout="20000" preload="false" />
                  </persistence>
              </invalidation-cache>
      
      
              <distributed-cache name="myDistCache" mode="SYNC"
                  owners="2" l1-lifespan="600000" />
      
          </cache-container>
      </infinispan>
      
      
      


      The standalone server is started firs, the embedded one is started after the standalone one finishes to initialize.


      When the clusters start the handshake, the embedded one produces the following log:


      =========================================================================

       

        JBoss Bootstrap Environment

       

        JBOSS_HOME: /home/kikosha/Apps/Infinispan/infinispan-server-7.0.2.Final

       

        JAVA: java

       

        JAVA_OPTS:  -server -Xms64m -Xmx512m -XX:MaxPermSize=256m -Djava.net.preferIPv4Stack=true -Djboss.modules.system.pkgs=org.jboss.byteman -Djava.awt.headless=true

       

      =========================================================================

       

      14:01:36,809 INFO  [org.jboss.modules] (main) JBoss Modules version 1.3.3.Final

      14:01:37,070 INFO  [org.jboss.msc] (main) JBoss MSC version 1.2.2.Final

      14:01:37,150 INFO  [org.jboss.as] (MSC service thread 1-6) JBAS015899: JBoss Infinispan Server 7.0.2.Final (WildFly 8.1.0.Final) starting

      14:01:37,905 INFO  [org.jboss.as.server] (Controller Boot Thread) JBAS015888: Creating http management service using socket-binding (management-http)

      14:01:37,917 INFO  [org.xnio] (MSC service thread 1-15) XNIO version 3.2.2.Final

      14:01:37,924 INFO  [org.xnio.nio] (MSC service thread 1-15) XNIO NIO Implementation Version 3.2.2.Final

      14:01:37,935 INFO  [org.jboss.as.clustering.infinispan] (ServerService Thread Pool -- 21) JBAS010280: Activating Infinispan subsystem.

      14:01:37,936 INFO  [org.wildfly.extension.io] (ServerService Thread Pool -- 22) WFLYIO001: Worker 'default' has auto-configured to 16 core threads with 128 task threads based on your 8 available processors

      14:01:37,949 INFO  [org.jboss.as.clustering.jgroups] (ServerService Thread Pool -- 25) JBAS010260: Activating JGroups subsystem.

      14:01:37,956 INFO  [org.jboss.as.security] (ServerService Thread Pool -- 30) JBAS013171: Activating Security Subsystem

      14:01:37,958 INFO  [org.jboss.as.naming] (ServerService Thread Pool -- 28) JBAS011800: Activating Naming Subsystem

      14:01:37,960 WARN  [org.jboss.as.txn] (ServerService Thread Pool -- 32) JBAS010153: Node identifier property is set to the default value. Please make sure it is unique.                                                                                                     

      14:01:37,966 INFO  [org.jboss.as.connector.logging] (MSC service thread 1-1) JBAS010408: Starting JCA Subsystem (IronJacamar 1.1.5.Final)

      14:01:37,974 INFO  [org.jboss.as.security] (MSC service thread 1-14) JBAS013170: Current PicketBox version=4.0.21.Beta1

      14:01:37,984 INFO  [org.jboss.remoting] (MSC service thread 1-15) JBoss Remoting version 4.0.3.Final

      14:01:38,007 INFO  [org.wildfly.extension.undertow] (ServerService Thread Pool -- 33) JBAS017502: Undertow 1.0.15.Final starting

      14:01:38,007 INFO  [org.wildfly.extension.undertow] (MSC service thread 1-9) JBAS017502: Undertow 1.0.15.Final starting

      14:01:38,024 INFO  [org.jboss.as.naming] (MSC service thread 1-7) JBAS011802: Starting Naming Service

      14:01:38,076 INFO  [org.wildfly.extension.undertow] (MSC service thread 1-2) JBAS017525: Started server default-server.

      14:01:38,084 INFO  [com.arjuna.ats.jbossatx] (MSC service thread 1-13) ARJUNA032010: JBossTS Recovery Service (tag: 9aa71) - JBoss Inc.

      14:01:38,092 INFO  [com.arjuna.ats.arjuna] (MSC service thread 1-13) ARJUNA012324: Start RecoveryActivators

      14:01:38,095 INFO  [com.arjuna.ats.arjuna] (Transaction Expired Entry Monitor) ARJUNA012296: ExpiredEntryMonitor running at Wed, 3 Dec 2014 14:01:38

      14:01:38,117 INFO  [com.arjuna.ats.arjuna] (MSC service thread 1-13) ARJUNA012343: RecoveryManagerImple is ready. Socket listener is turned off.

      14:01:38,117 INFO  [com.arjuna.ats.jbossatx] (MSC service thread 1-13) ARJUNA032013: Starting transaction recovery manager

      14:01:38,126 INFO  [org.wildfly.extension.undertow] (MSC service thread 1-7) JBAS017519: Undertow HTTP listener default listening on /127.0.0.1:8080

      14:01:38,152 INFO  [com.arjuna.ats.jbossatx] (MSC service thread 1-13) ARJUNA032017: JBossTS Transaction Service (JTA version - tag: 9aa71) - JBoss Inc.

      14:01:38,382 INFO  [org.jboss.as.server.deployment.scanner] (MSC service thread 1-11) JBAS015012: Started FileSystemDeploymentService for directory /home/kikosha/Apps/Infinispan/infinispan-server-7.0.2.Final/standalone/deployments

      14:01:38,763 INFO  [org.infinispan.remoting.transport.jgroups.JGroupsTransport] (MSC service thread 1-14) ISPN000078: Starting JGroups channel clustered

      14:01:38,775 INFO  [stdout] (MSC service thread 1-14)

      14:01:38,775 INFO  [stdout] (MSC service thread 1-14) -------------------------------------------------------------------

      14:01:38,775 INFO  [stdout] (MSC service thread 1-14) GMS: address=nodeB/clustered, cluster=clustered, physical address=127.0.0.1:7600

      14:01:38,775 INFO  [stdout] (MSC service thread 1-14) -------------------------------------------------------------------

      14:01:41,788 INFO  [org.infinispan.remoting.transport.jgroups.JGroupsTransport] (MSC service thread 1-14) ISPN000094: Received new cluster view for channel clustered: [nodeB/clustered|0] (1) [nodeB/clustered]

      14:01:41,900 INFO  [org.infinispan.remoting.transport.jgroups.JGroupsTransport] (MSC service thread 1-14) ISPN000079: Channel clustered local address is nodeB/clustered, physical addresses are [127.0.0.1:7600]

      14:01:41,904 INFO  [org.infinispan.factories.GlobalComponentRegistry] (MSC service thread 1-14) ISPN000128: Infinispan version: Infinispan 'Guinness' 7.0.2.Final

      14:01:42,083 INFO  [org.jboss.as.clustering.infinispan] (MSC service thread 1-14) JBAS010281: Started ___protobuf_metadata cache from clustered container

      14:01:42,089 INFO  [org.infinispan.eviction.impl.EvictionManagerImpl] (MSC service thread 1-14) ISPN000025: wakeUpInterval is <= 0, not starting expired purge thread

      14:01:42,093 INFO  [org.jboss.as.clustering.infinispan] (MSC service thread 1-14) JBAS010281: Started myDistCache cache from clustered container

      14:01:42,095 INFO  [org.infinispan.server.endpoint] (MSC service thread 1-4) JDGS010000: HotRodServer starting

      14:01:42,096 INFO  [org.infinispan.server.endpoint] (MSC service thread 1-4) JDGS010001: HotRodServer listening on 127.0.0.1:11222

      14:01:42,250 INFO  [org.jboss.as.clustering.infinispan] (MSC service thread 1-4) JBAS010281: Started ___hotRodTopologyCache cache from clustered container

      14:01:42,425 INFO  [org.jboss.as] (Controller Boot Thread) JBAS015961: Http management interface listening on http://127.0.0.1:9990/management

      14:01:42,426 INFO  [org.jboss.as] (Controller Boot Thread) JBAS015951: Admin console listening on http://127.0.0.1:9990

      14:01:42,426 INFO  [org.jboss.as] (Controller Boot Thread) JBAS015874: JBoss Infinispan Server 7.0.2.Final (WildFly 8.1.0.Final) started in 5911ms - Started 142 of 162 services (58 services are lazy, passive or on-demand)

       

      14:01:59,314 INFO  [org.infinispan.remoting.transport.jgroups.JGroupsTransport] (Incoming-2,shared=tcp) ISPN000094: Received new cluster view for channel clustered: [nodeB/clustered|1] (2) [nodeB/clustered, Node-A-41319, site-id=LON, rack-id=r1, machine-id=m1]

      14:01:59,598 INFO  [org.infinispan.CLUSTER] (remote-thread--p3-t1) ISPN000310: Starting cluster-wide rebalance for cache myDistCache, topology CacheTopology{id=1, rebalanceId=1, currentCH=DefaultConsistentHash{ns = 20, owners = (1)[nodeB/clustered: 20+0]}, pendingCH=DefaultConsistentHash{ns = 20, owners = (2)[nodeB/clustered: 10+0, Node-A-41319, site-id=LON, rack-id=r1, machine-id=m1: 10+0]}, unionCH=null}

      14:01:59,603 INFO  [org.infinispan.CLUSTER] (undefined) ISPN000328: Finished local rebalance for cache myDistCache on node nodeB/clustered, topology id = 1

      14:02:04,744 INFO  [org.infinispan.remoting.transport.jgroups.JGroupsTransport] (Incoming-4,shared=tcp) ISPN000094: Received new cluster view for channel clustered: [nodeB/clustered|2] (1) [nodeB/clustered]

      14:02:04,750 INFO  [org.infinispan.CLUSTER] (undefined) ISPN000328: Finished local rebalance for cache myDistCache on node nodeB/clustered, topology id = 2

       

      And the embedded one produces this:

       

      2014-12-03 14:01:58,948 INFO  org.infinispan.remoting.transport.jgroups.JGroupsTransport.start(JGroupsTransport.java:187) (main) ISPN000078: Starting JGroups channel clustered

      2014-12-03 14:01:59,071 WARN  org.jgroups.stack.Configurator.resolveAndAssignField(Configurator.java:1140) (main) JGRP000014: TP.loopback has been deprecated: enabled by default

      2014-12-03 14:01:59,080 WARN  org.jgroups.stack.Configurator.resolveAndAssignField(Configurator.java:1140) (main) JGRP000014: Discovery.timeout has been deprecated: GMS.join_timeout should be used instead

      2014-12-03 14:01:59,080 WARN  org.jgroups.stack.Configurator.resolveAndAssignField(Configurator.java:1140) (main) JGRP000014: Discovery.num_initial_members has been deprecated: will be ignored

       

      -------------------------------------------------------------------

      GMS: address=Node-A-41319, site-id=LON, rack-id=r1, machine-id=m1, cluster=clustered, physical address=10.194.5.123:7800

      -------------------------------------------------------------------

      2014-12-03 14:01:59,327 INFO  org.infinispan.remoting.transport.jgroups.JGroupsTransport.viewAccepted(JGroupsTransport.java:680) (main) ISPN000094: Received new cluster view for channel clustered: [nodeB/clustered|1] (2) [nodeB/clustered, Node-A-41319, site-id=LON, rack-id=r1, machine-id=m1]

      View Changed.  Details = EventImpl{type=VIEW_CHANGED, newMembers=[nodeB/clustered, Node-A-41319, site-id=LON, rack-id=r1, machine-id=m1], oldMembers=null, localAddress=Node-A-41319, site-id=LON, rack-id=r1, machine-id=m1, viewId=1, subgroupsMerged=null, mergeView=false}

      2014-12-03 14:01:59,439 INFO  org.infinispan.remoting.transport.jgroups.JGroupsTransport.startJGroupsChannelIfNeeded(JGroupsTransport.java:225) (main) ISPN000079: Channel clustered local address is Node-A-41319, site-id=LON, rack-id=r1, machine-id=m1, physical addresses are [10.194.5.123:7800]

      2014-12-03 14:01:59,853 ERROR org.jgroups.blocks.RequestCorrelator.receiveMessage(RequestCorrelator.java:394) (OOB-2,clustered,Node-A-41319, site-id=LON, rack-id=r1, machine-id=m1) failed unmarshalling buffer into return value

      java.io.EOFException

          at org.jboss.marshalling.UTFUtils.readUTFBytes(UTFUtils.java:153)

          at org.jboss.marshalling.river.RiverUnmarshaller.readString(RiverUnmarshaller.java:1226)

          at org.jboss.marshalling.river.RiverUnmarshaller.doReadNewObject(RiverUnmarshaller.java:1325)

          at org.jboss.marshalling.river.RiverUnmarshaller.doReadObject(RiverUnmarshaller.java:272)

          at org.jboss.marshalling.river.RiverUnmarshaller.doReadObject(RiverUnmarshaller.java:209)

          at org.jboss.marshalling.AbstractObjectInput.readObject(AbstractObjectInput.java:41)

          at org.infinispan.topology.CacheStatusResponse$Externalizer.readObject(CacheStatusResponse.java:74)

          at org.infinispan.topology.CacheStatusResponse$Externalizer.readObject(CacheStatusResponse.java:60)

          at org.infinispan.marshall.core.ExternalizerTable$ExternalizerAdapter.readObject(ExternalizerTable.java:424)

          at org.infinispan.marshall.core.ExternalizerTable.readObject(ExternalizerTable.java:221)

          at org.infinispan.marshall.core.JBossMarshaller$ExternalizerTableProxy.readObject(JBossMarshaller.java:148)

          at org.jboss.marshalling.river.RiverUnmarshaller.doReadObject(RiverUnmarshaller.java:351)

          at org.jboss.marshalling.river.RiverUnmarshaller.doReadObject(RiverUnmarshaller.java:209)

          at org.jboss.marshalling.AbstractObjectInput.readObject(AbstractObjectInput.java:41)

          at org.infinispan.remoting.responses.SuccessfulResponse$Externalizer.readObject(SuccessfulResponse.java:79)

          at org.infinispan.remoting.responses.SuccessfulResponse$Externalizer.readObject(SuccessfulResponse.java:64)

          at org.infinispan.marshall.core.ExternalizerTable$ExternalizerAdapter.readObject(ExternalizerTable.java:424)

          at org.infinispan.marshall.core.ExternalizerTable.readObject(ExternalizerTable.java:221)

          at org.infinispan.marshall.core.JBossMarshaller$ExternalizerTableProxy.readObject(JBossMarshaller.java:148)

          at org.jboss.marshalling.river.RiverUnmarshaller.doReadObject(RiverUnmarshaller.java:351)

          at org.jboss.marshalling.river.RiverUnmarshaller.doReadObject(RiverUnmarshaller.java:209)

          at org.jboss.marshalling.AbstractObjectInput.readObject(AbstractObjectInput.java:41)

          at org.infinispan.commons.marshall.jboss.AbstractJBossMarshaller.objectFromObjectStream(AbstractJBossMarshaller.java:135)

          at org.infinispan.marshall.core.VersionAwareMarshaller.objectFromByteBuffer(VersionAwareMarshaller.java:101)

          at org.infinispan.commons.marshall.AbstractDelegatingMarshaller.objectFromByteBuffer(AbstractDelegatingMarshaller.java:80)

          at org.infinispan.remoting.transport.jgroups.MarshallerAdapter.objectFromBuffer(MarshallerAdapter.java:28)

          at org.jgroups.blocks.RequestCorrelator.receiveMessage(RequestCorrelator.java:390)

          at org.jgroups.blocks.RequestCorrelator.receive(RequestCorrelator.java:250)

          at org.jgroups.blocks.MessageDispatcher$ProtocolAdapter.up(MessageDispatcher.java:677)

          at org.jgroups.JChannel.up(JChannel.java:733)

          at org.jgroups.stack.ProtocolStack.up(ProtocolStack.java:1029)

          at org.jgroups.protocols.pbcast.STATE_TRANSFER.up(STATE_TRANSFER.java:146)

          at org.jgroups.protocols.FRAG2.up(FRAG2.java:165)

          at org.jgroups.protocols.FlowControl.up(FlowControl.java:394)

          at org.jgroups.protocols.pbcast.GMS.up(GMS.java:1042)

          at org.jgroups.protocols.pbcast.STABLE.up(STABLE.java:234)

          at org.jgroups.protocols.UNICAST3.deliverMessage(UNICAST3.java:1064)

          at org.jgroups.protocols.UNICAST3.handleDataReceived(UNICAST3.java:779)

          at org.jgroups.protocols.UNICAST3.up(UNICAST3.java:426)

          at org.jgroups.protocols.pbcast.NAKACK.up(NAKACK.java:638)

          at org.jgroups.protocols.BARRIER.up(BARRIER.java:152)

          at org.jgroups.protocols.VERIFY_SUSPECT.up(VERIFY_SUSPECT.java:155)

          at org.jgroups.protocols.FD_ALL.up(FD_ALL.java:200)

          at org.jgroups.protocols.FD_SOCK.up(FD_SOCK.java:297)

          at org.jgroups.protocols.MERGE3.up(MERGE3.java:288)

          at org.jgroups.protocols.Discovery.up(Discovery.java:277)

          at org.jgroups.protocols.MPING.up(MPING.java:178)

          at org.jgroups.protocols.TP.passMessageUp(TP.java:1572)

          at org.jgroups.protocols.TP$MyHandler.run(TP.java:1791)

          at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)

          at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)

          at java.lang.Thread.run(Thread.java:745)

      Caused by: an exception which occurred:

          in object of type org.infinispan.partitionhandling.AvailabilityMode

              -> classloader hierarchy:

              -> type classloader = sun.misc.Launcher$AppClassLoader@6132b73b

              ->...file:/home/kikosha/Apps/Infinispan/workspace/InfinispanCluster/bin/

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/infinispan-embedded-7.0.2.Final.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/infinispan-embedded-query-7.0.2.Final.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/infinispan-remote-7.0.2.Final.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/activation-1.1.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/asm-2.2.3.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/asm-commons-2.2.3.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/asm-tree-2.2.3.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/avalon-framework-4.1.3.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/cache-api-1.0.0.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/commons-codec-1.4.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/commons-io-1.3.2.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/commons-lang-2.1.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/commons-logging-1.1.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/commons-pool-1.6.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/forms-1.0.7.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/forms_rt-6.0.5.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/freemarker-2.3.11.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/httpclient-4.2.4.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/httpcore-4.2.4.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/jackson-core-asl-1.9.13.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/jackson-mapper-asl-1.9.13.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/javassist-3.10.0.GA.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/jaxrs-api-3.0.7.Final.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/jboss-annotations-api_1.1_spec-1.0.1.Final.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/jboss-transaction-api_1.1_spec-1.0.1.Final.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/jdom-1.0.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/jsap-2.1.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/log4j-1.2.16.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/logkit-1.0.1.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/lucene-core-4.10.2.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/lucene-queries-4.10.2.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/lucene-queryparser-4.10.2.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/lucene-sandbox-4.10.2.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/milyn-commons-1.2.2.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/milyn-smooks-core-1.2.2.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/milyn-smooks-csv-1.2.2.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/milyn-smooks-javabean-1.2.2.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/mvel2-2.0.12.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/netty-all-4.0.20.Final.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/opencsv-1.8.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/resteasy-jaxrs-3.0.7.Final.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/scala-library-2.11.2.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/scala-xml_2.11-1.0.2.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/servlet-api-2.5.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/stream-2.2.0.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/webdav-servlet-2.0.1.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/xercesImpl-2.9.1.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/xml-apis-1.3.04.jar

              -> parent classloader = sun.misc.Launcher$ExtClassLoader@489bb457

              ->...file:/usr/lib/x86_64-linux-gnu/jni/libatk-wrapper.so.0.0.18

              ->...file:/usr/lib/jvm/java-7-openjdk-amd64/jre/lib/ext/sunjce_provider.jar

              ->...file:/usr/lib/jvm/java-7-openjdk-amd64/jre/lib/ext/zipfs.jar

              ->...file:/usr/lib/jvm/java-7-openjdk-amd64/jre/lib/ext/icedtea-sound.jar

              ->...file:/usr/lib/jvm/java-7-openjdk-amd64/jre/lib/ext/sunpkcs11.jar

              ->...file:/usr/lib/jvm/java-7-openjdk-amd64/jre/lib/ext/java-atk-wrapper.jar

              ->...file:/usr/lib/jvm/java-7-openjdk-amd64/jre/lib/ext/dnsns.jar

              ->...file:/usr/lib/jvm/java-7-openjdk-amd64/jre/lib/ext/localedata.jar

      2014-12-03 14:02:01,218 ERROR org.jgroups.blocks.RequestCorrelator.receiveMessage(RequestCorrelator.java:394) (OOB-2,clustered,Node-A-41319, site-id=LON, rack-id=r1, machine-id=m1) failed unmarshalling buffer into return value

      java.io.EOFException

          at org.jboss.marshalling.UTFUtils.readUTFBytes(UTFUtils.java:153)

          at org.jboss.marshalling.river.RiverUnmarshaller.readString(RiverUnmarshaller.java:1226)

          at org.jboss.marshalling.river.RiverUnmarshaller.doReadNewObject(RiverUnmarshaller.java:1325)

          at org.jboss.marshalling.river.RiverUnmarshaller.doReadObject(RiverUnmarshaller.java:272)

          at org.jboss.marshalling.river.RiverUnmarshaller.doReadObject(RiverUnmarshaller.java:209)

          at org.jboss.marshalling.AbstractObjectInput.readObject(AbstractObjectInput.java:41)

          at org.infinispan.topology.CacheStatusResponse$Externalizer.readObject(CacheStatusResponse.java:74)

          at org.infinispan.topology.CacheStatusResponse$Externalizer.readObject(CacheStatusResponse.java:60)

          at org.infinispan.marshall.core.ExternalizerTable$ExternalizerAdapter.readObject(ExternalizerTable.java:424)

          at org.infinispan.marshall.core.ExternalizerTable.readObject(ExternalizerTable.java:221)

          at org.infinispan.marshall.core.JBossMarshaller$ExternalizerTableProxy.readObject(JBossMarshaller.java:148)

          at org.jboss.marshalling.river.RiverUnmarshaller.doReadObject(RiverUnmarshaller.java:351)

          at org.jboss.marshalling.river.RiverUnmarshaller.doReadObject(RiverUnmarshaller.java:209)

          at org.jboss.marshalling.AbstractObjectInput.readObject(AbstractObjectInput.java:41)

          at org.infinispan.remoting.responses.SuccessfulResponse$Externalizer.readObject(SuccessfulResponse.java:79)

          at org.infinispan.remoting.responses.SuccessfulResponse$Externalizer.readObject(SuccessfulResponse.java:64)

          at org.infinispan.marshall.core.ExternalizerTable$ExternalizerAdapter.readObject(ExternalizerTable.java:424)

          at org.infinispan.marshall.core.ExternalizerTable.readObject(ExternalizerTable.java:221)

          at org.infinispan.marshall.core.JBossMarshaller$ExternalizerTableProxy.readObject(JBossMarshaller.java:148)

          at org.jboss.marshalling.river.RiverUnmarshaller.doReadObject(RiverUnmarshaller.java:351)

          at org.jboss.marshalling.river.RiverUnmarshaller.doReadObject(RiverUnmarshaller.java:209)

          at org.jboss.marshalling.AbstractObjectInput.readObject(AbstractObjectInput.java:41)

          at org.infinispan.commons.marshall.jboss.AbstractJBossMarshaller.objectFromObjectStream(AbstractJBossMarshaller.java:135)

          at org.infinispan.marshall.core.VersionAwareMarshaller.objectFromByteBuffer(VersionAwareMarshaller.java:101)

          at org.infinispan.commons.marshall.AbstractDelegatingMarshaller.objectFromByteBuffer(AbstractDelegatingMarshaller.java:80)

          at org.infinispan.remoting.transport.jgroups.MarshallerAdapter.objectFromBuffer(MarshallerAdapter.java:28)

          at org.jgroups.blocks.RequestCorrelator.receiveMessage(RequestCorrelator.java:390)

          at org.jgroups.blocks.RequestCorrelator.receive(RequestCorrelator.java:250)

          at org.jgroups.blocks.MessageDispatcher$ProtocolAdapter.up(MessageDispatcher.java:677)

          at org.jgroups.JChannel.up(JChannel.java:733)

          at org.jgroups.stack.ProtocolStack.up(ProtocolStack.java:1029)

          at org.jgroups.protocols.pbcast.STATE_TRANSFER.up(STATE_TRANSFER.java:146)

          at org.jgroups.protocols.FRAG2.up(FRAG2.java:165)

          at org.jgroups.protocols.FlowControl.up(FlowControl.java:394)

          at org.jgroups.protocols.pbcast.GMS.up(GMS.java:1042)

          at org.jgroups.protocols.pbcast.STABLE.up(STABLE.java:234)

          at org.jgroups.protocols.UNICAST3.deliverMessage(UNICAST3.java:1064)

          at org.jgroups.protocols.UNICAST3.handleDataReceived(UNICAST3.java:779)

          at org.jgroups.protocols.UNICAST3.up(UNICAST3.java:426)

          at org.jgroups.protocols.pbcast.NAKACK.up(NAKACK.java:638)

          at org.jgroups.protocols.BARRIER.up(BARRIER.java:152)

          at org.jgroups.protocols.VERIFY_SUSPECT.up(VERIFY_SUSPECT.java:155)

          at org.jgroups.protocols.FD_ALL.up(FD_ALL.java:200)

          at org.jgroups.protocols.FD_SOCK.up(FD_SOCK.java:297)

          at org.jgroups.protocols.MERGE3.up(MERGE3.java:288)

          at org.jgroups.protocols.Discovery.up(Discovery.java:277)

          at org.jgroups.protocols.MPING.up(MPING.java:178)

          at org.jgroups.protocols.TP.passMessageUp(TP.java:1572)

          at org.jgroups.protocols.TP$MyHandler.run(TP.java:1791)

          at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)

          at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)

          at java.lang.Thread.run(Thread.java:745)

      Caused by: an exception which occurred:

          in object of type org.infinispan.partitionhandling.AvailabilityMode

              -> classloader hierarchy:

              -> type classloader = sun.misc.Launcher$AppClassLoader@6132b73b

              ->...file:/home/kikosha/Apps/Infinispan/workspace/InfinispanCluster/bin/

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/infinispan-embedded-7.0.2.Final.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/infinispan-embedded-query-7.0.2.Final.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/infinispan-remote-7.0.2.Final.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/activation-1.1.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/asm-2.2.3.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/asm-commons-2.2.3.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/asm-tree-2.2.3.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/avalon-framework-4.1.3.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/cache-api-1.0.0.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/commons-codec-1.4.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/commons-io-1.3.2.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/commons-lang-2.1.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/commons-logging-1.1.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/commons-pool-1.6.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/forms-1.0.7.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/forms_rt-6.0.5.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/freemarker-2.3.11.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/httpclient-4.2.4.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/httpcore-4.2.4.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/jackson-core-asl-1.9.13.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/jackson-mapper-asl-1.9.13.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/javassist-3.10.0.GA.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/jaxrs-api-3.0.7.Final.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/jboss-annotations-api_1.1_spec-1.0.1.Final.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/jboss-transaction-api_1.1_spec-1.0.1.Final.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/jdom-1.0.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/jsap-2.1.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/log4j-1.2.16.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/logkit-1.0.1.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/lucene-core-4.10.2.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/lucene-queries-4.10.2.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/lucene-queryparser-4.10.2.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/lucene-sandbox-4.10.2.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/milyn-commons-1.2.2.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/milyn-smooks-core-1.2.2.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/milyn-smooks-csv-1.2.2.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/milyn-smooks-javabean-1.2.2.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/mvel2-2.0.12.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/netty-all-4.0.20.Final.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/opencsv-1.8.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/resteasy-jaxrs-3.0.7.Final.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/scala-library-2.11.2.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/scala-xml_2.11-1.0.2.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/servlet-api-2.5.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/stream-2.2.0.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/webdav-servlet-2.0.1.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/xercesImpl-2.9.1.jar

              ->...file:/home/kikosha/Apps/Infinispan/infinispan-7.0.2.Final-all/lib/xml-apis-1.3.04.jar

              -> parent classloader = sun.misc.Launcher$ExtClassLoader@489bb457

              ->...file:/usr/lib/x86_64-linux-gnu/jni/libatk-wrapper.so.0.0.18

              ->...file:/usr/lib/jvm/java-7-openjdk-amd64/jre/lib/ext/sunjce_provider.jar

              ->...file:/usr/lib/jvm/java-7-openjdk-amd64/jre/lib/ext/zipfs.jar

              ->...file:/usr/lib/jvm/java-7-openjdk-amd64/jre/lib/ext/icedtea-sound.jar

              ->...file:/usr/lib/jvm/java-7-openjdk-amd64/jre/lib/ext/sunpkcs11.jar

              ->...file:/usr/lib/jvm/java-7-openjdk-amd64/jre/lib/ext/java-atk-wrapper.jar

              ->...file:/usr/lib/jvm/java-7-openjdk-amd64/jre/lib/ext/dnsns.jar

              ->...file:/usr/lib/jvm/java-7-openjdk-amd64/jre/lib/ext/localedata.jar

       

       

      I tried debugging that specific part of the code:

       

      public static String readUTFBytes(final ByteInput input, final int len) throws IOException {
              final byte[] byteBuf = BYTES_HOLDER.get();
              final char[] chars = new char[len];
              int i = 0, cnt = 0, charIdx = 0;
              while (charIdx < len) {
                  if (i == cnt) {
                      cnt = input.read(byteBuf, 0, Math.min(UTF_BUFS_BYTE_CNT, len - charIdx));
                      if (cnt < 0) {
                          throw new EOFException(); <----- This is thrown
      
      

       

      and here's what I see:

       

      input         ExtendedRiverUnmarshaller  (id=95)  

      len           1041133426  

      byteBuf    [103, 46, 105, 110, 102, 105, 110, 105, 115, 112, 97, 110, 62, 4, 109, 97, 105, 110, 0, 0, 0, 9, 65, 86, 65, 73, 76, 65, 66, 76, 69, 103, 46, 65, 118, 97, 105, 108, 97, 98, 105, 108, 105, 116, 121, 77, 111, 100, 101, 0, 0, 0 ...............

      chars       [g, ., i, n, f, i, n, i, s, p, a, n, >,  , m, a, i, n,

      i              31  

      cnt          -1  

      charIdx    31  

       

      I'm running on:

      Linux 3.16.0-25-generic #33-Ubuntu SMP Tue Nov 4 12:06:54 UTC 2014 x86_64 x86_64 x86_64 GNU/Linux


      With Java:

       

      java version "1.7.0_65"

      OpenJDK Runtime Environment (IcedTea 2.5.3) (7u71-2.5.3-0ubuntu1)

      OpenJDK 64-Bit Server VM (build 24.65-b04, mixed mode)


      Here's a TCP Dump of what is happening on the wire:


      bela.....................TSJh3.]...w.d........site-id.LON.rack-id.r1

      machine-id.m1.D..].x.....B......... .....8..clustered.../..D..].x.....B........nodeB/clustered.....................D..].x.....B......TSJh3.]...w.d........site-id.LON.rack-id.r1

      machine-id.m1...........TSJh3.]...w.d........site-id.LON.rack-id.r1

      machine-id.m1.@.k.........8..clustered............TSJh3.]...w.d........site-id.LON.rack-id.r1

      machine-id.m1.D..].x.....B.............TSJh3.]...w.d........site-id.LON.rack-id.r1

      machine-id.m1........8..clustered............TSJh3.]...w.d........site-id.LON.rack-id.r1

      machine-id.m1.D..].x.....B...............@.k.........8..clustered...i...D..].x.....B.........D..].x.....B......TSJh3.]...w.d........site-id.LON.rack-id.r1

      machine-id.m1..................TSJh3.]...w.d........site-id.LON.rack-id.r1

      machine-id.m1.........TSJh3.]...w.d........site-id.LON.rack-id.r1

      machine-id.m1..

      ..{.........8..clustered........D..].x.....B......TSJh3.]...w.d........site-id.LON.rack-id.r1

      machine-id.m1..clustered................TSJh3.]...w.d........site-id.LON.rack-id.r1

      machine-id.m1..

      ..{............... ..................TSJh3.]...w.d........site-id.LON.rack-id.r1

      machine-id.m1.D..].x.....B....................8..clustered...`......TSJh3.]...w.d........site-id.LON.rack-id.r1

      machine-id.m1..

      ..{...D..].x.....B...............u.......D..].x.....B......TSJh3.]...w.d........site-id.LON.rack-id.r1

      machine-id.m1......

      ....@.k.........8..clustered...n........TSJh3.]...w.d........site-id.LON.rack-id.r1

      machine-id.m1.D..].x.....B.......@.k..........8..clustered.........#.D..].x.....B......TSJh3.]...w.d........site-id.LON.rack-id.r1

      machine-id.m1.....G.......@.k.........8..clustered........>....>.myDistCacheI..<..TSJh3.]...w.d........site-id.LON.rack-id.r1

      machine-id.m1.W.^.I...<..............?...K....K........K.............#..TSJh3.]...w.d........site-id.LON.rack-id.r1

      machine-id.m1.D..].x.....B.........G.......@.k.........8..clustered...4....>....>.myDistCacheI..'.D..].x.....B.....K....K.....3.............'.D..].x.....B.........'.D..].x.....B....N?....<..T.......d#......N....site-id.LON.rack-id.r1

      machine-id.m1N?....<..T.j1....d...6........site-id.LON.rack-id.r1

      machine-id.m1N?....I.z....'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].

      x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.....3.............'.D..].x.....B.....<..TSJh3.]...w.d........site-id.LON.rack-id.r1

      machine-id.m1.....<..TSJh3.]...w.d........site-id.LON.rack-id.r1

      machine-id.m1N?....'.D..].x.....B....N?....<..T.......d#......N....site-id.LON.rack-id.r1

      machine-id.m1N?....<..T.j1....d...6........site-id.LON.rack-id.r1

      machine-id.m1N?....I.z....'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......<..TSJh3.]...w.d........site-id.LON.rack-id.r1

      machine-id.m1...<..TSJh3.]...w.d........site-id.LON.rack-id.r1

      machine-id.m1...<..TSJh3.]...w.d........site-id.LON.rack-id.r1

      machine-id.m1...<..TSJh3.]...w.d........site-id.LON.rack-id.r1

      machine-id.m1...<..TSJh3.]...w.d........site-id.LON.rack-id.r1

      machine-id.m1...<..TSJh3.]...w.d........site-id.LON.rack-id.r1

      machine-id.m1...<..TSJh3.]...w.d........site-id.LON.rack-id.r1

      machine-id.m1...<..TSJh3.]...w.d........site-id.LON.rack-id.r1

      machine-id.m1...<..TSJh3.]...w.d........site-id.LON.rack-id.r1

      machine-id.m1...<..TSJh3.]...w.d........site-id.LON.rack-id.r1

      machine-id.m1..K.......F.....#..TSJh3.]...w.d........site-id.LON.rack-id.r1

      machine-id.m1.D..].x.....B.........G.......@.k.........8..clustered..............K.........3.............'.D..].x.....B.........'.D..].x.....B....N?....<..T.......d#......N....site-id.LON.rack-id.r1

      machine-id.m1N?....<..T.j1....d...6........site-id.LON.rack-id.r1

      machine-id.m1N?....I.z....'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..]

      .x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......K.........3.............'.D..].x.....B.........'.D..].x.....B....N?....I.z....'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B....

      ...'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B...........

      1org.infinispan.partitionhandling.AvailabilityMode>.org.infinispan>.main....AVAILABLE...n.......D..].x.....B......TSJh3.]...w.d........site-id.LON.rack-id.r1

      machine-id.m1...@.k..........8..clustered...n........TSJh3.]...w.d........site-id.LON.rack-id.r1

      machine-id.m1.D..].x.....B.......@.k..........8..clustered.........#.D..].x.....B......TSJh3.]...w.d........site-id.LON.rack-id.r1

      machine-id.m1.....G.......@.k.........8..clustered........>....>.myDistCacheI..<..TSJh3.]...w.d........site-id.LON.rack-id.r1

      machine-id.m1.W.^.I...<..............?...K....K........K.............#..TSJh3.]...w.d........site-id.LON.rack-id.r1

      machine-id.m1.D..].x.....B.........G.......@.k.........8..clustered...W..........K.........3.............'.D..].x.....B.........'.D..].x.....B....N?....<..T.......d#......N....site-id.LON.rack-id.r1

      machine-id.m1N?....<..T.j1....d...6........site-id.LON.rack-id.r1

      machine-id.m1N?....I.z....'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].

      x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.....3.............'.D..].x.....B.....<..TSJh3.]...w.d........site-id.LON.rack-id.r1

      machine-id.m1.....<..TSJh3.]...w.d........site-id.LON.rack-id.r1

      machine-id.m1N?....'.D..].x.....B....N?....<..T.......d#......N....site-id.LON.rack-id.r1

      machine-id.m1N?....<..T.j1....d...6........site-id.LON.rack-id.r1

      machine-id.m1N?....I.z....'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......<..TSJh3.]...w.d........site-id.LON.rack-id.r1

      machine-id.m1...<..TSJh3.]...w.d........site-id.LON.rack-id.r1

      machine-id.m1...<..TSJh3.]...w.d........site-id.LON.rack-id.r1

      machine-id.m1...<..TSJh3.]...w.d........site-id.LON.rack-id.r1

      machine-id.m1...<..TSJh3.]...w.d........site-id.LON.rack-id.r1

      machine-id.m1...<..TSJh3.]...w.d........site-id.LON.rack-id.r1

      machine-id.m1...<..TSJh3.]...w.d........site-id.LON.rack-id.r1

      machine-id.m1...<..TSJh3.]...w.d........site-id.LON.rack-id.r1

      machine-id.m1...<..TSJh3.]...w.d........site-id.LON.rack-id.r1

      machine-id.m1...<..TSJh3.]...w.d........site-id.LON.rack-id.r1

      machine-id.m1..K.........3.............'.D..].x.....B.........'.D..].x.....B....N?....I.z....'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D

      ..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B.......'.D..].x.....B...........1org.infinispan.partitionhandling.AvailabilityMode>.org.infinispan>.main....AVAILABLE...n........TSJh3.]...w.d........site-id.LON.rack-id.r1

      machine-id.m1.D..].x.....B.......@.k..........8..clustered...n.......D..].x.....B......TSJh3.]...w.d........site-id.LON.rack-id.r1

      machine-id.m1...@.k..........8..clustered...V........TSJh3.]...w.d........site-id.LON.rack-id.r1

      machine-id.m1.....=...8..clustered


      Needless to say I'm completely lost about what may be the problem...