Modeshape 2.8.3.Final + infinispan problem

[pjakub]

Hi, I'm trying to setup Jboss 4.2.3 (I must use that old version..) with Modeshape 2.8.3.Final and Infinispan 4.2.1.Final

My config files are as follow:

jcr-config.xml

<?xml version="1.0" encoding="UTF-8" ?>

<configuration xmlns:mode="http://www.modeshape.org/1.0" xmlns:jcr="http://www.jcp.org/jcr/1.0">

    <mode:repositories>

        <mode:repository jcr:name="tlk_repo" mode:source="tlk_repo_src">

        <mode:options jcr:primaryType="mode:options">

            <jaasLoginConfigName jcr:primaryType="mode:option" mode:value=""/>

            <systemSourceName jcr:primaryType="mode:option" mode:value=""/>

            <anonymousUserRoles jcr:primaryType="mode:option" mode:value="readonly,readwrite,admin,connect"/>

            <queryIndexDirectory jcr:primaryType="mode:option" mode:value="/opt/data20/content_repo/index"/>

            <queryIndexesUpdatedSynchronously jcr:primaryType="mode:option" mode:value="true"/>

            <rebuildQueryIndexOnStartup jcr:primaryType="mode:option"  mode:value="ifMissing"/>

        </mode:options>

        <jcr:nodeTypes>

            <mode:resource>//jboss/server/default/deploy/tlk-custom-node-types.cnd</mode:resource>

        </jcr:nodeTypes>

        </mode:repository>

    </mode:repositories>

  <mode:sources jcr:primaryType="nt:unstructured">

            <mode:source jcr:name="tlk_repo_src"

              mode:classname="org.modeshape.connector.infinispan.InfinispanSource"

              mode:retryLimit="3" mode:defaultWorkspaceName="default"

              mode:predefinedWorkspaceNames="default,global,219,1142,1143,1151"

        mode:updatesAllowed="true"

              mode:cacheConfigurationName="/jboss/server/default/deploy/modeshape3-jca.rar/jcr-infinispan.xml"

              mode:creatingWorkspaceAllowed="true"

              mode:rootNodeUuid="d878c1c0-684c-11df-a08a-0800200c9a66">

          </mode:source>

  </mode:sources>

      <mode:clustering clusterName="tlk-modeshape-cluster">  

        <configuration><![CDATA[

        <config>

            <UDP

          mcast_addr='224.1.8.8'

          mcast_port='3333'

          ip_ttl='64'

          ip_mcast='true'

              mcast_send_buf_size='120000'

          mcast_recv_buf_size='80000'

          ucast_send_buf_size='120000'

              ucast_recv_buf_size='80000'

          loopback='true' />

        <PING timeout='1000' num_initial_members='2' />

        <MERGE2 min_interval='10000' max_interval='20000' />

        <FD timeout='10000'/>

        <VERIFY_SUSPECT timeout='1500' />

        <pbcast.NAKACK retransmit_timeout='600,1200,2400,4800' discard_delivered_msgs='true' use_mcast_xmit='false' />

        <UNICAST timeout='600,1200,2400' />

        <pbcast.STABLE desired_avg_gossip='20000' max_bytes='400000' />

        <FRAG frag_size='8192' />

        <pbcast.GMS join_timeout='5000' print_local_addr='true' />

        <pbcast.STATE_TRANSFER />

        </config>]]>

        </configuration>

      </mode:clustering>

</configuration>

jcr-infinispan.xml

<?xml version="1.0" encoding="UTF-8"?>

<infinispan xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"

    xmlns="urn:infinispan:config:4.0">

    <global>

        <transport>

            <properties>

                <property name="configurationFile"

                    value="/jboss/server/default/deploy/modeshape3-jca.rar/jcr-infinispan-jgroups.xml" />

            </properties>

        </transport>

    </global>

    <default>

        <locking useLockStriping="false"/>

        <clustering mode="distribution">

            <sync />

            <hash numOwners="2" rehashWait="120000" rehashRpcTimeout="600000" />

            <l1 enabled="false" lifespan="600000" />

        </clustering>

        <loaders shared="true" preload="false">

            <loader class="org.infinispan.loaders.file.FileCacheStore"

                fetchPersistentState="true" ignoreModifications="false"

                purgeOnStartup="false">

                <properties>

                    <property name="location" value="/opt/data20/content_repo/l/store" />

                </properties>

            </loader>

        </loaders>

    </default>

</infinispan>

jcr-infinispan-jgroups.xml

<?xml version="1.0" encoding="UTF-8"?>

<config>

    <UDP mcast_addr='224.1.8.9' mcast_port='3337'

        ip_ttl='64' ip_mcast='true' mcast_send_buf_size='120000'

        mcast_recv_buf_size='80000' ucast_send_buf_size='120000'

        ucast_recv_buf_size='80000' loopback='false' />

    <PING timeout='1000' num_initial_members='2' />

    <MERGE2 min_interval='10000' max_interval='20000' />

    <FD timeout='5000'/>

    <VERIFY_SUSPECT timeout='1500' />

    <pbcast.NAKACK retransmit_timeout='600,1200,2400,5200'

        discard_delivered_msgs='true' use_mcast_xmit='false' />

    <UNICAST timeout='600,1200,2400' />

    <pbcast.STABLE desired_avg_gossip='20000' max_bytes='400000' />

    <FRAG frag_size='8192' />

    <pbcast.GMS join_timeout='5000' print_local_addr='true' />

    <pbcast.STATE_TRANSFER />

    <!-- https://issues.jboss.org/browse/ISPN-83, required for Infinispan 4.x -->

    <pbcast.FLUSH timeout="0"/>

</config>

/opt/data20 is a nfs shared between all machines.

I'm running one server + two webdavs (they have queryExecutionEnabled = false)

Everythink works great.

When I start another server everything still works great. It can query modeshape for files - everything is ok.

But when second instance try adding some files I'm getting strange exceptions:

DEBUG [RuleBasedOptimizer] Running query optimizer rule AddAccessNodes

11:41:55,879 DEBUG [RuleBasedOptimizer] Running query optimizer rule PushSelectCriteria

11:41:55,879 DEBUG [RuleBasedOptimizer] Running query optimizer rule PushProjects

11:41:55,880 DEBUG [RuleBasedOptimizer] Running query optimizer rule RewriteAsRangeCriteria

11:44:12,147 ERROR [RepositoryQueryManager] Error updating the query indexes: Error while committing changes to the indexes for the "1142" workspace of the "tlk_repo_src" source: Stale NFS file handle

org.modeshape.search.lucene.LuceneException: Error while committing changes to the indexes for the "1142" workspace of the "tlk_repo_src" source: Stale NFS file handle

    at org.modeshape.search.lucene.LuceneSearchSession.commit(LuceneSearchSession.java:347)

    at org.modeshape.search.lucene.AbstractLuceneSearchEngine$AbstractLuceneProcessor.commit(AbstractLuceneSearchEngine.java:200)

    at org.modeshape.graph.search.SearchEngineProcessor.close(SearchEngineProcessor.java:64)

    at org.modeshape.search.lucene.LuceneSearchProcessor.close(LuceneSearchProcessor.java:91)

    at org.modeshape.graph.search.SearchEngineIndexer.close(SearchEngineIndexer.java:482)

    at org.modeshape.search.lucene.LuceneSearchEngine.index(LuceneSearchEngine.java:282)

    at org.modeshape.jcr.RepositoryQueryManager$SelfContained.process(RepositoryQueryManager.java:418)

    at org.modeshape.jcr.RepositoryQueryManager$SelfContained$2.notify(RepositoryQueryManager.java:315)

    at org.modeshape.graph.observe.ChangeObservers.broadcast(ChangeObservers.java:114)

    at org.modeshape.clustering.ClusteredObservationBus$Receiver.receive(ClusteredObservationBus.java:394)

    at org.jgroups.JChannel.up(JChannel.java:1490)

    at org.jgroups.stack.ProtocolStack.up(ProtocolStack.java:1074)

    at org.jgroups.protocols.pbcast.STATE_TRANSFER.up(STATE_TRANSFER.java:150)

    at org.jgroups.protocols.pbcast.GMS.up(GMS.java:891)

    at org.jgroups.protocols.FRAG.unfragment(FRAG.java:269)

    at org.jgroups.protocols.FRAG.up(FRAG.java:151)

    at org.jgroups.protocols.pbcast.STABLE.up(STABLE.java:246)

    at org.jgroups.protocols.UNICAST.up(UNICAST.java:309)

    at org.jgroups.protocols.pbcast.NAKACK.handleMessage(NAKACK.java:838)

    at org.jgroups.protocols.pbcast.NAKACK.up(NAKACK.java:667)

    at org.jgroups.protocols.VERIFY_SUSPECT.up(VERIFY_SUSPECT.java:133)

    at org.jgroups.protocols.FD.up(FD.java:275)

    at org.jgroups.protocols.MERGE2.up(MERGE2.java:209)

    at org.jgroups.protocols.Discovery.up(Discovery.java:291)

    at org.jgroups.protocols.PING.up(PING.java:66)

    at org.jgroups.protocols.TP.passMessageUp(TP.java:1102)

    at org.jgroups.protocols.TP$3.run(TP.java:1025)

    at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)

    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)

    at java.lang.Thread.run(Thread.java:662)

Caused by: java.io.IOException: Stale NFS file handle

    at java.io.RandomAccessFile.close0(Native Method)

    at java.io.RandomAccessFile.close(RandomAccessFile.java:577)

    at org.apache.lucene.store.FSDirectory$FSIndexOutput.close(FSDirectory.java:469)

    at org.apache.lucene.index.TermInfosWriter.close(TermInfosWriter.java:243)

    at org.apache.lucene.index.TermInfosWriter.close(TermInfosWriter.java:246)

    at org.apache.lucene.util.IOUtils.close(IOUtils.java:121)

    at org.apache.lucene.index.FormatPostingsFieldsWriter.finish(FormatPostingsFieldsWriter.java:70)

    at org.apache.lucene.index.FreqProxTermsWriter.flush(FreqProxTermsWriter.java:138)

    at org.apache.lucene.index.TermsHash.flush(TermsHash.java:113)

    at org.apache.lucene.index.DocInverter.flush(DocInverter.java:70)

    at org.apache.lucene.index.DocFieldProcessor.flush(DocFieldProcessor.java:60)

    at org.apache.lucene.index.DocumentsWriter.flush(DocumentsWriter.java:581)

    at org.apache.lucene.index.IndexWriter.doFlush(IndexWriter.java:3600)

    at org.apache.lucene.index.IndexWriter.flush(IndexWriter.java:3565)

    at org.apache.lucene.index.IndexWriter.optimize(IndexWriter.java:2510)

    at org.apache.lucene.index.IndexWriter.optimize(IndexWriter.java:2486)

    at org.apache.lucene.index.IndexWriter.optimize(IndexWriter.java:2456)

    at org.modeshape.search.lucene.LuceneSearchSession.commit(LuceneSearchSession.java:325)

    ... 29 more

11:45:45,809 DEBUG [RuleBasedOptimizer] Running query optimizer rule org.modeshape.jcr.query.RewritePseudoColumns@4ba5dbd9

11:45:45,809 DEBUG [RuleBasedOptimizer] Running query optimizer rule ReplaceViews

11:45:45,810 DEBUG [RuleBasedOptimizer] Running query optimizer rule CopyCriteria

11:45:45,810 DEBUG [RuleBasedOptimizer] Running query optimizer rule RightOuterToLeftOuterJoins

11:45:45,810 DEBUG [RuleBasedOptim

or:

DEBUG [RuleBasedOptimizer] Running query optimizer rule PushSelectCriteria

12:36:26,900 DEBUG [RuleBasedOptimizer] Running query optimizer rule PushProjects

12:36:26,901 DEBUG [RuleBasedOptimizer] Running query optimizer rule RewriteAsRangeCriteria

12:36:58,417 ERROR [RepositoryQueryManager] Error updating the query indexes: Error while committing changes to the indexes for the "1142" workspace of the "tlk_repo_src" source: /opt/data20/content_repo/index/1142/content/_9.tis (No such file or directory)

org.modeshape.search.lucene.LuceneException: Error while committing changes to the indexes for the "1142" workspace of the "tlk_repo_src" source: /opt/data20/content_repo/index/1142/content/_9.tis (No such file or directory)

    at org.modeshape.search.lucene.LuceneSearchSession.commit(LuceneSearchSession.java:347)

    at org.modeshape.search.lucene.AbstractLuceneSearchEngine$AbstractLuceneProcessor.commit(AbstractLuceneSearchEngine.java:200)

    at org.modeshape.graph.search.SearchEngineProcessor.close(SearchEngineProcessor.java:64)

    at org.modeshape.search.lucene.LuceneSearchProcessor.close(LuceneSearchProcessor.java:91)

    at org.modeshape.graph.search.SearchEngineIndexer.close(SearchEngineIndexer.java:482)

    at org.modeshape.search.lucene.LuceneSearchEngine.index(LuceneSearchEngine.java:282)

    at org.modeshape.jcr.RepositoryQueryManager$SelfContained.process(RepositoryQueryManager.java:418)

    at org.modeshape.jcr.RepositoryQueryManager$SelfContained$2.notify(RepositoryQueryManager.java:315)

    at org.modeshape.graph.observe.ChangeObservers.broadcast(ChangeObservers.java:114)

    at org.modeshape.clustering.ClusteredObservationBus$Receiver.receive(ClusteredObservationBus.java:394)

    at org.jgroups.JChannel.up(JChannel.java:1490)

    at org.jgroups.stack.ProtocolStack.up(ProtocolStack.java:1074)

    at org.jgroups.protocols.pbcast.STATE_TRANSFER.up(STATE_TRANSFER.java:150)

    at org.jgroups.protocols.pbcast.GMS.up(GMS.java:891)

    at org.jgroups.protocols.FRAG.unfragment(FRAG.java:269)

    at org.jgroups.protocols.FRAG.up(FRAG.java:151)

    at org.jgroups.protocols.pbcast.STABLE.up(STABLE.java:246)

    at org.jgroups.protocols.UNICAST.up(UNICAST.java:309)

    at org.jgroups.protocols.pbcast.NAKACK.handleMessage(NAKACK.java:838)

    at org.jgroups.protocols.pbcast.NAKACK.up(NAKACK.java:667)

    at org.jgroups.protocols.VERIFY_SUSPECT.up(VERIFY_SUSPECT.java:133)

    at org.jgroups.protocols.FD.up(FD.java:275)

    at org.jgroups.protocols.MERGE2.up(MERGE2.java:209)

    at org.jgroups.protocols.Discovery.up(Discovery.java:291)

    at org.jgroups.protocols.PING.up(PING.java:66)

    at org.jgroups.protocols.TP.passMessageUp(TP.java:1102)

    at org.jgroups.protocols.TP$IncomingPacket.handleMyMessage(TP.java:1658)

    at org.jgroups.protocols.TP$IncomingPacket.run(TP.java:1635)

    at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)

    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)

    at java.lang.Thread.run(Thread.java:662)

Caused by: java.io.FileNotFoundException: /opt/data20/content_repo/index/1142/content/_9.tis (No such file or directory)

    at java.io.RandomAccessFile.open(Native Method)

    at java.io.RandomAccessFile.<init>(RandomAccessFile.java:216)

    at org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:214)

    at org.apache.lucene.index.TermInfosReader.<init>(TermInfosReader.java:104)

    at org.apache.lucene.index.SegmentCoreReaders.<init>(SegmentCoreReaders.java:76)

    at org.apache.lucene.index.SegmentReader.get(SegmentReader.java:115)

    at org.apache.lucene.index.IndexWriter$ReaderPool.get(IndexWriter.java:710)

    at org.apache.lucene.index.IndexWriter$ReaderPool.get(IndexWriter.java:685)

    at org.apache.lucene.index.BufferedDeletesStream.applyDeletes(BufferedDeletesStream.java:201)

    at org.apache.lucene.index.IndexWriter.doFlush(IndexWriter.java:3628)

    at org.apache.lucene.index.IndexWriter.flush(IndexWriter.java:3565)

    at org.apache.lucene.index.IndexWriter.optimize(IndexWriter.java:2510)

    at org.apache.lucene.index.IndexWriter.optimize(IndexWriter.java:2486)

    at org.apache.lucene.index.IndexWriter.optimize(IndexWriter.java:2456)

    at org.modeshape.search.lucene.LuceneSearchSession.commit(LuceneSearchSession.java:325)

    ... 30 more

12:37:33,788 DEBUG [RepositoryConnectionPool] Shutting down repository connection pool for tlk_repo_src

12:37:33,788 DEBUG [RepositoryConnectionPool] Terminated repository connection pool for tlk_repo_src

what could be reason for that ?

for me it seems like all files are visible and ok because of infinispan, but indexes are being used in bad way by servers, they override each other files.

I tried running same configuration, but with index files poining to other directories, but then I'm getting some exceptions too ( lik "trying to acces /X/Y/Z but /X/Y is the closest file avaliable [i don't remember exact error])

Am I missing some obvious config property? Help please, I've spend more than four days trying to figure this out .I have had other errors, related to classloading, like joda-time twice on classpath, and wrong jgroups version, but I managed to resolve those)

[rhauch]

Each repository in the cluster needs to have its own set of indexes, which means that each repository should have its own directory for indexes. When a cluster is created, changes in one repository instance will be propagated to the other repository instances via events over the JGroups channel, and the receiving repositories will update their own indexes.

Also, I don't think that Infinispan's FileCacheStore can use a shared directory. (The JDBC cache stores can share a single database, but file systems aren't transactional.) If you want to use the file cache store, then I'd recommend not sharing them and giving all their own directory. Clustering should keep everything in sync.

The easiest way to think about clustering ModeShape+Infinispan is that each process (including the Infinispan cache used by the connector) is entirely self-sufficient: it has its own storage, its own indexes, etc. Clustering (via JGroups) allows the different processes to know what's happening in each and to update themselves with changes made in other processes. It's sort of a "share-nothing" architecture (in terms of storage). The benefit is that should a process shut down, it contains no resources used/required by any other process.

Consider what happens if you're using a JDBC cache store. All the different processes running ModeShape (and Infinispan) are all separate, but if your database goes down everything is now hosed and the repository cannot be read or written-to. This single-point-of-failure can be eliminated by making each process own its own storage, and by relying upon network communication (via JGroups, preferrably over a high-speed network like in any data center) to keep the state in all of these independent processes synchronized.

(BTW, using ModeShape 2.x with an Infinispan connector is very similar to how ModeShape 3 always works. Therefore, the above guidelines also apply when using ModeShape 3. However, the guidelines do not necessarily apply to ModeShape 2.x when using other connectors, like the JPA connector.)

[pjakub]

After alittle bit of trying and chatting on IRC with Randall we figured out, as Rhandall stated one post above:

Indexes must be stored seperetley for each machine, if you by mistake, or by your own bad idea put indexes into NFS bad things happen, they get overwritten by different instances, and lots of FileNotFoundException and other bad things happens, basicly destroying your data, and purging repository is required.

Using NFS for FileCacheStore is also bad idea. We've got lots of data, and need few small servers to serve it, so we cant turn replication on. I'll try to setup without NFS, using some other kind of store.

I'll also try switching do modeshape 3 + infinispan 5 as Rhandall suggested, if that will be possible in my current setup.

Big thanks Rhandall for your time and help !

[pjakub]

I moved to ModeShape 3.10 + Infinispan 5.1.2.Final, changed connector to BDBJE and now im getting:

org.infinispan.CacheException: Could not invoke MapReduce task on remote nodes

    at org.infinispan.distexec.mapreduce.MapReduceTask.execute(MapReduceTask.java:244)

    at org.modeshape.jcr.value.binary.infinispan.InfinispanBinaryStore.removeValuesUnusedLongerThan(InfinispanBinaryStore.java:321)

    at org.modeshape.jcr.JcrRepository$RunningState.cleanUpBinaryValues(JcrRepository.java:1665)

    at org.modeshape.jcr.JcrRepository.cleanUp(JcrRepository.java:880)

    at org.modeshape.jcr.ModeShapeEngine$GarbageCollectionTask.run(ModeShapeEngine.java:135)

    at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:441)

    at java.util.concurrent.FutureTask$Sync.innerRunAndReset(FutureTask.java:317)

    at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:150)

    at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$101(ScheduledThreadPoolExecutor.java:98)

    at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.runPeriodic(ScheduledThreadPoolExecutor.java:180)

    at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:204)

    at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)

    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)

    at java.lang.Thread.run(Thread.java:662)

Caused by: org.infinispan.CacheException: java.lang.RuntimeException: Failure to marshal argument(s)

    at org.infinispan.util.Util.rewrapAsCacheException(Util.java:524)

    at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.invokeRemoteCommand(CommandAwareRpcDispatcher.java:168)

    at org.infinispan.remoting.transport.jgroups.JGroupsTransport.invokeRemotely(JGroupsTransport.java:478)

    at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:148)

    at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:169)

    at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:219)

    at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:206)

    at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:201)

    at org.infinispan.distexec.mapreduce.MapReduceTask.execute(MapReduceTask.java:240)

    ... 13 more

Caused by: java.lang.RuntimeException: Failure to marshal argument(s)

    at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.marshallCall(CommandAwareRpcDispatcher.java:247)

    at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.processSingleCall(CommandAwareRpcDispatcher.java:262)

    at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.invokeRemoteCommand(CommandAwareRpcDispatcher.java:163)

    ... 20 more

Caused by: org.infinispan.marshall.NotSerializableException: org.modeshape.jcr.value.binary.infinispan.InfinispanBinaryStore

Caused by: an exception which occurred:

    in field this$0

    in object org.modeshape.jcr.value.binary.infinispan.InfinispanBinaryStore$2@65314b1

    in object org.infinispan.commands.read.MapReduceCommand@25d5e64b

and on the second node:

Caused by: java.lang.ClassCastException: org.infinispan.schematic.internal.SchematicEntryWholeDelta cannot be cast to org.infinispan.schematic.SchematicEntry

    at org.infinispan.schematic.internal.CacheSchematicDb.get(CacheSchematicDb.java:235)

    at org.modeshape.jcr.cache.document.LocalDocumentStore.get(LocalDocumentStore.java:63)

    at org.modeshape.jcr.cache.RepositoryCache.refreshWorkspaces(RepositoryCache.java:378)

    at org.modeshape.jcr.cache.RepositoryCache.createWorkspace(RepositoryCache.java:699)

    at org.modeshape.jcr.JcrWorkspace.createWorkspace(JcrWorkspace.java:662)

what could be the cause?

[rhauch]

Actually, that first one looks like a bug in our Infinispan Binary Store implementation. Can you log it? You can get around this by using a different binary store.

The second one is a bit more serious. We've not seen anything like this in our testing. Is there any way for you to provide a test project that reproduces this, and log a second bug?

You could try Infinispan 5.1.8.Final, which may behave better. See our explanation for how to do this with Maven.

[pjakub]

Thanks for your reply.

I moved back to modeshape 2.8.3 and used RemoteInfinispanConnector and  few instances of hotrod server version 4.2.1. Now I'ts working beautifully.

Those strange errors might be caused by some strange classloading problems that we encountered in Jboss4 - we had many versions of infinispan running in one application, loaded ad different points in time

(that basicly caused 'instanceof' operator to work badly, because there were same classes loaded multiple time, and java.lang.ClassCastException might be caused by this issues too).

Right now it's very busy deploying it to production, but in near future I'll be trying to move to modeshape 3.1 + hotrod 5.1.2 . I think that moving to hotrod instead of build-in infinispan may resolve those strange errors (because hotrod have separate classpath), but if any of those remain, I'll then fill some jira ticket for you, and build full example.

Thanks for your help again, your first post basicly helped me resolve biggest issue there - shared index and NFS shared storage

[pjakub]

ok, I've switched to 3.1.1.Final, and I'm getting:

15:49:01,853 ERROR [JcrRepository] Error during background garbage collection: Could not invoke MapReduce task on remote nodes

org.infinispan.CacheException: Could not invoke MapReduce task on remote nodes

    at org.infinispan.distexec.mapreduce.MapReduceTask.execute(MapReduceTask.java:244)

    at org.modeshape.jcr.value.binary.infinispan.InfinispanBinaryStore.removeValuesUnusedLongerThan(InfinispanBinaryStore.java:321)

    at org.modeshape.jcr.JcrRepository$RunningState.cleanUpBinaryValues(JcrRepository.java:1665)

    at org.modeshape.jcr.JcrRepository.cleanUp(JcrRepository.java:880)

    at org.modeshape.jcr.ModeShapeEngine$GarbageCollectionTask.run(ModeShapeEngine.java:135)

    at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:441)

    at java.util.concurrent.FutureTask$Sync.innerRunAndReset(FutureTask.java:317)

    at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:150)

    at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$101(ScheduledThreadPoolExecutor.java:98)

    at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.runPeriodic(ScheduledThreadPoolExecutor.java:180)

    at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:204)

    at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)

    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)

    at java.lang.Thread.run(Thread.java:662)

Caused by: org.infinispan.CacheException: java.lang.RuntimeException: Failure to marshal argument(s)

    at org.infinispan.util.Util.rewrapAsCacheException(Util.java:524)

    at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.invokeRemoteCommand(CommandAwareRpcDispatcher.java:168)

    at org.infinispan.remoting.transport.jgroups.JGroupsTransport.invokeRemotely(JGroupsTransport.java:478)

    at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:148)

    at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:169)

    at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:219)

    at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:206)

    at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:201)

    at org.infinispan.distexec.mapreduce.MapReduceTask.execute(MapReduceTask.java:240)

    ... 13 more

Caused by: java.lang.RuntimeException: Failure to marshal argument(s)

    at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.marshallCall(CommandAwareRpcDispatcher.java:247)

    at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.processSingleCall(CommandAwareRpcDispatcher.java:262)

    at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.invokeRemoteCommand(CommandAwareRpcDispatcher.java:163)

    ... 20 more

Caused by: org.infinispan.marshall.NotSerializableException: org.modeshape.jcr.value.binary.infinispan.InfinispanBinaryStore

Caused by: an exception which occurred:

    in field this$0

    in object org.modeshape.jcr.value.binary.infinispan.InfinispanBinaryStore$2@5d15e8

    in object org.infinispan.commands.read.MapReduceCommand@887982c9

15:53:59,558 ERROR [JcrRepository] Error during background garbage collection: Could not invoke MapReduce task on remote nodes

org.infinispan.CacheException: Could not invoke MapReduce task on remote nodes

    at org.infinispan.distexec.mapreduce.MapReduceTask.execute(MapReduceTask.java:244)

    at org.modeshape.jcr.value.binary.infinispan.InfinispanBinaryStore.removeValuesUnusedLongerThan(InfinispanBinaryStore.java:321)

    at org.modeshape.jcr.JcrRepository$RunningState.cleanUpBinaryValues(JcrRepository.java:1665)

    at org.modeshape.jcr.JcrRepository.cleanUp(JcrRepository.java:880)

    at org.modeshape.jcr.ModeShapeEngine$GarbageCollectionTask.run(ModeShapeEngine.java:135)

    at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:441)

    at java.util.concurrent.FutureTask$Sync.innerRunAndReset(FutureTask.java:317)

    at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:150)

    at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$101(ScheduledThreadPoolExecutor.java:98)

    at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.runPeriodic(ScheduledThreadPoolExecutor.java:180)

    at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:204)

    at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)

    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)

    at java.lang.Thread.run(Thread.java:662)

Caused by: org.infinispan.CacheException: java.lang.RuntimeException: Failure to marshal argument(s)

    at org.infinispan.util.Util.rewrapAsCacheException(Util.java:524)

    at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.invokeRemoteCommand(CommandAwareRpcDispatcher.java:168)

    at org.infinispan.remoting.transport.jgroups.JGroupsTransport.invokeRemotely(JGroupsTransport.java:478)

    at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:148)

    at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:169)

    at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:219)

    at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:206)

    at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:201)

    at org.infinispan.distexec.mapreduce.MapReduceTask.execute(MapReduceTask.java:240)

    ... 13 more

Caused by: java.lang.RuntimeException: Failure to marshal argument(s)

    at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.marshallCall(CommandAwareRpcDispatcher.java:247)

    at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.processSingleCall(CommandAwareRpcDispatcher.java:262)

    at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.invokeRemoteCommand(CommandAwareRpcDispatcher.java:163)

    ... 20 more

Caused by: org.infinispan.marshall.NotSerializableException: org.modeshape.jcr.value.binary.infinispan.InfinispanBinaryStore

Caused by: an exception which occurred:

    in field this$0

    in object org.modeshape.jcr.value.binary.infinispan.InfinispanBinaryStore$2@1600256

    in object org.infinispan.commands.read.MapReduceCommand@e2e2578

15:58:59,555 ERROR [JcrRepository] Error during background garbage collection: Could not invoke MapReduce task on remote nodes

org.infinispan.CacheException: Could not invoke MapReduce task on remote nodes

    at org.infinispan.distexec.mapreduce.MapReduceTask.execute(MapReduceTask.java:244)

    at org.modeshape.jcr.value.binary.infinispan.InfinispanBinaryStore.removeValuesUnusedLongerThan(InfinispanBinaryStore.java:321)

    at org.modeshape.jcr.JcrRepository$RunningState.cleanUpBinaryValues(JcrRepository.java:1665)

    at org.modeshape.jcr.JcrRepository.cleanUp(JcrRepository.java:880)

    at org.modeshape.jcr.ModeShapeEngine$GarbageCollectionTask.run(ModeShapeEngine.java:135)

    at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:441)

    at java.util.concurrent.FutureTask$Sync.innerRunAndReset(FutureTask.java:317)

    at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:150)

    at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$101(ScheduledThreadPoolExecutor.java:98)

    at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.runPeriodic(ScheduledThreadPoolExecutor.java:180)

    at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:204)

    at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)

    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)

    at java.lang.Thread.run(Thread.java:662)

Caused by: org.infinispan.CacheException: java.lang.RuntimeException: Failure to marshal argument(s)

    at org.infinispan.util.Util.rewrapAsCacheException(Util.java:524)

    at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.invokeRemoteCommand(CommandAwareRpcDispatcher.java:168)

    at org.infinispan.remoting.transport.jgroups.JGroupsTransport.invokeRemotely(JGroupsTransport.java:478)

    at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:148)

    at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:169)

    at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:219)

    at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:206)

    at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:201)

    at org.infinispan.distexec.mapreduce.MapReduceTask.execute(MapReduceTask.java:240)

    ... 13 more

Caused by: java.lang.RuntimeException: Failure to marshal argument(s)

    at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.marshallCall(CommandAwareRpcDispatcher.java:247)

    at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.processSingleCall(CommandAwareRpcDispatcher.java:262)

    at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.invokeRemoteCommand(CommandAwareRpcDispatcher.java:163)

    ... 20 more

Caused by: org.infinispan.marshall.NotSerializableException: org.modeshape.jcr.value.binary.infinispan.InfinispanBinaryStore

Caused by: an exception which occurred:

    in field this$0

    in object org.modeshape.jcr.value.binary.infinispan.InfinispanBinaryStore$2@6fe8a8

    in object org.infinispan.commands.read.MapReduceCommand@d194cc81

16:03:59,556 ERROR [JcrRepository] Error during background garbage collection: Could not invoke MapReduce task on remote nodes

org.infinispan.CacheException: Could not invoke MapReduce task on remote nodes

    at org.infinispan.distexec.mapreduce.MapReduceTask.execute(MapReduceTask.java:244)

    at org.modeshape.jcr.value.binary.infinispan.InfinispanBinaryStore.removeValuesUnusedLongerThan(InfinispanBinaryStore.java:321)

    at org.modeshape.jcr.JcrRepository$RunningState.cleanUpBinaryValues(JcrRepository.java:1665)

    at org.modeshape.jcr.JcrRepository.cleanUp(JcrRepository.java:880)

    at org.modeshape.jcr.ModeShapeEngine$GarbageCollectionTask.run(ModeShapeEngine.java:135)

    at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:441)

    at java.util.concurrent.FutureTask$Sync.innerRunAndReset(FutureTask.java:317)

    at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:150)

    at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$101(ScheduledThreadPoolExecutor.java:98)

    at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.runPeriodic(ScheduledThreadPoolExecutor.java:180)

    at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:204)

    at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)

    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)

    at java.lang.Thread.run(Thread.java:662)

Caused by: org.infinispan.CacheException: java.lang.RuntimeException: Failure to marshal argument(s)

    at org.infinispan.util.Util.rewrapAsCacheException(Util.java:524)

    at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.invokeRemoteCommand(CommandAwareRpcDispatcher.java:168)

    at org.infinispan.remoting.transport.jgroups.JGroupsTransport.invokeRemotely(JGroupsTransport.java:478)

    at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:148)

    at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:169)

    at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:219)

    at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:206)

    at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:201)

    at org.infinispan.distexec.mapreduce.MapReduceTask.execute(MapReduceTask.java:240)

    ... 13 more

Caused by: java.lang.RuntimeException: Failure to marshal argument(s)

    at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.marshallCall(CommandAwareRpcDispatcher.java:247)

    at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.processSingleCall(CommandAwareRpcDispatcher.java:262)

    at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.invokeRemoteCommand(CommandAwareRpcDispatcher.java:163)

    ... 20 more

Caused by: org.infinispan.marshall.NotSerializableException: org.modeshape.jcr.value.binary.infinispan.InfinispanBinaryStore

Caused by: an exception which occurred:

    in field this$0

    in object org.modeshape.jcr.value.binary.infinispan.InfinispanBinaryStore$2@c9a1b0

    in object org.infinispan.commands.read.MapReduceCommand@3b6daf71

16:08:59,555 ERROR [JcrRepository] Error during background garbage collection: Could not invoke MapReduce task on remote nodes

org.infinispan.CacheException: Could not invoke MapReduce task on remote nodes

    at org.infinispan.distexec.mapreduce.MapReduceTask.execute(MapReduceTask.java:244)

    at org.modeshape.jcr.value.binary.infinispan.InfinispanBinaryStore.removeValuesUnusedLongerThan(InfinispanBinaryStore.java:321)

    at org.modeshape.jcr.JcrRepository$RunningState.cleanUpBinaryValues(JcrRepository.java:1665)

    at org.modeshape.jcr.JcrRepository.cleanUp(JcrRepository.java:880)

    at org.modeshape.jcr.ModeShapeEngine$GarbageCollectionTask.run(ModeShapeEngine.java:135)

    at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:441)

    at java.util.concurrent.FutureTask$Sync.innerRunAndReset(FutureTask.java:317)

    at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:150)

    at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$101(ScheduledThreadPoolExecutor.java:98)

    at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.runPeriodic(ScheduledThreadPoolExecutor.java:180)

    at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:204)

    at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)

    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)

    at java.lang.Thread.run(Thread.java:662)

Caused by: org.infinispan.CacheException: java.lang.RuntimeException: Failure to marshal argument(s)

    at org.infinispan.util.Util.rewrapAsCacheException(Util.java:524)

    at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.invokeRemoteCommand(CommandAwareRpcDispatcher.java:168)

    at org.infinispan.remoting.transport.jgroups.JGroupsTransport.invokeRemotely(JGroupsTransport.java:478)

    at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:148)

    at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:169)

    at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:219)

    at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:206)

    at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:201)

    at org.infinispan.distexec.mapreduce.MapReduceTask.execute(MapReduceTask.java:240)

    ... 13 more

Caused by: java.lang.RuntimeException: Failure to marshal argument(s)

    at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.marshallCall(CommandAwareRpcDispatcher.java:247)

    at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.processSingleCall(CommandAwareRpcDispatcher.java:262)

    at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.invokeRemoteCommand(CommandAwareRpcDispatcher.java:163)

    ... 20 more

Caused by: org.infinispan.marshall.NotSerializableException: org.modeshape.jcr.value.binary.infinispan.InfinispanBinaryStore

Caused by: an exception which occurred:

    in field this$0

    in object org.modeshape.jcr.value.binary.infinispan.InfinispanBinaryStore$2@14e4f10

    in object org.infinispan.commands.read.MapReduceCommand@db86e950

16:13:59,554 ERROR [JcrRepository] Error during background garbage collection: Could not invoke MapReduce task on remote nodes

org.infinispan.CacheException: Could not invoke MapReduce task on remote nodes

    at org.infinispan.distexec.mapreduce.MapReduceTask.execute(MapReduceTask.java:244)

    at org.modeshape.jcr.value.binary.infinispan.InfinispanBinaryStore.removeValuesUnusedLongerThan(InfinispanBinaryStore.java:321)

    at org.modeshape.jcr.JcrRepository$RunningState.cleanUpBinaryValues(JcrRepository.java:1665)

    at org.modeshape.jcr.JcrRepository.cleanUp(JcrRepository.java:880)

    at org.modeshape.jcr.ModeShapeEngine$GarbageCollectionTask.run(ModeShapeEngine.java:135)

    at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:441)

    at java.util.concurrent.FutureTask$Sync.innerRunAndReset(FutureTask.java:317)

    at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:150)

    at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$101(ScheduledThreadPoolExecutor.java:98)

    at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.runPeriodic(ScheduledThreadPoolExecutor.java:180)

    at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:204)

    at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)

    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)

    at java.lang.Thread.run(Thread.java:662)

Caused by: org.infinispan.CacheException: java.lang.RuntimeException: Failure to marshal argument(s)

    at org.infinispan.util.Util.rewrapAsCacheException(Util.java:524)

    at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.invokeRemoteCommand(CommandAwareRpcDispatcher.java:168)

    at org.infinispan.remoting.transport.jgroups.JGroupsTransport.invokeRemotely(JGroupsTransport.java:478)

    at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:148)

    at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:169)

    at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:219)

    at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:206)

    at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:201)

    at org.infinispan.distexec.mapreduce.MapReduceTask.execute(MapReduceTask.java:240)

    ... 13 more

Caused by: java.lang.RuntimeException: Failure to marshal argument(s)

    at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.marshallCall(CommandAwareRpcDispatcher.java:247)

    at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.processSingleCall(CommandAwareRpcDispatcher.java:262)

    at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.invokeRemoteCommand(CommandAwareRpcDispatcher.java:163)

    ... 20 more

Caused by: org.infinispan.marshall.NotSerializableException: org.modeshape.jcr.value.binary.infinispan.InfinispanBinaryStore

Caused by: an exception which occurred:

    in field this$0

    in object org.modeshape.jcr.value.binary.infinispan.InfinispanBinaryStore$2@d6882a

    in object org.infinispan.commands.read.MapReduceCommand@826f1a15

16:18:59,554 ERROR [JcrRepository] Error during background garbage collection: Could not invoke MapReduce task on remote nodes

org.infinispan.CacheException: Could not invoke MapReduce task on remote nodes

    at org.infinispan.distexec.mapreduce.MapReduceTask.execute(MapReduceTask.java:244)

    at org.modeshape.jcr.value.binary.infinispan.InfinispanBinaryStore.removeValuesUnusedLongerThan(InfinispanBinaryStore.java:321)

    at org.modeshape.jcr.JcrRepository$RunningState.cleanUpBinaryValues(JcrRepository.java:1665)

    at org.modeshape.jcr.JcrRepository.cleanUp(JcrRepository.java:880)

    at org.modeshape.jcr.ModeShapeEngine$GarbageCollectionTask.run(ModeShapeEngine.java:135)

    at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:441)

    at java.util.concurrent.FutureTask$Sync.innerRunAndReset(FutureTask.java:317)

    at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:150)

    at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$101(ScheduledThreadPoolExecutor.java:98)

    at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.runPeriodic(ScheduledThreadPoolExecutor.java:180)

    at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:204)

    at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)

    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)

    at java.lang.Thread.run(Thread.java:662)

Caused by: org.infinispan.CacheException: java.lang.RuntimeException: Failure to marshal argument(s)

    at org.infinispan.util.Util.rewrapAsCacheException(Util.java:524)

    at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.invokeRemoteCommand(CommandAwareRpcDispatcher.java:168)

    at org.infinispan.remoting.transport.jgroups.JGroupsTransport.invokeRemotely(JGroupsTransport.java:478)

    at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:148)

    at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:169)

    at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:219)

    at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:206)

    at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:201)

    at org.infinispan.distexec.mapreduce.MapReduceTask.execute(MapReduceTask.java:240)

    ... 13 more

Caused by: java.lang.RuntimeException: Failure to marshal argument(s)

    at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.marshallCall(CommandAwareRpcDispatcher.java:247)

    at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.processSingleCall(CommandAwareRpcDispatcher.java:262)

    at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.invokeRemoteCommand(CommandAwareRpcDispatcher.java:163)

    ... 20 more

Caused by: org.infinispan.marshall.NotSerializableException: org.modeshape.jcr.value.binary.infinispan.InfinispanBinaryStore

Caused by: an exception which occurred:

    in field this$0

    in object org.modeshape.jcr.value.binary.infinispan.InfinispanBinaryStore$2@1eb9805

    in object org.infinispan.commands.read.MapReduceCommand@ca5521a1

any idea what could be wrong?

[rhauch]

Hmmm... our task implementation is actually a static class nested within InfinispanBinaryStore. And I just noted this in the Infinispan JavaDoc:

Note that Mapper and Reducer should not be specified as inner classes. Inner classes declared in non-static contexts contain implicit non-transient references to enclosing class instances, serializing such an inner class instance will result in serialization of its associated outer class instance as well.

Can you file a bug for this last exception? I'll try to fix it today, and it should be available in 3.1.2 (due out next week).

BTW, if you wanted to test it locally before 3.1.2.Final is released (but after I commit/merge the fix for the bug), you could grab and build the latest code using the "3.1.x" branch to create a "3.1.2-SNAPSHOT" version in your local Maven repository.

[pjakub]

I've filled a bug:

https://issues.jboss.org/browse/MODE-1794

Please let me know when It's ready, I'll test it with SNAPSHOT.

Thanks for quick reaction.

[rhauch]

I just merged it into the '3.1.x' branch and in 'master'. Be sure to use '3.1.x', since that is the "3.1.2-SNAPSHOT" (what will become 3.1.2.Final) and contains only fixes for issues found in 3.1.1.Final.

[pjakub]

ok, I've build 3.1.2-SNAPSHOT

and now I'm getting:

11:55:50,901 ERROR [RemoteCacheStore] ISPN010001: RemoteCacheStore can only run in shared mode! This method shouldn't be called in shared mode

11:55:50,906 ERROR [JcrRepository] Error during background garbage collection: Error during background garbage collection: RemoteCacheStore can only run in shared mode! This method shouldn't be called in shared mode

org.modeshape.jcr.value.binary.BinaryStoreException: Error during background garbage collection: RemoteCacheStore can only run in shared mode! This method shouldn't be called in shared mode

    at org.modeshape.jcr.value.binary.infinispan.InfinispanBinaryStore.removeValuesUnusedLongerThan(InfinispanBinaryStore.java:368)

    at org.modeshape.jcr.JcrRepository$RunningState.cleanUpBinaryValues(JcrRepository.java:1665)

    at org.modeshape.jcr.JcrRepository.cleanUp(JcrRepository.java:880)

    at org.modeshape.jcr.ModeShapeEngine$GarbageCollectionTask.run(ModeShapeEngine.java:135)

    at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:441)

    at java.util.concurrent.FutureTask$Sync.innerRunAndReset(FutureTask.java:317)

    at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:150)

    at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$101(ScheduledThreadPoolExecutor.java:98)

    at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.runPeriodic(ScheduledThreadPoolExecutor.java:180)

    at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:204)

    at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)

    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)

    at java.lang.Thread.run(Thread.java:662)

12:00:50,927 ERROR [RemoteCacheStore] ISPN010001: RemoteCacheStore can only run in shared mode! This method shouldn't be called in shared mode

12:00:50,929 ERROR [JcrRepository] Error during background garbage collection: Error during background garbage collection: RemoteCacheStore can only run in shared mode! This method shouldn't be called in shared mode

org.modeshape.jcr.value.binary.BinaryStoreException: Error during background garbage collection: RemoteCacheStore can only run in shared mode! This method shouldn't be called in shared mode

    at org.modeshape.jcr.value.binary.infinispan.InfinispanBinaryStore.removeValuesUnusedLongerThan(InfinispanBinaryStore.java:368)

    at org.modeshape.jcr.JcrRepository$RunningState.cleanUpBinaryValues(JcrRepository.java:1665)

    at org.modeshape.jcr.JcrRepository.cleanUp(JcrRepository.java:880)

    at org.modeshape.jcr.ModeShapeEngine$GarbageCollectionTask.run(ModeShapeEngine.java:135)

    at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:441)

    at java.util.concurrent.FutureTask$Sync.innerRunAndReset(FutureTask.java:317)

    at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:150)

    at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$101(ScheduledThreadPoolExecutor.java:98)

    at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.runPeriodic(ScheduledThreadPoolExecutor.java:180)

    at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:204)

    at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)

    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)

    at java.lang.Thread.run(Thread.java:662)

12:05:50,885 ERROR [RemoteCacheStore] ISPN010001: RemoteCacheStore can only run in shared mode! This method shouldn't be called in shared mode

my store configuration right now is:

<loaders passivation="false" shared="true" preload="false">

            <loader class="org.infinispan.loaders.remote.RemoteCacheStore" fetchPersistentState="false"

                   ignoreModifications="false" purgeOnStartup="false"  >

                  <properties>

                 <property name="hotRodClientPropertiesFile" value="hotrod-client.properties"/>

                 <property name="remoteCacheName" value="tlk_content"/>

                  </properties>

             </loader>

        </loaders>

is there sth wrong in code? or am I missing some option to disable garbage collection in infinispan?

Good news is that it's all running, even after that exception cache is not being blocked or damaged anyhow.

[pjakub]

when the exception from post abowe is showing, I'm getting:

12:51:44,519 DEBUG FD:445 - sending are-you-alive msg to ubuntu-19148 (own address=ubuntu-38308)

12:51:47,521 DEBUG FD:445 - sending are-you-alive msg to ubuntu-19148 (own address=ubuntu-38308)

12:51:48,661 DEBUG JcrRepository:111 - Starting lock cleanup in the 'tlk_repo' repository

12:51:48,662 DEBUG WritableSessionCache:111 - Altered 1 node(s)

12:51:48,670 DEBUG JcrRepository:111 - Finishing lock cleanup in the 'tlk_repo' repository

12:51:48,670 DEBUG JcrRepository:111 - Starting binary value cleanup in the 'tlk_repo' repository

12:51:48,670 DEBUG JcrRepository:111 - Finishing binary value cleanup in the 'tlk_repo' repository

12:51:50,521 DEBUG FD:445 - sending are-you-alive msg to ubuntu-19148 (own address=ubuntu-38308)

12:51:53,522 DEBUG FD:445 - sending are-you-alive msg to ubuntu-19148 (own address=ubuntu-38308)

on another node

[rhauch]

ModeShape has a garbage collection (GC) mechanism that periodically requests the BinaryStore implementation to remove all persisted binary values that have been unused (meaning they are no longer referenced by a persisted property) for longer than X minutes. The InfinispanBinaryStore implements this request by using map-reduce to find all of the stored values that meet the criteria. Unfortunately, Infinispan's RemoteCache (and RemoteCacheLoader) do not suport map-reduce (or distributed execution). That means that the above error happens whenever the InfinispanBinaryStore is configured to use an Infinispan remote cache (e.g,. the cache really is running on a remote server, and ModeShape uses HotRod to talk to the remote cache) and ModeShape's GC process is run.

Now, although the error needs to be fixed (see MODE-1796), it may be possible to (at least for a while) ignore this error. What that would mean is that some Binary values would continue to be marked as unused (e.g., when a property that was referring to the Binary value is removed or changed, and there is no other reference to it), but these unused Binary values do not get removed from persistent storage. This does consume extra space, but as soon as ModeShape is upgraded to a version that fixes MODE-1796, ModeShape GC process will be run and will clean up all the unused binary values that have accumulated in the remote cache to date.

Note that this is only an issue when a repository is configured with an InfinispanBinaryStore that uses remote Infinispan caches.

[pjakub]

ok, another problem arised:

I changed my indexStore, to use infinispan as storage. I'm not specifying file name or jndi name, I want it to use existing infinispan configuration , and now I get:

org.hibernate.search.SearchException: HSEARCH000103: Unable to initialize IndexManager nodeinfo)
    at org.jboss.resource.connectionmanager.ConnectionFactoryBindingService.createConnectionFactory(ConnectionFactoryBindingService.java:132)
    at org.jboss.resource.connectionmanager.ConnectionFactoryBindingService.startService(ConnectionFactoryBindingService.java:65)
    at org.jboss.system.ServiceMBeanSupport.jbossInternalStart(ServiceMBeanSupport.java:289)
    at org.jboss.system.ServiceMBeanSupport.jbossInternalLifecycle(ServiceMBeanSupport.java:245)
    at sun.reflect.GeneratedMethodAccessor4.invoke(Unknown Source)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
    at java.lang.reflect.Method.invoke(Method.java:597)
    at org.jboss.mx.interceptor.ReflectedDispatcher.invoke(ReflectedDispatcher.java:155)
    at org.jboss.mx.server.Invocation.dispatch(Invocation.java:94)
    at org.jboss.mx.server.Invocation.invoke(Invocation.java:86)
    at org.jboss.mx.server.AbstractMBeanInvoker.invoke(AbstractMBeanInvoker.java:264)
    at org.jboss.mx.server.MBeanServerImpl.invoke(MBeanServerImpl.java:659)
    at org.jboss.system.server.jmx.LazyMBeanServer.invoke(LazyMBeanServer.java:291)
    at org.jboss.system.ServiceController$ServiceProxy.invoke(ServiceController.java:978)
    at $Proxy0.start(Unknown Source)
    at org.jboss.system.ServiceController.start(ServiceController.java:417)
    at org.jboss.system.ServiceController.start(ServiceController.java:435)
    at org.jboss.system.ServiceController.start(ServiceController.java:435)
    at org.jboss.system.ServiceController.start(ServiceController.java:435)
    at org.jboss.system.ServiceController.start(ServiceController.java:435)
    at sun.reflect.GeneratedMethodAccessor10.invoke(Unknown Source)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
    at java.lang.reflect.Method.invoke(Method.java:597)
    at org.jboss.mx.interceptor.ReflectedDispatcher.invoke(ReflectedDispatcher.java:155)
    at org.jboss.mx.server.Invocation.dispatch(Invocation.java:94)
    at org.jboss.mx.server.Invocation.invoke(Invocation.java:86)
    at org.jboss.mx.server.AbstractMBeanInvoker.invoke(AbstractMBeanInvoker.java:264)
    at org.jboss.mx.server.MBeanServerImpl.invoke(MBeanServerImpl.java:659)
    at org.jboss.system.server.jmx.LazyMBeanServer.invoke(LazyMBeanServer.java:291)
    at org.jboss.mx.util.MBeanProxyExt.invoke(MBeanProxyExt.java:210)
    at $Proxy39.start(Unknown Source)
    at org.jboss.deployment.SimpleSubDeployerSupport.startService(SimpleSubDeployerSupport.java:356)
    at org.jboss.deployment.SimpleSubDeployerSupport.start(SimpleSubDeployerSupport.java:127)
    at org.jboss.deployment.MainDeployer.start(MainDeployer.java:1025)
    at org.jboss.deployment.MainDeployer.start(MainDeployer.java:1015)
    at org.jboss.deployment.MainDeployer.deploy(MainDeployer.java:819)
    at org.jboss.deployment.MainDeployer.deploy(MainDeployer.java:782)
    at sun.reflect.GeneratedMethodAccessor21.invoke(Unknown Source)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
    at java.lang.reflect.Method.invoke(Method.java:597)
    at org.jboss.mx.interceptor.ReflectedDispatcher.invoke(ReflectedDispatcher.java:155)
    at org.jboss.mx.server.Invocation.dispatch(Invocation.java:94)
    at org.jboss.mx.interceptor.AbstractInterceptor.invoke(AbstractInterceptor.java:133)
    at org.jboss.mx.server.Invocation.invoke(Invocation.java:88)
    at org.jboss.mx.interceptor.ModelMBeanOperationInterceptor.invoke(ModelMBeanOperationInterceptor.java:142)
    at org.jboss.mx.server.Invocation.invoke(Invocation.java:88)
    at org.jboss.mx.server.AbstractMBeanInvoker.invoke(AbstractMBeanInvoker.java:264)
    at org.jboss.mx.server.MBeanServerImpl.invoke(MBeanServerImpl.java:659)
    at org.jboss.system.server.jmx.LazyMBeanServer.invoke(LazyMBeanServer.java:291)
    at org.jboss.mx.util.MBeanProxyExt.invoke(MBeanProxyExt.java:210)
    at $Proxy9.deploy(Unknown Source)
    at org.jboss.deployment.scanner.URLDeploymentScanner.deploy(URLDeploymentScanner.java:421)
    at org.jboss.deployment.scanner.URLDeploymentScanner.scan(URLDeploymentScanner.java:634)
    at org.jboss.deployment.scanner.AbstractDeploymentScanner$ScannerThread.doScan(AbstractDeploymentScanner.java:263)
    at org.jboss.deployment.scanner.AbstractDeploymentScanner.startService(AbstractDeploymentScanner.java:336)
    at org.jboss.system.ServiceMBeanSupport.jbossInternalStart(ServiceMBeanSupport.java:289)
    at org.jboss.system.ServiceMBeanSupport.jbossInternalLifecycle(ServiceMBeanSupport.java:245)
    at sun.reflect.GeneratedMethodAccessor4.invoke(Unknown Source)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
    at java.lang.reflect.Method.invoke(Method.java:597)
    at org.jboss.mx.interceptor.ReflectedDispatcher.invoke(ReflectedDispatcher.java:155)
    at org.jboss.mx.server.Invocation.dispatch(Invocation.java:94)
    at org.jboss.mx.server.Invocation.invoke(Invocation.java:86)
    at org.jboss.mx.server.AbstractMBeanInvoker.invoke(AbstractMBeanInvoker.java:264)
    at org.jboss.mx.server.MBeanServerImpl.invoke(MBeanServerImpl.java:659)
    at org.jboss.system.server.jmx.LazyMBeanServer.invoke(LazyMBeanServer.java:291)
    at org.jboss.system.ServiceController$ServiceProxy.invoke(ServiceController.java:978)
    at $Proxy0.start(Unknown Source)
    at org.jboss.system.ServiceController.start(ServiceController.java:417)
    at sun.reflect.GeneratedMethodAccessor10.invoke(Unknown Source)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
    at java.lang.reflect.Method.invoke(Method.java:597)
    at org.jboss.mx.interceptor.ReflectedDispatcher.invoke(ReflectedDispatcher.java:155)
    at org.jboss.mx.server.Invocation.dispatch(Invocation.java:94)
    at org.jboss.mx.server.Invocation.invoke(Invocation.java:86)
    at org.jboss.mx.server.AbstractMBeanInvoker.invoke(AbstractMBeanInvoker.java:264)
    at org.jboss.mx.server.MBeanServerImpl.invoke(MBeanServerImpl.java:659)
    at org.jboss.system.server.jmx.LazyMBeanServer.invoke(LazyMBeanServer.java:291)
    at org.jboss.mx.util.MBeanProxyExt.invoke(MBeanProxyExt.java:210)
    at $Proxy4.start(Unknown Source)
    at org.jboss.deployment.SARDeployer.start(SARDeployer.java:304)
    at org.jboss.deployment.MainDeployer.start(MainDeployer.java:1025)
    at org.jboss.deployment.MainDeployer.deploy(MainDeployer.java:819)
    at org.jboss.deployment.MainDeployer.deploy(MainDeployer.java:782)
    at org.jboss.deployment.MainDeployer.deploy(MainDeployer.java:766)
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
    at java.lang.reflect.Method.invoke(Method.java:597)
    at org.jboss.mx.interceptor.ReflectedDispatcher.invoke(ReflectedDispatcher.java:155)
    at org.jboss.mx.server.Invocation.dispatch(Invocation.java:94)
    at org.jboss.mx.interceptor.AbstractInterceptor.invoke(AbstractInterceptor.java:133)
    at org.jboss.mx.server.Invocation.invoke(Invocation.java:88)
    at org.jboss.mx.interceptor.ModelMBeanOperationInterceptor.invoke(ModelMBeanOperationInterceptor.java:142)
    at org.jboss.mx.server.Invocation.invoke(Invocation.java:88)
    at org.jboss.mx.server.AbstractMBeanInvoker.invoke(AbstractMBeanInvoker.java:264)
    at org.jboss.mx.server.MBeanServerImpl.invoke(MBeanServerImpl.java:659)
    at org.jboss.mx.util.MBeanProxyExt.invoke(MBeanProxyExt.java:210)
    at $Proxy5.deploy(Unknown Source)
    at org.jboss.system.server.ServerImpl.doStart(ServerImpl.java:482)
    at org.jboss.system.server.ServerImpl.start(ServerImpl.java:362)
    at org.jboss.Main.boot(Main.java:200)
    at org.jboss.Main$1.run(Main.java:508)
    at java.lang.Thread.run(Thread.java:662)
Caused by: javax.resource.ResourceException: javax.jcr.RepositoryException: org.hibernate.search.SearchException: HSEARCH000103: Unable to initialize IndexManager nodeinfo
    at com.lumesse.modeshape3.jca.Modeshape3ManagedConnectionFactory.createConnectionFactory(Modeshape3ManagedConnectionFactory.java:134)
    at org.jboss.resource.connectionmanager.ConnectionFactoryBindingService.createConnectionFactory(ConnectionFactoryBindingService.java:128)
    ... 103 more
Caused by: javax.jcr.RepositoryException: org.hibernate.search.SearchException: HSEARCH000103: Unable to initialize IndexManager nodeinfo
    at org.modeshape.jcr.JcrRepositoryFactory.getRepositoryFromConfigFile(JcrRepositoryFactory.java:373)
    at org.modeshape.jcr.JcrRepositoryFactory.getRepository(JcrRepositoryFactory.java:220)
    at org.modeshape.jcr.JcrRepositoryFactory.getRepository(JcrRepositoryFactory.java:205)
    at com.lumesse.modeshape3.jca.Modeshape3ManagedConnectionFactory.createConnectionFactory(Modeshape3ManagedConnectionFactory.java:122)
    ... 104 more
Caused by: org.hibernate.search.SearchException: HSEARCH000103: Unable to initialize IndexManager nodeinfo
    at org.hibernate.search.indexes.impl.IndexManagerHolder.createDirectoryManager(IndexManagerHolder.java:244)
    at org.hibernate.search.indexes.impl.IndexManagerHolder.buildEntityIndexBinding(IndexManagerHolder.java:111)
    at org.hibernate.search.spi.SearchFactoryBuilder.initDocumentBuilders(SearchFactoryBuilder.java:411)
    at org.hibernate.search.spi.SearchFactoryBuilder.buildNewSearchFactory(SearchFactoryBuilder.java:221)
    at org.hibernate.search.spi.SearchFactoryBuilder.buildSearchFactory(SearchFactoryBuilder.java:145)
    at org.modeshape.jcr.RepositoryQueryManager.queryEngine(RepositoryQueryManager.java:146)
    at org.modeshape.jcr.RepositoryQueryManager.getIndexes(RepositoryQueryManager.java:132)
    at org.modeshape.jcr.JcrRepository$RepositoryMonitorFactory.createMonitor(JcrRepository.java:1774)
    at org.modeshape.jcr.txn.Transactions.newMonitor(Transactions.java:220)
    at org.modeshape.jcr.txn.Transactions$BaseTransaction.createMonitor(Transactions.java:233)
    at org.modeshape.jcr.cache.document.WritableSessionCache.save(WritableSessionCache.java:377)
    at org.modeshape.jcr.cache.document.WritableSessionCache.save(WritableSessionCache.java:341)
    at org.modeshape.jcr.SystemContent.save(SystemContent.java:112)
    at org.modeshape.jcr.RepositoryNodeTypeManager.registerNodeTypes(RepositoryNodeTypeManager.java:537)
    at org.modeshape.jcr.NodeTypesImporter.importNodeTypes(NodeTypesImporter.java:75)
    at org.modeshape.jcr.JcrRepository$RunningState.postInitialize(JcrRepository.java:1260)
    at org.modeshape.jcr.JcrRepository.doStart(JcrRepository.java:358)
    at org.modeshape.jcr.JcrRepository.start(JcrRepository.java:279)
    at org.modeshape.jcr.JcrRepositoryFactory.getRepositoryFromConfigFile(JcrRepositoryFactory.java:368)
    ... 107 more
Caused by: org.hibernate.search.SearchException: Unable to retrieve CacheManager from JNDI [infinispan_configuration.xml]
    at org.hibernate.search.infinispan.CacheManagerServiceProvider.locateCacheManager(CacheManagerServiceProvider.java:119)
    at org.hibernate.search.infinispan.CacheManagerServiceProvider.start(CacheManagerServiceProvider.java:105)
    at org.hibernate.search.engine.ServiceManager$ServiceProviderWrapper.increaseCounter(ServiceManager.java:166)
    at org.hibernate.search.engine.ServiceManager.requestService(ServiceManager.java:118)
    at org.hibernate.search.spi.SearchFactoryBuilder$BuildContext.requestService(SearchFactoryBuilder.java:635)
    at org.hibernate.search.infinispan.impl.InfinispanDirectoryProvider.start(InfinispanDirectoryProvider.java:83)
    at org.hibernate.search.indexes.impl.DirectoryBasedIndexManager.initialize(DirectoryBasedIndexManager.java:93)
    at org.hibernate.search.indexes.impl.IndexManagerHolder.createDirectoryManager(IndexManagerHolder.java:241)
    ... 125 more

infinispan_configuration.xml is name of file that I use to configure my storage:

"storage" : {

        "cacheName" : "tlk_infinispan_repo",

        "cacheConfiguration" : "infinispan_configuration.xml",

...

Can anything be done to fix that?

[pjakub]

ok, I've fixed that,

now there are

14:40:19,679 ERROR [InvocationContextInterceptor] ISPN000136: Execution error

org.infinispan.util.concurrent.TimeoutException: Unable to acquire lock after [10 seconds] on key [segments_1|0|nodeinfo] for requestor [GlobalTransaction:<ubuntu-24079>:4:local]! Lock held by [null]

    at org.infinispan.util.concurrent.locks.LockManagerImpl.lock(LockManagerImpl.java:206)

    at org.infinispan.util.concurrent.locks.LockManagerImpl.acquireLock(LockManagerImpl.java:180)

    at org.infinispan.util.concurrent.locks.LockManagerImpl.acquireLock(LockManagerImpl.java:170)

    at org.infinispan.interceptors.locking.AbstractTxLockingInterceptor.lockKeyAndCheckOwnership(AbstractTxLockingInterceptor.java:209)

    at org.infinispan.interceptors.locking.AbstractTxLockingInterceptor.lockAndRegisterBackupLock(AbstractTxLockingInterceptor.java:136)

    at org.infinispan.interceptors.locking.OptimisticLockingInterceptor.acquireAllLocks(OptimisticLockingInterceptor.java:257)

    at org.infinispan.interceptors.locking.OptimisticLockingInterceptor.visitPrepareCommand(OptimisticLockingInterceptor.java:101)

    at org.infinispan.commands.tx.PrepareCommand.acceptVisitor(PrepareCommand.java:131)

...

...

14:40:20,828 ERROR [LogErrorHandler] HSEARCH000058: Exception occurred org.apache.lucene.store.LockObtainFailedException: Lock obtain timed out: org.infinispan.lucene.locking.BaseLuceneLock@175b0b5

Primary Failure:

    Entity org.modeshape.jcr.query.lucene.basic.NodeInfo  Id de0a805317f1e7mode:namespaces  Work Type  org.hibernate.search.backend.UpdateLuceneWork

org.apache.lucene.store.LockObtainFailedException: Lock obtain timed out: org.infinispan.lucene.locking.BaseLuceneLock@175b0b5

    at org.apache.lucene.store.Lock.obtain(Lock.java:84)

    at org.apache.lucene.index.IndexWriter.<init>(IndexWriter.java:1108)

    at org.hibernate.search.backend.impl.lucene.IndexWriterHolder.createNewIndexWriter(IndexWriterHolder.java:127)

    at org.hibernate.search.backend.impl.lucene.IndexWriterHolder.getIndexWriter(IndexWriterHolder.java:102)

    at org.hibernate.search.backend.impl.lucene.AbstractWorkspaceImpl.getIndexWriter(AbstractWorkspaceImpl.java:119)

    at org.hibernate.search.backend.impl.lucene.LuceneBackendQueueTask.applyUpdates(LuceneBackendQueueTask.java:99)

    at org.hibernate.search.backend.impl.lucene.LuceneBackendQueueTask.run(LuceneBackendQueueTask.java:67)

    at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:441)

    at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)

    at java.util.concurrent.FutureTask.run(FutureTask.java:138)

    at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)

    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)

    at java.lang.Thread.run(Thread.java:662)

flowing through the screen...