1 2 Previous Next 15 Replies Latest reply on Dec 18, 2012 3:15 AM by codeape

    Modeshape loses nodes during concurrency

    codeape

      Hello,

       

      We are playing around with Modeshape. Cool stuff.

       

      However, it looks like there is a concurrency problem caused by we are doing something wrong or Modeshape is broken.

       

      This is the code and other resources:

       

      Our webapp

      {code:java}package com.example.modeshape;

       

      import java.util.Iterator;

      import java.util.concurrent.atomic.AtomicInteger;

       

      import javax.inject.Inject;

      import javax.jcr.Node;

      import javax.jcr.NodeIterator;

      import javax.jcr.Property;

      import javax.jcr.PropertyIterator;

      import javax.jcr.Repository;

      import javax.jcr.RepositoryException;

      import javax.jcr.Session;

      import javax.ws.rs.Consumes;

      import javax.ws.rs.GET;

      import javax.ws.rs.PUT;

      import javax.ws.rs.Path;

      import javax.ws.rs.PathParam;

      import javax.ws.rs.Produces;

       

      import org.codehaus.jettison.json.JSONArray;

      import org.codehaus.jettison.json.JSONException;

      import org.codehaus.jettison.json.JSONObject;

      import org.infinispan.schematic.document.Json;

       

      import com.sun.jersey.json.impl.provider.entity.JSONArrayProvider;

       

      @Path("/")

      public class ModeshapeService {

          private final Repository _repository;

       

          static AtomicInteger call = new AtomicInteger();

       

          @Inject

          public ModeshapeService(Repository repository) {

              _repository = repository;

       

          }

       

          @Path("document/{path}")

          @GET

          @Produces("application/json")

          public JSONObject getDocument(@PathParam("path") String path) throws Exception {

              Session session = _repository.login("main");

              Node rootNode = session.getRootNode();

              Node node = rootNode.getNode(path);

       

              JSONObject json = jsonify(node);

       

              return json;

          }

       

          @Path("document/{path}/count")

          @GET

          @Produces("application/json")

          public JSONObject count(@PathParam("path") String path)

              throws Exception

          {

              Session session = _repository.login("main");

              JSONObject json = new JSONObject();

              try {

                  Node rootNode = session.getRootNode();

                  Node pnode = rootNode.getNode(path);

                  NodeIterator nodeIter = pnode.getNodes();

                  long size = nodeIter.getSize();

                  while (nodeIter.hasNext()) {

                      Node inode = nodeIter.nextNode();

                      System.out.println(inode.getPath());

                      NodeIterator subNodeIter = inode.getNodes();

                      size += subNodeIter.getSize();

                      while (subNodeIter.hasNext()) {

                          Node tnode = subNodeIter.nextNode();

                          System.out.println(tnode.getPath());

                      }

                  }

                  json.put("count", size);

              } finally {

                  session.logout();

              }

              return json;

          }

       

       

          @Path("document/{path}")

          @PUT

          @Consumes("application/json")

          @Produces("text/plain")

          public String createDocument(@PathParam("path") String path, JSONObject json)

              throws Exception

          {

              Integer incr = call.incrementAndGet();

              long start = System.nanoTime();

              Session session = _repository.login("main");

              int i = 0;

              try {

                  Node rootNode;

                  if (!session.getRootNode().hasNode(path)) {

                      rootNode = session.getRootNode().addNode(path);

                  } else {

                      rootNode = session.getRootNode().getNode(path);

                  }

                  Node numNode = rootNode.addNode(incr.toString());

                  for (;i < 10; i++) {

                      Node node = numNode.addNode(String.valueOf(i));

                      //session.getRootNode().addNode(path + "/" + incr.toString() + "/" + i);

                      node.setPrimaryType("nt:unstructured");

                      writeJsonToNode(json, node);

                  }

                  session.save();

                  //System.out.println("Doing PUT " + incr.toString());

       

              } finally {

                  session.logout();

              }

       

              long end = System.nanoTime();

              return String.format("Creating 10 nodes took %dms", (end - start) / 1000000);

          }

       

          private void writeJsonToNode(JSONObject json, Node node)

              throws Exception {

              Iterator keys = json.keys();

              while (keys.hasNext()) {

                  String key = (String) keys.next();

                  Object object = json.get(key);

                  if (object instanceof JSONObject) {

                      Node child = node.addNode(key);

                      writeJsonToNode((JSONObject) object, child);

                  }

                  else if (object instanceof JSONArray) {

                      JSONArray array = (JSONArray) object;

                      for (int i = 0; i < array.length(); i++) {

                          Object item = array.get(i);

                          if (item instanceof JSONObject) {

                              Node itemNode = node.addNode(key);

                              writeJsonToNode((JSONObject) item, itemNode);

                          }

                          else {

                              // ToDo: create list

                              setProperty(node, key, item);

                          }

                      }

                  }

                  else {

                      setProperty(node, key, object);

                  }

              }

          }

       

          private void setProperty(Node node, String key, Object object)

              throws Exception {

              if (object instanceof Long) {

                  node.setProperty(key, (Long)object);

              } else if (object instanceof Integer) {

                  node.setProperty(key, (Integer)object);

              } else if (object instanceof Double) {

                  node.setProperty(key, (Double)object);

              } else if (object instanceof String) {

                  node.setProperty(key, (String)object);

              } else if (object instanceof Boolean) {

                  node.setProperty(key, (Boolean)object);

              } else {

                  throw new IllegalArgumentException();

              }

          }

       

          private JSONObject jsonify(Node node)

              throws RepositoryException, JSONException

          {

              NodeIterator nodes = node.getNodes();

              JSONObject json = new JSONObject();

              while (nodes.hasNext()) {

                  Node next = (Node) nodes.next();

                  NodeIterator nodes2 = node.getNodes(next.getName());

                  if (nodes2.getSize() > 1) {

                      JSONArray array = new JSONArray();

                      nodes.skip(nodes2.getSize() - 1);

                      while (nodes2.hasNext()) {

                          array.put(jsonify(nodes2.nextNode()));

                      }

                      json.put(next.getName(), array);

                  }

                  else {

                      json.put(next.getName(), jsonify(next));

                  }

              }

       

              PropertyIterator properties = node.getProperties();

              while (properties.hasNext()) {

                  Property property = (Property) properties.next();

                  json.put(property.getName(), property.getString());

              }

              return json;

          }

      }{code}

       

       

      Modeshape conf

      {code}{

          "name" : "example",

          "jndiName" : "jcr/example",

          "monitoring" : {

              "enabled" : true

          },

          "workspaces" : {

              "default" : "main",

              "allowCreation" : true

          },

          "storage" : {

              "cacheConfiguration" : "infinispan.xml",

              "cacheName" : "example",

              "transactionManagerLookup" = "org.infinispan.transaction.lookup.GenericTransactionManagerLookup",

              "binaryStorage" : {

                  "directory" : "modeshape/binary",

                  "minimumBinarySizeInBytes" : 4096,

                  "type" = "file"

              }

          },

      }{code}

       

      Infinispan conf

      {code:xml}<infinispan>

        <namedCache name="example">

          <transaction

           transactionManagerLookupClass="org.infinispan.transaction.lookup.GenericTransactionManagerLookup"

           transactionMode="TRANSACTIONAL"

           lockingMode="OPTIMISTIC"/>

          <loaders shared="true"

                   passivation="false"

                   preload="false">

            <loader class="org.infinispan.loaders.bdbje.BdbjeCacheStore"

                    fetchPersistentState="true"

                    purgeOnStartup="false">

              <properties>

                <property name="location" value="/opt/modeshape/cache"/>

              </properties>

            </loader>

       

          </loaders>

        </namedCache>

      </infinispan>{code}

       

       

      We test this code by sending this piece of json to the web service (with cleard cache by deleting the berkeley db files etc.) with Apache Bench:

      {code}{

          "title": "Titulum",

          "author": "Patrik",

          "body": "Marie Söderqvist, vd för Livsmedelsföretagen, och Åke Rutegård, vd för Kött- och Charkföretagen, ger kommunerna skulden för att livsmedelsbutiker säljer färgat och förfalskat kött. Kommunernas livsmedelskontroller brister och staten borde därför ta över deras ansvar, skriver de i en debattartikel (Brännpunkt 22/11).\nDet Söderqvist och Rutegård ägnar sig åt här är att försöka skjuta ifrån sig ansvaret. Istället för att skylla ifrån sig behöver de ställa sig frågan varför chark- och livsmedelsföretagen brister i sitt ansvar. Regelverket markerar tydligt att egenkontroll är en förutsättning för att få driva livsmedelsverksamhet. Det är företagarna själva som ska se till att varorna inte är falska.\n\nKommunernas kontrollansvar handlar framför allt om att se till att livsmedel förvaras och hanteras på ett bra sätt. Det sker inga omfattande provtagningar vilket krävs för att upptäcka en sådan här avancerad brottslig verksamhet. Sådana provtagningar skulle kräva oerhörda resurser och bli otroligt kostsam för branschen.\n\nDet är dessutom fel att tro att livsmedelskontroller skulle bli effektivare bara för att staten tar över ansvaret. Så sent som 2010 riktade EU-kommissionen kritik mot Livsmedelsverkets kontrollverksamhet eftersom myndigheten för sällan vidtog åtgärder mot bristfälliga verksamheter.\n\nNär ansvaret skjuts uppåt går man miste om något viktigt: närheten. Det är den som ger kommunerna en god kännedom om det lokala näringslivet. Det är den som gör att inspektörer inte bara kan rycka ut med kort varsel utan också röra sig i butikerna i sin vardag.\n\nDetta väljer Söderqvist och Rutegård att bortse från. Istället hävdar de att kommunerna inte vill släppa ifrån sig ansvaret eftersom livsmedelskontrollerna är en intäktskälla. Detta stämmer inte. Ingen kommun går med vinst på sina livsmedelskontroller. Taxorna som företagen betalar täcker ofta bara en del av kommunernas kontrollverksamhet.\n\nKommunerna välkomnar att Livsmedelsverket utifrån sina revisioner ingriper i de fall då kommuner brister i sitt kontrolluppdrag. Men vid den här typen av brottsliga handlingar är det ointressant att diskutera vem som gör kontrollen bäst. Det som är viktigt är att hela livsmedelskontrollkedjan hjälps åt.\n\nHÅKAN SÖRMAN\n\nvd Sveriges Kommuner och Landsting\n",

          "rating": 3.0,

          "categorization" : {

              "subject": [ "subject-01", "subject-02033" ],

              "person": ["Håkan Sörman", "Marie Söderqvist"],

          },

      }{code}

       

      When we send this json with a single threaded client there is no problem:

       

      {code}>$ ab -T "application/json" -c 1 -n 1000 -u src/main/resources/json http://localhost:8080/document/snabel

      This is ApacheBench, Version 2.3 <$Revision: 1373084 $>

      Copyright 1996 Adam Twiss, Zeus Technology Ltd, http://www.zeustech.net/

      Licensed to The Apache Software Foundation, http://www.apache.org/

       

       

      Benchmarking localhost (be patient)

      Completed 100 requests

      Completed 200 requests

      Completed 300 requests

      Completed 400 requests

      Completed 500 requests

      Completed 600 requests

      Completed 700 requests

      Completed 800 requests

      Completed 900 requests

      Completed 1000 requests

      Finished 1000 requests

       

       

       

       

      Server Software:        Jetty(8.1.8.v20121106)

      Server Hostname:        localhost

      Server Port:            8080

       

       

      Document Path:          /document/snabel

      Document Length:        29 bytes

       

       

      Concurrency Level:      1

      Time taken for tests:   27.630 seconds

      Complete requests:      1000

      Failed requests:        999

         (Connect: 0, Receive: 0, Length: 999, Exceptions: 0)

      Write errors:           0

      Total transferred:      104008 bytes

      Total body sent:        2669000

      HTML transferred:       27008 bytes

      Requests per second:    36.19 [#/sec] (mean)

      Time per request:       27.630 [ms] (mean)

      Time per request:       27.630 [ms] (mean, across all concurrent requests)

      Transfer rate:          3.68 [Kbytes/sec] received

                              94.33 kb/s sent

                              98.01 kb/s total

       

       

      Connection Times (ms)

                    min  mean[+/-sd] median   max

      Connect:        0    0   0.0      0       0

      Processing:    17   27  66.4     22    2031

      Waiting:       17   27  66.4     22    2031

      Total:         18   28  66.4     22    2031

       

       

      Percentage of the requests served within a certain time (ms)

        50%     22

        66%     24

        75%     25

        80%     26

        90%     30

        95%     41

        98%     67

        99%     79

      100%   2031 (longest request)

       

      >$ curl http://localhost:8080/document/snabel/count

      {"count":11000}

      {code}

       

       

      We get the expected amount of nodes (11000).

       

      Then stop the web service and clear the cache then start the web service and run the same test but with multiple threads (16 in this case):

      {code}>$ ab -T "application/json" -c 16 -n 1000 -u src/main/resources/json http://localhost:8080/document/snabel

      This is ApacheBench, Version 2.3 <$Revision: 1373084 $>

      Copyright 1996 Adam Twiss, Zeus Technology Ltd, http://www.zeustech.net/

      Licensed to The Apache Software Foundation, http://www.apache.org/

       

       

      Benchmarking localhost (be patient)

      Completed 100 requests

      Completed 200 requests

      Completed 300 requests

      Completed 400 requests

      Completed 500 requests

      Completed 600 requests

      Completed 700 requests

      Completed 800 requests

      Completed 900 requests

      Completed 1000 requests

      Finished 1000 requests

       

       

       

       

      Server Software:        Jetty(8.1.8.v20121106)

      Server Hostname:        localhost

      Server Port:            8080

       

       

      Document Path:          /document/snabel

      Document Length:        29 bytes

       

       

      Concurrency Level:      16

      Time taken for tests:   19.845 seconds

      Complete requests:      1000

      Failed requests:        983

         (Connect: 0, Receive: 0, Length: 983, Exceptions: 0)

      Write errors:           0

      Total transferred:      105017 bytes

      Total body sent:        2669000

      HTML transferred:       28017 bytes

      Requests per second:    50.39 [#/sec] (mean)

      Time per request:       317.523 [ms] (mean)

      Time per request:       19.845 [ms] (mean, across all concurrent requests)

      Transfer rate:          5.17 [Kbytes/sec] received

                              131.34 kb/s sent

                              136.51 kb/s total

       

       

      Connection Times (ms)

                    min  mean[+/-sd] median   max

      Connect:        0    0   0.1      0       2

      Processing:   145  316 385.5    242    3620

      Waiting:      145  316 385.5    242    3620

      Total:        146  316 385.5    243    3621

       

       

      Percentage of the requests served within a certain time (ms)

        50%    243

        66%    263

        75%    277

        80%    289

        90%    388

        95%    530

        98%    780

        99%   3192

      100%   3621 (longest request)

       

      >$ curl http://localhost:8080/document/snabel/count

      {"count":6688}

      {code}

       

      Now we only find half of the nodes (6688). And this happens every time.

       

      The only Exception we get is the following and it appears in both cases (for the first call to the web service):

      {code}

      2012-dec-06 11:07:29 org.modeshape.jcr.JcrRepository$RunningState bindIntoJndi

      : Error while binding repository 'Polopoly' into JNDI at 'jcr/polopoly' : jcr is not bound

      javax.naming.NameNotFoundException: jcr is not bound

                at org.eclipse.jetty.jndi.local.localContextRoot.bind(localContextRoot.java:611)

                at org.eclipse.jetty.jndi.local.localContextRoot.bind(localContextRoot.java:550)

                at javax.naming.InitialContext.bind(InitialContext.java:400)

                at org.modeshape.jcr.JcrRepository$RunningState.bindIntoJndi(JcrRepository.java:1515)

                at org.modeshape.jcr.JcrRepository$RunningState.<init>(JcrRepository.java:1197)

                at org.modeshape.jcr.JcrRepository$RunningState.<init>(JcrRepository.java:960)

                at org.modeshape.jcr.JcrRepository.doStart(JcrRepository.java:352)

                at org.modeshape.jcr.JcrRepository.login(JcrRepository.java:609)

                at org.modeshape.jcr.JcrRepository.login(JcrRepository.java:578)

                at org.modeshape.jcr.JcrRepository.login(JcrRepository.java:147)

                at com.polopoly.modeshape.ModeshapeService.createDocument(ModeshapeService.java:93)

                at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)

                at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)

                at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)

                at java.lang.reflect.Method.invoke(Method.java:597)

                at com.sun.jersey.spi.container.JavaMethodInvokerFactory$1.invoke(JavaMethodInvokerFactory.java:60)

                at com.sun.jersey.server.impl.model.method.dispatch.AbstractResourceMethodDispatchProvider$TypeOutInvoker._dispatch(AbstractResourceMethodDispatchProvider.java:185)

                at com.sun.jersey.server.impl.model.method.dispatch.ResourceJavaMethodDispatcher.dispatch(ResourceJavaMethodDispatcher.java:75)

                at com.sun.jersey.server.impl.uri.rules.HttpMethodRule.accept(HttpMethodRule.java:302)

                at com.sun.jersey.server.impl.uri.rules.RightHandPathRule.accept(RightHandPathRule.java:147)

                at com.sun.jersey.server.impl.uri.rules.ResourceClassRule.accept(ResourceClassRule.java:108)

                at com.sun.jersey.server.impl.uri.rules.RightHandPathRule.accept(RightHandPathRule.java:147)

                at com.sun.jersey.server.impl.uri.rules.RootResourceClassesRule.accept(RootResourceClassesRule.java:84)

                at com.sun.jersey.server.impl.application.WebApplicationImpl._handleRequest(WebApplicationImpl.java:1480)

                at com.sun.jersey.server.impl.application.WebApplicationImpl._handleRequest(WebApplicationImpl.java:1411)

                at com.sun.jersey.server.impl.application.WebApplicationImpl.handleRequest(WebApplicationImpl.java:1360)

                at com.sun.jersey.server.impl.application.WebApplicationImpl.handleRequest(WebApplicationImpl.java:1350)

                at com.sun.jersey.spi.container.servlet.WebComponent.service(WebComponent.java:416)

                at com.sun.jersey.spi.container.servlet.ServletContainer.service(ServletContainer.java:538)

                at com.sun.jersey.spi.container.servlet.ServletContainer.service(ServletContainer.java:716)

                at javax.servlet.http.HttpServlet.service(HttpServlet.java:848)

                at com.google.inject.servlet.ServletDefinition.doService(ServletDefinition.java:263)

                at com.google.inject.servlet.ServletDefinition.service(ServletDefinition.java:178)

                at com.google.inject.servlet.ManagedServletPipeline.service(ManagedServletPipeline.java:91)

                at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:62)

                at com.google.inject.servlet.ManagedFilterPipeline.dispatch(ManagedFilterPipeline.java:118)

                at com.google.inject.servlet.GuiceFilter.doFilter(GuiceFilter.java:113)

                at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1307)

                at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:453)

                at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)

                at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:560)

                at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:231)

                at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1072)

                at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:382)

                at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)

                at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1006)

                at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)

                at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:255)

                at org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:154)

                at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)

                at org.eclipse.jetty.server.Server.handle(Server.java:365)

                at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:485)

                at org.eclipse.jetty.server.AbstractHttpConnection.content(AbstractHttpConnection.java:937)

                at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.content(AbstractHttpConnection.java:998)

                at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:856)

                at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:240)

                at org.eclipse.jetty.server.AsyncHttpConnection.handle(AsyncHttpConnection.java:82)

                at org.eclipse.jetty.io.nio.SelectChannelEndPoint.handle(SelectChannelEndPoint.java:628)

                at org.eclipse.jetty.io.nio.SelectChannelEndPoint$1.run(SelectChannelEndPoint.java:52)

                at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)

                at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)

                at java.lang.Thread.run(Thread.java:680){code}

       

       

      I have attached the code in modeshape-pp-test.tar.gz. You can run it by:

       

      {code}mvn clean install

      mvn jetty:run{code}

       

       

      We wonder:

      1. Why do nodes disappear (are we doing wrong or is it a bug)?

      2. Why do we get a repository binding error?

       

      Note: I have also tried using File chace instead of Berkely cache with the same result:

       

      {code:xml}<infinispan>

        <namedCache name="Polopoly">

          <transaction

           transactionManagerLookupClass="org.infinispan.transaction.lookup.GenericTransactionManagerLookup"

           transactionMode="TRANSACTIONAL"

           lockingMode="OPTIMISTIC"/>

          <loaders shared="true"

                   passivation="false"

                   preload="false">

            <loader

             class="org.infinispan.loaders.file.FileCacheStore"

             fetchPersistentState="true"

             ignoreModifications="false"

             purgeOnStartup="false">

              <properties>

                <property name="location" value="/opt/modeshape/cache"/>

              </properties>

            </loader>

          </loaders>

        </namedCache>

      </infinispan>{code}

       

      After starting with clean/empty cache i got this:

      {code}ab -T "application/json" -c 16 -n 1000 -u src/main/resources/json http://localhost:8080/document/snabel

      This is ApacheBench, Version 2.3 <$Revision: 1373084 $>

      Copyright 1996 Adam Twiss, Zeus Technology Ltd, http://www.zeustech.net/

      Licensed to The Apache Software Foundation, http://www.apache.org/

       

       

      Benchmarking localhost (be patient)

      Completed 100 requests

      Completed 200 requests

      Completed 300 requests

      Completed 400 requests

      Completed 500 requests

      Completed 600 requests

      Completed 700 requests

      Completed 800 requests

      Completed 900 requests

      Completed 1000 requests

      Finished 1000 requests

       

       

       

       

      Server Software:        Jetty(8.1.8.v20121106)

      Server Hostname:        localhost

      Server Port:            8080

       

       

      Document Path:          /document/snabel

      Document Length:        29 bytes

       

       

      Concurrency Level:      16

      Time taken for tests:   28.082 seconds

      Complete requests:      1000

      Failed requests:        907

         (Connect: 0, Receive: 0, Length: 907, Exceptions: 0)

      Write errors:           0

      Total transferred:      105092 bytes

      Total body sent:        2669000

      HTML transferred:       28092 bytes

      Requests per second:    35.61 [#/sec] (mean)

      Time per request:       449.314 [ms] (mean)

      Time per request:       28.082 [ms] (mean, across all concurrent requests)

      Transfer rate:          3.65 [Kbytes/sec] received

                              92.82 kb/s sent

                              96.47 kb/s total

       

       

      Connection Times (ms)

                    min  mean[+/-sd] median   max

      Connect:        0    0   0.2      0       2

      Processing:    58  447 460.0    297    3363

      Waiting:       58  447 460.1    296    3363

      Total:         59  448 460.1    297    3363

       

       

      Percentage of the requests served within a certain time (ms)

        50%    297

        66%    318

        75%    343

        80%    368

        90%    810

        95%   1534

        98%   1773

        99%   2976

      100%   3363 (longest request)

       

      >$ curl http://localhost:8080/document/snabel/count

      {"count":3729}{code}

       

      Only 3729 nodes found.

       

       

      Thank you in advance.

        • 1. Re: Modeshape looses nodes during concurrency
          codeape

          Hello again.

           

          I am using Modeshape 3.0.1.Final, I forgot to mention that.

          • 2. Re: Modeshape looses nodes during concurrency
            codeape

            Removed the jindi name like described in this link: https://community.jboss.org/thread/204527

             

            The "javax.naming.NameNotFoundException: jcr is not bound" is gone now.

             

            We still loose nodes.

            • 3. Re: Modeshape looses nodes during concurrency
              rhauch

              The JNDI error might be due to the way that Jetty's JNDI works. IIRC, Jetty requires the JNDI names to be fully-qualified, so try changing the "jndiName" value to "java:/comp/env/jcr/example" (rather than "jcr/example") in your ModeShape configuration.

               

              As for the other bigger issue, it will take a bit more time to investigate. Your configurations appear to be okay, but I may be missing something. Can you please log an issue in our JIRA with all of the relevant information? This is a big enough problem that we don't want to lose track of it. We'll investigate it and see if we can identify the problem.

               

              In the mean time, you might also try to change the Infinispan configuration from "OPTIMISTIC" to "PESSIMISTIC", and see how that affects the results.

              • 4. Re: Modeshape looses nodes during concurrency
                codeape

                Thanks for the input.

                 

                I think I am closer to fix the JINDI problem thanks to you. I did the change you told me to do but then I got this error:

                 

                Error while binding repository 'Polopoly' into JNDI at 'java:comp/env/jcr/polopoly' : This context is immutable

                javax.naming.NamingException: This context is immutable

                          at org.eclipse.jetty.jndi.NamingContext.bind(NamingContext.java:202)

                          at org.eclipse.jetty.jndi.NamingContext.bind(NamingContext.java:270)

                          at org.eclipse.jetty.jndi.NamingContext.bind(NamingContext.java:290)

                          at org.eclipse.jetty.jndi.java.javaRootURLContext.bind(javaRootURLContext.java:129)

                          at javax.naming.InitialContext.bind(InitialContext.java:400)

                          at org.modeshape.jcr.JcrRepository$RunningState.bindIntoJndi(JcrRepository.java:1515)

                          at org.modeshape.jcr.JcrRepository$RunningState.<init>(JcrRepository.java:1197)

                          at org.modeshape.jcr.JcrRepository$RunningState.<init>(JcrRepository.java:960)

                          at org.modeshape.jcr.JcrRepository.doStart(JcrRepository.java:352)

                          at org.modeshape.jcr.JcrRepository.login(JcrRepository.java:609)

                          at org.modeshape.jcr.JcrRepository.login(JcrRepository.java:578)

                          at org.modeshape.jcr.JcrRepository.login(JcrRepository.java:147)

                 

                It appears (I googled it) that the java:comp/env namespace is immutable once the webapp has been started because the servlet spec/j2ee spec requires this. I need to figure out how to get around this.

                 

                When I changed the locking to PESSIMISTIC I got this

                 

                Error while refreshing locks for the "Polopoly" repository

                java.lang.NullPointerException

                          at org.modeshape.jcr.RepositoryLockManager$ModeShapeLock.<init>(RepositoryLockManager.java:408)

                          at org.modeshape.jcr.RepositoryLockManager.refreshFromSystem(RepositoryLockManager.java:84)

                          at org.modeshape.jcr.JcrRepository$RunningState.<init>(JcrRepository.java:1087)

                          at org.modeshape.jcr.JcrRepository$RunningState.<init>(JcrRepository.java:960)

                          at org.modeshape.jcr.JcrRepository.doStart(JcrRepository.java:352)

                          at org.modeshape.jcr.JcrRepository.login(JcrRepository.java:609)

                          at org.modeshape.jcr.JcrRepository.login(JcrRepository.java:578)

                          at org.modeshape.jcr.JcrRepository.login(JcrRepository.java:147)

                 

                And 7566 nodes where found out of 11000. I let the server run and after a while I got this:

                 

                ALLVARLIG: Error during background garbage collection: null

                java.lang.NullPointerException

                          at org.modeshape.jcr.SystemContent.first(SystemContent.java:638)

                          at org.modeshape.jcr.SystemContent.first(SystemContent.java:632)

                          at org.modeshape.jcr.SystemContent.cleanUpLocks(SystemContent.java:800)

                          at org.modeshape.jcr.JcrRepository$RunningState.cleanUpLocks(JcrRepository.java:1616)

                          at org.modeshape.jcr.JcrRepository.cleanUp(JcrRepository.java:877)

                          at org.modeshape.jcr.ModeShapeEngine$GarbageCollectionTask.run(ModeShapeEngine.java:135)

                          at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:439)

                          at java.util.concurrent.FutureTask$Sync.innerRunAndReset(FutureTask.java:317)

                          at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:150)

                          at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$101(ScheduledThreadPoolExecutor.java:98)

                          at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.runPeriodic(ScheduledThreadPoolExecutor.java:180)

                          at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:204)

                          at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)

                          at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)

                          at java.lang.Thread.run(Thread.java:680)

                2012-dec-10 07:15:06 org.modeshape.jcr.JcrRepository$RunningState cleanUpLocks

                ALLVARLIG: Error during background garbage collection: null

                java.lang.NullPointerException

                          at org.modeshape.jcr.SystemContent.first(SystemContent.java:638)

                          at org.modeshape.jcr.SystemContent.first(SystemContent.java:632)

                          at org.modeshape.jcr.SystemContent.cleanUpLocks(SystemContent.java:800)

                          at org.modeshape.jcr.JcrRepository$RunningState.cleanUpLocks(JcrRepository.java:1616)

                          at org.modeshape.jcr.JcrRepository.cleanUp(JcrRepository.java:877)

                          at org.modeshape.jcr.ModeShapeEngine$GarbageCollectionTask.run(ModeShapeEngine.java:135)

                          at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:439)

                          at java.util.concurrent.FutureTask$Sync.innerRunAndReset(FutureTask.java:317)

                          at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:150)

                          at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$101(ScheduledThreadPoolExecutor.java:98)

                          at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.runPeriodic(ScheduledThreadPoolExecutor.java:180)

                          at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:204)

                          at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)

                          at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)

                          at java.lang.Thread.run(Thread.java:680)

                2012-dec-10 07:20:06 org.modeshape.jcr.JcrRepository$RunningState cleanUpLocks

                ALLVARLIG: Error during background garbage collection: null

                java.lang.NullPointerException

                          at org.modeshape.jcr.SystemContent.first(SystemContent.java:638)

                          at org.modeshape.jcr.SystemContent.first(SystemContent.java:632)

                          at org.modeshape.jcr.SystemContent.cleanUpLocks(SystemContent.java:800)

                          at org.modeshape.jcr.JcrRepository$RunningState.cleanUpLocks(JcrRepository.java:1616)

                          at org.modeshape.jcr.JcrRepository.cleanUp(JcrRepository.java:877)

                          at org.modeshape.jcr.ModeShapeEngine$GarbageCollectionTask.run(ModeShapeEngine.java:135)

                          at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:439)

                          at java.util.concurrent.FutureTask$Sync.innerRunAndReset(FutureTask.java:317)

                          at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:150)

                          at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$101(ScheduledThreadPoolExecutor.java:98)

                          at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.runPeriodic(ScheduledThreadPoolExecutor.java:180)

                          at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:204)

                          at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)

                          at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)

                          at java.lang.Thread.run(Thread.java:680)

                2012-dec-10 07:25:06 org.modeshape.jcr.JcrRepository$RunningState cleanUpLocks

                ALLVARLIG: Error during background garbage collection: null

                java.lang.NullPointerException

                          at org.modeshape.jcr.SystemContent.first(SystemContent.java:638)

                          at org.modeshape.jcr.SystemContent.first(SystemContent.java:632)

                          at org.modeshape.jcr.SystemContent.cleanUpLocks(SystemContent.java:800)

                          at org.modeshape.jcr.JcrRepository$RunningState.cleanUpLocks(JcrRepository.java:1616)

                          at org.modeshape.jcr.JcrRepository.cleanUp(JcrRepository.java:877)

                          at org.modeshape.jcr.ModeShapeEngine$GarbageCollectionTask.run(ModeShapeEngine.java:135)

                          at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:439)

                          at java.util.concurrent.FutureTask$Sync.innerRunAndReset(FutureTask.java:317)

                          at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:150)

                          at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$101(ScheduledThreadPoolExecutor.java:98)

                          at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.runPeriodic(ScheduledThreadPoolExecutor.java:180)

                          at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:204)

                          at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)

                          at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)

                          at java.lang.Thread.run(Thread.java:680)

                2012-dec-10 07:30:06 org.modeshape.jcr.JcrRepository$RunningState cleanUpLocks

                ALLVARLIG: Error during background garbage collection: null

                java.lang.NullPointerException

                          at org.modeshape.jcr.SystemContent.first(SystemContent.java:638)

                          at org.modeshape.jcr.SystemContent.first(SystemContent.java:632)

                          at org.modeshape.jcr.SystemContent.cleanUpLocks(SystemContent.java:800)

                          at org.modeshape.jcr.JcrRepository$RunningState.cleanUpLocks(JcrRepository.java:1616)

                          at org.modeshape.jcr.JcrRepository.cleanUp(JcrRepository.java:877)

                          at org.modeshape.jcr.ModeShapeEngine$GarbageCollectionTask.run(ModeShapeEngine.java:135)

                          at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:439)

                          at java.util.concurrent.FutureTask$Sync.innerRunAndReset(FutureTask.java:317)

                          at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:150)

                          at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$101(ScheduledThreadPoolExecutor.java:98)

                          at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.runPeriodic(ScheduledThreadPoolExecutor.java:180)

                          at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:204)

                          at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)

                          at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)

                          at java.lang.Thread.run(Thread.java:680)

                • 5. Re: Modeshape looses nodes during concurrency
                  codeape

                  Randall Hauch wrote:

                   

                  Can you please log an issue in our JIRA with all of the relevant information?

                   

                  Yes. I will do it today or tomorrow.

                  • 6. Re: Modeshape looses nodes during concurrency
                    codeape

                    Added: https://issues.jboss.org/browse/MODE-1734

                     

                    Edit: I attached to the JIRA issue a slightly uppdated version of the code without jindi (causing Exception) because that is a sepparate problem that we probably have introduced our selfs.

                    • 7. Re: Modeshape looses nodes during concurrency
                      rhauch

                      Thanks for logging the issue. I bumped the priority up to blocker, and I'll start looking at it today.

                       

                      As for the JNDI issue, the repository is bound to JNDI programmatically using this code, so maybe that will help track down some of the peculiarities of Jetty's JNDI.

                      • 8. Re: Modeshape looses nodes during concurrency
                        codeape

                        Randall Hauch wrote:

                         

                        Thanks for logging the issue. I bumped the priority up to blocker, and I'll start looking at it today.

                         

                        As for the JNDI issue, the repository is bound to JNDI programmatically using this code, so maybe that will help track down some of the peculiarities of Jetty's JNDI.

                         

                        Thank you. If you have any questions or anything, you can allways contact me. I will keep on digging =)

                         

                        I am going to look at the jndi now.

                        • 9. Re: Modeshape loses nodes during concurrency
                          rhauch

                          I was able to spend much of yesterday looking into this, and I think there are some issues to be fixed in ModeShape. I'll continue working today to identify the bugs and improvements to make situations like this easier for client applications, and I'll continue to track those on MODE-1734.

                           

                          But the application is not really written in a concurrent fashion, which leads to some other issues. I'll share the lessons here, hoping that it helps people looking for information on how to write concurrent JCR applications.

                           

                          Yes, the application is correctly using a static AtomicInteger to correctly maintain a count of the number of PUT calls and non-clashing node names created at the second level (below the root). However, an application or service that uses JCR still must use good concurrency techniques; there's nothing magical about JCR that obviates an application from having to do that. For example, the client really does nothing but call the "createDocument" method:

                           

                              public String createDocument(@PathParam("path") String path, JSONObject json) throws Exception {
                                  Integer incr = call.incrementAndGet();
                                  long start = System.nanoTime();
                                  Session session = _repository.login("main");
                                  int i = 0;
                                  try {
                                      Node rootNode;
                                      if (!session.getRootNode().hasNode(path)) {
                                          rootNode = session.getRootNode().addNode(path);
                                      } else {
                                          rootNode = session.getRootNode().getNode(path);
                                      }
                                      Node numNode = rootNode.addNode(incr.toString());
                                      for (;i < 10; i++) {
                                          Node node = numNode.addNode(String.valueOf(i));
                                          //session.getRootNode().addNode(path + "/" + incr.toString() + "/" + i);
                                          node.setPrimaryType("nt:unstructured");
                                          writeJsonToNode(json, node);
                                      }
                                      session.save();
                                      //System.out.println("Doing PUT " + incr.toString());
                          
                                  } finally {
                                      session.logout();
                                  }
                          
                                  long end = System.nanoTime();
                                  return String.format("Creating 10 nodes took %dms", (end - start) / 1000000);
                              }
                          

                           

                          Now, consider the case when Apache Bench starts running and has created it's 16 threads. (Because ModeShape lazily initializes the repository, one thread will be the first to get the repository and the others will block. Therefore, there's a good chance on a quad core machine that quite a few of these threads will run concurrently, albeit dependent upon thread context switching.) For concurrent calls to the RESTful service, a separate ModeshapeService instance will be created and injected with the Repository instance and each response thread will call "createDocument". Thus the "createDocument" method is being called against the same Repository instance, but the method is not really written to handle such concurrency.

                           

                          We can see the flaw in the application logic if we look at how just the first three server threads might interact. We'll call these threads A, B and C, and we'll assume (like in the test run) that the value of "path" is "snabel" for all invocations. Here's a perfectly probable timeline:

                           

                          1. Thread A obtains its session
                          2. Thread B obtains its session
                          3. Thread C obtains its session
                          4. Thread A gets the root node and looks for a child node named "snabel", but there is none.
                          5. Thread B gets the root node and looks for a child node named "snabel", but there is none.
                          6. Thread C gets the root node and looks for a child node named "snabel", but there is none.
                          7. Thread A creates "/snabel" using its session.
                          8. Thread B creates "/snabel" using its session.
                          9. Thread C creates "/snabel" using its session.
                          10. Thread A creates "/snabel/1" using its session.
                          11. Thread B creates "/snabel/2" using its session.
                          12. Thread C creates "/snabel/3" using its session.
                          13. Thread A creates "/snabel/1/1" through "/snabel/1/10" (and the nested nodes) using its session.
                          14. Thread B creates "/snabel/2/1" through "/snabel/2/10" (and the nested nodes) using its session.
                          15. Thread C creates "/snabel/3/1" through "/snabel/3/10" (and the nested nodes) using its session.


                          At this point, each of these Sessions is trying to create the "/snabel" node. The path bug is getting in the way, but assuming this is fixed the three threads would each create a node named "snabel", and all would succeed because the root node allows child nodes with the same name (different same-name-sibling indexes would be generated by ModeShape). And all of the children created within the session would be placed under the respective "/snabel[n]" node. BTW, the worst case is that as many as 16 top-level "snabel[n]" nodes are created if all 16 threads call "createDocument" and get to the "if (!session.getRootNode().hasNode(path)) " line before any of the first 16 threads reach the "session.save()" line.
                          (I've actually run the test lots of times, and all but once the path bug prevents the creation of multiple top-level nodes. Only once did I actually see "/snabel[1]" and "/snabel[2]" nodes, and no other same-name-sibling nodes were created because of the path bug.)The proper way to create one and only one top-level node is to change the application to:

                          1. create that top-level node with initial content (which happens only once when the repository is first initialized, regardless of how many processes in a cluster are started at the same time), or
                          2. use JCR locks; or
                          3. use normal Java concurrency techniques (e.g. locks) to create that top-level node only once (this does not work in a clustered application).

                           

                          For example, option #3 can be done with the following addition to the code:

                           

                              private static final Lock pathWriterLock = new ReentrantLock();
                          
                              private Node getTopNode( Session session, String path ) throws Exception {
                                  if (!session.getRootNode().hasNode(path)) {
                                      // The node doesn't yet exist, so get a lock and try again ...
                                      pathWriterLock.lock();
                                      try {
                                          // Another thread may have created the node while we waited for the lock, so
                                          // so look for the node again ...
                                          session.refresh(false);
                                          if (!session.getRootNode().hasNode(path)) {
                                              session.getRootNode().addNode(path);
                                              System.out.println("Created node at " + path );
                                              session.save();
                                              System.out.println("Saved new node at " + path );
                                              long numTopLevelNodes = session.getRootNode().getNodes().getSize();
                                              System.out.println("Number of top-level nodes: " + numTopLevelNodes );
                                          }
                                      } finally {
                                          pathWriterLock.unlock();
                                      }
                                  }
                                  return session.getRootNode().getNode(path);
                              }
                          

                           

                          and then changing the "createDocument(...)" method to call this method:

                           

                             public String createDocument(@PathParam("path") String path, JSONObject json)
                                  throws Exception
                              {
                                  try {
                                      Integer incr = call.incrementAndGet();
                                      long start = System.nanoTime();
                                      Session session = _repository.login("main");
                                      int i = 0;
                                      try {
                                          Node rootNode = getTopNode(session,path);
                                          String nodeName = incr.toString();
                                          Node numNode = rootNode.addNode(nodeName);
                                          for (;i < 10; i++) {
                                              Node node = numNode.addNode(String.valueOf(i));
                                              node.setPrimaryType("nt:unstructured");
                                              writeJsonToNode(json, node);
                                          }
                                          session.save();
                                          System.out.println("Doing PUT " + nodeName + " @ " + path);
                                      } finally {
                                          session.logout();
                                      }
                                      long end = System.nanoTime();
                                      return String.format("Creating 10 nodes took %dms", (end - start) / 1000000);
                                  } catch ( Exception e ) {
                                      e.printStackTrace(System.out);
                                      throw e;
                                  }
                              }
                          

                           

                          Here, the "getTopNode" method is called each time the "createDocument" method is called, but the "getTopNode" method uses a static lock to ensure that only one thread can create the node at the specified path. This fixes how the top-level node is created, and now the test runs with no exceptions. However, there still is the problem of nodes being dropped.

                           

                          I'll continue to work on the bugs that are causing the nodes to be dropped and paths to throw exceptions (in the unmodified web app).

                          1 of 1 people found this helpful
                          • 10. Re: Modeshape loses nodes during concurrency
                            rhauch

                            I've successfully created some unit tests that reliably replicate the problem of lost nodes. See the issue for details.

                            • 11. Re: Modeshape loses nodes during concurrency
                              codeape

                              Thanks for the input Randal. I have played around with your code now

                               

                              We are going to use several fronts (clustered application) so we are going to use JCR locks. It took a while before I could wrap my head around the LockManager but I have something that works

                               

                              I started playing with this and changed ModeshapeService to this:

                               

                               

                              {code:java}

                                  private static final ReentrantLock pathWriterLock = new ReentrantLock();

                               

                                  @Inject

                                  public ModeshapeService(Repository repository)

                                      throws LoginException, NoSuchWorkspaceException, RepositoryException

                                  {

                                      _repository = repository;

                                      Session session = _repository.login("main");

                                      try {

                                          if (!session.getRootNode().isNodeType("mix:lockable")) {

                                              pathWriterLock.lock();

                                              try {

                                                  session.refresh(false);

                                                  if (!session.getRootNode().isNodeType("mix:lockable")) {

                                                      session.getRootNode().addMixin("mix:lockable");

                                                      session.save();

                                                      System.out.println("Added lockable to root (thread " + Thread.currentThread().getId() + ")");

                                                  }

                                              } finally {

                                                  System.out.println("Root unlocked  (thread " + Thread.currentThread().getId() + ")");

                                                  pathWriterLock.unlock();

                                              }

                                          }

                                      } finally {

                                          session.logout();

                                      }

                                  }

                               

                                  private Node getTopNode( Session session, String path ) throws Exception {

                                      if (!session.getRootNode().hasNode(path)) {

                                          // The node doesn't yet exist, so get a lock and try again ...

                                          Boolean retry = true;

                                          while (retry) {

                                              retry = false;

                                              Lock lock = null;

                                              if(!session.getWorkspace().getLockManager().isLocked(session.getRootNode().getPath())) {

                                                  try {

                                                      do {

                                                          Thread.sleep(10);

                                                          session.refresh(false);

                                                      } while(session.getWorkspace().getLockManager().isLocked(

                                                              session.getRootNode().getPath()));

                               

                                                      // Another thread may have created the node while we waited for the lock,

                                                      // so look for the node again

                                                      if (!session.getRootNode().hasNode(path)) {

                                                          lock = session.getWorkspace().getLockManager().lock(

                                                                  session.getRootNode().getPath(),

                                                                  false, true, Long.MAX_VALUE, path);

                                                          session.getRootNode().addNode(path);

                                                          System.out.println("Created node at " + path );

                                                          session.save();

                                                          System.out.println("Saved new node (thread " + Thread.currentThread().getId() + ") at " + path );

                                                          long numTopLevelNodes = session.getRootNode().getNodes().getSize();

                                                          System.out.println("Number of top-level nodes: " + numTopLevelNodes );

                                                      }

                                                  } catch (LockException e) {

                                                      // There is a lock at the root node, lets try again!

                                                      retry = true;

                                                      System.out.println("Lock retry path: (thread " + Thread.currentThread().getId() + ") " + path);

                                                  }

                                                  finally {

                                                      if (null != lock) {

                                                          System.out.println("Unlock path " + session.getRootNode().getPath());

                                                          session.getWorkspace().getLockManager().unlock(lock.getNode().getPath());

                                                      }

                                                  }

                                              } else {

                                                  lock = session.getWorkspace().getLockManager().getLock(

                                                          session.getRootNode().getPath());

                                                  // Is the right node for waiting created?

                                                  if (lock.getLockOwner().equals(path)) {

                                                      while (!session.getRootNode().hasNode(path)) {

                                                          System.out.println("Find retry path: (thread " + Thread.currentThread().getId() + ") "+ path);

                                                          Thread.sleep(10);

                                                          session.refresh(false);

                                                      }

                                                  } else {

                                                      // No we need a lock!

                                                      retry = true;

                                                  }

                                              }

                                          }

                                      }

                                      return session.getRootNode().getNode(path);

                                  }

                               

                                  @Path("document/{path}")

                                  @PUT

                                  @Consumes("application/json")

                                  @Produces("text/plain")

                                  public String createDocument(@PathParam("path") String path, JSONObject json)

                                      throws Exception

                                  {

                                      Integer incr = call.incrementAndGet();

                                      long start = System.nanoTime();

                                      Session session = _repository.login("main");

                                      try {

                                          Node rootNode = getTopNode(session,path);

                                          String nodeName = incr.toString();

                                          Node numNode = rootNode.addNode(nodeName);

                                          for (int i = 0;i < 10; i++) {

                                              Node node = numNode.addNode(String.valueOf(i));

                                              //session.getRootNode().addNode(path + "/" + incr.toString() + "/" + i);

                                              node.setPrimaryType("nt:unstructured");

                                              writeJsonToNode(json, node);

                                          }

                                          session.save();

                                          //System.out.println("Doing PUT " + incr.toString());

                               

                                      } finally {

                                          session.logout();

                                      }

                               

                                      long end = System.nanoTime();

                                      return String.format("Creating 10 nodes took %dms", (end - start) / 1000000);

                                  }

                              {code}

                               

                              Maybe it became a over complicated solution but I got stuck on "javax.jcr.lock.LockException: The node at '/' is already locked"; in this piece of code:

                               

                              {code:java}

                                                      do {

                                                          Thread.sleep(10);

                                                          session.refresh(false);

                                                      } while(session.getWorkspace().getLockManager().isLocked(

                                                              session.getRootNode().getPath()));

                               

                                                      // Another thread may have created the node while we waited for the lock,

                                                      // so look for the node again

                                                      if (!session.getRootNode().hasNode(path)) {

                                                          lock = session.getWorkspace().getLockManager().lock(

                                                                  session.getRootNode().getPath(),

                                                                  false, true, Long.MAX_VALUE, path);

                                                                  ...

                                                      }

                              {code}

                               

                              So I figured that if the path I want is locked then it will be created. So when I can not lock a path I stop trying to lock it and start to wait for the node to appear. I maybe abuse the API by setting the owner to the path and using that as a marker for if I should wait or try to lock again:

                               

                              {code:java}

                                                  lock = session.getWorkspace().getLockManager().getLock(

                                                          session.getRootNode().getPath());

                                                  // Is the right node for waiting created?

                                                  if (lock.getLockOwner().equals(path)) {

                                                      while (!session.getRootNode().hasNode(path)) {

                                                          System.out.println("Find retry path: (thread " + Thread.currentThread().getId() + ") "+ path);

                                                          Thread.sleep(10);

                                                          session.refresh(false);

                                                      }

                                                  } else {

                                                      // No we need a lock!

                                                      retry = true;

                                                  }

                              {code}

                               

                              However, I should probably use the absolute path instead, and there is the question if my locking strategy is a good one . I had realy nice coding session thoug

                               

                              Note: The locking in Modeshape is fast! Good job!

                               

                              Message was edited by: Oscar Norlander - Updated some code

                              • 12. Re: Modeshape loses nodes during concurrency
                                rhauch

                                Be careful about locking the root node. You really want to create a hierarchy with different areas for different purposes, and one of the best ways to do that is to initialize the repository with that skeleton data structure.

                                 

                                Glad you like ModeShape, and that you're looking at some of the different features. I'll keep working one the concurrency issue, though, so stay tuned.

                                • 13. Re: Modeshape loses nodes during concurrency
                                  codeape

                                  Randall Hauch wrote:

                                   

                                  Be careful about locking the root node. You really want to create a hierarchy with different areas for different purposes, and one of the best ways to do that is to initialize the repository with that skeleton data structure.

                                   

                                  We have a hierarchy in mind but we are testing the extreem cases right now.

                                  • 14. Re: Modeshape loses nodes during concurrency
                                    rhauch

                                    Okay, I've figured out the best way to fix this problem, which is known as "write-skew". I've updated the issue with a lot of detail about the failure mechanism and how/why this solution works. The unit tests that I was able to create earlier duplicated this problem before these changes, but they now pass with these changes.

                                     

                                    The solution does rely upon a particular configuration setting in Infinispan. By default, an Infinispan cache uses optimistic locking, but this (coupled with the bug in ModeShape) results in write skew when there are multiple sessions that are concurrently updating the same node. As shown by many of our users, this is actually a pretty good default for many applications. However, when an application has the potential for concurrently updating the same node, the Infinispan cache should be configured with pessimistic locking. That, coupled with the bug fix, will eliminate the write skew problem.

                                     

                                    You can enable pessimistic locking in your Infinispan configuration with the "lockingMode" attribute on the "transaction" element:

                                     

                                    <?xml version="1.0" encoding="UTF-8"?>
                                    <infinispan ... >
                                    
                                        <namedCache ...>
                                            <transaction lockingMode="PESSIMISTIC" .../>
                                        </namedCache>
                                    </infinispan>
                                    

                                     

                                    Again, you'll need to do this and use the 3.1 codebase (when it is released, or locally build and use the latest pre-3.1 code from our GitHub repository's 'master' branch).

                                     

                                    I've updated your test case (as submitted on the issue, with some local changes) so that the Infinispan configuration uses pessimistic locking, and I was able to successfully get 11000 nodes. So I closed the issue.

                                     

                                    Thanks again for providing such a useful and easy-to-follow test case. It was really instrumental in narrowing down the problem, coming up with narrower unit tests that replicated the same problem, and verifying that the problem has been fixed!

                                    1 2 Previous Next