2 Replies Latest reply on Sep 10, 2018 8:36 AM by Paul Ferraro

    Wildfly 10.1 -> 13 upgrade cluster problem WFLYCTL0013

    Michael Hansen Newbie

      I am upgrading a SSO cluster with 2 servers from  wildfly 10.1 to wildfly 13.0

       

      When starting the first server using wildfly 13.0 I get a WFLYCTL0013 error, and the clustering fails to start. I have added part of the log and configuration files below:

       

      2018-09-10 03:21:41,082 WARN                          [Validator] WELD-001478: Interceptor class com.one.webservice.utils.interceptor.RemoteCallInterceptor is enabled for the application and for the bean archive auth.ear.external.vfs:/content/sharedutils.ear/lib/common-webservice-utils.jar/META-INF/beans.xml. It will only be invoked in the @Priority part of the chain.
      2018-09-10 03:21:42,095 ERROR                         [management-operation] WFLYCTL0013: Operation ("add") failed - address: ([
          ("subsystem" => "undertow"),
          ("server" => "default-server"),
          ("host" => "default-host"),
          ("setting" => "single-sign-on")
      ]) - failure description: {
          "WFLYCTL0080: Failed services" => {"org.wildfly.clustering.infinispan.cache.web.default-host" => "org.infinispan.commons.CacheException: Unable to invoke method public void org.infinispan.persistence.manager.PersistenceManagerImpl.start() on object of type PersistenceManagerImpl
          Caused by: org.infinispan.commons.CacheException: Unable to invoke method public void org.infinispan.persistence.manager.PersistenceManagerImpl.start() on object of type PersistenceManagerImpl
          Caused by: org.infinispan.commons.CacheException: Unable to start cache loaders
          Caused by: org.infinispan.persistence.spi.PersistenceException: java.lang.NullPointerException
          Caused by: java.lang.NullPointerException"},
          "WFLYCTL0288: One or more services were unable to start due to one or more indirect dependencies not being available." => {
              "Services that were unable to start:" => ["jboss.clustering.sso.default-host"],
              "Services that may be the cause:" => [
                  "jboss.clustering.sso.default-host",
                  "jboss.clustering.web.route.default-server",
                  "jboss.deployment.subunit.\"auth.ear\".\"auth-war.war\".deploymentCompleteService",
                  "jboss.deployment.subunit.\"auth.ear\".\"auth-ws.jar\".deploymentCompleteService",
                  "jboss.deployment.subunit.\"mailer.ear\".\"mailer-ejb-impl.jar\".deploymentCompleteService",
                  "jboss.deployment.subunit.\"monitor.ear\".\"monitor-war.war\".deploymentCompleteService",
                  "jboss.deployment.unit.\"auth.ear\".moduleDeploymentRuntimeInformation",
                  "jboss.deployment.unit.\"babel.ear\".PARSE",
                  "jboss.deployment.unit.\"mailer.ear\".moduleDeploymentRuntimeInformation",
                  "jboss.deployment.unit.\"monitor.ear\".deploymentCompleteService",
                  "jboss.deployment.unit.\"monitor.ear\".moduleDeploymentRuntimeInformation",
                  "jboss.deployment.unit.\"sharedi18n.ear\".moduleDeploymentRuntimeInformation",
                  "jboss.deployment.unit.\"sharedjmx.ear\".moduleDeploymentRuntimeInformation",
                  "jboss.deployment.unit.\"sharedutils.ear\".moduleDeploymentRuntimeInformation",
                  "jboss.undertow.deployment.default-server.default-host./b-one-user",
                  "jboss.undertow.deployment.default-server.default-host./b-one-user.UndertowDeploymentInfoService",
                  "jboss.undertow.deployment.default-server.default-host./javadevmonitor",
                  "jboss.undertow.deployment.default-server.default-host./javadevmonitor.UndertowDeploymentInfoService",
                  "jboss.undertow.deployment.default-server.default-host./jmx-console-admin",
                  "jboss.undertow.deployment.default-server.default-host./jmx-console-admin.UndertowDeploymentInfoService",
                  "jboss.undertow.deployment.default-server.default-host./mail",
                  "jboss.undertow.deployment.default-server.default-host./mail.UndertowDeploymentInfoService",
                  "jboss.undertow.deployment.default-server.default-host./principal",
                  "jboss.undertow.deployment.default-server.default-host./principal.UndertowDeploymentInfoService",
                  "jboss.undertow.server.default-server.default-host.single-sign-on.manager",
                  "jboss.undertow.server.default-server.default-host.single-sign-on.manager.listener",
                  "jboss.undertow.server.default-server.default-host.single-sign-on.manager.manager",
                  "jboss.undertow.server.default-server.default-host.single-sign-on.manager.registry",
                  "jboss.xts.handlers",
                  "org.wildfly.clustering.cache.default-service-provider-registry.ejb",
                  "org.wildfly.clustering.cache.default-service-provider-registry.web",
                  "org.wildfly.clustering.cache.group.ejb.dist",
                  "org.wildfly.clustering.cache.group.hibernate.entity",
                  "org.wildfly.clustering.cache.group.hibernate.local-query",
                  "org.wildfly.clustering.cache.group.hibernate.timestamps",
                  "org.wildfly.clustering.cache.group.server.client-mappings",
                  "org.wildfly.clustering.cache.group.server.default",
                  "org.wildfly.clustering.cache.group.web.client-mappings",
                  "org.wildfly.clustering.cache.group.web.default-server",
                  "org.wildfly.clustering.cache.group.web.repl",
                  "org.wildfly.clustering.cache.registry.ejb.dist",
                  "org.wildfly.clustering.cache.registry.server.default",
                  "org.wildfly.clustering.cache.registry.web.repl",
                  "org.wildfly.clustering.cache.registry-entry.ejb.dist",
                  "org.wildfly.clustering.cache.registry-entry.hibernate.entity",
                  "org.wildfly.clustering.cache.registry-entry.hibernate.local-query",
                  "org.wildfly.clustering.cache.registry-entry.hibernate.timestamps",
                  "org.wildfly.clustering.cache.registry-entry.server.client-mappings",
                  "org.wildfly.clustering.cache.registry-entry.server.default",
                  "org.wildfly.clustering.cache.registry-entry.web.client-mappings",
                  "org.wildfly.clustering.cache.registry-entry.web.default-server",
                  "org.wildfly.clustering.cache.registry-entry.web.repl",
                  "org.wildfly.clustering.cache.registry-factory.ejb.dist",
                  "org.wildfly.clustering.cache.registry-factory.hibernate.entity",
                  "org.wildfly.clustering.cache.registry-factory.hibernate.local-query",
                  "org.wildfly.clustering.cache.registry-factory.hibernate.timestamps",
                  "org.wildfly.clustering.cache.registry-factory.server.client-mappings",
                  "org.wildfly.clustering.cache.registry-factory.server.default",
                  "org.wildfly.clustering.cache.registry-factory.web.client-mappings",
                  "org.wildfly.clustering.cache.registry-factory.web.default-server",
                  "org.wildfly.clustering.cache.registry-factory.web.repl",
                  "org.wildfly.clustering.cache.service-provider-registry.ejb.client-mappings",
                  "org.wildfly.clustering.cache.service-provider-registry.ejb.dist",
                  "org.wildfly.clustering.cache.service-provider-registry.hibernate.entity",
                  "org.wildfly.clustering.cache.service-provider-registry.hibernate.local-query",
                  "org.wildfly.clustering.cache.service-provider-registry.hibernate.timestamps",
                  "org.wildfly.clustering.cache.service-provider-registry.server.client-mappings",
                  "org.wildfly.clustering.cache.service-provider-registry.server.default",
                  "org.wildfly.clustering.cache.service-provider-registry.web.client-mappings",
                  "org.wildfly.clustering.cache.service-provider-registry.web.default-server",
                  "org.wildfly.clustering.cache.service-provider-registry.web.repl",
                  "org.wildfly.clustering.command-dispatcher-factory.ejb",
                  "org.wildfly.clustering.command-dispatcher-factory.hibernate",
                  "org.wildfly.clustering.command-dispatcher-factory.local",
                  "org.wildfly.clustering.command-dispatcher-factory.server",
                  "org.wildfly.clustering.command-dispatcher-factory.web",
                  "org.wildfly.clustering.group.hibernate",
                  "org.wildfly.clustering.group.hq-cluster",
                  "org.wildfly.clustering.group.local",
                  "org.wildfly.clustering.group.server",
                  "org.wildfly.clustering.group.web",
                  "org.wildfly.clustering.infinispan.cache.ejb.dist",
                  "org.wildfly.clustering.infinispan.cache.hibernate.entity",
                  "org.wildfly.clustering.infinispan.cache.hibernate.local-query",
                  "org.wildfly.clustering.infinispan.cache.hibernate.timestamps",
                  "org.wildfly.clustering.infinispan.cache.server.client-mappings",
                  "org.wildfly.clustering.infinispan.cache.server.default",
                  "org.wildfly.clustering.infinispan.cache.store.hibernate.timestamps",
                  "org.wildfly.clustering.infinispan.cache.store.server.default",
                  "org.wildfly.clustering.infinispan.cache.web.client-mappings",
                  "org.wildfly.clustering.infinispan.cache.web.default-server",
                  "org.wildfly.clustering.infinispan.cache.web.repl",
                  "org.wildfly.clustering.infinispan.cache-configuration.hibernate.timestamps",
                  "org.wildfly.clustering.infinispan.cache-configuration.hibernate.timestamps.backup",
                  "org.wildfly.clustering.infinispan.cache-configuration.hibernate.timestamps.expiration",
                  "org.wildfly.clustering.infinispan.cache-configuration.hibernate.timestamps.locking",
                  "org.wildfly.clustering.infinispan.cache-configuration.hibernate.timestamps.memory",
                  "org.wildfly.clustering.infinispan.cache-configuration.hibernate.timestamps.partition-handling",
                  "org.wildfly.clustering.infinispan.cache-configuration.hibernate.timestamps.state-transfer",
                  "org.wildfly.clustering.infinispan.cache-configuration.hibernate.timestamps.transaction",
                  "org.wildfly.clustering.infinispan.cache-configuration.server.client-mappings",
                  "org.wildfly.clustering.infinispan.cache-configuration.server.default",
                  "org.wildfly.clustering.infinispan.cache-configuration.server.default.backup",
                  "org.wildfly.clustering.infinispan.cache-configuration.server.default.expiration",
                  "org.wildfly.clustering.infinispan.cache-configuration.server.default.locking",
                  "org.wildfly.clustering.infinispan.cache-configuration.server.default.memory",
                  "org.wildfly.clustering.infinispan.cache-configuration.server.default.partition-handling",
                  "org.wildfly.clustering.infinispan.cache-configuration.server.default.state-transfer",
                  "org.wildfly.clustering.infinispan.cache-configuration.server.default.transaction",
                  "org.wildfly.clustering.infinispan.cache-configuration.web.client-mappings",
                  "org.wildfly.clustering.infinispan.cache-configuration.web.default-server",
                  "org.wildfly.clustering.infinispan.default-cache.ejb",
                  "org.wildfly.clustering.infinispan.default-cache.web",
                  "org.wildfly.clustering.infinispan.default-cache-configuration.server",
                  "org.wildfly.undertow.host.sso.default-server.default-host"
              ]
          }
      }

      The infinispan section of standalone-full.xml on my server looks like:

       

              <subsystem xmlns="urn:jboss:domain:infinispan:6.0">

                  <cache-container name="server" default-cache="default" module="org.wildfly.clustering.server" aliases="singleton cluster">

                      <transport lock-timeout="60000"/>

                      <replicated-cache name="default" >

                          <transaction mode="BATCH"/>

                      </replicated-cache>

                  </cache-container>

                  <cache-container name="web" default-cache="repl" module="org.wildfly.clustering.web.infinispan" statistics-enabled="true">

                      <transport lock-timeout="60000"/>

                      <replicated-cache name="repl" statistics-enabled="true">

                          <transaction mode="BATCH"/>

                          <transaction locking="OPTIMISTIC"/>

                          <locking isolation="READ_COMMITTED" acquire-timeout="20000" concurrency-level="1500"/>

                          <file-store/>

                      </replicated-cache>

                  </cache-container>

                  <cache-container name="ejb" default-cache="dist" module="org.wildfly.clustering.ejb.infinispan" aliases="sfsb">

                      <transport lock-timeout="60000"/>

                      <distributed-cache name="dist" owners="2" l1-lifespan="0">

                          <locking isolation="REPEATABLE_READ"/>

                          <transaction mode="BATCH"/>

                          <file-store/>

                      </distributed-cache>

                  </cache-container>

                  <cache-container name="hibernate" module="org.infinispan.hibernate-cache">

                      <transport lock-timeout="60000"/>

                      <local-cache name="local-query">

                          <object-memory size="10000"/>

                          <expiration max-idle="100000"/>

                      </local-cache>

                      <invalidation-cache name="entity">

                          <transaction mode="NON_XA"/>

                          <object-memory size="10000"/>

                          <expiration max-idle="100000"/>

                      </invalidation-cache>

                      <replicated-cache name="timestamps" />

                  </cache-container>

              </subsystem>