Wildfly 10.1 -> 13 upgrade cluster problem WFLYCTL0013
I am upgrading a SSO cluster with 2 servers from wildfly 10.1 to wildfly 13.0
When starting the first server using wildfly 13.0 I get a WFLYCTL0013 error, and the clustering fails to start. I have added part of the log and configuration files below:
2018-09-10 03:21:41,082 WARN [Validator] WELD-001478: Interceptor class com.one.webservice.utils.interceptor.RemoteCallInterceptor is enabled for the application and for the bean archive auth.ear.external.vfs:/content/sharedutils.ear/lib/common-webservice-utils.jar/META-INF/beans.xml. It will only be invoked in the @Priority part of the chain.
2018-09-10 03:21:42,095 ERROR [management-operation] WFLYCTL0013: Operation ("add") failed - address: ([
("subsystem" => "undertow"),
("server" => "default-server"),
("host" => "default-host"),
("setting" => "single-sign-on")
]) - failure description: {
"WFLYCTL0080: Failed services" => {"org.wildfly.clustering.infinispan.cache.web.default-host" => "org.infinispan.commons.CacheException: Unable to invoke method public void org.infinispan.persistence.manager.PersistenceManagerImpl.start() on object of type PersistenceManagerImpl
Caused by: org.infinispan.commons.CacheException: Unable to invoke method public void org.infinispan.persistence.manager.PersistenceManagerImpl.start() on object of type PersistenceManagerImpl
Caused by: org.infinispan.commons.CacheException: Unable to start cache loaders
Caused by: org.infinispan.persistence.spi.PersistenceException: java.lang.NullPointerException
Caused by: java.lang.NullPointerException"},
"WFLYCTL0288: One or more services were unable to start due to one or more indirect dependencies not being available." => {
"Services that were unable to start:" => ["jboss.clustering.sso.default-host"],
"Services that may be the cause:" => [
"jboss.clustering.sso.default-host",
"jboss.clustering.web.route.default-server",
"jboss.deployment.subunit.\"auth.ear\".\"auth-war.war\".deploymentCompleteService",
"jboss.deployment.subunit.\"auth.ear\".\"auth-ws.jar\".deploymentCompleteService",
"jboss.deployment.subunit.\"mailer.ear\".\"mailer-ejb-impl.jar\".deploymentCompleteService",
"jboss.deployment.subunit.\"monitor.ear\".\"monitor-war.war\".deploymentCompleteService",
"jboss.deployment.unit.\"auth.ear\".moduleDeploymentRuntimeInformation",
"jboss.deployment.unit.\"babel.ear\".PARSE",
"jboss.deployment.unit.\"mailer.ear\".moduleDeploymentRuntimeInformation",
"jboss.deployment.unit.\"monitor.ear\".deploymentCompleteService",
"jboss.deployment.unit.\"monitor.ear\".moduleDeploymentRuntimeInformation",
"jboss.deployment.unit.\"sharedi18n.ear\".moduleDeploymentRuntimeInformation",
"jboss.deployment.unit.\"sharedjmx.ear\".moduleDeploymentRuntimeInformation",
"jboss.deployment.unit.\"sharedutils.ear\".moduleDeploymentRuntimeInformation",
"jboss.undertow.deployment.default-server.default-host./b-one-user",
"jboss.undertow.deployment.default-server.default-host./b-one-user.UndertowDeploymentInfoService",
"jboss.undertow.deployment.default-server.default-host./javadevmonitor",
"jboss.undertow.deployment.default-server.default-host./javadevmonitor.UndertowDeploymentInfoService",
"jboss.undertow.deployment.default-server.default-host./jmx-console-admin",
"jboss.undertow.deployment.default-server.default-host./jmx-console-admin.UndertowDeploymentInfoService",
"jboss.undertow.deployment.default-server.default-host./mail",
"jboss.undertow.deployment.default-server.default-host./mail.UndertowDeploymentInfoService",
"jboss.undertow.deployment.default-server.default-host./principal",
"jboss.undertow.deployment.default-server.default-host./principal.UndertowDeploymentInfoService",
"jboss.undertow.server.default-server.default-host.single-sign-on.manager",
"jboss.undertow.server.default-server.default-host.single-sign-on.manager.listener",
"jboss.undertow.server.default-server.default-host.single-sign-on.manager.manager",
"jboss.undertow.server.default-server.default-host.single-sign-on.manager.registry",
"jboss.xts.handlers",
"org.wildfly.clustering.cache.default-service-provider-registry.ejb",
"org.wildfly.clustering.cache.default-service-provider-registry.web",
"org.wildfly.clustering.cache.group.ejb.dist",
"org.wildfly.clustering.cache.group.hibernate.entity",
"org.wildfly.clustering.cache.group.hibernate.local-query",
"org.wildfly.clustering.cache.group.hibernate.timestamps",
"org.wildfly.clustering.cache.group.server.client-mappings",
"org.wildfly.clustering.cache.group.server.default",
"org.wildfly.clustering.cache.group.web.client-mappings",
"org.wildfly.clustering.cache.group.web.default-server",
"org.wildfly.clustering.cache.group.web.repl",
"org.wildfly.clustering.cache.registry.ejb.dist",
"org.wildfly.clustering.cache.registry.server.default",
"org.wildfly.clustering.cache.registry.web.repl",
"org.wildfly.clustering.cache.registry-entry.ejb.dist",
"org.wildfly.clustering.cache.registry-entry.hibernate.entity",
"org.wildfly.clustering.cache.registry-entry.hibernate.local-query",
"org.wildfly.clustering.cache.registry-entry.hibernate.timestamps",
"org.wildfly.clustering.cache.registry-entry.server.client-mappings",
"org.wildfly.clustering.cache.registry-entry.server.default",
"org.wildfly.clustering.cache.registry-entry.web.client-mappings",
"org.wildfly.clustering.cache.registry-entry.web.default-server",
"org.wildfly.clustering.cache.registry-entry.web.repl",
"org.wildfly.clustering.cache.registry-factory.ejb.dist",
"org.wildfly.clustering.cache.registry-factory.hibernate.entity",
"org.wildfly.clustering.cache.registry-factory.hibernate.local-query",
"org.wildfly.clustering.cache.registry-factory.hibernate.timestamps",
"org.wildfly.clustering.cache.registry-factory.server.client-mappings",
"org.wildfly.clustering.cache.registry-factory.server.default",
"org.wildfly.clustering.cache.registry-factory.web.client-mappings",
"org.wildfly.clustering.cache.registry-factory.web.default-server",
"org.wildfly.clustering.cache.registry-factory.web.repl",
"org.wildfly.clustering.cache.service-provider-registry.ejb.client-mappings",
"org.wildfly.clustering.cache.service-provider-registry.ejb.dist",
"org.wildfly.clustering.cache.service-provider-registry.hibernate.entity",
"org.wildfly.clustering.cache.service-provider-registry.hibernate.local-query",
"org.wildfly.clustering.cache.service-provider-registry.hibernate.timestamps",
"org.wildfly.clustering.cache.service-provider-registry.server.client-mappings",
"org.wildfly.clustering.cache.service-provider-registry.server.default",
"org.wildfly.clustering.cache.service-provider-registry.web.client-mappings",
"org.wildfly.clustering.cache.service-provider-registry.web.default-server",
"org.wildfly.clustering.cache.service-provider-registry.web.repl",
"org.wildfly.clustering.command-dispatcher-factory.ejb",
"org.wildfly.clustering.command-dispatcher-factory.hibernate",
"org.wildfly.clustering.command-dispatcher-factory.local",
"org.wildfly.clustering.command-dispatcher-factory.server",
"org.wildfly.clustering.command-dispatcher-factory.web",
"org.wildfly.clustering.group.hibernate",
"org.wildfly.clustering.group.hq-cluster",
"org.wildfly.clustering.group.local",
"org.wildfly.clustering.group.server",
"org.wildfly.clustering.group.web",
"org.wildfly.clustering.infinispan.cache.ejb.dist",
"org.wildfly.clustering.infinispan.cache.hibernate.entity",
"org.wildfly.clustering.infinispan.cache.hibernate.local-query",
"org.wildfly.clustering.infinispan.cache.hibernate.timestamps",
"org.wildfly.clustering.infinispan.cache.server.client-mappings",
"org.wildfly.clustering.infinispan.cache.server.default",
"org.wildfly.clustering.infinispan.cache.store.hibernate.timestamps",
"org.wildfly.clustering.infinispan.cache.store.server.default",
"org.wildfly.clustering.infinispan.cache.web.client-mappings",
"org.wildfly.clustering.infinispan.cache.web.default-server",
"org.wildfly.clustering.infinispan.cache.web.repl",
"org.wildfly.clustering.infinispan.cache-configuration.hibernate.timestamps",
"org.wildfly.clustering.infinispan.cache-configuration.hibernate.timestamps.backup",
"org.wildfly.clustering.infinispan.cache-configuration.hibernate.timestamps.expiration",
"org.wildfly.clustering.infinispan.cache-configuration.hibernate.timestamps.locking",
"org.wildfly.clustering.infinispan.cache-configuration.hibernate.timestamps.memory",
"org.wildfly.clustering.infinispan.cache-configuration.hibernate.timestamps.partition-handling",
"org.wildfly.clustering.infinispan.cache-configuration.hibernate.timestamps.state-transfer",
"org.wildfly.clustering.infinispan.cache-configuration.hibernate.timestamps.transaction",
"org.wildfly.clustering.infinispan.cache-configuration.server.client-mappings",
"org.wildfly.clustering.infinispan.cache-configuration.server.default",
"org.wildfly.clustering.infinispan.cache-configuration.server.default.backup",
"org.wildfly.clustering.infinispan.cache-configuration.server.default.expiration",
"org.wildfly.clustering.infinispan.cache-configuration.server.default.locking",
"org.wildfly.clustering.infinispan.cache-configuration.server.default.memory",
"org.wildfly.clustering.infinispan.cache-configuration.server.default.partition-handling",
"org.wildfly.clustering.infinispan.cache-configuration.server.default.state-transfer",
"org.wildfly.clustering.infinispan.cache-configuration.server.default.transaction",
"org.wildfly.clustering.infinispan.cache-configuration.web.client-mappings",
"org.wildfly.clustering.infinispan.cache-configuration.web.default-server",
"org.wildfly.clustering.infinispan.default-cache.ejb",
"org.wildfly.clustering.infinispan.default-cache.web",
"org.wildfly.clustering.infinispan.default-cache-configuration.server",
"org.wildfly.undertow.host.sso.default-server.default-host"
]
}
}
The infinispan section of standalone-full.xml on my server looks like:
<subsystem xmlns="urn:jboss:domain:infinispan:6.0">
<cache-container name="server" default-cache="default" module="org.wildfly.clustering.server" aliases="singleton cluster">
<transport lock-timeout="60000"/>
<replicated-cache name="default" >
<transaction mode="BATCH"/>
</replicated-cache>
</cache-container>
<cache-container name="web" default-cache="repl" module="org.wildfly.clustering.web.infinispan" statistics-enabled="true">
<transport lock-timeout="60000"/>
<replicated-cache name="repl" statistics-enabled="true">
<transaction mode="BATCH"/>
<transaction locking="OPTIMISTIC"/>
<locking isolation="READ_COMMITTED" acquire-timeout="20000" concurrency-level="1500"/>
<file-store/>
</replicated-cache>
</cache-container>
<cache-container name="ejb" default-cache="dist" module="org.wildfly.clustering.ejb.infinispan" aliases="sfsb">
<transport lock-timeout="60000"/>
<distributed-cache name="dist" owners="2" l1-lifespan="0">
<locking isolation="REPEATABLE_READ"/>
<transaction mode="BATCH"/>
<file-store/>
</distributed-cache>
</cache-container>
<cache-container name="hibernate" module="org.infinispan.hibernate-cache">
<transport lock-timeout="60000"/>
<local-cache name="local-query">
<object-memory size="10000"/>
<expiration max-idle="100000"/>
</local-cache>
<invalidation-cache name="entity">
<transaction mode="NON_XA"/>
<object-memory size="10000"/>
<expiration max-idle="100000"/>
</invalidation-cache>
<replicated-cache name="timestamps" />
</cache-container>
</subsystem>
