0 Replies Latest reply on Aug 20, 2016 2:28 AM by mohilkhare

    Wildfly10 Cluster: Singleton Issues

    mohilkhare

      We upgraded our cluster of 3 nodes and 6 nodes from wildfly9 to wildfly10.0.0.final and it seems singleton is not behaving properly and is in fact worst than wildfly9.  I am pretty sure we are missing something in our config or code and this can't be this bad with wldfly10.0.0.final.

       

      FYI: In a cluster of 6 nodes, we have 3 nodes in one data center and 3 in other. We have max latency of 10ms across data centers. Our OS kernel's keep alive timer is 10 minutes. We use wildfly mainly for session replication and singleton service. We are not using hibernate and ejb as such.

       

      When we upgraded nodes from wildfly 9 to wildfly 10, we are witnessing that singleton dying more often now. Even simple tests like -- taking down node running singleton should cause singleton service to switch from old node to new node, is NOT working.

       

      This is our wildfly config which we took from standalone-ha.xml:

       

      <?xml version='1.0' encoding='UTF-8'?>

      <!-- TEMPLATE_VERSION:10.0.0.5  -->

       

       

      <server xmlns="urn:jboss:domain:4.0">

       

       

          <extensions>

              <extension module="org.jboss.as.clustering.infinispan"/>

              <extension module="org.jboss.as.clustering.jgroups"/>

              <extension module="org.jboss.as.connector"/>

              <extension module="org.jboss.as.deployment-scanner"/>

              <extension module="org.jboss.as.ee"/>

              <extension module="org.jboss.as.ejb3"/>

              <extension module="org.jboss.as.jaxrs"/>

              <extension module="org.jboss.as.jdr"/>

              <extension module="org.jboss.as.jmx"/>

              <extension module="org.jboss.as.jpa"/>

              <extension module="org.jboss.as.jsf"/>

              <extension module="org.jboss.as.logging"/>

              <extension module="org.jboss.as.mail"/>

              <extension module="org.jboss.as.naming"/>

              <extension module="org.jboss.as.pojo"/>

              <extension module="org.jboss.as.remoting"/>

              <extension module="org.jboss.as.sar"/>

              <extension module="org.jboss.as.security"/>

              <extension module="org.jboss.as.transactions"/>

              <extension module="org.jboss.as.webservices"/>

              <extension module="org.jboss.as.weld"/>

              <extension module="org.wildfly.extension.batch.jberet"/>

              <extension module="org.wildfly.extension.bean-validation"/>

              <extension module="org.wildfly.extension.clustering.singleton"/>

              <extension module="org.wildfly.extension.io"/>

              <extension module="org.wildfly.extension.request-controller"/>

              <extension module="org.wildfly.extension.security.manager"/>

              <extension module="org.wildfly.extension.undertow"/>

          </extensions>

       

       

          <paths>

              <path name="viptela.deploy.dir" path="/opt/web-app"/>

              <path name="viptela.log.dir" path="/var/log/nms"/>

          </paths>

       

       

          <management>

              <security-realms>

                  <security-realm name="SSLRealm">

                      <server-identities>

                          <ssl>

                              <keystore path="etc/server.keystore" relative-to="viptela.deploy.dir" keystore-password="vipvmanage" alias="server" key-password="vipvmanage"/>

                          </ssl>

                      </server-identities>

                  </security-realm>

                  <security-realm name="ManagementRealm">

                      <authentication>

                          <local default-user="$local" skip-group-loading="true"/>

                          <properties path="mgmt-users.properties" relative-to="jboss.server.config.dir"/>

                      </authentication>

                      <authorization map-groups-to-roles="false">

                          <properties path="mgmt-groups.properties" relative-to="jboss.server.config.dir"/>

                      </authorization>

                  </security-realm>

                  <security-realm name="ApplicationRealm">

                      <authentication>

                          <local default-user="$local" allowed-users="*" skip-group-loading="true"/>

                          <properties path="application-users.properties" relative-to="jboss.server.config.dir"/>

                      </authentication>

                      <authorization>

                          <properties path="application-roles.properties" relative-to="jboss.server.config.dir"/>

                      </authorization>

                  </security-realm>

              </security-realms>

              <audit-log>

                  <formatters>

                      <json-formatter name="json-formatter"/>

                  </formatters>

                  <handlers>

                      <file-handler name="file" formatter="json-formatter" path="audit-log.log" relative-to="jboss.server.data.dir"/>

                  </handlers>

                  <logger log-boot="true" log-read-only="false" enabled="false">

                      <handlers>

                          <handler name="file"/>

                      </handlers>

                  </logger>

              </audit-log>

              <management-interfaces>

                  <http-interface security-realm="ManagementRealm" http-upgrade-enabled="true">

                      <socket-binding http="management-http"/>

                  </http-interface>

              </management-interfaces>

              <access-control provider="simple">

                  <role-mapping>

                      <role name="SuperUser">

                          <include>

                              <user name="$local"/>

                          </include>

                      </role>

                  </role-mapping>

              </access-control>

          </management>

       

       

          <profile>

              <subsystem xmlns="urn:jboss:domain:logging:3.0">

                <!--

                  <console-handler name="CONSOLE">

                      <level name="INFO"/>

                      <formatter>

                          <named-formatter name="COLOR-PATTERN"/>

                      </formatter>

                  </console-handler>

                -->

                  <size-rotating-file-handler name="FILE" autoflush="true">

                      <level name="INFO"/>

                      <formatter>

                          <pattern-formatter pattern="%d{dd-MMM-yyyy HH:mm:ss,SSS} %-5p [%c{1}] (%t) %s%E%n"/>

                      </formatter>

                      <file relative-to="viptela.log.dir" path="vmanage-server.log"/>

                      <rotate-size value="10000k"/>

                      <max-backup-index value="5"/>

                      <append value="true"/>

                  </size-rotating-file-handler>

                  <size-rotating-file-handler name="CONFIGLOG" autoflush="true">

                      <level name="INFO"/>

                      <formatter>

                          <pattern-formatter pattern="%d{dd-MMM-yyyy HH:mm:ss,SSS} %-5p [%c{1}] (%t) %s%E%n"/>

                      </formatter>

                      <file relative-to="viptela.log.dir" path="vmanage-server-device-config.log"/>

                      <rotate-size value="10000k"/>

                      <max-backup-index value="5"/>

                      <append value="true"/>

                  </size-rotating-file-handler>

                  <logger category="com.arjuna">

                      <level name="WARN"/>

                  </logger>

                  <logger category="org.apache.tomcat.util.modeler">

                      <level name="WARN"/>

                  </logger>

                  <logger category="org.jboss.as.config">

                      <level name="DEBUG"/>

                  </logger>

                  <logger category="sun.rmi">

                      <level name="WARN"/>

                  </logger>

                  <logger category="jacorb">

                      <level name="WARN"/>

                  </logger>

                  <logger category="jacorb.config">

                      <level name="ERROR"/>

                  </logger>

                  <logger category="org.jboss.security">

                      <level name="INFO"/>

                  </logger>

                  <logger category="org.wildfly.clustering.ejb.infinispan">

                      <level name="ERROR"/>

                  </logger>

                  <logger category="org.wildfly.clustering.web.infinispan">

                      <level name="ERROR"/>

                  </logger>

                  <logger category="com.tinkerpop.blueprints.impls.orient">

                      <level name="ERROR"/>

                  </logger>

                  <logger category="com.orientechnologies.orient.client.remote.OStorageRemote">

                      <level name="ERROR"/>

                  </logger>

                 <logger category="org.elasticsearch.plugins">

                      <level name="WARN"/>

                  </logger>

                  <logger category="kafka.consumer">

                      <level name="ERROR"/>

                  </logger>

                  <logger category="kafka.producer">

                      <level name="ERROR"/>

                  </logger>

                  <logger category="kafka.utils">

                      <level name="ERROR"/>

                  </logger>

                  <logger category="kafka.client">

                      <level name="ERROR"/>

                  </logger>

                  <logger category="org.apache.zookeeper">

                      <level name="ERROR"/>

                  </logger>

                  <logger category="org.I0Itec.zkclient">

                      <level name="ERROR"/>

                  </logger>

                  <logger category="com.tailf.maapi">

                      <level name="WARN"/>

                      <handlers>

                          <handler name="CONFIGLOG"/>

                      </handlers>

                  </logger>

                  <logger category="com.viptela.vmanage.server.ncs">

                      <level name="INFO"/>

                      <handlers>

                          <handler name="CONFIGLOG"/>

                      </handlers>

                  </logger>

                  <root-logger>

                      <level name="INFO"/>

                      <handlers>

                        <!--

                          <handler name="CONSOLE"/>

                        -->

                          <handler name="FILE"/>

                      </handlers>

                  </root-logger>

                  <formatter name="PATTERN">

                      <pattern-formatter pattern="%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p [%c] (%t) %s%e%n"/>

                  </formatter>

                  <formatter name="COLOR-PATTERN">

                      <pattern-formatter pattern="%K{level}%d{HH:mm:ss,SSS} %-5p [%c] (%t) %s%e%n"/>

                  </formatter>

              </subsystem>

              <subsystem xmlns="urn:jboss:domain:batch-jberet:1.0">

                  <default-job-repository name="in-memory"/>

                  <default-thread-pool name="batch"/>

                  <job-repository name="in-memory">

                      <in-memory/>

                  </job-repository>

                  <thread-pool name="batch">

                      <max-threads count="10"/>

                      <keepalive-time time="30" unit="seconds"/>

                  </thread-pool>

              </subsystem>

              <subsystem xmlns="urn:jboss:domain:bean-validation:1.0"/>

              <subsystem xmlns="urn:jboss:domain:datasources:4.0">

                  <datasources>

                      <datasource jndi-name="java:jboss/datasources/ExampleDS" pool-name="ExampleDS" enabled="true" use-java-context="true">

                          <connection-url>jdbc:h2:mem:test;DB_CLOSE_DELAY=-1;DB_CLOSE_ON_EXIT=FALSE</connection-url>

                          <driver>h2</driver>

                          <security>

                              <user-name>sa</user-name>

                              <password>sa</password>

                          </security>

                      </datasource>

                      <drivers>

                          <driver name="h2" module="com.h2database.h2">

                              <xa-datasource-class>org.h2.jdbcx.JdbcDataSource</xa-datasource-class>

                          </driver>

                      </drivers>

                  </datasources>

              </subsystem>

              <subsystem xmlns="urn:jboss:domain:deployment-scanner:2.0">

                  <deployment-scanner path="deployments" relative-to="jboss.server.base.dir" scan-interval="5000" runtime-failure-causes-rollback="${jboss.deployment.scanner.rollback.on.failure:false}"/>

                  <deployment-scanner name="viptela" path="webapps" relative-to="viptela.deploy.dir" scan-interval="0"/>

              </subsystem>

              <subsystem xmlns="urn:jboss:domain:ee:4.0">

                  <spec-descriptor-property-replacement>false</spec-descriptor-property-replacement>

                  <concurrent>

                      <context-services>

                          <context-service name="default" jndi-name="java:jboss/ee/concurrency/context/default" use-transaction-setup-provider="true"/>

                      </context-services>

                      <managed-thread-factories>

                          <managed-thread-factory name="default" jndi-name="java:jboss/ee/concurrency/factory/default" context-service="default"/>

                      </managed-thread-factories>

                      <managed-executor-services>

                          <managed-executor-service name="default" jndi-name="java:jboss/ee/concurrency/executor/default" context-service="default" hung-task-threshold="60000" core-threads="5" max-threads="25" keepalive-time="5000"/>

                      </managed-executor-services>

                      <managed-scheduled-executor-services>

                          <managed-scheduled-executor-service name="default" jndi-name="java:jboss/ee/concurrency/scheduler/default" context-service="default" hung-task-threshold="60000" core-threads="2" keepalive-time="3000"/>

                      </managed-scheduled-executor-services>

                  </concurrent>

                  <default-bindings context-service="java:jboss/ee/concurrency/context/default" datasource="java:jboss/datasources/ExampleDS" managed-executor-service="java:jboss/ee/concurrency/executor/default" managed-scheduled-executor-service="java:jboss/ee/concurrency/scheduler/default" managed-thread-factory="java:jboss/ee/concurrency/factory/default"/>

              </subsystem>

              <subsystem xmlns="urn:jboss:domain:ejb3:4.0">

                  <session-bean>

                      <stateless>

                          <bean-instance-pool-ref pool-name="slsb-strict-max-pool"/>

                      </stateless>

                      <stateful default-access-timeout="5000" cache-ref="distributable" passivation-disabled-cache-ref="simple"/>

                      <singleton default-access-timeout="5000"/>

                  </session-bean>

                  <pools>

                      <bean-instance-pools>

                          <!-- Automatically configure pools. Alternatively, max-pool-size can be set to a specific value -->

                          <strict-max-pool name="slsb-strict-max-pool" derive-size="from-worker-pools" instance-acquisition-timeout="5" instance-acquisition-timeout-unit="MINUTES"/>

                          <strict-max-pool name="mdb-strict-max-pool" derive-size="from-cpu-count" instance-acquisition-timeout="5" instance-acquisition-timeout-unit="MINUTES"/>

                      </bean-instance-pools>

                  </pools>

                  <caches>

                      <cache name="simple"/>

                      <cache name="distributable" passivation-store-ref="infinispan" aliases="passivating clustered"/>

                  </caches>

                  <passivation-stores>

                      <passivation-store name="infinispan" cache-container="ejb" max-size="10000"/>

                  </passivation-stores>

                  <async thread-pool-name="default"/>

                  <timer-service thread-pool-name="default" default-data-store="default-file-store">

                      <data-stores>

                          <file-data-store name="default-file-store" path="timer-service-data" relative-to="jboss.server.data.dir"/>

                      </data-stores>

                  </timer-service>

                  <remote connector-ref="http-remoting-connector" thread-pool-name="default"/>

                  <thread-pools>

                      <thread-pool name="default">

                          <max-threads count="10"/>

                          <keepalive-time time="100" unit="milliseconds"/>

                      </thread-pool>

                  </thread-pools>

                  <default-security-domain value="other"/>

                  <default-missing-method-permissions-deny-access value="true"/>

                  <log-system-exceptions value="true"/>

              </subsystem>

              <subsystem xmlns="urn:jboss:domain:io:1.1">

                  <worker name="default"/>

                  <buffer-pool name="default"/>

              </subsystem>

              <subsystem xmlns="urn:jboss:domain:infinispan:4.0">

                  <cache-container aliases="singleton cluster" default-cache="default" module="org.wildfly.clustering.server" name="server">

                      <transport lock-timeout="120000"/>

                      <replicated-cache mode="ASYNC" name="default">

                          <state-transfer timeout="300000"/>

                          <transaction locking="OPTIMISTIC" mode="BATCH"/>

                      </replicated-cache>

                  </cache-container>

                  <cache-container default-cache="session" module="org.wildfly.clustering.web.infinispan" name="web">

                      <transport lock-timeout="120000"/>

                      <replicated-cache mode="ASYNC" name="session">

                          <state-transfer timeout="300000"/>

                          <locking isolation="READ_COMMITTED" acquire-timeout="60000"/>

                          <transaction locking="PESSIMISTIC" mode="BATCH"/>

                      </replicated-cache>

                  </cache-container>

                  <cache-container name="ejb" default-cache="dist" module="org.wildfly.clustering.ejb.infinispan" aliases="sfsb">

                      <transport lock-timeout="60000"/>

                      <distributed-cache name="dist" mode="ASYNC" owners="2" l1-lifespan="0">

                          <locking isolation="REPEATABLE_READ"/>

                          <transaction mode="BATCH"/>

                          <file-store/>

                      </distributed-cache>

                  </cache-container>

                  <cache-container name="hibernate" default-cache="local-query" module="org.hibernate.infinispan">

                      <transport lock-timeout="60000"/>

                      <local-cache name="local-query">

                          <eviction strategy="LRU" max-entries="10000"/>

                          <expiration max-idle="100000"/>

                      </local-cache>

                      <invalidation-cache name="entity" mode="SYNC">

                          <transaction mode="NON_XA"/>

                          <eviction strategy="LRU" max-entries="10000"/>

                          <expiration max-idle="100000"/>

                      </invalidation-cache>

                      <replicated-cache name="timestamps" mode="ASYNC"/>

                  </cache-container>

              </subsystem>

              <subsystem xmlns="urn:jboss:domain:jaxrs:1.0"/>

              <subsystem xmlns="urn:jboss:domain:jca:4.0">

                  <archive-validation enabled="true" fail-on-error="true" fail-on-warn="false"/>

                  <bean-validation enabled="true"/>

                  <default-workmanager>

                      <short-running-threads>

                          <core-threads count="50"/>

                          <queue-length count="50"/>

                          <max-threads count="50"/>

                          <keepalive-time time="10" unit="seconds"/>

                      </short-running-threads>

                      <long-running-threads>

                          <core-threads count="50"/>

                          <queue-length count="50"/>

                          <max-threads count="50"/>

                          <keepalive-time time="10" unit="seconds"/>

                      </long-running-threads>

                  </default-workmanager>

                  <cached-connection-manager/>

              </subsystem>

              <subsystem xmlns="urn:jboss:domain:jdr:1.0"/>

              <subsystem xmlns="urn:jboss:domain:jgroups:4.0">

                  <channels default="ee">

                      <channel name="ee"/>

                  </channels>

                  <stacks default="tcp">

                      <stack name="tcp">

                          <transport type="TCP" socket-binding="jgroups-tcp"/>

                          <protocol type="TCPPING">

                              <property name="initial_hosts">

                              </property>

                              <property name="port_range">

                                  0

                              </property>

                          </protocol>

                          <protocol type="MERGE3"/>

                          <protocol type="FD_SOCK" socket-binding="jgroups-tcp-fd"/>

                          <protocol type="FD">

                               <property name="timeout">

                                  10000

                               </property>

                               <property name="max_tries">

                                  5

                               </property>

                          </protocol>

                          <protocol type="VERIFY_SUSPECT"/>

                          <protocol type="pbcast.NAKACK2"/>

                          <protocol type="UNICAST3"/>

                          <protocol type="pbcast.STABLE"/>

                          <protocol type="pbcast.GMS">

                              <property name="join_timeout">

                                 10000

                              </property>

                          </protocol>

                          <protocol type="MFC"/>

                          <protocol type="FRAG2"/>

                          <protocol type="RSVP"/>

                      </stack>

                  </stacks>

              </subsystem>

              <subsystem xmlns="urn:jboss:domain:jmx:1.3">

                  <expose-resolved-model/>

                  <expose-expression-model/>

                  <remoting-connector/>

              </subsystem>

              <subsystem xmlns="urn:jboss:domain:jpa:1.1">

                  <jpa default-datasource="" default-extended-persistence-inheritance="DEEP"/>

              </subsystem>

              <subsystem xmlns="urn:jboss:domain:jsf:1.0"/>

              <subsystem xmlns="urn:jboss:domain:mail:2.0">

                  <mail-session name="default" jndi-name="java:jboss/mail/Default">

                      <smtp-server outbound-socket-binding-ref="mail-smtp"/>

                  </mail-session>

              </subsystem>

              <subsystem xmlns="urn:jboss:domain:naming:2.0">

                  <remote-naming/>

              </subsystem>

              <subsystem xmlns="urn:jboss:domain:pojo:1.0"/>

              <subsystem xmlns="urn:jboss:domain:remoting:3.0">

                  <endpoint worker="default"/>

                  <http-connector name="http-remoting-connector" connector-ref="default" security-realm="ApplicationRealm"/>

              </subsystem>

              <subsystem xmlns="urn:jboss:domain:resource-adapters:4.0"/>

              <subsystem xmlns="urn:jboss:domain:request-controller:1.0"/>

              <subsystem xmlns="urn:jboss:domain:sar:1.0"/>

              <subsystem xmlns="urn:jboss:domain:security-manager:1.0">

                  <deployment-permissions>

                      <maximum-set>

                          <permission class="java.security.AllPermission"/>

                      </maximum-set>

                  </deployment-permissions>

              </subsystem>

              <subsystem xmlns="urn:jboss:domain:security:1.2">

                  <security-domains>

                      <security-domain name="ServerAuthRealm" cache-type="default">

                          <authentication>

                              <login-module code="com.viptela.vmanage.server.auth.AppServerLoginModule" flag="required"/>

                          </authentication>

                      </security-domain>

                      <security-domain name="other" cache-type="default">

                          <authentication>

                              <login-module code="Remoting" flag="optional">

                                  <module-option name="password-stacking" value="useFirstPass"/>

                              </login-module>

                              <login-module code="RealmDirect" flag="required">

                                  <module-option name="password-stacking" value="useFirstPass"/>

                              </login-module>

                          </authentication>

                      </security-domain>

                      <security-domain name="jboss-web-policy" cache-type="default">

                          <authorization>

                              <policy-module code="Delegating" flag="required"/>

                          </authorization>

                      </security-domain>

                      <security-domain name="jboss-ejb-policy" cache-type="default">

                          <authorization>

                              <policy-module code="Delegating" flag="required"/>

                          </authorization>

                      </security-domain>

                  </security-domains>

              </subsystem>

               <subsystem xmlns="urn:jboss:domain:singleton:1.0">

                  <singleton-policies default="default">

                      <singleton-policy name="default" cache-container="server">

                          <simple-election-policy/>

                      </singleton-policy>

                  </singleton-policies>

              </subsystem>

              <subsystem xmlns="urn:jboss:domain:transactions:3.0">

                  <core-environment>

                      <process-id>

                          <uuid/>

                      </process-id>

                  </core-environment>

                  <recovery-environment socket-binding="txn-recovery-environment" status-socket-binding="txn-status-manager"/>

              </subsystem>

              <subsystem xmlns="urn:jboss:domain:undertow:3.0">

                  <buffer-cache name="default"/>

                  <server name="default-server">

                      <http-listener name="default" socket-binding="http" tcp-keep-alive="true" enable-http2="false" redirect-socket="https"/>

                      <https-listener name="https" socket-binding="https" tcp-keep-alive="true" enable-http2="false" security-realm="SSLRealm"/>

                      <host name="default-host" alias="localhost">

                          <location name="/" handler="vmanage-content"/>

                          <filter-ref name="gzipFilter" predicate="exists('%{o,Content-Type}') and regex[pattern='(?:application/javascript|text/css|text/html|text/xml|application/json)(;.*)?', value=%{o,Content-Type}, full-match=true]"/>

                          <filter-ref name="Vary-header"/>

                          <filter-ref name="custom-max-age" predicate="path-suffix('.js') or path-suffix ('.html') or path-suffix ('.css') or path-suffix ('.jpg') or path-suffix ('.jpeg') or path-suffix ('.png') or path-suffix ('.gif')"/>

                          <filter-ref name="data-service-cache" predicate="path-prefix('/dataservice')"/>

                      </host>

                  </server>

                  <servlet-container name="default" disable-caching-for-secured-pages="false">

                      <jsp-config/>

                      <websockets/>

                  </servlet-container>

                  <handlers>

                      <file name="welcome-content" path="${jboss.home.dir}/welcome-content"/>

                      <file name="vmanage-content" path="/opt/web-app/webapps/vmanage.war"/>

                  </handlers>

                  <filters>

                      <response-header name="Vary-header" header-name="Vary" header-value="Accept-Encoding"/>

                      <response-header name="transport-security" header-name="Strict-Transport-Security" header-value="max-age=31536000; includeSubDomains"/>

                      <response-header name="x-frame-options" header-name="X-Frame-Options" header-value="DENY"/>

                      <gzip name="gzipFilter"/>

                      <response-header name="custom-max-age" header-name="Cache-Control" header-value="max-age=86400, private"/>

                      <response-header name="data-service-cache" header-name="Cache-Control" header-value="no-cache, no-store, must-revalidate"/>

                  </filters>

              </subsystem>

              <subsystem xmlns="urn:jboss:domain:webservices:2.0">

                  <wsdl-host>${jboss.bind.address:127.0.0.1}</wsdl-host>

                  <endpoint-config name="Standard-Endpoint-Config"/>

                  <endpoint-config name="Recording-Endpoint-Config">

                      <pre-handler-chain name="recording-handlers" protocol-bindings="##SOAP11_HTTP ##SOAP11_HTTP_MTOM ##SOAP12_HTTP ##SOAP12_HTTP_MTOM">

                          <handler name="RecordingHandler" class="org.jboss.ws.common.invocation.RecordingServerHandler"/>

                      </pre-handler-chain>

                  </endpoint-config>

                  <client-config name="Standard-Client-Config"/>

              </subsystem>

              <subsystem xmlns="urn:jboss:domain:weld:2.0"/>

          </profile>

       

       

          <interfaces>

              <interface name="management">

                  <inet-address value="${jboss.bind.address.management:0.0.0.0}"/>

              </interface>

              <interface name="public">

                  <inet-address value="${jboss.bind.address:0.0.0.0}"/>

              </interface>

              <interface name="unsecure">

                  <inet-address value="${jboss.bind.address.unsecure:127.0.0.1}"/>

              </interface>

              <interface name="jgroup-tcp-interface">

              </interface>

          </interfaces>

       

       

          <socket-binding-group name="standard-sockets" default-interface="public" port-offset="${jboss.socket.binding.port-offset:0}">

              <socket-binding name="management-http" interface="management" port="${jboss.management.http.port:9990}"/>

              <socket-binding name="management-https" interface="management" port="${jboss.management.https.port:9993}"/>

              <socket-binding name="http" port="${jboss.http.port:8080}"/>

              <socket-binding name="https" port="${jboss.https.port:8443}"/>

              <socket-binding name="jgroups-tcp" interface="jgroup-tcp-interface" port="7600"/>

              <socket-binding name="jgroups-tcp-fd" interface="jgroup-tcp-interface" port="57600"/>

              <socket-binding name="txn-recovery-environment" port="4712"/>

              <socket-binding name="txn-status-manager" port="4713"/>

              <outbound-socket-binding name="mail-smtp">

                  <remote-destination host="localhost" port="25"/>

              </outbound-socket-binding>

          </socket-binding-group>

      </server>

       

       

      SingletonServiceActivator code:

       

      // this is pretty much like quickstart/HATimerServiceActivator.java at 10.x · wildfly/quickstart · GitHub

      public class SingletonServiceActivator implements ServiceActivator {

       

          private final static Logger LOGGER =

                  LoggerFactory.getLogger(SingletonServiceActivator.class);

       

          public static final ServiceName SINGLETON_SERVICE_NAME =

                  ServiceName.JBOSS.append("ha", "singleton");  // in your example you guys use  ServiceName.JBOSS.append("quickstart", "ha", "singleton"); Can this make a difference ?

       

          private static final String CONTAINER_NAME = "server";

          private static final String CACHE_NAME = "default";

       

          @Override

          public void activate(ServiceActivatorContext context) throws ServiceRegistryException {

              LOGGER.info("SingletonServiceActivator will be installed");

              int quorum =  4 ;  // for 6 nodes and 2 for 3 nodes

              InjectedValue<ServerEnvironment> env = new InjectedValue<>();

              SingletonServiceClient srv = new SingletonServiceClient(env);

              ServiceController<?> factoryService = context.getServiceRegistry().getRequiredService(SingletonServiceName.BUILDER.getServiceName(CONTAINER_NAME, CACHE_NAME));

              SingletonServiceBuilderFactory factory = (SingletonServiceBuilderFactory) factoryService.getValue();   // I think we can use awaitValue() as well  ??

         

              factory.createSingletonServiceBuilder(SINGLETON_SERVICE_NAME, srv)

              .requireQuorum(quorum)

              .build(new DelegatingServiceContainer(context.getServiceTarget(),context.getServiceRegistry()))

              .addDependency(ServerEnvironmentService.SERVICE_NAME, ServerEnvironment.class, env)

              .setInitialMode(ServiceController.Mode.ACTIVE)

              .install();

              LOGGER.info("SingletonServiceActivator installation SUCCESSFUL");

          }

      }

       

      // client

       

       

      public final class SingletonServiceClient extends AbstractService<Serializable> {

       

        

          private static AtomicBoolean masterStatus = new AtomicBoolean(false);

       

          private final Value<ServerEnvironment> env;

       

          public SingletonServiceClient(Value<ServerEnvironment> env) {

              this.env = env;

          }

       

          @Override

          public String getValue() {

              if (!this.masterStatus.get()) {

                  throw new IllegalStateException();

              }

              // do something

          }

       

          @Override

          public void start(StartContext startContext) {

              //        startContext.

              LOGGER.info("SingletonService started");

              masterStatus.set(true);

              SingletonServiceManager.getInstance().startAllServices(masterStatus.get());

          }

       

          @Override

          public void stop(StopContext stopContext) {

              LOGGER.info("SingletonService stopped");

              masterStatus.set(false);

              SingletonServiceManager.getInstance().stopAllServices(masterStatus.get());

          }

       

          public static boolean isMaster() {

              return masterStatus.get();

          }

      }

       

      Are we missing something ?

       

      Thanks

      Mohil