7 Replies Latest reply on Nov 4, 2016 9:42 AM by mbabacek

    Apache 2.4.20 / mod_cluster 1.3.2 memory leak

    j_sykora

      Hi,

       

       

      I have problem with memory leak in Apache 2.4 with native mod_cluster 1.3.2 on Solaris 10 (x64)

      As I checked in prstat monitor there is memory growing in child process where STATUS request from mod_security is comming.

      Memory is growing up till httpd server is restarted or all usable memory is consumed.

      I found that same problem was also in older version od mod_cluster so I try to set workaround configuration (KeepAliveTimeout and MaxKeepAliveRequests)

      to my config files but there was no affect. 

      Because I build Apache with mpms-shared so I tested it in prefork, work and also event mode of Apache. But situation is same in all modes.

      Memory is growing up in child process which is used for STATUS request

       

      Do you have an experience with similar problem on Apache 2.4 ?

       

      H.

       

       

      Env:

      Apache/2.4.20 (Unix) OpenSSL/1.0.2h mod_cluster/1.3.2.Final configured

      SunOS 5.10 Generic_150401-27 i86pc i386 i86pc

       

      Apache server details:

      Server version: Apache/2.4.20 (Unix)

      Server built:   May 23 2016 14:23:52

      Server's Module Magic Number: 20120211:57

      Server loaded:  APR 1.5.2, APR-UTIL 1.5.4

      Compiled using: APR 1.5.2, APR-UTIL 1.5.4

      Architecture:   64-bit

      Server MPM:     prefork

        threaded:     no

          forked:     yes (variable process count)

      Server compiled with....

      -D APR_HAS_SENDFILE

      -D APR_HAS_MMAP

      -D APR_HAVE_IPV6 (IPv4-mapped addresses enabled)

      -D APR_USE_PROC_PTHREAD_SERIALIZE

      -D APR_USE_PTHREAD_SERIALIZE

      -D SINGLE_LISTEN_UNSERIALIZED_ACCEPT

      -D APR_HAS_OTHER_CHILD

      -D AP_HAVE_RELIABLE_PIPED_LOGS

      -D DYNAMIC_MODULE_LIMIT=256

      -D HTTPD_ROOT="/app01/apache2"

      -D SUEXEC_BIN="/app01/apache2/bin/suexec"

      -D DEFAULT_PIDLOG="logs/httpd.pid"

      -D DEFAULT_SCOREBOARD="logs/apache_runtime_status"

      -D DEFAULT_ERRORLOG="logs/error_log"

      -D AP_TYPES_CONFIG_FILE="conf/mime.types"

      -D SERVER_CONFIG_FILE="conf/httpd.conf"

       

       

      relevant information form httpd.conf:

      #

      # Dynamic Shared Object (DSO) Support

      #

      # To be able to use the functionality of a module which was built as a DSO you

      # have to place corresponding `LoadModule' lines at this location so the

      # directives contained in it are actually available _before_ they are used.

      # Statically compiled modules (those listed by `httpd -l') do not need

      # to be loaded here.

      #

      # Example:

      # LoadModule foo_module modules/mod_foo.so

      #

       

      LoadModule mpm_prefork_module modules/mod_mpm_prefork.so

      #LoadModule mpm_worker_module modules/mod_mpm_worker.so

      #LoadModule mpm_event_module modules/mod_mpm_event.so

       

      LoadModule authn_core_module modules/mod_authn_core.so

      LoadModule authz_core_module modules/mod_authz_core.so

      LoadModule authz_host_module modules/mod_authz_host.so

      LoadModule access_compat_module modules/mod_access_compat.so

      LoadModule auth_basic_module modules/mod_auth_basic.so

      LoadModule allowmethods_module modules/mod_allowmethods.so

      LoadModule reqtimeout_module modules/mod_reqtimeout.so

      LoadModule filter_module modules/mod_filter.so

      LoadModule mime_module modules/mod_mime.so

      LoadModule deflate_module modules/mod_deflate.so

      LoadModule log_config_module modules/mod_log_config.so

      LoadModule env_module modules/mod_env.so

      LoadModule headers_module modules/mod_headers.so

      LoadModule unique_id_module modules/mod_unique_id.so

      LoadModule setenvif_module modules/mod_setenvif.so

      LoadModule version_module modules/mod_version.so

      LoadModule proxy_module modules/mod_proxy.so

      LoadModule proxy_connect_module modules/mod_proxy_connect.so

      LoadModule proxy_http_module modules/mod_proxy_http.so

      LoadModule proxy_ajp_module modules/mod_proxy_ajp.so

      LoadModule ssl_module modules/mod_ssl.so

      LoadModule unixd_module modules/mod_unixd.so

      LoadModule status_module modules/mod_status.so

      LoadModule autoindex_module modules/mod_autoindex.so

      LoadModule dir_module modules/mod_dir.so

      LoadModule alias_module modules/mod_alias.so

      LoadModule rewrite_module modules/mod_rewrite.so

      LoadModule vhost_alias_module modules/mod_vhost_alias.so

       

      LoadModule security2_module modules/mod_security2.so

       

      LoadModule proxy_cluster_module modules/mod_proxy_cluster.so

      LoadModule manager_module modules/mod_manager.so

      LoadModule advertise_module modules/mod_advertise.so

      LoadModule cluster_slotmem_module modules/mod_cluster_slotmem.so

       

      <IfModule ssl_module>

        SSLPassPhraseDialog  builtin

        SSLSessionCache none

        #SSLSessionCache        "shmcb:/usr/local/apache2/logs/ssl_scache(512000)"

        #SSLSessionCacheTimeout  300

        #SSLMutex  "file:/usr/local/apache2/logs/ssl_mutex"

        SSLRandomSeed startup builtin

        SSLRandomSeed connect builtin

        #ErrorLog /usr/local/apache2/logs/ssl_engine.log

        #LogLevel debug

      </IfModule>

       

      ####### My config ########

       

      Listen 0.0.0.0:80

      Listen 0.0.0.0:443

      Listen 0.0.0.0:6666

       

      #NameVirtualHost *:443

      #NameVirtualHost *:80

      #NameVirtualHost *:6666

       

      ServerName myServer

       

      MaxNode 200

       

      <IfModule prefork.c>

        ServerLimit 100

        MinSpareServers 5

        MaxSpareServers 100

        MaxRequestWorkers 100

      </IfModule>

      <IfModule worker.c>

        StartServers 2

        MinSpareThreads 25

        MaxSpareThreads 100

        ThreadsPerChild 25

        MaxRequestWorkers 500

      </IfModule>

      <IfModule event.c>

        StartServers 2

        ServerLimit 30

        MinSpareThreads 25

        MaxSpareThreads 100

        AsyncRequestWorkerFactor 4

        ThreadsPerChild 20

        MaxRequestWorkers 400

      </IfModule>

       

       

       

       

      Config for VH of mod_cluster listener

      PersistSlots on

      <VirtualHost *:6666>

        ServerName  myServer

        ServerAlias myServer

        <Location />

          Require all granted

        </Location>

        #KeepAlive off

        KeepAliveTimeout 60

        #MaxKeepAliveRequests 0

        MaxKeepAliveRequests 200

        EnableMCPMReceive Off

        ServerAdvertise Off

        CustomLog "|/usr/apache2/bin/rotatelogs /usr/apache2/logs/MC-access_6666_log-%Y-%m-%d 86400" combined

        ErrorLog  "|/usr/apache2/bin/rotatelogs /usr/apache2/logs/MC-error_6666_log-%Y-%m-%d 86400"

      </VirtualHost>

       

      Message was edited by: Jan Sykora

       

      Message was edited by: Jan Sykora

        • 1. Re: Apache 2.4.20 / mod_cluster 1.3.2 memory leak
          mbabacek

          Hi Jan, I recall dealing with a similar issue on RHEL back in the day. Lemme give it a shot with this particular setup on a Solaris box. I'll get back to you.

          BTW: You might want to sanitize your config in the post...there are certain domains mentioned

          • 2. Re: Apache 2.4.20 / mod_cluster 1.3.2 memory leak
            j_sykora

            Hi Michal,

            thank you of course can use these setup, if is useful.

            It is my mistake, I try to sanitize it now, if is possible.

            Thanks

            H.

            • 3. Re: Apache 2.4.20 / mod_cluster 1.3.2 memory leak
              j_sykora

              Hi Michal,

               

              do you have some new information about this issue ?

               

              Thanks

              Jan

              • 4. Re: Apache 2.4.20 / mod_cluster 1.3.2 memory leak
                mbabacek

                I did take a look only yesterday and I seem to be able to reproduce it. I don't have a concrete result yet though. Stay tuned, please.

                • 5. Re: Apache 2.4.20 / mod_cluster 1.3.2 memory leak
                  jmsantuci

                  Hi Michal,

                   

                  I tested the same configuration with the same result on two different OS: RHEL 6.5 and Windows Server 2008 R2. In both OS the Apache memory stay stable while WildFly (8.2 version) servers are not running. After start them the memory is growing up until the Apache servers become slow and a restart is necessary.

                   

                   

                  Apache server configuration (http.conf):

                   

                  LoadModule cluster_slotmem_module modules/mod_cluster_slotmem.so

                  LoadModule manager_module modules/mod_manager.so

                  LoadModule proxy_cluster_module modules/mod_proxy_cluster.so

                  LoadModule advertise_module modules/mod_advertise.so

                   

                  Listen apache-server-ip:6666

                   

                  <IfModule manager_module>

                    <VirtualHost *:6666>

                   

                      KeepAliveTimeout 60

                      MaxKeepAliveRequests 0

                   

                      ManagerBalancerName my-balancer

                      ServerAdvertise Off

                      EnableMCPMReceive On

                      AllowDisplay On

                   

                      <Location />

                          Order deny,allow

                          Allow from all

                      </Location>

                   

                      <Location /mod_cluster-manager>

                          SetHandler mod_cluster-manager

                          Order deny,allow

                          Allow from all

                      </Location>

                   

                    </VirtualHost>

                  </IfModule>

                   

                   

                  WildFly configuration:

                   

                  <subsystem xmlns="urn:jboss:domain:modcluster:1.2">

                      <mod-cluster-config proxy-list="apache-server-ip:6666" balancer="my-balancer" advertise="false" sticky-session="true" connector="ajp">

                          <dynamic-load-provider history="10" decay="2">

                              <load-metric type="heap" weight="7"/>

                              <load-metric type="cpu" weight="3"/>

                          </dynamic-load-provider>

                      </mod-cluster-config>

                  </subsystem>

                   

                  Tks,

                   

                  José Marcelo

                  • 6. Re: Apache 2.4.20 / mod_cluster 1.3.2 memory leak
                    mbabacek
                    • 7. Re: Apache 2.4.20 / mod_cluster 1.3.2 memory leak
                      mbabacek

                      j_sykora,   mod_cluster 1.3.5.Final have these weird leaks resolved. Please, give it a shot:

                      GitHub - modcluster/mod_cluster at 1.3.5.Final

                       

                      If you have any troubles building Solaris binaries, tell me and I could build a test release for you.