Mongrel over mod_proxy unacceptably slow

Question

Here is the situation

C:\>ab -n 100 -c 5 http://webserver/track/
This is ApacheBench, Version 2.3 <$Revision: 655654 $>
Copyright 1996 Adam Twiss, Zeus Technology Ltd, http://www.zeustech.net/
Licensed to The Apache Software Foundation, http://www.apache.org/

Benchmarking webserver (be patient).....done


Server Software:        Mongrel
Server Hostname:        webserver
Server Port:            80

Document Path:          /track/
Document Length:        111 bytes

Concurrency Level:      5
Time taken for tests:   20.176 seconds
Complete requests:      100
Failed requests:        0
Write errors:           0
Non-2xx responses:      100
Total transferred:      58104 bytes
HTML transferred:       11100 bytes
Requests per second:    4.96 [#/sec] (mean)
Time per request:       1008.801 [ms] (mean)
Time per request:       201.760 [ms] (mean, across all concurrent requests)
Transfer rate:          2.81 [Kbytes/sec] received

Connection Times (ms)
              min  mean[+/-sd] median   max
Connect:        0    1   0.5      1       2
Processing:   994 1005  12.8    999    1055
Waiting:      994 1004  12.8    997    1055
Total:        995 1006  12.9    999    1056

Percentage of the requests served within a certain time (ms)
  50%    999
  66%   1009
  75%   1013
  80%   1014
  90%   1015
  95%   1045
  98%   1055
  99%   1056
 100%   1056 (longest request)

C:\>ab -n 100 -c 5 http://webserver:8001/track/
This is ApacheBench, Version 2.3 <$Revision: 655654 $>
Copyright 1996 Adam Twiss, Zeus Technology Ltd, http://www.zeustech.net/
Licensed to The Apache Software Foundation, http://www.apache.org/

Benchmarking webserver (be patient).....done


Server Software:        Mongrel
Server Hostname:        webserver
Server Port:            8001

Document Path:          /track/
Document Length:        116 bytes

Concurrency Level:      5
Time taken for tests:   0.711 seconds
Complete requests:      100
Failed requests:        0
Write errors:           0
Non-2xx responses:      100
Total transferred:      59106 bytes
HTML transferred:       11600 bytes
Requests per second:    140.63 [#/sec] (mean)
Time per request:       35.554 [ms] (mean)
Time per request:       7.111 [ms] (mean, across all concurrent requests)
Transfer rate:          81.17 [Kbytes/sec] received

Connection Times (ms)
              min  mean[+/-sd] median   max
Connect:        0    1   0.3      1       2
Processing:    17   33  13.8     30      92
Waiting:       16   32  13.8     28      91
Total:         19   34  13.8     31      93

Percentage of the requests served within a certain time (ms)
  50%     31
  66%     32
  75%     34
  80%     34
  90%     35
  95%     93
  98%     93
  99%     93
 100%     93 (longest request)

As you can see going through Apache is several orders of magnitude slower than hitting Mongrel directly. Here is my httpd.conf:

LoadModule actions_module modules/mod_actions.so
LoadModule alias_module modules/mod_alias.so
LoadModule asis_module modules/mod_asis.so
LoadModule auth_basic_module modules/mod_auth_basic.so
#LoadModule auth_digest_module modules/mod_auth_digest.so
#LoadModule authn_alias_module modules/mod_authn_alias.so
#LoadModule authn_anon_module modules/mod_authn_anon.so
#LoadModule authn_dbd_module modules/mod_authn_dbd.so
#LoadModule authn_dbm_module modules/mod_authn_dbm.so
LoadModule authn_default_module modules/mod_authn_default.so
LoadModule authn_file_module modules/mod_authn_file.so
#LoadModule authnz_ldap_module modules/mod_authnz_ldap.so
#LoadModule authz_dbm_module modules/mod_authz_dbm.so
LoadModule authz_default_module modules/mod_authz_default.so
LoadModule authz_groupfile_module modules/mod_authz_groupfile.so
LoadModule authz_host_module modules/mod_authz_host.so
#LoadModule authz_owner_module modules/mod_authz_owner.so
LoadModule authz_user_module modules/mod_authz_user.so
LoadModule autoindex_module modules/mod_autoindex.so
#LoadModule cache_module modules/mod_cache.so
#LoadModule cern_meta_module modules/mod_cern_meta.so
LoadModule cgi_module modules/mod_cgi.so
#LoadModule charset_lite_module modules/mod_charset_lite.so
#LoadModule dav_module modules/mod_dav.so
#LoadModule dav_fs_module modules/mod_dav_fs.so
#LoadModule dav_lock_module modules/mod_dav_lock.so
#LoadModule dbd_module modules/mod_dbd.so
#LoadModule deflate_module modules/mod_deflate.so
LoadModule dir_module modules/mod_dir.so
#LoadModule disk_cache_module modules/mod_disk_cache.so
#LoadModule dumpio_module modules/mod_dumpio.so
LoadModule env_module modules/mod_env.so
#LoadModule expires_module modules/mod_expires.so
#LoadModule ext_filter_module modules/mod_ext_filter.so
#LoadModule file_cache_module modules/mod_file_cache.so
#LoadModule filter_module modules/mod_filter.so
LoadModule headers_module modules/mod_headers.so
#LoadModule ident_module modules/mod_ident.so
#LoadModule imagemap_module modules/mod_imagemap.so
LoadModule include_module modules/mod_include.so
#LoadModule info_module modules/mod_info.so
LoadModule isapi_module modules/mod_isapi.so
#LoadModule ldap_module modules/mod_ldap.so
#LoadModule logio_module modules/mod_logio.so
LoadModule log_config_module modules/mod_log_config.so
#LoadModule log_forensic_module modules/mod_log_forensic.so
#LoadModule mem_cache_module modules/mod_mem_cache.so
LoadModule mime_module modules/mod_mime.so
#LoadModule mime_magic_module modules/mod_mime_magic.so
LoadModule negotiation_module modules/mod_negotiation.so
LoadModule proxy_module modules/mod_proxy.so
#LoadModule proxy_ajp_module modules/mod_proxy_ajp.so
LoadModule proxy_balancer_module modules/mod_proxy_balancer.so
#LoadModule proxy_connect_module modules/mod_proxy_connect.so
#LoadModule proxy_ftp_module modules/mod_proxy_ftp.so
LoadModule proxy_http_module modules/mod_proxy_http.so
#LoadModule proxy_scgi_module modules/mod_proxy_scgi.so
#LoadModule reqtimeout_module modules/mod_reqtimeout.so
LoadModule rewrite_module modules/mod_rewrite.so
LoadModule setenvif_module modules/mod_setenvif.so
#LoadModule speling_module modules/mod_speling.so
LoadModule ssl_module modules/mod_ssl.so
#LoadModule status_module modules/mod_status.so
#LoadModule substitute_module modules/mod_substitute.so
#LoadModule unique_id_module modules/mod_unique_id.so
#LoadModule userdir_module modules/mod_userdir.so
#LoadModule usertrack_module modules/mod_usertrack.so
#LoadModule version_module modules/mod_version.so
#LoadModule vhost_alias_module modules/mod_vhost_alias.so
LoadModule sspi_auth_module modules/mod_auth_sspi.so
LoadModule fcgid_module modules/mod_fcgid.so

# Server configuration

ServerRoot "c:/apache2"
KeepAlive On
KeepAliveTimeout 5
TimeOut 30
ThreadsPerChild 10
Listen [redacted]:80
ServerAdmin [redacted]
ServerName webserver.domain
ErrorLog "logs/error.log"
LogLevel warn
DefaultType text/plain
DocumentRoot "c:/www"

<Directory "c:/www">
    RedirectMatch ^/$ /wiki/
    Options -Indexes -FollowSymLinks
    AllowOverride None
    Order allow,deny
    Allow from all
</Directory>

<Directory "c:/www/wiki">
    AuthName "Local Intranet"
    AuthType SSPI
    SSPIPerRequestAuth On
    SSPIAuth On
    require valid-user
</Directory>

<Proxy balancer://track>
    BalancerMember http://localhost:8001/track
    BalancerMember http://localhost:8002/track
    BalancerMember http://localhost:8003/track
</Proxy>

<Location /track>
    ProxyPass balancer://track
    ProxyPassReverse http://localhost:8001/track
</Location>

<Location /audit>
    ProxyPass http://localhost:8004/audit
    ProxyPassReverse http://localhost:8004/audit
</Location>

<Location /dev>
    ProxyPass http://localhost:8010/dev
    ProxyPassReverse http://localhost:8010/dev
</Location>

<Location /auth>
    AuthName "Local Intranet"
    AuthType SSPI
    SSPIPerRequestAuth On
    SSPIAuth On
    require valid-user

    RewriteEngine On
    RewriteCond %{REMOTE_USER} (.+)
    RewriteRule ^.*$ - [E=RU:%1]
    RequestHeader add X-Forwarded-User %{RU}e

    ProxyPass  balancer://track
    ProxyPassReverse http://localhost:8001/auth
</Location>

<Location /bin>
    Order allow,deny
    Allow from all
    Satisfy Any
</Location>

<IfModule dir_module>
    DirectoryIndex index.php
</IfModule>

<FilesMatch "^\.ht">
    Order allow,deny
    Deny from all
    Satisfy All
</FilesMatch>

<IfModule log_config_module>
    LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined
    LogFormat "%h %l %u %t \"%r\" %>s %b" common

    CustomLog "logs/access.log" common
</IfModule>

<IfModule mime_module>
    TypesConfig conf/mime.types
    AddType application/x-compress .Z
    AddType application/x-gzip .gz .tgz
</IfModule>

<IfModule mod_fcgid.c>
    AddHandler fcgid-script .fcgi .php
    # Where to look for the php.ini file?
    FcgidInitialEnv PHPRC        "c:/php64"
    # Set PHP_FCGI_MAX_REQUESTS to greater than or equal to FcgidMaxRequestsPerProcess
    # to prevent php-cgi process from exiting before all requests completed
    FcgidInitialEnv PHP_FCGI_MAX_REQUESTS      1000
    # Maximum requests a process should handle before it is terminated
    FcgidMaxRequestsPerProcess       1000
    # Maximum number of PHP processes
    FcgidMaxProcesses             15
    # Number of seconds of idle time before a php-cgi process is terminated
    FcgidIOTimeout             120
    #Path to php-cgi
    FcgidWrapper "c:/php64/php-cgi.exe" .php
    # Define the MIME-Type for ".php" files
    AddType application/x-httpd-php .php
    FcgidMaxRequestLen 2097152
</IfModule>

I'm running MediaWiki along side my three load-balanced Mongrel processes. I have additional web apps mounted at different directories. I have tried stripping it down to a bare-minimum server that does nothing but forward to proxies. It makes no difference, going through Apache is significantly and noticeably slower.

The Mongrel processes are services created by the mongrel_service gem, and their start line in the registry looks like this:

"c:/ruby/bin/mongrel_service.exe" service single -e production -p 8005 -a 0.0.0.0 -l "log/authentication.log" -P "log/mongrel.pid" -c "c:/track" -t 0 -r "public" -n 1024 --prefix "/track"

What could be causing this slowdown? How do I troubleshoot?

Mongrel 0.4.0
Rails 2.3.3
Ruby 1.8.7
Apache 2.2.22.0
Windows Server 2008

score 1 · Accepted Answer · answered May 03 '13 at 20:41

1

I have seen that changing "localhost" to "127.0.0.1" drastically reduces the time mod_proxy takes. But I do not know why. I would actually love to see an answer to that. I can't imagine it takes a whole second to resolve localhost.

answered May 03 '13 at 20:41

Kasius

371
1
3
12

This worked! I changed every instance of 'localhost' with '127.0.0.1' in httpb.conf. Requests are faster now than they've ever been. I guess I'm left with the question, why would using localhost be so much slower? And more generally, what kind of troubleshooting steps could I have taken to come to that conclusion? – Monban May 04 '13 at 15:00

Mongrel over mod_proxy unacceptably slow

1 Answers1