Nginx + php5-fpm + CodeIgniter Response Too Slow

Question

I have installed nginx, php5-fpm and setup codeigniter with REST controller on Ubuntu 12.04 LTS. The configurations work but response is too slow at times, even with just a single user. Sometimes the nginx would respond with 504 status code. Please find below the snippet of my configuration file. Also, please let me know if I need to provide any more configurations information.

Thank you.

    location / {                                                                                                                                                                                       
            # First attempt to serve request as file, then                                                                                                                                             
            # as directory, then fall back to index.html                                                                                                                                               
            try_files $uri $uri/ /index.php;                                                                                                                                                           
            # Uncomment to enable naxsi on this location                                                                                                                                               
            # include /etc/nginx/naxsi.rules                                                                                                                                                           
    }                                                                                                                                                                                                  

    location /doc/ {                                                                                                                                                                                   
            alias /usr/share/doc/;                                                                                                                                                                     
            autoindex on;                                                                                                                                                                              
            allow 127.0.0.1;                                                                                                                                                                           
            deny all;                                                                                                                                                                                  
    }                                                                                                                                                                                                  

    # Only for nginx-naxsi : process denied requests                                                                                                                                                   
    #location /RequestDenied {                                                                                                                                                                         
            # For example, return an error code                                                                                                                                                        
            #return 418;                                                                                                                                                                               
    #}                                                                                                                                                                                                 

    error_page 404 /404.html;                                                                                                                                                                          

    # redirect server error pages to the static page /50x.html                                                                                                                                         
    #                                                                                                                                                                                                  
    #error_page 500 502 503 504 /50x.html;                                                                                                                                                             
    #location = /50x.html {                                                                                                                                                                            
    #       root /usr/share/nginx/www;                                                                                                                                                                 
    #}                                                                                                                                                                                                 

    # pass the PHP scripts to FastCGI server listening on 127.0.0.1:9000                                                                                                                               
    #                                                                                                                                                                                                  
    location ~ \.php$ {                                                                                                                                                                                
    #       fastcgi_split_path_info ^(.+\.php)(/.+)$;                                                                                                                                                  
    #       # NOTE: You should have "cgi.fix_pathinfo = 0;" in php.ini                                                                                                                                 
    #                                                                                                                                                                                                  
    #       # With php5-cgi alone:                                                                                                                                                                     
              #     fastcgi_pass 127.0.0.1:9000;                                                                                                                                                       
    #       # With php5-fpm:                                                                                                                                                                           
            fastcgi_pass unix:/var/run/php5-fpm.sock;                                                                                                                                                  
            fastcgi_index index.php;                                                                                                                                                                   
            include fastcgi_params;                                                                                                                                                                    
    }