How to configure Apache not to keep access.log or error.log open all the time for virtualhosts?

Question

I want to lower number of open files on my VPS. With lsof I noticed that Apache constantly keeps access.log and error.log open for each domain hosted on server.

Is there a way to change this behaviour?

Answer 1

Many open files are usually not a problem. There's a system limit, which you can check using command sysctl fs.file-max - on my CentOS 6 system with 4GB RAM it is 382299, which should be plenty, and on 256MB VM I started it was 23539, which can be somewhat too little. But you can easily increase this limit for example by adding to /etc/sysctl.conf:

fs.file-max = 500000

And then running sysctl -p or rebooting.

You can write a program which will open, write, close a log file every time data is received from stdin, and use it like this:

CustomLog "||open-write-close /var/log/httpd/access_log" common

but it won't be efficient. And it wan't be easy, as it would need to use async IO.

---

I've written a program for this in C:

#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <fcntl.h>
#include <string.h>
#include <stdio.h>
#include <errno.h>
#include <limits.h>

int outfd;

int main(int argn, char* argv[])
{
    if (argn != 2) {
        fprintf(stderr, "Usage %s [output_file]\n", argv[0]);
        return 1;
    }

    {
        long stdinflags = fcntl(0,F_GETFL);
        if ( stdinflags == -1 ) {
            perror("fcntl: Can't read stdin status");
            return 2;
        }
        if ( fcntl(0,F_SETFL,stdinflags|O_NONBLOCK) == -1 ) {
            perror("fcntl: Can't set stdin to non-blocking mode");
            return 2;
        }
    }


    do {
        #define BUFSIZE PIPE_BUF
        char buf[BUFSIZE];
        ssize_t bytesread = read(0, buf, BUFSIZE);

        {
            fd_set select_fdset;
            FD_ZERO(&select_fdset);
            FD_SET(0, &select_fdset);
            if ( select(1, &select_fdset, NULL, NULL, NULL) == -1 ) {
                if ( errno != EINTR ) {
                    perror("select");
                    return 2;
                }
            };
        }
        if ( bytesread==-1 ) {
            perror("Can't read from stdin");
            return 2;
        }
        if ( bytesread == 0 ) break;
        outfd = open(argv[1],O_WRONLY|O_APPEND|O_CREAT);
        if ( outfd < 0 ) {
            fprintf(stderr, "Can't open file \"%s\": %s\n", argv[1], strerror(errno));
            return 2;
        }
        do {
            if ( write(outfd, buf, bytesread) == -1 ) {
                fprintf(stderr, "Can't write to file \"%s\": %s\n", argv[1], strerror(errno));
                return 2;
            };
            bytesread = read(0, buf, BUFSIZE);
            if ( bytesread==-1 ) {
                if ( errno==EAGAIN ) break;
                perror("Can't read from stdin");
            }
        } while ( bytesread>0 );
        if ( close(outfd) != 0) {
            fprintf(stderr, "Can't close file \"%s\": %s\n", argv[1], strerror(errno));
            return 2;
        }
    } while ( 1 );
    return 0;
}

Answer 2

If you have really big number of vhosts with separate access log files, I suggest to log everything to one file (Yes, eliminate CustomLog in every vhost), and later use split-logfile application to separate them. You can join it with logresolve script, to resolve DNS names.

To split-logfile work, you need to set first column in CustomLog to ServerName.