Tail -f on new files

Question

Is there a way to do something like this:

tail -f logs/

and make the stdout to be updated on every line added to each file already present in logs/ and to every file that will be created in logs/ after the command is issued?

Answer 1

Thanks all for support, but since nor mutitail nor tail -F nor watch tail seems to help for what I need, I developed a small solution in C. I post the code here, since maybe someone can find it useful. (There are missing checks and some weakness I know, but so far it's enough)

#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <unistd.h>
#include <errno.h>
#include <sys/types.h>
#include <sys/inotify.h>
#include <sys/stat.h>
#include <signal.h>
#include <dirent.h>
#include <linux/limits.h>
#define    CHAR_BACK   500

// * File handler structure
struct file_followed { long last_position; char filename[NAME_MAX]; struct file_followed * next; };
struct file_followed * file_list = NULL;

// * To quit peacefully
int cycle = 1;
void stopCycle(int u) { cycle = 0; }

// * Last tailed filename
char last_tailed[NAME_MAX];

void fileAdd(char * file) {
    struct file_followed ** list = &file_list;
    struct stat statdesc;

    if(stat(file, &statdesc) || !S_ISREG(statdesc.st_mode)) { return; }
    while(*list) { list = &((*list)->next); }
    *list = malloc(sizeof(struct file_followed));
    (*list)->last_position = -1;
    strcpy((*list)->filename, file);
    (*list)->next = NULL;
}

int fileTail(struct file_followed * item) {
    int ret = 0;
    FILE * fp = fopen(item->filename, "r");
    fseek(fp, 0, SEEK_END);
    long end_position = ftell(fp);

    if( end_position != item->last_position ) {
        if(strcmp(item->filename, last_tailed)) { strcpy(last_tailed, item->filename); printf("\n** %s **:\n", item->filename); }

        int start_position = item->last_position == -1 || item->last_position > end_position ? (end_position-CHAR_BACK > 0 ? end_position-CHAR_BACK : 0) : item->last_position;
        fseek(fp, start_position, SEEK_SET);

        int len = end_position - start_position;
        char * buf = malloc(len+1);
        fread(buf, len, 1, fp);
        buf[len] = '\0';
        printf("%s%s", len == CHAR_BACK ? "[...]" : "", buf);
        free(buf);

        item->last_position = end_position;
        ret = 1;
    }

    fclose(fp);
    return ret;
}

void fileRem(char * file) {
    struct file_followed ** list = &file_list;
    while(*list && strcmp((*list)->filename, file)) { list = &((*list)->next); }
    if(*list) { struct file_followed * todel = *list; *list = (*list)->next; free(todel); }
}

int main(int argc, char ** argv) {

    struct dirent **namelist;
    struct stat statdesc;
    struct timeval tv;
    fd_set set;
    int fd;
    int wd;
    int r;

    // * Help
    if(stat(argv[1], &statdesc) || !S_ISDIR(statdesc.st_mode)) { printf("[usage] %s dir-to-monitor\n", argv[0]); exit(EXIT_FAILURE); }

    // * Init
    chdir(argv[1]);
    memset(last_tailed, 0, sizeof(last_tailed));
    signal(SIGINT, stopCycle);
    signal(SIGTERM, stopCycle);

    // * Inotify
    if( (fd = inotify_init()) < 0) { perror("inotify_init"); }
    if( (wd = inotify_add_watch( fd, ".", IN_CREATE | IN_DELETE ) < 0)) { perror("inotify_add_watch"); }

    // * File add recursively on dirscan
    if( (r = scandir(".", &namelist, 0, alphasort)) < 0) { perror("scandir"); }
    while (r--) { fileAdd(namelist[r]->d_name); free(namelist[r]); }
    free(namelist);

    // * Neverending cycle
    while(cycle) {
        // * Select on inotify
        FD_ZERO(&set);
        FD_SET(fd, &set);
        tv.tv_sec = 0;
        tv.tv_usec = 1000;
        if( (r = select(fd+1, &set, NULL, NULL, &tv)) == -1) { perror("select"); }

        // * New add or del on inotify
        if(r) {
            struct inotify_event * event;
            char buf[1024];
            if(read(fd, buf, 1024) <= 0) { perror("read"); }
            event = (struct inotify_event *) buf;
            if(event->mask & IN_CREATE) { fileAdd(event->name); } 
            else if(event->mask & IN_DELETE) { fileRem(event->name); }
        }

        // * Check for new tails
        struct file_followed * list = file_list;
        int tailers = 0;
        while(list) { tailers += fileTail(list); list = list->next; }
        if(!tailers) { usleep(500000); }
    }

    // * Stop inotify
    inotify_rm_watch( fd, wd );
    close(fd);

    return EXIT_SUCCESS;
}

Answer 2

I made changes to https://serverfault.com/a/542580/203373 to fix a couple of compile errors on my system (using Ubuntu linux). I added casts to (struct file_followed*) and (char*), and included IN_MODIFY in the add watch list to watch for modifications to current files. Added this line:

if(event->mask & IN_MODIFY) { fileMod(event->name, file_list); }

and the fileMod function

void fileMod(char* fileName, struct file_followed* file_list)

to check if a modified file was truncated, and print out if it had been as well as update item->last_position = -1so that it reprints the file.

#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <unistd.h>
#include <errno.h>
#include <sys/types.h>
#include <sys/inotify.h>
#include <sys/stat.h>
#include <signal.h>
#include <dirent.h>
#include <linux/limits.h>
#define    CHAR_BACK   500

// * File handler structure
struct file_followed { long last_position; char filename[NAME_MAX]; struct file_followed * next; };
struct file_followed * file_list = NULL;

// * To quit peacefully
int cycle = 1;
void stopCycle(int u) { cycle = 0; }

// * Last tailed filename
char last_tailed[NAME_MAX];

void fileAdd(char * file) {
    struct file_followed ** list = &file_list;
    struct stat statdesc;

    if(stat(file, &statdesc) || !S_ISREG(statdesc.st_mode)) { return; }
    while(*list) { list = &((*list)->next); }
    *list = (struct file_followed*)malloc(sizeof(struct file_followed));
    (*list)->last_position = -1;
    strcpy((*list)->filename, file);
    (*list)->next = NULL;
}

void fileMod(char* fileName, struct file_followed* file_list) {
    struct file_followed* item = file_list;
    while(item) { 
        if(strcmp(item->filename, fileName) == 0) {
            FILE* fp = fopen(item->filename, "r");
            fseek(fp, 0, SEEK_END);
            long end_position = ftell(fp);
            fclose(fp);
            if (end_position <= item->last_position) {
                printf("\n** %s truncated **\n", fileName);
                item->last_position = -1;
            }
            usleep(100);
            return;
        }
        item = item->next;
    }
}

int fileTail(struct file_followed * item) {
    int ret = 0;
    FILE * fp = fopen(item->filename, "r");
    fseek(fp, 0, SEEK_END);
    long end_position = ftell(fp);

    if( end_position != item->last_position ) {
        if(strcmp(item->filename, last_tailed)) { strcpy(last_tailed, item->filename); printf("\n** %s **:\n", item->filename); }

        int start_position = item->last_position == -1 || item->last_position > end_position ? (end_position-CHAR_BACK > 0 ? end_position-CHAR_BACK : 0) : item->last_position;
                    fseek(fp, start_position, SEEK_SET);

        int len = end_position - start_position;
        char * buf = (char*)malloc(len+1);
        fread(buf, len, 1, fp);
        buf[len] = '\0';
        printf("%s%s", len == CHAR_BACK ? "[...]" : "", buf);
        free(buf);

        item->last_position = end_position;
        ret = 1;
    }

    fclose(fp);
    return ret;
}

void fileRem(char * file) {
    struct file_followed ** list = &file_list;
    while(*list && strcmp((*list)->filename, file)) { list = &((*list)->next); }
    if(*list) { struct file_followed * todel = *list; *list = (*list)->next; free(todel); }
}

int main(int argc, char ** argv) {

    struct dirent **namelist;
    struct stat statdesc;
    struct timeval tv;
    fd_set set;
    int fd;
    int wd;
    int r;

    // * Help
    if(stat(argv[1], &statdesc) || !S_ISDIR(statdesc.st_mode)) { printf("[usage] %s dir-to-monitor\n", argv[0]); exit(EXIT_FAILURE); }

    // * Init
    chdir(argv[1]);
    memset(last_tailed, 0, sizeof(last_tailed));
    signal(SIGINT, stopCycle);
    signal(SIGTERM, stopCycle);

    // * Inotify
    if( (fd = inotify_init()) < 0) { perror("inotify_init"); }
    if( (wd = inotify_add_watch( fd, ".", IN_CREATE | IN_MODIFY |IN_DELETE ) < 0)) { perror("inotify_add_watch"); }

    // * File add recursively on dirscan
    if( (r = scandir(".", &namelist, 0, alphasort)) < 0) { perror("scandir"); }
    while (r--) { fileAdd(namelist[r]->d_name); free(namelist[r]); }
    free(namelist);

    // * Neverending cycle
    while(cycle) {
        // * Select on inotify
        FD_ZERO(&set);
        FD_SET(fd, &set);
        tv.tv_sec = 0;
        tv.tv_usec = 1000;
        if( (r = select(fd+1, &set, NULL, NULL, &tv)) == -1) { perror("select"); }

        // * New add or del on inotify
        if(r) {
            struct inotify_event * event;
            char buf[1024];
            if(read(fd, buf, 1024) <= 0) { perror("read"); }
            event = (struct inotify_event *) buf;
            if(event->mask & IN_MODIFY) { fileMod(event->name, file_list);} 
            else if(event->mask & IN_CREATE) { fileAdd(event->name); } 
            else if(event->mask & IN_DELETE) { fileRem(event->name); }
        }

        // * Check for new tails
        struct file_followed * list = file_list;
        int tailers = 0;
        while(list) { tailers += fileTail(list); list = list->next; }
        if(!tailers) { usleep(500000); }
    }

    // * Stop inotify
    inotify_rm_watch( fd, wd );
    close(fd);

    return EXIT_SUCCESS;
}

Answer 3

I don't think there is a way with using only tail, but you should be able to achieve the same effect using watch together with tail. The only danger then becomes making sure you don't get a directory created instead of just a new file, which can be mitigated by making sure you use an appropriate shell glob passed to tail. Example: watch -n 2 tail *.log

Answer 4

This thread inspired me to write my own version of the retail script. My version of retail works cross-platform, including on macos.

Answer 5

You could use: tail -F logs/*

Bonus tip: Check out multitail, it's a great little command.

For example: Merge ALL apache logfiles (*access_log/*error_log) into one window:

multitail -cS apache --mergeall /var/log/apache2/*access_log --no-mergeall \  
  -cS apache_error --mergeall /var/log/apache2/*error_log --no-mergeall

Show 5 logfiles while merging 2 and put them in 2 columns with only one in the left column:

multitail -s 2 -sn 1,3  /var/log/apache/access.log -I /var/log/apache/error.log \ 
  /var/log/messages /var/log/mail.log /var/log/syslog

Answer 6

You might be able to use something like multitail to tail multiple files at the same time. If you combine that with the -F option (follow with retry, in case the file doesn't exist) it could get you what you're looking for.