5

I have a FreeNAS 8 machine (FreeBSD 8.2-RELEASE-p1) with two different shares on it. I'm migrating files from Share1 to Share2. The Shares have different directory structures and naming conventions. I'm looking for a way to verify that a file on Share1 actually exists (and is accurate) on Share2.

My plan is basically to checksum every file on both Shares and make sure that all the checksums in the Share1 set exist in the Share2 set. I can do this relatively easily with a bash script, but it'll take a long time to generate each of the checksum sets. My question is: Is it possible to access the native ZFS checksums to use them for this comparison?

Alan W. Smith
  • 219
  • 4
  • 10

1 Answers1

5

I don't believe it is possible to extract the block level checksums from a ZFS filesystem, but since the checksums are at the block level, not the file level, it probably wouldn't help you anyway.

A while back I wrote a couple of programs for producing a manifest of a filesystem and for comparing an existing filesystem with that manifest.

Here's chksum.pl

#! /usr/bin/perl
#
# generate a system manifest
#

use strict;

use File::Find;
use Digest::MD5;

our %config;

$| = 1;

&main;
exit;

sub initialize {
    my ($exclude_next, $include_next, @ex, @in, @ex_pats, @in_pats, %in, @x);

    # the exclude/include list looks like what you would pass to rsync
    my $elist = ['--exclude' => '/var/log',
                 '--exclude' => '/var/tmp',
                 '--exclude' => '/tmp',
                ];

    foreach my $op (@$elist) {
        if ($op eq '--exclude') {
            $exclude_next = 1;
        } elsif ($op eq '--include') {
            $include_next = 1;
        } elsif ($op eq '--delete') {
            # ignore
        } else {
            if ($exclude_next) {
                push(@ex, $op);
            } elsif ($include_next) {
                push(@in, $op);
            } else {
                warn "don't know what to do with $op\n";
            }
            $include_next = 0; $exclude_next = 0;
        }
    }

    $config{'exclude_list'} = \@ex;
    $config{'include_list'} = \@in;
    @ex_pats = map(glob($_), @ex);
    @in_pats = map(glob($_), @in);

#    print STDERR "exclusion patterns:\n", join("\n", @ex_pats), "\n\n";
#    print STDERR "inclusion patterns:\n", join("\n", @in_pats), "\n\n";

    # remove exclusions that exactly match inclusions
    foreach my $pat (@in_pats) {
        $in{$pat}++;
    }
    foreach my $pat (@ex_pats) {
        next if $in{$pat};
        push(@x, $pat);
    }

    $config{'ex_pats'} = \@x;
}

sub wanted {
    my($type, $extra);

    my $path = $File::Find::name;

    my @ex = grep($path =~ m"^$_", @{$config{'ex_pats'}});

    my($mode, $uid, $gid, $dev) = (lstat($path))[2,4,5,6];

    if (scalar(@ex) > 0) {
        # if we're excluding, and it was a dir, don't bother to descend
        $File::Find::prune = -d _;
        return;
    }

    $mode &= 07777;                             # mask off mode bits
    if (-d _) {
        $type = "dir ";
    } elsif (-l _) {
        $type = "link";
        $extra = "\"" . readlink($path) . "\"";
    } elsif (-f _) {
        $type = "file";
        $extra = &md5sum($path);
        return unless defined($extra);
    } elsif (-p _) {
        $type = "pipe";
    } elsif (-S _) {
        $type = "sock";
    } elsif (-b _) {
        $type = "bdev";
        $extra = "\"" . ($dev >> 8) . " " . ($dev & 0377) . "\"";
    } elsif (-c _) {
        $type = "cdev";
        $extra = "\"" . ($dev >> 8) . " " . ($dev & 0377) . "\"";
    } else {
        $type = "unk ";
    }


    printf "%s 0%04o %d %d \"%s\"", $type, $mode, $uid, $gid, $path;
    print " $extra" if $extra;
    print "\n";
}

sub md5sum {
    my $file = shift;
    my $context = Digest::MD5->new;
    unless(open(F, $file)) {
        warn "WARNING: couldn't open $file: $!\n";
        return;
    }
    $context->addfile(*F);
    close(F);
    my $digest = $context->hexdigest;
    return($digest);
}

sub main {
    &initialize;

    &find({no_chdir => 1, wanted => \&wanted}, '/');
}

and here's verify_sum.pl

#! /usr/bin/perl

# verify that a system matches the sum file produced by chksum.pl

use strict;

use File::Find;
use Digest::MD5;
use Text::ParseWords;

our %config;

$| = 1;

&main;
exit;

sub initialize {
    my ($exclude_next, $include_next, @ex, @in, @ex_pats, @in_pats, %in, @x);

    my $elist = ['--exclude' => '/var/log',
                 '--exclude' => '/var/tmp',
                 '--exclude' => '/tmp',
                ];

    foreach my $op (@$elist) {
        if ($op eq '--exclude') {
            $exclude_next = 1;
        } elsif ($op eq '--include') {
            $include_next = 1;
        } elsif ($op eq '--delete') {
            # ignore
        } else {
            if ($exclude_next) {
                push(@ex, $op);
            } elsif ($include_next) {
                push(@in, $op);
            } else {
                warn "don't know what to do with $op\n";
            }
            $include_next = 0; $exclude_next = 0;
        }
    }

    $config{'exclude_list'} = \@ex;
    $config{'include_list'} = \@in;
    @ex_pats = map(glob($_), @ex);
    @in_pats = map(glob($_), @in);

#    print STDERR "exclusion patterns:\n", join("\n", @ex_pats), "\n\n";
#    print STDERR "inclusion patterns:\n", join("\n", @in_pats), "\n\n";

    # remove exclusions that exactly match inclusions
    foreach my $pat (@in_pats) {
        $in{$pat}++;
    }
    foreach my $pat (@ex_pats) {
        next if $in{$pat};
        push(@x, $pat);
    }

    $config{'ex_pats'} = \@x;
}

sub md5sum {
    my $file = shift;
    my $context = Digest::MD5->new;
    unless(open(F, $file)) {
        warn "WARNING: couldn't open $file: $!\n";
        return;
    }
    $context->addfile(*F);
    close(F);
    my $digest = $context->hexdigest;
    return($digest);
}

sub wanted {
    my $path = $File::Find::name;

    my @ex = grep($path =~ m"^$_", @{$config{'ex_pats'}});

    if (scalar(@ex) > 0) {
        # if we're excluding, and it was a dir, don't bother to descend
        $file::Find::prune = -d $path;
        return;
    }

    if (! $config{'seen'}->{$path}) {
        warn "new file: $path\n";
        $config{'new'}++;
    }
}

sub main {
    $config{'mismatch'} = 0;
    $config{'missing'} = 0;
    $config{'new'} = 0;

    &initialize;

    while (<>) {
        chomp;
        my ($type, $mode, $uid, $gid, $path, $extra) = &shellwords($_);
        my ($etype, $eextra);

        $mode = oct($mode);
        my($emode, $euid, $egid, $edev) = (lstat($path))[2,4,5,6];

        if (! -e _) {
            warn "missing file: $path\n";
            $config{'missing'}++;
            next;
        }

        $emode &= 07777;
        if (-d _) {
            $etype = "dir";
        } elsif (-l _) {
            $etype = "link";
            $eextra = readlink($path);
        } elsif (-f _) {
            $etype = "file";
            $eextra = &md5sum($path);
            return unless defined($eextra);
        } elsif (-p _) {
            $etype = "pipe";
        } elsif (-S _) {
            $etype = "sock";
        } elsif (-b _) {
            $etype = "bdev";
            $eextra = ($edev >> 8) . " " . ($edev & 0377);
        } elsif (-c _) {
            $etype = "cdev";
            $eextra = ($edev >> 8) . " " . ($edev & 0377);
        } else {
            $etype = "unk";
        }

        if ($type ne $etype) {
            warn "mismatch file type ($type vs $etype): $path\n";
            $config{'mismatch'}++;
        } else {
            # types are the same, compare other factors

            if ($mode != $emode) {
                warn sprintf("mismatch mode (0%o vs 0%o): %s\n", $mode, $emode, $path);
                $config{'mismatch'}++;
            }
            if ($uid != $euid) {
                warn "mismatch uid ($uid vs $euid): $path\n";
                $config{'mismatch'}++;
            }
            if ($gid != $egid) {
                warn "mismatch gid ($gid vs $egid): $path\n";
                $config{'mismatch'}++;
            }
            if ($extra ne $eextra) {
                if ($etype eq 'link') {
                    warn "mismatch link target ($extra vs $eextra): $path\n";
                } elsif ($etype eq 'file') {
                    warn "mismatch file checksum ($extra vs $eextra): $path\n";
                } elsif ($etype eq 'bdev' or $etype eq 'cdev') {
                    warn "mismatch device node ($extra vs $eextra): $path\n";
                }
                $config{'mismatch'}++;
            }
        }

        $config{'seen'}->{$path}++;
    }

    # now walk the filesystem looking for "new" files

    &find({no_chdir => 1, wanted => \&wanted}, '/');

    print "$config{'mismatch'} files changed\n";
    print "$config{'missing'} files missing\n";
    print "$config{'new'} new files\n";

    if ($config{'mismatch'} > 0 or
        $config{'missing'} > 0 or
        $config{'new'} > 0) {
        exit 1;
    }

    exit 0;
}

You'll probably want to tweak these scripts a bit for things like the exclude list, root directory of the comparison, etc.

jlp
  • 401
  • 2
  • 5
  • 1
    Thank you. Unfortunately, the FreeNAS release I have doesn't have perl on it. I wrote a simple bash script to create a list of hashes that I'll move to another machine with perl to check. The script I wrote is based off [this post](http://www.cafewebmaster.com/how-recursively-create-md5-and-sha1-sum-your-files). Works well for me when updated to ignore hidden files and setup to output to a file. (e.g. `find /mnt/tank01 \( ! -regex '.*/\..*' \) -type f -exec ~/sum.sh '{}' \; > full-tank01-list.txt`) – Alan W. Smith Feb 13 '13 at 14:41
  • If you've installed bash in FreeNAS (`pkg install bash`), you can use bash 4's "globstar" option to make a list: `shopt -s globstar; md5 **/* > /tmp/files.md5`. Then on your target, use process recirection: `shopt -s globstar; diff /path/to/files.md5 <(md5 **/*)`. The result show show you missing or added files, along with mismatched MD5s. You could do similar things with more typing without bash, of course. – ghoti Jun 06 '16 at 18:35