5

I have an azure File Storage share mounted in a Ubuntu 16.04 VM following this how-to. Everything seems to work fine. So I run some process that produces a lot of data, e.g. 10k files for each run, collect them into a directory for each run, and move the directory where the storage has been mounted. Some of those directory are kind of broken and if I run a ls on them, the command hangs indefinitely. Any hint? Thanks in advance.

EDIT as suggested by Andrew in the comments, I run a strace for the ls command and the output is the following:

08:54:14.405929 execve("/bin/ls", ["ls"], [/* 29 vars */]) = 0
08:54:14.406240 brk(NULL)               = 0x9b1000
08:54:14.406325 access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
08:54:14.406422 mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ff0a669d000
08:54:14.406487 access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory)
08:54:14.406536 open("/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
08:54:14.406593 fstat(3, {st_mode=S_IFREG|0644, st_size=35482, ...}) = 0
08:54:14.406652 mmap(NULL, 35482, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7ff0a6694000
08:54:14.406700 close(3)                = 0
08:54:14.406766 access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
08:54:14.406814 open("/lib/x86_64-linux-gnu/libselinux.so.1", O_RDONLY|O_CLOEXEC) = 3
08:54:14.406857 read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\260Z\0\0\0\0\0\0"..., 832) = 832
08:54:14.406898 fstat(3, {st_mode=S_IFREG|0644, st_size=130224, ...}) = 0
08:54:14.406944 mmap(NULL, 2234080, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7ff0a6258000
08:54:14.406995 mprotect(0x7ff0a6277000, 2093056, PROT_NONE) = 0
08:54:14.407063 mmap(0x7ff0a6476000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1e000) = 0x7ff0a6476000
08:54:14.407124 mmap(0x7ff0a6478000, 5856, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7ff0a6478000
08:54:14.407182 close(3)                = 0
08:54:14.407237 access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
08:54:14.407280 open("/lib/x86_64-linux-gnu/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
08:54:14.407342 read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0P\t\2\0\0\0\0\0"..., 832) = 832
08:54:14.407392 fstat(3, {st_mode=S_IFREG|0755, st_size=1868984, ...}) = 0
08:54:14.407437 mmap(NULL, 3971488, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7ff0a5e8e000
08:54:14.407503 mprotect(0x7ff0a604e000, 2097152, PROT_NONE) = 0
08:54:14.407547 mmap(0x7ff0a624e000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1c0000) = 0x7ff0a624e000
08:54:14.407593 mmap(0x7ff0a6254000, 14752, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7ff0a6254000
08:54:14.407648 close(3)                = 0
08:54:14.407704 access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
08:54:14.407754 open("/lib/x86_64-linux-gnu/libpcre.so.3", O_RDONLY|O_CLOEXEC) = 3
08:54:14.407806 read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0000\25\0\0\0\0\0\0"..., 832) = 832
08:54:14.407849 fstat(3, {st_mode=S_IFREG|0644, st_size=456632, ...}) = 0
08:54:14.407891 mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ff0a6693000
08:54:14.407932 mmap(NULL, 2552072, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7ff0a5c1e000
08:54:14.407981 mprotect(0x7ff0a5c8c000, 2097152, PROT_NONE) = 0
08:54:14.408038 mmap(0x7ff0a5e8c000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6e000) = 0x7ff0a5e8c000
08:54:14.408094 close(3)                = 0
08:54:14.408144 access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
08:54:14.408208 open("/lib/x86_64-linux-gnu/libdl.so.2", O_RDONLY|O_CLOEXEC) = 3
08:54:14.408259 read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\240\r\0\0\0\0\0\0"..., 832) = 832
08:54:14.408301 fstat(3, {st_mode=S_IFREG|0644, st_size=14608, ...}) = 0
08:54:14.408343 mmap(NULL, 2109680, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7ff0a5a1a000
08:54:14.408391 mprotect(0x7ff0a5a1d000, 2093056, PROT_NONE) = 0
08:54:14.408433 mmap(0x7ff0a5c1c000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x7ff0a5c1c000
08:54:14.408490 close(3)                = 0
08:54:14.408537 access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
08:54:14.408607 open("/lib/x86_64-linux-gnu/libpthread.so.0", O_RDONLY|O_CLOEXEC) = 3
08:54:14.408652 read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\260`\0\0\0\0\0\0"..., 832) = 832
08:54:14.408695 fstat(3, {st_mode=S_IFREG|0755, st_size=138696, ...}) = 0
08:54:14.408738 mmap(NULL, 2212904, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7ff0a57fd000
08:54:14.408788 mprotect(0x7ff0a5815000, 2093056, PROT_NONE) = 0
08:54:14.408831 mmap(0x7ff0a5a14000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x17000) = 0x7ff0a5a14000
08:54:14.408882 mmap(0x7ff0a5a16000, 13352, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7ff0a5a16000
08:54:14.408954 close(3)                = 0
08:54:14.409008 mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ff0a6692000
08:54:14.409091 mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ff0a6690000
08:54:14.409152 arch_prctl(ARCH_SET_FS, 0x7ff0a6690800) = 0
08:54:14.409252 mprotect(0x7ff0a624e000, 16384, PROT_READ) = 0
08:54:14.409352 mprotect(0x7ff0a5a14000, 4096, PROT_READ) = 0
08:54:14.409409 mprotect(0x7ff0a5c1c000, 4096, PROT_READ) = 0
08:54:14.409490 mprotect(0x7ff0a5e8c000, 4096, PROT_READ) = 0
08:54:14.409574 mprotect(0x7ff0a6476000, 4096, PROT_READ) = 0
08:54:14.409630 mprotect(0x61d000, 4096, PROT_READ) = 0
08:54:14.409674 mprotect(0x7ff0a669f000, 4096, PROT_READ) = 0
08:54:14.409721 munmap(0x7ff0a6694000, 35482) = 0
08:54:14.409761 set_tid_address(0x7ff0a6690ad0) = 2107
08:54:14.409801 set_robust_list(0x7ff0a6690ae0, 24) = 0
08:54:14.409842 rt_sigaction(SIGRTMIN, {0x7ff0a5802b50, [], SA_RESTORER|SA_SIGINFO, 0x7ff0a580e390}, NULL, 8) = 0
08:54:14.409889 rt_sigaction(SIGRT_1, {0x7ff0a5802be0, [], SA_RESTORER|SA_RESTART|SA_SIGINFO, 0x7ff0a580e390}, NULL, 8) = 0
08:54:14.409930 rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0
08:54:14.409969 getrlimit(RLIMIT_STACK, {rlim_cur=8192*1024, rlim_max=RLIM64_INFINITY}) = 0
08:54:14.410106 statfs("/sys/fs/selinux", 0x7ffe99d30390) = -1 ENOENT (No such file or directory)
08:54:14.410209 statfs("/selinux", 0x7ffe99d30390) = -1 ENOENT (No such file or directory)
08:54:14.410345 brk(NULL)               = 0x9b1000
08:54:14.410397 brk(0x9d2000)           = 0x9d2000
08:54:14.410452 open("/proc/filesystems", O_RDONLY) = 3
08:54:14.410540 fstat(3, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0
08:54:14.410607 read(3, "nodev\tsysfs\nnodev\trootfs\nnodev\tr"..., 1024) = 362
08:54:14.410698 read(3, "", 1024)       = 0
08:54:14.410763 close(3)                = 0
08:54:14.410857 open("/usr/lib/locale/locale-archive", O_RDONLY|O_CLOEXEC) = 3
08:54:14.410918 fstat(3, {st_mode=S_IFREG|0644, st_size=1668976, ...}) = 0
08:54:14.410964 mmap(NULL, 1668976, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7ff0a64f8000
08:54:14.410997 close(3)                = 0
08:54:14.411040 open("/usr/share/locale/locale.alias", O_RDONLY|O_CLOEXEC) = 3
08:54:14.411092 fstat(3, {st_mode=S_IFREG|0644, st_size=2995, ...}) = 0
08:54:14.411139 read(3, "# Locale name alias data base.\n#"..., 4096) = 2995
08:54:14.411247 read(3, "", 4096)       = 0
08:54:14.411299 close(3)                = 0
08:54:14.411405 open("/usr/lib/locale/it_IT.UTF-8/LC_IDENTIFICATION", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
08:54:14.411460 open("/usr/lib/locale/it_IT.utf8/LC_IDENTIFICATION", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
08:54:14.411511 open("/usr/lib/locale/it_IT/LC_IDENTIFICATION", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
08:54:14.411550 open("/usr/lib/locale/it.UTF-8/LC_IDENTIFICATION", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
08:54:14.411603 open("/usr/lib/locale/it.utf8/LC_IDENTIFICATION", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
08:54:14.411668 open("/usr/lib/locale/it/LC_IDENTIFICATION", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
08:54:14.411711 ioctl(1, TCGETS, {B38400 opost isig icanon echo ...}) = 0
08:54:14.411759 ioctl(1, TIOCGWINSZ, {ws_row=88, ws_col=272, ws_xpixel=0, ws_ypixel=0}) = 0
08:54:14.411873 open(".", O_RDONLY|O_NONBLOCK|O_DIRECTORY|O_CLOEXEC) = 3
08:54:14.411918 fstat(3, {st_mode=S_IFDIR|0777, st_size=0, ...}) = 0
08:54:14.873927 getdents(3,

and here everything hangs.

P.S. On the azure dashboard, I can see all the files but (AFAIK) there is no way to run a bulk download on the directory. Any similar workaround is appreciated, as well. UPDATE for this particular task, you can use the Storage Explorer.

petrux
  • 151
  • 5
  • 1
    Are you running the `ls` command from Ubuntu? If you are, you can try running it under `strace` to see what system call(s) it's getting hung up in: `strace -tt -o /path/to/output/file ls ...`. Use the `-tt` option to get time values so you can tell how long `ls` takes for each system call. – Andrew Henle Jul 31 '17 at 22:43
  • @AndrewHenle added the `strace` dump to the original question. Thanks. – petrux Aug 05 '17 at 14:15
  • 1
    @petrux did you ever get this resolved? I am encountering a very similar issue, `strace` even hangs on the same line. – doydoy Aug 16 '18 at 14:50
  • 1
    @doydoy no, never. I think it is something related to `cifs` and/or the underlying NFS protocol. From my experience, the Storage Explorer can save your life. HTH. – petrux Aug 17 '18 at 15:22
  • I'm experiencing this exact same issue on an Azure data disk. An `strace` dump reveals `ls` is hanging on the exact same syscall: `getdents(3,`. Any thoughts on resolving this? – Brannon Oct 14 '20 at 16:54

0 Answers0