Segmentation fault diagnoses

Question

I am trying to diagnose a segmentation fault issue that started to occur after upgrading to Ubuntu 14.04 (32 bit). I am running a TSM client 6.2.5-4 which worked fine up to the upgrade. I tried to use strace to get to the core of the problem, but I don't see what the problem is.

The SIGSEGV occurs right after an mmap2 call, so I thought it might be a lack of swap space of memory in general, but there is plenty of both available. Just before the mmap call there is also a timeout of a select call, so perhaps it has something to do with that.

Honestly, I am bit out of my depth here. I tried to ask the Ubuntu Community and the TSM community, but no word from either. So I am hoping that someone here will be able to tell me how to read this strace, the end of which I append (the rest can be found here http://pastebin.com/dz1faACE). I am also happy to provide any other information that might be of help! Thanks a lot in advance!

open("/etc/mtab", O_RDONLY|O_CLOEXEC)   = 4
futex(0xb718be2c, FUTEX_WAKE_PRIVATE, 2147483647) = 0
fstat64(4, {st_mode=S_IFREG|0644, st_size=780, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb6a50000
read(4, "/dev/sda1 / ext4 rw,errors=remou"..., 4096) = 780
read(4, "", 4096)                       = 0
close(4)                                = 0
munmap(0xb6a50000, 4096)                = 0
brk(0x9fa8000)                          = 0x9fa8000
open("/etc/mtab", O_RDONLY|O_CLOEXEC)   = 4
fstat64(4, {st_mode=S_IFREG|0644, st_size=780, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb6a50000
read(4, "/dev/sda1 / ext4 rw,errors=remou"..., 4096) = 780
brk(0x9fd9000)                          = 0x9fd9000
read(4, "", 4096)                       = 0
close(4)                                = 0
munmap(0xb6a50000, 4096)                = 0
open("/proc/mounts", O_RDONLY|O_LARGEFILE) = 4
fstat64(4, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb6a50000
read(4, "rootfs / rootfs rw 0 0\nsysfs /sy"..., 1024) = 1024
read(4, "lv001 /data ext4 rw,relatime,dat"..., 1024) = 46
read(4, "", 1024)                       = 0
close(4)                                = 0
munmap(0xb6a50000, 4096)                = 0
brk(0x9fc0000)                          = 0x9fc0000
brk(0x9fa8000)                          = 0x9fa8000
brk(0x9f8f000)                          = 0x9f8f000
openat(AT_FDCWD, "/tmp/TSM", O_RDONLY|O_NONBLOCK|O_LARGEFILE|O_DIRECTORY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
mmap2(NULL, 512000, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb638b000
open("/etc/mtab", O_RDONLY|O_CLOEXEC)   = 4
fstat64(4, {st_mode=S_IFREG|0644, st_size=780, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb6a50000
read(4, "/dev/sda1 / ext4 rw,errors=remou"..., 4096) = 780
read(4, "", 4096)                       = 0
close(4)                                = 0
munmap(0xb6a50000, 4096)                = 0
open("/etc/mtab", O_RDONLY|O_CLOEXEC)   = 4
fstat64(4, {st_mode=S_IFREG|0644, st_size=780, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb6a50000
read(4, "/dev/sda1 / ext4 rw,errors=remou"..., 4096) = 780
read(4, "", 4096)                       = 0
close(4)                                = 0
munmap(0xb6a50000, 4096)                = 0
open("/proc/mounts", O_RDONLY|O_LARGEFILE) = 4
fstat64(4, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb6a50000
read(4, "rootfs / rootfs rw 0 0\nsysfs /sy"..., 1024) = 1024
read(4, "lv001 /data ext4 rw,relatime,dat"..., 1024) = 46
read(4, "", 1024)                       = 0
close(4)                                = 0
munmap(0xb6a50000, 4096)                = 0
statfs64("/", 84, {f_type="EXT2_SUPER_MAGIC", f_bsize=4096, f_blocks=3063340, f_bfree=708785, f_bavail=547416, f_files=786432, f_ffree=540306, f_fsid={-371423568, -29257012}, f_namelen=255, f_frsize=4096, f_flags=4128}) = 0
stat64("/", {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
open("/etc/mtab", O_RDONLY|O_CLOEXEC)   = 4
fstat64(4, {st_mode=S_IFREG|0644, st_size=780, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb6a50000
read(4, "/dev/sda1 / ext4 rw,errors=remou"..., 4096) = 780
close(4)                                = 0
munmap(0xb6a50000, 4096)                = 0
statfs64("/proc", 84, {f_type="PROC_SUPER_MAGIC", f_bsize=4096, f_blocks=0, f_bfree=0, f_bavail=0, f_files=0, f_ffree=0, f_fsid={0, 0}, f_namelen=255, f_frsize=4096, f_flags=4142}) = 0
stat64("/proc", {st_mode=S_IFDIR|0555, st_size=0, ...}) = 0
statfs64("/sys", 84, {f_type="SYSFS_MAGIC", f_bsize=4096, f_blocks=0, f_bfree=0, f_bavail=0, f_files=0, f_ffree=0, f_fsid={0, 0}, f_namelen=255, f_frsize=4096, f_flags=4142}) = 0
stat64("/sys", {st_mode=S_IFDIR|0555, st_size=0, ...}) = 0
statfs64("/sys/fs/cgroup", 84, {f_type=0x1021994, f_bsize=4096, f_blocks=1, f_bfree=1, f_bavail=1, f_files=202220, f_ffree=202218, f_fsid={0, 0}, f_namelen=255, f_frsize=4096, f_flags=4128}) = 0
stat64("/sys/fs/cgroup", {st_mode=S_IFDIR|0755, st_size=60, ...}) = 0
statfs64("/sys/fs/fuse/connections", 84, {f_type=0x65735543, f_bsize=4096, f_blocks=0, f_bfree=0, f_bavail=0, f_files=0, f_ffree=0, f_fsid={0, 0}, f_namelen=255, f_frsize=4096, f_flags=4128}) = 0
stat64("/sys/fs/fuse/connections", {st_mode=S_IFDIR|0755, st_size=0, ...}) = 0
statfs64("/sys/kernel/debug", 84, {f_type=0x64626720, f_bsize=4096, f_blocks=0, f_bfree=0, f_bavail=0, f_files=0, f_ffree=0, f_fsid={0, 0}, f_namelen=255, f_frsize=4096, f_flags=4128}) = 0
stat64("/sys/kernel/debug", {st_mode=S_IFDIR|0700, st_size=0, ...}) = 0
statfs64("/sys/kernel/security", 84, {f_type=0x73636673, f_bsize=4096, f_blocks=0, f_bfree=0, f_bavail=0, f_files=0, f_ffree=0, f_fsid={0, 0}, f_namelen=255, f_frsize=4096, f_flags=4128}) = 0
stat64("/sys/kernel/security", {st_mode=S_IFDIR|0755, st_size=0, ...}) = 0
statfs64("/dev", 84, {f_type=0x1021994, f_bsize=4096, f_blocks=1033006, f_bfree=1033003, f_bavail=1033003, f_files=197227, f_ffree=196785, f_fsid={0, 0}, f_namelen=255, f_frsize=4096, f_flags=4128}) = 0
stat64("/dev", {st_mode=S_IFDIR|0755, st_size=4240, ...}) = 0
open("/etc/fstab", O_RDONLY|O_CLOEXEC)  = 4
fstat64(4, {st_mode=S_IFREG|0644, st_size=989, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb6a50000
read(4, "# /etc/fstab: static file system"..., 4096) = 989
read(4, "", 4096)                       = 0
close(4)                                = 0
munmap(0xb6a50000, 4096)                = 0
open("/etc/mtab", O_RDONLY|O_CLOEXEC)   = 4
fstat64(4, {st_mode=S_IFREG|0644, st_size=780, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb6a50000
read(4, "/dev/sda1 / ext4 rw,errors=remou"..., 4096) = 780
read(4, "", 4096)                       = 0
close(4)                                = 0
munmap(0xb6a50000, 4096)                = 0
statfs64("/run", 84, {f_type=0x1021994, f_bsize=4096, f_blocks=207101, f_bfree=206972, f_bavail=206972, f_files=202220, f_ffree=201823, f_fsid={0, 0}, f_namelen=255, f_frsize=4096, f_flags=4138}) = 0
stat64("/run", {st_mode=S_IFDIR|0755, st_size=660, ...}) = 0
statfs64("/run/lock", 84, {f_type=0x1021994, f_bsize=4096, f_blocks=1280, f_bfree=1280, f_bavail=1280, f_files=202220, f_ffree=202214, f_fsid={0, 0}, f_namelen=255, f_frsize=4096, f_flags=4142}) = 0
stat64("/run/lock", {st_mode=S_IFDIR|S_ISVTX|0777, st_size=120, ...}) = 0
statfs64("/run/shm", 84, {f_type=0x1021994, f_bsize=4096, f_blocks=1035503, f_bfree=1035503, f_bavail=1035503, f_files=202220, f_ffree=202219, f_fsid={0, 0}, f_namelen=255, f_frsize=4096, f_flags=4134}) = 0
stat64("/run/shm", {st_mode=S_IFDIR|S_ISVTX|0777, st_size=40, ...}) = 0
statfs64("/run/user", 84, {f_type=0x1021994, f_bsize=4096, f_blocks=25600, f_bfree=25600, f_bavail=25600, f_files=202220, f_ffree=202217, f_fsid={0, 0}, f_namelen=255, f_frsize=4096, f_flags=4142}) = 0
stat64("/run/user", {st_mode=S_IFDIR|0755, st_size=80, ...}) = 0
statfs64("/sys/fs/pstore", 84, {f_type=0x6165676c, f_bsize=4096, f_blocks=0, f_bfree=0, f_bavail=0, f_files=0, f_ffree=0, f_fsid={0, 0}, f_namelen=255, f_frsize=4096, f_flags=4128}) = 0
stat64("/sys/fs/pstore", {st_mode=S_IFDIR|0755, st_size=0, ...}) = 0
statfs64("/sys/fs/cgroup/systemd", 84, {f_type=0x27e0eb, f_bsize=4096, f_blocks=0, f_bfree=0, f_bavail=0, f_files=0, f_ffree=0, f_fsid={0, 0}, f_namelen=255, f_frsize=4096, f_flags=4142}) = 0
stat64("/sys/fs/cgroup/systemd", {st_mode=S_IFDIR|0755, st_size=0, ...}) = 0
statfs64("/data", 84, {f_type="EXT2_SUPER_MAGIC", f_bsize=4096, f_blocks=128981614, f_bfree=93342103, f_bavail=86784459, f_files=32768000, f_ffree=32451931, f_fsid={-1906668704, -244025993}, f_namelen=255, f_frsize=4096, f_flags=4128}) = 0
stat64("/data", {st_mode=S_IFDIR|0775, st_size=4096, ...}) = 0
open("/etc/mtab", O_RDONLY|O_CLOEXEC)   = 4
fstat64(4, {st_mode=S_IFREG|0644, st_size=780, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb6a50000
read(4, "/dev/sda1 / ext4 rw,errors=remou"..., 4096) = 780
close(4)                                = 0
munmap(0xb6a50000, 4096)                = 0
munmap(0xb638b000, 512000)              = 0
statfs64("/", 84, {f_type="EXT2_SUPER_MAGIC", f_bsize=4096, f_blocks=3063340, f_bfree=708785, f_bavail=547416, f_files=786432, f_ffree=540306, f_fsid={-371423568, -29257012}, f_namelen=255, f_frsize=4096, f_flags=4128}) = 0
stat64("/", {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
open("/etc/mtab", O_RDONLY|O_CLOEXEC)   = 4
fstat64(4, {st_mode=S_IFREG|0644, st_size=780, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb6a50000
read(4, "/dev/sda1 / ext4 rw,errors=remou"..., 4096) = 780
close(4)                                = 0
munmap(0xb6a50000, 4096)                = 0
mmap2(NULL, 8392704, PROT_READ|PROT_WRITE|PROT_EXEC, MAP_PRIVATE|MAP_ANONYMOUS|MAP_STACK, -1, 0) = 0xb5c07000
mprotect(0xb5c07000, 4096, PROT_NONE)   = 0
clone(child_stack=0xb6407424, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID, parent_tidptr=0xb6407ba8, {entry_number:6, base_addr:0xb6407b40, limit:1048575, seg_32bit:1, contents:0, read_exec_only:0, limit_in_pages:1, seg_not_present:0, useable:1}, child_tidptr=0xb6407ba8) = 13914
select(0, NULL, NULL, NULL, {0, 50000}) = 0 (Timeout)
sched_yield()                           = 0
mmap2(NULL, 2101248, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb5a06000
--- SIGSEGV {si_signo=SIGSEGV, si_code=SEGV_MAPERR, si_addr=0x2} ---
time(NULL)                              = 1425498940
open("/etc/localtime", O_RDONLY|O_CLOEXEC) = 4
fstat64(4, {st_mode=S_IFREG|0644, st_size=2309, ...}) = 0
fstat64(4, {st_mode=S_IFREG|0644, st_size=2309, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb6a50000
read(4, "TZif2\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\10\0\0\0\10\0\0\0\0"..., 4096) = 2309
_llseek(4, -28, [2281], SEEK_CUR)       = 0
read(4, "\nCET-1CEST,M3.5.0,M10.5.0/3\n", 4096) = 28
close(4)                                = 0
munmap(0xb6a50000, 4096)                = 0
write(3, "03/04/2015 20:55:40 ANS0361I DIA"..., 71) = 71
rt_sigaction(SIGABRT, {SIG_DFL, [], SA_RESTORER|SA_STACK|SA_RESTART|SA_INTERRUPT|SA_NODEFER|SA_RESETHAND|SA_SIGINFO|SA_NOCLDSTOP|SA_NOCLDWAIT|0x3fffff8, 0xb770c100}, NULL, 8) = 0
rt_sigprocmask(SIG_UNBLOCK, [ABRT], NULL, 8) = 0
tgkill(13911, 13911, SIGABRT)           = 0
--- SIGABRT {si_signo=SIGABRT, si_code=SI_TKILL, si_pid=13911, si_uid=0} ---
+++ killed by SIGABRT (core dumped) +++
Aborted (core dumped)

score 1 · Answer 1 · answered Mar 05 '15 at 19:08

An option could be that after the mmap2 call the application accessed an address that was previously unmapped by a munmap call. This would also throw a SIGSEGV. But you'll need more than strace to see this, some memory debugger could help you at runtime like valgrind.

From man 2 munmap :

int munmap(void *addr, size_t length); 
[...]

The address addr must be a multiple of the page size. All pages containing a part of the indicated range are unmapped, and subsequent references to these pages will generate SIGSEGV. It is not an error if the indicated range does not contain any mapped pages.

Thanks for the answer. Hm, this sounds like you suspect a problem with the programme itself. But the problem must be somehow related to the OS, since the problem only occurred after upgrading. Perhaps I'll try my luck at askubuntu... — Philipp, Mar 06 '15 at 09:41

Segmentation fault diagnoses

1 Answers1