RFC: New version of abrt-action-generate-core-backtrace - Crash-catcher - Fedora mailing-lists

1 Mar 2012

      The libunwind-based coredump-level backtrace generator is ready
for first review.
On my test problem directory, old one was generating this
core_backtrace:
ec1fd70dbee0db36eff9527254d9d2bbfd260f13 0x34a7 - [exe] -
ec1fd70dbee0db36eff9527254d9d2bbfd260f13 0x3dd8 - [exe] -
ec1fd70dbee0db36eff9527254d9d2bbfd260f13 0x2093 - [exe] -
ec1fd70dbee0db36eff9527254d9d2bbfd260f13 0x1659 - [exe] -
And new one generates this:
ec1fd70dbee0db36eff9527254d9d2bbfd260f13 0x34a7 close_stdout /usr/bin/md5sum -
ec1fd70dbee0db36eff9527254d9d2bbfd260f13 0x3dd8 close_stdout /usr/bin/md5sum -
ec1fd70dbee0db36eff9527254d9d2bbfd260f13 0x2093 - /usr/bin/md5sum -
ec1fd70dbee0db36eff9527254d9d2bbfd260f13 0x1659 - /usr/bin/md5sum -
879571cf1e8dd31736d0f24b55e0daffc1591278 0x196b3 __libc_start_main /lib/libc-2.14.90.so -
ec1fd70dbee0db36eff9527254d9d2bbfd260f13 0x1f99 - /usr/bin/md5sum -
It does not use gdb or eu-unstrip: search for build-ids
is implemented in the tool itself, backtracing is provided
by (patched) libunwind.
Segmentation faults are rather likely when libunwind tries to access missing
parts of coredump. Therefore the tool has SIGSEGV handler, which needs
to be extended so that we can survive it and continue extracting useful data,
if possible.
New abrt-action-generate-core-backtrace.c is below.
(Note: so far it prints results to stdout instead of saving them
in core_backtrace element.)
-- 
vda

/*
     Copyright (C) 2011  ABRT team
     Copyright (C) 2011  RedHat Inc

     This program is free software; you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
     the Free Software Foundation; either version 2 of the License, or
     (at your option) any later version.

     This program is distributed in the hope that it will be useful,
     but WITHOUT ANY WARRANTY; without even the implied warranty of
     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
     GNU General Public License for more details.

     You should have received a copy of the GNU General Public License along
     with this program; if not, write to the Free Software Foundation, Inc.,
     51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
#include "libabrt.h"
#include <libunwind-coredump.h>

/* For coredump parsing */
/* Endian detection */
#include <limits.h>
#include <byteswap.h>
#include <endian.h>
#if defined(__BYTE_ORDER) && __BYTE_ORDER == __BIG_ENDIAN
# define WE_ARE_BIG_ENDIAN    1
# define WE_ARE_LITTLE_ENDIAN 0
#elif defined(__BYTE_ORDER) && __BYTE_ORDER == __LITTLE_ENDIAN
# define WE_ARE_BIG_ENDIAN    0
# define WE_ARE_LITTLE_ENDIAN 1
#elif defined(_BYTE_ORDER) && _BYTE_ORDER == _BIG_ENDIAN
# define WE_ARE_BIG_ENDIAN    1
# define WE_ARE_LITTLE_ENDIAN 0
#elif defined(_BYTE_ORDER) && _BYTE_ORDER == _LITTLE_ENDIAN
# define WE_ARE_BIG_ENDIAN    0
# define WE_ARE_LITTLE_ENDIAN 1
#elif defined(BYTE_ORDER) && BYTE_ORDER == BIG_ENDIAN
# define WE_ARE_BIG_ENDIAN    1
# define WE_ARE_LITTLE_ENDIAN 0
#elif defined(BYTE_ORDER) && BYTE_ORDER == LITTLE_ENDIAN
# define WE_ARE_BIG_ENDIAN    0
# define WE_ARE_LITTLE_ENDIAN 1
#elif defined(__386__)
# define WE_ARE_BIG_ENDIAN    0
# define WE_ARE_LITTLE_ENDIAN 1
#else
# error "Can't determine endianness"
#endif
#include <elf.h>
#include <sys/procfs.h> /* struct elf_prstatus */

/* For SIGSEGV handler code */
#include <execinfo.h>
#include <sys/ucontext.h>

static
void handle_sigsegv(int sig, siginfo_t *info, void *ucontext)
{
         long ip;
         ucontext_t *uc;

         uc = ucontext;
         ip = uc->uc_mcontext.gregs[REG_EIP];
         dprintf(2, "signal:%d address:0x%lx ip:0x%lx\n",
                         sig,
                         /* this is void*, but using %p would print "(null)"
                          * even for ptrs which are not exactly 0, but, say, 0x123:
                          */
                         (long)info->si_addr,
                         ip);
         {
                 /* glibc extension */
                 void *array[50];
                 int size;
                 size = backtrace(array, 50);
                 backtrace_symbols_fd(array, size, 2);
         }
         _exit(1);
}

static void install_sigsegv_handler(void)
{
     struct sigaction sa;
     memset(&sa, 0, sizeof(sa));
     sa.sa_sigaction = handle_sigsegv;
     sa.sa_flags = SA_SIGINFO;
     sigaction(SIGSEGV, &sa, NULL);
     sigaction(SIGILL, &sa, NULL);
     sigaction(SIGFPE, &sa, NULL);
     sigaction(SIGBUS, &sa, NULL);
}

#if SIZEOF_OFF_T == 4
typedef uint32_t uoff_t;
#elif SIZEOF_OFF_T == 8
typedef uint64_t uoff_t;
#else
# error Unknown size of off_t!
#endif

union elf_header_t {
     Elf32_Ehdr h32;
     Elf64_Ehdr h64;
};
typedef union elf_header_t elf_header_t;

struct mapping_data_t {
     uoff_t start;
     char  *filename;
};
typedef struct mapping_data_t mapping_data_t;

struct coredump_segment_data_t
{
     uint32_t p_type;
     uint32_t p_flags;
     uoff_t   p_offset;
     uoff_t   p_vaddr;
     uoff_t   p_filesz;
     uoff_t   p_memsz;
     uoff_t   p_align;
     char    *filename;
     char    *build_id;
};
typedef struct coredump_segment_data_t coredump_segment_data_t;

/* For error messages only */
static const char *elf_filename;

/* Returns the list of mapping_data's, one per executable file
  * mentioned in map file.
  */
static GList *load_map_file(const char *filename)
{
     FILE *fp = fopen(filename, "r");
     if (!fp)
         //error_msg_and_die("Can't open '%s'", filename);
         return NULL; /* we do support "bare" coredump, w/o map file */

     GList *list = NULL;
     char *line;
     while ((line = xmalloc_fgetline(fp)) != NULL)
     {
         /* Parse lines of this form:
          * 08048000-08050000 r-xp 00000000 fd:01 665656 /usr/bin/md5sum
          * 08050000-08051000 r--p 00007000 fd:01 665656 /usr/bin/md5sum
          * 08051000-08052000 rw-p 00008000 fd:01 665656 /usr/bin/md5sum
          * 088ad000-088ce000 rw-p 00000000 00:00 0      [heap]
          * b76fe000-b7700000 rw-p 00000000 00:00 0
          * b7700000-b7701000 r-xp 00000000 00:00 0      [vdso]
          * bfdc3000-bfde4000 rw-p 00000000 00:00 0      [stack]
          */
         unsigned long long start;//, end, fileofs;
         //unsigned maj, min;
         //unsigned long long inode;
         char mode[20];
         char filename[1024];
         /* %c conversion doesn't store terminating NUL, need to prepare for that! */
         memset(filename, 0, sizeof(filename));
         int r = sscanf(line, "%llx-%*s %19s %*s %*s %*s %1023c",
                         &start, /*&end,*/ mode, /*&fileofs, &maj, &min, &inode,*/ filename);
         VERB3 log("mapping: line:'%s' r:%d", line, r);
         if (r < 2)
             error_msg("Malformed line in maps file, ignored: '%s'", line);
         free(line);
         if (r < 2)
             continue;
         if (strcmp(mode, "r-xp") != 0) /* not a read/execute private mapping? */
             continue;
         if (filename[0] != '/') /* filename is missing? */
             continue;
         mapping_data_t *mapping = xzalloc(sizeof(*mapping));
         mapping->start = start;
         mapping->filename = xstrdup(filename);
         VERB3 log("mapping: vaddr:%llx name:'%s'", (unsigned long long)start, filename);
         list = g_list_prepend(list, mapping);
     }
     list = g_list_reverse(list);

     fclose(fp);

     return list;
}

/* Reads ELF header. Dies if it does not pass sanity checks.
  */
static void read_elf_hdr(int fd, elf_header_t *elf_header)
{
#define elf_header32 elf_header->h32
#define elf_header64 elf_header->h64
     bool _64bits;

     /* No sane ELF32 file is going to be smaller than ELF64 _header_,
      * so let's just read 64-bit sized one.
      */
     xread(fd, &elf_header64, sizeof(elf_header64));

     if (memcmp(elf_header32.e_ident, "\x7f""ELF", 4) != 0)
         error_msg_and_die("'%s' is not an ELF file", elf_filename);

     if (elf_header32.e_ident[EI_CLASS] != ELFCLASS32
      && elf_header32.e_ident[EI_CLASS] != ELFCLASS64
     ) {
         error_msg_and_die("'%s' is not a 32/64 bit ELF file", elf_filename);
     }

     if (WE_ARE_LITTLE_ENDIAN != (elf_header32.e_ident[EI_DATA] == ELFDATA2LSB))
         error_msg_and_die("'%s' is endian-incompatible", elf_filename);

     _64bits = (elf_header32.e_ident[EI_CLASS] == ELFCLASS64);
     if (_64bits && sizeof(elf_header64.e_entry) > sizeof(off_t))
         error_msg_and_die("Can't process '%s': 64-bit file "
                                 "while only %u bits are supported",
                                 elf_filename, 8 * sizeof(off_t));

     /* paranoia checks */
     if (_64bits
             ? 0 /* todo: (elf_header64.e_ehsize != NN || elf_header64.e_phentsize != NN) */
             : (elf_header32.e_ehsize != 52 || elf_header32.e_phentsize != 32
                || (elf_header32.e_shnum != 0 && elf_header32.e_shentsize != 40)
               )
     ) {
         error_msg_and_die("'%s' has wrong e_ehsize, e_phentsize or e_shentsize", elf_filename);
     }
#undef elf_header32
#undef elf_header64
}

static char *get_build_id_from_elf(int fd, uoff_t offset_in_coredump, uoff_t size_in_coredump)
{
     elf_header_t elf_header;
#define elf_header32 elf_header.h32
#define elf_header64 elf_header.h64
     bool _64bits;

     VERB3 log("%s: offset:%llx size:%llx", __func__, (unsigned long long)offset_in_coredump, (unsigned long long)size_in_coredump);

     if (size_in_coredump < sizeof(elf_header))
         return NULL;

     xlseek(fd, offset_in_coredump, SEEK_SET);
     read_elf_hdr(fd, &elf_header);
     _64bits = (elf_header32.e_ident[EI_CLASS] == ELFCLASS64);

     /* We want to retrieve the following (example):
      * Program Headers:
      * Type           Offset   VirtAddr   PhysAddr   FileSiz MemSiz  Flg Align
      * NOTE           0x000168 0x08048168 0x08048168 0x00044 0x00044 R   0x4
      * Section Headers:
      * [Nr] Name               Type            Addr     Off    Size   ES Flg Lk Inf Al
      * [ 2] .note.ABI-tag      NOTE            08048168 000168 000020 00   A  0   0  4
      * [ 3] .note.gnu.build-id NOTE            08048188 000188 000024 00   A  0   0  4
      * Notes at offset 0x00000188 with length 0x00000024:
      *  Owner                 Data size       Description
      *  GNU                  0x00000014       NT_GNU_BUILD_ID (unique build ID bitstring)
      *        Build ID: f7802b9085ea10b5e8ea2074aa14659f83338712
      *
      * The same data (notes) is accessible through both phdrs and sections.
      * But section table is usually not present in the coredump.
      * Therefore we search for NOTE *segment* and try to find NT_GNU_BUILD_ID
      * note in the area it points to.
      */
     unsigned segments_cnt = (_64bits ? elf_header64.e_phnum : elf_header32.e_phnum);
     VERB3 log("segments_cnt:%u", segments_cnt);
     if (segments_cnt == 0)
         return NULL;
     if (segments_cnt > 0xffff) /* paranoia */
         error_msg_and_die("Too many ELF segments: %u", segments_cnt);
     unsigned phentsize = (_64bits ? elf_header64.e_phentsize : elf_header32.e_phentsize);

     uoff_t ofs = (_64bits ? elf_header64.e_phoff : elf_header32.e_phoff);
     VERB3 log("ofs:%llx segments_cnt*phentsize:%u", (unsigned long long)ofs, segments_cnt * phentsize);
     if (ofs + segments_cnt * phentsize >= size_in_coredump)
         return NULL;
     xlseek(fd, offset_in_coredump + ofs, SEEK_SET);
     char *segment_tab_buffer = xmalloc(segments_cnt * phentsize);
     xread(fd, segment_tab_buffer, segments_cnt * phentsize);
     const int MAX_NOTE_SIZE = 0xff00;
     char *note_buffer = xzalloc(MAX_NOTE_SIZE);

     char *result = NULL;
     unsigned i;
     for (i = 0; i < segments_cnt; i++)
     {
         VERB3 log("segment[%u]", i);
         void *hdr = segment_tab_buffer + i * phentsize;
         uoff_t p_offset;
         unsigned p_filesz;
         if (_64bits)
         {
             Elf64_Phdr *hdr64 = hdr;
             if (hdr64->p_type != PT_NOTE)
                 continue;
             VERB3 log("PT_NOTE");
             if (hdr64->p_filesz < sizeof(Elf32_Nhdr) + sizeof("GNU") + 1)
                 continue;
             if (hdr64->p_filesz > MAX_NOTE_SIZE)
                 continue;
             VERB3 log("p_filesz is good");
             p_offset = hdr64->p_offset;
             p_filesz = hdr64->p_filesz; /* or p_memsz, for notes they are the same */
         } else {
             Elf32_Phdr *hdr32 = hdr;
             if (hdr32->p_type != PT_NOTE)
                 continue;
             VERB3 log("PT_NOTE");
             if (hdr32->p_filesz < sizeof(Elf32_Nhdr) + sizeof("GNU") + 1)
                 continue;
             if (hdr32->p_filesz > MAX_NOTE_SIZE)
                 continue;
             VERB3 log("p_filesz is good");
             p_offset = hdr32->p_offset;
             p_filesz = hdr32->p_filesz;
         }
         if (p_offset + p_filesz > size_in_coredump)
             continue;
         VERB3 log("present in coredump");
         xlseek(fd, offset_in_coredump + p_offset, SEEK_SET);
         xread(fd, note_buffer, p_filesz);

         /* There may be more than one note. Loop over them. */
         unsigned note_size;
         char *ptr = note_buffer;
         do {
             /* Notes have the same format in 32 and 64 bits, can use Elf32_Nhdr type for both */
             Elf32_Nhdr *note_data = (void*)ptr;
             note_size = sizeof(*note_data) + note_data->n_namesz + note_data->n_descsz;
             note_size = (note_size + 3) & (~3);
             VERB3 log("note_size:%u", note_size);
             if (note_size > p_filesz)
                 error_msg_and_die("Malformed NT_GNU_BUILD_ID note (bad size fields)");
             if (note_data->n_type != NT_GNU_BUILD_ID)
                 continue;
             VERB3 log("NT_GNU_BUILD_ID");
             const char *note_name = (char*)(note_data + 1);
             if (note_data->n_namesz < 4 || strcmp(note_name, "GNU") != 0)
                 continue;
             VERB3 log("good name");
             if (note_data->n_descsz < 1) /* NT_GNU_BUILD_ID notes must have at least 1 byte */
                 continue;
             VERB3 log("good n_descsz");
             /* Ok, all checks passed, looks like we have it! */
             const char *note_desc = note_name + note_data->n_namesz;
             result = xmalloc(note_data->n_descsz * 2 + 1);
             bin2hex(result, note_desc, note_data->n_descsz)[0] = '\0';
             VERB3 log("result:'%s'", result);
             goto found;
         } while (ptr += note_size, p_filesz -= note_size, p_filesz > sizeof(Elf32_Nhdr));
     }
  found:
     free(segment_tab_buffer);
     free(note_buffer);

     return result;
#undef elf_header32
#undef elf_header64
}

/* Returns the list of coredump_segment_data's, one per segment
  * in the core dump.
  */
static GList *analyze_coredump(const char *filename, GList *mapping_list)
{
     elf_header_t elf_header;
#define elf_header32 elf_header.h32
#define elf_header64 elf_header.h64
     bool _64bits;

     int fd = xopen(filename, O_RDONLY);

     read_elf_hdr(fd, &elf_header);
     _64bits = (elf_header32.e_ident[EI_CLASS] == ELFCLASS64);

     /* paranoia checks */
     if (elf_header32.e_type != ET_CORE)
         error_msg_and_die("'%s' is not a core dump file (wrong e_type:%u)",
                         filename, elf_header32.e_type);

     uoff_t ofs = (_64bits ? elf_header64.e_phoff : elf_header32.e_phoff);
     xlseek(fd, ofs, SEEK_SET);
     unsigned segments_cnt = (_64bits ? elf_header64.e_phnum : elf_header32.e_phnum);

     GList *list = NULL;
     unsigned i;
     for (i = 0; i < segments_cnt; i++)
     {
         coredump_segment_data_t *cur = xzalloc(sizeof(*cur));
         if (_64bits)
         {
             Elf64_Phdr hdr64;
             xread(fd, &hdr64, sizeof(hdr64));
             cur->p_type   = hdr64.p_type  ;
             cur->p_flags  = hdr64.p_flags ;
             cur->p_offset = hdr64.p_offset;
             cur->p_vaddr  = hdr64.p_vaddr ;
             /*cur->p_paddr  = hdr64.p_paddr ; always 0 */
             if (hdr64.p_paddr != 0)
                 error_msg_and_die("'%s' has nonzero p_paddr in phdr[%u]", filename, i);
             cur->p_filesz = hdr64.p_filesz;
             cur->p_memsz  = hdr64.p_memsz ;
             cur->p_align  = hdr64.p_align ;
         } else {
             Elf32_Phdr hdr32;
             xread(fd, &hdr32, sizeof(hdr32));
             cur->p_type   = hdr32.p_type  ;
             cur->p_flags  = hdr32.p_flags ;
             cur->p_offset = hdr32.p_offset;
             cur->p_vaddr  = hdr32.p_vaddr ;
             /*cur->p_paddr  = hdr32.p_paddr ; always 0 */
             if (hdr32.p_paddr != 0)
                 error_msg_and_die("'%s' has nonzero p_paddr in phdr[%u]", filename, i);
             cur->p_filesz = hdr32.p_filesz;
             cur->p_memsz  = hdr32.p_memsz ;
             cur->p_align  = hdr32.p_align ;
         }
         list = g_list_prepend(list, cur);
     }
     list = g_list_reverse(list);

     /* Typical program headers in core dump:
      * Type Offset   VirtAddr           FileSiz  MemSiz   Flg Align
      * NOTE 0x0003c0 0x0000000000000000 0x000538 0x000000     0
      * LOAD 0x001000 0x0000000000400000 0x000000 0x004000 R E 0x1000
      * LOAD 0x001000 0x0000000000604000 0x001000 0x001000 RW  0x1000
      * LOAD 0x002000 0x0000000003387000 0x021000 0x021000 RW  0x1000
      * LOAD 0x023000 0x00002b4f04aaa000 0x000000 0x01c000 R E 0x1000
      * LOAD 0x023000 0x00002b4f04ac6000 0x002000 0x002000 RW  0x1000
      * LOAD 0x025000 0x00002b4f04cc6000 0x001000 0x001000 R   0x1000
      * LOAD 0x026000 0x00002b4f04cc7000 0x001000 0x001000 RW  0x1000
      * LOAD 0x027000 0x00002b4f04cc8000 0x000000 0x14e000 R E 0x1000
      * LOAD 0x027000 0x00002b4f04e16000 0x000000 0x200000     0x1000
      * LOAD 0x027000 0x00002b4f05016000 0x004000 0x004000 R   0x1000
      * LOAD 0x02b000 0x00002b4f0501a000 0x001000 0x001000 RW  0x1000
      * LOAD 0x02c000 0x00002b4f0501b000 0x006000 0x006000 RW  0x1000
      * LOAD 0x032000 0x00002b4f05021000 0x000000 0x35ce000 R  0x1000
      * LOAD 0x032000 0x00007fffcdf61000 0x016000 0x016000 RW  0x1000
      * LOAD 0x048000 0x00007fffcdffd000 0x003000 0x003000 R E 0x1000
      *
      * Some segments are only partially present, or not present at all
      * (p_filesz < p_memsz). Usually all code segments (RE) are like that.
      *
      * We need to examine code segments and if they are at least
      * partially present, retrieve build-ids from their notes.
      */
     for (GList *l = list; l; l = l->next)
     {
         coredump_segment_data_t *cur = l->data;
         if (cur->p_type != PT_LOAD)
             continue;
         if (!(cur->p_flags & PF_X))
             continue;
         cur->build_id = get_build_id_from_elf(fd, cur->p_offset, cur->p_filesz);
     }

//TODO: we can use mapping_list to retrieve build ids from the mapped files
//even if coredump does not contain ELF headers.

     for (GList *l = list; l; l = l->next)
     {
         coredump_segment_data_t *seg = l->data;
         for (GList *m = mapping_list; m; m = m->next)
         {
             mapping_data_t *map = m->data;
             if (seg->p_vaddr == map->start)
             {
                 seg->filename = xstrdup(map->filename);
                 break;
             }
         }
     }

     close(fd);

     return list;
#undef elf_header32
#undef elf_header64
}

int main(int argc, char **argv)
{
     /* I18n */
     setlocale(LC_ALL, "");
#if ENABLE_NLS
     bindtextdomain(PACKAGE, LOCALEDIR);
     textdomain(PACKAGE);
#endif

     abrt_init(argv);

     const char *dump_dir_name = ".";

     /* Can't keep these strings/structs static: _() doesn't support that */
     const char *program_usage_string = _(
         "& [-v] -d DIR\n"
         "\n"
         "Creates coredump-level backtrace from core dump and corresponding binary"
     );
     enum {
         OPT_v = 1 << 0,
         OPT_d = 1 << 1,
     };
     /* Keep enum above and order of options below in sync! */
     struct options program_options[] = {
         OPT__VERBOSE(&g_verbose),
         OPT_STRING('d', NULL, &dump_dir_name, "DIR", _("Problem directory")),
         OPT_END()
     };
     /*unsigned opts =*/ parse_opts(argc, argv, program_options, program_usage_string);

     export_abrt_envvars(0);

//    struct dump_dir *dd = dd_opendir(dump_dir_name, /*flags:*/ 0);
//    if (!dd)
//        return xfunc_die();

     install_sigsegv_handler();

     unw_addr_space_t as;
     struct UCD_info *ui;
     unw_cursor_t c;
     as = unw_create_addr_space(&_UCD_accessors, 0);
     if (!as)
         error_msg_and_die("unw_create_addr_space() failed");

     char *filename;

     filename = concat_path_file(dump_dir_name, FILENAME_MAPS);
     GList *mapping_list = load_map_file(filename);
     free(filename);

     filename = concat_path_file(dump_dir_name, FILENAME_COREDUMP);
     elf_filename = filename;
     GList *segment_list = analyze_coredump(filename, mapping_list);
     ui = _UCD_create(filename);
     if (!ui)
         error_msg_and_die("_UCD_create('%s') failed", filename);
     int ret = unw_init_remote(&c, as, ui);
     if (ret < 0)
         error_msg_and_die("unw_init_remote() failed: ret=%d\n", ret);
     free(filename);
     elf_filename = NULL;

     for (GList *l = mapping_list; l; l = l->next)
     {
         mapping_data_t *mapping = l->data;
         VERB3 log("add_backing_file_at_vaddr: vaddr:%llx name:'%s'", (unsigned long long)mapping->start, mapping->filename);
         if (_UCD_add_backing_file_at_vaddr(ui, mapping->start, mapping->filename) < 0)
             error_msg("Can't add backing file '%s'", filename);
     }

     struct strbuf *result = strbuf_new();
     int count = 1000;
     while (--count != 0)
     {
         unw_word_t ip;
         ret = unw_get_reg(&c, UNW_REG_IP, &ip);
         if (ret < 0)
             error_msg_and_die("unw_get_reg(UNW_REG_IP) failed: ret=%d\n", ret);

         coredump_segment_data_t *ip_seg = NULL;
         for (GList *l = segment_list; l; l = l->next)
         {
             coredump_segment_data_t *cur = l->data;
             if (cur->p_vaddr <= ip && cur->p_vaddr + cur->p_memsz > ip)
             {
                 ip_seg = cur;
                 break;
             }
         }

         //unw_proc_info_t pi;
         //ret = unw_get_proc_info(&c, &pi);
         //if (ret < 0)
         //    error_msg_and_die("unw_get_proc_info(ip=0x%lx) failed: ret=%d\n", (long) ip, ret);
         /* pi.start_ip, pi.end_ip might be useful */

         char funcname[10*1024]; /* mangled C++ names are HUGE */
         unw_word_t off;
         ret = unw_get_proc_name(&c, funcname, sizeof(funcname)-1, &off);
         if (ret != 0)
             strcpy(funcname, "-");

         /* Output line formam:
          * BUILD_ID OFFSET SYMBOL MODNAME FINGERPRINT
          * BUILD_ID: build id of the binary file the address is mapped to.
          * OFFSET: offset from the start of the executable section of the file
          * the stored instruction pointer points to.
          * SYMBOL: name of the function if it is present in the binary (which is often
          * the case for shared libraries).
          * MODNAME: name of the binary or library.
          * FINGERPRINT: fingerprint of the function the instruction pointer points to.
          * Not yet implemented.
          */
         strbuf_append_strf(result, "%s 0x%llx %s %s -\n",
                 (ip_seg && ip_seg->build_id) ? ip_seg->build_id : "-",
                 (unsigned long long)(ip_seg ? ip - ip_seg->p_vaddr : ip),
                 funcname,
                 (ip_seg && ip_seg->filename) ? ip_seg->filename : "-"
         );

         VERB3 log("step");
         ret = unw_step(&c);
         VERB3 log("step done:%d", ret);
         if (ret < 0)
             error_msg_and_die("FAILURE: unw_step() returned %d", ret);
         if (ret == 0)
             break;
     }
     VERB3 log("stepping ended");
     printf("%s", result->buf);

     _UCD_destroy(ui);

     return 0;
}