Need help with jemalloc cores

Tue Sep 13 22:41:25 PDT 2016

Hi,

We are using jemalloc library (compiled with --enable-fill and
--enable-debug options) in a multi-threaded daemon and seeing multiple
cores during malloc/realloc with jemalloc library. The core files are
approximately 61GB and 79GB in size, so this is typically seen under heavy
memory usage situation.

Are these issues in jemalloc code or the application code that is using
jemalloc?

Can someone please clarify what these stacks / aborts mean and how to deal
with them?

stack for corefile1 -

(gdb) bt
#0  0x00007fdc399155f7 in raise () from /lib64/libc.so.6
#1  0x00007fdc39916ce8 in abort () from /lib64/libc.so.6
#2  0x00007fdc3a317fe4 in je_extent_heap_remove (ph=0x7fd44530de90,
phn=0x7fd0c022e0c0) at src/extent.c:191
#3  0x00007fdc3a3180e3 in extent_heaps_remove (extent_heaps=0x7fd44530dda0,
extent=0x7fd0c022e0c0) at src/extent.c:206
#4  0x00007fdc3a318c3f in extent_recycle (tsdn=0x7fdc329a6580,
arena=0x7fd44530b680, r_extent_hooks=0x7fdc329a5438,
extent_heaps=0x7fd44530dda0,
    cache=false, new_addr=0x7fd0bfd0b740, usize=16384, pad=0, alignment=64,
zero=0x7fdc329a5447, commit=0x7fdc329a5437, slab=false) at src/extent.c:424
#5  0x00007fdc3a3197f3 in extent_alloc_retained (tsdn=0x7fdc329a6580,
arena=0x7fd44530b680, r_extent_hooks=0x7fdc329a5438,
new_addr=0x7fd0bfd0b740,
    usize=16384, pad=0, alignment=64, zero=0x7fdc329a5447,
commit=0x7fdc329a5437, slab=false) at src/extent.c:608
#6  0x00007fdc3a319a91 in je_extent_alloc_wrapper (tsdn=0x7fdc329a6580,
arena=0x7fd44530b680, r_extent_hooks=0x7fdc329a5438,
new_addr=0x7fd0bfd0b740,
    usize=16384, pad=0, alignment=64, zero=0x7fdc329a5447,
commit=0x7fdc329a5437, slab=false) at src/extent.c:664
#7  0x00007fdc3a31d2c2 in large_ralloc_no_move_expand (tsdn=0x7fdc329a6580,
extent=0x7fd3412ba6c0, usize=32768, zero=false) at src/large.c:149
#8  0x00007fdc3a31d651 in je_large_ralloc_no_move (tsdn=0x7fdc329a6580,
extent=0x7fd3412ba6c0, usize_min=32768, usize_max=32768, zero=false)
    at src/large.c:205
#9  0x00007fdc3a31d92f in je_large_ralloc (tsdn=0x7fdc329a6580, arena=0x0,
extent=0x7fd3412ba6c0, usize=32768, alignment=0, zero=false,
    tcache=0x7fd538d85780) at src/large.c:264
#10 0x00007fdc3a305b10 in je_arena_ralloc (tsdn=0x7fdc329a6580, arena=0x0,
extent=0x7fd3412ba6c0, ptr=0x7fd0bfd06740, oldsize=16384, size=32768,
    alignment=0, zero=false, tcache=0x7fd538d85780) at src/arena.c:1567
#11 0x00007fdc3a2f9124 in je_iralloct (tsdn=0x7fdc329a6580,
extent=0x7fd3412ba6c0, ptr=0x7fd0bfd06740, oldsize=16384, size=32768,
alignment=0,
    zero=false, tcache=0x7fd538d85780, arena=0x0) at
include/jemalloc/internal/jemalloc_internal.h:1163
#12 0x00007fdc3a2f91ad in je_iralloc (tsd=0x7fdc329a6580,
extent=0x7fd3412ba6c0, ptr=0x7fd0bfd06740, oldsize=16384, size=32768,
alignment=0, zero=false)
    at include/jemalloc/internal/jemalloc_internal.h:1172
#13 0x00007fdc3a2fd883 in realloc (ptr=0x7fd0bfd06740, size=32768) at
src/jemalloc.c:1796

stack for corefile2 -
#0  0x00007f9170d7f5f7 in raise () from /lib64/libc.so.6
#1  0x00007f9170d80ce8 in abort () from /lib64/libc.so.6
#2  0x00007f917176b616 in je_arena_extent_cache_maybe_remove
(arena=0x7f8a0d652180, extent=0x7f874860e180, dirty=true) at src/arena.c:119
#3  0x00007f9171782c5c in extent_recycle (tsdn=0x7f916be14580,
arena=0x7f8a0d652180, r_extent_hooks=0x7f916be13388,
    extent_heaps=0x7f8a0d654268, cache=true, new_addr=0x0, usize=4096,
pad=0, alignment=4096, zero=0x7f916be13387, commit=0x7f916be13297,
    slab=true) at src/extent.c:425
#4  0x00007f91717834df in je_extent_alloc_cache (tsdn=0x7f916be14580,
arena=0x7f8a0d652180, r_extent_hooks=0x7f916be13388, new_addr=0x0,
    usize=4096, pad=0, alignment=4096, zero=0x7f916be13387, slab=true) at
src/extent.c:554
#5  0x00007f917176b3b6 in arena_extent_cache_alloc_locked
(tsdn=0x7f916be14580, arena=0x7f8a0d652180, r_extent_hooks=0x7f916be13388,
    new_addr=0x0, usize=4096, pad=0, alignment=4096, zero=0x7f916be13387,
slab=true) at src/arena.c:63
#6  0x00007f917176dfd3 in arena_slab_alloc (tsdn=0x7f916be14580,
arena=0x7f8a0d652180, binind=1,
    bin_info=0x7f91717a2e48 <je_arena_bin_info+40>) at src/arena.c:1024
#7  0x00007f917176e196 in arena_bin_nonfull_slab_get (tsdn=0x7f916be14580,
arena=0x7f8a0d652180, bin=0x7f8a0d6550e0, binind=1)
    at src/arena.c:1067
#8  0x00007f917176e2be in arena_bin_malloc_hard (tsdn=0x7f916be14580,
arena=0x7f8a0d652180, bin=0x7f8a0d6550e0, binind=1)
    at src/arena.c:1105
#9  0x00007f917176e5a5 in je_arena_tcache_fill_small (tsdn=0x7f916be14580,
arena=0x7f8a0d652180, tbin=0x7f8a6e0d5008, binind=1,
    prof_accumbytes=0) at src/arena.c:1171
#10 0x00007f917179b014 in je_tcache_alloc_small_hard (tsdn=0x7f916be14580,
arena=0x7f8a0d652180, tcache=0x7f8a6e0d4fc0,
    tbin=0x7f8a6e0d5008, binind=1, tcache_success=0x7f916be13576) at
src/tcache.c:79
#11 0x00007f917179a425 in je_tcache_alloc_small (tsd=0x7f916be14580,
arena=0x7f8a0d652180, tcache=0x7f8a6e0d4fc0, size=16, binind=1,
    zero=false, slow_path=true) at include/jemalloc/internal/tcache.h:297
#12 0x00007f917176acd0 in je_arena_malloc (tsdn=0x7f916be14580, arena=0x0,
size=16, ind=1, zero=false, tcache=0x7f8a6e0d4fc0,
    slow_path=true) at include/jemalloc/internal/arena.h:538
#13 0x00007f9171762612 in je_iallocztm (tsdn=0x7f916be14580, size=16,
ind=1, zero=false, tcache=0x7f8a6e0d4fc0, is_metadata=false,
    arena=0x0, slow_path=true) at
include/jemalloc/internal/jemalloc_internal.h:1005
#14 0x00007f9171762706 in je_ialloc (tsd=0x7f916be14580, size=16, ind=1,
zero=false, slow_path=true)
    at include/jemalloc/internal/jemalloc_internal.h:1017
#15 0x00007f917176655e in ialloc_body (size=16, zero=false,
tsdn=0x7f916be13770, usize=0x7f916be13768, slow_path=true)
    at src/jemalloc.c:1434
---Type <return> to continue, or q <return> to quit---
#16 0x00007f91717667b8 in malloc (size=16) at src/jemalloc.c:1476

Regards,
Asmita
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://jemalloc.net/mailman/jemalloc-discuss/attachments/20160914/81c85dba/attachment.html>