Need help with jemalloc cores

Tue Sep 13 22:57:14 PDT 2016

Forgot to mention the version, it is jemalloc 4.2.1

On Wed, Sep 14, 2016 at 11:11 AM, Asmita Jagtap <ajopensrc at gmail.com> wrote:

> Hi,
>
> We are using jemalloc library (compiled with --enable-fill and
> --enable-debug options) in a multi-threaded daemon and seeing multiple
> cores during malloc/realloc with jemalloc library. The core files are
> approximately 61GB and 79GB in size, so this is typically seen under heavy
> memory usage situation.
>
> Are these issues in jemalloc code or the application code that is using
> jemalloc?
>
> Can someone please clarify what these stacks / aborts mean and how to deal
> with them?
>
> stack for corefile1 -
>
> (gdb) bt
> #0  0x00007fdc399155f7 in raise () from /lib64/libc.so.6
> #1  0x00007fdc39916ce8 in abort () from /lib64/libc.so.6
> #2  0x00007fdc3a317fe4 in je_extent_heap_remove (ph=0x7fd44530de90,
> phn=0x7fd0c022e0c0) at src/extent.c:191
> #3  0x00007fdc3a3180e3 in extent_heaps_remove
> (extent_heaps=0x7fd44530dda0, extent=0x7fd0c022e0c0) at src/extent.c:206
> #4  0x00007fdc3a318c3f in extent_recycle (tsdn=0x7fdc329a6580,
> arena=0x7fd44530b680, r_extent_hooks=0x7fdc329a5438,
> extent_heaps=0x7fd44530dda0,
>     cache=false, new_addr=0x7fd0bfd0b740, usize=16384, pad=0,
> alignment=64, zero=0x7fdc329a5447, commit=0x7fdc329a5437, slab=false) at
> src/extent.c:424
> #5  0x00007fdc3a3197f3 in extent_alloc_retained (tsdn=0x7fdc329a6580,
> arena=0x7fd44530b680, r_extent_hooks=0x7fdc329a5438,
> new_addr=0x7fd0bfd0b740,
>     usize=16384, pad=0, alignment=64, zero=0x7fdc329a5447,
> commit=0x7fdc329a5437, slab=false) at src/extent.c:608
> #6  0x00007fdc3a319a91 in je_extent_alloc_wrapper (tsdn=0x7fdc329a6580,
> arena=0x7fd44530b680, r_extent_hooks=0x7fdc329a5438,
> new_addr=0x7fd0bfd0b740,
>     usize=16384, pad=0, alignment=64, zero=0x7fdc329a5447,
> commit=0x7fdc329a5437, slab=false) at src/extent.c:664
> #7  0x00007fdc3a31d2c2 in large_ralloc_no_move_expand
> (tsdn=0x7fdc329a6580, extent=0x7fd3412ba6c0, usize=32768, zero=false) at
> src/large.c:149
> #8  0x00007fdc3a31d651 in je_large_ralloc_no_move (tsdn=0x7fdc329a6580,
> extent=0x7fd3412ba6c0, usize_min=32768, usize_max=32768, zero=false)
>     at src/large.c:205
> #9  0x00007fdc3a31d92f in je_large_ralloc (tsdn=0x7fdc329a6580, arena=0x0,
> extent=0x7fd3412ba6c0, usize=32768, alignment=0, zero=false,
>     tcache=0x7fd538d85780) at src/large.c:264
> #10 0x00007fdc3a305b10 in je_arena_ralloc (tsdn=0x7fdc329a6580, arena=0x0,
> extent=0x7fd3412ba6c0, ptr=0x7fd0bfd06740, oldsize=16384, size=32768,
>     alignment=0, zero=false, tcache=0x7fd538d85780) at src/arena.c:1567
> #11 0x00007fdc3a2f9124 in je_iralloct (tsdn=0x7fdc329a6580,
> extent=0x7fd3412ba6c0, ptr=0x7fd0bfd06740, oldsize=16384, size=32768,
> alignment=0,
>     zero=false, tcache=0x7fd538d85780, arena=0x0) at
> include/jemalloc/internal/jemalloc_internal.h:1163
> #12 0x00007fdc3a2f91ad in je_iralloc (tsd=0x7fdc329a6580,
> extent=0x7fd3412ba6c0, ptr=0x7fd0bfd06740, oldsize=16384, size=32768,
> alignment=0, zero=false)
>     at include/jemalloc/internal/jemalloc_internal.h:1172
> #13 0x00007fdc3a2fd883 in realloc (ptr=0x7fd0bfd06740, size=32768) at
> src/jemalloc.c:1796
>
> stack for corefile2 -
> #0  0x00007f9170d7f5f7 in raise () from /lib64/libc.so.6
> #1  0x00007f9170d80ce8 in abort () from /lib64/libc.so.6
> #2  0x00007f917176b616 in je_arena_extent_cache_maybe_remove
> (arena=0x7f8a0d652180, extent=0x7f874860e180, dirty=true) at src/arena.c:119
> #3  0x00007f9171782c5c in extent_recycle (tsdn=0x7f916be14580,
> arena=0x7f8a0d652180, r_extent_hooks=0x7f916be13388,
>     extent_heaps=0x7f8a0d654268, cache=true, new_addr=0x0, usize=4096,
> pad=0, alignment=4096, zero=0x7f916be13387, commit=0x7f916be13297,
>     slab=true) at src/extent.c:425
> #4  0x00007f91717834df in je_extent_alloc_cache (tsdn=0x7f916be14580,
> arena=0x7f8a0d652180, r_extent_hooks=0x7f916be13388, new_addr=0x0,
>     usize=4096, pad=0, alignment=4096, zero=0x7f916be13387, slab=true) at
> src/extent.c:554
> #5  0x00007f917176b3b6 in arena_extent_cache_alloc_locked
> (tsdn=0x7f916be14580, arena=0x7f8a0d652180, r_extent_hooks=0x7f916be13388,
>     new_addr=0x0, usize=4096, pad=0, alignment=4096, zero=0x7f916be13387,
> slab=true) at src/arena.c:63
> #6  0x00007f917176dfd3 in arena_slab_alloc (tsdn=0x7f916be14580,
> arena=0x7f8a0d652180, binind=1,
>     bin_info=0x7f91717a2e48 <je_arena_bin_info+40>) at src/arena.c:1024
> #7  0x00007f917176e196 in arena_bin_nonfull_slab_get (tsdn=0x7f916be14580,
> arena=0x7f8a0d652180, bin=0x7f8a0d6550e0, binind=1)
>     at src/arena.c:1067
> #8  0x00007f917176e2be in arena_bin_malloc_hard (tsdn=0x7f916be14580,
> arena=0x7f8a0d652180, bin=0x7f8a0d6550e0, binind=1)
>     at src/arena.c:1105
> #9  0x00007f917176e5a5 in je_arena_tcache_fill_small (tsdn=0x7f916be14580,
> arena=0x7f8a0d652180, tbin=0x7f8a6e0d5008, binind=1,
>     prof_accumbytes=0) at src/arena.c:1171
> #10 0x00007f917179b014 in je_tcache_alloc_small_hard (tsdn=0x7f916be14580,
> arena=0x7f8a0d652180, tcache=0x7f8a6e0d4fc0,
>     tbin=0x7f8a6e0d5008, binind=1, tcache_success=0x7f916be13576) at
> src/tcache.c:79
> #11 0x00007f917179a425 in je_tcache_alloc_small (tsd=0x7f916be14580,
> arena=0x7f8a0d652180, tcache=0x7f8a6e0d4fc0, size=16, binind=1,
>     zero=false, slow_path=true) at include/jemalloc/internal/tcache.h:297
> #12 0x00007f917176acd0 in je_arena_malloc (tsdn=0x7f916be14580, arena=0x0,
> size=16, ind=1, zero=false, tcache=0x7f8a6e0d4fc0,
>     slow_path=true) at include/jemalloc/internal/arena.h:538
> #13 0x00007f9171762612 in je_iallocztm (tsdn=0x7f916be14580, size=16,
> ind=1, zero=false, tcache=0x7f8a6e0d4fc0, is_metadata=false,
>     arena=0x0, slow_path=true) at include/jemalloc/internal/
> jemalloc_internal.h:1005
> #14 0x00007f9171762706 in je_ialloc (tsd=0x7f916be14580, size=16, ind=1,
> zero=false, slow_path=true)
>     at include/jemalloc/internal/jemalloc_internal.h:1017
> #15 0x00007f917176655e in ialloc_body (size=16, zero=false,
> tsdn=0x7f916be13770, usize=0x7f916be13768, slow_path=true)
>     at src/jemalloc.c:1434
> ---Type <return> to continue, or q <return> to quit---
> #16 0x00007f91717667b8 in malloc (size=16) at src/jemalloc.c:1476
>
>
> Regards,
> Asmita
>
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://jemalloc.net/mailman/jemalloc-discuss/attachments/20160914/abc79643/attachment-0001.html>