[PATCH] Hook jemalloc into glibc's malloc

[Mike Hommey]

From: Mike Hommey <mh at glandium.org>

When jemalloc is used as a libc malloc replacement (i.e. not prefixed),
some particular setups may end up inconsistently calling malloc from
libc and free from jemalloc, or the other way around.

Glibc provides hooks to make its functions use alternative
implementations. Use them.

Original patch by: Karl Tomlinson <karlt+ at karlt.net>
---
 configure.ac                        |   10 +++++++-
 include/jemalloc/jemalloc_defs.h.in |    5 ++++
 src/jemalloc.c                      |   44 +++++++++++++++++++++++++++++++++++
 3 files changed, 58 insertions(+), 1 deletions(-)

diff --git a/configure.ac b/configure.ac
index fdbf1ba..5ce16eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -714,6 +714,12 @@ AC_CHECK_LIB([pthread], [pthread_create], [LIBS="$LIBS -lpthread"],
 
 CPPFLAGS="$CPPFLAGS -D_REENTRANT"
 
+AC_CHECK_HEADERS([dlfcn.h], has_dlfcn=yes, has_dlfcn=no)
+
+if test "x${has_dlfcn}" = "xyes" ; then
+  AC_DEFINE([JEMALLOC_DLFCN])
+fi
+
 dnl Disable lazy locking by default.
 AC_ARG_ENABLE([lazy_lock],
   [AS_HELP_STRING([--enable-lazy-lock],
@@ -727,7 +733,9 @@ fi
 [enable_lazy_lock="0"]
 )
 if test "x$enable_lazy_lock" = "x1" ; then
-  AC_CHECK_HEADERS([dlfcn.h], , [AC_MSG_ERROR([dlfcn.h is missing])])
+  if test "x${has_dlfcn}" != "xyes"; then
+    AC_MSG_ERROR([dlfcn.h is missing])
+  fi
   AC_CHECK_LIB([dl], [dlopen], [LIBS="$LIBS -ldl"],
                [AC_MSG_ERROR([libdl is missing])])
   AC_DEFINE([JEMALLOC_LAZY_LOCK], [ ])
diff --git a/include/jemalloc/jemalloc_defs.h.in b/include/jemalloc/jemalloc_defs.h.in
index 66da6f3..d1622fb 100644
--- a/include/jemalloc/jemalloc_defs.h.in
+++ b/include/jemalloc/jemalloc_defs.h.in
@@ -34,6 +34,11 @@
 #undef CPU_SPINWAIT
 
 /*
+ * Defined if the dlfcn.h header is available.
+ */
+#undef JEMALLOC_DLFCN
+
+/*
  * Defined if OSAtomic*() functions are available, as provided by Darwin, and
  * documented in the atomic(3) manual page.
  */
diff --git a/src/jemalloc.c b/src/jemalloc.c
index 81829fe..f678358 100644
--- a/src/jemalloc.c
+++ b/src/jemalloc.c
@@ -1666,3 +1666,47 @@ jemalloc_postfork(void)
 }
 
 /******************************************************************************/
+
+#ifndef JEMALLOC_PREFIX
+
+#if defined(__GLIBC__) && !defined(__UCLIBC__)
+/*
+ * glibc provides the RTLD_DEEPBIND flag for dlopen which can make it possible
+ * to inconsistently reference libc's malloc(3)-compatible functions
+ * (https://bugzilla.mozilla.org/show_bug.cgi?id=493541)
+ *
+ * These definitions interpose hooks in glibc.  The functions are actually
+ * passed an extra argument for the caller return address, which will be
+ * ignored.
+ */
+JEMALLOC_ATTR(visibility("default"))
+void (* const __free_hook)(void *ptr) = free;
+
+JEMALLOC_ATTR(visibility("default"))
+void *(* const __malloc_hook)(size_t size) = malloc;
+
+JEMALLOC_ATTR(visibility("default"))
+void *(* const __realloc_hook)(void *ptr, size_t size) = realloc;
+
+JEMALLOC_ATTR(visibility("default"))
+void *(* const __memalign_hook)(size_t alignment, size_t size) = memalign;
+
+#else
+#ifdef JEMALLOC_DLFCN
+#include <dlfcn.h>
+
+#ifdef RTLD_DEEPBIND
+/*
+ * XXX On systems that support RTLD_GROUP or DF_1_GROUP, do their
+ * implementations permit similar inconsistencies?  Should STV_SINGLETON
+ * visibility be used for interposition where available?
+ */
+#  error "Interposing malloc is unsafe on this system without libc malloc hooks."
+
+#endif
+
+#endif
+
+#endif
+
+#endif
-- 
1.7.9