[PATCH 05/11] Fix malloc_vsnprintf handling of %o, %u and %x

[Mike Hommey]

From: Mike Hommey <mh at glandium.org>

These flags take unsigned values, but they were fed with signed values
taken with va_arg, and that led to sign extension in cases where the
corresponding value has the most significant bit set.
---
 src/util.c |   20 ++++++++++++++++----
 1 file changed, 16 insertions(+), 4 deletions(-)

diff --git a/src/util.c b/src/util.c
index 2aab61f..99ae26d 100644
--- a/src/util.c
+++ b/src/util.c
@@ -320,12 +320,21 @@ malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap)
 	case '?':							\
 		val = va_arg(ap, int);					\
 		break;							\
+	case '?' | 0x80:						\
+		val = va_arg(ap, unsigned int);				\
+		break;							\
 	case 'l':							\
 		val = va_arg(ap, long);					\
 		break;							\
+	case 'l' | 0x80:						\
+		val = va_arg(ap, unsigned long);			\
+		break;							\
 	case 'q':							\
 		val = va_arg(ap, long long);				\
 		break;							\
+	case 'q' | 0x80:						\
+		val = va_arg(ap, unsigned long long);			\
+		break;							\
 	case 'j':							\
 		val = va_arg(ap, intmax_t);				\
 		break;							\
@@ -335,6 +344,9 @@ malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap)
 	case 'z':							\
 		val = va_arg(ap, ssize_t);				\
 		break;							\
+	case 'z' | 0x80:						\
+		val = va_arg(ap, size_t);				\
+		break;							\
 	case 'p': /* Synthetic; used for %p. */				\
 		val = va_arg(ap, uintptr_t);				\
 		break;							\
@@ -358,7 +370,7 @@ malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap)
 			bool plus_plus = false;
 			int prec = -1;
 			int width = -1;
-			char len = '?';
+			unsigned char len = '?';
 
 			f++;
 			if (*f == '%') {
@@ -480,7 +492,7 @@ malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap)
 				uintmax_t val JEMALLOC_CC_SILENCE_INIT(0);
 				char buf[O2S_BUFSIZE];
 
-				GET_ARG_NUMERIC(val, len);
+				GET_ARG_NUMERIC(val, len | 0x80);
 				s = o2s(val, alt_form, buf, &slen);
 				APPEND_PADDED_S(s, slen, width, left_justify);
 				f++;
@@ -489,7 +501,7 @@ malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap)
 				uintmax_t val JEMALLOC_CC_SILENCE_INIT(0);
 				char buf[U2S_BUFSIZE];
 
-				GET_ARG_NUMERIC(val, len);
+				GET_ARG_NUMERIC(val, len | 0x80);
 				s = u2s(val, 10, false, buf, &slen);
 				APPEND_PADDED_S(s, slen, width, left_justify);
 				f++;
@@ -498,7 +510,7 @@ malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap)
 				uintmax_t val JEMALLOC_CC_SILENCE_INIT(0);
 				char buf[X2S_BUFSIZE];
 
-				GET_ARG_NUMERIC(val, len);
+				GET_ARG_NUMERIC(val, len | 0x80);
 				s = x2s(val, alt_form, *f == 'X', buf, &slen);
 				APPEND_PADDED_S(s, slen, width, left_justify);
 				f++;
-- 
1.7.10