commit 8f651326a5cbec5ca7cf0bad0205d4c87dc8c2d0
Author: Matt Caswell <matt@openssl.org>
Date:   Mon Feb 22 10:27:18 2016 +0000

    Fix BN_hex2bn/BN_dec2bn NULL ptr/heap corruption
    
    In the BN_hex2bn function the number of hex digits is calculated using
    an int value |i|. Later |bn_expand| is called with a value of |i * 4|.
    For large values of |i| this can result in |bn_expand| not allocating any
    memory because |i * 4| is negative. This leaves ret->d as NULL leading
    to a subsequent NULL ptr deref. For very large values of |i|, the
    calculation |i * 4| could be a positive value smaller than |i|. In this
    case memory is allocated to ret->d, but it is insufficiently sized
    leading to heap corruption. A similar issue exists in BN_dec2bn.
    
    This could have security consequences if BN_hex2bn/BN_dec2bn is ever
    called by user applications with very large untrusted hex/dec data. This is
    anticipated to be a rare occurrence.
    
    All OpenSSL internal usage of this function uses data that is not expected
    to be untrusted, e.g. config file data or application command line
    arguments. If user developed applications generate config file data based
    on untrusted data then it is possible that this could also lead to security
    consequences. This is also anticipated to be a rare.
    
    Issue reported by Guido Vranken.
    
    CVE-2016-0797
    
    Reviewed-by: Andy Polyakov <appro@openssl.org>
    (cherry picked from commit c175308407858afff3fc8c2e5e085d94d12edc7d)

diff --git a/crypto/bn/bn.h b/crypto/bn/bn.h
index 47d8c71..b39258d 100644
--- a/crypto/bn/bn.h
+++ b/crypto/bn/bn.h
@@ -125,6 +125,7 @@
 #ifndef HEADER_BN_H
 # define HEADER_BN_H
 
+# include <limits.h>
 # include <openssl/e_os2.h>
 # ifndef OPENSSL_NO_FP_API
 #  include <stdio.h>            /* FILE */
@@ -739,8 +740,17 @@ const BIGNUM *BN_get0_nist_prime_521(void);
 
 /* library internal functions */
 
-# define bn_expand(a,bits) ((((((bits+BN_BITS2-1))/BN_BITS2)) <= (a)->dmax)?\
-        (a):bn_expand2((a),(bits+BN_BITS2-1)/BN_BITS2))
+# define bn_expand(a,bits) \
+    ( \
+        bits > (INT_MAX - BN_BITS2 + 1) ? \
+            NULL \
+        : \
+            (((bits+BN_BITS2-1)/BN_BITS2) <= (a)->dmax) ? \
+                (a) \
+            : \
+                bn_expand2((a),(bits+BN_BITS2-1)/BN_BITS2) \
+    )
+
 # define bn_wexpand(a,words) (((words) <= (a)->dmax)?(a):bn_expand2((a),(words)))
 BIGNUM *bn_expand2(BIGNUM *a, int words);
 # ifndef OPENSSL_NO_DEPRECATED
diff --git a/crypto/bn/bn_print.c b/crypto/bn/bn_print.c
index ab10b95..bfa31ef 100644
--- a/crypto/bn/bn_print.c
+++ b/crypto/bn/bn_print.c
@@ -58,6 +58,7 @@
 
 #include <stdio.h>
 #include <ctype.h>
+#include <limits.h>
 #include "cryptlib.h"
 #include <openssl/buffer.h>
 #include "bn_lcl.h"
@@ -189,7 +190,11 @@ int BN_hex2bn(BIGNUM **bn, const char *a)
         a++;
     }
 
-    for (i = 0; isxdigit((unsigned char)a[i]); i++) ;
+    for (i = 0; i <= (INT_MAX/4) && isxdigit((unsigned char)a[i]); i++)
+        continue;
+
+    if (i > INT_MAX/4)
+        goto err;
 
     num = i + neg;
     if (bn == NULL)
@@ -204,7 +209,7 @@ int BN_hex2bn(BIGNUM **bn, const char *a)
         BN_zero(ret);
     }
 
-    /* i is the number of hex digests; */
+    /* i is the number of hex digits */
     if (bn_expand(ret, i * 4) == NULL)
         goto err;
 
@@ -260,7 +265,11 @@ int BN_dec2bn(BIGNUM **bn, const char *a)
         a++;
     }
 
-    for (i = 0; isdigit((unsigned char)a[i]); i++) ;
+    for (i = 0; i <= (INT_MAX/4) && isdigit((unsigned char)a[i]); i++)
+        continue;
+
+    if (i > INT_MAX/4)
+        goto err;
 
     num = i + neg;
     if (bn == NULL)
@@ -278,7 +287,7 @@ int BN_dec2bn(BIGNUM **bn, const char *a)
         BN_zero(ret);
     }
 
-    /* i is the number of digests, a bit of an over expand; */
+    /* i is the number of digits, a bit of an over expand */
     if (bn_expand(ret, i * 4) == NULL)
         goto err;