bug report for Lua 5.2.4: signed integer overflow

[Josh Haberman <jhaberman@...>]

At Google we discovered a signed integer overflow bug in Lua 5.2.4. This was discovered with "clang -fsanitize=signed-integer-overflow".

A fix for the bug is:

==== /src/llimits.h ====

--- src/llimits.h 2017-04-13 11:11:17.000000000 -0700

+++ src/llimits.h 2017-06-12 09:35:15.000000000 -0700

@@ -229,9 +229,13 @@

     volatile union luai_Cast u; u.l_d = (n) + 6755399441055744.0; \

     (i) = (t)u.l_p[LUA_IEEEENDIANLOC]; }

 

-#define luai_hashnum(i,n)  \

-  { volatile union luai_Cast u; u.l_d = (n) + 1.0;  /* avoid -0 */ \

-    (i) = u.l_p[0]; (i) += u.l_p[1]; }  /* add double bits for his hash */

+#define luai_hashnum(i, n)            \

+  {                                   \

+    volatile union luai_Cast u;       \

+    u.l_d = (n) + 1.0; /* avoid -0 */ \

+    (i) = u.l_p[0];                   \

+    (i) += (lu_int32)(u.l_p[1]);      \

+  } /* add double bits for his hash */

 

 #define lua_number2int(i,n) lua_number2int32(i, n, int)

 #define lua_number2unsigned(i,n) lua_number2int32(i, n, lua_Unsigned)