Patch to use colored NaNs on LP64

[Florian Weimer <fw@...>]

Here's an experimental patch to shrink the size of a Lua value to 64
bit on LP64 architectures, by encoding non-double values as special
NaNs.  I think this was initially suggested by Mike Pall, but I'm not
sure if it's been implemented yet.

On my Core 2 Duo, "binarytrees 16" results in slightly less resident
memory usage (90m vs 103m).  Execution time is also about 10% less.  I
had hoped for more.  Moving values from the ALU to the FPU and back is
probably a bit expensive even today.  It might be possible to gain a
few more percent by careful tuning.

In theory, it should be possible to speed up arithmetic, by first
attempting the calculation on unchecked values and falling back to the
current logic if the result is a NaN.  But in my tests, I didn't see a
speedup.

I'm positively surprised that the patch is so small, and that the
external ABI is completely unchanged.

diff --git a/src/lgc.c b/src/lgc.c
index d9e0b78..b943c2c 100644
--- a/src/lgc.c
+++ b/src/lgc.c
@@ -663,7 +663,7 @@ void luaC_barrierf (lua_State *L, GCObject *o, GCObject *v) {
   global_State *g = G(L);
   lua_assert(isblack(o) && iswhite(v) && !isdead(g, v) && !isdead(g, o));
   lua_assert(g->gcstate != GCSfinalize && g->gcstate != GCSpause);
-  lua_assert(ttype(&o->gch) != LUA_TTABLE);
+  lua_assert(o->gch.tt != LUA_TTABLE);
   /* must keep invariant? */
   if (g->gcstate == GCSpropagate)
     reallymarkobject(g, v);  /* restore invariant */
diff --git a/src/lobject.c b/src/lobject.c
index 4ff5073..54339d2 100644
--- a/src/lobject.c
+++ b/src/lobject.c
@@ -24,7 +24,7 @@
 
 
 
-const TValue luaO_nilobject_ = {{NULL}, LUA_TNIL};
+const TValue luaO_nilobject_ = {ttvnil};
 
 
 /*
diff --git a/src/lobject.h b/src/lobject.h
index f1e447e..8ad9f2a 100644
--- a/src/lobject.h
+++ b/src/lobject.h
@@ -68,99 +68,131 @@ typedef union {
 ** Tagged Values
 */
 
-#define TValuefields	Value value; int tt
+#define TValuefields	unsigned long l
 
 typedef struct lua_TValue {
   TValuefields;
 } TValue;
 
+#define tttotag(v) ((((unsigned long)v) | 0xfff0UL) << 48)
+#define ttvnil tttotag(LUA_TNIL)
+#define ttvnan tttotag(LUA_TNUMBER)
+#define ttvfalse tttotag(LUA_TBOOLEAN)
+#define ttvtrue (ttvfalse | 1)
+
+#define magic_check(o, t) (((unsigned) ((o)->l >> 48)) == (0xfff0 | (t)))
+#define magic_wrap(v, t) (((unsigned long) v) | tttotag(t))
+#define magic_unwrap(o) ((o)->l & 0xFFFFFFFFFFFF)
+
+static inline unsigned long magic_d2l(double d)
+{
+  if (isnan(d)) {
+    // Canonicalize NaNs to a single value.
+    return ttvnan;
+  }
+  union {
+    double d;
+    unsigned long l;
+  } u;
+  u.d = d;
+  return u.l;
+}
+
+static inline double magic_l2d(unsigned long l)
+{
+  union {
+    double d;
+    unsigned long l;
+  } u;
+  u.l = l;
+  return u.d;
+}
+
+static inline int magic_ttisnumber(unsigned long l)
+{
+  unsigned top = l >> 48;
+  return top < 0xFFF0 || l == ttvnan;
+}
 
 /* Macros to test type */
-#define ttisnil(o)	(ttype(o) == LUA_TNIL)
-#define ttisnumber(o)	(ttype(o) == LUA_TNUMBER)
-#define ttisstring(o)	(ttype(o) == LUA_TSTRING)
-#define ttistable(o)	(ttype(o) == LUA_TTABLE)
-#define ttisfunction(o)	(ttype(o) == LUA_TFUNCTION)
-#define ttisboolean(o)	(ttype(o) == LUA_TBOOLEAN)
-#define ttisuserdata(o)	(ttype(o) == LUA_TUSERDATA)
-#define ttisthread(o)	(ttype(o) == LUA_TTHREAD)
-#define ttislightuserdata(o)	(ttype(o) == LUA_TLIGHTUSERDATA)
+#define ttisnil(o)	(magic_check(o, LUA_TNIL))
+#define ttisnumber(o)	(magic_ttisnumber((o)->l))
+#define ttisstring(o)	(magic_check(o, LUA_TSTRING))
+#define ttistable(o)	(magic_check(o, LUA_TTABLE))
+#define ttisfunction(o)	(magic_check(o, LUA_TFUNCTION))
+#define ttisboolean(o)	(magic_check(o, LUA_TBOOLEAN))
+#define ttisuserdata(o)	(magic_check(o, LUA_TUSERDATA))
+#define ttisthread(o)	(magic_check(o, LUA_TTHREAD))
+#define ttislightuserdata(o)	(magic_check(o, LUA_TLIGHTUSERDATA))
 
 /* Macros to access values */
-#define ttype(o)	((o)->tt)
-#define gcvalue(o)	check_exp(iscollectable(o), (o)->value.gc)
-#define pvalue(o)	check_exp(ttislightuserdata(o), (o)->value.p)
-#define nvalue(o)	check_exp(ttisnumber(o), (o)->value.n)
-#define rawtsvalue(o)	check_exp(ttisstring(o), &(o)->value.gc->ts)
+#define ttype(o)			     \
+  ({ unsigned i_top = (o)->l >> 48;	     \
+     i_top < 0xFFF0 ? LUA_TNUMBER : i_top & 0xF; \
+  })
+#define magic_gcvalue(o) ((GCObject *) magic_unwrap(o))
+#define gcvalue(o)	check_exp(iscollectable(o), magic_gcvalue(o))
+#define pvalue(o)	check_exp(ttislightuserdata(o), (void *)(magic_unwrap(o)))
+#define nvalue(o)	check_exp(ttisnumber(o), magic_l2d((o)->l))
+#define rawtsvalue(o)	check_exp(ttisstring(o), (TString *)magic_unwrap(o))
 #define tsvalue(o)	(&rawtsvalue(o)->tsv)
-#define rawuvalue(o)	check_exp(ttisuserdata(o), &(o)->value.gc->u)
+#define rawuvalue(o)	check_exp(ttisuserdata(o), (Udata *)magic_unwrap(o))
 #define uvalue(o)	(&rawuvalue(o)->uv)
-#define clvalue(o)	check_exp(ttisfunction(o), &(o)->value.gc->cl)
-#define hvalue(o)	check_exp(ttistable(o), &(o)->value.gc->h)
-#define bvalue(o)	check_exp(ttisboolean(o), (o)->value.b)
-#define thvalue(o)	check_exp(ttisthread(o), &(o)->value.gc->th)
+#define clvalue(o)	check_exp(ttisfunction(o), (Closure *)magic_unwrap(o))
+#define hvalue(o)	check_exp(ttistable(o), (Table *)magic_unwrap(o))
+#define bvalue(o)	check_exp(ttisboolean(o), (o)->l - ttvfalse)
+#define thvalue(o)	check_exp(ttisthread(o), (lua_State *)magic_unwrap(o))
 
-#define l_isfalse(o)	(ttisnil(o) || (ttisboolean(o) && bvalue(o) == 0))
+#define l_isfalse(o)	((o)->l == ttvnil || (o)->l == ttvfalse)
 
 /*
 ** for internal debug only
 */
 #define checkconsistency(obj) \
-  lua_assert(!iscollectable(obj) || (ttype(obj) == (obj)->value.gc->gch.tt))
+  lua_assert(!iscollectable(obj) || (ttype(obj) == magic_gcvalue(obj)->gch.tt))
 
 #define checkliveness(g,obj) \
-  lua_assert(!iscollectable(obj) || \
-  ((ttype(obj) == (obj)->value.gc->gch.tt) && !isdead(g, (obj)->value.gc)))
+  lua_assert(!iscollectable(obj) ||				\
+	     ((ttype(obj) == magic_gcvalue(obj)->gch.tt)	\
+	      && !isdead(g, magic_gcvalue(obj))))
 
 
 /* Macros to set values */
-#define setnilvalue(obj) ((obj)->tt=LUA_TNIL)
+#define setnilvalue(obj) (obj)->l = ttvnil
 
 #define setnvalue(obj,x) \
-  { TValue *i_o=(obj); i_o->value.n=(x); i_o->tt=LUA_TNUMBER; }
+  { TValue *i_o=(obj); double i_x = (x); \
+    i_o->l = magic_d2l(i_x); \
+  }
 
 #define setpvalue(obj,x) \
-  { TValue *i_o=(obj); i_o->value.p=(x); i_o->tt=LUA_TLIGHTUSERDATA; }
+  { TValue *i_o=(obj); void *i_x = x;			\
+    i_o->l = magic_wrap(i_x, LUA_TLIGHTUSERDATA);	\
+  }
 
 #define setbvalue(obj,x) \
-  { TValue *i_o=(obj); i_o->value.b=(x); i_o->tt=LUA_TBOOLEAN; }
-
-#define setsvalue(L,obj,x) \
-  { TValue *i_o=(obj); \
-    i_o->value.gc=cast(GCObject *, (x)); i_o->tt=LUA_TSTRING; \
-    checkliveness(G(L),i_o); }
-
-#define setuvalue(L,obj,x) \
-  { TValue *i_o=(obj); \
-    i_o->value.gc=cast(GCObject *, (x)); i_o->tt=LUA_TUSERDATA; \
-    checkliveness(G(L),i_o); }
-
-#define setthvalue(L,obj,x) \
-  { TValue *i_o=(obj); \
-    i_o->value.gc=cast(GCObject *, (x)); i_o->tt=LUA_TTHREAD; \
-    checkliveness(G(L),i_o); }
-
-#define setclvalue(L,obj,x) \
-  { TValue *i_o=(obj); \
-    i_o->value.gc=cast(GCObject *, (x)); i_o->tt=LUA_TFUNCTION; \
-    checkliveness(G(L),i_o); }
-
-#define sethvalue(L,obj,x) \
-  { TValue *i_o=(obj); \
-    i_o->value.gc=cast(GCObject *, (x)); i_o->tt=LUA_TTABLE; \
-    checkliveness(G(L),i_o); }
-
-#define setptvalue(L,obj,x) \
-  { TValue *i_o=(obj); \
-    i_o->value.gc=cast(GCObject *, (x)); i_o->tt=LUA_TPROTO; \
-    checkliveness(G(L),i_o); }
+  { TValue *i_o=(obj); int i_x = x; \
+    i_o->l = ttvfalse + !!i_x;		    \
+  }
 
+#define setptr_internal(L,obj,x,tv)		\
+  { TValue *i_o=(obj);				\
+    GCObject *i_x = cast(GCObject *, (x));	\
+    i_o->l = magic_wrap(i_x, tv);		\
+    checkliveness(G(L),i_o);			\
+  }
 
+#define setsvalue(L,obj,x)  setptr_internal(L,obj,x,LUA_TSTRING)
+#define setuvalue(L,obj,x)  setptr_internal(L,obj,x,LUA_TUSERDATA)
+#define setthvalue(L,obj,x) setptr_internal(L,obj,x,LUA_TTHREAD)
+#define setclvalue(L,obj,x) setptr_internal(L,obj,x,LUA_TFUNCTION)
+#define sethvalue(L,obj,x)  setptr_internal(L,obj,x,LUA_TTABLE)
+#define setptvalue(L,obj,x) setptr_internal(L,obj,x,LUA_TPROTO)
 
 
 #define setobj(L,obj1,obj2) \
   { const TValue *o2=(obj2); TValue *o1=(obj1); \
-    o1->value = o2->value; o1->tt=o2->tt; \
+    *o1 = *o2; \
     checkliveness(G(L),o1); }
 
 
@@ -183,8 +215,10 @@ typedef struct lua_TValue {
 #define setobj2n	setobj
 #define setsvalue2n	setsvalue
 
-#define setttype(obj, tt) (ttype(obj) = (tt))
-
+#define setttype(obj, ttv)	  \
+  { __typeof__(obj) iii_o = (obj);	  \
+    iii_o->l = magic_wrap(magic_unwrap(iii_o), ttv); \
+  }
 
 #define iscollectable(o)	(ttype(o) >= LUA_TSTRING)
 
diff --git a/src/ltable.c b/src/ltable.c
index ec84f4f..6b8c9a6 100644
--- a/src/ltable.c
+++ b/src/ltable.c
@@ -73,8 +73,8 @@
 #define dummynode		(&dummynode_)
 
 static const Node dummynode_ = {
-  {{NULL}, LUA_TNIL},  /* value */
-  {{{NULL}, LUA_TNIL, NULL}}  /* key */
+  {ttvnil},  /* value */
+  {{ttvnil, NULL}}  /* key */
 };
 
 
@@ -422,7 +422,7 @@ static TValue *newkey (lua_State *L, Table *t, const TValue *key) {
       mp = n;
     }
   }
-  gkey(mp)->value = key->value; gkey(mp)->tt = key->tt;
+  gkey(mp)->l = key->l;
   luaC_barriert(L, t, key);
   lua_assert(ttisnil(gval(mp)));
   return gval(mp);