repro: unballanced luai_userstatefree call, risk of heap corruption

["Erik Cassel" <erik@...>]

I have a reproducible situation where luai_userstatefree is called on a
thread that never had luai_userstatethread called on it. This looks like a
bug in Lua, and it can cause heap corruption.

The problem occurs if you run out of memory while calling lua_newthread. If
the second allocation inside lua_newthread fails, then apparently the thread
is kept around in a non-initialized state. When you shutdown with lua_close,
then luai_userstatefree is called on this thread, and the corresponding
userstate is in an undefined state.

Below are the steps to reproduce the bug on a stock 5.1.4 source. All it
does is open a state, attempt to create a thread, and then close the state:

luaconf.h:

#include "stdio.h"
#define luai_userstateopen(L)		((void)L)
#define luai_userstateclose(L)		((void)L)
#define luai_userstatethread(L,L1)
(fprintf(stderr,"luai_userstatethread\n"))
#define luai_userstatefree(L)
(fprintf(stderr,"luai_userstatefree\n"))
#define luai_userstateresume(L,n)	((void)L)
#define luai_userstateyield(L,n)	((void)L)



Lua.c:

#define lua_c

#include "lua.h"

static int trigger = 0;

// A simple override of l_alloc
static void *allocator (void *ud, void *ptr, size_t osize, size_t nsize) {
  (void)ud;
  (void)osize;
  if (nsize == 0) {
    free(ptr);
    return NULL;
  }
  else if (nsize>osize && (--trigger==0))
  {
	  // Simulate an out-of-memory condition
	  return 0;
  }
  else
    return realloc(ptr, nsize);
}

static int pmain (lua_State *L) {

  // setting trigger to 2 will cause the 2nd allocation to fail.
  // setting trigger to 0 will cause no memory error, 
  // and you will see the correct behavior.
  trigger = 2;		
  
  lua_newthread(L);
  
  return 0;
}

int main (int argc, char **argv) {
  lua_State *L = lua_newstate(allocator, NULL);
  lua_cpcall(L, &pmain, 0);
  lua_close(L);

  fgetc(stdin);
  // At this point the console will display how many times
  // luai_userstatethread and luai_userstatefree where called
  return 0;
}



-Erik