Interesting interaction between debug hooks, the garbage collector, and "open" VM instructions

[Peter Cawley <lua@...>]

In extremely convoluted situations, if a debug hook is called
immediately before an open VM instruction, and then proceeds to do a
garbage collection, then the behaviour of the program can change. This
is shown in the example code below, which will print "fail" if the
debug hook is set, and "pass" if it isn't:

extern "C" {
// Standard Lua includes
#include <lua.h>
#include <lauxlib.h>
// Internal Lua includes
#include <lopcodes.h>
#include <lstate.h>
// For memcpy
#include <memory.h>
}

// Simple Lua debug hook which just does a garbage collection
void hook_gc(lua_State *L, lua_Debug *ar)
{
  lua_gc(L, LUA_GCCOLLECT, 0);
}

// Simple (and insecure) implementation of a lua_Write-r
struct simple_dump_ud
{
  char buffer[1024];
  size_t length;
};

int simple_writer (lua_State *L, const void* p, size_t sz, simple_dump_ud* ud)
{
  memcpy(ud->buffer + ud->length, p, sz);
  ud->length += sz;
  return 0;
}

// Actual program
int main()
{
  lua_State *L;
  L = luaL_newstate();

  // With this line commented out, the program prints "pass". However, with
  // it left uncommented, it prints "fail".
  lua_sethook(L, hook_gc, LUA_MASKCOUNT, 1);

  // Start with some standard Lua code
  luaL_loadstring(L, "local a, b, c, d, e, f, g; g = 'pass'; a =
{...}; return g");

  // Change a the {...} construct to use the un-used b through to f stack slots.
  // The Lua internals were included to allow this readable and portable method
  // for modifying the opcodes of a Lua function.
  {
    Proto *pProto = ((LClosure*)lua_topointer(L, -1))->p;
    for(int i = 0; i < pProto->sizecode; ++i)
    {
      if(GET_OPCODE(pProto->code[i]) == OP_VARARG)
      {
        pProto->code[i    ] = CREATE_ABC(OP_VARARG , 2, 0, 0);
        pProto->code[i + 1] = CREATE_ABC(OP_SETLIST, 1, 0, 1);
        break;
      }
    }
  }

  // To show that we could have have passed a specially crafted binary
chunk to lua_load
  // rather than messing with the Lua internals, we'll dump our
contruction and re-pass
  // it through lua_load.
  {
    simple_dump_ud buffer;
    buffer.length = 0;
    lua_dump(L, (lua_Writer)simple_writer, &buffer);
    lua_settop(L, 0);
    luaL_loadbuffer(L, buffer.buffer, buffer.length, "reloaded");
  }

  // Call the function and print the result
  lua_pcall(L, 0, 1, 0);
  printf("%s\n", lua_isstring(L, -1) ? lua_tostring(L, -1) : "fail");

  // Cleanup
  lua_close(L);
}