[Date Prev][Date Next][Thread Prev][Thread Next]
[Date Index]
[Thread Index]
- Subject: Re: PATCH: fixes bug with calling garbage collector from custom lua_Alloc
- From: "Robert G. Jakabosky" <bobby@...>
- Date: Sat, 3 May 2008 20:18:09 -0700
Updated patch attached "emergency_gc_fixes.patch".
The patch fixes some bugs where the garbage collector (GC) will free an object
that is being created and hasn't been put onto the Lua stack or added to a
table yet. Also it protects the GC from recursive calls
(allocator->GC->allocator->GC or GC->allocator->GC).
Since this patch only fixes bug related to running the garbage collector from
the allocator function, I only recommend using it if you need to restrict the
amount of memory used by a Lua script. Also there maybe more bugs that I
haven't found yet, so use at your own risk.
Also attached is a new version of lua_memlimit.c + some lua scripts for stress
testing. The new version adds a lot of checks and debug messages to the
allocator for debugging GC bugs. There are some #define's at the top to
control how many debug messages are enabled and what checks to run. Does
anyone know a good set of Lua scripts for testing all Lua features? Do the
scripts that come with lua 5.1.3 test all the language features?
Summery of bug that the patch fixes (warning long read):
Function "lua_setfield" in lapi.c
This function created a Lua string and stored it in a variable on the C
stack, had to move the string to the Lua stack so the GC wouldn't free the
string before the key/value was added to the table. This is something that
can effect other C libraries that create temporary Lua objects on the C
stack. If another allocation happens before the object is put on the Lua
stack or added to a table the GC might free the object.
Function "lua_gc" in lapi.c
Added protection from recursive LUA_GCCOLLECT calls. Also LUA_GCSTOP &
LUA_GCRESTART can be used to stop a full GC.
Function "f_parser" in ldo.c
I haven't look to deeply into the Lua parser yet, but it looks like there
are a lot of places where objects are created and not placed somewhere that
the GC can find. So for now I just stop the GC before the parser runs, then
restart it after.
Function "checkSizes" in lgc.c
This function was shrinking a temp. buffer used to concatenate strings. Now
it will not shrink the size below the in use size needed by the string
concat. function.
Function "luaE_newthread" in lstate.c
The new lua_State object wasn't fully allocated before a link to it was
added to the GC. I only had to move the call to "luaC_link" to the end of
the function.
Function "luaH_new" in ltable.c
The table object wasn't fully allocated before a link to it was added to the
GC. I only had to move the call to "luaC_link" to the end of the function.
Function "newlstr" in lstring.c
This function creates a new Lua string object and adds it to the global
string hashtable, it also calls "luaS_resize" to expand the string hashtable
when it gets too crowded. The change needed here was to delay adding the
string to the hashtable until after the hashtable was resized.
Function "luaV_concat" in lvm.c
Same as last patch, the temp. buffer has it's "in use" size updated so the
GC will not shrink the buffer below the "in use" size. The "in use" size is
reset to 0 after the buffer's contents are copied to a new Lua string.
--
Robert G. Jakabosky
--- lua-5.1.3.orig_patched/src/lapi.c 2008-05-03 17:08:11.000000000 -0700
+++ lua-5.1.3/src/lapi.c 2008-05-03 16:52:03.000000000 -0700
@@ -656,14 +656,14 @@
LUA_API void lua_setfield (lua_State *L, int idx, const char *k) {
StkId t;
- TValue key;
lua_lock(L);
api_checknelems(L, 1);
t = index2adr(L, idx);
api_checkvalidindex(L, t);
- setsvalue(L, &key, luaS_new(L, k));
- luaV_settable(L, t, &key, L->top - 1);
- L->top--; /* pop value */
+ setsvalue2s(L, L->top, luaS_new(L, k));
+ api_incr_top(L);
+ luaV_settable(L, t, L->top - 1, L->top - 2);
+ L->top -= 2; /* pop key and value */
lua_unlock(L);
}
@@ -911,7 +911,12 @@
break;
}
case LUA_GCCOLLECT: {
- luaC_fullgc(L);
+ lu_mem old_thres = g->GCthreshold;
+ if(g->GCthreshold != MAX_LUMEM) {
+ g->GCthreshold = MAX_LUMEM;
+ luaC_fullgc(L);
+ g->GCthreshold = old_thres;
+ }
break;
}
case LUA_GCCOUNT: {
--- lua-5.1.3.orig_patched/src/ldo.c 2008-01-18 14:31:22.000000000 -0800
+++ lua-5.1.3/src/ldo.c 2008-05-02 17:47:55.000000000 -0700
@@ -494,6 +494,7 @@
struct SParser *p = cast(struct SParser *, ud);
int c = luaZ_lookahead(p->z);
luaC_checkGC(L);
+ lua_gc(L, LUA_GCSTOP, 0); /* stop collector during parsing */
tf = ((c == LUA_SIGNATURE[0]) ? luaU_undump : luaY_parser)(L, p->z,
&p->buff, p->name);
cl = luaF_newLclosure(L, tf->nups, hvalue(gt(L)));
@@ -502,6 +503,7 @@
cl->l.upvals[i] = luaF_newupval(L);
setclvalue(L, L->top, cl);
incr_top(L);
+ lua_gc(L, LUA_GCRESTART, 0);
}
--- lua-5.1.3.orig_patched/src/lgc.c 2007-12-27 05:02:25.000000000 -0800
+++ lua-5.1.3/src/lgc.c 2008-05-03 17:37:44.000000000 -0700
@@ -437,7 +437,10 @@
/* check size of buffer */
if (luaZ_sizebuffer(&g->buff) > LUA_MINBUFFER*2) { /* buffer too big? */
size_t newsize = luaZ_sizebuffer(&g->buff) / 2;
- luaZ_resizebuffer(L, &g->buff, newsize);
+ /* make sure newsize is larger then the buffer's in use size. */
+ newsize = (luaZ_bufflen(&g->buff) > newsize) ? luaZ_bufflen(&g->buff) : newsize;
+ if(newsize < luaZ_sizebuffer(&g->buff))
+ luaZ_resizebuffer(L, &g->buff, newsize);
}
}
--- lua-5.1.3.orig_patched/src/lstate.c 2008-01-03 07:20:39.000000000 -0800
+++ lua-5.1.3/src/lstate.c 2008-05-03 11:32:22.000000000 -0700
@@ -118,7 +118,6 @@
lua_State *luaE_newthread (lua_State *L) {
lua_State *L1 = tostate(luaM_malloc(L, state_size(lua_State)));
- luaC_link(L, obj2gco(L1), LUA_TTHREAD);
preinit_state(L1, G(L));
stack_init(L1, L); /* init stack */
setobj2n(L, gt(L1), gt(L)); /* share table of globals */
@@ -126,6 +125,7 @@
L1->basehookcount = L->basehookcount;
L1->hook = L->hook;
resethookcount(L1);
+ luaC_link(L, obj2gco(L1), LUA_TTHREAD);
lua_assert(iswhite(obj2gco(L1)));
return L1;
}
--- lua-5.1.3.orig_patched/src/lstring.c 2007-12-27 05:02:25.000000000 -0800
+++ lua-5.1.3/src/lstring.c 2008-05-03 13:20:31.000000000 -0700
@@ -53,6 +53,9 @@
stringtable *tb;
if (l+1 > (MAX_SIZET - sizeof(TString))/sizeof(char))
luaM_toobig(L);
+ tb = &G(L)->strt;
+ if ((tb->nuse + 1) > cast(lu_int32, tb->size) && tb->size <= MAX_INT/2)
+ luaS_resize(L, tb->size*2); /* too crowded */
ts = cast(TString *, luaM_malloc(L, (l+1)*sizeof(char)+sizeof(TString)));
ts->tsv.len = l;
ts->tsv.hash = h;
@@ -61,13 +64,10 @@
ts->tsv.reserved = 0;
memcpy(ts+1, str, l*sizeof(char));
((char *)(ts+1))[l] = '\0'; /* ending 0 */
- tb = &G(L)->strt;
h = lmod(h, tb->size);
ts->tsv.next = tb->hash[h]; /* chain new entry */
tb->hash[h] = obj2gco(ts);
tb->nuse++;
- if (tb->nuse > cast(lu_int32, tb->size) && tb->size <= MAX_INT/2)
- luaS_resize(L, tb->size*2); /* too crowded */
return ts;
}
--- lua-5.1.3.orig_patched/src/ltable.c 2007-12-28 07:32:23.000000000 -0800
+++ lua-5.1.3/src/ltable.c 2008-05-02 16:50:49.000000000 -0700
@@ -357,7 +357,6 @@
Table *luaH_new (lua_State *L, int narray, int nhash) {
Table *t = luaM_new(L, Table);
- luaC_link(L, obj2gco(t), LUA_TTABLE);
t->metatable = NULL;
t->flags = cast_byte(~0);
/* temporary values (kept only if some malloc fails) */
@@ -367,6 +366,7 @@
t->node = cast(Node *, dummynode);
setarrayvector(L, t, narray);
setnodevector(L, t, nhash);
+ luaC_link(L, obj2gco(t), LUA_TTABLE);
return t;
}
--- lua-5.1.3.orig_patched/src/lvm.c 2007-12-28 07:32:23.000000000 -0800
+++ lua-5.1.3/src/lvm.c 2008-05-03 12:41:41.000000000 -0700
@@ -295,6 +295,7 @@
if (l >= MAX_SIZET - tl) luaG_runerror(L, "string length overflow");
tl += l;
}
+ G(L)->buff.n = tl;
buffer = luaZ_openspace(L, &G(L)->buff, tl);
tl = 0;
for (i=n; i>0; i--) { /* concat all strings */
@@ -303,6 +304,7 @@
tl += l;
}
setsvalue2s(L, top-n, luaS_newlstr(L, buffer, tl));
+ luaZ_resetbuffer(&G(L)->buff);
}
total -= n-1; /* got `n' strings to create 1 new */
last -= n-1;
-- test array_fill.
--
local function stats()
-- normal lua runtime doesn't have print_memstats function.
if print_memstats then
print_memstats()
end
end
local size = 100
local val = "test"
local arr
local debug = false
stats()
for x=0,10 do
arr = array_fill(val, size)
if debug then print("arr len=", #arr) end
for i=0,#arr do
local val2 = arr[i]
assert(val2 == val, string.format("expected '%s' at index %d, instead got '%s'", val, i, val2))
if debug then print(i, " = ", arr[i]) end
end
stats()
end
-- test coroutine.create()
--
local function stats()
-- normal lua runtime doesn't have print_memstats function.
if print_memstats then
print_memstats()
end
end
function foo()
print("foo", 1)
coroutine.yield();
print("foo", 2)
end
debug = true
stats()
for x=0,10 do
local co = coroutine.create(foo)
coroutine.resume(co)
stats()
coroutine.resume(co)
stats()
end
-- test array_fill.
--
local function stats()
-- normal lua runtime doesn't have print_memstats function.
collectgarbage("collect")
if print_memstats then
print_memstats()
end
end
local size = 200
local val = "test"
local hashtable
local debug = false
local key
stats()
for x=0,10 do
hashtable = {}
for i=0,size do
key = tostring(i)
hashtable[key] = val
if debug then print(key, " = ", hashtable[key]) end
end
for i=0,size do
key = tostring(i)
local val2 = hashtable[key]
assert(val2 == val,string.format("expected '%s' at key '%s', instead got '%s'",val,key,val2))
if debug then print(key, " = ", hashtable[key]) end
end
stats()
end
-- test bug: calling lua_gc(L, LUA_GCCOLLECT, 0); from lua_Alloc function.
--
local function stats()
-- normal lua runtime doesn't have print_memstats function.
collectgarbage("collect")
if print_memstats then
print_memstats()
end
end
function replace(s, index, char)
local result = ""
if index > 1 then
result = s:sub(1, index - 1)
end
result = result .. char
if index < s:len() then
result = result .. s:sub(index + 1, -1)
end
return result
end
function test(s, index, char, def)
for i=1,#s do
local c = s:sub(i,i)
local expect = def
if i == index then
expect = char
end
assert(c == expect, string.format("expected '%s' at index %d, but got '%s'",expect,i,c))
end
end
stats()
local mem_limit = 64 * 1024
local large_string = string.rep("1", mem_limit/10)
local tmp
stats()
for i=50,500,50 do
tmp = replace(large_string, i, "x")
--print("tmp len=",#tmp,", tmp=", tmp)
test(tmp, i, "x", "1")
end
stats()
large_string = nil
tmp = nil
stats()
/*
* Run Lua scripts with a memory limit.
*
* compile:
* gcc -Wall -llua -o lua_memlimit lua_memlimit.c
*
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <assert.h>
#include <lua.h>
#include <lauxlib.h>
#include <lualib.h>
/*
* DEBUG_LEVEL -
* 1 - non-debug GC allocator, minimal debug messages & no checks. See function 'script_alloc'
* it is a very basic memlimit + GC allocator.
* 5 - debug GC allocator, with some checks enabled.
* 10 - enables some more debug messages.
* 20 - enables block size counter & block header. Extra leak checking + block size validator.
* 40 - enables a lot of debug messages.
* 50 - enables all debug messages.
*
* ZERO_FREE_MEM -
* 1 - fills all freed & allocated memory with zeros.
*
* ALWAYS_GC -
* 1 - Do a full garbage collect on all allocations (where nsize > 0)
*
* ENABLE_GC -
* 1 - Enables full garbage collection when max memory limit is hit.
* 0 - No GC just returns NULL when max memory limit is hit.
*/
#define DEBUG_LEVEL 1
#define ZERO_FREE_MEM 0
#define ALWAYS_GC 0
#define ENABLE_GC 1
#define MAX_BLOCK_SIZE 128 * 1024
#if DEBUG_LEVEL >= 20
#define BLOCK_HEADER (sizeof(size_t)*2 + 16)
#else
#define BLOCK_HEADER (0)
#endif
typedef struct {
char *name;
lua_State *L;
size_t memused;
size_t peak_memused;
size_t max_memused;
#if DEBUG_LEVEL >= 20
void *gc_ptr;
int block_count[MAX_BLOCK_SIZE + 1];
size_t max_blocksize;
#endif
} script_info_t;
#if DEBUG_LEVEL >= 20
static void script_check_blocks(script_info_t *info)
{
int i = 0;
int c = 0;
size_t tot = 0;
for(i = 1; i < MAX_BLOCK_SIZE && i <= info->max_blocksize; i++) {
c = info->block_count[i];
if(c > 0) {
#if DEBUG_LEVEL >= 50
printf("block size=%d, count=%d\n", i, c);
#endif
tot += i * c;
}
}
c = info->block_count[MAX_BLOCK_SIZE];
if(c > 0) {
#if DEBUG_LEVEL >= 50
printf("block size >= %d, count=%d\n", MAX_BLOCK_SIZE, c);
#endif
tot += i * c;
}
#if DEBUG_LEVEL >= 40
printf("mem_total=%zd, max_blocksize=%zd\n", tot, info->max_blocksize);
#endif
assert(tot == info->memused);
}
static void script_update_block_counts(script_info_t *info, size_t osize, size_t nsize)
{
if(osize > 0) {
if(osize < MAX_BLOCK_SIZE) {
assert(--(info->block_count[osize]) >= 0);
} else {
assert(--(info->block_count[MAX_BLOCK_SIZE]) >= 0);
}
}
if(nsize > 0) {
if(nsize < MAX_BLOCK_SIZE) {
assert(++(info->block_count[nsize]) >= 0);
} else {
assert(++(info->block_count[MAX_BLOCK_SIZE]) >= 0);
}
}
}
#endif
static void *script_debug_alloc(void *ud, void *ptr, size_t osize, size_t nsize)
{
script_info_t *info=(script_info_t *)ud;
size_t old_size = info->memused;
int run_gc = 0;
#if DEBUG_LEVEL >= 20
size_t bsize = -1;
#endif
#if DEBUG_LEVEL >= 40
if(osize > info->memused) {
printf("osize(%zd) > memused(%zd)\n", osize, info->memused);
}
#endif
assert(osize <= info->memused);
#if DEBUG_LEVEL >= 20
if(ptr) {
ptr -= BLOCK_HEADER;
bsize = *((size_t *)ptr);
assert(bsize == osize);
assert(info->gc_ptr != ptr); /* check for second alloc call on same ptr. */
}
#endif
info->memused -= osize;
if (nsize == 0) {
#if DEBUG_LEVEL >= 40
if(ptr) printf("free 1 (%p), osize=%zd, bsize=%zd\n", ptr, osize, bsize);
#endif
#if DEBUG_LEVEL >= 20
script_update_block_counts(info, osize, 0);
script_check_blocks(info);
#endif
#if ZERO_FREE_MEM
if(osize > 0) {
#if DEBUG_LEVEL >= 45
printf("1 memset(%p,0,%zd)\n", ptr, osize + BLOCK_HEADER);
#endif
memset(ptr, 0, osize + BLOCK_HEADER);
}
#endif
free(ptr);
return NULL;
}
info->memused += nsize;
if(nsize > osize && info->memused >= info->max_memused) {
#if ENABLE_GC
run_gc = 1;
#if ALWAYS_GC
} else if(info->L != NULL) {
run_gc = 1;
#endif
}
if(run_gc) {
#if DEBUG_LEVEL >= 40
printf("LOW MEM: 1 osize=%zd, nsize=%zd, used=%zu, peak=%zu, need=%zd\n", osize, nsize,
info->memused, info->peak_memused, (info->memused - info->max_memused));
#endif
info->memused = old_size;
#if DEBUG_LEVEL >= 20
info->gc_ptr = ptr;
#endif
/* try to free memory by collecting garbage. */
lua_gc(info->L, LUA_GCCOLLECT, 0);
#if DEBUG_LEVEL >= 20
info->gc_ptr = NULL;
#endif
#if DEBUG_LEVEL >= 40
printf("LOW MEM: 2 used=%zu, peak=%zu\n", info->memused, info->peak_memused);
#endif
/* check memory usage again. */
old_size = info->memused;
info->memused -= osize;
info->memused += nsize;
if(info->memused >= info->max_memused) {
info->memused = old_size;
printf("OUT OF MEMORY: memused=%zd, osize=%zd, nsize=%zd\n", info->memused, osize, nsize);
return NULL;
}
#else
info->memused = old_size;
return NULL;
#endif
}
if(info->memused > info->peak_memused) info->peak_memused = info->memused;
#if ZERO_FREE_MEM
if(osize > nsize) {
#if DEBUG_LEVEL >= 45
printf("2 memset(%p,0,%zd)\n", ptr + nsize + BLOCK_HEADER, osize - nsize);
#endif
memset(ptr + nsize + BLOCK_HEADER, 0, osize - nsize);
}
#endif
#if DEBUG_LEVEL >= 40
if(ptr) printf("free 2 (%p), osize=%zd, bsize=%zd\n", ptr, osize, bsize);
#endif
#if DEBUG_LEVEL >= 20
if(nsize > info->max_blocksize) info->max_blocksize = nsize;
script_update_block_counts(info, osize, nsize);
script_check_blocks(info);
#endif
ptr = realloc(ptr, nsize + BLOCK_HEADER);
#if ZERO_FREE_MEM
if(osize < nsize) {
#if DEBUG_LEVEL >= 45
printf("3 memset(%p,0,%zd)\n", ptr + osize + BLOCK_HEADER, nsize - osize);
#endif
memset(ptr + osize + BLOCK_HEADER, 0, nsize - osize);
}
#endif
#if DEBUG_LEVEL >= 40
if(ptr) printf("alloc (%p), nsize=%zd\n", ptr, nsize);
#endif
#if DEBUG_LEVEL >= 20
*((size_t *)ptr) = nsize;
ptr += BLOCK_HEADER;
#endif
return ptr;
}
static void *script_alloc(void *ud, void *ptr, size_t osize, size_t nsize)
{
script_info_t *info=(script_info_t *)ud;
size_t old_size = info->memused;
info->memused -= osize;
if (nsize == 0) {
free(ptr);
return NULL;
}
info->memused += nsize;
if(nsize > osize && info->memused >= info->max_memused) {
#if ENABLE_GC
info->memused = old_size;
/* try to free memory by collecting garbage. */
lua_gc(info->L, LUA_GCCOLLECT, 0);
/* check memory usage again. */
old_size = info->memused;
info->memused -= osize;
info->memused += nsize;
if(info->memused >= info->max_memused) {
info->memused = old_size;
#if DEBUG_LEVEL >= 1
printf("OUT OF MEMORY: memused=%zd, osize=%zd, nsize=%zd\n", info->memused, osize, nsize);
#endif
return NULL;
}
#else
info->memused = old_size;
return NULL;
#endif
}
if(info->memused > info->peak_memused) info->peak_memused = info->memused;
return realloc(ptr, nsize);
}
static int print_memstats(lua_State *L)
{
script_info_t *info;
lua_getallocf(L, (void *)(&info));
#if DEBUG_LEVEL >= 10
printf("%s: before GC memused=%zd, peak_memused=%zd\n", info->name,
info->memused, info->peak_memused);
lua_gc(L, LUA_GCCOLLECT, 0);
printf("%s: after GC memused=%zd, peak_memused=%zd\n", info->name,
info->memused, info->peak_memused);
#else
printf("%s: memused=%zd, peak_memused=%zd\n", info->name,
info->memused, info->peak_memused);
#endif
return 0;
}
static int array_fill(lua_State *L)
{
size_t l;
const char *s = luaL_checklstring(L, 1, &l);
int n = luaL_checkint(L, 2);
int i;
lua_createtable(L, n, 0);
for(i=0; i < n; i++) {
lua_pushstring(L, s);
lua_rawseti(L, -2, i);
}
return 1;
}
static lua_State *create_newstate(script_info_t *info)
{
lua_State *L;
#if DEBUG_LEVEL >= 5
L = lua_newstate(script_debug_alloc, info);
#else
L = lua_newstate(script_alloc, info);
#endif
if(L == NULL) return L;
/* load libs */
info->L = L;
luaL_openlibs(L);
lua_register(L,"print_memstats", print_memstats);
lua_register(L,"array_fill", array_fill);
return L;
}
static int run_script(char *script_name)
{
script_info_t *info;
lua_State *L;
int status;
/* run script */
info = (script_info_t *)calloc(1, sizeof(script_info_t));
info->max_memused = 64 * 1024;
info->name = script_name;
info->memused = 0;
info->peak_memused = 0;
/* create lua state & load script file. */
L=create_newstate(info);
status = luaL_loadfile(L, info->name);
if(status != 0) {
fprintf(stderr,"Failed to load script: %s\n",lua_tostring(L,-1));
lua_close(L);
free(info);
return -1;
}
/* execute script. */
status = lua_pcall(L, 0, LUA_MULTRET, 0);
if(status != 0) {
fprintf(stderr,"%s: %s\n", script_name,lua_tostring(L,-1));
}
lua_close(L);
printf("%s: memused=%zd, peak_memused=%zd\n", info->name,
info->memused, info->peak_memused);
/* check for memory leak. */
assert(info->memused == 0);
free(info);
return 0;
}
int main(int argc, char *argv[])
{
int rc = 0;
int i = 0;
if(argc < 2) {
printf("usage: %s <lua script 1> [<lua script 2>]\n", argv[0]);
exit(1);
}
/* run scripts. */
for(i=1; i < argc && rc == 0; i++) {
rc = run_script(argv[i]);
}
return 0;
}
