[Date Prev][Date Next][Thread Prev][Thread Next]
[Date Index]
[Thread Index]
- Subject: Re: PATCH: fixes bug with calling garbage collector from custom lua_Alloc
- From: "Robert G. Jakabosky" <bobby@...>
- Date: Fri, 9 May 2008 19:04:05 -0700
Updated patch attached 'emergency_gc-5.1.3.patch'. Fixed the infinite loop
bug and added memory limit support to the Lua core.
This new patch removes the need for a custom allocator like the one used in my
lua_memlimit program.
Additions to C API:
/* get current memlimit in Kbytes */
int memlimit = lua_gc(L, LUA_GCGETMEMLIMIT, 0);
/* set memlimit to 64Kbytes */
lua_gc(L, LUA_GCSETMEMLIMIT, 64);
Additions to Lua API:
-- get current memlimit in Kbytes
memlimit = collectgarbage("getmemlimit");
-- set memlimt to 64Kbytes
collectgarbage("setmemlimit", 64);
If memlimit > 0, then the Lua core will do a step collection when the script
tries to allocate more memory then the limit allows. If the step collection
finishes a full collection cycle and still can't free enough memory a "not
enough memory" error will be thrown.
If the allocator fails (returning NULL when nsize > 0) a full garbage
collection cycle will be do and then the allocator will be tried one last
time before throwing the "not enought memory" error. Right now this can't be
disabled by setting "memlimit" to 0.
Trying to set memlimit to a value less then the currently allocated byte count
will cause a full garbage collection. if the allocated byte count is still
larger then memlimit, then then new memlimit will be the allocated byte count
rounded up to the next multiple of 1024.
Also I added a '-m limit' option to the command line interpreter.
To limit life.lua to 64Kbytes run this command:
lua -m 64 life.lua
On 64bit builds of Lua the life.lua script will atleast 85Kbytes.
Any comments or suggestions about these changes are welcome.
--
Robert G. Jakabosky
diff --git a/src/lapi.c b/src/lapi.c
index 9a39513..fe20f44 100644
--- a/src/lapi.c
+++ b/src/lapi.c
@@ -656,14 +656,14 @@ LUA_API void lua_settable (lua_State *L, int idx) {
LUA_API void lua_setfield (lua_State *L, int idx, const char *k) {
StkId t;
- TValue key;
lua_lock(L);
api_checknelems(L, 1);
t = index2adr(L, idx);
api_checkvalidindex(L, t);
- setsvalue(L, &key, luaS_new(L, k));
- luaV_settable(L, t, &key, L->top - 1);
- L->top--; /* pop value */
+ setsvalue2s(L, L->top, luaS_new(L, k));
+ api_incr_top(L);
+ luaV_settable(L, t, L->top - 1, L->top - 2);
+ L->top -= 2; /* pop key and value */
lua_unlock(L);
}
@@ -903,11 +903,11 @@ LUA_API int lua_gc (lua_State *L, int what, int data) {
g = G(L);
switch (what) {
case LUA_GCSTOP: {
- g->GCthreshold = MAX_LUMEM;
+ set_block_gc(L);
break;
}
case LUA_GCRESTART: {
- g->GCthreshold = g->totalbytes;
+ unset_block_gc(L);
break;
}
case LUA_GCCOLLECT: {
@@ -924,6 +924,10 @@ LUA_API int lua_gc (lua_State *L, int what, int data) {
break;
}
case LUA_GCSTEP: {
+ if(is_block_gc(L)) {
+ res = 1; /* gc is block so we need to pretend that the collection cycle finished. */
+ break;
+ }
lu_mem a = (cast(lu_mem, data) << 10);
if (a <= g->totalbytes)
g->GCthreshold = g->totalbytes - a;
@@ -945,6 +949,24 @@ LUA_API int lua_gc (lua_State *L, int what, int data) {
g->gcstepmul = data;
break;
}
+ case LUA_GCSETMEMLIMIT: {
+ /* GC values are expressed in Kbytes: #bytes/2^10 */
+ lu_mem new_memlimit = (cast(lu_mem, data) << 10);
+ if(new_memlimit > 0 && new_memlimit < g->totalbytes) {
+ /* run a full GC to make totalbytes < the new limit. */
+ luaC_fullgc(L);
+ if(new_memlimit < g->totalbytes)
+ new_memlimit = (g->totalbytes + 1024) & ~(1024-1); /* round up to next multiple of 1024 */
+ }
+ g->memlimit = new_memlimit;
+ /* new memlimit might be > then requested memlimit. */
+ res = cast_int(new_memlimit >> 10);
+ break;
+ }
+ case LUA_GCGETMEMLIMIT: {
+ res = cast_int(g->memlimit >> 10);
+ break;
+ }
default: res = -1; /* invalid option */
}
lua_unlock(L);
diff --git a/src/lbaselib.c b/src/lbaselib.c
index fcb2da6..6b137e9 100644
--- a/src/lbaselib.c
+++ b/src/lbaselib.c
@@ -192,9 +192,10 @@ static int luaB_gcinfo (lua_State *L) {
static int luaB_collectgarbage (lua_State *L) {
static const char *const opts[] = {"stop", "restart", "collect",
- "count", "step", "setpause", "setstepmul", NULL};
+ "count", "step", "setpause", "setstepmul","setmemlimit","getmemlimit", NULL};
static const int optsnum[] = {LUA_GCSTOP, LUA_GCRESTART, LUA_GCCOLLECT,
- LUA_GCCOUNT, LUA_GCSTEP, LUA_GCSETPAUSE, LUA_GCSETSTEPMUL};
+ LUA_GCCOUNT, LUA_GCSTEP, LUA_GCSETPAUSE, LUA_GCSETSTEPMUL,
+ LUA_GCSETMEMLIMIT,LUA_GCGETMEMLIMIT};
int o = luaL_checkoption(L, 1, "collect", opts);
int ex = luaL_optint(L, 2, 0);
int res = lua_gc(L, optsnum[o], ex);
diff --git a/src/ldo.c b/src/ldo.c
index 8de05f7..4c32b93 100644
--- a/src/ldo.c
+++ b/src/ldo.c
@@ -494,6 +494,7 @@ static void f_parser (lua_State *L, void *ud) {
struct SParser *p = cast(struct SParser *, ud);
int c = luaZ_lookahead(p->z);
luaC_checkGC(L);
+ set_block_gc(L); /* stop collector during parsing */
tf = ((c == LUA_SIGNATURE[0]) ? luaU_undump : luaY_parser)(L, p->z,
&p->buff, p->name);
cl = luaF_newLclosure(L, tf->nups, hvalue(gt(L)));
@@ -502,6 +503,7 @@ static void f_parser (lua_State *L, void *ud) {
cl->l.upvals[i] = luaF_newupval(L);
setclvalue(L, L->top, cl);
incr_top(L);
+ unset_block_gc(L);
}
diff --git a/src/lfunc.c b/src/lfunc.c
index 813e88f..d2ce63d 100644
--- a/src/lfunc.c
+++ b/src/lfunc.c
@@ -66,7 +66,6 @@ UpVal *luaF_findupval (lua_State *L, StkId level) {
}
uv = luaM_new(L, UpVal); /* not found: create a new one */
uv->tt = LUA_TUPVAL;
- uv->marked = luaC_white(g);
uv->v = level; /* current value lives in the stack */
uv->next = *pp; /* chain it in the proper position */
*pp = obj2gco(uv);
@@ -74,6 +73,7 @@ UpVal *luaF_findupval (lua_State *L, StkId level) {
uv->u.l.next = g->uvhead.u.l.next;
uv->u.l.next->u.l.prev = uv;
g->uvhead.u.l.next = uv;
+ luaC_marknew(L, obj2gco(uv));
lua_assert(uv->u.l.next->u.l.prev == uv && uv->u.l.prev->u.l.next == uv);
return uv;
}
diff --git a/src/lgc.c b/src/lgc.c
index d9e0b78..5ffc51b 100644
--- a/src/lgc.c
+++ b/src/lgc.c
@@ -232,8 +232,10 @@ static void traverseclosure (global_State *g, Closure *cl) {
int i;
lua_assert(cl->l.nupvalues == cl->l.p->nups);
markobject(g, cl->l.p);
- for (i=0; i<cl->l.nupvalues; i++) /* mark its upvalues */
- markobject(g, cl->l.upvals[i]);
+ for (i=0; i<cl->l.nupvalues; i++) { /* mark its upvalues */
+ if(cl->l.upvals[i])
+ markobject(g, cl->l.upvals[i]);
+ }
}
}
@@ -258,6 +260,7 @@ static void traversestack (global_State *g, lua_State *l) {
CallInfo *ci;
markvalue(g, gt(l));
lim = l->top;
+ if(l->stack == NULL) return; /* no stack to traverse */
for (ci = l->base_ci; ci <= l->ci; ci++) {
lua_assert(ci->top <= l->stack_last);
if (lim < ci->top) lim = ci->top;
@@ -419,8 +422,6 @@ static GCObject **sweeplist (lua_State *L, GCObject **p, lu_mem count) {
else { /* must erase `curr' */
lua_assert(isdead(g, curr) || deadmask == bitmask(SFIXEDBIT));
*p = curr->gch.next;
- if (curr == g->rootgc) /* is the first element of the list? */
- g->rootgc = curr->gch.next; /* adjust first */
freeobj(L, curr);
}
}
@@ -437,7 +438,10 @@ static void checkSizes (lua_State *L) {
/* check size of buffer */
if (luaZ_sizebuffer(&g->buff) > LUA_MINBUFFER*2) { /* buffer too big? */
size_t newsize = luaZ_sizebuffer(&g->buff) / 2;
- luaZ_resizebuffer(L, &g->buff, newsize);
+ /* make sure newsize is larger then the buffer's in use size. */
+ newsize = (luaZ_bufflen(&g->buff) > newsize) ? luaZ_bufflen(&g->buff) : newsize;
+ if(newsize < luaZ_sizebuffer(&g->buff))
+ luaZ_resizebuffer(L, &g->buff, newsize);
}
}
@@ -609,10 +613,14 @@ static l_mem singlestep (lua_State *L) {
void luaC_step (lua_State *L) {
global_State *g = G(L);
+ if(is_block_gc(L)) return;
+ set_block_gc(L);
l_mem lim = (GCSTEPSIZE/100) * g->gcstepmul;
if (lim == 0)
lim = (MAX_LUMEM-1)/2; /* no limit */
g->gcdept += g->totalbytes - g->GCthreshold;
+ if (g->estimate > g->totalbytes)
+ g->estimate = g->totalbytes;
do {
lim -= singlestep(L);
if (g->gcstate == GCSpause)
@@ -630,11 +638,14 @@ void luaC_step (lua_State *L) {
lua_assert(g->totalbytes >= g->estimate);
setthreshold(g);
}
+ unset_block_gc(L);
}
void luaC_fullgc (lua_State *L) {
global_State *g = G(L);
+ if(is_block_gc(L)) return;
+ set_block_gc(L);
if (g->gcstate <= GCSpropagate) {
/* reset sweep marks to sweep all elements (returning them to white) */
g->sweepstrgc = 0;
@@ -656,6 +667,7 @@ void luaC_fullgc (lua_State *L) {
singlestep(L);
}
setthreshold(g);
+ unset_block_gc(L);
}
@@ -683,6 +695,14 @@ void luaC_barrierback (lua_State *L, Table *t) {
}
+void luaC_marknew (lua_State *L, GCObject *o) {
+ global_State *g = G(L);
+ o->gch.marked = luaC_white(g);
+ if (g->gcstate == GCSpropagate)
+ reallymarkobject(g, o); /* mark new objects as gray during propagate state. */
+}
+
+
void luaC_link (lua_State *L, GCObject *o, lu_byte tt) {
global_State *g = G(L);
o->gch.next = g->rootgc;
diff --git a/src/lgc.h b/src/lgc.h
index 5a8dc60..a623703 100644
--- a/src/lgc.h
+++ b/src/lgc.h
@@ -37,6 +37,18 @@
#define test2bits(x,b1,b2) testbits(x, (bit2mask(b1, b2)))
+/*
+** Possible Garbage Collector flags.
+** Layout for bit use in 'gsflags' field in global_State structure.
+** bit 0 - Protect GC from recursive calls.
+*/
+#define GCFlagsNone 0
+#define GCBlockGCBit 0
+
+
+#define is_block_gc(L) testbit(G(L)->gcflags, GCBlockGCBit)
+#define set_block_gc(L) l_setbit(G(L)->gcflags, GCBlockGCBit)
+#define unset_block_gc(L) resetbit(G(L)->gcflags, GCBlockGCBit)
/*
** Layout for bit use in `marked' field:
@@ -101,6 +113,7 @@ LUAI_FUNC void luaC_callGCTM (lua_State *L);
LUAI_FUNC void luaC_freeall (lua_State *L);
LUAI_FUNC void luaC_step (lua_State *L);
LUAI_FUNC void luaC_fullgc (lua_State *L);
+LUAI_FUNC void luaC_marknew (lua_State *L, GCObject *o);
LUAI_FUNC void luaC_link (lua_State *L, GCObject *o, lu_byte tt);
LUAI_FUNC void luaC_linkupval (lua_State *L, UpVal *uv);
LUAI_FUNC void luaC_barrierf (lua_State *L, GCObject *o, GCObject *v);
diff --git a/src/lmem.c b/src/lmem.c
index ae7d8c9..72abf72 100644
--- a/src/lmem.c
+++ b/src/lmem.c
@@ -14,6 +14,7 @@
#include "ldebug.h"
#include "ldo.h"
+#include "lgc.h"
#include "lmem.h"
#include "lobject.h"
#include "lstate.h"
@@ -69,6 +70,21 @@ void *luaM_toobig (lua_State *L) {
}
+void luaM_check_memlimit(lua_State *L, size_t needbytes) {
+ global_State *g = G(L);
+ int cycle_count = 0;
+ lu_mem limit = g->memlimit - needbytes;
+ /* make sure the GC is not disabled. */
+ if (!is_block_gc(L)) {
+ while (g->totalbytes >= limit) {
+ /* only allow the GC to finished atleast 1 full cycle. */
+ if (g->gcstate == GCSpause && ++cycle_count > 1) break;
+ luaC_step(L);
+ }
+ }
+ if (g->totalbytes >= limit)
+ luaD_throw(L, LUA_ERRMEM);
+}
/*
** generic allocation routine.
@@ -76,9 +92,15 @@ void *luaM_toobig (lua_State *L) {
void *luaM_realloc_ (lua_State *L, void *block, size_t osize, size_t nsize) {
global_State *g = G(L);
lua_assert((osize == 0) == (block == NULL));
+ if (g->memlimit > 0 && nsize > osize)
+ luaM_check_memlimit(L, nsize - osize);
block = (*g->frealloc)(g->ud, block, osize, nsize);
- if (block == NULL && nsize > 0)
- luaD_throw(L, LUA_ERRMEM);
+ if (block == NULL && nsize > 0) {
+ luaC_fullgc(L); /* emergency full collection. */
+ block = (*g->frealloc)(g->ud, block, osize, nsize); /* retry allocation. */
+ if (block == NULL)
+ luaD_throw(L, LUA_ERRMEM); /* allocation still failed. */
+ }
lua_assert((nsize == 0) == (block == NULL));
g->totalbytes = (g->totalbytes - osize) + nsize;
return block;
diff --git a/src/lstate.c b/src/lstate.c
index 4313b83..64dd71f 100644
--- a/src/lstate.c
+++ b/src/lstate.c
@@ -119,6 +119,8 @@ static void close_state (lua_State *L) {
lua_State *luaE_newthread (lua_State *L) {
lua_State *L1 = tostate(luaM_malloc(L, state_size(lua_State)));
luaC_link(L, obj2gco(L1), LUA_TTHREAD);
+ setthvalue(L, L->top, L1); /* put thread on stack */
+ incr_top(L);
preinit_state(L1, G(L));
stack_init(L1, L); /* init stack */
setobj2n(L, gt(L1), gt(L)); /* share table of globals */
@@ -126,7 +128,8 @@ lua_State *luaE_newthread (lua_State *L) {
L1->basehookcount = L->basehookcount;
L1->hook = L->hook;
resethookcount(L1);
- lua_assert(iswhite(obj2gco(L1)));
+ lua_assert(!isdead(G(L), obj2gco(L1)));
+ L->top--; /* remove thread from stack */
return L1;
}
@@ -160,6 +163,7 @@ LUA_API lua_State *lua_newstate (lua_Alloc f, void *ud) {
g->uvhead.u.l.prev = &g->uvhead;
g->uvhead.u.l.next = &g->uvhead;
g->GCthreshold = 0; /* mark it as unfinished state */
+ g->estimate = 0;
g->strt.size = 0;
g->strt.nuse = 0;
g->strt.hash = NULL;
@@ -167,6 +171,7 @@ LUA_API lua_State *lua_newstate (lua_Alloc f, void *ud) {
luaZ_initbuffer(L, &g->buff);
g->panic = NULL;
g->gcstate = GCSpause;
+ g->gcflags = GCFlagsNone;
g->rootgc = obj2gco(L);
g->sweepstrgc = 0;
g->sweepgc = &g->rootgc;
@@ -175,6 +180,7 @@ LUA_API lua_State *lua_newstate (lua_Alloc f, void *ud) {
g->weak = NULL;
g->tmudata = NULL;
g->totalbytes = sizeof(LG);
+ g->memlimit = 0;
g->gcpause = LUAI_GCPAUSE;
g->gcstepmul = LUAI_GCMUL;
g->gcdept = 0;
diff --git a/src/lstate.h b/src/lstate.h
index 3bc575b..93fc594 100644
--- a/src/lstate.h
+++ b/src/lstate.h
@@ -71,6 +71,7 @@ typedef struct global_State {
void *ud; /* auxiliary data to `frealloc' */
lu_byte currentwhite;
lu_byte gcstate; /* state of garbage collector */
+ lu_byte gcflags; /* flags for the garbage collector */
int sweepstrgc; /* position of sweep in `strt' */
GCObject *rootgc; /* list of all collectable objects */
GCObject **sweepgc; /* position of sweep in `rootgc' */
@@ -81,6 +82,7 @@ typedef struct global_State {
Mbuffer buff; /* temporary buffer for string concatentation */
lu_mem GCthreshold;
lu_mem totalbytes; /* number of bytes currently allocated */
+ lu_mem memlimit; /* maximum number of bytes that can be allocated, 0 = no limit. */
lu_mem estimate; /* an estimate of number of bytes actually in use */
lu_mem gcdept; /* how much GC is `behind schedule' */
int gcpause; /* size of pause between successive GCs */
diff --git a/src/lstring.c b/src/lstring.c
index 4911315..a84cfab 100644
--- a/src/lstring.c
+++ b/src/lstring.c
@@ -53,6 +53,9 @@ static TString *newlstr (lua_State *L, const char *str, size_t l,
stringtable *tb;
if (l+1 > (MAX_SIZET - sizeof(TString))/sizeof(char))
luaM_toobig(L);
+ tb = &G(L)->strt;
+ if ((tb->nuse + 1) > cast(lu_int32, tb->size) && tb->size <= MAX_INT/2)
+ luaS_resize(L, tb->size*2); /* too crowded */
ts = cast(TString *, luaM_malloc(L, (l+1)*sizeof(char)+sizeof(TString)));
ts->tsv.len = l;
ts->tsv.hash = h;
@@ -61,13 +64,10 @@ static TString *newlstr (lua_State *L, const char *str, size_t l,
ts->tsv.reserved = 0;
memcpy(ts+1, str, l*sizeof(char));
((char *)(ts+1))[l] = '\0'; /* ending 0 */
- tb = &G(L)->strt;
h = lmod(h, tb->size);
ts->tsv.next = tb->hash[h]; /* chain new entry */
tb->hash[h] = obj2gco(ts);
tb->nuse++;
- if (tb->nuse > cast(lu_int32, tb->size) && tb->size <= MAX_INT/2)
- luaS_resize(L, tb->size*2); /* too crowded */
return ts;
}
diff --git a/src/ltable.c b/src/ltable.c
index ec84f4f..31162fe 100644
--- a/src/ltable.c
+++ b/src/ltable.c
@@ -358,6 +358,8 @@ static void rehash (lua_State *L, Table *t, const TValue *ek) {
Table *luaH_new (lua_State *L, int narray, int nhash) {
Table *t = luaM_new(L, Table);
luaC_link(L, obj2gco(t), LUA_TTABLE);
+ sethvalue2s(L, L->top, t); /* put table on stack */
+ incr_top(L);
t->metatable = NULL;
t->flags = cast_byte(~0);
/* temporary values (kept only if some malloc fails) */
@@ -367,6 +369,7 @@ Table *luaH_new (lua_State *L, int narray, int nhash) {
t->node = cast(Node *, dummynode);
setarrayvector(L, t, narray);
setnodevector(L, t, nhash);
+ L->top--; /* remove table from stack */
return t;
}
diff --git a/src/lua.c b/src/lua.c
index 3a46609..670b1de 100644
--- a/src/lua.c
+++ b/src/lua.c
@@ -45,6 +45,7 @@ static void print_usage (void) {
"Available options are:\n"
" -e stat execute string " LUA_QL("stat") "\n"
" -l name require library " LUA_QL("name") "\n"
+ " -m limit set memory limit. (units are in Kbytes)\n"
" -i enter interactive mode after executing " LUA_QL("script") "\n"
" -v show version information\n"
" -- stop handling options\n"
@@ -278,6 +279,7 @@ static int collectargs (char **argv, int *pi, int *pv, int *pe) {
break;
case 'e':
*pe = 1; /* go through */
+ case 'm': /* go through */
case 'l':
if (argv[i][2] == '\0') {
i++;
@@ -305,6 +307,15 @@ static int runargs (lua_State *L, char **argv, int n) {
return 1;
break;
}
+ case 'm': {
+ const char *limit = argv[i] + 2;
+ int memlimit=0;
+ if (*limit == '\0') limit = argv[++i];
+ lua_assert(limit != NULL);
+ memlimit = atoi(limit);
+ lua_gc(L, LUA_GCSETMEMLIMIT, memlimit);
+ break;
+ }
case 'l': {
const char *filename = argv[i] + 2;
if (*filename == '\0') filename = argv[++i];
diff --git a/src/lua.h b/src/lua.h
index 5bc97b7..a21f56f 100644
--- a/src/lua.h
+++ b/src/lua.h
@@ -226,6 +226,8 @@ LUA_API int (lua_status) (lua_State *L);
#define LUA_GCSTEP 5
#define LUA_GCSETPAUSE 6
#define LUA_GCSETSTEPMUL 7
+#define LUA_GCSETMEMLIMIT 8
+#define LUA_GCGETMEMLIMIT 9
LUA_API int (lua_gc) (lua_State *L, int what, int data);
diff --git a/src/lvm.c b/src/lvm.c
index ee3256a..8b5085b 100644
--- a/src/lvm.c
+++ b/src/lvm.c
@@ -295,6 +295,7 @@ void luaV_concat (lua_State *L, int total, int last) {
if (l >= MAX_SIZET - tl) luaG_runerror(L, "string length overflow");
tl += l;
}
+ G(L)->buff.n = tl;
buffer = luaZ_openspace(L, &G(L)->buff, tl);
tl = 0;
for (i=n; i>0; i--) { /* concat all strings */
@@ -303,6 +304,7 @@ void luaV_concat (lua_State *L, int total, int last) {
tl += l;
}
setsvalue2s(L, top-n, luaS_newlstr(L, buffer, tl));
+ luaZ_resetbuffer(&G(L)->buff);
}
total -= n-1; /* got `n' strings to create 1 new */
last -= n-1;
@@ -723,6 +725,7 @@ void luaV_execute (lua_State *L, int nexeccalls) {
p = cl->p->p[GETARG_Bx(i)];
nup = p->nups;
ncl = luaF_newLclosure(L, nup, cl->env);
+ setclvalue(L, ra, ncl);
ncl->l.p = p;
for (j=0; j<nup; j++, pc++) {
if (GET_OPCODE(*pc) == OP_GETUPVAL)
@@ -732,7 +735,6 @@ void luaV_execute (lua_State *L, int nexeccalls) {
ncl->l.upvals[j] = luaF_findupval(L, base + GETARG_B(*pc));
}
}
- setclvalue(L, ra, ncl);
Protect(luaC_checkGC(L));
continue;
}
