[Date Prev][Date Next][Thread Prev][Thread Next]
[Date Index]
[Thread Index]
- Subject: PATCH: fixes bug with calling garbage collector from custom lua_Alloc
- From: "Robert G. Jakabosky" <bobby@...>
- Date: Fri, 2 May 2008 01:56:13 -0700
I made a custom allocator function to limit the memory used by Lua scripts.
The allocator runs the garbage collector when the script hits it's memory
limit instead of just returning NULL. See the attached lua_memlimit.c for a
example program that uses the custom allocator.
Calling the garbage collector from the allocator exposed a bug with the temp.
buffer used for concatenating strings. If lua_gc(L, LUA_GCCOLLECT, 0) gets
called during the concatenation of two large (>1024 chars) strings, then the
temp buffer used to hold both strings will be shrunken before the contents
are copied to the new string.
The attached gc_bug.patch file fixes this bug by marking the "used" size of
the temp. buffer and not allowing the garbage collector to shrink the buffer
size below the "used" size.
This bug can be reproduced with the attached lua_memlimit.c and
str_replace.lua files. You might have to use valgrind's memcheck tool to see
the reads from freed memory. See the attached valgrind.log for the memory
errors caused by this bug. The lua_memlimit.c program has a very simple
memory leak checker that will sometimes fail showing a memory used value < 0,
because of this bug. I have only tested this on 64bit Gentoo Linux (Lua &
lua_memlimit compiled as 64bit code).
While fixing this bug I noticed that the "global_State" structure has a
member "totalbytes" that keeps track of the amount of memory allocated. It
would be easy to add another memory "maxbytes" and doing the same memory
limiting work as my custom allocator in luaM_realloc(). Later I will look
into creating a bigger patch that add the memory limiting feature to the Lua
core. Right now I am just experimenting with Lua and don't want to make
major changes to the core.
P.S. This is my first post to this mailing list. Should I upload this patch
to the lua-users wiki?
--
Robert G. Jakabosky
/*
* Run Lua scripts with a memory limit.
*
* compile:
* gcc -llua -o lua_memlimit lua_memlimit.c
*
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <assert.h>
#include <lua.h>
#include <lauxlib.h>
#include <lualib.h>
#define LOW_MEM_DEBUG 1
typedef struct {
char *name;
lua_State *L;
size_t memused;
size_t peak_memused;
size_t max_memused;
int collecting;
} script_info_t;
static void *script_alloc(void *ud, void *ptr, size_t osize, size_t nsize)
{
script_info_t *info=(script_info_t *)ud;
size_t old_size = info->memused;
info->memused -= osize;
if (nsize == 0) {
free(ptr);
return NULL;
}
info->memused += nsize;
if(nsize > osize && info->memused >= info->max_memused) {
#if LOW_MEM_DEBUG
printf("LOW MEM: 1 osize=%zd, nsize=%zd, used=%zu, peak=%zu, need=%zd\n", osize, nsize,
info->memused, info->peak_memused, (info->memused - info->max_memused));
#endif
info->memused = old_size;
/* don't allow a recursive garbage collection call. */
if(info->collecting != 0) {
return NULL;
}
info->collecting = 1;
/* try to free memory by collecting garbage. */
lua_gc(info->L, LUA_GCCOLLECT, 0);
info->collecting = 0;
#if LOW_MEM_DEBUG
printf("LOW MEM: 2 used=%zu, peak=%zu\n", info->memused, info->peak_memused);
#endif
/* check memory usage again. */
old_size = info->memused;
info->memused -= osize;
info->memused += nsize;
if(info->memused >= info->max_memused) {
info->memused = old_size;
printf("OUT OF MEMORY: memused=%zd, osize=%zd, nsize=%zd\n", info->memused, osize, nsize);
return NULL;
}
}
if(info->memused > info->peak_memused) info->peak_memused = info->memused;
return realloc(ptr, nsize);
}
static int print_memstats(lua_State *L)
{
script_info_t *info;
lua_getallocf(L, (void *)(&info));
printf("%s: before GC memused=%zd, peak_memused=%zd\n", info->name,
info->memused, info->peak_memused);
lua_gc(L, LUA_GCCOLLECT, 0);
printf("%s: after GC memused=%zd, peak_memused=%zd\n", info->name,
info->memused, info->peak_memused);
return 0;
}
static lua_State *create_newstate(script_info_t *info)
{
lua_State *L;
L = lua_newstate(script_alloc, info);
if(L == NULL) return L;
/* load libs */
luaL_openlibs(L);
lua_register(L,"print_memstats", print_memstats);
return L;
}
static int run_script(char *script_name)
{
script_info_t *info;
lua_State *L;
int status;
/* run script */
info = (script_info_t *)calloc(1, sizeof(script_info_t));
info->max_memused = 64 * 1024;
info->collecting = 0;
info->name = script_name;
info->memused = 0;
info->peak_memused = 0;
/* create lua state & load script file. */
L=create_newstate(info);
status = luaL_loadfile(L, info->name);
if(status != 0) {
fprintf(stderr,"Failed to load script: %s\n",lua_tostring(L,-1));
lua_close(L);
free(info);
return -1;
}
info->L = L;
/* execute script. */
status = lua_pcall(L, 0, LUA_MULTRET, 0);
if(status != 0) {
fprintf(stderr,"script_runner: %s\n",lua_tostring(L,-1));
}
lua_close(L);
printf("%s: memused=%zd, peak_memused=%zd\n", info->name,
info->memused, info->peak_memused);
/* check for memory leak. */
assert(info->memused == 0);
free(info);
return 0;
}
int main(int argc, char *argv[])
{
int rc = 0;
int i = 0;
if(argc < 2) {
printf("usage: %s <lua script 1> [<lua script 2>]\n", argv[0]);
exit(1);
}
/* run scripts. */
for(i=1; i < argc && rc == 0; i++) {
rc = run_script(argv[i]);
}
return 0;
}
--- lua-5.1.3.orig/src/lgc.c 2007-12-27 05:02:25.000000000 -0800
+++ lua-5.1.3/src/lgc.c 2008-05-01 23:27:08.000000000 -0700
@@ -437,7 +437,8 @@
/* check size of buffer */
if (luaZ_sizebuffer(&g->buff) > LUA_MINBUFFER*2) { /* buffer too big? */
size_t newsize = luaZ_sizebuffer(&g->buff) / 2;
- luaZ_resizebuffer(L, &g->buff, newsize);
+ if(luaZ_bufflen(&g->buff) < newsize)
+ luaZ_resizebuffer(L, &g->buff, newsize);
}
}
--- lua-5.1.3.orig/src/lvm.c 2007-12-28 07:32:23.000000000 -0800
+++ lua-5.1.3/src/lvm.c 2008-05-01 16:00:24.000000000 -0700
@@ -296,6 +296,7 @@
tl += l;
}
buffer = luaZ_openspace(L, &G(L)->buff, tl);
+ G(L)->buff.n = tl;
tl = 0;
for (i=n; i>0; i--) { /* concat all strings */
size_t l = tsvalue(top-i)->len;
@@ -303,6 +304,7 @@
tl += l;
}
setsvalue2s(L, top-n, luaS_newlstr(L, buffer, tl));
+ luaZ_resetbuffer(&G(L)->buff);
}
total -= n-1; /* got `n' strings to create 1 new */
last -= n-1;
-- test bug: calling lua_gc(L, LUA_GCCOLLECT, 0); from lua_Alloc function.
--
local function stats()
-- normal lua runtime doesn't have print_memstats function.
if print_memstats then
print_memstats()
end
end
function replace(s, index, char)
local result = ""
if index > 1 then
result = s:sub(1, index - 1)
end
result = result .. char
if index < s:len() then
result = result .. s:sub(index + 1, -1)
end
return result
end
stats()
local mem_limit = 64 * 1024
local large_string = string.rep("1", mem_limit/10)
local tmp
stats()
for i=50,300,50 do
tmp = replace(large_string, i, "x")
end
stats()
large_string = nil
tmp = nil
stats()
==24272== Memcheck, a memory error detector.
==24272== Copyright (C) 2002-2007, and GNU GPL'd, by Julian Seward et al.
==24272== Using LibVEX rev 1804, a library for dynamic binary translation.
==24272== Copyright (C) 2004-2007, and GNU GPL'd, by OpenWorks LLP.
==24272== Using valgrind-3.3.0, a dynamic binary instrumentation framework.
==24272== Copyright (C) 2000-2007, and GNU GPL'd, by Julian Seward et al.
==24272== For more details, rerun with: -v
==24272==
==24272== Invalid read of size 1
==24272== at 0x4C23108: memcpy (mc_replace_strmem.c:402)
==24272== by 0x4E391E0: luaS_newlstr (lstring.c:62)
==24272== by 0x4E3B3A6: luaV_concat (lvm.c:306)
==24272== by 0x4E3B861: luaV_execute (lvm.c:535)
==24272== by 0x4E327CC: luaD_call (ldo.c:377)
==24272== by 0x4E31F06: luaD_rawrunprotected (ldo.c:116)
==24272== by 0x4E31F84: luaD_pcall (ldo.c:463)
==24272== by 0x4E2E014: lua_pcall (lapi.c:821)
==24272== by 0x400F7D: run_script (in /home/bobby/opt/lua/misc/lua_memlimit)
==24272== by 0x40107D: main (in /home/bobby/opt/lua/misc/lua_memlimit)
==24272== Address 0x5834c40 is not stack'd, malloc'd or (recently) free'd
==24272==
==24272== Invalid read of size 1
==24272== at 0x4C23111: memcpy (mc_replace_strmem.c:402)
==24272== by 0x4E391E0: luaS_newlstr (lstring.c:62)
==24272== by 0x4E3B3A6: luaV_concat (lvm.c:306)
==24272== by 0x4E3B861: luaV_execute (lvm.c:535)
==24272== by 0x4E327CC: luaD_call (ldo.c:377)
==24272== by 0x4E31F06: luaD_rawrunprotected (ldo.c:116)
==24272== by 0x4E31F84: luaD_pcall (ldo.c:463)
==24272== by 0x4E2E014: lua_pcall (lapi.c:821)
==24272== by 0x400F7D: run_script (in /home/bobby/opt/lua/misc/lua_memlimit)
==24272== by 0x40107D: main (in /home/bobby/opt/lua/misc/lua_memlimit)
==24272== Address 0x5834c3f is not stack'd, malloc'd or (recently) free'd
==24272==
==24272== Invalid read of size 1
==24272== at 0x4C23118: memcpy (mc_replace_strmem.c:402)
==24272== by 0x4E391E0: luaS_newlstr (lstring.c:62)
==24272== by 0x4E3B3A6: luaV_concat (lvm.c:306)
==24272== by 0x4E3B861: luaV_execute (lvm.c:535)
==24272== by 0x4E327CC: luaD_call (ldo.c:377)
==24272== by 0x4E31F06: luaD_rawrunprotected (ldo.c:116)
==24272== by 0x4E31F84: luaD_pcall (ldo.c:463)
==24272== by 0x4E2E014: lua_pcall (lapi.c:821)
==24272== by 0x400F7D: run_script (in /home/bobby/opt/lua/misc/lua_memlimit)
==24272== by 0x40107D: main (in /home/bobby/opt/lua/misc/lua_memlimit)
==24272== Address 0x5834c3e is not stack'd, malloc'd or (recently) free'd
==24272==
==24272== Invalid read of size 1
==24272== at 0x4C2311F: memcpy (mc_replace_strmem.c:402)
==24272== by 0x4E391E0: luaS_newlstr (lstring.c:62)
==24272== by 0x4E3B3A6: luaV_concat (lvm.c:306)
==24272== by 0x4E3B861: luaV_execute (lvm.c:535)
==24272== by 0x4E327CC: luaD_call (ldo.c:377)
==24272== by 0x4E31F06: luaD_rawrunprotected (ldo.c:116)
==24272== by 0x4E31F84: luaD_pcall (ldo.c:463)
==24272== by 0x4E2E014: lua_pcall (lapi.c:821)
==24272== by 0x400F7D: run_script (in /home/bobby/opt/lua/misc/lua_memlimit)
==24272== by 0x40107D: main (in /home/bobby/opt/lua/misc/lua_memlimit)
==24272== Address 0x5834c3d is not stack'd, malloc'd or (recently) free'd
lua_memlimit: lua_memlimit.c:138: run_script: Assertion `info->memused == 0' failed.
str_replace.lua: before GC memused=31277, peak_memused=31683
str_replace.lua: after GC memused=28563, peak_memused=31683
str_replace.lua: before GC memused=47778, peak_memused=47778
str_replace.lua: after GC memused=36747, peak_memused=47778
LOW MEM: 1 osize=0, nsize=6578, used=69038, peak=62460, need=3502
LOW MEM: 2 used=50072, peak=62460
LOW MEM: 1 osize=1638, nsize=6553, used=68342, peak=63427, need=2806
LOW MEM: 2 used=48843, peak=63427
LOW MEM: 1 osize=0, nsize=6378, used=67163, peak=63427, need=1627
LOW MEM: 2 used=42465, peak=63427
LOW MEM: 1 osize=0, nsize=6328, used=67213, peak=63427, need=1677
LOW MEM: 2 used=42515, peak=63427
LOW MEM: 1 osize=0, nsize=6278, used=67263, peak=63427, need=1727
LOW MEM: 2 used=42565, peak=63427
str_replace.lua: before GC memused=60336, peak_memused=63427
str_replace.lua: after GC memused=42240, peak_memused=63427
str_replace.lua: before GC memused=42240, peak_memused=63427
str_replace.lua: after GC memused=27855, peak_memused=63427
str_replace.lua: memused=-1229, peak_memused=63427
==24272==
==24272== ERROR SUMMARY: 4915 errors from 4 contexts (suppressed: 4 from 1)
==24272== malloc/free: in use at exit: 48 bytes in 1 blocks.
==24272== malloc/free: 561 allocs, 560 frees, 214,475 bytes allocated.
==24272== For counts of detected errors, rerun with: -v
==24272== searching for pointers to 1 not-freed blocks.
==24272== checked 83,680 bytes.
==24272==
==24272== LEAK SUMMARY:
==24272== definitely lost: 0 bytes in 0 blocks.
==24272== possibly lost: 0 bytes in 0 blocks.
==24272== still reachable: 48 bytes in 1 blocks.
==24272== suppressed: 0 bytes in 0 blocks.
==24272== Rerun with --leak-check=full to see details of leaked memory.
