Re: Lua 5.4.0-rc5 segfault in low memory conditions

[Sergey Zakharchenko <doublef.mobile@...>]

Hello Andrew,

Andrew Gierth <andrew@tao11.riddles.org.uk>:
> I couldn't reproduce this on freebsd, even with malloc debugging options
> enabled.

I'm afraid the memory patterns may be different across libcs and the
allocation that needs to fail may vary as well. My base system is
Debian Buster and libc is glibc 2.28. I could run a similar test on a
relatively  ancient FreeBSD installation I have around if you think
it's worth it. I'm attaching the brute force patch and helper script I
mentioned before for convenience.

> Can you get a backtrace of where your allocation #345 is happening?

Sure, it's during proto construction:

00000344 (nil) 5 0x5555555a9d40 56 <--- this one is OK
fail_alloc_at: 00000345: failing

Breakpoint 1, l_alloc (ud=0x0, ptr=0x0, osize=10, nsize=128) at lauxlib.c:998
998           failed = 1;
(gdb) bt
#0  l_alloc (ud=0x0, ptr=0x0, osize=10, nsize=128) at lauxlib.c:998
#1  0x0000555555569895 in luaM_malloc_ (L=0x5555555a2268, size=128,
tag=10) at lmem.c:193
#2  0x0000555555565e7f in luaC_newobj (L=0x5555555a2268, tt=10,
sz=128) at lgc.c:242
#3  0x00005555555658c4 in luaF_newproto (L=0x5555555a2268) at lfunc.c:246
#4  0x000055555556da08 in luaY_parser (L=0x5555555a2268,
z=0x7fffffffbd20, buff=0x7fffffffbc78, dyd=0x7fffffffbc90,
name=0x5555555a8ab8 "@test.lua", firstchar=114) at lparser.c:1979
#5  0x000055555556451c in f_parser (L=0x5555555a2268,
ud=0x7fffffffbc70) at ldo.c:796
#6  0x00005555555629be in luaD_rawrunprotected (L=0x5555555a2268,
f=0x55555556441f <f_parser>, ud=0x7fffffffbc70) at ldo.c:148
#7  0x0000555555564316 in luaD_pcall (L=0x5555555a2268,
func=0x55555556441f <f_parser>, u=0x7fffffffbc70, old_top=80, ef=0) at
ldo.c:749
#8  0x0000555555564607 in luaD_protectedparser (L=0x5555555a2268,
z=0x7fffffffbd20, name=0x5555555a8ab8 "@test.lua", mode=0x0) at
ldo.c:813
#9  0x000055555555f79a in lua_load (L=0x5555555a2268,
reader=0x55555557e905 <getF>, data=0x7fffffffbdc0,
chunkname=0x5555555a8ab8 "@test.lua", mode=0x0) at lapi.c:1053
#10 0x000055555557ed3b in luaL_loadfilex (L=0x5555555a2268,
filename=0x7fffffffe562 "test.lua", mode=0x0) at lauxlib.c:776
#11 0x000055555555be2f in handle_script (L=0x5555555a2268,
argv=0x7fffffffe260) at lua.c:225
#12 0x000055555555ca08 in pmain (L=0x5555555a2268) at lua.c:603
#13 0x00005555555638d5 in luaD_call (L=0x5555555a2268,
func=0x5555555a28d0, nresults=1) at ldo.c:482
#14 0x0000555555563b75 in luaD_callnoyield (L=0x5555555a2268,
func=0x5555555a28d0, nResults=1) at ldo.c:526
#15 0x000055555555f544 in f_call (L=0x5555555a2268, ud=0x7fffffffe110)
at lapi.c:997
#16 0x00005555555629be in luaD_rawrunprotected (L=0x5555555a2268,
f=0x55555555f50f <f_call>, ud=0x7fffffffe110) at ldo.c:148
#17 0x0000555555564316 in luaD_pcall (L=0x5555555a2268,
func=0x55555555f50f <f_call>, u=0x7fffffffe110, old_top=16, ef=0) at
ldo.c:749
#18 0x000055555555f614 in lua_pcallk (L=0x5555555a2268, nargs=2,
nresults=1, errfunc=0, ctx=0, k=0x0) at lapi.c:1023
#19 0x000055555555cb2d in main (argc=2, argv=0x7fffffffe258) at lua.c:629

Not sure if this directly gives you anything. The "interesting" things
in 'failed' vs 'non-failed' cases happen later on, when a bunch of
protos are seemingly disposed of by the GC in the 'failed' case, at
least that's what I recall from testing on MIPS. However, this
deallocation of protos doesn't happen during the full GC triggered by
the allocation failure.

> (Then, of course, let it continue and verify it crashes in the expected place)

Verified. Thanks for your interest!

Best regards,

-- 
DoubleF

diff --git a/src/lauxlib.c b/src/lauxlib.c
index e3d9be3..ec5c419 100644
--- a/src/lauxlib.c
+++ b/src/lauxlib.c
@@ -982,15 +982,39 @@ LUALIB_API const char *luaL_gsub (lua_State *L, const char *s,
   return lua_tostring(L, -1);
 }
 
-
+static int alloc_idx = 0;
+static int fail_alloc_at = -1;
 static void *l_alloc (void *ud, void *ptr, size_t osize, size_t nsize) {
+  void* ret;
+  char failed = 0;
   (void)ud; (void)osize;  /* not used */
-  if (nsize == 0) {
-    free(ptr);
-    return NULL;
+  if (fail_alloc_at == -1) {
+    const char* fail_alloc_at_str = getenv("LUA_FAIL_ALLOC_AT");
+    fail_alloc_at = strtoul(fail_alloc_at_str ? fail_alloc_at_str : "0", NULL, 0);
   }
-  else
-    return realloc(ptr, nsize);
+  alloc_idx++;
+  if (alloc_idx == fail_alloc_at)
+  {
+    if (nsize <= osize)
+      fprintf(stderr, "fail_alloc_at: %08d: cannot fail\n", fail_alloc_at);
+    else {
+      fprintf(stderr, "fail_alloc_at: %08d: failing\n", fail_alloc_at);
+      failed = 1;
+      ret = NULL;
+    }
+  }
+  if (!failed)
+  {
+    if (nsize == 0) {
+      free(ptr);
+      ret = NULL;
+    }
+    else {
+      ret = realloc(ptr, nsize);
+    }
+  }
+  fprintf(stderr, "%08d %p %u %p %u\n", alloc_idx, ptr, (unsigned)osize, ret, (unsigned)nsize);
+  return ret;
 }

Attachment: brute-force.sh
Description: application/shellscript