Re: Possible bug in luaD_shrinkstack

[Szymon Modzelewski <szmodzelewski@...>]

Here is a test case using the C API, provided as a patch against lua 5.3.6

run using:
make
src/lua

Actual output:
taskMain()
lua: ldo.c:216: stackinuse: Assertion `lim <= L->stack_last' failed.
Aborted (core dumped)

Expected output:
taskMain()
dummyFunc()

HARDMEMTESTS needs to be defined for easy reproduction (included in
the patched luaconf.h)

Best regards,
Szymon Modzelewski

From 5e3db53bcd00d91475a8fec874d77fe185209585 Mon Sep 17 00:00:00 2001
From: Szymon Modzelewski <szmodzelewski@gmail.com>
Date: Thu, 10 Mar 2022 00:07:06 +0100
Subject: [PATCH] luaD_shrinkstack bug case

---
 Makefile      |   2 +-
 src/lua.c     | 107 ++++++++++++++++++++++++++++++++++++++++++++++++++
 src/luaconf.h |   7 ++++
 3 files changed, 115 insertions(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index a2820e0..b156c44 100644
--- a/Makefile
+++ b/Makefile
@@ -4,7 +4,7 @@
 # == CHANGE THE SETTINGS BELOW TO SUIT YOUR ENVIRONMENT =======================
 
 # Your platform. See PLATS for possible values.
-PLAT= none
+PLAT= linux
 
 # Where to install. The installation starts in the src and doc directories,
 # so take care if INSTALL_TOP is not an absolute path. See the local target.
diff --git a/src/lua.c b/src/lua.c
index ca5b298..4d8dbee 100644
--- a/src/lua.c
+++ b/src/lua.c
@@ -1,3 +1,4 @@
+#if 0
 /*
 ** $Id: lua.c,v 1.230.1.1 2017/04/19 17:29:57 roberto Exp $
 ** Lua stand-alone interpreter
@@ -609,4 +610,110 @@ int main (int argc, char **argv) {
   lua_close(L);
   return (result && status == LUA_OK) ? EXIT_SUCCESS : EXIT_FAILURE;
 }
+#endif
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <assert.h>
+
+#include "lua.h"
+
+#include "lauxlib.h"
+#include "lualib.h"
+
+static int dummyContinuation(lua_State *L, int status, lua_KContext ctx) {
+  (void)L; (void)status; (void)ctx;
+  //printf("status: %d\n", status);
+  return 0;
+}
+
+static int taskError(lua_State *L) {
+  const char *str = luaL_tolstring(L, -1, NULL);
+  lua_writestringerror("%s\n", str);
+  lua_pop(L, 1);
+  return 1;
+}
+
+static int taskMain(lua_State *L) {
+  lua_pushcfunction(L, taskError);
+  lua_replace(L, 1);
+
+  printf("taskMain()\n");
+
+  lua_pcallk(L, 0, LUA_MULTRET, 1, 0, dummyContinuation);
+
+  return 0;
+}
+
+static int dummyFunc(lua_State *L) {
+  printf("dummyFunc()\n");
+  lua_pushnil(L);
+  return 1;
+}
+
+static int testIndex(lua_State *L) {
+  //printf("testIndex()\n");
+
+  lua_pushcfunction(L, dummyFunc);
+  return 1;
+}
+
+/*
+** Main body of stand-alone interpreter (to be called in protected mode).
+** Reads the options and handles them all.
+*/
+static int pmain (lua_State *L) {
+  luaL_checkversion(L);  /* check that interpreter has correct version */
+ 
+  lua_pushboolean(L, 1);  /* signal for libraries to ignore env. vars. */
+  lua_setfield(L, LUA_REGISTRYINDEX, "LUA_NOENV");
+
+  luaL_openlibs(L);  /* open standard libraries */
+
+  lua_newuserdata(L, 0);
+
+  if (luaL_newmetatable(L, "testmt")) {
+    lua_pushcfunction(L, testIndex);
+    lua_setfield(L, -2, "__index");
+  }
+
+  lua_setmetatable(L, -2);
+  lua_setglobal(L, "udTest");
+
+  static const char rstr[] = "return udTest.whatever(0, 1, 2, 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z')";
+
+  int res = luaL_loadstring(L, rstr);
+  assert(res == LUA_OK);
+
+  lua_State *co = lua_newthread(L);
+  lua_insert(L, -2); // move above chunk
+
+  lua_pushcfunction(co, taskMain);
+
+  // dummy arg
+  lua_pushlightuserdata(co, NULL);
+
+  lua_xmove(L, co, 1);
+
+  lua_resume(co, L, 2);
+
+  lua_pop(L, 1); // pop thread
+
+  return 0;
+}
+
+
+int main (int argc, char **argv) {
+  (void)argc; (void)argv;
+  int status;
+  lua_State *L = luaL_newstate();  /* create state */
+  if (L == NULL) {
+    return EXIT_FAILURE;
+  }
+  lua_pushcfunction(L, &pmain);  /* to call 'pmain' in protected mode */
+  status = lua_pcall(L, 0, 0, 0);  /* do the call */
+  lua_close(L);
+  return (status == LUA_OK) ? EXIT_SUCCESS : EXIT_FAILURE;
+}
 
diff --git a/src/luaconf.h b/src/luaconf.h
index 9eeeea6..e745394 100644
--- a/src/luaconf.h
+++ b/src/luaconf.h
@@ -18,6 +18,13 @@
 ** ===================================================================
 */
 
+#define LUA_USE_APICHECK
+//#define HARDSTACKTESTS
+#define HARDMEMTESTS
+
+#include <assert.h>
+#define lua_assert	assert
+
 
 /*
 ** {====================================================================
-- 
2.25.1