Re: Handling conflicts caused by command-line editing, revisited

[Andrew Gierth <andrew@...>]

>>>>> "Andrew" == Andrew Gierth <andrew@tao11.riddles.org.uk> writes:

 Andrew> 2) patch lua.c to make it load libedit/libreadline dynamically
 Andrew> rather than explicitly linking it, and not load it until it's
 Andrew> needed (and maybe add an option to disable it even then)

So I did a proof-of-concept patch for this, and tested that it (a) seems
to work and (b) resolves the issue.

Here's the patch against lua 5.3.5 (as a bonus, it also backpatches the
rl_readline_name / rl_inhibit_completion settings). This is intended for
FreeBSD, but nothing in it is OS-dependent except the pathname for
libedit or readline.

Open questions:

 - stealing an option letter (currently -N) to disable editing might
   cause conflicts in future, and obviously isn't portable to unpatched
   systems; I _think_ that it should be unnecessary, since loading
   libedit with RTLD_LOCAL should prevent it interfering with lua
   modules, and some preliminary testing suggests that this is so

 - currently it silently falls back to no editing if dlopen fails,
   should there be a warning saying why?

-- 
Andrew.

--- src/lua.c.orig	2017-04-19 17:29:57 UTC
+++ src/lua.c
@@ -77,16 +77,69 @@
 */
 #if !defined(lua_readline)	/* { */
 
-#if defined(LUA_USE_READLINE)	/* { */
+#if defined(LUA_USE_READLINE_DL)/* { */
+
+#include <dlfcn.h>
+
+#ifndef LUA_READLINE_LIBPATH
+#define LUA_READLINE_LIBPATH "/usr/local/lib/libedit.so"
+#endif
+
+typedef char *readline_functype(const char *);
+typedef int add_history_functype(const char *);
+
+static readline_functype *lua_readline_p = NULL;
+static add_history_functype *lua_saveline_p = NULL;
+static int disable_readline = 0;
+
+static void lua_initreadline(lua_State *L)
+{
+  void *editlib = NULL;
+  union dl_func_hack {
+    void *ptr;
+    readline_functype *rlfunc;
+    add_history_functype *ahfunc;
+    char **rlnamevar;
+    int *icompvar;
+  } u;
+  (void) L;
+  if (disable_readline)
+    return;
+  if ((editlib = dlopen(LUA_READLINE_LIBPATH, RTLD_LAZY | RTLD_LOCAL))) {
+    u.ptr = dlsym(editlib, "readline");
+    lua_readline_p = u.rlfunc;
+    u.ptr = dlsym(editlib, "add_history");
+    lua_saveline_p = u.ahfunc;
+    if ((u.ptr = dlsym(editlib, "rl_readline_name")))
+      *u.rlnamevar = "lua";
+    if ((u.ptr = dlsym(editlib, "rl_inhibit_completion")))
+      *u.icompvar = 1;
+  }
+}
+
+#define lua_readline(L,b,p) \
+  ((void)L,                                                          \
+   (lua_readline_p)                                                  \
+   ? (((b)=lua_readline_p(p)) != NULL)                               \
+   : (fputs(p, stdout), fflush(stdout), fgets(b, LUA_MAXINPUT, stdin) != NULL))
+#define lua_saveline(L,line)	\
+  do { (void)L; if (lua_saveline_p) lua_saveline_p(line); } while(0)
+#define lua_freeline(L,b)	\
+  do { (void)L; if (lua_readline_p) free(b); } while(0)
+
+#elif defined(LUA_USE_READLINE)	/* { */
 
 #include <readline/readline.h>
 #include <readline/history.h>
+#define lua_initreadline(L)  \
+	((void)L, rl_readline_name="lua", rl_inhibit_completion=1)
 #define lua_readline(L,b,p)	((void)L, ((b)=readline(p)) != NULL)
 #define lua_saveline(L,line)	((void)L, add_history(line))
 #define lua_freeline(L,b)	((void)L, free(b))
 
 #else				/* }{ */
 
+#define lua_initreadline(L)	((void) L)
 #define lua_readline(L,b,p) \
         ((void)L, fputs(p, stdout), fflush(stdout),  /* show prompt */ \
         fgets(b, LUA_MAXINPUT, stdin) != NULL)  /* get line */
@@ -141,6 +194,9 @@ static void print_usage (const char *bad
   "  -l name  require library 'name' into global 'name'\n"
   "  -v       show version information\n"
   "  -E       ignore environment variables\n"
+#if defined(LUA_USE_READLINE_DL)
+  "  -N       disable command-line editing\n"
+#endif
   "  --       stop handling options\n"
   "  -        stop handling options and execute stdin\n"
   ,
@@ -406,6 +462,7 @@ static void doREPL (lua_State *L) {
   int status;
   const char *oldprogname = progname;
   progname = NULL;  /* no 'progname' on errors in interactive mode */
+  lua_initreadline(L);
   while ((status = loadline(L)) != -1) {
     if (status == LUA_OK)
       status = docall(L, 0, LUA_MULTRET);
@@ -455,6 +512,7 @@ static int handle_script (lua_State *L, 
 #define has_v		4	/* -v */
 #define has_e		8	/* -e */
 #define has_E		16	/* -E */
+#define has_N		32	/* -N */
 
 /*
 ** Traverses all arguments from 'argv', returning a mask with those
@@ -482,6 +540,13 @@ static int collectargs (char **argv, int
           return has_error;  /* invalid option */
         args |= has_E;
         break;
+#if defined(LUA_USE_READLINE_DL)
+      case 'N':
+        if (argv[i][2] != '\0')  /* extra characters after 1st? */
+          return has_error;  /* invalid option */
+        args |= has_N;
+        break;
+#endif
       case 'i':
         args |= has_i;  /* (-i implies -v) *//* FALLTHROUGH */
       case 'v':
@@ -579,6 +644,10 @@ static int pmain (lua_State *L) {
   if (script < argc &&  /* execute main script (if there is one) */
       handle_script(L, argv + script) != LUA_OK)
     return 0;
+#if defined(LUA_USE_READLINE_DL)
+  if (args & has_N)
+    disable_readline = 1;
+#endif
   if (args & has_i)  /* -i option? */
     doREPL(L);  /* do read-eval-print loop */
   else if (script == argc && !(args & (has_e | has_v))) {  /* no arguments? */