Re: Status of ltokens?

[Luiz Henrique de Figueiredo <lhf@...>]

> Are there any plans of bringing ltokens up to date to Lua 5.1 in
> the near future?

I'm sorry but I have no immediate plans to update ltokens.
Too much to do, too little time...

However, please find attached a version of ltokens that runs with Lua 5.1.
Note that it's not just an update of the 5.0 version because it only works
on files, not strings, and it does not work as a dynamic library because
the internal API needed by ltokens is not normally visible when built under
Linux. So, you'll have to link this library statically, e.g., by adapting
linit.c.

As alternatives, you can try tokenf or lbci, both available at
	http://www.tecgraf.puc-rio.br/~lhf/ftp/lua/
	
> I need it for extracting translatable strings from
> Lua code (in Ion), and currently I can't do that.

If all you need is to extract strings from Lua code and you can use a
standalone program, it is easier to adapt my lstrip tool, also available
at the URL above.

If you need further help, please get in touch.
--lhf

/*
* ltokens.c
* lexer library -- uses private functions (politely)
* Luiz Henrique de Figueiredo <lhf@tecgraf.puc-rio.br>
* 16 Oct 2006 13:21:22
* This code is hereby placed in the public domain.
*/

#include <errno.h>
#include <string.h>
#include <stdio.h>

#define LUA_CORE

#include "lua.h"
#include "lauxlib.h"

#include "llex.h"
#include "lparser.h"
#include "ldo.h"
#include "lstring.h"
#include "ltable.h"
#include "lzio.h"

static const char *reader(lua_State *L, void *data, size_t *size)
{
 static char b[512];
 FILE *f = (FILE*) data;
 if (feof(f)) return NULL;
 *size = fread(b, 1, sizeof(b), f);
 return (*size > 0) ? b : NULL;
}

static int utokens(lua_State *L)
{
 FILE *f;
 const char *filename;
 int n=0;
 ZIO z;
 Mbuffer B;
 LexState X;
 struct FuncState fs;
 if (lua_isnone(L,1))
 {
  filename="=stdin";
  f=stdin;
 }
 else
 {
  filename=luaL_checkstring(L,1);
  f=fopen(filename,"r");
  if (f==NULL)
  {
   lua_pushnil(L);
   lua_pushfstring(L, "%s: %s", filename, strerror(errno));
   lua_pushnumber(L, errno);
   return 3;
  }
  filename=lua_pushfstring(L, "@%s", filename);
 }
 luaZ_init(L,&z,reader,f);
 luaZ_initbuffer(L,&B);
 X.buff=&B;
 luaX_setinput(L,&X,&z,luaS_new(L,filename));
 X.fs = &fs;
 fs.h = luaH_new(L, 0, 0);
 /* anchor table of constants (to avoid being collected) */
 sethvalue2s(L,L->top,fs.h);
 incr_top(L);
 lua_newtable(L);
 for (;;)
 {
  int t;
  luaX_next(&X);
  t=X.t.token;
#if 0
printf("tokens: text=[%s]\n",X.buff->buffer);
#endif
  if (t==TK_EOS)
  {
#if 0
   lua_pushliteral(L,"n");
   lua_pushnumber(L,n);
   lua_settable(L,-3);
#endif
   luaZ_freebuffer(L,&B);
   if (f!=stdin) fclose(f);
   return 1;
  }
  lua_pushnumber(L,++n);
  lua_newtable(L);

  lua_pushliteral(L,"line");
  lua_pushnumber(L,X.linenumber);
  lua_settable(L,-3);

#if 0
  lua_pushliteral(L,"TOKEN");
  lua_pushnumber(L,t);
  lua_settable(L,-3);

  lua_pushliteral(L,"TEXT");
  lua_pushlstring(L,X.buff->buffer,X.buff->n);
  lua_settable(L,-3);
#endif

  lua_pushliteral(L,"token");
  switch (t)
  {
   case TK_STRING:
    lua_pushliteral(L,"string");
    lua_settable(L,-3);
    lua_pushliteral(L,"value");
    lua_pushstring(L,getstr(X.t.seminfo.ts));
    lua_settable(L,-3);
    break;
   case TK_NAME:
    lua_pushliteral(L,"name");
    lua_settable(L,-3);
    lua_pushliteral(L,"value");
    lua_pushstring(L,getstr(X.t.seminfo.ts));
    lua_settable(L,-3);
    break;
   case TK_NUMBER:
    lua_pushliteral(L,"number");
    lua_settable(L,-3);
    lua_pushliteral(L,"value");
    lua_pushnumber(L,X.t.seminfo.r);
    lua_settable(L,-3);
    break;
   default:
   {
    const char *s=luaX_token2str(&X,t);
    if (t<FIRST_RESERVED) ; else lua_pushstring(L,s);
#if 0
printf("tokens: luaX_token2str s=[%s]\n",s);
#endif
    lua_settable(L,-3);
    break;
   }
  }
  lua_settable(L,-3);
 }
}

static int ptokens(lua_State *L)
{
 lua_pushvalue(L,lua_upvalueindex(1));
 lua_insert(L,1);
 if (lua_pcall(L, 1, LUA_MULTRET, 0))
 {
  lua_pushnil(L);
  lua_insert(L,-2);
  return 2;
 }
 return lua_gettop(L);
}

LUALIB_API int luaopen_tokens(lua_State *L)
{
 lua_pushcclosure(L,utokens,0);
 lua_pushcclosure(L,ptokens,1);
 lua_setglobal(L,"tokens");
 return 0;
}