lua-5.0 performance problems

[Simon at the Threshold <dweller@...>]

Lo there,

I searched the internet for a small, fast embedible scripting/language 
to integrate in with my carrier grade web application (millions of 
users) to replace php (which is very slow).

Not wanting to rewrite the entire interface in c/c++ I figured it would 
be worth looking at other languages. I've evaluated (and rejected) the 
likes of Java, Python, PHP, Perl, oCaml and finally came to rest on lua.

In multi-threaded C tests with a single process connecting to a socket 
and driving a ascii based command line I can get 400 TPS (transactions 
per seconds) across a 100 MBS network.

I wrappered in lua (pretty easy all things considered) directly into the 
core code and wrote myself a multi-threaded c program to run anything up 
to 300 threads each running a lua parser.

I got 2 TPS

I figured maybe I was being dim and optimised my code so that it ran 
compiled byte-code and got 4 TPS.  I then figured it was a waste of time 
 each iteration of the lua script loading the same shared library so I 
linked it in staticly and only call the registration once for each 
thread and got 6 TPS.

On the plus side lua doesn't seem to thrash the machine (80% idle at 100 
threads) but the performance is dire.

Am I being dim or is lua really this so..

I've attached my lua test script (test.lua) my C lua wrapper 
(lua_mmcore.c) and my multi-threaded test program (luatest.c).Apols if 
they are a little rough around the edges but this was a test bed not a 
long-term maintainable code base.

Thanks in advance for any advice..

Regards
Simon

--
If you want to know what god thinks of money, just look at the people he 
gave
it to.
		-- Dorthy Parker
--
This email and any attachments hereto are strictly confidential and 
intended solely for the addressee. It may contain information which is 
covered by legal, professional or other privilege. If you are not the 
indended addressee, you must not disclose, forward, copy or take any 
action in reliance of this email or attachments. If you have received 
this email in error, please notify us as soon as possible.
//
//  LUA interface layer, this file defines all of the Core Mediator Lua
//  wrappers which allow script language to resources and protocols like
//	HPI.
//
//  Revision History:-
//
//  $Log$
//
static const char* const cvsId = "@(#) $Header$";

#include <stdio.h>
#include <stdarg.h>
#include <errno.h>
#include <unistd.h>
#include <string.h>
#include <strings.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <signal.h>
#include <pthread.h>
#include "common.h"
#include "hpi.h"
#include "lua_mmcore.h"

// Internal/Private method (single hpi call point)
// 
static HPIRequest doHPI(lua_State* lua, char cmd[])
{
	HPIRequest	hpi = NULL;

	// Attempt to obtain a HPI handle from the registery
	//
	lua_getglobal(lua, "hpi");
	hpi = lua_touserdata(lua, -1);
	
	// Make the parse attempt even if we don't have a HPI handle
	//
	hpi = parseHPIRequest(hpi, cmd);

	// Now stick the latest socket/hpi values onto the registary so we
	// can make use of them in other lua/c calls.
	//
	lua_pushlightuserdata(lua, hpi);
	lua_setglobal(lua, "hpi");

	return hpi;
}

// Wrapper action for attempting a HPI command
//
static int lua_doHPI(lua_State* lua)
{
	HPIRequest  hpi = lua_touserdata(lua, lua_upvalueindex(1));	
	char*		cmd = (char*)luaL_checkstring(lua, 1);

	hpi = doHPI(lua, cmd);
	if(hpi == NULL)
	{	
		lua_pushnil(lua);
	}
	else
	{
		HPIToken	tok;
		Profile		prof;

		lua_newtable(lua);

		lua_pushstring(lua, "RESP");
		lua_pushnumber(lua, (lua_Number)hpi->errcode);
		lua_settable(lua, -3);

		lua_pushstring(lua, "ERROR");
		if(hpi->retstr)
		{
			lua_pushstring(lua, hpi->retstr);
		}
		else
		{
			lua_pushnil(lua);
		}
		lua_settable(lua, -3);

		// Convert the info fields into the table return
		//
		for(tok=hpi->argout; tok; tok=tok->next)
		{
			lua_pushstring(lua, tok->name);
			if(tok->numdata > 0)
			{
				lua_pushstring(lua, tok->data[0]);
			}
			else
			{
				lua_pushnil(lua);
			}
			lua_settable(lua, -3);
		}

		// Convert the profile data (persist/io data) into the table return
		//
		for(prof=hpi->out; prof; prof=prof->next)
		{
			char	value[MAXSTRING+1];

			lua_pushstring(lua, prof->feature->name);
			if(numProfileData(hpi->prop, prof) > 0
			&& getProfileData(hpi->prop, prof, value, 0, STRING_O))
			{
				lua_pushstring(lua, value);
			}
			else
			{
				lua_pushnil(lua);
			}
			lua_settable(lua, -3);
		}
	}

	return 1;
}

// Register the defined lua wrapper methods to MMLIBNAME (mmcore)
//
LUALIB_API int luaopen_mmcore(lua_State*	lua)
{
	static struct luaL_reg	reg[] =
	{
		{	"hpi",		lua_doHPI		},
		{	NULL,			NULL			}
	};

	// Now register the methods
	//
	luaL_openlib(lua, "mmcore", reg, 0);

	return 1;
}

// End of File

// My attempt at multi-threading some lua parsers
//
#include <stdio.h>
#include <stdlib.h>
#include <stdarg.h>
#include <unistd.h>
#include <string.h>
#include <fcntl.h>
#include <signal.h>
#include <sys/socket.h>
#include <sys/time.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <pthread.h>
#include <hpi.h>
#include <lua.h>
#include <lualib.h>
#include <lauxlib.h>
#include <lua_mmcore.h>

extern char*	optarg;
extern int		optind;

/*
    We have been asked to shutdown gracefully
*/
int shutdownModule(int sig)
{
    ModuleShutdown = TRUE;
    return sig;
}


/*
    We have been asked to Reconfigure
*/
int configModule(int sig)
{
    if(loadConfig(SERVICE_DATA) < 0)
    {
        return -1;
    }

    if(loadConfig(QUEUE_DATA) < 0)
    {
        return -1;
    }

    if(loadConfig(DB_DATA) < 0)
    {
        return -1;
    }

    if(loadConfig(PROPERTY_DATA) < 0)
    {
        return -1;
    }

    if(loadConfig(ELEMENT_DATA) < 0)
    {
        return -1;
    }

    if(loadConfig(USER_DATA) < 0)
    {
        return -1;
    }

    if(loadConfig(LOGIN_DATA) < 0)
    {
        return -1;
    }

    if(loadConfig(RESPONSE_DATA) < 0)
    {
        return -1;
    }

    if(loadConfig(MAPPING_DATA) < 0)
    {
        return -1;
    }

    if(loadConfig(ROUTING_DATA) < 0)
    {
        return -1;
    }

    if(loadConfig(REMOTE_DATA) < 0)
    {
        return -1;
    }

    return 0;
}   


void report(FILE* fp, char format[], ...)
{
	struct timeval	tyme;
	va_list			ap;

	if(gettimeofday(&tyme, NULL) == 0)
	{
		fprintf(fp, "%d %d ", (int)tyme.tv_sec, (int)tyme.tv_usec);
		va_start(ap, format);
		vfprintf(fp, format, ap);
		va_end(ap);
	}
}

static char*	luaBinary		= NULL;
static int		luaBinarySize	= 0;

static char*	luaScript		= "test.lub";
static int		loopCount		= 3;
static unsigned	startNum		= 1000;
static int		numThreads		= 51;

void loadluabin()
{
	struct stat	st;

	if(stat(luaScript, &st) == 0)
	{
		luaBinarySize = st.st_size;
		FILE* fp = fopen(luaScript, "r");
		if(fp)
		{
			luaBinary = calloc(st.st_size, sizeof(char));
			fread(luaBinary, sizeof(char), st.st_size, fp);
			fclose(fp);
		}
	}
}


void* luaThread(void* arg)
{
	int				id = (int)arg;
	int				i;
	char			strid[64];
	char			file[64];
	FILE*			fp;
	lua_State*		lua = lua_open();

	luaopen_base(lua);
	luaopen_table(lua);
	luaopen_loadlib(lua);
	luaopen_mmcore(lua);
	
	sprintf(strid, "%d", startNum+id);
	sprintf(file, "log/%d-log-%d.out", startNum+id, id);
	printf("Thread %d loging to %s\n", id, file);

	fp = fopen(file,"w+");
	if(fp == NULL)
	{
		printf("Failed to open %s\n", file);
	}

	//
	// Set the subid in the lua script..
	lua_pushstring(lua, strid);
	lua_setglobal(lua, "subid");

	for(i=0; i < loopCount; ++i)
	{
		report(fp, "START LUA %s ", luaScript);
		if(luaBinary)
		{
			lua_dobuffer(lua, luaBinary, luaBinarySize, "script");
		}
		else
		{
			lua_dofile(lua, luaScript);
		}
		report(fp, "END LUA %s\n", luaScript);
	}

	// Free and close off the output file
	//
	fclose(fp);

	// Shutdown lua
	//
	lua_close(lua);

	return arg;
}

int main(int argc, char*argv[])
{
	int					opt;

	if(initModule("luatest", 0) < 0)
	{
		printf("Failed to init\n");
		exit(-1);
	}

	while((opt = getopt(argc, argv, "lz:t:n:s:S:c:")) != -1)
	{
		switch(opt)
		{
			case 'l': // lua loop test
			{
				int				i;
				pthread_t		tid[1024];
				pthread_attr_t  tattr;


				loadluabin();

				printf("LUA SCRIPT TEST\n");

				pthread_attr_init(&tattr);
				
				// Create the threads
				//
				for(i=0; i < numThreads; ++i)
				{
					if(pthread_create(&tid[i], &tattr, luaThread, (void*)i) < 0)
					{
						printf("Failed to create thread %d\n", i);
					}
					else
					{
						printf("Created thread %d\n", i);
					}
				}

				// Wait for the threads to terminate and cleanup
				//
				for(i=0; i < numThreads; ++i)
				{
					pthread_join(tid[i], NULL);
				}
			}
			break;

			case 'z': // Sleep
			{
				int winks = atoi(optarg);
				printf("Sleeping %d winks\n", winks);
				sleep(winks);
			}
			break;

			case 'c': // Number of iterations
			{
				loopCount = atoi(optarg);
				printf("loopCount %d\n", loopCount);
			}
			break;

			case 't': // Number of threads
			{
				numThreads = atoi(optarg);
				printf("numThreads %d\n", numThreads);
			}
			break;

			case 's': // Start number
			{
				startNum = atoi(optarg);
				printf("startNum %d\n", startNum);
			}
			break;

			case 'S': // Lua script file to run
			{
				luaScript = optarg;
				printf("luaScript %s\n", luaScript);
			}
			break;

			default:
				printf("Useless usage\n");
			break;
		}
	}

	// this should terminate the process
	printf("Calling deinitModule\n");


	deinitModule(SIGTERM);
	// should not get this far
	exit(0);
}

print ("About to login subid "..subid)

resp = mmcore.hpi("LOGIN:SUBID,"..subid..":PIN,1234:type,subscriber;")
if(resp)
then
	sid = resp["SID"]

	print ("Dumping response "..sid);
	table.foreach(resp, print);

	print ("About to info")
	resp = mmcore.hpi("INFO;");
	if(resp)
	then
		print ("Dumping response");
		table.foreach(resp, print);
	end

	print ("About to info")
	resp = mmcore.hpi("GET:VIRTUAL,120:ERROR,0;");
	if(resp)
	then
		print ("Dumping response");
		table.foreach(resp, print);
	end

	print ("About to logout ")
	resp = mmcore.hpi("LOGOUT;");
	if(resp)
	then
		print ("Dumping response");
		table.foreach(resp, print);
	end
else
	print("No response");
end