Isn't the context switch between OS threads fairly costly? Are you sure
this isn't a OS thread performance issue?
Tom
----- Original Message -----
From: "Simon at the Threshold" <dweller@the-threshold.org>
To: <lua@bazar2.conectiva.com.br>
Sent: Thursday, October 02, 2003 3:11 AM
Subject: lua-5.0 performance problems
Lo there,
I searched the internet for a small, fast embedible scripting/language
to integrate in with my carrier grade web application (millions of
users) to replace php (which is very slow).
Not wanting to rewrite the entire interface in c/c++ I figured it would
be worth looking at other languages. I've evaluated (and rejected) the
likes of Java, Python, PHP, Perl, oCaml and finally came to rest on lua.
In multi-threaded C tests with a single process connecting to a socket
and driving a ascii based command line I can get 400 TPS (transactions
per seconds) across a 100 MBS network.
I wrappered in lua (pretty easy all things considered) directly into the
core code and wrote myself a multi-threaded c program to run anything up
to 300 threads each running a lua parser.
I got 2 TPS
I figured maybe I was being dim and optimised my code so that it ran
compiled byte-code and got 4 TPS. I then figured it was a waste of time
each iteration of the lua script loading the same shared library so I
linked it in staticly and only call the registration once for each
thread and got 6 TPS.
On the plus side lua doesn't seem to thrash the machine (80% idle at 100
threads) but the performance is dire.
Am I being dim or is lua really this so..
I've attached my lua test script (test.lua) my C lua wrapper
(lua_mmcore.c) and my multi-threaded test program (luatest.c).Apols if
they are a little rough around the edges but this was a test bed not a
long-term maintainable code base.
Thanks in advance for any advice..
Regards
Simon
--
If you want to know what god thinks of money, just look at the people he
gave
it to.
-- Dorthy Parker
--
This email and any attachments hereto are strictly confidential and
intended solely for the addressee. It may contain information which is
covered by legal, professional or other privilege. If you are not the
indended addressee, you must not disclose, forward, copy or take any
action in reliance of this email or attachments. If you have received
this email in error, please notify us as soon as possible.
----------------------------------------------------------------------------
----
//
// LUA interface layer, this file defines all of the Core Mediator Lua
// wrappers which allow script language to resources and protocols like
// HPI.
//
// Revision History:-
//
// $Log$
//
static const char* const cvsId = "@(#) $Header$";
#include <stdio.h>
#include <stdarg.h>
#include <errno.h>
#include <unistd.h>
#include <string.h>
#include <strings.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <signal.h>
#include <pthread.h>
#include "common.h"
#include "hpi.h"
#include "lua_mmcore.h"
// Internal/Private method (single hpi call point)
//
static HPIRequest doHPI(lua_State* lua, char cmd[])
{
HPIRequest hpi = NULL;
// Attempt to obtain a HPI handle from the registery
//
lua_getglobal(lua, "hpi");
hpi = lua_touserdata(lua, -1);
// Make the parse attempt even if we don't have a HPI handle
//
hpi = parseHPIRequest(hpi, cmd);
// Now stick the latest socket/hpi values onto the registary so we
// can make use of them in other lua/c calls.
//
lua_pushlightuserdata(lua, hpi);
lua_setglobal(lua, "hpi");
return hpi;
}
// Wrapper action for attempting a HPI command
//
static int lua_doHPI(lua_State* lua)
{
HPIRequest hpi = lua_touserdata(lua, lua_upvalueindex(1));
char* cmd = (char*)luaL_checkstring(lua, 1);
hpi = doHPI(lua, cmd);
if(hpi == NULL)
{
lua_pushnil(lua);
}
else
{
HPIToken tok;
Profile prof;
lua_newtable(lua);
lua_pushstring(lua, "RESP");
lua_pushnumber(lua, (lua_Number)hpi->errcode);
lua_settable(lua, -3);
lua_pushstring(lua, "ERROR");
if(hpi->retstr)
{
lua_pushstring(lua, hpi->retstr);
}
else
{
lua_pushnil(lua);
}
lua_settable(lua, -3);
// Convert the info fields into the table return
//
for(tok=hpi->argout; tok; tok=tok->next)
{
lua_pushstring(lua, tok->name);
if(tok->numdata > 0)
{
lua_pushstring(lua, tok->data[0]);
}
else
{
lua_pushnil(lua);
}
lua_settable(lua, -3);
}
// Convert the profile data (persist/io data) into the table return
//
for(prof=hpi->out; prof; prof=prof->next)
{
char value[MAXSTRING+1];
lua_pushstring(lua, prof->feature->name);
if(numProfileData(hpi->prop, prof) > 0
&& getProfileData(hpi->prop, prof, value, 0, STRING_O))
{
lua_pushstring(lua, value);
}
else
{
lua_pushnil(lua);
}
lua_settable(lua, -3);
}
}
return 1;
}
// Register the defined lua wrapper methods to MMLIBNAME (mmcore)
//
LUALIB_API int luaopen_mmcore(lua_State* lua)
{
static struct luaL_reg reg[] =
{
{ "hpi", lua_doHPI },
{ NULL, NULL }
};
// Now register the methods
//
luaL_openlib(lua, "mmcore", reg, 0);
return 1;
}
// End of File
----------------------------------------------------------------------------
----
// My attempt at multi-threading some lua parsers
//
#include <stdio.h>
#include <stdlib.h>
#include <stdarg.h>
#include <unistd.h>
#include <string.h>
#include <fcntl.h>
#include <signal.h>
#include <sys/socket.h>
#include <sys/time.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <pthread.h>
#include <hpi.h>
#include <lua.h>
#include <lualib.h>
#include <lauxlib.h>
#include <lua_mmcore.h>
extern char* optarg;
extern int optind;
/*
We have been asked to shutdown gracefully
*/
int shutdownModule(int sig)
{
ModuleShutdown = TRUE;
return sig;
}
/*
We have been asked to Reconfigure
*/
int configModule(int sig)
{
if(loadConfig(SERVICE_DATA) < 0)
{
return -1;
}
if(loadConfig(QUEUE_DATA) < 0)
{
return -1;
}
if(loadConfig(DB_DATA) < 0)
{
return -1;
}
if(loadConfig(PROPERTY_DATA) < 0)
{
return -1;
}
if(loadConfig(ELEMENT_DATA) < 0)
{
return -1;
}
if(loadConfig(USER_DATA) < 0)
{
return -1;
}
if(loadConfig(LOGIN_DATA) < 0)
{
return -1;
}
if(loadConfig(RESPONSE_DATA) < 0)
{
return -1;
}
if(loadConfig(MAPPING_DATA) < 0)
{
return -1;
}
if(loadConfig(ROUTING_DATA) < 0)
{
return -1;
}
if(loadConfig(REMOTE_DATA) < 0)
{
return -1;
}
return 0;
}
void report(FILE* fp, char format[], ...)
{
struct timeval tyme;
va_list ap;
if(gettimeofday(&tyme, NULL) == 0)
{
fprintf(fp, "%d %d ", (int)tyme.tv_sec, (int)tyme.tv_usec);
va_start(ap, format);
vfprintf(fp, format, ap);
va_end(ap);
}
}
static char* luaBinary = NULL;
static int luaBinarySize = 0;
static char* luaScript = "test.lub";
static int loopCount = 3;
static unsigned startNum = 1000;
static int numThreads = 51;
void loadluabin()
{
struct stat st;
if(stat(luaScript, &st) == 0)
{
luaBinarySize = st.st_size;
FILE* fp = fopen(luaScript, "r");
if(fp)
{
luaBinary = calloc(st.st_size, sizeof(char));
fread(luaBinary, sizeof(char), st.st_size, fp);
fclose(fp);
}
}
}
void* luaThread(void* arg)
{
int id = (int)arg;
int i;
char strid[64];
char file[64];
FILE* fp;
lua_State* lua = lua_open();
luaopen_base(lua);
luaopen_table(lua);
luaopen_loadlib(lua);
luaopen_mmcore(lua);
sprintf(strid, "%d", startNum+id);
sprintf(file, "log/%d-log-%d.out", startNum+id, id);
printf("Thread %d loging to %s\n", id, file);
fp = fopen(file,"w+");
if(fp == NULL)
{
printf("Failed to open %s\n", file);
}
//
// Set the subid in the lua script..
lua_pushstring(lua, strid);
lua_setglobal(lua, "subid");
for(i=0; i < loopCount; ++i)
{
report(fp, "START LUA %s ", luaScript);
if(luaBinary)
{
lua_dobuffer(lua, luaBinary, luaBinarySize, "script");
}
else
{
lua_dofile(lua, luaScript);
}
report(fp, "END LUA %s\n", luaScript);
}
// Free and close off the output file
//
fclose(fp);
// Shutdown lua
//
lua_close(lua);
return arg;
}
int main(int argc, char*argv[])
{
int opt;
if(initModule("luatest", 0) < 0)
{
printf("Failed to init\n");
exit(-1);
}
while((opt = getopt(argc, argv, "lz:t:n:s:S:c:")) != -1)
{
switch(opt)
{
case 'l': // lua loop test
{
int i;
pthread_t tid[1024];
pthread_attr_t tattr;
loadluabin();
printf("LUA SCRIPT TEST\n");
pthread_attr_init(&tattr);
// Create the threads
//
for(i=0; i < numThreads; ++i)
{
if(pthread_create(&tid[i], &tattr, luaThread, (void*)i) < 0)
{
printf("Failed to create thread %d\n", i);
}
else
{
printf("Created thread %d\n", i);
}
}
// Wait for the threads to terminate and cleanup
//
for(i=0; i < numThreads; ++i)
{
pthread_join(tid[i], NULL);
}
}
break;
case 'z': // Sleep
{
int winks = atoi(optarg);
printf("Sleeping %d winks\n", winks);
sleep(winks);
}
break;
case 'c': // Number of iterations
{
loopCount = atoi(optarg);
printf("loopCount %d\n", loopCount);
}
break;
case 't': // Number of threads
{
numThreads = atoi(optarg);
printf("numThreads %d\n", numThreads);
}
break;
case 's': // Start number
{
startNum = atoi(optarg);
printf("startNum %d\n", startNum);
}
break;
case 'S': // Lua script file to run
{
luaScript = optarg;
printf("luaScript %s\n", luaScript);
}
break;
default:
printf("Useless usage\n");
break;
}
}
// this should terminate the process
printf("Calling deinitModule\n");
deinitModule(SIGTERM);
// should not get this far
exit(0);
}
----------------------------------------------------------------------------
----
print ("About to login subid "..subid)
resp = mmcore.hpi("LOGIN:SUBID,"..subid..":PIN,1234:type,subscriber;")
if(resp)
then
sid = resp["SID"]
print ("Dumping response "..sid);
table.foreach(resp, print);
print ("About to info")
resp = mmcore.hpi("INFO;");
if(resp)
then
print ("Dumping response");
table.foreach(resp, print);
end
print ("About to info")
resp = mmcore.hpi("GET:VIRTUAL,120:ERROR,0;");
if(resp)
then
print ("Dumping response");
table.foreach(resp, print);
end
print ("About to logout ")
resp = mmcore.hpi("LOGOUT;");
if(resp)
then
print ("Dumping response");
table.foreach(resp, print);
end
else
print("No response");
end
