From 2e66aca35722e7fee786027d545fe371786fc01f Mon Sep 17 00:00:00 2001 From: sapier Date: Tue, 26 Nov 2013 18:15:31 +0100 Subject: Fix modstore/favourites hang by adding asynchronous lua job support --- src/script/lua_api/CMakeLists.txt | 2 + src/script/lua_api/l_async_events.cpp | 396 ++++++++++++++++++++++++ src/script/lua_api/l_async_events.h | 228 ++++++++++++++ src/script/lua_api/l_internal.h | 6 +- src/script/lua_api/l_mainmenu.cpp | 72 +++-- src/script/lua_api/l_mainmenu.h | 6 + src/script/lua_api/l_util.cpp | 16 + src/script/lua_api/l_util.h | 4 + src/script/lua_api/marshall.c | 551 ++++++++++++++++++++++++++++++++++ src/script/scripting_mainmenu.cpp | 24 +- src/script/scripting_mainmenu.h | 9 + 11 files changed, 1286 insertions(+), 28 deletions(-) create mode 100644 src/script/lua_api/l_async_events.cpp create mode 100644 src/script/lua_api/l_async_events.h create mode 100644 src/script/lua_api/marshall.c (limited to 'src/script') diff --git a/src/script/lua_api/CMakeLists.txt b/src/script/lua_api/CMakeLists.txt index 08960d2ad..0b89df6a3 100644 --- a/src/script/lua_api/CMakeLists.txt +++ b/src/script/lua_api/CMakeLists.txt @@ -16,6 +16,8 @@ set(common_SCRIPT_LUA_API_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/l_util.cpp ${CMAKE_CURRENT_SOURCE_DIR}/l_vmanip.cpp ${CMAKE_CURRENT_SOURCE_DIR}/l_settings.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/l_async_events.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/marshall.c PARENT_SCOPE) # Used by client only diff --git a/src/script/lua_api/l_async_events.cpp b/src/script/lua_api/l_async_events.cpp new file mode 100644 index 000000000..cc4644cdf --- /dev/null +++ b/src/script/lua_api/l_async_events.cpp @@ -0,0 +1,396 @@ +/* +Minetest +Copyright (C) 2013 sapier, + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU Lesser General Public License as published by +the Free Software Foundation; either version 2.1 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public License along +with this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +*/ + +extern "C" { +#include "lua.h" +#include "lauxlib.h" +#include "lualib.h" +int luaopen_marshal(lua_State *L); +} +#include + +#include "l_async_events.h" +#include "log.h" +#include "filesys.h" +#include "porting.h" + +//TODO replace by ShadowNinja version not yet merged to master +static int script_error_handler(lua_State *L) { + lua_getfield(L, LUA_GLOBALSINDEX, "debug"); + if (!lua_istable(L, -1)) { + lua_pop(L, 1); + return 1; + } + lua_getfield(L, -1, "traceback"); + if (!lua_isfunction(L, -1)) { + lua_pop(L, 2); + return 1; + } + lua_pushvalue(L, 1); + lua_pushinteger(L, 2); + lua_call(L, 2, 1); + return 1; +} + +/******************************************************************************/ +static void scriptError(const char *fmt, ...) +{ + va_list argp; + va_start(argp, fmt); + char buf[10000]; + vsnprintf(buf, 10000, fmt, argp); + va_end(argp); + errorstream<<"ERROR: "<::iterator i= m_WorkerThreads.begin(); + i != m_WorkerThreads.end();i++) { + (*i)->Stop(); + } + + + /** wakeup all threads **/ + for (std::vector::iterator i= m_WorkerThreads.begin(); + i != m_WorkerThreads.end();i++) { + m_JobQueueCounter.Post(); + } + + /** wait for threads to finish **/ + for (std::vector::iterator i= m_WorkerThreads.begin(); + i != m_WorkerThreads.end();i++) { + (*i)->Wait(); + } + + /** force kill all threads **/ + for (std::vector::iterator i= m_WorkerThreads.begin(); + i != m_WorkerThreads.end();i++) { + (*i)->Kill(); + delete *i; + } + + m_JobQueueMutex.Lock(); + m_JobQueue.clear(); + m_JobQueueMutex.Unlock(); + m_WorkerThreads.clear(); +} + +/******************************************************************************/ +bool AsyncEngine::registerFunction(const char* name, lua_CFunction fct) { + + if (m_initDone) return false; + m_FunctionList[name] = fct; + return true; +} + +/******************************************************************************/ +void AsyncEngine::Initialize(unsigned int numengines) { + m_initDone = true; + + for (unsigned int i=0; i < numengines; i ++) { + + AsyncWorkerThread* toadd = new AsyncWorkerThread(this,i); + m_WorkerThreads.push_back(toadd); + toadd->Start(); + } +} + +/******************************************************************************/ +unsigned int AsyncEngine::doAsyncJob(std::string fct, std::string params) { + + m_JobQueueMutex.Lock(); + LuaJobInfo toadd; + toadd.JobId = m_JobIdCounter++; + toadd.serializedFunction = fct; + toadd.serializedParams = params; + + m_JobQueue.push_back(toadd); + + m_JobQueueCounter.Post(); + + m_JobQueueMutex.Unlock(); + + return toadd.JobId; +} + +/******************************************************************************/ +LuaJobInfo AsyncEngine::getJob() { + + m_JobQueueCounter.Wait(); + m_JobQueueMutex.Lock(); + + LuaJobInfo retval; + + if (m_JobQueue.size() != 0) { + retval = m_JobQueue.front(); + m_JobQueue.erase((m_JobQueue.begin())); + } + m_JobQueueMutex.Unlock(); + + return retval; +} + +/******************************************************************************/ +void AsyncEngine::putJobResult(LuaJobInfo result) { + m_ResultQueueMutex.Lock(); + m_ResultQueue.push_back(result); + m_ResultQueueMutex.Unlock(); +} + +/******************************************************************************/ +void AsyncEngine::Step(lua_State *L) { + m_ResultQueueMutex.Lock(); + while(!m_ResultQueue.empty()) { + + LuaJobInfo jobdone = m_ResultQueue.front(); + m_ResultQueue.erase(m_ResultQueue.begin()); + lua_getglobal(L, "engine"); + + lua_getfield(L, -1, "async_event_handler"); + + if(lua_isnil(L, -1)) + assert("Someone managed to destroy a async callback in engine!" == 0); + + luaL_checktype(L, -1, LUA_TFUNCTION); + + lua_pushinteger(L, jobdone.JobId); + lua_pushlstring(L, jobdone.serializedResult.c_str(), + jobdone.serializedResult.length()); + + if(lua_pcall(L, 2, 0, 0)) { + scriptError("Async ENGINE step: %s", lua_tostring(L, -1)); + } + + lua_pop(L,1); + } + m_ResultQueueMutex.Unlock(); +} + +/******************************************************************************/ +void AsyncEngine::PushFinishedJobs(lua_State* L) { + //Result Table + m_ResultQueueMutex.Lock(); + + unsigned int index=1; + lua_newtable(L); + int top = lua_gettop(L); + + while(!m_ResultQueue.empty()) { + + LuaJobInfo jobdone = m_ResultQueue.front(); + m_ResultQueue.erase(m_ResultQueue.begin()); + + lua_pushnumber(L,index); + + lua_newtable(L); + int top_lvl2 = lua_gettop(L); + + lua_pushstring(L,"jobid"); + lua_pushnumber(L,jobdone.JobId); + lua_settable(L, top_lvl2); + + lua_pushstring(L,"retval"); + lua_pushstring(L, jobdone.serializedResult.c_str()); + lua_settable(L, top_lvl2); + + lua_settable(L, top); + index++; + } + + m_ResultQueueMutex.Unlock(); + +} +/******************************************************************************/ +void AsyncEngine::PrepareEnvironment(lua_State* L, int top) { + for(std::map::iterator i = m_FunctionList.begin(); + i != m_FunctionList.end(); i++) { + + lua_pushstring(L,i->first.c_str()); + lua_pushcfunction(L,i->second); + lua_settable(L, top); + + } +} + +/******************************************************************************/ +int async_worker_ErrorHandler(lua_State *L) { + lua_getfield(L, LUA_GLOBALSINDEX, "debug"); + if (!lua_istable(L, -1)) { + lua_pop(L, 1); + return 1; + } + lua_getfield(L, -1, "traceback"); + if (!lua_isfunction(L, -1)) { + lua_pop(L, 2); + return 1; + } + lua_pushvalue(L, 1); + lua_pushinteger(L, 2); + lua_call(L, 2, 1); + return 1; +} + +/******************************************************************************/ +AsyncWorkerThread::AsyncWorkerThread(AsyncEngine* jobdispatcher, + unsigned int numthreadnumber) : + m_JobDispatcher(jobdispatcher), + m_luaerrorhandler(-1), + m_threadnum(numthreadnumber) +{ + // create luastack + m_LuaStack = luaL_newstate(); + + // load basic lua modules + luaL_openlibs(m_LuaStack); + + // load serialization functions + luaopen_marshal(m_LuaStack); +} + +/******************************************************************************/ +AsyncWorkerThread::~AsyncWorkerThread() { + + assert(IsRunning() == false); + lua_close(m_LuaStack); +} + +/******************************************************************************/ +void* AsyncWorkerThread::worker_thread_main() { + + //register thread for error logging + char number[21]; + snprintf(number,sizeof(number),"%d",m_threadnum); + log_register_thread(std::string("AsyncWorkerThread_") + number); + + /** prepare job lua environment **/ + lua_newtable(m_LuaStack); + lua_setglobal(m_LuaStack, "engine"); + + lua_getglobal(m_LuaStack, "engine"); + int top = lua_gettop(m_LuaStack); + + lua_pushstring(m_LuaStack, DIR_DELIM); + lua_setglobal(m_LuaStack, "DIR_DELIM"); + + lua_pushstring(m_LuaStack, + std::string(porting::path_share + DIR_DELIM + "builtin").c_str()); + lua_setglobal(m_LuaStack, "SCRIPTDIR"); + + + m_JobDispatcher->PrepareEnvironment(m_LuaStack,top); + + std::string asyncscript = + porting::path_share + DIR_DELIM + "builtin" + + DIR_DELIM + "async_env.lua"; + + lua_pushcfunction(m_LuaStack, async_worker_ErrorHandler); + m_luaerrorhandler = lua_gettop(m_LuaStack); + + if(!runScript(asyncscript)) { + infostream + << "AsyncWorkderThread::worker_thread_main execution of async base environment failed!" + << std::endl; + assert("no future with broken builtin async environment scripts" == 0); + } + /** main loop **/ + while(IsRunning()) { + //wait for job + LuaJobInfo toprocess = m_JobDispatcher->getJob(); + + if (!IsRunning()) { continue; } + + //first push error handler + lua_pushcfunction(m_LuaStack, script_error_handler); + int errorhandler = lua_gettop(m_LuaStack); + + lua_getglobal(m_LuaStack, "engine"); + if(lua_isnil(m_LuaStack, -1)) + assert("unable to find engine within async environment" == 0); + + lua_getfield(m_LuaStack, -1, "job_processor"); + if(lua_isnil(m_LuaStack, -1)) + assert("Someone managed to destroy a async worker engine!" == 0); + + luaL_checktype(m_LuaStack, -1, LUA_TFUNCTION); + + //call it + lua_pushlstring(m_LuaStack, + toprocess.serializedFunction.c_str(), + toprocess.serializedFunction.length()); + lua_pushlstring(m_LuaStack, + toprocess.serializedParams.c_str(), + toprocess.serializedParams.length()); + + if (!IsRunning()) { continue; } + if(lua_pcall(m_LuaStack, 2, 2, errorhandler)) { + scriptError("Async WORKER thread: %s\n", lua_tostring(m_LuaStack, -1)); + toprocess.serializedResult="ERROR"; + } + else { + //fetch result + const char *retval = lua_tostring(m_LuaStack, -2); + unsigned int lenght = lua_tointeger(m_LuaStack,-1); + toprocess.serializedResult = std::string(retval,lenght); + } + + if (!IsRunning()) { continue; } + //put job result + m_JobDispatcher->putJobResult(toprocess); + } + log_deregister_thread(); + return 0; +} + +/******************************************************************************/ +bool AsyncWorkerThread::runScript(std::string script) { + + int ret = luaL_loadfile(m_LuaStack, script.c_str()) || + lua_pcall(m_LuaStack, 0, 0, m_luaerrorhandler); + if(ret){ + errorstream<<"==== ERROR FROM LUA WHILE INITIALIZING ASYNC ENVIRONMENT ====="<worker_thread_main(); +} diff --git a/src/script/lua_api/l_async_events.h b/src/script/lua_api/l_async_events.h new file mode 100644 index 000000000..4aaf3bdfe --- /dev/null +++ b/src/script/lua_api/l_async_events.h @@ -0,0 +1,228 @@ +/* +Minetest +Copyright (C) 2013 sapier, + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU Lesser General Public License as published by +the Free Software Foundation; either version 2.1 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public License along +with this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +*/ + +#ifndef C_ASYNC_EVENTS_H_ +#define C_ASYNC_EVENTS_H_ + +#include +#include +#include + +/******************************************************************************/ +/* Includes */ +/******************************************************************************/ +#include "jthread/jthread.h" +#include "jthread/jmutex.h" +#include "jthread/jsemaphore.h" +#include "debug.h" +#include "lua.h" + +/******************************************************************************/ +/* Typedefs and macros */ +/******************************************************************************/ +#define MAINMENU_NUMBER_OF_ASYNC_THREADS 4 + +/******************************************************************************/ +/* forward declarations */ +/******************************************************************************/ +class AsyncEngine; + +/******************************************************************************/ +/* declarations */ +/******************************************************************************/ + +/** a struct to encapsulate data required to queue a job **/ +struct LuaJobInfo { + /** function to be called in async environment **/ + std::string serializedFunction; + /** parameter table to be passed to function **/ + std::string serializedParams; + /** result of function call **/ + std::string serializedResult; + /** jobid used to identify a job and match it to callback **/ + unsigned int JobId; +}; + +/** class encapsulating a asynchronous working environment **/ +class AsyncWorkerThread : public JThread { +public: + /** + * default constructor + * @param pointer to job dispatcher + */ + AsyncWorkerThread(AsyncEngine* jobdispatcher, unsigned int threadnumber); + + /** + * default destructor + */ + virtual ~AsyncWorkerThread(); + + /** + * thread function + */ + void* Thread() { + ThreadStarted(); + return worker_thread_wrapper(this); + } + + /** + * wait for thread to stop + */ + void Wait() { + while(IsRunning()) { + sleep(1); + } + } + +private: + /** + * helper function to run a lua script + * @param path of script + */ + bool runScript(std::string script); + + /** + * main function of thread + */ + void* worker_thread_main(); + + /** + * static wrapper for thread creation + * @param this pointer to the thread to be created + */ + static void* worker_thread_wrapper(void* thread); + + /** + * pointer to job dispatcher + */ + AsyncEngine* m_JobDispatcher; + + /** + * the lua stack to run at + */ + lua_State* m_LuaStack; + + /** + * lua internal stack number of error handler + */ + int m_luaerrorhandler; + + /** + * thread number used for debug output + */ + unsigned int m_threadnum; + +}; + +/** asynchornous thread and job management **/ +class AsyncEngine { + friend AsyncWorkerThread; +public: + /** + * default constructor + */ + AsyncEngine(); + /** + * default destructor + */ + ~AsyncEngine(); + + /** + * register function to be used within engines + * @param name function name to be used within lua environment + * @param fct c-function to be called + */ + bool registerFunction(const char* name, lua_CFunction fct); + + /** + * create async engine tasks and lock function registration + * @param numengines number of async threads to be started + */ + void Initialize(unsigned int numengines); + + /** + * queue/run a async job + * @param fct serialized lua function + * @param params serialized parameters + * @return jobid the job is queued + */ + unsigned int doAsyncJob(std::string fct, std::string params); + + /** + * engine step to process finished jobs + * the engine step is one way to pass events back, PushFinishedJobs another + * @param L the lua environment to do the step in + */ + void Step(lua_State *L); + + + void PushFinishedJobs(lua_State* L); + +protected: + /** + * Get a Job from queue to be processed + * this function blocks until a job is ready + * @return a job to be processed + */ + LuaJobInfo getJob(); + + /** + * put a Job result back to result queue + * @param result result of completed job + */ + void putJobResult(LuaJobInfo result); + + /** + * initialize environment with current registred functions + * this function adds all functions registred by registerFunction to the + * passed lua stack + * @param L lua stack to initialize + * @param top stack position + */ + void PrepareEnvironment(lua_State* L, int top); + +private: + + /** variable locking the engine against further modification **/ + bool m_initDone; + + /** internal store for registred functions **/ + std::map m_FunctionList; + + /** internal counter to create job id's **/ + unsigned int m_JobIdCounter; + + /** mutex to protect job queue **/ + JMutex m_JobQueueMutex; + /** job queue **/ + std::vector m_JobQueue; + + /** mutext to protect result queue **/ + JMutex m_ResultQueueMutex; + /** result queue **/ + std::vector m_ResultQueue; + + /** list of current worker threads **/ + std::vector m_WorkerThreads; + + /** counter semaphore for job dispatching **/ + JSemaphore m_JobQueueCounter; +}; + +#endif /* C_ASYNC_EVENTS_H_ */ diff --git a/src/script/lua_api/l_internal.h b/src/script/lua_api/l_internal.h index 14215ee5d..5936ac046 100644 --- a/src/script/lua_api/l_internal.h +++ b/src/script/lua_api/l_internal.h @@ -30,14 +30,16 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "common/c_internal.h" #define luamethod(class, name) {#name, class::l_##name} -#define API_FCT(name) registerFunction(L,#name,l_##name,top) +#define API_FCT(name) registerFunction(L, #name, l_##name,top) +#define ASYNC_API_FCT(name) engine.registerFunction(#name, l_##name) #if (defined(WIN32) || defined(_WIN32_WCE)) #define NO_MAP_LOCK_REQUIRED #else #include "main.h" #include "profiler.h" -#define NO_MAP_LOCK_REQUIRED ScopeProfiler nolocktime(g_profiler,"Scriptapi: unlockable time",SPT_ADD) +#define NO_MAP_LOCK_REQUIRED \ + ScopeProfiler nolocktime(g_profiler,"Scriptapi: unlockable time",SPT_ADD) #endif #endif /* L_INTERNAL_H_ */ diff --git a/src/script/lua_api/l_mainmenu.cpp b/src/script/lua_api/l_mainmenu.cpp index 498ac0383..42ddd0b14 100644 --- a/src/script/lua_api/l_mainmenu.cpp +++ b/src/script/lua_api/l_mainmenu.cpp @@ -20,6 +20,7 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "lua_api/l_mainmenu.h" #include "lua_api/l_internal.h" #include "common/c_content.h" +#include "lua_api/l_async_events.h" #include "guiEngine.h" #include "guiMainMenu.h" #include "guiKeyChangeMenu.h" @@ -200,9 +201,6 @@ int ModApiMainMenu::l_get_textlist_index(lua_State *L) /******************************************************************************/ int ModApiMainMenu::l_get_worlds(lua_State *L) { - GUIEngine* engine = getGuiEngine(L); - assert(engine != 0); - std::vector worlds = getAvailableWorlds(); lua_newtable(L); @@ -237,9 +235,6 @@ int ModApiMainMenu::l_get_worlds(lua_State *L) /******************************************************************************/ int ModApiMainMenu::l_get_games(lua_State *L) { - GUIEngine* engine = getGuiEngine(L); - assert(engine != 0); - std::vector games = getAvailableGames(); lua_newtable(L); @@ -365,9 +360,6 @@ int ModApiMainMenu::l_get_modstore_details(lua_State *L) /******************************************************************************/ int ModApiMainMenu::l_get_modstore_list(lua_State *L) { - GUIEngine* engine = getGuiEngine(L); - assert(engine != 0); - std::string listtype = "local"; if (!lua_isnone(L,1)) { @@ -421,9 +413,6 @@ int ModApiMainMenu::l_get_modstore_list(lua_State *L) /******************************************************************************/ int ModApiMainMenu::l_get_favorites(lua_State *L) { - GUIEngine* engine = getGuiEngine(L); - assert(engine != 0); - std::string listtype = "local"; if (!lua_isnone(L,1)) { @@ -545,9 +534,6 @@ int ModApiMainMenu::l_get_favorites(lua_State *L) /******************************************************************************/ int ModApiMainMenu::l_delete_favorite(lua_State *L) { - GUIEngine* engine = getGuiEngine(L); - assert(engine != 0); - std::vector servers; std::string listtype = "local"; @@ -599,9 +585,6 @@ int ModApiMainMenu::l_show_keys_menu(lua_State *L) /******************************************************************************/ int ModApiMainMenu::l_create_world(lua_State *L) { - GUIEngine* engine = getGuiEngine(L); - assert(engine != 0); - const char *name = luaL_checkstring(L, 1); int gameidx = luaL_checkinteger(L,2) -1; @@ -632,9 +615,6 @@ int ModApiMainMenu::l_create_world(lua_State *L) /******************************************************************************/ int ModApiMainMenu::l_delete_world(lua_State *L) { - GUIEngine* engine = getGuiEngine(L); - assert(engine != 0); - int worldidx = luaL_checkinteger(L,1) -1; std::vector worlds = getAvailableWorlds(); @@ -962,9 +942,6 @@ int ModApiMainMenu::l_sound_stop(lua_State *L) /******************************************************************************/ int ModApiMainMenu::l_download_file(lua_State *L) { - GUIEngine* engine = getGuiEngine(L); - assert(engine != 0); - const char *url = luaL_checkstring(L, 1); const char *target = luaL_checkstring(L, 2); @@ -972,7 +949,7 @@ int ModApiMainMenu::l_download_file(lua_State *L) std::string absolute_destination = fs::RemoveRelativePathComponents(target); if (ModApiMainMenu::isMinetestPath(absolute_destination)) { - if (engine->downloadFile(url,absolute_destination)) { + if (GUIEngine::downloadFile(url,absolute_destination)) { lua_pushboolean(L,true); return 1; } @@ -990,6 +967,28 @@ int ModApiMainMenu::l_gettext(lua_State *L) return 1; } +/******************************************************************************/ +int ModApiMainMenu::l_do_async_callback(lua_State *L) +{ + GUIEngine* engine = getGuiEngine(L); + + const char* serialized_fct_raw = luaL_checkstring(L, 1); + unsigned int lenght_fct = luaL_checkint(L, 2); + + const char* serialized_params_raw = luaL_checkstring(L, 3); + unsigned int lenght_params = luaL_checkint(L, 4); + + assert(serialized_fct_raw != 0); + assert(serialized_params_raw != 0); + + std::string serialized_fct = std::string(serialized_fct_raw,lenght_fct); + std::string serialized_params = std::string(serialized_params_raw,lenght_params); + + lua_pushinteger(L,engine->DoAsync(serialized_fct,serialized_params)); + + return 1; +} + /******************************************************************************/ void ModApiMainMenu::Initialize(lua_State *L, int top) { @@ -1024,4 +1023,27 @@ void ModApiMainMenu::Initialize(lua_State *L, int top) API_FCT(sound_play); API_FCT(sound_stop); API_FCT(gettext); + API_FCT(do_async_callback); +} + +/******************************************************************************/ +void ModApiMainMenu::InitializeAsync(AsyncEngine& engine) +{ + + ASYNC_API_FCT(get_worlds); + ASYNC_API_FCT(get_games); + ASYNC_API_FCT(get_favorites); + ASYNC_API_FCT(get_modpath); + ASYNC_API_FCT(get_gamepath); + ASYNC_API_FCT(get_texturepath); + ASYNC_API_FCT(get_dirlist); + ASYNC_API_FCT(create_dir); + ASYNC_API_FCT(delete_dir); + ASYNC_API_FCT(copy_dir); + //ASYNC_API_FCT(extract_zip); //TODO remove dependency to GuiEngine + ASYNC_API_FCT(get_version); + ASYNC_API_FCT(download_file); + ASYNC_API_FCT(get_modstore_details); + ASYNC_API_FCT(get_modstore_list); + //ASYNC_API_FCT(gettext); (gettext lib isn't threadsafe) } diff --git a/src/script/lua_api/l_mainmenu.h b/src/script/lua_api/l_mainmenu.h index d0f3d6f72..e185f0a37 100644 --- a/src/script/lua_api/l_mainmenu.h +++ b/src/script/lua_api/l_mainmenu.h @@ -22,6 +22,8 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "lua_api/l_base.h" +class AsyncEngine; + /** Implementation of lua api support for mainmenu */ class ModApiMainMenu : public ModApiBase { @@ -125,6 +127,8 @@ private: static int l_download_file(lua_State *L); + // async + static int l_do_async_callback(lua_State *L); public: /** @@ -134,6 +138,8 @@ public: */ static void Initialize(lua_State *L, int top); + static void InitializeAsync(AsyncEngine& engine); + }; #endif /* L_MAINMENU_H_ */ diff --git a/src/script/lua_api/l_util.cpp b/src/script/lua_api/l_util.cpp index af9c19210..fe10e4f57 100644 --- a/src/script/lua_api/l_util.cpp +++ b/src/script/lua_api/l_util.cpp @@ -21,6 +21,7 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "lua_api/l_internal.h" #include "common/c_converter.h" #include "common/c_content.h" +#include "lua_api/l_async_events.h" #include "debug.h" #include "log.h" #include "tool.h" @@ -257,3 +258,18 @@ void ModApiUtil::Initialize(lua_State *L, int top) API_FCT(is_yes); } +void ModApiUtil::InitializeAsync(AsyncEngine& engine) +{ + ASYNC_API_FCT(debug); + ASYNC_API_FCT(log); + + //ASYNC_API_FCT(setting_set); + ASYNC_API_FCT(setting_get); + //ASYNC_API_FCT(setting_setbool); + ASYNC_API_FCT(setting_getbool); + //ASYNC_API_FCT(setting_save); + + ASYNC_API_FCT(parse_json); + + ASYNC_API_FCT(is_yes); +} diff --git a/src/script/lua_api/l_util.h b/src/script/lua_api/l_util.h index bb99e1562..d91c880cf 100644 --- a/src/script/lua_api/l_util.h +++ b/src/script/lua_api/l_util.h @@ -22,6 +22,8 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "lua_api/l_base.h" +class AsyncEngine; + class ModApiUtil : public ModApiBase { private: /* @@ -77,6 +79,8 @@ private: public: static void Initialize(lua_State *L, int top); + static void InitializeAsync(AsyncEngine& engine); + }; #endif /* L_UTIL_H_ */ diff --git a/src/script/lua_api/marshall.c b/src/script/lua_api/marshall.c new file mode 100644 index 000000000..ef70566cb --- /dev/null +++ b/src/script/lua_api/marshall.c @@ -0,0 +1,551 @@ +/* +* lmarshal.c +* A Lua library for serializing and deserializing Lua values +* Richard Hundt +* +* License: MIT +* +* Copyright (c) 2010 Richard Hundt +* +* Permission is hereby granted, free of charge, to any person +* obtaining a copy of this software and associated documentation +* files (the "Software"), to deal in the Software without +* restriction, including without limitation the rights to use, +* copy, modify, merge, publish, distribute, sublicense, and/or sell +* copies of the Software, and to permit persons to whom the +* Software is furnished to do so, subject to the following +* conditions: +* +* The above copyright notice and this permission notice shall be +* included in all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +* OTHER DEALINGS IN THE SOFTWARE. +*/ + +#include +#include +#include + +#include "lua.h" +#include "lualib.h" +#include "lauxlib.h" + + +#define MAR_TREF 1 +#define MAR_TVAL 2 +#define MAR_TUSR 3 + +#define MAR_CHR 1 +#define MAR_I32 4 +#define MAR_I64 8 + +#define MAR_MAGIC 0x8e +#define SEEN_IDX 3 + +typedef struct mar_Buffer { + size_t size; + size_t seek; + size_t head; + char* data; +} mar_Buffer; + +static int mar_encode_table(lua_State *L, mar_Buffer *buf, size_t *idx); +static int mar_decode_table(lua_State *L, const char* buf, size_t len, size_t *idx); + +static void buf_init(lua_State *L, mar_Buffer *buf) +{ + buf->size = 128; + buf->seek = 0; + buf->head = 0; + if (!(buf->data = malloc(buf->size))) luaL_error(L, "Out of memory!"); +} + +static void buf_done(lua_State* L, mar_Buffer *buf) +{ + free(buf->data); +} + +static int buf_write(lua_State* L, const char* str, size_t len, mar_Buffer *buf) +{ + if (len > UINT32_MAX) luaL_error(L, "buffer too long"); + if (buf->size - buf->head < len) { + size_t new_size = buf->size << 1; + size_t cur_head = buf->head; + while (new_size - cur_head <= len) { + new_size = new_size << 1; + } + if (!(buf->data = realloc(buf->data, new_size))) { + luaL_error(L, "Out of memory!"); + } + buf->size = new_size; + } + memcpy(&buf->data[buf->head], str, len); + buf->head += len; + return 0; +} + +static const char* buf_read(lua_State *L, mar_Buffer *buf, size_t *len) +{ + if (buf->seek < buf->head) { + buf->seek = buf->head; + *len = buf->seek; + return buf->data; + } + *len = 0; + return NULL; +} + +static void mar_encode_value(lua_State *L, mar_Buffer *buf, int val, size_t *idx) +{ + size_t l; + int val_type = lua_type(L, val); + lua_pushvalue(L, val); + + buf_write(L, (void*)&val_type, MAR_CHR, buf); + switch (val_type) { + case LUA_TBOOLEAN: { + int int_val = lua_toboolean(L, -1); + buf_write(L, (void*)&int_val, MAR_CHR, buf); + break; + } + case LUA_TSTRING: { + const char *str_val = lua_tolstring(L, -1, &l); + buf_write(L, (void*)&l, MAR_I32, buf); + buf_write(L, str_val, l, buf); + break; + } + case LUA_TNUMBER: { + lua_Number num_val = lua_tonumber(L, -1); + buf_write(L, (void*)&num_val, MAR_I64, buf); + break; + } + case LUA_TTABLE: { + int tag, ref; + lua_pushvalue(L, -1); + lua_rawget(L, SEEN_IDX); + if (!lua_isnil(L, -1)) { + ref = lua_tointeger(L, -1); + tag = MAR_TREF; + buf_write(L, (void*)&tag, MAR_CHR, buf); + buf_write(L, (void*)&ref, MAR_I32, buf); + lua_pop(L, 1); + } + else { + mar_Buffer rec_buf; + lua_pop(L, 1); /* pop nil */ + if (luaL_getmetafield(L, -1, "__persist")) { + tag = MAR_TUSR; + + lua_pushvalue(L, -2); /* self */ + lua_call(L, 1, 1); + if (!lua_isfunction(L, -1)) { + luaL_error(L, "__persist must return a function"); + } + + lua_remove(L, -2); /* __persist */ + + lua_newtable(L); + lua_pushvalue(L, -2); /* callback */ + lua_rawseti(L, -2, 1); + + buf_init(L, &rec_buf); + mar_encode_table(L, &rec_buf, idx); + + buf_write(L, (void*)&tag, MAR_CHR, buf); + buf_write(L, (void*)&rec_buf.head, MAR_I32, buf); + buf_write(L, rec_buf.data, rec_buf.head, buf); + buf_done(L, &rec_buf); + lua_pop(L, 1); + } + else { + tag = MAR_TVAL; + + lua_pushvalue(L, -1); + lua_pushinteger(L, (*idx)++); + lua_rawset(L, SEEN_IDX); + + lua_pushvalue(L, -1); + buf_init(L, &rec_buf); + mar_encode_table(L, &rec_buf, idx); + lua_pop(L, 1); + + buf_write(L, (void*)&tag, MAR_CHR, buf); + buf_write(L, (void*)&rec_buf.head, MAR_I32, buf); + buf_write(L, rec_buf.data,rec_buf.head, buf); + buf_done(L, &rec_buf); + } + } + break; + } + case LUA_TFUNCTION: { + int tag, ref; + lua_pushvalue(L, -1); + lua_rawget(L, SEEN_IDX); + if (!lua_isnil(L, -1)) { + ref = lua_tointeger(L, -1); + tag = MAR_TREF; + buf_write(L, (void*)&tag, MAR_CHR, buf); + buf_write(L, (void*)&ref, MAR_I32, buf); + lua_pop(L, 1); + } + else { + mar_Buffer rec_buf; + int i; + lua_Debug ar; + lua_pop(L, 1); /* pop nil */ + + lua_pushvalue(L, -1); + lua_getinfo(L, ">nuS", &ar); + if (ar.what[0] != 'L') { + luaL_error(L, "attempt to persist a C function '%s'", ar.name); + } + tag = MAR_TVAL; + lua_pushvalue(L, -1); + lua_pushinteger(L, (*idx)++); + lua_rawset(L, SEEN_IDX); + + lua_pushvalue(L, -1); + buf_init(L, &rec_buf); + lua_dump(L, (lua_Writer)buf_write, &rec_buf); + + buf_write(L, (void*)&tag, MAR_CHR, buf); + buf_write(L, (void*)&rec_buf.head, MAR_I32, buf); + buf_write(L, rec_buf.data, rec_buf.head, buf); + buf_done(L, &rec_buf); + lua_pop(L, 1); + + lua_newtable(L); + for (i=1; i <= ar.nups; i++) { + lua_getupvalue(L, -2, i); + lua_rawseti(L, -2, i); + } + + buf_init(L, &rec_buf); + mar_encode_table(L, &rec_buf, idx); + + buf_write(L, (void*)&rec_buf.head, MAR_I32, buf); + buf_write(L, rec_buf.data, rec_buf.head, buf); + buf_done(L, &rec_buf); + lua_pop(L, 1); + } + + break; + } + case LUA_TUSERDATA: { + int tag, ref; + lua_pushvalue(L, -1); + lua_rawget(L, SEEN_IDX); + if (!lua_isnil(L, -1)) { + ref = lua_tointeger(L, -1); + tag = MAR_TREF; + buf_write(L, (void*)&tag, MAR_CHR, buf); + buf_write(L, (void*)&ref, MAR_I32, buf); + lua_pop(L, 1); + } + else { + mar_Buffer rec_buf; + lua_pop(L, 1); /* pop nil */ + if (luaL_getmetafield(L, -1, "__persist")) { + tag = MAR_TUSR; + + lua_pushvalue(L, -2); + lua_pushinteger(L, (*idx)++); + lua_rawset(L, SEEN_IDX); + + lua_pushvalue(L, -2); + lua_call(L, 1, 1); + if (!lua_isfunction(L, -1)) { + luaL_error(L, "__persist must return a function"); + } + lua_newtable(L); + lua_pushvalue(L, -2); + lua_rawseti(L, -2, 1); + lua_remove(L, -2); + + buf_init(L, &rec_buf); + mar_encode_table(L, &rec_buf, idx); + + buf_write(L, (void*)&tag, MAR_CHR, buf); + buf_write(L, (void*)&rec_buf.head, MAR_I32, buf); + buf_write(L, rec_buf.data, rec_buf.head, buf); + buf_done(L, &rec_buf); + } + else { + luaL_error(L, "attempt to encode userdata (no __persist hook)"); + } + lua_pop(L, 1); + } + break; + } + case LUA_TNIL: break; + default: + luaL_error(L, "invalid value type (%s)", lua_typename(L, val_type)); + } + lua_pop(L, 1); +} + +static int mar_encode_table(lua_State *L, mar_Buffer *buf, size_t *idx) +{ + lua_pushnil(L); + while (lua_next(L, -2) != 0) { + mar_encode_value(L, buf, -2, idx); + mar_encode_value(L, buf, -1, idx); + lua_pop(L, 1); + } + return 1; +} + +#define mar_incr_ptr(l) \ + if (((*p)-buf)+(l) > len) luaL_error(L, "bad code"); (*p) += (l); + +#define mar_next_len(l,T) \ + if (((*p)-buf)+sizeof(T) > len) luaL_error(L, "bad code"); \ + l = *(T*)*p; (*p) += sizeof(T); + +static void mar_decode_value + (lua_State *L, const char *buf, size_t len, const char **p, size_t *idx) +{ + size_t l; + char val_type = **p; + mar_incr_ptr(MAR_CHR); + switch (val_type) { + case LUA_TBOOLEAN: + lua_pushboolean(L, *(char*)*p); + mar_incr_ptr(MAR_CHR); + break; + case LUA_TNUMBER: + lua_pushnumber(L, *(lua_Number*)*p); + mar_incr_ptr(MAR_I64); + break; + case LUA_TSTRING: + mar_next_len(l, uint32_t); + lua_pushlstring(L, *p, l); + mar_incr_ptr(l); + break; + case LUA_TTABLE: { + char tag = *(char*)*p; + mar_incr_ptr(MAR_CHR); + if (tag == MAR_TREF) { + int ref; + mar_next_len(ref, int); + lua_rawgeti(L, SEEN_IDX, ref); + } + else if (tag == MAR_TVAL) { + mar_next_len(l, uint32_t); + lua_newtable(L); + lua_pushvalue(L, -1); + lua_rawseti(L, SEEN_IDX, (*idx)++); + mar_decode_table(L, *p, l, idx); + mar_incr_ptr(l); + } + else if (tag == MAR_TUSR) { + mar_next_len(l, uint32_t); + lua_newtable(L); + mar_decode_table(L, *p, l, idx); + lua_rawgeti(L, -1, 1); + lua_call(L, 0, 1); + lua_remove(L, -2); + lua_pushvalue(L, -1); + lua_rawseti(L, SEEN_IDX, (*idx)++); + mar_incr_ptr(l); + } + else { + luaL_error(L, "bad encoded data"); + } + break; + } + case LUA_TFUNCTION: { + size_t nups; + int i; + mar_Buffer dec_buf; + char tag = *(char*)*p; + mar_incr_ptr(1); + if (tag == MAR_TREF) { + int ref; + mar_next_len(ref, int); + lua_rawgeti(L, SEEN_IDX, ref); + } + else { + mar_next_len(l, uint32_t); + dec_buf.data = (char*)*p; + dec_buf.size = l; + dec_buf.head = l; + dec_buf.seek = 0; + lua_load(L, (lua_Reader)buf_read, &dec_buf, "=marshal"); + mar_incr_ptr(l); + + lua_pushvalue(L, -1); + lua_rawseti(L, SEEN_IDX, (*idx)++); + + mar_next_len(l, uint32_t); + lua_newtable(L); + mar_decode_table(L, *p, l, idx); + nups = lua_objlen(L, -1); + for (i=1; i <= nups; i++) { + lua_rawgeti(L, -1, i); + lua_setupvalue(L, -3, i); + } + lua_pop(L, 1); + mar_incr_ptr(l); + } + break; + } + case LUA_TUSERDATA: { + char tag = *(char*)*p; + mar_incr_ptr(MAR_CHR); + if (tag == MAR_TREF) { + int ref; + mar_next_len(ref, int); + lua_rawgeti(L, SEEN_IDX, ref); + } + else if (tag == MAR_TUSR) { + mar_next_len(l, uint32_t); + lua_newtable(L); + mar_decode_table(L, *p, l, idx); + lua_rawgeti(L, -1, 1); + lua_call(L, 0, 1); + lua_remove(L, -2); + lua_pushvalue(L, -1); + lua_rawseti(L, SEEN_IDX, (*idx)++); + mar_incr_ptr(l); + } + else { /* tag == MAR_TVAL */ + lua_pushnil(L); + } + break; + } + case LUA_TNIL: + case LUA_TTHREAD: + lua_pushnil(L); + break; + default: + luaL_error(L, "bad code"); + } +} + +static int mar_decode_table(lua_State *L, const char* buf, size_t len, size_t *idx) +{ + const char* p; + p = buf; + while (p - buf < len) { + mar_decode_value(L, buf, len, &p, idx); + mar_decode_value(L, buf, len, &p, idx); + lua_settable(L, -3); + } + return 1; +} + +static int mar_encode(lua_State* L) +{ + const unsigned char m = MAR_MAGIC; + size_t idx, len; + mar_Buffer buf; + + if (lua_isnone(L, 1)) { + lua_pushnil(L); + } + if (lua_isnoneornil(L, 2)) { + lua_newtable(L); + } + else if (!lua_istable(L, 2)) { + luaL_error(L, "bad argument #2 to encode (expected table)"); + } + lua_settop(L, 2); + + len = lua_objlen(L, 2); + lua_newtable(L); + for (idx = 1; idx <= len; idx++) { + lua_rawgeti(L, 2, idx); + if (lua_isnil(L, -1)) { + lua_pop(L, 1); + continue; + } + lua_pushinteger(L, idx); + lua_rawset(L, SEEN_IDX); + } + lua_pushvalue(L, 1); + + buf_init(L, &buf); + buf_write(L, (void*)&m, 1, &buf); + + mar_encode_value(L, &buf, -1, &idx); + + lua_pop(L, 1); + + lua_pushlstring(L, buf.data, buf.head); + + buf_done(L, &buf); + + lua_remove(L, SEEN_IDX); + + return 1; +} + +static int mar_decode(lua_State* L) +{ + size_t l, idx, len; + const char *p; + const char *s = luaL_checklstring(L, 1, &l); + + if (l < 1) luaL_error(L, "bad header"); + if (*(unsigned char *)s++ != MAR_MAGIC) luaL_error(L, "bad magic"); + l -= 1; + + if (lua_isnoneornil(L, 2)) { + lua_newtable(L); + } + else if (!lua_istable(L, 2)) { + luaL_error(L, "bad argument #2 to decode (expected table)"); + } + lua_settop(L, 2); + + len = lua_objlen(L, 2); + lua_newtable(L); + for (idx = 1; idx <= len; idx++) { + lua_rawgeti(L, 2, idx); + lua_rawseti(L, SEEN_IDX, idx); + } + + p = s; + mar_decode_value(L, s, l, &p, &idx); + + lua_remove(L, SEEN_IDX); + lua_remove(L, 2); + + return 1; +} + +static int mar_clone(lua_State* L) +{ + mar_encode(L); + lua_replace(L, 1); + mar_decode(L); + return 1; +} + +static const luaL_reg R[] = +{ + {"encode", mar_encode}, + {"decode", mar_decode}, + {"clone", mar_clone}, + {NULL, NULL} +}; + +int luaopen_marshal(lua_State *L) +{ + lua_newtable(L); + luaL_register(L, "marshal", R); + return 1; +} + + + + + diff --git a/src/script/scripting_mainmenu.cpp b/src/script/scripting_mainmenu.cpp index 31581a1bf..a4619e9da 100644 --- a/src/script/scripting_mainmenu.cpp +++ b/src/script/scripting_mainmenu.cpp @@ -28,8 +28,9 @@ with this program; if not, write to the Free Software Foundation, Inc., extern "C" { #include "lualib.h" + int luaopen_marshal(lua_State *L); } - +/******************************************************************************/ MainMenuScripting::MainMenuScripting(GUIEngine* guiengine) { setGuiEngine(guiengine); @@ -37,6 +38,7 @@ MainMenuScripting::MainMenuScripting(GUIEngine* guiengine) //TODO add security luaL_openlibs(getStack()); + luaopen_marshal(getStack()); SCRIPTAPI_PRECHECKHEADER @@ -58,6 +60,7 @@ MainMenuScripting::MainMenuScripting(GUIEngine* guiengine) infostream << "SCRIPTAPI: initialized mainmenu modules" << std::endl; } +/******************************************************************************/ void MainMenuScripting::InitializeModApi(lua_State *L, int top) { // Initialize mod api modules @@ -66,4 +69,23 @@ void MainMenuScripting::InitializeModApi(lua_State *L, int top) // Register reference classes (userdata) LuaSettings::Register(L); + + // Register functions to async environment + ModApiMainMenu::InitializeAsync(m_AsyncEngine); + ModApiUtil::InitializeAsync(m_AsyncEngine); + + // Initialize async environment + //TODO possibly make number of async threads configurable + m_AsyncEngine.Initialize(MAINMENU_NUMBER_OF_ASYNC_THREADS); +} + +/******************************************************************************/ +void MainMenuScripting::Step() { + m_AsyncEngine.Step(getStack()); +} + +/******************************************************************************/ +unsigned int MainMenuScripting::DoAsync(std::string serialized_fct, + std::string serialized_params) { + return m_AsyncEngine.doAsyncJob(serialized_fct,serialized_params); } diff --git a/src/script/scripting_mainmenu.h b/src/script/scripting_mainmenu.h index 7592c8e23..f4d78f664 100644 --- a/src/script/scripting_mainmenu.h +++ b/src/script/scripting_mainmenu.h @@ -22,6 +22,7 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "cpp_api/s_base.h" #include "cpp_api/s_mainmenu.h" +#include "lua_api/l_async_events.h" /*****************************************************************************/ /* Scripting <-> Main Menu Interface */ @@ -37,8 +38,16 @@ public: // use ScriptApiBase::loadMod() or ScriptApiBase::loadScript() // to load scripts + /* global step handler to pass back async events */ + void Step(); + + /* pass async events from engine to async threads */ + unsigned int DoAsync(std::string serialized_fct, + std::string serialized_params); private: void InitializeModApi(lua_State *L, int top); + + AsyncEngine m_AsyncEngine; }; -- cgit v1.2.3