Remove dependency on marshal and many other async changes

This makes a number of changes:
  * Remove the dependency on marshal by using string.dump and loadstring.
  * Use lua_tolstring rather than having Lua functions pass string lengths to C++.
  * Move lua_api/l_async_events.* to cpp_api/s_async.*, where it belongs.
  * Make AsyncWorkerThread a child of ScriptApiBase, this removes some duplicate functionality.
  * Don't wait for async threads to shut down.  (Is this safe?  Might result in corruption if the thread is writing to a file.)
  * Pop more unused items from the stack
  * Code style fixes
  * Other misc changes

6 files changed, 10 insertions, 1154 deletions

diff --git a/src/script/lua_api/CMakeLists.txt b/src/script/lua_api/CMakeLists.txt
index 0b89df6a3..08960d2ad 100644
--- a/src/script/lua_api/CMakeLists.txt
+++ b/src/script/lua_api/CMakeLists.txt
@@ -16,8 +16,6 @@ set(common_SCRIPT_LUA_API_SRCS
 	${CMAKE_CURRENT_SOURCE_DIR}/l_util.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/l_vmanip.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/l_settings.cpp
-	${CMAKE_CURRENT_SOURCE_DIR}/l_async_events.cpp
-	${CMAKE_CURRENT_SOURCE_DIR}/marshall.c
 	PARENT_SCOPE)
 
 # Used by client only
diff --git a/src/script/lua_api/l_async_events.cpp b/src/script/lua_api/l_async_events.cpp
deleted file mode 100644
index f5c27a235..000000000
--- a/src/script/lua_api/l_async_events.cpp
+++ /dev/null
@@ -1,370 +0,0 @@
-/*
-Minetest
-Copyright (C) 2013 sapier, <sapier AT gmx DOT net>
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU Lesser General Public License as published by
-the Free Software Foundation; either version 2.1 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Lesser General Public License for more details.
-
-You should have received a copy of the GNU Lesser General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-*/
-
-extern "C" {
-#include "lua.h"
-#include "lauxlib.h"
-#include "lualib.h"
-int luaopen_marshal(lua_State *L);
-}
-#include <stdio.h>
-
-#include "l_async_events.h"
-#include "log.h"
-#include "filesys.h"
-#include "porting.h"
-#include "common/c_internal.h"
-
-/******************************************************************************/
-AsyncEngine::AsyncEngine() :
-	m_initDone(false),
-	m_JobIdCounter(0)
-{
-}
-
-/******************************************************************************/
-AsyncEngine::~AsyncEngine()
-{
-	/** request all threads to stop **/
-	for (std::vector<AsyncWorkerThread*>::iterator i= m_WorkerThreads.begin();
-			i != m_WorkerThreads.end();i++) {
-		(*i)->Stop();
-	}
-
-
-	/** wakeup all threads **/
-	for (std::vector<AsyncWorkerThread*>::iterator i= m_WorkerThreads.begin();
-				i != m_WorkerThreads.end();i++) {
-		m_JobQueueCounter.Post();
-	}
-
-	/** wait for threads to finish **/
-	for (std::vector<AsyncWorkerThread*>::iterator i= m_WorkerThreads.begin();
-			i != m_WorkerThreads.end();i++) {
-		(*i)->Wait();
-	}
-
-	/** force kill all threads **/
-	for (std::vector<AsyncWorkerThread*>::iterator i= m_WorkerThreads.begin();
-			i != m_WorkerThreads.end();i++) {
-		(*i)->Kill();
-		delete *i;
-	}
-
-	m_JobQueueMutex.Lock();
-	m_JobQueue.clear();
-	m_JobQueueMutex.Unlock();
-	m_WorkerThreads.clear();
-}
-
-/******************************************************************************/
-bool AsyncEngine::registerFunction(const char* name, lua_CFunction fct) {
-
-	if (m_initDone) return false;
-	m_FunctionList[name] = fct;
-	return true;
-}
-
-/******************************************************************************/
-void AsyncEngine::Initialize(unsigned int numengines) {
-	m_initDone = true;
-
-	for (unsigned int i=0; i < numengines; i ++) {
-
-		AsyncWorkerThread* toadd = new AsyncWorkerThread(this,i);
-		m_WorkerThreads.push_back(toadd);
-		toadd->Start();
-	}
-}
-
-/******************************************************************************/
-unsigned int AsyncEngine::doAsyncJob(std::string fct, std::string params) {
-
-	m_JobQueueMutex.Lock();
-	LuaJobInfo toadd;
-	toadd.JobId = m_JobIdCounter++;
-	toadd.serializedFunction = fct;
-	toadd.serializedParams = params;
-
-	m_JobQueue.push_back(toadd);
-
-	m_JobQueueCounter.Post();
-
-	m_JobQueueMutex.Unlock();
-
-	return toadd.JobId;
-}
-
-/******************************************************************************/
-LuaJobInfo AsyncEngine::getJob() {
-
-	m_JobQueueCounter.Wait();
-	m_JobQueueMutex.Lock();
-
-	LuaJobInfo retval;
-	retval.valid = false;
-
-	if (m_JobQueue.size() != 0) {
-		retval = m_JobQueue.front();
-		retval.valid = true;
-		m_JobQueue.erase((m_JobQueue.begin()));
-	}
-	m_JobQueueMutex.Unlock();
-
-	return retval;
-}
-
-/******************************************************************************/
-void AsyncEngine::putJobResult(LuaJobInfo result) {
-	m_ResultQueueMutex.Lock();
-	m_ResultQueue.push_back(result);
-	m_ResultQueueMutex.Unlock();
-}
-
-/******************************************************************************/
-void AsyncEngine::Step(lua_State *L) {
-	lua_pushcfunction(L, script_error_handler);
-	int errorhandler = lua_gettop(L);
-	lua_getglobal(L, "engine");
-	m_ResultQueueMutex.Lock();
-	while(!m_ResultQueue.empty()) {
-		LuaJobInfo jobdone = m_ResultQueue.front();
-		m_ResultQueue.erase(m_ResultQueue.begin());
-
-		lua_getfield(L, -1, "async_event_handler");
-
-		if(lua_isnil(L, -1))
-			assert("Someone managed to destroy a async callback in engine!" == 0);
-
-		luaL_checktype(L, -1, LUA_TFUNCTION);
-
-		lua_pushinteger(L, jobdone.JobId);
-		lua_pushlstring(L, jobdone.serializedResult.c_str(),
-				jobdone.serializedResult.length());
-
-		if(lua_pcall(L, 2, 0, errorhandler)) {
-			script_error(L);
-		}
-	}
-	m_ResultQueueMutex.Unlock();
-	lua_pop(L, 2); // Pop engine and error handler
-}
-
-/******************************************************************************/
-void AsyncEngine::PushFinishedJobs(lua_State* L) {
-	//Result Table
-	m_ResultQueueMutex.Lock();
-
-	unsigned int index=1;
-	lua_newtable(L);
-	int top = lua_gettop(L);
-
-	while(!m_ResultQueue.empty()) {
-
-		LuaJobInfo jobdone = m_ResultQueue.front();
-		m_ResultQueue.erase(m_ResultQueue.begin());
-
-		lua_pushnumber(L,index);
-
-		lua_newtable(L);
-		int top_lvl2 = lua_gettop(L);
-
-		lua_pushstring(L,"jobid");
-		lua_pushnumber(L,jobdone.JobId);
-		lua_settable(L, top_lvl2);
-
-		lua_pushstring(L,"retval");
-		lua_pushstring(L, jobdone.serializedResult.c_str());
-		lua_settable(L, top_lvl2);
-
-		lua_settable(L, top);
-		index++;
-	}
-
-	m_ResultQueueMutex.Unlock();
-
-}
-/******************************************************************************/
-void AsyncEngine::PrepareEnvironment(lua_State* L, int top) {
-	for(std::map<std::string,lua_CFunction>::iterator i = m_FunctionList.begin();
-			i != m_FunctionList.end(); i++) {
-
-		lua_pushstring(L,i->first.c_str());
-		lua_pushcfunction(L,i->second);
-		lua_settable(L, top);
-
-	}
-}
-
-/******************************************************************************/
-int async_worker_ErrorHandler(lua_State *L) {
-	lua_getfield(L, LUA_GLOBALSINDEX, "debug");
-	if (!lua_istable(L, -1)) {
-	lua_pop(L, 1);
-	return 1;
-	}
-	lua_getfield(L, -1, "traceback");
-	if (!lua_isfunction(L, -1)) {
-	lua_pop(L, 2);
-	return 1;
-	}
-	lua_pushvalue(L, 1);
-	lua_pushinteger(L, 2);
-	lua_call(L, 2, 1);
-	return 1;
-}
-
-/******************************************************************************/
-AsyncWorkerThread::AsyncWorkerThread(AsyncEngine* jobdispatcher,
-									unsigned int numthreadnumber) :
-	m_JobDispatcher(jobdispatcher),
-	m_luaerrorhandler(-1),
-	m_threadnum(numthreadnumber)
-{
-	// create luastack
-	m_LuaStack = luaL_newstate();
-
-	// load basic lua modules
-	luaL_openlibs(m_LuaStack);
-
-	// load serialization functions
-	luaopen_marshal(m_LuaStack);
-}
-
-/******************************************************************************/
-AsyncWorkerThread::~AsyncWorkerThread() {
-
-	assert(IsRunning() == false);
-	lua_close(m_LuaStack);
-}
-
-/******************************************************************************/
-void* AsyncWorkerThread::worker_thread_main() {
-
-	//register thread for error logging
-	char number[21];
-	snprintf(number,sizeof(number),"%d",m_threadnum);
-	log_register_thread(std::string("AsyncWorkerThread_") + number);
-
-	porting::setThreadName(
-			std::string(std::string("AsyncWorkTh_") + number).c_str());
-
-	/** prepare job lua environment **/
-	lua_newtable(m_LuaStack);
-	lua_setglobal(m_LuaStack, "engine");
-
-	lua_getglobal(m_LuaStack, "engine");
-	int top = lua_gettop(m_LuaStack);
-
-	lua_pushstring(m_LuaStack, DIR_DELIM);
-	lua_setglobal(m_LuaStack, "DIR_DELIM");
-
-	lua_pushstring(m_LuaStack,
-			std::string(porting::path_share + DIR_DELIM + "builtin").c_str());
-	lua_setglobal(m_LuaStack, "SCRIPTDIR");
-
-
-	m_JobDispatcher->PrepareEnvironment(m_LuaStack,top);
-
-	std::string asyncscript =
-	porting::path_share + DIR_DELIM + "builtin"
-			+ DIR_DELIM + "async_env.lua";
-
-	lua_pushcfunction(m_LuaStack, async_worker_ErrorHandler);
-	m_luaerrorhandler = lua_gettop(m_LuaStack);
-
-	if(!runScript(asyncscript)) {
-		infostream
-			<< "AsyncWorkderThread::worker_thread_main execution of async base environment failed!"
-			<< std::endl;
-			assert("no future with broken builtin async environment scripts" == 0);
-	}
-	/** main loop **/
-	while(!StopRequested()) {
-		//wait for job
-		LuaJobInfo toprocess = m_JobDispatcher->getJob();
-
-		if (toprocess.valid == false) { continue; }
-		if (StopRequested()) { continue; }
-
-		//first push error handler
-		lua_pushcfunction(m_LuaStack, script_error_handler);
-		int errorhandler = lua_gettop(m_LuaStack);
-
-		lua_getglobal(m_LuaStack, "engine");
-		if(lua_isnil(m_LuaStack, -1))
-			assert("unable to find engine within async environment" == 0);
-
-		lua_getfield(m_LuaStack, -1, "job_processor");
-		if(lua_isnil(m_LuaStack, -1))
-			assert("Someone managed to destroy a async worker engine!" == 0);
-
-		luaL_checktype(m_LuaStack, -1, LUA_TFUNCTION);
-
-		//call it
-		lua_pushlstring(m_LuaStack,
-						toprocess.serializedFunction.c_str(),
-						toprocess.serializedFunction.length());
-		lua_pushlstring(m_LuaStack,
-						toprocess.serializedParams.c_str(),
-						toprocess.serializedParams.length());
-
-		if (StopRequested()) { continue; }
-		if(lua_pcall(m_LuaStack, 2, 2, errorhandler)) {
-			script_error(m_LuaStack);
-			toprocess.serializedResult = "ERROR";
-		} else {
-			//fetch result
-			const char *retval = lua_tostring(m_LuaStack, -2);
-			unsigned int lenght = lua_tointeger(m_LuaStack,-1);
-			toprocess.serializedResult = std::string(retval,lenght);
-		}
-
-		if (StopRequested()) { continue; }
-		//put job result
-		m_JobDispatcher->putJobResult(toprocess);
-	}
-	log_deregister_thread();
-	return 0;
-}
-
-/******************************************************************************/
-bool AsyncWorkerThread::runScript(std::string script) {
-
-	int ret = 	luaL_loadfile(m_LuaStack, script.c_str()) ||
-				lua_pcall(m_LuaStack, 0, 0, m_luaerrorhandler);
-	if(ret){
-		errorstream<<"==== ERROR FROM LUA WHILE INITIALIZING ASYNC ENVIRONMENT ====="<<std::endl;
-		errorstream<<"Failed to load and run script from "<<std::endl;
-		errorstream<<script<<":"<<std::endl;
-		errorstream<<std::endl;
-		errorstream<<lua_tostring(m_LuaStack, -1)<<std::endl;
-		errorstream<<std::endl;
-		errorstream<<"=================== END OF ERROR FROM LUA ===================="<<std::endl;
-		lua_pop(m_LuaStack, 1); // Pop error message from stack
-		lua_pop(m_LuaStack, 1); // Pop the error handler from stack
-		return false;
-	}
-	return true;
-}
-
-/******************************************************************************/
-void* AsyncWorkerThread::worker_thread_wrapper(void* thread) {
-	return ((AsyncWorkerThread*) thread)->worker_thread_main();
-}
diff --git a/src/script/lua_api/l_async_events.h b/src/script/lua_api/l_async_events.h
deleted file mode 100644
index 9d42b07cf..000000000
--- a/src/script/lua_api/l_async_events.h
+++ /dev/null
@@ -1,220 +0,0 @@
-/*
-Minetest
-Copyright (C) 2013 sapier, <sapier AT gmx DOT net>
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU Lesser General Public License as published by
-the Free Software Foundation; either version 2.1 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Lesser General Public License for more details.
-
-You should have received a copy of the GNU Lesser General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-*/
-
-#ifndef L_ASYNC_EVENTS_H_
-#define L_ASYNC_EVENTS_H_
-
-#include <vector>
-#include <map>
-
-/******************************************************************************/
-/* Includes                                                                   */
-/******************************************************************************/
-#include "jthread/jthread.h"
-#include "jthread/jmutex.h"
-#include "jthread/jsemaphore.h"
-#include "debug.h"
-#include "lua.h"
-
-/******************************************************************************/
-/* Typedefs and macros                                                        */
-/******************************************************************************/
-#define MAINMENU_NUMBER_OF_ASYNC_THREADS                   4
-
-/******************************************************************************/
-/* forward declarations                                                       */
-/******************************************************************************/
-class AsyncEngine;
-
-/******************************************************************************/
-/* declarations                                                               */
-/******************************************************************************/
-
-/** a struct to encapsulate data required to queue a job **/
-struct LuaJobInfo {
-	/** function to be called in async environment **/
-	std::string serializedFunction;
-	/** parameter table to be passed to function **/
-	std::string serializedParams;
-	/** result of function call **/
-	std::string serializedResult;
-	/** jobid used to identify a job and match it to callback **/
-	unsigned int JobId;
-	/** valid marker **/
-	bool valid;
-};
-
-/** class encapsulating a asynchronous working environment **/
-class AsyncWorkerThread : public JThread {
-public:
-	/**
-	 * default constructor
-	 * @param pointer to job dispatcher
-	 */
-	AsyncWorkerThread(AsyncEngine* jobdispatcher, unsigned int threadnumber);
-
-	/**
-	 * default destructor
-	 */
-	virtual ~AsyncWorkerThread();
-
-	/**
-	 * thread function
-	 */
-	void* Thread() {
-			ThreadStarted();
-			return worker_thread_wrapper(this);
-	}
-
-private:
-	/**
-	 * helper function to run a lua script
-	 * @param path of script
-	 */
-	bool runScript(std::string script);
-
-	/**
-	 * main function of thread
-	 */
-	void* worker_thread_main();
-
-	/**
-	 * static wrapper for thread creation
-	 * @param this pointer to the thread to be created
-	 */
-	static void* worker_thread_wrapper(void* thread);
-
-	/**
-	 * pointer to job dispatcher
-	 */
-	AsyncEngine* m_JobDispatcher;
-
-	/**
-	 * the lua stack to run at
-	 */
-	lua_State*   m_LuaStack;
-
-	/**
-	 * lua internal stack number of error handler
-	 */
-	int m_luaerrorhandler;
-
-	/**
-	 * thread number used for debug output
-	 */
-	unsigned int m_threadnum;
-
-};
-
-/** asynchornous thread and job management **/
-class AsyncEngine {
-	friend class AsyncWorkerThread;
-public:
-	/**
-	 * default constructor
-	 */
-	AsyncEngine();
-	/**
-	 * default destructor
-	 */
-	~AsyncEngine();
-
-	/**
-	 * register function to be used within engines
-	 * @param name function name to be used within lua environment
-	 * @param fct c-function to be called
-	 */
-	bool registerFunction(const char* name, lua_CFunction fct);
-
-	/**
-	 * create async engine tasks and lock function registration
-	 * @param numengines number of async threads to be started
-	 */
-	void Initialize(unsigned int numengines);
-
-	/**
-	 * queue/run a async job
-	 * @param fct serialized lua function
-	 * @param params serialized parameters
-	 * @return jobid the job is queued
-	 */
-	unsigned int doAsyncJob(std::string fct, std::string params);
-
-	/**
-	 * engine step to process finished jobs
-	 *   the engine step is one way to pass events back, PushFinishedJobs another
-	 * @param L the lua environment to do the step in
-	 */
-	void Step(lua_State *L);
-
-
-	void PushFinishedJobs(lua_State* L);
-
-protected:
-	/**
-	 * Get a Job from queue to be processed
-	 *  this function blocks until a job is ready
-	 * @return a job to be processed
-	 */
-	LuaJobInfo getJob();
-
-	/**
-	 * put a Job result back to result queue
-	 * @param result result of completed job
-	 */
-	void putJobResult(LuaJobInfo result);
-
-	/**
-	 * initialize environment with current registred functions
-	 *  this function adds all functions registred by registerFunction to the
-	 *  passed lua stack
-	 * @param L lua stack to initialize
-	 * @param top stack position
-	 */
-	void PrepareEnvironment(lua_State* L, int top);
-
-private:
-
-	/** variable locking the engine against further modification **/
-	bool m_initDone;
-
-	/** internal store for registred functions **/
-	std::map<std::string,lua_CFunction> m_FunctionList;
-
-	/** internal counter to create job id's **/
-	unsigned int m_JobIdCounter;
-
-	/** mutex to protect job queue **/
-	JMutex m_JobQueueMutex;
-	/** job queue **/
-	std::vector<LuaJobInfo> m_JobQueue;
-
-	/** mutext to protect result queue **/
-	JMutex m_ResultQueueMutex;
-	/** result queue **/
-	std::vector<LuaJobInfo> m_ResultQueue;
-
-	/** list of current worker threads **/
-	std::vector<AsyncWorkerThread*> m_WorkerThreads;
-
-	/** counter semaphore for job dispatching **/
-	JSemaphore m_JobQueueCounter;
-};
-
-#endif /* L_ASYNC_EVENTS_H_ */
diff --git a/src/script/lua_api/l_mainmenu.cpp b/src/script/lua_api/l_mainmenu.cpp
index 5de1c77f0..31fc45ce2 100644
--- a/src/script/lua_api/l_mainmenu.cpp
+++ b/src/script/lua_api/l_mainmenu.cpp
@@ -20,7 +20,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include "lua_api/l_mainmenu.h"
 #include "lua_api/l_internal.h"
 #include "common/c_content.h"
-#include "lua_api/l_async_events.h"
+#include "cpp_api/s_async.h"
 #include "guiEngine.h"
 #include "guiMainMenu.h"
 #include "guiKeyChangeMenu.h"
@@ -1034,19 +1034,18 @@ int ModApiMainMenu::l_do_async_callback(lua_State *L)
 {
 	GUIEngine* engine = getGuiEngine(L);
 
-	const char* serialized_fct_raw = luaL_checkstring(L, 1);
-	unsigned int lenght_fct = luaL_checkint(L, 2);
+	size_t func_length, param_length;
+	const char* serialized_func_raw = luaL_checklstring(L, 1, &func_length);
 
-	const char* serialized_params_raw = luaL_checkstring(L, 3);
-	unsigned int lenght_params = luaL_checkint(L, 4);
+	const char* serialized_param_raw = luaL_checklstring(L, 2, &param_length);
 
-	assert(serialized_fct_raw != 0);
-	assert(serialized_params_raw != 0);
+	assert(serialized_func_raw != NULL);
+	assert(serialized_param_raw != NULL);
 
-	std::string serialized_fct = std::string(serialized_fct_raw,lenght_fct);
-	std::string serialized_params = std::string(serialized_params_raw,lenght_params);
+	std::string serialized_func = std::string(serialized_func_raw, func_length);
+	std::string serialized_param = std::string(serialized_param_raw, param_length);
 
-	lua_pushinteger(L,engine->DoAsync(serialized_fct,serialized_params));
+	lua_pushinteger(L, engine->DoAsync(serialized_func, serialized_param));
 
 	return 1;
 }
diff --git a/src/script/lua_api/l_util.cpp b/src/script/lua_api/l_util.cpp
index f2bbf181d..90a1d77ab 100644
--- a/src/script/lua_api/l_util.cpp
+++ b/src/script/lua_api/l_util.cpp
@@ -21,7 +21,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include "lua_api/l_internal.h"
 #include "common/c_converter.h"
 #include "common/c_content.h"
-#include "lua_api/l_async_events.h"
+#include "cpp_api/s_async.h"
 #include "debug.h"
 #include "log.h"
 #include "tool.h"
diff --git a/src/script/lua_api/marshall.c b/src/script/lua_api/marshall.c
deleted file mode 100644
index ef70566cb..000000000
--- a/src/script/lua_api/marshall.c
+++ /dev/null
@@ -1,551 +0,0 @@
-/*
-* lmarshal.c
-* A Lua library for serializing and deserializing Lua values
-* Richard Hundt <richardhundt@gmail.com>
-*
-* License: MIT
-*
-* Copyright (c) 2010 Richard Hundt
-*
-* Permission is hereby granted, free of charge, to any person
-* obtaining a copy of this software and associated documentation
-* files (the "Software"), to deal in the Software without
-* restriction, including without limitation the rights to use,
-* copy, modify, merge, publish, distribute, sublicense, and/or sell
-* copies of the Software, and to permit persons to whom the
-* Software is furnished to do so, subject to the following
-* conditions:
-*
-* The above copyright notice and this permission notice shall be
-* included in all copies or substantial portions of the Software.
-*
-* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
-* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
-* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
-* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
-* OTHER DEALINGS IN THE SOFTWARE.
-*/
-
-#include <stdlib.h>
-#include <string.h>
-#include <stdint.h>
-
-#include "lua.h"
-#include "lualib.h"
-#include "lauxlib.h"
-
-
-#define MAR_TREF 1
-#define MAR_TVAL 2
-#define MAR_TUSR 3
-
-#define MAR_CHR 1
-#define MAR_I32 4
-#define MAR_I64 8
-
-#define MAR_MAGIC 0x8e
-#define SEEN_IDX  3
-
-typedef struct mar_Buffer {
-    size_t size;
-    size_t seek;
-    size_t head;
-    char*  data;
-} mar_Buffer;
-
-static int mar_encode_table(lua_State *L, mar_Buffer *buf, size_t *idx);
-static int mar_decode_table(lua_State *L, const char* buf, size_t len, size_t *idx);
-
-static void buf_init(lua_State *L, mar_Buffer *buf)
-{
-    buf->size = 128;
-    buf->seek = 0;
-    buf->head = 0;
-    if (!(buf->data = malloc(buf->size))) luaL_error(L, "Out of memory!");
-}
-
-static void buf_done(lua_State* L, mar_Buffer *buf)
-{
-    free(buf->data);
-}
-
-static int buf_write(lua_State* L, const char* str, size_t len, mar_Buffer *buf)
-{
-    if (len > UINT32_MAX) luaL_error(L, "buffer too long");
-    if (buf->size - buf->head < len) {
-        size_t new_size = buf->size << 1;
-        size_t cur_head = buf->head;
-        while (new_size - cur_head <= len) {
-            new_size = new_size << 1;
-        }
-        if (!(buf->data = realloc(buf->data, new_size))) {
-            luaL_error(L, "Out of memory!");
-        }
-        buf->size = new_size;
-    }
-    memcpy(&buf->data[buf->head], str, len);
-    buf->head += len;
-    return 0;
-}
-
-static const char* buf_read(lua_State *L, mar_Buffer *buf, size_t *len)
-{
-    if (buf->seek < buf->head) {
-        buf->seek = buf->head;
-        *len = buf->seek;
-        return buf->data;
-    }
-    *len = 0;
-    return NULL;
-}
-
-static void mar_encode_value(lua_State *L, mar_Buffer *buf, int val, size_t *idx)
-{
-    size_t l;
-    int val_type = lua_type(L, val);
-    lua_pushvalue(L, val);
-
-    buf_write(L, (void*)&val_type, MAR_CHR, buf);
-    switch (val_type) {
-    case LUA_TBOOLEAN: {
-        int int_val = lua_toboolean(L, -1);
-        buf_write(L, (void*)&int_val, MAR_CHR, buf);
-        break;
-    }
-    case LUA_TSTRING: {
-        const char *str_val = lua_tolstring(L, -1, &l);
-        buf_write(L, (void*)&l, MAR_I32, buf);
-        buf_write(L, str_val, l, buf);
-        break;
-    }
-    case LUA_TNUMBER: {
-        lua_Number num_val = lua_tonumber(L, -1);
-        buf_write(L, (void*)&num_val, MAR_I64, buf);
-        break;
-    }
-    case LUA_TTABLE: {
-        int tag, ref;
-        lua_pushvalue(L, -1);
-        lua_rawget(L, SEEN_IDX);
-        if (!lua_isnil(L, -1)) {
-            ref = lua_tointeger(L, -1);
-            tag = MAR_TREF;
-            buf_write(L, (void*)&tag, MAR_CHR, buf);
-            buf_write(L, (void*)&ref, MAR_I32, buf);
-            lua_pop(L, 1);
-        }
-        else {
-            mar_Buffer rec_buf;
-            lua_pop(L, 1); /* pop nil */
-            if (luaL_getmetafield(L, -1, "__persist")) {
-                tag = MAR_TUSR;
-
-                lua_pushvalue(L, -2); /* self */
-                lua_call(L, 1, 1);
-                if (!lua_isfunction(L, -1)) {
-                    luaL_error(L, "__persist must return a function");
-                }
-
-                lua_remove(L, -2); /* __persist */
-
-                lua_newtable(L);
-                lua_pushvalue(L, -2); /* callback */
-                lua_rawseti(L, -2, 1);
-
-                buf_init(L, &rec_buf);
-                mar_encode_table(L, &rec_buf, idx);
-
-                buf_write(L, (void*)&tag, MAR_CHR, buf);
-                buf_write(L, (void*)&rec_buf.head, MAR_I32, buf);
-                buf_write(L, rec_buf.data, rec_buf.head, buf);
-                buf_done(L, &rec_buf);
-                lua_pop(L, 1);
-            }
-            else {
-                tag = MAR_TVAL;
-
-                lua_pushvalue(L, -1);
-                lua_pushinteger(L, (*idx)++);
-                lua_rawset(L, SEEN_IDX);
-
-                lua_pushvalue(L, -1);
-                buf_init(L, &rec_buf);
-                mar_encode_table(L, &rec_buf, idx);
-                lua_pop(L, 1);
-
-                buf_write(L, (void*)&tag, MAR_CHR, buf);
-                buf_write(L, (void*)&rec_buf.head, MAR_I32, buf);
-                buf_write(L, rec_buf.data,rec_buf.head, buf);
-                buf_done(L, &rec_buf);
-            }
-        }
-        break;
-    }
-    case LUA_TFUNCTION: {
-        int tag, ref;
-        lua_pushvalue(L, -1);
-        lua_rawget(L, SEEN_IDX);
-        if (!lua_isnil(L, -1)) {
-            ref = lua_tointeger(L, -1);
-            tag = MAR_TREF;
-            buf_write(L, (void*)&tag, MAR_CHR, buf);
-            buf_write(L, (void*)&ref, MAR_I32, buf);
-            lua_pop(L, 1);
-        }
-        else {
-            mar_Buffer rec_buf;
-            int i;
-            lua_Debug ar;
-            lua_pop(L, 1); /* pop nil */
-
-            lua_pushvalue(L, -1);
-            lua_getinfo(L, ">nuS", &ar);
-            if (ar.what[0] != 'L') {
-                luaL_error(L, "attempt to persist a C function '%s'", ar.name);
-            }
-            tag = MAR_TVAL;
-            lua_pushvalue(L, -1);
-            lua_pushinteger(L, (*idx)++);
-            lua_rawset(L, SEEN_IDX);
-
-            lua_pushvalue(L, -1);
-            buf_init(L, &rec_buf);
-            lua_dump(L, (lua_Writer)buf_write, &rec_buf);
-
-            buf_write(L, (void*)&tag, MAR_CHR, buf);
-            buf_write(L, (void*)&rec_buf.head, MAR_I32, buf);
-            buf_write(L, rec_buf.data, rec_buf.head, buf);
-            buf_done(L, &rec_buf);
-            lua_pop(L, 1);
-
-            lua_newtable(L);
-            for (i=1; i <= ar.nups; i++) {
-                lua_getupvalue(L, -2, i);
-                lua_rawseti(L, -2, i);
-            }
-
-            buf_init(L, &rec_buf);
-            mar_encode_table(L, &rec_buf, idx);
-
-            buf_write(L, (void*)&rec_buf.head, MAR_I32, buf);
-            buf_write(L, rec_buf.data, rec_buf.head, buf);
-            buf_done(L, &rec_buf);
-            lua_pop(L, 1);
-        }
-
-        break;
-    }
-    case LUA_TUSERDATA: {
-        int tag, ref;
-        lua_pushvalue(L, -1);
-        lua_rawget(L, SEEN_IDX);
-        if (!lua_isnil(L, -1)) {
-            ref = lua_tointeger(L, -1);
-            tag = MAR_TREF;
-            buf_write(L, (void*)&tag, MAR_CHR, buf);
-            buf_write(L, (void*)&ref, MAR_I32, buf);
-            lua_pop(L, 1);
-        }
-        else {
-            mar_Buffer rec_buf;
-            lua_pop(L, 1); /* pop nil */
-            if (luaL_getmetafield(L, -1, "__persist")) {
-                tag = MAR_TUSR;
-
-                lua_pushvalue(L, -2);
-                lua_pushinteger(L, (*idx)++);
-                lua_rawset(L, SEEN_IDX);
-
-                lua_pushvalue(L, -2);
-                lua_call(L, 1, 1);
-                if (!lua_isfunction(L, -1)) {
-                    luaL_error(L, "__persist must return a function");
-                }
-                lua_newtable(L);
-                lua_pushvalue(L, -2);
-                lua_rawseti(L, -2, 1);
-                lua_remove(L, -2);
-
-                buf_init(L, &rec_buf);
-                mar_encode_table(L, &rec_buf, idx);
-
-                buf_write(L, (void*)&tag, MAR_CHR, buf);
-                buf_write(L, (void*)&rec_buf.head, MAR_I32, buf);
-		buf_write(L, rec_buf.data, rec_buf.head, buf);
-		buf_done(L, &rec_buf);
-            }
-            else {
-                luaL_error(L, "attempt to encode userdata (no __persist hook)");
-            }
-            lua_pop(L, 1);
-        }
-        break;
-    }
-    case LUA_TNIL: break;
-    default:
-        luaL_error(L, "invalid value type (%s)", lua_typename(L, val_type));
-    }
-    lua_pop(L, 1);
-}
-
-static int mar_encode_table(lua_State *L, mar_Buffer *buf, size_t *idx)
-{
-    lua_pushnil(L);
-    while (lua_next(L, -2) != 0) {
-        mar_encode_value(L, buf, -2, idx);
-        mar_encode_value(L, buf, -1, idx);
-        lua_pop(L, 1);
-    }
-    return 1;
-}
-
-#define mar_incr_ptr(l) \
-    if (((*p)-buf)+(l) > len) luaL_error(L, "bad code"); (*p) += (l);
-
-#define mar_next_len(l,T) \
-    if (((*p)-buf)+sizeof(T) > len) luaL_error(L, "bad code"); \
-    l = *(T*)*p; (*p) += sizeof(T);
-
-static void mar_decode_value
-    (lua_State *L, const char *buf, size_t len, const char **p, size_t *idx)
-{
-    size_t l;
-    char val_type = **p;
-    mar_incr_ptr(MAR_CHR);
-    switch (val_type) {
-    case LUA_TBOOLEAN:
-        lua_pushboolean(L, *(char*)*p);
-        mar_incr_ptr(MAR_CHR);
-        break;
-    case LUA_TNUMBER:
-        lua_pushnumber(L, *(lua_Number*)*p);
-        mar_incr_ptr(MAR_I64);
-        break;
-    case LUA_TSTRING:
-        mar_next_len(l, uint32_t);
-        lua_pushlstring(L, *p, l);
-        mar_incr_ptr(l);
-        break;
-    case LUA_TTABLE: {
-        char tag = *(char*)*p;
-        mar_incr_ptr(MAR_CHR);
-        if (tag == MAR_TREF) {
-            int ref;
-            mar_next_len(ref, int);
-            lua_rawgeti(L, SEEN_IDX, ref);
-        }
-        else if (tag == MAR_TVAL) {
-            mar_next_len(l, uint32_t);
-            lua_newtable(L);
-            lua_pushvalue(L, -1);
-            lua_rawseti(L, SEEN_IDX, (*idx)++);
-            mar_decode_table(L, *p, l, idx);
-            mar_incr_ptr(l);
-        }
-        else if (tag == MAR_TUSR) {
-            mar_next_len(l, uint32_t);
-            lua_newtable(L);
-            mar_decode_table(L, *p, l, idx);
-            lua_rawgeti(L, -1, 1);
-            lua_call(L, 0, 1);
-            lua_remove(L, -2);
-            lua_pushvalue(L, -1);
-            lua_rawseti(L, SEEN_IDX, (*idx)++);
-            mar_incr_ptr(l);
-        }
-        else {
-            luaL_error(L, "bad encoded data");
-        }
-        break;
-    }
-    case LUA_TFUNCTION: {
-        size_t nups;
-        int i;
-        mar_Buffer dec_buf;
-        char tag = *(char*)*p;
-        mar_incr_ptr(1);
-        if (tag == MAR_TREF) {
-            int ref;
-            mar_next_len(ref, int);
-            lua_rawgeti(L, SEEN_IDX, ref);
-        }
-        else {
-            mar_next_len(l, uint32_t);
-            dec_buf.data = (char*)*p;
-            dec_buf.size = l;
-            dec_buf.head = l;
-            dec_buf.seek = 0;
-            lua_load(L, (lua_Reader)buf_read, &dec_buf, "=marshal");
-            mar_incr_ptr(l);
-
-            lua_pushvalue(L, -1);
-            lua_rawseti(L, SEEN_IDX, (*idx)++);
-
-            mar_next_len(l, uint32_t);
-            lua_newtable(L);
-            mar_decode_table(L, *p, l, idx);
-            nups = lua_objlen(L, -1);
-            for (i=1; i <= nups; i++) {
-                lua_rawgeti(L, -1, i);
-                lua_setupvalue(L, -3, i);
-            }
-            lua_pop(L, 1);
-            mar_incr_ptr(l);
-        }
-        break;
-    }
-    case LUA_TUSERDATA: {
-        char tag = *(char*)*p;
-        mar_incr_ptr(MAR_CHR);
-        if (tag == MAR_TREF) {
-            int ref;
-            mar_next_len(ref, int);
-            lua_rawgeti(L, SEEN_IDX, ref);
-        }
-        else if (tag == MAR_TUSR) {
-            mar_next_len(l, uint32_t);
-            lua_newtable(L);
-            mar_decode_table(L, *p, l, idx);
-            lua_rawgeti(L, -1, 1);
-            lua_call(L, 0, 1);
-            lua_remove(L, -2);
-            lua_pushvalue(L, -1);
-            lua_rawseti(L, SEEN_IDX, (*idx)++);
-            mar_incr_ptr(l);
-        }
-        else { /* tag == MAR_TVAL */
-            lua_pushnil(L);
-        }
-        break;
-    }
-    case LUA_TNIL:
-    case LUA_TTHREAD:
-        lua_pushnil(L);
-        break;
-    default:
-        luaL_error(L, "bad code");
-    }
-}
-
-static int mar_decode_table(lua_State *L, const char* buf, size_t len, size_t *idx)
-{
-    const char* p;
-    p = buf;
-    while (p - buf < len) {
-        mar_decode_value(L, buf, len, &p, idx);
-        mar_decode_value(L, buf, len, &p, idx);
-        lua_settable(L, -3);
-    }
-    return 1;
-}
-
-static int mar_encode(lua_State* L)
-{
-    const unsigned char m = MAR_MAGIC;
-    size_t idx, len;
-    mar_Buffer buf;
-
-    if (lua_isnone(L, 1)) {
-        lua_pushnil(L);
-    }
-    if (lua_isnoneornil(L, 2)) {
-        lua_newtable(L);
-    }
-    else if (!lua_istable(L, 2)) {
-        luaL_error(L, "bad argument #2 to encode (expected table)");
-    }
-    lua_settop(L, 2);
-
-    len = lua_objlen(L, 2);
-    lua_newtable(L);
-    for (idx = 1; idx <= len; idx++) {
-        lua_rawgeti(L, 2, idx);
-        if (lua_isnil(L, -1)) {
-            lua_pop(L, 1);
-            continue;
-        }
-        lua_pushinteger(L, idx);
-        lua_rawset(L, SEEN_IDX);
-    }
-    lua_pushvalue(L, 1);
-
-    buf_init(L, &buf);
-    buf_write(L, (void*)&m, 1, &buf);
-
-    mar_encode_value(L, &buf, -1, &idx);
-
-    lua_pop(L, 1);
-
-    lua_pushlstring(L, buf.data, buf.head);
-
-    buf_done(L, &buf);
-
-    lua_remove(L, SEEN_IDX);
-
-    return 1;
-}
-
-static int mar_decode(lua_State* L)
-{
-    size_t l, idx, len;
-    const char *p;
-    const char *s = luaL_checklstring(L, 1, &l);
-
-    if (l < 1) luaL_error(L, "bad header");
-    if (*(unsigned char *)s++ != MAR_MAGIC) luaL_error(L, "bad magic");
-    l -= 1;
-
-    if (lua_isnoneornil(L, 2)) {
-        lua_newtable(L);
-    }
-    else if (!lua_istable(L, 2)) {
-        luaL_error(L, "bad argument #2 to decode (expected table)");
-    }
-    lua_settop(L, 2);
-
-    len = lua_objlen(L, 2);
-    lua_newtable(L);
-    for (idx = 1; idx <= len; idx++) {
-        lua_rawgeti(L, 2, idx);
-        lua_rawseti(L, SEEN_IDX, idx);
-    }
-
-    p = s;
-    mar_decode_value(L, s, l, &p, &idx);
-
-    lua_remove(L, SEEN_IDX);
-    lua_remove(L, 2);
-
-    return 1;
-}
-
-static int mar_clone(lua_State* L)
-{
-    mar_encode(L);
-    lua_replace(L, 1);
-    mar_decode(L);
-    return 1;
-}
-
-static const luaL_reg R[] =
-{
-    {"encode",      mar_encode},
-    {"decode",      mar_decode},
-    {"clone",       mar_clone},
-    {NULL,	    NULL}
-};
-
-int luaopen_marshal(lua_State *L)
-{
-    lua_newtable(L);
-    luaL_register(L, "marshal", R);
-    return 1;
-}
-
-
-
-
-