Add Lua interface to HTTPFetchRequest

This allows mods to perform both asynchronous and synchronous HTTP
requests. Mods are only granted access to HTTP APIs if either mod
security is disabled or if they are whitelisted in any of the
the secure.http_mods and secure.trusted_mods settings.

Adds httpfetch_caller_alloc_secure to generate random, non-predictable
caller IDs so that lua mods cannot spy on each others HTTP queries.

9 files changed, 280 insertions, 1 deletions

diff --git a/src/defaultsettings.cpp b/src/defaultsettings.cpp
index b1e754870..49e03a260 100644
--- a/src/defaultsettings.cpp
+++ b/src/defaultsettings.cpp
@@ -285,6 +285,7 @@ void set_default_settings(Settings *settings)
 	settings->setDefault("num_emerge_threads", "1");
 	settings->setDefault("secure.enable_security", "false");
 	settings->setDefault("secure.trusted_mods", "");
+	settings->setDefault("secure.http_mods", "");
 
 	// physics stuff
 	settings->setDefault("movement_acceleration_default", "3");
diff --git a/src/httpfetch.cpp b/src/httpfetch.cpp
index 1a19dd082..f64c9f717 100644
--- a/src/httpfetch.cpp
+++ b/src/httpfetch.cpp
@@ -18,7 +18,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 */
 
 #include "socket.h" // for select()
-#include "porting.h" // for sleep_ms(), get_sysinfo()
+#include "porting.h" // for sleep_ms(), get_sysinfo(), secure_rand_fill_buf()
 #include "httpfetch.h"
 #include <iostream>
 #include <sstream>
@@ -34,9 +34,11 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include "util/thread.h"
 #include "version.h"
 #include "settings.h"
+#include "noise.h"
 
 Mutex g_httpfetch_mutex;
 std::map<unsigned long, std::queue<HTTPFetchResult> > g_httpfetch_results;
+PcgRandom g_callerid_randomness;
 
 HTTPFetchRequest::HTTPFetchRequest()
 {
@@ -84,6 +86,34 @@ unsigned long httpfetch_caller_alloc()
 	return discard;
 }
 
+unsigned long httpfetch_caller_alloc_secure()
+{
+	MutexAutoLock lock(g_httpfetch_mutex);
+
+	// Generate random caller IDs and make sure they're not
+	// already used or equal to HTTPFETCH_DISCARD
+	// Give up after 100 tries to prevent infinite loop
+	u8 tries = 100;
+	unsigned long caller;
+
+	do {
+		caller = (((u64) g_callerid_randomness.next()) << 32) |
+				g_callerid_randomness.next();
+
+		if (--tries < 1) {
+			FATAL_ERROR("httpfetch_caller_alloc_secure: ran out of caller IDs");
+			return HTTPFETCH_DISCARD;
+		}
+	} while (g_httpfetch_results.find(caller) != g_httpfetch_results.end());
+
+	verbosestream << "httpfetch_caller_alloc_secure: allocating "
+		<< caller << std::endl;
+
+	// Access element to create it
+	g_httpfetch_results[caller];
+	return caller;
+}
+
 void httpfetch_caller_free(unsigned long caller)
 {
 	verbosestream<<"httpfetch_caller_free: freeing "
@@ -710,6 +740,11 @@ void httpfetch_init(int parallel_limit)
 	FATAL_ERROR_IF(res != CURLE_OK, "CURL init failed");
 
 	g_httpfetch_thread = new CurlFetchThread(parallel_limit);
+
+	// Initialize g_callerid_randomness for httpfetch_caller_alloc_secure
+	u64 randbuf[2];
+	porting::secure_rand_fill_buf(randbuf, sizeof(u64) * 2);
+	g_callerid_randomness = PcgRandom(randbuf[0], randbuf[1]);
 }
 
 void httpfetch_cleanup()
diff --git a/src/httpfetch.h b/src/httpfetch.h
index c44c8d2d3..f57ed8789 100644
--- a/src/httpfetch.h
+++ b/src/httpfetch.h
@@ -116,6 +116,9 @@ bool httpfetch_async_get(unsigned long caller, HTTPFetchResult &fetch_result);
 // Not required if you want to set caller = HTTPFETCH_DISCARD
 unsigned long httpfetch_caller_alloc();
 
+// Allocates a non-predictable caller ID for httpfetch_async
+unsigned long httpfetch_caller_alloc_secure();
+
 // Frees a caller ID allocated with httpfetch_caller_alloc
 // Note: This can be expensive, because the httpfetch thread is told
 // to stop any ongoing fetches for the given caller.
diff --git a/src/script/common/c_converter.cpp b/src/script/common/c_converter.cpp
index f1d3cc421..55c4a5f5a 100644
--- a/src/script/common/c_converter.cpp
+++ b/src/script/common/c_converter.cpp
@@ -517,6 +517,15 @@ bool getboolfield_default(lua_State *L, int table,
 	return result;
 }
 
+void setstringfield(lua_State *L, int table,
+		const char *fieldname, const char *value)
+{
+	lua_pushstring(L, value);
+	if(table < 0)
+		table -= 1;
+	lua_setfield(L, table, fieldname);
+}
+
 void setintfield(lua_State *L, int table,
 		const char *fieldname, int value)
 {
diff --git a/src/script/common/c_converter.h b/src/script/common/c_converter.h
index 18a045d2a..eefac0ed7 100644
--- a/src/script/common/c_converter.h
+++ b/src/script/common/c_converter.h
@@ -69,6 +69,8 @@ bool               getfloatfield(lua_State *L, int table,
 std::string        checkstringfield(lua_State *L, int table,
                              const char *fieldname);
 
+void               setstringfield(lua_State *L, int table,
+                             const char *fieldname, const char *value);
 void               setintfield(lua_State *L, int table,
                              const char *fieldname, int value);
 void               setfloatfield(lua_State *L, int table,
diff --git a/src/script/lua_api/CMakeLists.txt b/src/script/lua_api/CMakeLists.txt
index 2501ce6d6..d507dcf70 100644
--- a/src/script/lua_api/CMakeLists.txt
+++ b/src/script/lua_api/CMakeLists.txt
@@ -16,6 +16,7 @@ set(common_SCRIPT_LUA_API_SRCS
 	${CMAKE_CURRENT_SOURCE_DIR}/l_util.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/l_vmanip.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/l_settings.cpp
+	${CMAKE_CURRENT_SOURCE_DIR}/l_http.cpp
 	PARENT_SCOPE)
 
 set(client_SCRIPT_LUA_API_SRCS
diff --git a/src/script/lua_api/l_http.cpp b/src/script/lua_api/l_http.cpp
new file mode 100644
index 000000000..b0357e3e0
--- /dev/null
+++ b/src/script/lua_api/l_http.cpp
@@ -0,0 +1,176 @@
+/*
+Minetest
+Copyright (C) 2013 celeron55, Perttu Ahola <celeron55@gmail.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU Lesser General Public License as published by
+the Free Software Foundation; either version 2.1 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+*/
+
+#include "lua_api/l_internal.h"
+#include "common/c_converter.h"
+#include "common/c_content.h"
+#include "lua_api/l_http.h"
+#include "httpfetch.h"
+#include "settings.h"
+#include "log.h"
+
+#include <algorithm>
+#include <iomanip>
+#include <cctype>
+
+#define HTTP_API(name) \
+	lua_pushstring(L, #name); \
+	lua_pushcfunction(L, l_http_##name); \
+	lua_settable(L, -3);
+
+#if USE_CURL
+void ModApiHttp::read_http_fetch_request(lua_State *L, HTTPFetchRequest &req)
+{
+	luaL_checktype(L, 1, LUA_TTABLE);
+
+	req.caller = httpfetch_caller_alloc_secure();
+	getstringfield(L, 1, "url", req.url);
+	lua_getfield(L, 1, "user_agent");
+	if (lua_isstring(L, -1))
+		req.useragent = getstringfield_default(L, 1, "user_agent", "");
+	lua_pop(L, 1);
+	req.multipart = getboolfield_default(L, 1, "multipart", false);
+	req.timeout = getintfield_default(L, 1, "timeout", 3) * 1000;
+
+	// post_data: if table, post form data, otherwise raw data
+	lua_getfield(L, 1, "post_data");
+	if (lua_istable(L, 2)) {
+		lua_pushnil(L);
+		while (lua_next(L, 2) != 0)
+		{
+			req.post_fields[luaL_checkstring(L, -2)] = luaL_checkstring(L, -1);
+			lua_pop(L, 1);
+		}
+	} else if (lua_isstring(L, 2)) {
+		req.post_data = lua_tostring(L, 2);
+	}
+	lua_pop(L, 1);
+
+	lua_getfield(L, 1, "extra_headers");
+	if (lua_istable(L, 2)) {
+		lua_pushnil(L);
+		while (lua_next(L, 2) != 0)
+		{
+			const char *header = luaL_checkstring(L, -1);
+			req.extra_headers.push_back(header);
+			lua_pop(L, 1);
+		}
+	}
+	lua_pop(L, 1);
+}
+
+void ModApiHttp::push_http_fetch_result(lua_State *L, HTTPFetchResult &res, bool completed)
+{
+	lua_newtable(L);
+	setboolfield(L, -1, "succeeded", res.succeeded);
+	setboolfield(L, -1, "timeout", res.timeout);
+	setboolfield(L, -1, "completed", completed);
+	setintfield(L, -1, "code", res.response_code);
+	setstringfield(L, -1, "data", res.data.c_str());
+}
+
+// http_api.fetch_async(HTTPRequest definition)
+int ModApiHttp::l_http_fetch_async(lua_State *L)
+{
+	NO_MAP_LOCK_REQUIRED;
+
+	HTTPFetchRequest req;
+	read_http_fetch_request(L, req);
+
+	actionstream << "Mod performs HTTP request with URL " << req.url << std::endl;
+	httpfetch_async(req);
+
+	// Convert handle to hex string since lua can't handle 64-bit integers
+	std::stringstream handle_conversion_stream;
+	handle_conversion_stream << std::hex << req.caller;
+	std::string caller_handle(handle_conversion_stream.str());
+
+	lua_pushstring(L, caller_handle.c_str());
+	return 1;
+}
+
+// http_api.fetch_async_get(handle)
+int ModApiHttp::l_http_fetch_async_get(lua_State *L)
+{
+	NO_MAP_LOCK_REQUIRED;
+
+	std::string handle_str = luaL_checkstring(L, 1);
+
+	// Convert hex string back to 64-bit handle
+	u64 handle;
+	std::stringstream handle_conversion_stream;
+	handle_conversion_stream << std::hex << handle_str;
+	handle_conversion_stream >> handle;
+
+	HTTPFetchResult res;
+	bool completed = httpfetch_async_get(handle, res);
+
+	push_http_fetch_result(L, res, completed);
+
+	return 1;
+}
+
+int ModApiHttp::l_request_http_api(lua_State *L)
+{
+	NO_MAP_LOCK_REQUIRED;
+
+	// Mod must be listed in secure.http_mods or secure.trusted_mods
+	lua_rawgeti(L, LUA_REGISTRYINDEX, CUSTOM_RIDX_CURRENT_MOD_NAME);
+	if (!lua_isstring(L, -1)) {
+		lua_pushnil(L);
+		return 1;
+	}
+
+	const char *mod_name = lua_tostring(L, -1);
+	std::string http_mods = g_settings->get("secure.http_mods");
+	http_mods.erase(std::remove(http_mods.begin(), http_mods.end(), ' '), http_mods.end());
+	std::vector<std::string> mod_list_http = str_split(http_mods, ',');
+
+	std::string trusted_mods = g_settings->get("secure.trusted_mods");
+	trusted_mods.erase(std::remove(trusted_mods.begin(), trusted_mods.end(), ' '), trusted_mods.end());
+	std::vector<std::string> mod_list_trusted = str_split(trusted_mods, ',');
+
+	mod_list_http.insert(mod_list_http.end(), mod_list_trusted.begin(), mod_list_trusted.end());
+	if (std::find(mod_list_http.begin(), mod_list_http.end(), mod_name) == mod_list_http.end()) {
+		lua_pushnil(L);
+		return 1;
+	}
+
+	lua_getglobal(L, "core");
+	lua_getfield(L, -1, "http_add_fetch");
+
+	lua_newtable(L);
+	HTTP_API(fetch_async);
+	HTTP_API(fetch_async_get);
+
+	// Stack now looks like this:
+	// <core.http_add_fetch> <table with fetch_async, fetch_async_get>
+	// Now call core.http_add_fetch to append .fetch(request, callback) to table
+	lua_call(L, 1, 1);
+
+	return 1;
+}
+#endif
+
+void ModApiHttp::Initialize(lua_State *L, int top)
+{
+#if USE_CURL
+	API_FCT(request_http_api);
+#endif
+}
diff --git a/src/script/lua_api/l_http.h b/src/script/lua_api/l_http.h
new file mode 100644
index 000000000..077ade691
--- /dev/null
+++ b/src/script/lua_api/l_http.h
@@ -0,0 +1,50 @@
+/*
+Minetest
+Copyright (C) 2013 celeron55, Perttu Ahola <celeron55@gmail.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU Lesser General Public License as published by
+the Free Software Foundation; either version 2.1 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+*/
+
+#ifndef L_HTTP_H_
+#define L_HTTP_H_
+
+#include "lua_api/l_base.h"
+#include "config.h"
+
+struct HTTPFetchRequest;
+struct HTTPFetchResult;
+
+class ModApiHttp : public ModApiBase {
+private:
+#if USE_CURL
+	// Helpers for HTTP fetch functions
+	static void read_http_fetch_request(lua_State *L, HTTPFetchRequest &req);
+	static void push_http_fetch_result(lua_State *L, HTTPFetchResult &res, bool completed = true);
+
+	// http_fetch_async({url=, timeout=, post_data=})
+	static int l_http_fetch_async(lua_State *L);
+
+	// http_fetch_async_get(handle)
+	static int l_http_fetch_async_get(lua_State *L);
+
+	// request_http_api()
+	static int l_request_http_api(lua_State *L);
+#endif
+
+public:
+	static void Initialize(lua_State *L, int top);
+};
+
+#endif /* L_HTTP_H_ */
diff --git a/src/script/scripting_game.cpp b/src/script/scripting_game.cpp
index 33bc5c2a7..e313d55f8 100644
--- a/src/script/scripting_game.cpp
+++ b/src/script/scripting_game.cpp
@@ -39,6 +39,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include "lua_api/l_util.h"
 #include "lua_api/l_vmanip.h"
 #include "lua_api/l_settings.h"
+#include "lua_api/l_http.h"
 
 extern "C" {
 #include "lualib.h"
@@ -89,6 +90,7 @@ void GameScripting::InitializeModApi(lua_State *L, int top)
 	ModApiRollback::Initialize(L, top);
 	ModApiServer::Initialize(L, top);
 	ModApiUtil::Initialize(L, top);
+	ModApiHttp::Initialize(L, top);
 
 	// Register reference classes (userdata)
 	InvRef::Register(L);