advtrains/textures/advtrains_boiler.png

/*
Minetest
Copyright (C) 2013 celeron55, Perttu Ahola <celeron55@gmail.com>

This program is free software; you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation; either version 2.1 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU Lesser General Public License for more details.

You should have received a copy of the GNU Lesser General Public License along
with this program; if not, write to the Free Software Foundation, Inc.,
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/

#include "socket.h" // for select()
#include "porting.h" // for sleep_ms(), get_sysinfo(), secure_rand_fill_buf()
#include "httpfetch.h"
#include <iostream>
#include <sstream>
#include <list>
#include <map>
#include <errno.h>
#include "threading/event.h"
#include "config.h"
#include "exceptions.h"
#include "debug.h"
#include "log.h"
#include "util/container.h"
#include "util/thread.h"
#include "version.h"
#include "settings.h"
#include "noise.h"

Mutex g_httpfetch_mutex;
std::map<unsigned long, std::queue<HTTPFetchResult> > g_httpfetch_results;
PcgRandom g_callerid_randomness;

HTTPFetchRequest::HTTPFetchRequest() :
	url(""),
	caller(HTTPFETCH_DISCARD),
	request_id(0),
	timeout(g_settings->getS32("curl_timeout")),
	connect_timeout(timeout),
	multipart(false),
	useragent(std::string(PROJECT_NAME_C "/") + g_version_hash + " (" + porting::get_sysinfo() + ")")
{
}


static void httpfetch_deliver_result(const HTTPFetchResult &fetch_result)
{
	unsigned long caller = fetch_result.caller;
	if (caller != HTTPFETCH_DISCARD) {
		MutexAutoLock lock(g_httpfetch_mutex);
		g_httpfetch_results[caller].push(fetch_result);
	}
}

static void httpfetch_request_clear(unsigned long caller);

unsigned long httpfetch_caller_alloc()
{
	MutexAutoLock lock(g_httpfetch_mutex);

	// Check each caller ID except HTTPFETCH_DISCARD
	const unsigned long discard = HTTPFETCH_DISCARD;
	for (unsigned long caller = discard + 1; caller != discard; ++caller) {
		std::map<unsigned long, std::queue<HTTPFetchResult> >::iterator
			it = g_httpfetch_results.find(caller);
		if (it == g_httpfetch_results.end()) {
			verbosestream << "httpfetch_caller_alloc: allocating "
					<< caller << std::endl;
			// Access element to create it
			g_httpfetch_results[caller];
			return caller;
		}
	}

	FATAL_ERROR("httpfetch_caller_alloc: ran out of caller IDs");
	return discard;
}

unsigned long httpfetch_caller_alloc_secure()
{
	MutexAutoLock lock(g_httpfetch_mutex);

	// Generate random caller IDs and make sure they're not
	// already used or equal to HTTPFETCH_DISCARD
	// Give up after 100 tries to prevent infinite loop
	u8 tries = 100;
	unsigned long caller;

	do {
		caller = (((u64) g_callerid_randomness.next()) << 32) |
				g_callerid_randomness.next();

		if (--tries < 1) {
			FATAL_ERROR("httpfetch_caller_alloc_secure: ran out of caller IDs");
			return HTTPFETCH_DISCARD;
		}
	} while (g_httpfetch_results.find(caller) != g_httpfetch_results.end());

	verbosestream << "httpfetch_caller_alloc_secure: allocating "
		<< caller << std::endl;

	// Access element to create it
	g_httpfetch_results[caller];
	return caller;
}

void httpfetch_caller_free(unsigned long caller)
{
	verbosestream<<"httpfetch_caller_free: freeing "
			<<caller<<std::endl;

	httpfetch_request_clear(caller);
	if (caller != HTTPFETCH_DISCARD) {
		MutexAutoLock lock(g_httpfetch_mutex);
		g_httpfetch_results.erase(caller);
	}
}

bool httpfetch_async_get(unsigned long caller, HTTPFetchResult &fetch_result)
{
	MutexAutoLock lock(g_httpfetch_mutex);

	// Check that caller exists
	std::map<unsigned long, std::queue<HTTPFetchResult> >::iterator
		it = g_httpfetch_results.find(caller);
	if (it == g_httpfetch_results.end())
		return false;

	// Check that result queue is nonempty
	std::queue<HTTPFetchResult> &caller_results = it->second;
	if (caller_results.empty())
		return false;

	// Pop first result
	fetch_result = caller_results.front();
	caller_results.pop();
	return true;
}

#if USE_CURL
#include <curl/curl.h>

/*
	USE_CURL is on: use cURL based httpfetch implementation
*/

static size_t httpfetch_writefunction(
		char *ptr, size_t size, size_t nmemb, void *userdata)
{
	std::ostringstream *stream = (std::ostringstream*)userdata;
	size_t count = size * nmemb;
	stream->write(ptr, count);
	return count;
}

static size_t httpfetch_discardfunction(
		char *ptr, size_t size, size_t nmemb, void *userdata)
{
	return size * nmemb;
}

class CurlHandlePool
{
	std::list<CURL*> handles;

public:
	CurlHandlePool() {}
	~CurlHandlePool()
	{
		for (std::list<CURL*>::iterator it = handles.begin();
				it != handles.end(); ++it) {
			curl_easy_cleanup(*it);
		}
	}
	CURL * alloc()
	{
		CURL *curl;
		if (handles.empty()) {
			curl = curl_easy_init();
			if (curl == NULL) {
				errorstream<<"curl_easy_init returned NULL"<<std::endl;
			}
		}
		else {
			curl = handles.front();
			handles.pop_front();
		}
		return curl;
	}
	void free(CURL *handle)
	{
		if (handle)
			handles.push_back(handle);
	}
};

class HTTPFetchOngoing
{
public:
	HTTPFetchOngoing(const HTTPFetchRequest &request, CurlHandlePool *pool);
	~HTTPFetchOngoing();

	CURLcode start(CURLM *multi);
	const HTTPFetchResult * complete(CURLcode res);

	const HTTPFetchRequest &getRequest()    const { return request; };
	const CURL             *getEasyHandle() const { return curl; };

private:
	CurlHandlePool *pool;
	CURL *curl;
	CURLM *multi;
	HTTPFetchRequest request;
	HTTPFetchResult result;
	std::ostringstream oss;
	struct curl_slist *http_header;
	curl_httppost *post;
};


HTTPFetchOngoing::HTTPFetchOngoing(const HTTPFetchRequest &request_,
		CurlHandlePool *pool_):
	pool(pool_),
	curl(NULL),
	multi(NULL),
	request(request_),
	result(request_),
	oss(std::ios::binary),
	http_header(NULL),
	post(NULL)
{
	curl = pool->alloc();
	if (curl == NULL) {
		return;
	}

	// Set static cURL options
	curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1);
	curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1);
	curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1);
	curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 1);

	std::string bind_address = g_settings->get("bind_address");
	if (!bind_address.empty()) {
		curl_easy_setopt(curl, CURLOPT_INTERFACE, bind_address.c_str());
	}

#if LIBCURL_VERSION_NUM >= 0x071304
	// Restrict protocols so that curl vulnerabilities in
	// other protocols don't affect us.
	// These settings were introduced in curl 7.19.4.
	long protocols =
		CURLPROTO_HTTP |
		CURLPROTO_HTTPS |
		CURLPROTO_FTP |
		CURLPROTO_FTPS;
	curl_easy_setopt(curl, CURLOPT_PROTOCOLS, protocols);
	curl_easy_setopt(curl, CURLOPT_REDIR_PROTOCOLS, protocols);
#endif

	// Set cURL options based on HTTPFetchRequest
	curl_easy_setopt(curl, CURLOPT_URL,
			request.url.c_str());
	curl_easy_setopt(curl, CURLOPT_TIMEOUT_MS,
			request.timeout);
	curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT_MS,
			request.connect_timeout);

	if (request.useragent != "")
		curl_easy_setopt(curl, CURLOPT_USERAGENT, request.useragent.c_str());

	// Set up a write callback that writes to the
	// ostringstream ongoing->oss, unless the data
	// is to be discarded
	if (request.caller == HTTPFETCH_DISCARD) {
		curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION,
				httpfetch_discardfunction);
		curl_easy_setopt(curl, CURLOPT_WRITEDATA, NULL);
	} else {
		curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION,
				httpfetch_writefunction);
		curl_easy_setopt(curl, CURLOPT_WRITEDATA, &oss);
	}

	// Set POST (or GET) data
	if (request.post_fields.empty() && request.post_data.empty()) {
		curl_easy_setopt(curl, CURLOPT_HTTPGET, 1);
	} else if (request.multipart) {
		curl_httppost *last = NULL;
		for (StringMap::iterator it = request.post_fields.begin();
				it != request.post_fields.end(); ++it) {
			curl_formadd(&post, &last,
					CURLFORM_NAMELENGTH, it->first.size(),
					CURLFORM_PTRNAME, it->first.c_str(),
					CURLFORM_CONTENTSLENGTH, it->second.size(),
					CURLFORM_PTRCONTENTS, it->second.c_str(),
					CURLFORM_END);
		}
		curl_easy_setopt(curl, CURLOPT_HTTPPOST, post);
		// request.post_fields must now *never* be
		// modified until CURLOPT_HTTPPOST is cleared
	} else if (request.post_data.empty()) {
		curl_easy_setopt(curl, CURLOPT_POST, 1);
		std::string str;
		for (StringMap::iterator it = request.post_fields.begin();
				it != request.post_fields.end(); ++it) {
			if (str != "")
				str += "&";
			str += urlencode(it->first);
			str += "=";
			str += urlencode(it->second);
		}
		curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE,
				str.size());
		curl_easy_setopt(curl, CURLOPT_COPYPOSTFIELDS,
				str.c_str());
	} else {
		curl_easy_setopt(curl, CURLOPT_POST, 1);
		curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE,
				request.post_data.size());
		curl_easy_setopt(curl, CURLOPT_POSTFIELDS,
				request.post_data.c_str());
		// request.post_data must now *never* be
		// modified until CURLOPT_POSTFIELDS is cleared
	}
	// Set additional HTTP headers
	for (std::vector<std::string>::iterator it = request.extra_headers.begin();
			it != request.extra_headers.end(); ++it) {
		http_header = curl_slist_append(http_header, it->c_str());
	}
	curl_easy_setopt(curl, CURLOPT_HTTPHEADER, http_header);

	if (!g_settings->getBool("curl_verify_cert")) {
		curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, false);
	}
}

CURLcode HTTPFetchOngoing::start(CURLM *multi_)
{
	if (!curl)
		return CURLE_FAILED_INIT;

	if (!multi_) {
		// Easy interface (sync)
		return curl_easy_perform(curl);
	}

	// Multi interface (async)
	CURLMcode mres = curl_multi_add_handle(multi_, curl);
	if (mres != CURLM_OK) {
		errorstream << "curl_multi_add_handle"
			<< " returned error code " << mres
			<< std::endl;
		return CURLE_FAILED_INIT;
	}
	multi = multi_; // store for curl_multi_remove_handle
	return CURLE_OK;
}

const HTTPFetchResult * HTTPFetchOngoing::complete(CURLcode res)
{
	result.succeeded = (res == CURLE_OK);
	result.timeout = (res == CURLE_OPERATION_TIMEDOUT);
	result.data = oss.str();

	// Get HTTP/FTP response code
	result.response_code = 0;
	if (curl && (curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE,
				&result.response_code) != CURLE_OK)) {
		// We failed to get a return code, make sure it is still 0
		result.response_code = 0;
	}

	if (res != CURLE_OK) {
		errorstream << request.url << " not found ("
			<< curl_easy_strerror(res) << ")"
			<< " (response code " << result.response_code << ")"
			<< std::endl;
	}

	return &result;
}

HTTPFetchOngoing::~HTTPFetchOngoing()
{
	if (multi) {
		CURLMcode mres = curl_multi_remove_handle(multi, curl);
		if (mres != CURLM_OK) {
			errorstream << "curl_multi_remove_handle"
				<< " returned error code " << mres
				<< std::endl;
		}
	}

	// Set safe options for the reusable cURL handle
	curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION,
			httpfetch_discardfunction);
	curl_easy_setopt(curl, CURLOPT_WRITEDATA, NULL);
	curl_easy_setopt(curl, CURLOPT_POSTFIELDS, NULL);
	if (http_header) {
		curl_easy_setopt(curl, CURLOPT_HTTPHEADER, NULL);
		curl_slist_free_all(http_header);
	}
	if (post) {
		curl_easy_setopt(curl, CURLOPT_HTTPPOST, NULL);
		curl_formfree(post);
	}

	// Store the cURL handle for reuse
	pool->free(curl);
}


class CurlFetchThread : public Thread
{
protected:
	enum RequestType {
		RT_FETCH,
		RT_CLEAR,
		RT_WAKEUP,
	};

	struct Request {
		RequestType type;
		HTTPFetchRequest fetch_request;
		Event *event;
	};

	CURLM *m_multi;
	MutexedQueue<Request> m_requests;
	size_t m_parallel_limit;

	// Variables exclusively used within thread
	std::vector<HTTPFetchOngoing*> m_all_ongoing;
	std::list<HTTPFetchRequest> m_queued_fetches;

public:
	CurlFetchThread(int parallel_limit) :
		Thread("CurlFetch")
	{
		if (parallel_limit >= 1)
			m_parallel_limit = parallel_limit;
		else
			m_parallel_limit = 1;
	}

	void requestFetch(const HTTPFetchRequest &fetch_request)
	{
		Request req;
		req.type = RT_FETCH;
		req.fetch_request = fetch_request;
		req.event = NULL;
		m_requests.push_back(req);
	}

	void requestClear(unsigned long caller, Event *event)
	{
		Request req;
		req.type = RT_CLEAR;
		req.fetch_request.caller = caller;
		req.event = event;
		m_requests.push_back(req);
	}

	void requestWakeUp()
	{
ofs	hex dump	ascii
0000	89 50 4e 47 0d 0a 1a 0a 00 00 00 0d 49 48 44 52 00 00 00 10 00 00 00 10 08 06 00 00 00 1f f3 ff	.PNG........IHDR................
0020	61 00 00 00 06 62 4b 47 44 00 ff 00 ff 00 ff a0 bd a7 93 00 00 00 09 70 48 59 73 00 00 0b 13 00	a....bKGD..............pHYs.....
0040	00 0b 13 01 00 9a 9c 18 00 00 00 07 74 49 4d 45 07 e1 01 1f 0a 09 15 95 5e d6 8a 00 00 00 45 69	............tIME........^.....Ei
0060	54 58 74 43 6f 6d 6d 65 6e 74 00 00 00 00 00 43 52 45 41 54 4f 52 3a 20 67 64 2d 6a 70 65 67 20	TXtComment.....CREATOR:.gd-jpeg.
0080	76 31 2e 30 20 28 75 73 69 6e 67 20 49 4a 47 20 4a 50 45 47 20 76 38 30 29 2c 20 71 75 61 6c 69	v1.0.(using.IJG.JPEG.v80),.quali
00a0	74 79 20 3d 20 39 30 0a aa 8c c5 0e 00 00 00 d9 49 44 41 54 38 cb ad 52 4b 0e 82 30 14 1c 2b a4	ty.=.90.........IDAT8..RK..0..+.
00c0	69 57 2c 9b ae ca c6 9b e8 19 8c 27 71 65 0c 26 1e c5 1b b8 f4 36 1c c0 0d 22 01 aa 8b 7e d2 8a	iW,........'qe.&.....6..."...~..
00e0	24 24 30 9b 79 99 d7 cc 9b be 16 8c 31 30 c6 30 05 52 4a 48 29 23 8d 50 4a 41 29 9d 64 a0 b5 86	$$0.y.......10.0.RJH)#.PJA).d...
0100	d6 3a 36 c0 4c 24 4d d3 4c 3e 5c 55 d5 40 9b 9d 60 05 e0 03 00 8f b3 11 76 97 b5 6f a6 69 0a 00	.:6.L$M.L>\U.@..`.......v..o.i..
0120	b8 1f df 00 80 6d 61 e6 11 42 90 24 c9 42 3b 70 c5 f3 65 b8 ef 7b df 0c 6b f7 0a 8e bb ae 8b 0d	.....ma..B.$.B;p..e..{..k.......
0140	32 3e 3e c5 99 ff c3 72 57 70 e0 dc 44 69 db d6 47 ce 78 1f f5 ea ba f6 0b 9e 9d c0 43 08 01 21	2>>....rWp..Di..G.x.........C..!
0160	44 a4 29 a5 a0 94 1a 9c cd f3 7c c1 8f 74 b0 c5 cd f2 de f2 06 40 61 eb 93 e5 ce f2 35 5c 22 19	D.).......\|..t.......@a.....5\".
0180	d9 6a 19 68 e5 8f 41 88 2f bd 2e 38 61 0f c3 ad 57 00 00 00 00 49 45 4e 44 ae 42 60 82	.j.h..A./..8a...W....IEND.B`.