Rewrite client media download and support hash-based remote download

Move most of the media-related code in client.cpp into a new class
ClientMediaDownloader (clientmedia.cpp, clientmedia.h). Among other
things, this class does the following things:

- Download [remote_server][sha1] instead of [remote_server][name]. This
is to support servers that provide the same file name with different
contents.
- Initially fetch [remote_server]index.mth. This file should follow the
Minetest Hashset format (currently version 1) and contain a list of SHA1
hashes that exist on the server.
- The list of needed SHA1s is uploaded (via HTTP POST) when index.mth is
requested, so servers can optionally narrow down the list to the needs
of the client.
- If index.mth is missing (HTTP response code 404), we enter compat mode,
fetching [remote_server][name] as before this commit.
- remote_server can now contain multiple servers, separated by commas.
The downloader code attempts to split requests between the different
servers, as permitted by each server's index.mth. If one server claims
to have a file but actually doesn't (or something fails), we ask a
different server that also claims to have it.
- As before, when none of the remote servers provide a particular
file, we download it via the conventional method, i.e. using
the minetest protocol: TOSERVER_REQUEST_MEDIA / TOCLIENT_MEDIA.
- Bugfix: Every downloaded file's SHA1 is now verified against the SHA1
announced by the minetest server (before loading it and inserting it
into the file cache).
- Bugfix: Only send TOSERVER_RECEIVED_MEDIA when we actually have all
media. This should fix #863.

1 files changed, 150 insertions, 0 deletions

diff --git a/src/clientmedia.h b/src/clientmedia.h
new file mode 100644
index 000000000..540044239
--- /dev/null
+++ b/src/clientmedia.h
@@ -0,0 +1,150 @@
+/*
+Minetest
+Copyright (C) 2013 celeron55, Perttu Ahola <celeron55@gmail.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU Lesser General Public License as published by
+the Free Software Foundation; either version 2.1 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+*/
+
+#ifndef CLIENTMEDIA_HEADER
+#define CLIENTMEDIA_HEADER
+
+#include "irrlichttypes.h"
+#include "filecache.h"
+#include <ostream>
+#include <map>
+#include <set>
+#include <vector>
+
+class Client;
+struct HTTPFetchResult;
+
+#define MTHASHSET_FILE_SIGNATURE 0x4d544853 // 'MTHS'
+#define MTHASHSET_FILE_NAME "index.mth"
+
+class ClientMediaDownloader
+{
+public:
+	ClientMediaDownloader();
+	~ClientMediaDownloader();
+
+	float getProgress() const {
+		if (m_uncached_count >= 1)
+			return 1.0 * m_uncached_received_count /
+				m_uncached_count;
+		else
+			return 0.0;
+	}
+
+	bool isStarted() const {
+		return m_initial_step_done;
+	}
+
+	// If this returns true, the downloader is done and can be deleted
+	bool isDone() const {
+		return m_initial_step_done &&
+			m_uncached_received_count == m_uncached_count;
+	}
+
+	// Add a file to the list of required file (but don't fetch it yet)
+	void addFile(std::string name, std::string sha1);
+
+	// Add a remote server to the list; ignored if not built with cURL
+	void addRemoteServer(std::string baseurl);
+
+	// Steps the media downloader:
+	// - May load media into client by calling client->loadMedia()
+	// - May check media cache for files
+	// - May add files to media cache
+	// - May start remote transfers by calling httpfetch_async
+	// - May check for completion of current remote transfers
+	// - May start conventional transfers by calling client->request_media()
+	// - May inform server that all media has been loaded
+	//   by calling client->received_media()
+	// After step has been called once, don't call addFile/addRemoteServer.
+	void step(Client *client);
+
+	// Must be called for each file received through TOCLIENT_MEDIA
+	void conventionalTransferDone(
+			const std::string &name,
+			const std::string &data,
+			Client *client);
+
+private:
+	struct FileStatus {
+		bool received;
+		std::string sha1;
+		s32 current_remote;
+		std::vector<s32> available_remotes;
+	};
+
+	struct RemoteServerStatus {
+		std::string baseurl;
+		s32 active_count;
+		bool request_by_filename;
+	};
+
+	void initialStep(Client *client);
+	void remoteHashSetReceived(const HTTPFetchResult &fetchresult);
+	void remoteMediaReceived(const HTTPFetchResult &fetchresult,
+			Client *client);
+	s32 selectRemoteServer(FileStatus *filestatus);
+	void startRemoteMediaTransfers();
+	void startConventionalTransfers(Client *client);
+
+	bool checkAndLoad(const std::string &name, const std::string &sha1,
+			const std::string &data, bool is_from_cache,
+			Client *client);
+
+	std::string serializeRequiredHashSet();
+	static void deSerializeHashSet(const std::string &data,
+			std::set<std::string> &result);
+
+	// Maps filename to file status
+	std::map<std::string, FileStatus*> m_files;
+
+	// Array of remote media servers
+	std::vector<RemoteServerStatus*> m_remotes;
+
+	// Filesystem-based media cache
+	FileCache m_media_cache;
+
+	// Has an attempt been made to load media files from the file cache?
+	// Have hash sets been requested from remote servers?
+	bool m_initial_step_done;
+
+	// Total number of media files to load
+	s32 m_uncached_count;
+
+	// Number of media files that have been received
+	s32 m_uncached_received_count;
+
+	// Status of remote transfers
+	unsigned long m_httpfetch_caller;
+	unsigned long m_httpfetch_next_id;
+	long m_httpfetch_timeout;
+	s32 m_httpfetch_active;
+	s32 m_httpfetch_active_limit;
+	s32 m_outstanding_hash_sets;
+	std::map<unsigned long, std::string> m_remote_file_transfers;
+
+	// All files up to this name have either been received from a
+	// remote server or failed on all remote servers, so those files
+	// don't need to be looked at again
+	// (use m_files.upper_bound(m_name_bound) to get an iterator)
+	std::string m_name_bound;
+
+};
+
+#endif // !CLIENTMEDIA_HEADER