diff options
author | Ilya Zhuravlev <zhuravlevilya@ya.ru> | 2012-12-14 15:30:17 +0400 |
---|---|---|
committer | Ilya Zhuravlev <zhuravlevilya@ya.ru> | 2012-12-16 15:20:18 +0400 |
commit | 3578e1d4a711a32e2e768dcc6ff207ffc1bccbad (patch) | |
tree | 9215740fb64bbb89ce9b9ba89f8ebf0db709ee35 | |
parent | aa46e5c5e7285a8ac5c9e096a55c3fa02b90280e (diff) | |
download | minetest-3578e1d4a711a32e2e768dcc6ff207ffc1bccbad.tar.gz minetest-3578e1d4a711a32e2e768dcc6ff207ffc1bccbad.tar.bz2 minetest-3578e1d4a711a32e2e768dcc6ff207ffc1bccbad.zip |
Added ability to fetch media from remote server (using cURL library)
-rw-r--r-- | cmake/Modules/FindCURL.cmake | 17 | ||||
-rw-r--r-- | minetest.conf.example | 5 | ||||
-rw-r--r-- | src/CMakeLists.txt | 23 | ||||
-rw-r--r-- | src/client.cpp | 219 | ||||
-rw-r--r-- | src/client.h | 36 | ||||
-rw-r--r-- | src/clientserver.h | 6 | ||||
-rw-r--r-- | src/cmake_config.h.in | 1 | ||||
-rw-r--r-- | src/config.h | 3 | ||||
-rw-r--r-- | src/defaultsettings.cpp | 3 | ||||
-rw-r--r-- | src/server.cpp | 5 | ||||
-rw-r--r-- | src/util/string.cpp | 6 | ||||
-rw-r--r-- | src/util/string.h | 1 |
12 files changed, 277 insertions, 48 deletions
diff --git a/cmake/Modules/FindCURL.cmake b/cmake/Modules/FindCURL.cmake new file mode 100644 index 000000000..ec0503fd9 --- /dev/null +++ b/cmake/Modules/FindCURL.cmake @@ -0,0 +1,17 @@ +# - Find curl +# Find the native CURL headers and libraries. +# +# CURL_INCLUDE_DIR - where to find curl/curl.h, etc. +# CURL_LIBRARY - List of libraries when using curl. +# CURL_FOUND - True if curl found. + +# Look for the header file. +FIND_PATH(CURL_INCLUDE_DIR NAMES curl/curl.h) + +# Look for the library. +FIND_LIBRARY(CURL_LIBRARY NAMES curl) + +# handle the QUIETLY and REQUIRED arguments and set CURL_FOUND to TRUE if +# all listed variables are TRUE +INCLUDE(FindPackageHandleStandardArgs) +FIND_PACKAGE_HANDLE_STANDARD_ARGS(CURL DEFAULT_MSG CURL_LIBRARY CURL_INCLUDE_DIR) diff --git a/minetest.conf.example b/minetest.conf.example index a0aa69125..e80991936 100644 --- a/minetest.conf.example +++ b/minetest.conf.example @@ -141,6 +141,10 @@ # 2: enable high level shaders #enable_shaders = 2 +# will only work for servers which use remote_media setting +# and only for clients compiled with cURL +#media_fetch_threads = 8 + # # Server stuff # @@ -217,3 +221,4 @@ #congestion_control_aim_rtt = 0.2 #congestion_control_max_rate = 400 #congestion_control_min_rate = 10 +#remote_media = diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 3830ef3b6..38410f7d2 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -6,6 +6,19 @@ mark_as_advanced(EXECUTABLE_OUTPUT_PATH LIBRARY_OUTPUT_PATH) mark_as_advanced(JTHREAD_INCLUDE_DIR JTHREAD_LIBRARY) mark_as_advanced(SQLITE3_INCLUDE_DIR SQLITE3_LIBRARY) +option(ENABLE_CURL "Enable cURL support for fetching media" 1) + +if (NOT ENABLE_CURL) + mark_as_advanced(CLEAR CURL_LIBRARY CURL_INCLUDE_DIR) +endif(NOT ENABLE_CURL) + +find_package(CURL) +set(USE_CURL 0) +if (CURL_FOUND AND ENABLE_CURL) + message(STATUS "cURL support enabled") + set(USE_CURL 1) +endif(CURL_FOUND AND ENABLE_CURL) + # user-visible option to enable/disable gettext usage OPTION(ENABLE_GETTEXT "Use GetText for internationalization" 0) @@ -307,6 +320,16 @@ if(BUILD_CLIENT) ${PLATFORM_LIBS} ${CLIENT_PLATFORM_LIBS} ) + + if(USE_CURL) + target_link_libraries( + ${PROJECT_NAME} + ${CURL_LIBRARY} + ) + include_directories( + ${CURL_INCLUDE_DIR} + ) + endif(USE_CURL) endif(BUILD_CLIENT) if(BUILD_SERVER) diff --git a/src/client.cpp b/src/client.cpp index 3463e9262..46b53c6d7 100644 --- a/src/client.cpp +++ b/src/client.cpp @@ -44,21 +44,17 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "hex.h" #include "IMeshCache.h" #include "util/serialize.h" +#include "config.h" + +#if USE_CURL +#include <curl/curl.h> +#endif static std::string getMediaCacheDir() { return porting::path_user + DIR_DELIM + "cache" + DIR_DELIM + "media"; } -struct MediaRequest -{ - std::string name; - - MediaRequest(const std::string &name_=""): - name(name_) - {} -}; - /* QueuedMeshUpdate */ @@ -223,6 +219,45 @@ void * MeshUpdateThread::Thread() return NULL; } +void * MediaFetchThread::Thread() +{ + ThreadStarted(); + + log_register_thread("MediaFetchThread"); + + DSTACK(__FUNCTION_NAME); + + BEGIN_DEBUG_EXCEPTION_HANDLER + + #if USE_CURL + CURL *curl; + CURLcode res; + for (core::list<MediaRequest>::Iterator i = m_file_requests.begin(); + i != m_file_requests.end(); i++) { + curl = curl_easy_init(); + assert(curl); + curl_easy_setopt(curl, CURLOPT_URL, (m_remote_url + i->name).c_str()); + curl_easy_setopt(curl, CURLOPT_FAILONERROR, true); + std::ostringstream stream; + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curl_write_data); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &stream); + res = curl_easy_perform(curl); + if (res == CURLE_OK) { + std::string data = stream.str(); + m_file_data.push_back(make_pair(i->name, data)); + } else { + m_failed.push_back(*i); + infostream << "cURL request failed for " << i->name << std::endl; + } + curl_easy_cleanup(curl); + } + #endif + + END_DEBUG_EXCEPTION_HANDLER(errorstream) + + return NULL; +} + Client::Client( IrrlichtDevice *device, const char *playername, @@ -263,8 +298,9 @@ Client::Client( m_password(password), m_access_denied(false), m_media_cache(getMediaCacheDir()), - m_media_receive_progress(0), - m_media_received(false), + m_media_receive_started(false), + m_media_count(0), + m_media_received_count(0), m_itemdef_received(false), m_nodedef_received(false), m_time_of_day_set(false), @@ -731,6 +767,63 @@ void Client::step(float dtime) } /* + Load fetched media + */ + if (m_media_receive_started) { + bool all_stopped = true; + for (core::list<MediaFetchThread>::Iterator thread = m_media_fetch_threads.begin(); + thread != m_media_fetch_threads.end(); thread++) { + all_stopped &= !thread->IsRunning(); + while (thread->m_file_data.size() > 0) { + std::pair <std::string, std::string> out = thread->m_file_data.pop_front(); + ++m_media_received_count; + + bool success = loadMedia(out.second, out.first); + if(success){ + verbosestream<<"Client: Loaded received media: " + <<"\""<<out.first<<"\". Caching."<<std::endl; + } else{ + infostream<<"Client: Failed to load received media: " + <<"\""<<out.first<<"\". Not caching."<<std::endl; + continue; + } + + bool did = fs::CreateAllDirs(getMediaCacheDir()); + if(!did){ + errorstream<<"Could not create media cache directory" + <<std::endl; + } + + { + core::map<std::string, std::string>::Node *n; + n = m_media_name_sha1_map.find(out.first); + if(n == NULL) + errorstream<<"The server sent a file that has not " + <<"been announced."<<std::endl; + else + m_media_cache.update_sha1(out.second); + } + } + } + if (all_stopped) { + core::list<MediaRequest> fetch_failed; + for (core::list<MediaFetchThread>::Iterator thread = m_media_fetch_threads.begin(); + thread != m_media_fetch_threads.end(); thread++) { + for (core::list<MediaRequest>::Iterator request = thread->m_failed.begin(); + request != thread->m_failed.end(); request++) + fetch_failed.push_back(*request); + thread->m_failed.clear(); + } + if (fetch_failed.size() > 0) { + infostream << "Failed to remote-fetch " << fetch_failed.size() << " files. " + << "Requesting them the usual way." << std::endl; + request_media(fetch_failed); + } + m_media_fetch_threads.clear(); + } + } + + /* If the server didn't update the inventory in a while, revert the local inventory (so the player notices the lag problem and knows something is wrong). @@ -907,6 +1000,34 @@ void Client::deletingPeer(con::Peer *peer, bool timeout) <<"(timeout="<<timeout<<")"<<std::endl; } +/* + u16 command + u16 number of files requested + for each file { + u16 length of name + string name + } +*/ +void Client::request_media(const core::list<MediaRequest> &file_requests) +{ + std::ostringstream os(std::ios_base::binary); + writeU16(os, TOSERVER_REQUEST_MEDIA); + writeU16(os, file_requests.size()); + + for(core::list<MediaRequest>::ConstIterator i = file_requests.begin(); + i != file_requests.end(); i++) { + os<<serializeString(i->name); + } + + // Make data buffer + std::string s = os.str(); + SharedBuffer<u8> data((u8*)s.c_str(), s.size()); + // Send as reliable + Send(0, data, true); + infostream<<"Client: Sending media request list to server (" + <<file_requests.size()<<" files)"<<std::endl; +} + void Client::ReceiveAll() { DSTACK(__FUNCTION_NAME); @@ -1514,37 +1635,56 @@ void Client::ProcessData(u8 *data, u32 datasize, u16 sender_peer_id) file_requests.push_back(MediaRequest(name)); } - ClientEvent event; - event.type = CE_TEXTURES_UPDATED; - m_client_event_queue.push_back(event); + std::string remote_media = ""; + try { + remote_media = deSerializeString(is); + } + catch(SerializationError) { + // not supported by server or turned off + } - /* - u16 command - u16 number of files requested - for each file { - u16 length of name - string name + m_media_count = file_requests.size(); + m_media_receive_started = true; + + if (remote_media == "" || !USE_CURL) { + request_media(file_requests); + } else { + #if USE_CURL + for (size_t i = 0; i < g_settings->getU16("media_fetch_threads"); ++i) { + m_media_fetch_threads.push_back(MediaFetchThread(this)); } - */ - std::ostringstream os(std::ios_base::binary); - writeU16(os, TOSERVER_REQUEST_MEDIA); - writeU16(os, file_requests.size()); - for(core::list<MediaRequest>::Iterator i = file_requests.begin(); - i != file_requests.end(); i++) { - os<<serializeString(i->name); - } + core::list<MediaFetchThread>::Iterator cur = m_media_fetch_threads.begin(); + for(core::list<MediaRequest>::Iterator i = file_requests.begin(); + i != file_requests.end(); i++) { + cur->m_file_requests.push_back(*i); + cur++; + if (cur == m_media_fetch_threads.end()) + cur = m_media_fetch_threads.begin(); + } + for (core::list<MediaFetchThread>::Iterator i = m_media_fetch_threads.begin(); + i != m_media_fetch_threads.end(); i++) { + i->m_remote_url = remote_media; + i->Start(); + } + #endif - // Make data buffer - std::string s = os.str(); - SharedBuffer<u8> data((u8*)s.c_str(), s.size()); - // Send as reliable - Send(0, data, true); - infostream<<"Client: Sending media request list to server (" - <<file_requests.size()<<" files)"<<std::endl; + // notify server we received everything + std::ostringstream os(std::ios_base::binary); + writeU16(os, TOSERVER_RECEIVED_MEDIA); + std::string s = os.str(); + SharedBuffer<u8> data((u8*)s.c_str(), s.size()); + // Send as reliable + Send(0, data, true); + } + ClientEvent event; + event.type = CE_TEXTURES_UPDATED; + m_client_event_queue.push_back(event); } else if(command == TOCLIENT_MEDIA) { + if (m_media_count == 0) + return; std::string datastring((char*)&data[2], datasize-2); std::istringstream is(datastring, std::ios_base::binary); @@ -1566,17 +1706,12 @@ void Client::ProcessData(u8 *data, u32 datasize, u16 sender_peer_id) */ int num_bunches = readU16(is); int bunch_i = readU16(is); - if(num_bunches >= 2) - m_media_receive_progress = (float)bunch_i / (float)(num_bunches - 1); - else - m_media_receive_progress = 1.0; - if(bunch_i == num_bunches - 1) - m_media_received = true; int num_files = readU32(is); infostream<<"Client: Received files: bunch "<<bunch_i<<"/" <<num_bunches<<" files="<<num_files <<" size="<<datasize<<std::endl; for(int i=0; i<num_files; i++){ + m_media_received_count++; std::string name = deSerializeString(is); std::string data = deSerializeLongString(is); @@ -2458,7 +2593,7 @@ void Client::afterContentReceived() infostream<<"Client::afterContentReceived() started"<<std::endl; assert(m_itemdef_received); assert(m_nodedef_received); - assert(m_media_received); + assert(texturesReceived()); // remove the information about which checksum each texture // ought to have diff --git a/src/client.h b/src/client.h index f85e8ac7b..c6858f549 100644 --- a/src/client.h +++ b/src/client.h @@ -33,6 +33,7 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "filesys.h" #include "filecache.h" #include "localplayer.h" +#include "server.h" #include "util/pointedthing.h" struct MeshMakeData; @@ -129,6 +130,24 @@ public: IGameDef *m_gamedef; }; +class MediaFetchThread : public SimpleThread +{ +public: + + MediaFetchThread(IGameDef *gamedef): + m_gamedef(gamedef) + { + } + + void * Thread(); + + core::list<MediaRequest> m_file_requests; + MutexedQueue<std::pair<std::string, std::string> > m_file_data; + core::list<MediaRequest> m_failed; + std::string m_remote_url; + IGameDef *m_gamedef; +}; + enum ClientEventType { CE_NONE, @@ -289,10 +308,13 @@ public: { return m_access_denied_reason; } float mediaReceiveProgress() - { return m_media_receive_progress; } + { + if (!m_media_receive_started) return 0; + return 1.0 * m_media_received_count / m_media_count; + } bool texturesReceived() - { return m_media_received; } + { return m_media_receive_started && m_media_received_count == m_media_count; } bool itemdefReceived() { return m_itemdef_received; } bool nodedefReceived() @@ -318,7 +340,9 @@ private: // Insert a media file appropriately into the appropriate manager bool loadMedia(const std::string &data, const std::string &filename); - + + void request_media(const core::list<MediaRequest> &file_requests); + // Virtual methods from con::PeerHandler void peerAdded(con::Peer *peer); void deletingPeer(con::Peer *peer, bool timeout); @@ -347,6 +371,7 @@ private: MtEventManager *m_event; MeshUpdateThread m_mesh_update_thread; + core::list<MediaFetchThread> m_media_fetch_threads; ClientEnvironment m_env; con::Connection m_con; IrrlichtDevice *m_device; @@ -375,8 +400,9 @@ private: FileCache m_media_cache; // Mapping from media file name to SHA1 checksum core::map<std::string, std::string> m_media_name_sha1_map; - float m_media_receive_progress; - bool m_media_received; + bool m_media_receive_started; + u32 m_media_count; + u32 m_media_received_count; bool m_itemdef_received; bool m_nodedef_received; friend class FarMesh; diff --git a/src/clientserver.h b/src/clientserver.h index 6f9396c02..db551a90c 100644 --- a/src/clientserver.h +++ b/src/clientserver.h @@ -267,6 +267,8 @@ enum ToClientCommand u32 length of data data } + u16 length of remote media server url (if applicable) + string url */ TOCLIENT_TOOLDEF = 0x39, @@ -571,6 +573,10 @@ enum ToServerCommand } */ + TOSERVER_RECEIVED_MEDIA = 0x41, + /* + u16 command + */ }; #endif diff --git a/src/cmake_config.h.in b/src/cmake_config.h.in index c2bdc9670..4853d854f 100644 --- a/src/cmake_config.h.in +++ b/src/cmake_config.h.in @@ -7,6 +7,7 @@ #define CMAKE_VERSION_STRING "@VERSION_STRING@" #define CMAKE_RUN_IN_PLACE @RUN_IN_PLACE@ #define CMAKE_USE_GETTEXT @USE_GETTEXT@ +#define CMAKE_USE_CURL @USE_CURL@ #define CMAKE_USE_SOUND @USE_SOUND@ #define CMAKE_STATIC_SHAREDIR "@SHAREDIR@" diff --git a/src/config.h b/src/config.h index aedca8b20..f37ec0fed 100644 --- a/src/config.h +++ b/src/config.h @@ -11,6 +11,7 @@ #define RUN_IN_PLACE 0 #define USE_GETTEXT 0 #define USE_SOUND 0 +#define USE_CURL 0 #define STATIC_SHAREDIR "" #define BUILD_INFO "non-cmake" @@ -26,6 +27,8 @@ #define USE_GETTEXT CMAKE_USE_GETTEXT #undef USE_SOUND #define USE_SOUND CMAKE_USE_SOUND + #undef USE_CURL + #define USE_CURL CMAKE_USE_CURL #undef STATIC_SHAREDIR #define STATIC_SHAREDIR CMAKE_STATIC_SHAREDIR #undef BUILD_INFO diff --git a/src/defaultsettings.cpp b/src/defaultsettings.cpp index d824d2923..7cb781276 100644 --- a/src/defaultsettings.cpp +++ b/src/defaultsettings.cpp @@ -119,6 +119,8 @@ void set_default_settings(Settings *settings) settings->setDefault("preload_item_visuals", "true"); settings->setDefault("enable_shaders", "2"); + settings->setDefault("media_fetch_threads", "8"); + // Server stuff // "map-dir" doesn't exist by default. settings->setDefault("default_game", "minetest"); @@ -158,5 +160,6 @@ void set_default_settings(Settings *settings) settings->setDefault("congestion_control_aim_rtt", "0.2"); settings->setDefault("congestion_control_max_rate", "400"); settings->setDefault("congestion_control_min_rate", "10"); + settings->setDefault("remote_media", ""); } diff --git a/src/server.cpp b/src/server.cpp index 2449f4236..4b43a3205 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -2885,6 +2885,9 @@ void Server::ProcessData(u8 *data, u32 datasize, u16 peer_id) // (definitions and files) getClient(peer_id)->definitions_sent = true; } + else if(command == TOSERVER_RECEIVED_MEDIA) { + getClient(peer_id)->definitions_sent = true; + } else if(command == TOSERVER_INTERACT) { std::string datastring((char*)&data[2], datasize-2); @@ -4217,6 +4220,7 @@ void Server::sendMediaAnnouncement(u16 peer_id) os<<serializeString(j->name); os<<serializeString(j->sha1_digest); } + os<<serializeString(g_settings->get("remote_media")); // Make data buffer std::string s = os.str(); @@ -4224,7 +4228,6 @@ void Server::sendMediaAnnouncement(u16 peer_id) // Send as reliable m_con.Send(peer_id, 0, data, true); - } struct SendableMedia diff --git a/src/util/string.cpp b/src/util/string.cpp index fb39a24c3..215ac299d 100644 --- a/src/util/string.cpp +++ b/src/util/string.cpp @@ -41,3 +41,9 @@ std::string translatePassword(std::string playername, std::wstring password) return pwd; } +size_t curl_write_data(char *ptr, size_t size, size_t nmemb, void *userdata) { + std::ostringstream *stream = (std::ostringstream*)userdata; + size_t count = size * nmemb; + stream->write(ptr, count); + return count; +} diff --git a/src/util/string.h b/src/util/string.h index 71b11de3d..58274c677 100644 --- a/src/util/string.h +++ b/src/util/string.h @@ -282,6 +282,7 @@ inline std::string wrap_rows(const std::string &from, u32 rowlen) } std::string translatePassword(std::string playername, std::wstring password); +size_t curl_write_data(char *ptr, size_t size, size_t nmemb, void *userdata); #endif |