From 86a3c8ce566f28112dc032aa48c40e12fde97be2 Mon Sep 17 00:00:00 2001 From: ShadowNinja Date: Sun, 14 Sep 2014 20:46:45 -0400 Subject: Reduce indentation of HTTPFetchOngoing Also clean up some related things. --- src/httpfetch.cpp | 437 +++++++++++++++++++++++++++--------------------------- 1 file changed, 222 insertions(+), 215 deletions(-) (limited to 'src/httpfetch.cpp') diff --git a/src/httpfetch.cpp b/src/httpfetch.cpp index c651055bc..f61dbbf71 100644 --- a/src/httpfetch.cpp +++ b/src/httpfetch.cpp @@ -52,12 +52,12 @@ HTTPFetchRequest::HTTPFetchRequest() } -static void httpfetch_deliver_result(const HTTPFetchResult &fetchresult) +static void httpfetch_deliver_result(const HTTPFetchResult &fetch_result) { - unsigned long caller = fetchresult.caller; + unsigned long caller = fetch_result.caller; if (caller != HTTPFETCH_DISCARD) { JMutexAutoLock lock(g_httpfetch_mutex); - g_httpfetch_results[caller].push_back(fetchresult); + g_httpfetch_results[caller].push_back(fetch_result); } } @@ -97,7 +97,7 @@ void httpfetch_caller_free(unsigned long caller) } } -bool httpfetch_async_get(unsigned long caller, HTTPFetchResult &fetchresult) +bool httpfetch_async_get(unsigned long caller, HTTPFetchResult &fetch_result) { JMutexAutoLock lock(g_httpfetch_mutex); @@ -108,13 +108,13 @@ bool httpfetch_async_get(unsigned long caller, HTTPFetchResult &fetchresult) return false; // Check that result queue is nonempty - std::list &callerresults = it->second; - if (callerresults.empty()) + std::list &caller_results = it->second; + if (caller_results.empty()) return false; // Pop first result - fetchresult = callerresults.front(); - callerresults.pop_front(); + fetch_result = caller_results.front(); + caller_results.pop_front(); return true; } @@ -175,8 +175,19 @@ public: } }; -struct HTTPFetchOngoing +class HTTPFetchOngoing { +public: + HTTPFetchOngoing(HTTPFetchRequest request, CurlHandlePool *pool); + ~HTTPFetchOngoing(); + + CURLcode start(CURLM *multi); + const HTTPFetchResult * complete(CURLcode res); + + const HTTPFetchRequest &getRequest() const { return request; }; + const CURL *getEasyHandle() const { return curl; }; + +private: CurlHandlePool *pool; CURL *curl; CURLM *multi; @@ -184,201 +195,200 @@ struct HTTPFetchOngoing HTTPFetchResult result; std::ostringstream oss; char *post_fields; - struct curl_slist *httpheader; + struct curl_slist *http_header; curl_httppost *post; +}; - HTTPFetchOngoing(HTTPFetchRequest request_, CurlHandlePool *pool_): - pool(pool_), - curl(NULL), - multi(NULL), - request(request_), - result(request_), - oss(std::ios::binary), - httpheader(NULL), - post(NULL) - { - curl = pool->alloc(); - if (curl != NULL) { - // Set static cURL options - curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1); - curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1); - curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1); - curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 1); -#if LIBCURL_VERSION_NUM >= 0x071304 - // Restrict protocols so that curl vulnerabilities in - // other protocols don't affect us. - // These settings were introduced in curl 7.19.4. - long protocols = - CURLPROTO_HTTP | - CURLPROTO_HTTPS | - CURLPROTO_FTP | - CURLPROTO_FTPS; - curl_easy_setopt(curl, CURLOPT_PROTOCOLS, protocols); - curl_easy_setopt(curl, CURLOPT_REDIR_PROTOCOLS, protocols); -#endif +HTTPFetchOngoing::HTTPFetchOngoing(HTTPFetchRequest request_, CurlHandlePool *pool_): + pool(pool_), + curl(NULL), + multi(NULL), + request(request_), + result(request_), + oss(std::ios::binary), + http_header(NULL), + post(NULL) +{ + curl = pool->alloc(); + if (curl == NULL) { + return; + } - // Set cURL options based on HTTPFetchRequest - curl_easy_setopt(curl, CURLOPT_URL, - request.url.c_str()); - curl_easy_setopt(curl, CURLOPT_TIMEOUT_MS, - request.timeout); - curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT_MS, - request.connect_timeout); - - if (request.useragent != "") - curl_easy_setopt(curl, CURLOPT_USERAGENT, request.useragent.c_str()); - - // Set up a write callback that writes to the - // ostringstream ongoing->oss, unless the data - // is to be discarded - if (request.caller == HTTPFETCH_DISCARD) { - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, - httpfetch_discardfunction); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, NULL); - } - else { - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, - httpfetch_writefunction); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, &oss); - } + // Set static cURL options + curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1); + curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1); + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1); + curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 1); - // Set POST (or GET) data - if (request.post_fields.empty()) { - curl_easy_setopt(curl, CURLOPT_HTTPGET, 1); - } else if (request.multipart) { - curl_httppost *last = NULL; - for (std::map::iterator it = - request.post_fields.begin(); - it != request.post_fields.end(); - ++it) { - curl_formadd(&post, &last, - CURLFORM_NAMELENGTH, it->first.size(), - CURLFORM_PTRNAME, it->first.c_str(), - CURLFORM_CONTENTSLENGTH, it->second.size(), - CURLFORM_PTRCONTENTS, it->second.c_str(), - CURLFORM_END); - } - curl_easy_setopt(curl, CURLOPT_HTTPPOST, post); - // request.post_fields must now *never* be - // modified until CURLOPT_HTTPPOST is cleared - } else { - curl_easy_setopt(curl, CURLOPT_POST, 1); - if (request.post_data.empty()) { - std::string str; - for (std::map::iterator it = - request.post_fields.begin(); - it != request.post_fields.end(); - ++it) { - if (str != "") - str += "&"; - str += urlencode(it->first); - str += "="; - str += urlencode(it->second); - } - curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, - str.size()); - curl_easy_setopt(curl, CURLOPT_COPYPOSTFIELDS, - str.c_str()); - } else { - curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, - request.post_data.size()); - curl_easy_setopt(curl, CURLOPT_POSTFIELDS, - request.post_data.c_str()); - // request.post_data must now *never* be - // modified until CURLOPT_POSTFIELDS is cleared - } - } - // Set additional HTTP headers - for (size_t i = 0; i < request.extra_headers.size(); ++i) { - httpheader = curl_slist_append( - httpheader, - request.extra_headers[i].c_str()); - } - curl_easy_setopt(curl, CURLOPT_HTTPHEADER, httpheader); +#if LIBCURL_VERSION_NUM >= 0x071304 + // Restrict protocols so that curl vulnerabilities in + // other protocols don't affect us. + // These settings were introduced in curl 7.19.4. + long protocols = + CURLPROTO_HTTP | + CURLPROTO_HTTPS | + CURLPROTO_FTP | + CURLPROTO_FTPS; + curl_easy_setopt(curl, CURLOPT_PROTOCOLS, protocols); + curl_easy_setopt(curl, CURLOPT_REDIR_PROTOCOLS, protocols); +#endif - if (!g_settings->getBool("curl_verify_cert")) { - curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, false); - } - } + // Set cURL options based on HTTPFetchRequest + curl_easy_setopt(curl, CURLOPT_URL, + request.url.c_str()); + curl_easy_setopt(curl, CURLOPT_TIMEOUT_MS, + request.timeout); + curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT_MS, + request.connect_timeout); + + if (request.useragent != "") + curl_easy_setopt(curl, CURLOPT_USERAGENT, request.useragent.c_str()); + + // Set up a write callback that writes to the + // ostringstream ongoing->oss, unless the data + // is to be discarded + if (request.caller == HTTPFETCH_DISCARD) { + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, + httpfetch_discardfunction); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, NULL); + } else { + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, + httpfetch_writefunction); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &oss); } - CURLcode start(CURLM *multi_) - { - if (curl == NULL) - return CURLE_FAILED_INIT; - - if (multi_) { - // Multi interface (async) - CURLMcode mres = curl_multi_add_handle(multi_, curl); - if (mres != CURLM_OK) { - errorstream<<"curl_multi_add_handle" - <<" returned error code "<::iterator it = + request.post_fields.begin(); + it != request.post_fields.end(); ++it) { + curl_formadd(&post, &last, + CURLFORM_NAMELENGTH, it->first.size(), + CURLFORM_PTRNAME, it->first.c_str(), + CURLFORM_CONTENTSLENGTH, it->second.size(), + CURLFORM_PTRCONTENTS, it->second.c_str(), + CURLFORM_END); } - else { - // Easy interface (sync) - return curl_easy_perform(curl); + curl_easy_setopt(curl, CURLOPT_HTTPPOST, post); + // request.post_fields must now *never* be + // modified until CURLOPT_HTTPPOST is cleared + } else if (request.post_data.empty()) { + curl_easy_setopt(curl, CURLOPT_POST, 1); + std::string str; + for (std::map::iterator it = + request.post_fields.begin(); + it != request.post_fields.end(); + ++it) { + if (str != "") + str += "&"; + str += urlencode(it->first); + str += "="; + str += urlencode(it->second); } + curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, + str.size()); + curl_easy_setopt(curl, CURLOPT_COPYPOSTFIELDS, + str.c_str()); + } else { + curl_easy_setopt(curl, CURLOPT_POST, 1); + curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, + request.post_data.size()); + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, + request.post_data.c_str()); + // request.post_data must now *never* be + // modified until CURLOPT_POSTFIELDS is cleared + } + // Set additional HTTP headers + for (std::vector::iterator it = request.extra_headers.begin(); + it != request.extra_headers.end(); ++it) { + http_header = curl_slist_append(http_header, it->c_str()); } + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, http_header); - void complete(CURLcode res) - { - result.succeeded = (res == CURLE_OK); - result.timeout = (res == CURLE_OPERATION_TIMEDOUT); - result.data = oss.str(); + if (!g_settings->getBool("curl_verify_cert")) { + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, false); + } +} + +CURLcode HTTPFetchOngoing::start(CURLM *multi_) +{ + if (!curl) + return CURLE_FAILED_INIT; + + if (!multi_) { + // Easy interface (sync) + return curl_easy_perform(curl); + } - // Get HTTP/FTP response code + // Multi interface (async) + CURLMcode mres = curl_multi_add_handle(multi_, curl); + if (mres != CURLM_OK) { + errorstream << "curl_multi_add_handle" + << " returned error code " << mres + << std::endl; + return CURLE_FAILED_INIT; + } + multi = multi_; // store for curl_multi_remove_handle + return CURLE_OK; +} + +const HTTPFetchResult * HTTPFetchOngoing::complete(CURLcode res) +{ + result.succeeded = (res == CURLE_OK); + result.timeout = (res == CURLE_OPERATION_TIMEDOUT); + result.data = oss.str(); + + // Get HTTP/FTP response code + result.response_code = 0; + if (curl && (curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, + &result.response_code) != CURLE_OK)) { + // We failed to get a return code, make sure it is still 0 result.response_code = 0; - if (curl != NULL) { - if (curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, - &result.response_code) != CURLE_OK) { - //we failed to get a return code make sure it is still 0 - result.response_code = 0; - } - } + } - if (res != CURLE_OK) { - errorstream<free(curl); + // Set safe options for the reusable cURL handle + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, + httpfetch_discardfunction); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, NULL); + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, NULL); + if (http_header) { + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, NULL); + curl_slist_free_all(http_header); } -}; + if (post) { + curl_easy_setopt(curl, CURLOPT_HTTPPOST, NULL); + curl_formfree(post); + } + + // Store the cURL handle for reuse + pool->free(curl); +} + class CurlFetchThread : public JThread { @@ -391,7 +401,7 @@ protected: struct Request { RequestType type; - HTTPFetchRequest fetchrequest; + HTTPFetchRequest fetch_request; Event *event; }; @@ -412,11 +422,11 @@ public: m_parallel_limit = 1; } - void requestFetch(const HTTPFetchRequest &fetchrequest) + void requestFetch(const HTTPFetchRequest &fetch_request) { Request req; req.type = RT_FETCH; - req.fetchrequest = fetchrequest; + req.fetch_request = fetch_request; req.event = NULL; m_requests.push_back(req); } @@ -425,7 +435,7 @@ public: { Request req; req.type = RT_CLEAR; - req.fetchrequest.caller = caller; + req.fetch_request.caller = caller; req.event = event; m_requests.push_back(req); } @@ -446,24 +456,24 @@ protected: if (req.type == RT_FETCH) { // New fetch, queue until there are less // than m_parallel_limit ongoing fetches - m_queued_fetches.push_back(req.fetchrequest); + m_queued_fetches.push_back(req.fetch_request); // see processQueued() for what happens next } else if (req.type == RT_CLEAR) { - unsigned long caller = req.fetchrequest.caller; + unsigned long caller = req.fetch_request.caller; // Abort all ongoing fetches for the caller for (std::vector::iterator it = m_all_ongoing.begin(); it != m_all_ongoing.end();) { - if ((*it)->request.caller == caller) { + if ((*it)->getRequest().caller == caller) { delete (*it); it = m_all_ongoing.erase(it); - } - else + } else { ++it; + } } // Also abort all queued fetches for the caller @@ -503,8 +513,7 @@ protected: m_all_ongoing.push_back(ongoing); } else { - ongoing->complete(res); - httpfetch_deliver_result(ongoing->result); + httpfetch_deliver_result(*ongoing->complete(res)); delete ongoing; } } @@ -517,7 +526,7 @@ protected: size_t i = 0; bool found = false; for (i = 0; i < m_all_ongoing.size(); ++i) { - if (m_all_ongoing[i]->curl == msg->easy_handle) { + if (m_all_ongoing[i]->getEasyHandle() == msg->easy_handle) { found = true; break; } @@ -525,8 +534,7 @@ protected: if (msg->msg == CURLMSG_DONE && found) { // m_all_ongoing[i] succeeded or failed. HTTPFetchOngoing *ongoing = m_all_ongoing[i]; - ongoing->complete(msg->data.result); - httpfetch_deliver_result(ongoing->result); + httpfetch_deliver_result(*ongoing->complete(msg->data.result)); delete ongoing; m_all_ongoing.erase(m_all_ongoing.begin() + i); } @@ -719,9 +727,9 @@ void httpfetch_cleanup() curl_global_cleanup(); } -void httpfetch_async(const HTTPFetchRequest &fetchrequest) +void httpfetch_async(const HTTPFetchRequest &fetch_request) { - g_httpfetch_thread->requestFetch(fetchrequest); + g_httpfetch_thread->requestFetch(fetch_request); if (!g_httpfetch_thread->IsRunning()) g_httpfetch_thread->Start(); } @@ -738,18 +746,17 @@ static void httpfetch_request_clear(unsigned long caller) } } -void httpfetch_sync(const HTTPFetchRequest &fetchrequest, - HTTPFetchResult &fetchresult) +void httpfetch_sync(const HTTPFetchRequest &fetch_request, + HTTPFetchResult &fetch_result) { // Create ongoing fetch data and make a cURL handle // Set cURL options based on HTTPFetchRequest CurlHandlePool pool; - HTTPFetchOngoing ongoing(fetchrequest, &pool); + HTTPFetchOngoing ongoing(fetch_request, &pool); // Do the fetch (curl_easy_perform) CURLcode res = ongoing.start(NULL); - // Update fetchresult - ongoing.complete(res); - fetchresult = ongoing.result; + // Update fetch result + fetch_result = *ongoing.complete(res); } #else // USE_CURL @@ -768,26 +775,26 @@ void httpfetch_cleanup() { } -void httpfetch_async(const HTTPFetchRequest &fetchrequest) +void httpfetch_async(const HTTPFetchRequest &fetch_request) { - errorstream<<"httpfetch_async: unable to fetch "<