From ad75ebe5b3f10e77f1150d2d8111e6a60cb9039a Mon Sep 17 00:00:00 2001 From: Tay Ray Chuan Date: Fri, 27 Nov 2009 23:42:26 +0800 Subject: http: maintain curl sessions Allow curl sessions to be kept alive (ie. not ended with curl_easy_cleanup()) even after the request is completed, the number of which is determined by the configuration setting http.minSessions. Add a count for curl sessions, and update it, across slots, when starting and ending curl sessions. Signed-off-by: Tay Ray Chuan Signed-off-by: Junio C Hamano diff --git a/Documentation/config.txt b/Documentation/config.txt index a8e0876..b77d66d 100644 --- a/Documentation/config.txt +++ b/Documentation/config.txt @@ -1132,6 +1132,12 @@ http.maxRequests:: How many HTTP requests to launch in parallel. Can be overridden by the 'GIT_HTTP_MAX_REQUESTS' environment variable. Default is 5. +http.minSessions:: + The number of curl sessions (counted across slots) to be kept across + requests. They will not be ended with curl_easy_cleanup() until + http_cleanup() is invoked. If USE_CURL_MULTI is not defined, this + value will be capped at 1. Defaults to 1. + http.postBuffer:: Maximum size in bytes of the buffer used by smart HTTP transports when POSTing data to the remote system. diff --git a/http.c b/http.c index ed6414a..fb0a97b 100644 --- a/http.c +++ b/http.c @@ -7,6 +7,8 @@ int active_requests; int http_is_verbose; size_t http_post_buffer = 16 * LARGE_PACKET_MAX; +static int min_curl_sessions = 1; +static int curl_session_count; #ifdef USE_CURL_MULTI static int max_requests = -1; static CURLM *curlm; @@ -152,6 +154,14 @@ static int http_options(const char *var, const char *value, void *cb) ssl_cert_password_required = 1; return 0; } + if (!strcmp("http.minsessions", var)) { + min_curl_sessions = git_config_int(var, value); +#ifndef USE_CURL_MULTI + if (min_curl_sessions > 1) + min_curl_sessions = 1; +#endif + return 0; + } #ifdef USE_CURL_MULTI if (!strcmp("http.maxrequests", var)) { max_requests = git_config_int(var, value); @@ -372,6 +382,7 @@ void http_init(struct remote *remote) if (curl_ssl_verify == -1) curl_ssl_verify = 1; + curl_session_count = 0; #ifdef USE_CURL_MULTI if (max_requests < 1) max_requests = DEFAULT_MAX_REQUESTS; @@ -480,6 +491,7 @@ struct active_request_slot *get_active_slot(void) #else slot->curl = curl_easy_duphandle(curl_default); #endif + curl_session_count++; } active_requests++; @@ -558,9 +570,11 @@ void fill_active_slots(void) } while (slot != NULL) { - if (!slot->in_use && slot->curl != NULL) { + if (!slot->in_use && slot->curl != NULL + && curl_session_count > min_curl_sessions) { curl_easy_cleanup(slot->curl); slot->curl = NULL; + curl_session_count--; } slot = slot->next; } @@ -633,12 +647,13 @@ static void closedown_active_slot(struct active_request_slot *slot) void release_active_slot(struct active_request_slot *slot) { closedown_active_slot(slot); - if (slot->curl) { + if (slot->curl && curl_session_count > min_curl_sessions) { #ifdef USE_CURL_MULTI curl_multi_remove_handle(curlm, slot->curl); #endif curl_easy_cleanup(slot->curl); slot->curl = NULL; + curl_session_count--; } #ifdef USE_CURL_MULTI fill_active_slots(); -- cgit v0.10.2-6-g49f6 From b8ac923010484908d8426cb8ded5ad7e8c21a7f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Storsj=C3=B6?= Date: Fri, 27 Nov 2009 23:43:08 +0800 Subject: Add an option for using any HTTP authentication scheme, not only basic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This adds the configuration option http.authAny (overridable with the environment variable GIT_HTTP_AUTH_ANY), for instructing curl to allow any HTTP authentication scheme, not only basic (which sends the password in plaintext). When this is enabled, curl has to do double requests most of the time, in order to discover which HTTP authentication method to use, which lowers the performance slightly. Therefore this isn't enabled by default. One example of another authentication scheme to use is digest, which doesn't send the password in plaintext, but uses a challenge-response mechanism instead. Using digest authentication in practice requires at least curl 7.18.1, due to bugs in the digest handling in earlier versions of curl. Signed-off-by: Martin Storsjö Signed-off-by: Tay Ray Chuan Signed-off-by: Junio C Hamano diff --git a/Documentation/config.txt b/Documentation/config.txt index b77d66d..a54ede3 100644 --- a/Documentation/config.txt +++ b/Documentation/config.txt @@ -1158,6 +1158,13 @@ http.noEPSV:: support EPSV mode. Can be overridden by the 'GIT_CURL_FTP_NO_EPSV' environment variable. Default is false (curl will use EPSV). +http.authAny:: + Allow any HTTP authentication method, not only basic. Enabling + this lowers the performance slightly, by having to do requests + without any authentication to discover the authentication method + to use. Can be overridden by the 'GIT_HTTP_AUTH_ANY' + environment variable. Default is false. + i18n.commitEncoding:: Character encoding the commit messages are stored in; git itself does not care per se, but this information is necessary e.g. when diff --git a/http.c b/http.c index fb0a97b..aeb69b3 100644 --- a/http.c +++ b/http.c @@ -7,6 +7,10 @@ int active_requests; int http_is_verbose; size_t http_post_buffer = 16 * LARGE_PACKET_MAX; +#if LIBCURL_VERSION_NUM >= 0x070a06 +#define LIBCURL_CAN_HANDLE_AUTH_ANY +#endif + static int min_curl_sessions = 1; static int curl_session_count; #ifdef USE_CURL_MULTI @@ -36,6 +40,9 @@ static long curl_low_speed_time = -1; static int curl_ftp_no_epsv; static const char *curl_http_proxy; static char *user_name, *user_pass; +#ifdef LIBCURL_CAN_HANDLE_AUTH_ANY +static int curl_http_auth_any = 0; +#endif #if LIBCURL_VERSION_NUM >= 0x071700 /* Use CURLOPT_KEYPASSWD as is */ @@ -190,6 +197,12 @@ static int http_options(const char *var, const char *value, void *cb) http_post_buffer = LARGE_PACKET_MAX; return 0; } +#ifdef LIBCURL_CAN_HANDLE_AUTH_ANY + if (!strcmp("http.authany", var)) { + curl_http_auth_any = git_config_bool(var, value); + return 0; + } +#endif /* Fall back on the default ones */ return git_default_config(var, value, cb); @@ -240,6 +253,10 @@ static CURL *get_curl_handle(void) #if LIBCURL_VERSION_NUM >= 0x070907 curl_easy_setopt(result, CURLOPT_NETRC, CURL_NETRC_OPTIONAL); #endif +#ifdef LIBCURL_CAN_HANDLE_AUTH_ANY + if (curl_http_auth_any) + curl_easy_setopt(result, CURLOPT_HTTPAUTH, CURLAUTH_ANY); +#endif init_curl_http_auth(result); @@ -391,6 +408,11 @@ void http_init(struct remote *remote) if (getenv("GIT_CURL_FTP_NO_EPSV")) curl_ftp_no_epsv = 1; +#ifdef LIBCURL_CAN_HANDLE_AUTH_ANY + if (getenv("GIT_HTTP_AUTH_ANY")) + curl_http_auth_any = 1; +#endif + if (remote && remote->url && remote->url[0]) { http_auth_init(remote->url[0]); if (!ssl_cert_password_required && -- cgit v0.10.2-6-g49f6 From 6c81a9908206799ccbc9a17bde17f1d766a8eef4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Storsj=C3=B6?= Date: Tue, 1 Dec 2009 12:33:39 +0200 Subject: Allow curl to rewind the RPC read buffer When using multi-pass authentication methods, the curl library may need to rewind the read buffers used for providing data to HTTP POST, if data has been output before a 401 error is received. This is needed only when the first request (when the multi-pass authentication method isn't initialized and hasn't received its challenge yet) for a certain curl session is a chunked HTTP POST. As long as the current rpc read buffer is the first one, we're able to rewind without need for additional buffering. The curl library currently starts sending data without waiting for a response to the Expect: 100-continue header, due to a bug in curl that exists up to curl version 7.19.7. If the HTTP server doesn't handle Expect: 100-continue headers properly (e.g. Lighttpd), the library has to start sending data without knowing if the request will be successfully authenticated. In this case, this rewinding solution is not sufficient - the whole request will be sent before the 401 error is received. Signed-off-by: Martin Storsjo Acked-by: Shawn O. Pearce Signed-off-by: Junio C Hamano diff --git a/remote-curl.c b/remote-curl.c index a331bae..28b2a31 100644 --- a/remote-curl.c +++ b/remote-curl.c @@ -290,6 +290,7 @@ struct rpc_state { int out; struct strbuf result; unsigned gzip_request : 1; + unsigned initial_buffer : 1; }; static size_t rpc_out(void *ptr, size_t eltsize, @@ -300,6 +301,7 @@ static size_t rpc_out(void *ptr, size_t eltsize, size_t avail = rpc->len - rpc->pos; if (!avail) { + rpc->initial_buffer = 0; avail = packet_read_line(rpc->out, rpc->buf, rpc->alloc); if (!avail) return 0; @@ -314,6 +316,29 @@ static size_t rpc_out(void *ptr, size_t eltsize, return avail; } +#ifndef NO_CURL_IOCTL +curlioerr rpc_ioctl(CURL *handle, int cmd, void *clientp) +{ + struct rpc_state *rpc = clientp; + + switch (cmd) { + case CURLIOCMD_NOP: + return CURLIOE_OK; + + case CURLIOCMD_RESTARTREAD: + if (rpc->initial_buffer) { + rpc->pos = 0; + return CURLIOE_OK; + } + fprintf(stderr, "Unable to rewind rpc post data - try increasing http.postBuffer\n"); + return CURLIOE_FAILRESTART; + + default: + return CURLIOE_UNKNOWNCMD; + } +} +#endif + static size_t rpc_in(const void *ptr, size_t eltsize, size_t nmemb, void *buffer_) { @@ -370,8 +395,13 @@ static int post_rpc(struct rpc_state *rpc) */ headers = curl_slist_append(headers, "Expect: 100-continue"); headers = curl_slist_append(headers, "Transfer-Encoding: chunked"); + rpc->initial_buffer = 1; curl_easy_setopt(slot->curl, CURLOPT_READFUNCTION, rpc_out); curl_easy_setopt(slot->curl, CURLOPT_INFILE, rpc); +#ifndef NO_CURL_IOCTL + curl_easy_setopt(slot->curl, CURLOPT_IOCTLFUNCTION, rpc_ioctl); + curl_easy_setopt(slot->curl, CURLOPT_IOCTLDATA, rpc); +#endif if (options.verbosity > 1) { fprintf(stderr, "POST %s (chunked)\n", rpc->service_name); fflush(stderr); -- cgit v0.10.2-6-g49f6 From 525ecd26c6cc8d1dc0c6190d8266e5d30b06da51 Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Mon, 28 Dec 2009 10:04:24 -0800 Subject: Remove http.authAny Back when the feature to use different HTTP authentication methods was originally written, it needed an extra HTTP request for everything when the feature was in effect, because we didn't reuse curl sessions. However, b8ac923 (Add an option for using any HTTP authentication scheme, not only basic, 2009-11-27) builds on top of an updated codebase that does reuse curl sessions; there is no need to manually avoid the extra overhead by making this configurable anymore. Acked-by: Martin Storsjo Acked-by: Shawn O. Pearce Signed-off-by: Junio C Hamano diff --git a/Documentation/config.txt b/Documentation/config.txt index a54ede3..b77d66d 100644 --- a/Documentation/config.txt +++ b/Documentation/config.txt @@ -1158,13 +1158,6 @@ http.noEPSV:: support EPSV mode. Can be overridden by the 'GIT_CURL_FTP_NO_EPSV' environment variable. Default is false (curl will use EPSV). -http.authAny:: - Allow any HTTP authentication method, not only basic. Enabling - this lowers the performance slightly, by having to do requests - without any authentication to discover the authentication method - to use. Can be overridden by the 'GIT_HTTP_AUTH_ANY' - environment variable. Default is false. - i18n.commitEncoding:: Character encoding the commit messages are stored in; git itself does not care per se, but this information is necessary e.g. when diff --git a/http.c b/http.c index aeb69b3..01e0fdc 100644 --- a/http.c +++ b/http.c @@ -40,9 +40,6 @@ static long curl_low_speed_time = -1; static int curl_ftp_no_epsv; static const char *curl_http_proxy; static char *user_name, *user_pass; -#ifdef LIBCURL_CAN_HANDLE_AUTH_ANY -static int curl_http_auth_any = 0; -#endif #if LIBCURL_VERSION_NUM >= 0x071700 /* Use CURLOPT_KEYPASSWD as is */ @@ -197,12 +194,6 @@ static int http_options(const char *var, const char *value, void *cb) http_post_buffer = LARGE_PACKET_MAX; return 0; } -#ifdef LIBCURL_CAN_HANDLE_AUTH_ANY - if (!strcmp("http.authany", var)) { - curl_http_auth_any = git_config_bool(var, value); - return 0; - } -#endif /* Fall back on the default ones */ return git_default_config(var, value, cb); @@ -254,8 +245,7 @@ static CURL *get_curl_handle(void) curl_easy_setopt(result, CURLOPT_NETRC, CURL_NETRC_OPTIONAL); #endif #ifdef LIBCURL_CAN_HANDLE_AUTH_ANY - if (curl_http_auth_any) - curl_easy_setopt(result, CURLOPT_HTTPAUTH, CURLAUTH_ANY); + curl_easy_setopt(result, CURLOPT_HTTPAUTH, CURLAUTH_ANY); #endif init_curl_http_auth(result); @@ -408,11 +398,6 @@ void http_init(struct remote *remote) if (getenv("GIT_CURL_FTP_NO_EPSV")) curl_ftp_no_epsv = 1; -#ifdef LIBCURL_CAN_HANDLE_AUTH_ANY - if (getenv("GIT_HTTP_AUTH_ANY")) - curl_http_auth_any = 1; -#endif - if (remote && remote->url && remote->url[0]) { http_auth_init(remote->url[0]); if (!ssl_cert_password_required && -- cgit v0.10.2-6-g49f6